Upstream composer/composer state of the touched files before b05f4e7d53
This commit is contained in:
upstream committed
1 parent
d4a3091c46
commit
44656d3be2
2 files changed
+1448
No files matched your search
@@ -0,0 +1,968 @@
|
|||||||
|
<?php declare(strict_types=1);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This file is part of Composer.
|
||||||
|
*
|
||||||
|
* (c) Nils Adermann <naderman@naderman.de>
|
||||||
|
* Jordi Boggiano <j.boggiano@seld.be>
|
||||||
|
*
|
||||||
|
* For the full copyright and license information, please view the LICENSE
|
||||||
|
* file that was distributed with this source code.
|
||||||
|
*/
|
||||||
|
|
||||||
|
namespace Composer\Util\Http;
|
||||||
|
|
||||||
|
use Composer\Config;
|
||||||
|
use Composer\Downloader\MaxFileSizeExceededException;
|
||||||
|
use Composer\IO\IOInterface;
|
||||||
|
use Composer\Downloader\TransportException;
|
||||||
|
use Composer\Pcre\Preg;
|
||||||
|
use Composer\Util\Platform;
|
||||||
|
use Composer\Util\StreamContextFactory;
|
||||||
|
use Composer\Util\AuthHelper;
|
||||||
|
use Composer\Util\Url;
|
||||||
|
use React\Promise\Promise;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @internal
|
||||||
|
* @author Jordi Boggiano <j.boggiano@seld.be>
|
||||||
|
* @author Nicolas Grekas <p@tchwork.com>
|
||||||
|
* @phpstan-type Attributes array{retryAuthFailure: bool, redirects: int<0, max>, retries: int<0, max>, storeAuth: 'prompt'|bool, ipResolve: 4|6|null}
|
||||||
|
* @phpstan-type Job array{url: non-empty-string, origin: string, attributes: Attributes, options: mixed[], progress: mixed[], curlHandle: \CurlHandle, filename: string|null, headerHandle: resource, bodyHandle: resource, resolve: callable, reject: callable, primaryIp: string}
|
||||||
|
* @phpstan-type RetryAttributes array{retryAuthFailure?: bool, redirects?: int<0, max>, storeAuth?: 'prompt'|bool, retries: int<1, max>, ipResolve?: 4|6}
|
||||||
|
* @phpstan-type DelayedJob array{job: Job, url: non-empty-string, attributes: RetryAttributes, at: float}
|
||||||
|
*/
|
||||||
|
class CurlDownloader
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Known libcurl's broken versions when proxy is in use with HTTP/2
|
||||||
|
* multiplexing.
|
||||||
|
*
|
||||||
|
* @var list<non-empty-string>
|
||||||
|
*/
|
||||||
|
private const BAD_MULTIPLEXING_CURL_VERSIONS = ['7.87.0', '7.88.0', '7.88.1'];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Longest Retry-After interval which is waited out, in seconds
|
||||||
|
*
|
||||||
|
* Without a ceiling a server could park an install for as long as it likes, see isStatusCodeRetryNeeded()
|
||||||
|
*/
|
||||||
|
private const MAX_RETRY_AFTER = 60;
|
||||||
|
|
||||||
|
/** @var \CurlMultiHandle */
|
||||||
|
private $multiHandle;
|
||||||
|
/** @var \CurlShareHandle */
|
||||||
|
private $shareHandle;
|
||||||
|
/** @var Job[] */
|
||||||
|
private $jobs = [];
|
||||||
|
/**
|
||||||
|
* Retries which are waiting for their delay to elapse, see restartJobWithDelay()
|
||||||
|
*
|
||||||
|
* @var array<int, DelayedJob>
|
||||||
|
*/
|
||||||
|
private $delayedJobs = [];
|
||||||
|
/**
|
||||||
|
* Origins which asked for requests to be retried later, keyed by origin, see restartJobWithDelay()
|
||||||
|
*
|
||||||
|
* until is the latest point in time the origin asked to be left alone for, see getDueAt()
|
||||||
|
*
|
||||||
|
* @var array<string, array{statusCode: int, announced: bool, until: float}>
|
||||||
|
*/
|
||||||
|
private $retryAfterOrigins = [];
|
||||||
|
/** @var IOInterface */
|
||||||
|
private $io;
|
||||||
|
/** @var Config */
|
||||||
|
private $config;
|
||||||
|
/** @var AuthHelper */
|
||||||
|
private $authHelper;
|
||||||
|
/** @var float */
|
||||||
|
private $selectTimeout = 5.0;
|
||||||
|
/** @var int */
|
||||||
|
private $maxRedirects = 20;
|
||||||
|
/** @var int */
|
||||||
|
private $maxRetries = 3;
|
||||||
|
/** @var array<int, string[]> */
|
||||||
|
protected $multiErrors = [
|
||||||
|
CURLM_BAD_HANDLE => ['CURLM_BAD_HANDLE', 'The passed-in handle is not a valid CURLM handle.'],
|
||||||
|
CURLM_BAD_EASY_HANDLE => ['CURLM_BAD_EASY_HANDLE', "An easy handle was not good/valid. It could mean that it isn't an easy handle at all, or possibly that the handle already is in used by this or another multi handle."],
|
||||||
|
CURLM_OUT_OF_MEMORY => ['CURLM_OUT_OF_MEMORY', 'You are doomed.'],
|
||||||
|
CURLM_INTERNAL_ERROR => ['CURLM_INTERNAL_ERROR', 'This can only be returned if libcurl bugs. Please report it to us!'],
|
||||||
|
];
|
||||||
|
|
||||||
|
/** @var mixed[] */
|
||||||
|
private static $options = [
|
||||||
|
'http' => [
|
||||||
|
'method' => CURLOPT_CUSTOMREQUEST,
|
||||||
|
'content' => CURLOPT_POSTFIELDS,
|
||||||
|
'header' => CURLOPT_HTTPHEADER,
|
||||||
|
'timeout' => CURLOPT_TIMEOUT,
|
||||||
|
],
|
||||||
|
'ssl' => [
|
||||||
|
'cafile' => CURLOPT_CAINFO,
|
||||||
|
'capath' => CURLOPT_CAPATH,
|
||||||
|
'verify_peer' => CURLOPT_SSL_VERIFYPEER,
|
||||||
|
'verify_peer_name' => CURLOPT_SSL_VERIFYHOST,
|
||||||
|
'local_cert' => CURLOPT_SSLCERT,
|
||||||
|
'local_pk' => CURLOPT_SSLKEY,
|
||||||
|
'passphrase' => CURLOPT_SSLKEYPASSWD,
|
||||||
|
],
|
||||||
|
];
|
||||||
|
|
||||||
|
/** @var array<string, true> */
|
||||||
|
private static $timeInfo = [
|
||||||
|
'total_time' => true,
|
||||||
|
'namelookup_time' => true,
|
||||||
|
'connect_time' => true,
|
||||||
|
'pretransfer_time' => true,
|
||||||
|
'starttransfer_time' => true,
|
||||||
|
'redirect_time' => true,
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param mixed[] $options
|
||||||
|
*/
|
||||||
|
public function __construct(IOInterface $io, Config $config, array $options = [], bool $disableTls = false)
|
||||||
|
{
|
||||||
|
$this->io = $io;
|
||||||
|
$this->config = $config;
|
||||||
|
|
||||||
|
$this->multiHandle = $mh = curl_multi_init();
|
||||||
|
if (function_exists('curl_multi_setopt')) {
|
||||||
|
if (ProxyManager::getInstance()->hasProxy() && ($version = curl_version()) !== false && in_array($version['version'], self::BAD_MULTIPLEXING_CURL_VERSIONS, true)) {
|
||||||
|
/**
|
||||||
|
* Disable HTTP/2 multiplexing for some broken versions of libcurl.
|
||||||
|
*
|
||||||
|
* In certain versions of libcurl when proxy is in use with HTTP/2
|
||||||
|
* multiplexing, connections will continue stacking up. This was
|
||||||
|
* fixed in libcurl 8.0.0 in curl/curl@821f6e2a89de8aec1c7da3c0f381b92b2b801efc
|
||||||
|
*/
|
||||||
|
curl_multi_setopt($mh, CURLMOPT_PIPELINING, /* CURLPIPE_NOTHING */ 0);
|
||||||
|
} else {
|
||||||
|
curl_multi_setopt($mh, CURLMOPT_PIPELINING, \PHP_VERSION_ID >= 70400 ? /* CURLPIPE_MULTIPLEX */ 2 : /*CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX*/ 3);
|
||||||
|
}
|
||||||
|
if (defined('CURLMOPT_MAX_HOST_CONNECTIONS') && !defined('HHVM_VERSION')) {
|
||||||
|
curl_multi_setopt($mh, CURLMOPT_MAX_HOST_CONNECTIONS, 8);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (function_exists('curl_share_init')) {
|
||||||
|
$this->shareHandle = $sh = curl_share_init();
|
||||||
|
curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
|
||||||
|
curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
|
||||||
|
curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->authHelper = new AuthHelper($io, $config);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param mixed[] $options
|
||||||
|
* @param non-empty-string $url
|
||||||
|
*
|
||||||
|
* @return int internal job id
|
||||||
|
*/
|
||||||
|
public function download(callable $resolve, callable $reject, string $origin, string $url, array $options, ?string $copyTo = null): int
|
||||||
|
{
|
||||||
|
$attributes = [];
|
||||||
|
if (isset($options['retry-auth-failure'])) {
|
||||||
|
$attributes['retryAuthFailure'] = $options['retry-auth-failure'];
|
||||||
|
unset($options['retry-auth-failure']);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $this->initDownload($resolve, $reject, $origin, $url, $options, $copyTo, $attributes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param mixed[] $options
|
||||||
|
*
|
||||||
|
* @param array{retryAuthFailure?: bool, redirects?: int<0, max>, retries?: int<0, max>, storeAuth?: 'prompt'|bool, ipResolve?: 4|6|null} $attributes
|
||||||
|
* @param non-empty-string $url
|
||||||
|
*
|
||||||
|
* @return int internal job id
|
||||||
|
*/
|
||||||
|
private function initDownload(callable $resolve, callable $reject, string $origin, string $url, array $options, ?string $copyTo = null, array $attributes = []): int
|
||||||
|
{
|
||||||
|
$attributes = array_merge([
|
||||||
|
'retryAuthFailure' => true,
|
||||||
|
'redirects' => 0,
|
||||||
|
'retries' => 0,
|
||||||
|
'storeAuth' => false,
|
||||||
|
'ipResolve' => null,
|
||||||
|
], $attributes);
|
||||||
|
|
||||||
|
if ($attributes['ipResolve'] === null && Platform::getEnv('COMPOSER_IPRESOLVE') === '4') {
|
||||||
|
$attributes['ipResolve'] = 4;
|
||||||
|
} elseif ($attributes['ipResolve'] === null && Platform::getEnv('COMPOSER_IPRESOLVE') === '6') {
|
||||||
|
$attributes['ipResolve'] = 6;
|
||||||
|
}
|
||||||
|
|
||||||
|
$originalOptions = $options;
|
||||||
|
|
||||||
|
// check URL can be accessed (i.e. is not insecure), but allow insecure Packagist calls to $hashed providers as file integrity is verified with sha256
|
||||||
|
if (!Preg::isMatch('{^http://(repo\.)?packagist\.org/p/}', $url) || (false === strpos($url, '$') && false === strpos($url, '%24'))) {
|
||||||
|
$this->config->prohibitUrlByConfig($url, $this->io, $options);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
isset($options['prevent_url_access_callable']) &&
|
||||||
|
is_callable($options['prevent_url_access_callable']) &&
|
||||||
|
$options['prevent_url_access_callable']($url)
|
||||||
|
) {
|
||||||
|
throw new TransportException('Access to "'.Url::sanitize($url).'" is blocked.');
|
||||||
|
}
|
||||||
|
|
||||||
|
$curlHandle = curl_init();
|
||||||
|
$headerHandle = fopen('php://temp/maxmemory:32768', 'w+b');
|
||||||
|
if (false === $headerHandle) {
|
||||||
|
throw new \RuntimeException('Failed to open a temp stream to store curl headers');
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($copyTo !== null) {
|
||||||
|
$bodyTarget = $copyTo.'~';
|
||||||
|
} else {
|
||||||
|
$bodyTarget = 'php://temp/maxmemory:524288';
|
||||||
|
}
|
||||||
|
|
||||||
|
$errorMessage = '';
|
||||||
|
set_error_handler(static function (int $code, string $msg) use (&$errorMessage): bool {
|
||||||
|
if ($errorMessage) {
|
||||||
|
$errorMessage .= "\n";
|
||||||
|
}
|
||||||
|
$errorMessage .= Preg::replace('{^fopen\(.*?\): }', '', $msg);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
$bodyHandle = fopen($bodyTarget, 'w+b');
|
||||||
|
restore_error_handler();
|
||||||
|
if (false === $bodyHandle) {
|
||||||
|
throw new TransportException('The "'.Url::sanitize($url).'" file could not be written to '.($copyTo ?? 'a temporary file').': '.$errorMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
curl_setopt($curlHandle, CURLOPT_URL, $url);
|
||||||
|
curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, false);
|
||||||
|
curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 10);
|
||||||
|
curl_setopt($curlHandle, CURLOPT_TIMEOUT, max((int) ini_get("default_socket_timeout"), 300));
|
||||||
|
curl_setopt($curlHandle, CURLOPT_WRITEHEADER, $headerHandle);
|
||||||
|
curl_setopt($curlHandle, CURLOPT_FILE, $bodyHandle);
|
||||||
|
curl_setopt($curlHandle, CURLOPT_ENCODING, ""); // let cURL set the Accept-Encoding header to what it supports
|
||||||
|
curl_setopt($curlHandle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
||||||
|
|
||||||
|
if ($attributes['ipResolve'] === 4) {
|
||||||
|
curl_setopt($curlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
|
||||||
|
} elseif ($attributes['ipResolve'] === 6) {
|
||||||
|
curl_setopt($curlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($attributes['retries'] > 0) {
|
||||||
|
// when retrying, do not re-use a kept-alive connection from the pool as it may be the cause of the failure
|
||||||
|
curl_setopt($curlHandle, CURLOPT_FRESH_CONNECT, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (function_exists('curl_share_init')) {
|
||||||
|
curl_setopt($curlHandle, CURLOPT_SHARE, $this->shareHandle);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isset($options['http']['header'])) {
|
||||||
|
$options['http']['header'] = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
$options['http']['header'] = array_diff($options['http']['header'], ['Connection: close']);
|
||||||
|
$options['http']['header'][] = 'Connection: keep-alive';
|
||||||
|
|
||||||
|
$version = curl_version();
|
||||||
|
$features = $version['features'];
|
||||||
|
|
||||||
|
$proxy = ProxyManager::getInstance()->getProxyForRequest($url);
|
||||||
|
|
||||||
|
// HTTP/3 is deliberately not requested: packagist.org, GitHub, GitLab and Bitbucket do not support it yet, and
|
||||||
|
// CURL_HTTP_VERSION_3 makes curl wait for a QUIC handshake before falling back to TCP on every new connection.
|
||||||
|
// Worth revisiting once HTTP/3 is more widespread, ideally via CURLOPT_ALTSVC so only advertising hosts use it,
|
||||||
|
// see https://github.com/composer/composer/issues/12987
|
||||||
|
if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features) !== 0) {
|
||||||
|
curl_setopt($curlHandle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// curl 8.7.0 - 8.7.1 has a bug whereas automatic accept-encoding header results in an error when reading the response
|
||||||
|
// https://github.com/composer/composer/issues/11913
|
||||||
|
if (isset($version['version']) && in_array($version['version'], ['8.7.0', '8.7.1'], true) && \defined('CURL_VERSION_LIBZ') && (CURL_VERSION_LIBZ & $features) !== 0) {
|
||||||
|
curl_setopt($curlHandle, CURLOPT_ENCODING, "gzip");
|
||||||
|
}
|
||||||
|
|
||||||
|
$options = $this->authHelper->addAuthenticationOptions($options, $origin, $url);
|
||||||
|
$options = StreamContextFactory::initOptions($url, $options, true);
|
||||||
|
|
||||||
|
foreach (self::$options as $type => $curlOptions) {
|
||||||
|
foreach ($curlOptions as $name => $curlOption) {
|
||||||
|
if (isset($options[$type][$name])) {
|
||||||
|
if ($type === 'ssl' && $name === 'verify_peer_name') {
|
||||||
|
curl_setopt($curlHandle, $curlOption, $options[$type][$name] === true ? 2 : $options[$type][$name]);
|
||||||
|
} else {
|
||||||
|
curl_setopt($curlHandle, $curlOption, $options[$type][$name]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
curl_setopt_array($curlHandle, $proxy->getCurlOptions($options['ssl'] ?? []));
|
||||||
|
|
||||||
|
$progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo);
|
||||||
|
|
||||||
|
$this->jobs[(int) $curlHandle] = [
|
||||||
|
'url' => $url,
|
||||||
|
'origin' => $origin,
|
||||||
|
'attributes' => $attributes,
|
||||||
|
'options' => $originalOptions,
|
||||||
|
'progress' => $progress,
|
||||||
|
'curlHandle' => $curlHandle,
|
||||||
|
'filename' => $copyTo,
|
||||||
|
'headerHandle' => $headerHandle,
|
||||||
|
'bodyHandle' => $bodyHandle,
|
||||||
|
'resolve' => $resolve,
|
||||||
|
'reject' => $reject,
|
||||||
|
'primaryIp' => '',
|
||||||
|
];
|
||||||
|
|
||||||
|
$usingProxy = $proxy->getStatus(' using proxy (%s)');
|
||||||
|
$ifModified = false !== stripos(implode(',', $options['http']['header']), 'if-modified-since:') ? ' if modified' : '';
|
||||||
|
if ($attributes['redirects'] === 0 && $attributes['retries'] === 0) {
|
||||||
|
$this->io->writeError('Downloading ' . Url::sanitize($url) . $usingProxy . $ifModified, true, IOInterface::DEBUG);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $curlHandle));
|
||||||
|
// TODO progress
|
||||||
|
|
||||||
|
return (int) $curlHandle;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function abortRequest(int $id): void
|
||||||
|
{
|
||||||
|
if (isset($this->jobs[$id], $this->jobs[$id]['curlHandle'])) {
|
||||||
|
$job = $this->jobs[$id];
|
||||||
|
curl_multi_remove_handle($this->multiHandle, $job['curlHandle']);
|
||||||
|
if (\PHP_VERSION_ID < 80000) {
|
||||||
|
curl_close($job['curlHandle']);
|
||||||
|
}
|
||||||
|
if (is_resource($job['headerHandle'])) {
|
||||||
|
fclose($job['headerHandle']);
|
||||||
|
}
|
||||||
|
if (is_resource($job['bodyHandle'])) {
|
||||||
|
fclose($job['bodyHandle']);
|
||||||
|
}
|
||||||
|
if (null !== $job['filename']) {
|
||||||
|
@unlink($job['filename'].'~');
|
||||||
|
}
|
||||||
|
unset($this->jobs[$id]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// a retry waiting for its delay is not in flight but must not be started either
|
||||||
|
foreach ($this->delayedJobs as $i => $delayedJob) {
|
||||||
|
if ((int) $delayedJob['job']['curlHandle'] === $id) {
|
||||||
|
if (null !== $delayedJob['job']['filename']) {
|
||||||
|
@unlink($delayedJob['job']['filename'].'~');
|
||||||
|
}
|
||||||
|
unset($this->delayedJobs[$i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return int number of retries waiting for their delay, which hold no transfer open
|
||||||
|
*/
|
||||||
|
public function countDelayedJobs(): int
|
||||||
|
{
|
||||||
|
return count($this->delayedJobs);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return bool whether the origin asked for requests to be held back until later, see restartJobWithDelay()
|
||||||
|
*/
|
||||||
|
public function isOriginOnHold(string $origin): bool
|
||||||
|
{
|
||||||
|
return ($this->retryAfterOrigins[$origin]['until'] ?? 0.0) > microtime(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function tick(): void
|
||||||
|
{
|
||||||
|
if (count($this->jobs) === 0 && count($this->delayedJobs) === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->restartDueJobs();
|
||||||
|
|
||||||
|
if (count($this->jobs) === 0) {
|
||||||
|
// only delayed retries are left, so there is no transfer this could hold up and the
|
||||||
|
// short sleep merely avoids spinning the CPU until the next one comes due
|
||||||
|
usleep(10000);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$active = true;
|
||||||
|
$this->checkCurlResult(curl_multi_exec($this->multiHandle, $active));
|
||||||
|
if (-1 === curl_multi_select($this->multiHandle, $this->getSelectTimeout())) {
|
||||||
|
// sleep in case select returns -1 as it can happen on old php versions or some platforms where curl does not manage to do the select
|
||||||
|
usleep(150);
|
||||||
|
}
|
||||||
|
|
||||||
|
while ($progress = curl_multi_info_read($this->multiHandle)) {
|
||||||
|
$curlHandle = $progress['handle'];
|
||||||
|
$result = $progress['result'];
|
||||||
|
$i = (int) $curlHandle;
|
||||||
|
if (!isset($this->jobs[$i])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$progress = curl_getinfo($curlHandle);
|
||||||
|
if (false === $progress) {
|
||||||
|
throw new \RuntimeException('Failed getting info from curl handle '.$i.' ('.Url::sanitize($this->jobs[$i]['url']).')');
|
||||||
|
}
|
||||||
|
$job = $this->jobs[$i];
|
||||||
|
unset($this->jobs[$i]);
|
||||||
|
$error = curl_error($curlHandle);
|
||||||
|
$errno = curl_errno($curlHandle);
|
||||||
|
curl_multi_remove_handle($this->multiHandle, $curlHandle);
|
||||||
|
if (\PHP_VERSION_ID < 80000) {
|
||||||
|
curl_close($curlHandle);
|
||||||
|
}
|
||||||
|
|
||||||
|
$headers = null;
|
||||||
|
$statusCode = null;
|
||||||
|
$response = null;
|
||||||
|
try {
|
||||||
|
// TODO progress
|
||||||
|
if (CURLE_OK !== $errno || $error || $result !== CURLE_OK) {
|
||||||
|
$errno = $errno ?: $result;
|
||||||
|
if (!$error && function_exists('curl_strerror')) {
|
||||||
|
$error = curl_strerror($errno);
|
||||||
|
}
|
||||||
|
$progress['error_code'] = $errno;
|
||||||
|
|
||||||
|
if (
|
||||||
|
(!isset($job['options']['http']['method']) || $job['options']['http']['method'] === 'GET')
|
||||||
|
&& (
|
||||||
|
in_array($errno, [7 /* CURLE_COULDNT_CONNECT */, 16 /* CURLE_HTTP2 */, 92 /* CURLE_HTTP2_STREAM */, 6 /* CURLE_COULDNT_RESOLVE_HOST */, 28 /* CURLE_OPERATION_TIMEDOUT */], true)
|
||||||
|
|| (in_array($errno, [56 /* CURLE_RECV_ERROR */, 35 /* CURLE_SSL_CONNECT_ERROR */], true) && str_contains((string) $error, 'Connection reset by peer'))
|
||||||
|
) && $job['attributes']['retries'] < $this->maxRetries
|
||||||
|
) {
|
||||||
|
$attributes = ['retries' => $job['attributes']['retries'] + 1];
|
||||||
|
if ($errno === 7 && !isset($job['attributes']['ipResolve'])) { // CURLE_COULDNT_CONNECT, retry forcing IPv4 if no IP stack was selected
|
||||||
|
$attributes['ipResolve'] = 4;
|
||||||
|
}
|
||||||
|
$this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to curl error '. $errno, true, IOInterface::DEBUG);
|
||||||
|
$this->restartJobWithDelay($job, $job['url'], $attributes);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Remove this as soon as https://github.com/curl/curl/issues/10591 is resolved
|
||||||
|
if ($errno === 55 /* CURLE_SEND_ERROR */) {
|
||||||
|
$this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to curl error '. $errno, true, IOInterface::DEBUG);
|
||||||
|
$this->restartJobWithDelay($job, $job['url'], ['retries' => $job['attributes']['retries'] + 1]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new TransportException('curl error '.$errno.' while downloading '.Url::sanitize($progress['url']).': '.$error);
|
||||||
|
}
|
||||||
|
$statusCode = $progress['http_code'];
|
||||||
|
rewind($job['headerHandle']);
|
||||||
|
$headers = explode("\r\n", rtrim(stream_get_contents($job['headerHandle'])));
|
||||||
|
fclose($job['headerHandle']);
|
||||||
|
|
||||||
|
if ($statusCode === 0) {
|
||||||
|
throw new \LogicException('Received unexpected http status code 0 without error for '.Url::sanitize($progress['url']).': headers '.var_export($headers, true).' curl info '.var_export($progress, true));
|
||||||
|
}
|
||||||
|
|
||||||
|
// prepare response object
|
||||||
|
if (null !== $job['filename']) {
|
||||||
|
$contents = $job['filename'].'~';
|
||||||
|
if ($statusCode >= 300) {
|
||||||
|
rewind($job['bodyHandle']);
|
||||||
|
$contents = stream_get_contents($job['bodyHandle']);
|
||||||
|
}
|
||||||
|
$response = new CurlResponse(['url' => $job['url']], $statusCode, $headers, $contents, $progress);
|
||||||
|
$this->io->writeError('['.$statusCode.'] '.Url::sanitize($job['url']), true, IOInterface::DEBUG);
|
||||||
|
} else {
|
||||||
|
$maxFileSize = $job['options']['max_file_size'] ?? null;
|
||||||
|
rewind($job['bodyHandle']);
|
||||||
|
if ($maxFileSize !== null) {
|
||||||
|
$contents = stream_get_contents($job['bodyHandle'], $maxFileSize);
|
||||||
|
// Gzipped responses with missing Content-Length header cannot be detected during the file download
|
||||||
|
// because $progress['size_download'] refers to the gzipped size downloaded, not the actual file size
|
||||||
|
if ($contents !== false && Platform::strlen($contents) >= $maxFileSize) {
|
||||||
|
throw new MaxFileSizeExceededException('Maximum allowed download size reached. Downloaded ' . Platform::strlen($contents) . ' of allowed ' . $maxFileSize . ' bytes for ' . Url::sanitize($job['url']));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$contents = stream_get_contents($job['bodyHandle']);
|
||||||
|
}
|
||||||
|
|
||||||
|
$response = new CurlResponse(['url' => $job['url']], $statusCode, $headers, $contents, $progress);
|
||||||
|
$this->io->writeError('['.$statusCode.'] '.Url::sanitize($job['url']), true, IOInterface::DEBUG);
|
||||||
|
}
|
||||||
|
fclose($job['bodyHandle']);
|
||||||
|
|
||||||
|
$warningsOutput = false;
|
||||||
|
if ($response->getStatusCode() >= 300 && self::isJsonResponse($response)) {
|
||||||
|
$warningsOutput = ResponseWarnings::output($this->io, $job['origin'], json_decode($response->getBody(), true));
|
||||||
|
}
|
||||||
|
|
||||||
|
$result = $this->isAuthenticatedRetryNeeded($job, $response);
|
||||||
|
if ($result['retry']) {
|
||||||
|
$this->restartJob($job, $job['url'], ['storeAuth' => $result['storeAuth'], 'retries' => $job['attributes']['retries'] + 1]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// handle 3xx redirects, 304 Not Modified is excluded
|
||||||
|
if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['attributes']['redirects'] < $this->maxRedirects) {
|
||||||
|
$location = $this->handleRedirect($job, $response);
|
||||||
|
if ($location) {
|
||||||
|
$this->restartJob($job, $location, ['redirects' => $job['attributes']['redirects'] + 1]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fail 4xx and 5xx responses and capture the response
|
||||||
|
if ($statusCode >= 400 && $statusCode <= 599) {
|
||||||
|
$statusRetry = $this->isStatusCodeRetryNeeded($job, $response);
|
||||||
|
if ($statusRetry['retry']) {
|
||||||
|
$this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to status code '. $statusCode . (null !== $statusRetry['delay'] ? ' in '.$statusRetry['delay'].'s as requested by Retry-After' : ''), true, IOInterface::DEBUG);
|
||||||
|
$this->restartJobWithDelay($job, $job['url'], ['retries' => $job['attributes']['retries'] + 1], $statusRetry['delay'], $statusCode);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw $this->failResponse($job, $response, self::getStatusFailureMessage($response), $warningsOutput);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($job['attributes']['storeAuth'] !== false) {
|
||||||
|
$this->authHelper->storeAuth($job['origin'], $job['attributes']['storeAuth']);
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolve promise
|
||||||
|
if (null !== $job['filename']) {
|
||||||
|
rename($job['filename'].'~', $job['filename']);
|
||||||
|
$job['resolve']($response);
|
||||||
|
} else {
|
||||||
|
$job['resolve']($response);
|
||||||
|
}
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
if ($e instanceof TransportException) {
|
||||||
|
if (null !== $headers) {
|
||||||
|
$e->setHeaders($headers);
|
||||||
|
$e->setStatusCode($statusCode);
|
||||||
|
}
|
||||||
|
if (null !== $response) {
|
||||||
|
$e->setResponse($response->getBody());
|
||||||
|
}
|
||||||
|
$e->setResponseInfo($progress);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->rejectJob($job, $e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->announceRetryAfterWaits();
|
||||||
|
|
||||||
|
foreach ($this->jobs as $i => $curlHandle) {
|
||||||
|
$curlHandle = $this->jobs[$i]['curlHandle'];
|
||||||
|
$progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo);
|
||||||
|
|
||||||
|
if ($this->jobs[$i]['progress'] !== $progress) {
|
||||||
|
$this->jobs[$i]['progress'] = $progress;
|
||||||
|
|
||||||
|
if (isset($this->jobs[$i]['options']['max_file_size'])) {
|
||||||
|
// Compare max_file_size with the content-length header this value will be -1 until the header is parsed
|
||||||
|
if ($this->jobs[$i]['options']['max_file_size'] < $progress['download_content_length']) {
|
||||||
|
$this->rejectJob($this->jobs[$i], new MaxFileSizeExceededException('Maximum allowed download size reached. Content-length header indicates ' . $progress['download_content_length'] . ' bytes. Allowed ' . $this->jobs[$i]['options']['max_file_size'] . ' bytes for ' . Url::sanitize($this->jobs[$i]['url'])));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare max_file_size with the download size in bytes
|
||||||
|
if ($this->jobs[$i]['options']['max_file_size'] < $progress['size_download']) {
|
||||||
|
$this->rejectJob($this->jobs[$i], new MaxFileSizeExceededException('Maximum allowed download size reached. Downloaded ' . $progress['size_download'] . ' of allowed ' . $this->jobs[$i]['options']['max_file_size'] . ' bytes for ' . Url::sanitize($this->jobs[$i]['url'])));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isset($progress['primary_ip']) && $progress['primary_ip'] !== $this->jobs[$i]['primaryIp']) {
|
||||||
|
if (
|
||||||
|
isset($this->jobs[$i]['options']['prevent_ip_access_callable']) &&
|
||||||
|
is_callable($this->jobs[$i]['options']['prevent_ip_access_callable']) &&
|
||||||
|
$this->jobs[$i]['options']['prevent_ip_access_callable']($progress['primary_ip'])
|
||||||
|
) {
|
||||||
|
$this->rejectJob($this->jobs[$i], new TransportException(sprintf('IP "%s" is blocked for "%s".', $progress['primary_ip'], Url::sanitize($progress['url']))));
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->jobs[$i]['primaryIp'] = (string) $progress['primary_ip'];
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO progress
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Job $job
|
||||||
|
*/
|
||||||
|
private function handleRedirect(array $job, Response $response): string
|
||||||
|
{
|
||||||
|
if ($locationHeader = $response->getHeader('location')) {
|
||||||
|
if (parse_url($locationHeader, PHP_URL_SCHEME)) {
|
||||||
|
// Absolute URL; e.g. https://example.com/composer
|
||||||
|
$targetUrl = $locationHeader;
|
||||||
|
} elseif (parse_url($locationHeader, PHP_URL_HOST)) {
|
||||||
|
// Scheme relative; e.g. //example.com/foo
|
||||||
|
$targetUrl = parse_url($job['url'], PHP_URL_SCHEME).':'.$locationHeader;
|
||||||
|
} elseif ('/' === $locationHeader[0]) {
|
||||||
|
// Absolute path; e.g. /foo
|
||||||
|
$urlHost = parse_url($job['url'], PHP_URL_HOST);
|
||||||
|
|
||||||
|
// Replace path using hostname as an anchor.
|
||||||
|
$targetUrl = Preg::replace('{^(.+(?://|@)'.preg_quote($urlHost).'(?::\d+)?)(?:[/\?].*)?$}', '\1'.$locationHeader, $job['url']);
|
||||||
|
} else {
|
||||||
|
// Relative path; e.g. foo
|
||||||
|
// This actually differs from PHP which seems to add duplicate slashes.
|
||||||
|
$targetUrl = Preg::replace('{^(.+/)[^/?]*(?:\?.*)?$}', '\1'.$locationHeader, $job['url']);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!empty($targetUrl)) {
|
||||||
|
if (!Url::isAllowedRedirect($targetUrl)) {
|
||||||
|
throw new TransportException('Could not follow the redirect to "'.Url::sanitize($targetUrl).'" because only http and https redirects are supported.');
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->io->writeError(sprintf('Following redirect (%u) %s', $job['attributes']['redirects'] + 1, Url::sanitize($targetUrl)), true, IOInterface::DEBUG);
|
||||||
|
|
||||||
|
return $targetUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new TransportException('The "'.Url::sanitize($job['url']).'" file could not be downloaded, got redirect without Location ('.$response->getStatusMessage().')');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Job $job
|
||||||
|
* @return array{retry: bool, storeAuth: 'prompt'|bool}
|
||||||
|
*/
|
||||||
|
private function isAuthenticatedRetryNeeded(array $job, Response $response): array
|
||||||
|
{
|
||||||
|
if (in_array($response->getStatusCode(), [401, 403]) && $job['attributes']['retryAuthFailure']) {
|
||||||
|
$result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], $response->getStatusCode(), $response->getStatusMessage(), $response->getHeaders(), $job['attributes']['retries'], $response->getBody());
|
||||||
|
|
||||||
|
if ($result['retry']) {
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$locationHeader = $response->getHeader('location');
|
||||||
|
$needsAuthRetry = false;
|
||||||
|
|
||||||
|
// check for bitbucket login page asking to authenticate
|
||||||
|
if (
|
||||||
|
$job['origin'] === 'bitbucket.org'
|
||||||
|
&& !$this->authHelper->isPublicBitBucketDownload($job['url'])
|
||||||
|
&& substr($job['url'], -4) === '.zip'
|
||||||
|
&& (!$locationHeader || substr($locationHeader, -4) !== '.zip')
|
||||||
|
&& Preg::isMatch('{^text/html\b}i', $response->getHeader('content-type'))
|
||||||
|
) {
|
||||||
|
$needsAuthRetry = 'Bitbucket requires authentication and it was not provided';
|
||||||
|
}
|
||||||
|
|
||||||
|
// check for gitlab 404 when downloading archives
|
||||||
|
if (
|
||||||
|
$response->getStatusCode() === 404
|
||||||
|
&& in_array($job['origin'], $this->config->get('gitlab-domains'), true)
|
||||||
|
&& false !== strpos($job['url'], 'archive.zip')
|
||||||
|
) {
|
||||||
|
$needsAuthRetry = 'GitLab requires authentication and it was not provided';
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($needsAuthRetry) {
|
||||||
|
if ($job['attributes']['retryAuthFailure']) {
|
||||||
|
$result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], 401, null, [], $job['attributes']['retries']);
|
||||||
|
if ($result['retry']) {
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw $this->failResponse($job, $response, $needsAuthRetry);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ['retry' => false, 'storeAuth' => false];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decides whether a 4xx/5xx response should be retried, and how long to wait beforehand
|
||||||
|
*
|
||||||
|
* @param Job $job
|
||||||
|
* @return array{retry: bool, delay: float|null} delay is null when the default backoff applies
|
||||||
|
*/
|
||||||
|
private function isStatusCodeRetryNeeded(array $job, Response $response): array
|
||||||
|
{
|
||||||
|
$noRetry = ['retry' => false, 'delay' => null];
|
||||||
|
|
||||||
|
if (isset($job['options']['http']['method']) && $job['options']['http']['method'] !== 'GET') {
|
||||||
|
return $noRetry;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($job['attributes']['retries'] >= $this->maxRetries) {
|
||||||
|
return $noRetry;
|
||||||
|
}
|
||||||
|
|
||||||
|
$statusCode = $response->getStatusCode();
|
||||||
|
$retryAfter = self::getRetryAfterDelay($response);
|
||||||
|
|
||||||
|
$retryableByStatus = in_array($statusCode, [423, 425, 500, 502, 503, 504, 507, 510], true)
|
||||||
|
// codeload.github.com intermittently returns 400 on reused connections, retry those specifically, see #12958
|
||||||
|
|| ($statusCode === 400 && parse_url($job['url'], PHP_URL_HOST) === 'codeload.github.com');
|
||||||
|
|
||||||
|
// a rate limit is only retried when the server said when to come back
|
||||||
|
$retryableByHeader = $statusCode === 429 && null !== $retryAfter;
|
||||||
|
|
||||||
|
if (!$retryableByStatus && !$retryableByHeader) {
|
||||||
|
return $noRetry;
|
||||||
|
}
|
||||||
|
|
||||||
|
// an interval beyond the cap is not waited for: a status which is retryable anyway falls
|
||||||
|
// back to the usual backoff, a rate limit has nothing else to go on and fails
|
||||||
|
if (null !== $retryAfter && $retryAfter > self::MAX_RETRY_AFTER) {
|
||||||
|
if (!$retryableByStatus) {
|
||||||
|
return $noRetry;
|
||||||
|
}
|
||||||
|
|
||||||
|
$retryAfter = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// a wait of 0s is no wait, so the usual backoff spaces such retries out instead
|
||||||
|
return ['retry' => true, 'delay' => null !== $retryAfter && $retryAfter > 0 ? (float) $retryAfter : null];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads the Retry-After header, in either of the forms RFC 9110 allows, as a number of seconds to wait
|
||||||
|
*
|
||||||
|
* @return int|null seconds to wait, or null if there is no usable Retry-After header
|
||||||
|
*/
|
||||||
|
private static function getRetryAfterDelay(Response $response): ?int
|
||||||
|
{
|
||||||
|
$retryAfter = $response->getHeader('retry-after');
|
||||||
|
if (null === $retryAfter) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// more digits would overflow an int, and are no interval anyone can mean
|
||||||
|
if (Preg::isMatch('{^\d{1,9}$}', $retryAfter)) {
|
||||||
|
return (int) $retryAfter;
|
||||||
|
}
|
||||||
|
|
||||||
|
// the three date formats RFC 9110 requires recipients to accept, with the leading day name
|
||||||
|
// skipped as it is redundant and DateTime would otherwise move the date to match it
|
||||||
|
foreach (['!*, d M Y H:i:s \G\M\T', '!*, d-M-y H:i:s \G\M\T', '!* M j H:i:s Y'] as $format) {
|
||||||
|
$retryAt = \DateTime::createFromFormat($format, $retryAfter, new \DateTimeZone('UTC'));
|
||||||
|
if (false !== $retryAt) {
|
||||||
|
// a date which has already passed means the wait is over and we can retry straight away
|
||||||
|
return max(0, $retryAt->getTimestamp() - time());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Starts the delayed retries whose delay has elapsed
|
||||||
|
*/
|
||||||
|
private function restartDueJobs(): void
|
||||||
|
{
|
||||||
|
$now = microtime(true);
|
||||||
|
foreach ($this->delayedJobs as $i => $delayedJob) {
|
||||||
|
if ($this->getDueAt($delayedJob) > $now) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
unset($this->delayedJobs[$i]);
|
||||||
|
try {
|
||||||
|
$this->restartJob($delayedJob['job'], $delayedJob['url'], $delayedJob['attributes']);
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
// an immediate restart is covered by tick()'s try/catch, a deferred one is not
|
||||||
|
$this->rejectJob($delayedJob['job'], $e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return float how long curl_multi_select may block for, in seconds
|
||||||
|
*/
|
||||||
|
private function getSelectTimeout(): float
|
||||||
|
{
|
||||||
|
$timeout = $this->selectTimeout;
|
||||||
|
$now = microtime(true);
|
||||||
|
foreach ($this->delayedJobs as $delayedJob) {
|
||||||
|
// never sleep past the point where a delayed retry comes due
|
||||||
|
$timeout = min($timeout, max(0.0, $this->getDueAt($delayedJob) - $now));
|
||||||
|
}
|
||||||
|
|
||||||
|
return $timeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A Retry-After speaks for the whole origin, so a retry also waits for a later hold the origin asked for
|
||||||
|
*
|
||||||
|
* @param DelayedJob $delayedJob
|
||||||
|
* @return float when the retry may be started, as a timestamp
|
||||||
|
*/
|
||||||
|
private function getDueAt(array $delayedJob): float
|
||||||
|
{
|
||||||
|
return max($delayedJob['at'], $this->retryAfterOrigins[$delayedJob['job']['origin']]['until'] ?? 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Job $job
|
||||||
|
* @param non-empty-string $url
|
||||||
|
*
|
||||||
|
* @param array{retryAuthFailure?: bool, redirects?: int<0, max>, storeAuth?: 'prompt'|bool, retries?: int<1, max>, ipResolve?: 4|6} $attributes
|
||||||
|
*/
|
||||||
|
private function restartJob(array $job, string $url, array $attributes = []): void
|
||||||
|
{
|
||||||
|
if (null !== $job['filename']) {
|
||||||
|
@unlink($job['filename'].'~');
|
||||||
|
}
|
||||||
|
|
||||||
|
$attributes = array_merge($job['attributes'], $attributes);
|
||||||
|
$origin = Url::getOrigin($this->config, $url);
|
||||||
|
|
||||||
|
$this->initDownload($job['resolve'], $job['reject'], $origin, $url, $job['options'], $job['filename'], $attributes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Job $job
|
||||||
|
* @param non-empty-string $url
|
||||||
|
*
|
||||||
|
* @param RetryAttributes $attributes
|
||||||
|
* @param float|null $retryAfter seconds the response asked to wait before retrying, null to use the default backoff
|
||||||
|
* @param int $statusCode status code of the response which asked for the wait, only used with $retryAfter
|
||||||
|
*/
|
||||||
|
private function restartJobWithDelay(array $job, string $url, array $attributes, ?float $retryAfter = null, int $statusCode = 0): void
|
||||||
|
{
|
||||||
|
$now = microtime(true);
|
||||||
|
if (null !== $retryAfter) {
|
||||||
|
$delay = $retryAfter;
|
||||||
|
$until = $now + $retryAfter;
|
||||||
|
$origin = $this->retryAfterOrigins[$job['origin']] ?? null;
|
||||||
|
// a hold which has already run out is over, so the origin is announced anew rather than extended
|
||||||
|
if (null === $origin || $origin['until'] <= $now) {
|
||||||
|
$this->retryAfterOrigins[$job['origin']] = ['statusCode' => $statusCode, 'announced' => false, 'until' => $until];
|
||||||
|
} else {
|
||||||
|
$this->retryAfterOrigins[$job['origin']]['until'] = max($origin['until'], $until);
|
||||||
|
}
|
||||||
|
} elseif ($attributes['retries'] >= 3) {
|
||||||
|
$delay = 0.5; // half a second delay for 3rd retry and beyond
|
||||||
|
} elseif ($attributes['retries'] >= 2) {
|
||||||
|
$delay = 0.1; // 100ms delay for 2nd retry
|
||||||
|
} else {
|
||||||
|
$delay = 0.0; // no delay for the first retry
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($delay <= 0.0 && ($this->retryAfterOrigins[$job['origin']]['until'] ?? 0.0) <= $now) {
|
||||||
|
$this->restartJob($job, $url, $attributes);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// sleeping here would stall every other transfer tick() drives, so the retry is scheduled instead
|
||||||
|
$this->delayedJobs[] = ['job' => $job, 'url' => $url, 'attributes' => $attributes, 'at' => $now + $delay];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tells the user when an origin asked for requests to be retried later, once per origin and episode
|
||||||
|
*
|
||||||
|
* The wait can last up to MAX_RETRY_AFTER seconds, which without a word would look like a hang
|
||||||
|
*/
|
||||||
|
private function announceRetryAfterWaits(): void
|
||||||
|
{
|
||||||
|
$now = microtime(true);
|
||||||
|
foreach ($this->retryAfterOrigins as $origin => $retryAfterOrigin) {
|
||||||
|
$pending = 0;
|
||||||
|
$lastDue = $now;
|
||||||
|
foreach ($this->delayedJobs as $delayedJob) {
|
||||||
|
if ($delayedJob['job']['origin'] === $origin) {
|
||||||
|
$pending++;
|
||||||
|
$lastDue = max($lastDue, $this->getDueAt($delayedJob));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (0 === $pending) {
|
||||||
|
unset($this->retryAfterOrigins[$origin]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($retryAfterOrigin['announced']) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$reason = 429 === $retryAfterOrigin['statusCode'] ? 'is rate limiting requests' : 'responded with status code '.$retryAfterOrigin['statusCode'];
|
||||||
|
$this->io->writeError('<warning>'.$origin.' '.$reason.' ('.$pending.' pending), waiting '.(int) ceil($lastDue - $now).'s before retrying</warning>');
|
||||||
|
$this->retryAfterOrigins[$origin]['announced'] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Job $job
|
||||||
|
*/
|
||||||
|
private function failResponse(array $job, Response $response, string $errorMessage, bool $warningsOutput = false): TransportException
|
||||||
|
{
|
||||||
|
if (null !== $job['filename']) {
|
||||||
|
@unlink($job['filename'].'~');
|
||||||
|
}
|
||||||
|
|
||||||
|
$details = '';
|
||||||
|
// skip dumping the raw JSON body when ResponseWarnings::output() already presented it cleanly, to avoid duplicate/messy output
|
||||||
|
if (!$warningsOutput && self::isJsonResponse($response)) {
|
||||||
|
$details = ':'.PHP_EOL.substr($response->getBody(), 0, 200).(strlen($response->getBody()) > 200 ? '...' : '');
|
||||||
|
}
|
||||||
|
|
||||||
|
return new TransportException('The "'.Url::sanitize($job['url']).'" file could not be downloaded ('.$errorMessage.')' . $details, $response->getStatusCode());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Describes why a 4xx/5xx response failed the request
|
||||||
|
*
|
||||||
|
* A Retry-After which is longer than we wait for is spelled out, as it tells the user whether
|
||||||
|
* trying again straight away can help or whether to come back later.
|
||||||
|
*/
|
||||||
|
private static function getStatusFailureMessage(Response $response): string
|
||||||
|
{
|
||||||
|
$message = (string) $response->getStatusMessage();
|
||||||
|
$retryAfter = self::getRetryAfterDelay($response);
|
||||||
|
if (null !== $retryAfter && $retryAfter > self::MAX_RETRY_AFTER) {
|
||||||
|
$message .= ', the server asked to retry in '.$retryAfter.'s which is longer than the '.self::MAX_RETRY_AFTER.'s Composer is willing to wait, try again later';
|
||||||
|
}
|
||||||
|
|
||||||
|
return $message;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function isJsonResponse(Response $response): bool
|
||||||
|
{
|
||||||
|
return in_array(strtolower((string) $response->getHeader('content-type')), ['application/json', 'application/json; charset=utf-8'], true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Job $job
|
||||||
|
*/
|
||||||
|
private function rejectJob(array $job, \Exception $e): void
|
||||||
|
{
|
||||||
|
if (is_resource($job['headerHandle'])) {
|
||||||
|
fclose($job['headerHandle']);
|
||||||
|
}
|
||||||
|
if (is_resource($job['bodyHandle'])) {
|
||||||
|
fclose($job['bodyHandle']);
|
||||||
|
}
|
||||||
|
if (null !== $job['filename']) {
|
||||||
|
@unlink($job['filename'].'~');
|
||||||
|
}
|
||||||
|
$job['reject']($e);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function checkCurlResult(int $code): void
|
||||||
|
{
|
||||||
|
if ($code !== CURLM_OK && $code !== CURLM_CALL_MULTI_PERFORM) {
|
||||||
|
throw new \RuntimeException(
|
||||||
|
isset($this->multiErrors[$code])
|
||||||
|
? "cURL error: {$code} ({$this->multiErrors[$code][0]}): cURL message: {$this->multiErrors[$code][1]}"
|
||||||
|
: 'Unexpected cURL error: ' . $code
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,480 @@
|
|||||||
|
<?php declare(strict_types=1);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This file is part of Composer.
|
||||||
|
*
|
||||||
|
* (c) Nils Adermann <naderman@naderman.de>
|
||||||
|
* Jordi Boggiano <j.boggiano@seld.be>
|
||||||
|
*
|
||||||
|
* For the full copyright and license information, please view the LICENSE
|
||||||
|
* file that was distributed with this source code.
|
||||||
|
*/
|
||||||
|
|
||||||
|
namespace Composer\Test\Util\Http;
|
||||||
|
|
||||||
|
use Composer\IO\BufferIO;
|
||||||
|
use Composer\IO\IOInterface;
|
||||||
|
use Composer\Test\TestCase;
|
||||||
|
use Composer\Util\Http\CurlDownloader;
|
||||||
|
use Composer\Util\Http\Response;
|
||||||
|
|
||||||
|
class CurlDownloaderTest extends TestCase
|
||||||
|
{
|
||||||
|
protected function setUp(): void
|
||||||
|
{
|
||||||
|
if (!extension_loaded('curl')) {
|
||||||
|
self::markTestSkipped('curl extension is required');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryAfterIsReadAsDelaySeconds(): void
|
||||||
|
{
|
||||||
|
self::assertSame(42, self::parseRetryAfter($this->createResponse(429, ['Retry-After: 42'])));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryAfterIsReadAsHttpDate(): void
|
||||||
|
{
|
||||||
|
$delay = self::parseRetryAfter($this->createResponse(429, ['Retry-After: '.gmdate('D, d M Y H:i:s \G\M\T', time() + 120)]));
|
||||||
|
|
||||||
|
self::assertIsInt($delay);
|
||||||
|
self::assertGreaterThan(115, $delay);
|
||||||
|
self::assertLessThanOrEqual(120, $delay);
|
||||||
|
|
||||||
|
// the obsolete RFC 850 format must be accepted too
|
||||||
|
self::assertSame(0, self::parseRetryAfter($this->createResponse(429, ['Retry-After: Sunday, 06-Nov-94 08:49:37 GMT'])));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryAfterInThePastMeansNoWait(): void
|
||||||
|
{
|
||||||
|
self::assertSame(0, self::parseRetryAfter($this->createResponse(429, ['Retry-After: '.gmdate('D, d M Y H:i:s \G\M\T', time() - 120)])));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testMalformedRetryAfterIsIgnored(): void
|
||||||
|
{
|
||||||
|
self::assertNull(self::parseRetryAfter($this->createResponse(429, ['Retry-After: not a date'])));
|
||||||
|
self::assertNull(self::parseRetryAfter($this->createResponse(429, ['Retry-After: 99999999999999999999'])));
|
||||||
|
self::assertNull(self::parseRetryAfter($this->createResponse(429, ['Retry-After: '])));
|
||||||
|
self::assertNull(self::parseRetryAfter($this->createResponse(429, [])));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRateLimitedResponseIsRetriedAfterTheInterval(): void
|
||||||
|
{
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => true, 'delay' => 5.0],
|
||||||
|
$this->decideRetry($this->createJob(), $this->createResponse(429, ['Retry-After: 5']))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testZeroRetryAfterUsesTheDefaultBackoff(): void
|
||||||
|
{
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => true, 'delay' => null],
|
||||||
|
$this->decideRetry($this->createJob(), $this->createResponse(429, ['Retry-After: 0']))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRateLimitedResponseWithoutRetryAfterIsNotRetried(): void
|
||||||
|
{
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => false, 'delay' => null],
|
||||||
|
$this->decideRetry($this->createJob(), $this->createResponse(429, []))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryableStatusCodeHonorsRetryAfter(): void
|
||||||
|
{
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => true, 'delay' => 30.0],
|
||||||
|
$this->decideRetry($this->createJob(), $this->createResponse(503, ['Retry-After: 30']))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryableStatusCodeWithoutRetryAfterUsesTheDefaultBackoff(): void
|
||||||
|
{
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => true, 'delay' => null],
|
||||||
|
$this->decideRetry($this->createJob(), $this->createResponse(503, []))
|
||||||
|
);
|
||||||
|
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => true, 'delay' => null],
|
||||||
|
$this->decideRetry($this->createJob(), $this->createResponse(503, ['Retry-After: whenever']))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryAfterBeyondTheCapIsNotWaitedOut(): void
|
||||||
|
{
|
||||||
|
// 429 is only retryable because the header says when to come back, so an interval we will
|
||||||
|
// not wait for leaves nothing to act on
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => false, 'delay' => null],
|
||||||
|
$this->decideRetry($this->createJob(), $this->createResponse(429, ['Retry-After: 3600']))
|
||||||
|
);
|
||||||
|
|
||||||
|
// a status code which was retryable before this header was read must stay retryable, on the
|
||||||
|
// default backoff, so the cap cannot turn a retry we would have made anyway into a failure
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => true, 'delay' => null],
|
||||||
|
$this->decideRetry($this->createJob(), $this->createResponse(503, ['Retry-After: 3600']))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryAfterDoesNotDefeatMaxRetries(): void
|
||||||
|
{
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => false, 'delay' => null],
|
||||||
|
$this->decideRetry($this->createJob(3), $this->createResponse(429, ['Retry-After: 5']))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testNonGetRequestsAreNotRetried(): void
|
||||||
|
{
|
||||||
|
$job = $this->createJob();
|
||||||
|
$job['options'] = ['http' => ['method' => 'POST']];
|
||||||
|
|
||||||
|
self::assertSame(
|
||||||
|
['retry' => false, 'delay' => null],
|
||||||
|
$this->decideRetry($job, $this->createResponse(429, ['Retry-After: 5']))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testDelayedRetryDoesNotBlockTheDownloadLoop(): void
|
||||||
|
{
|
||||||
|
$downloader = $this->createDownloader();
|
||||||
|
|
||||||
|
// tick() drives every transfer which is in flight, so a delay which is waited out rather
|
||||||
|
// than scheduled stalls all of them, not just the request which is being retried
|
||||||
|
$start = microtime(true);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 30.0);
|
||||||
|
$downloader->tick();
|
||||||
|
$elapsed = microtime(true) - $start;
|
||||||
|
|
||||||
|
self::assertLessThan(1.0, $elapsed);
|
||||||
|
self::assertCount(1, $this->readDelayedJobs($downloader));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testSelectTimeoutIsShortenedToTheNextDueRetry(): void
|
||||||
|
{
|
||||||
|
$downloader = $this->createDownloader();
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 0.25);
|
||||||
|
|
||||||
|
$timeout = self::callPrivate($downloader, 'getSelectTimeout');
|
||||||
|
|
||||||
|
self::assertIsFloat($timeout);
|
||||||
|
self::assertGreaterThan(0.0, $timeout);
|
||||||
|
self::assertLessThanOrEqual(0.25, $timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testDueRetryIsStartedWhileALaterOneKeepsWaiting(): void
|
||||||
|
{
|
||||||
|
$downloader = $this->createDownloader();
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(0, 'other.org'), 30.0);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 0.0001);
|
||||||
|
usleep(1000);
|
||||||
|
|
||||||
|
self::callPrivate($downloader, 'restartDueJobs');
|
||||||
|
|
||||||
|
self::assertCount(1, $this->readDelayedJobs($downloader), 'the retry which is not due yet must still be waiting');
|
||||||
|
|
||||||
|
$jobs = $this->readProperty($downloader, 'jobs');
|
||||||
|
self::assertIsArray($jobs);
|
||||||
|
self::assertCount(1, $jobs, 'the due retry must have been started');
|
||||||
|
|
||||||
|
foreach (array_keys($jobs) as $id) {
|
||||||
|
$downloader->abortRequest((int) $id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testAbortedRequestIsNotRetried(): void
|
||||||
|
{
|
||||||
|
$downloader = $this->createDownloader();
|
||||||
|
$job = $this->createJob();
|
||||||
|
$this->scheduleRetry($downloader, $job, 30.0);
|
||||||
|
|
||||||
|
// a request is cancelled by the id of the handle it was first sent on
|
||||||
|
$downloader->abortRequest((int) $job['curlHandle']);
|
||||||
|
self::assertSame([], $this->readDelayedJobs($downloader));
|
||||||
|
|
||||||
|
$this->letWaitsRunOut($downloader);
|
||||||
|
self::callPrivate($downloader, 'restartDueJobs');
|
||||||
|
self::assertSame([], $this->readProperty($downloader, 'jobs'), 'a cancelled request must not be sent again');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryWaitsForTheLatestRetryAfterOfItsOrigin(): void
|
||||||
|
{
|
||||||
|
$io = new BufferIO();
|
||||||
|
$downloader = $this->createDownloader($io);
|
||||||
|
// another request to example.org was told to come back in 30s, which holds for the next ones too
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 30.0);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 0.0001);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(3), null);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(0, 'other.org'), 0.0001);
|
||||||
|
usleep(1000);
|
||||||
|
|
||||||
|
self::callPrivate($downloader, 'restartDueJobs');
|
||||||
|
|
||||||
|
$delayedJobs = $this->readDelayedJobs($downloader);
|
||||||
|
self::assertCount(3, $delayedJobs, 'retries to example.org must wait for its deadline, whether or not they got a Retry-After');
|
||||||
|
foreach ($delayedJobs as $delayedJob) {
|
||||||
|
self::assertSame('example.org', $delayedJob['job']['origin']);
|
||||||
|
}
|
||||||
|
|
||||||
|
$jobs = $this->readProperty($downloader, 'jobs');
|
||||||
|
self::assertIsArray($jobs);
|
||||||
|
self::assertCount(1, $jobs, 'a retry to another origin must not be held up');
|
||||||
|
foreach (array_keys($jobs) as $id) {
|
||||||
|
$downloader->abortRequest((int) $id);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->announceRetryAfterWaits($downloader);
|
||||||
|
self::assertStringContainsString('(3 pending), waiting 30s before retrying', $io->getOutput());
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testFirstRetryToABlockedOriginIsNotStartedStraightAway(): void
|
||||||
|
{
|
||||||
|
$downloader = $this->createDownloader();
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 30.0);
|
||||||
|
|
||||||
|
// the first retry has no backoff of its own
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(0), null);
|
||||||
|
|
||||||
|
self::assertCount(2, $this->readDelayedJobs($downloader));
|
||||||
|
self::assertSame([], $this->readProperty($downloader, 'jobs'));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryAfterWaitIsAnnouncedOncePerOrigin(): void
|
||||||
|
{
|
||||||
|
$io = new BufferIO();
|
||||||
|
$downloader = $this->createDownloader($io);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 10.0);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 30.0);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(0, 'other.org'), 30.0);
|
||||||
|
|
||||||
|
$this->announceRetryAfterWaits($downloader);
|
||||||
|
$this->announceRetryAfterWaits($downloader);
|
||||||
|
|
||||||
|
self::assertSame(
|
||||||
|
'<warning>example.org is rate limiting requests (2 pending), waiting 30s before retrying</warning>'.PHP_EOL
|
||||||
|
.'<warning>other.org is rate limiting requests (1 pending), waiting 30s before retrying</warning>'.PHP_EOL,
|
||||||
|
$io->getOutput()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryAfterWaitIsAnnouncedWhileRequestsToTheOriginAreStillInFlight(): void
|
||||||
|
{
|
||||||
|
$io = new BufferIO();
|
||||||
|
$downloader = $this->createDownloader($io);
|
||||||
|
|
||||||
|
// a long transfer to the origin can outlast the whole wait, so the notice must not wait for it
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 30.0);
|
||||||
|
$this->writeProperty($downloader, 'jobs', [1 => ['origin' => 'example.org']]);
|
||||||
|
$this->announceRetryAfterWaits($downloader);
|
||||||
|
|
||||||
|
self::assertSame('<warning>example.org is rate limiting requests (1 pending), waiting 30s before retrying</warning>'.PHP_EOL, $io->getOutput());
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRetryAfterWaitIsAnnouncedAgainOnceTheOriginTurnsRequestsAwayAnew(): void
|
||||||
|
{
|
||||||
|
$io = new BufferIO();
|
||||||
|
$downloader = $this->createDownloader($io);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(), 1.0, 503);
|
||||||
|
$this->announceRetryAfterWaits($downloader);
|
||||||
|
|
||||||
|
// the wait runs out and the retry is started, which ends the episode
|
||||||
|
$this->letWaitsRunOut($downloader);
|
||||||
|
self::callPrivate($downloader, 'restartDueJobs');
|
||||||
|
$this->announceRetryAfterWaits($downloader);
|
||||||
|
$jobs = $this->readProperty($downloader, 'jobs');
|
||||||
|
self::assertIsArray($jobs);
|
||||||
|
foreach (array_keys($jobs) as $id) {
|
||||||
|
$downloader->abortRequest((int) $id);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(1), 5.0);
|
||||||
|
$this->announceRetryAfterWaits($downloader);
|
||||||
|
|
||||||
|
self::assertSame(
|
||||||
|
'<warning>example.org responded with status code 503 (1 pending), waiting 1s before retrying</warning>'.PHP_EOL
|
||||||
|
.'<warning>example.org is rate limiting requests (1 pending), waiting 5s before retrying</warning>'.PHP_EOL,
|
||||||
|
$io->getOutput()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testDefaultBackoffIsNotAnnounced(): void
|
||||||
|
{
|
||||||
|
$io = new BufferIO();
|
||||||
|
$downloader = $this->createDownloader($io);
|
||||||
|
$this->scheduleRetry($downloader, $this->createJob(3), null);
|
||||||
|
|
||||||
|
$this->announceRetryAfterWaits($downloader);
|
||||||
|
|
||||||
|
self::assertCount(1, $this->readDelayedJobs($downloader));
|
||||||
|
self::assertSame('', $io->getOutput());
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testJsonResponseIsRecognisedWithCharset(): void
|
||||||
|
{
|
||||||
|
self::assertTrue(self::callPrivate(CurlDownloader::class, 'isJsonResponse', $this->createResponse(429, ['Content-Type: application/json'])));
|
||||||
|
self::assertTrue(self::callPrivate(CurlDownloader::class, 'isJsonResponse', $this->createResponse(429, ['Content-Type: application/json; charset=utf-8'])));
|
||||||
|
self::assertTrue(self::callPrivate(CurlDownloader::class, 'isJsonResponse', $this->createResponse(429, ['Content-Type: Application/JSON; charset=UTF-8'])));
|
||||||
|
self::assertFalse(self::callPrivate(CurlDownloader::class, 'isJsonResponse', $this->createResponse(429, ['Content-Type: text/html'])));
|
||||||
|
self::assertFalse(self::callPrivate(CurlDownloader::class, 'isJsonResponse', $this->createResponse(429, [])));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testFailureReportsARetryAfterWhichWasNotWaitedOut(): void
|
||||||
|
{
|
||||||
|
self::assertSame(
|
||||||
|
'HTTP/1.1 429 Status, the server asked to retry in 3600s which is longer than the 60s Composer is willing to wait, try again later',
|
||||||
|
self::callPrivate(CurlDownloader::class, 'getStatusFailureMessage', $this->createResponse(429, ['Retry-After: 3600']))
|
||||||
|
);
|
||||||
|
|
||||||
|
// an interval which was waited out, or no interval at all, has nothing to add
|
||||||
|
self::assertSame('HTTP/1.1 429 Status', self::callPrivate(CurlDownloader::class, 'getStatusFailureMessage', $this->createResponse(429, ['Retry-After: 5'])));
|
||||||
|
self::assertSame('HTTP/1.1 429 Status', self::callPrivate(CurlDownloader::class, 'getStatusFailureMessage', $this->createResponse(429, [])));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param list<string> $headers
|
||||||
|
*/
|
||||||
|
private function createResponse(int $statusCode, array $headers): Response
|
||||||
|
{
|
||||||
|
array_unshift($headers, 'HTTP/1.1 '.$statusCode.' Status');
|
||||||
|
|
||||||
|
return new Response(['url' => 'https://example.org/packages.json'], $statusCode, $headers, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return mixed[]
|
||||||
|
*/
|
||||||
|
private function createJob(int $retries = 0, string $origin = 'example.org'): array
|
||||||
|
{
|
||||||
|
return [
|
||||||
|
'url' => 'https://'.$origin.'/packages.json',
|
||||||
|
'origin' => $origin,
|
||||||
|
'attributes' => ['retryAuthFailure' => false, 'redirects' => 0, 'retries' => $retries, 'storeAuth' => false, 'ipResolve' => null],
|
||||||
|
'options' => [],
|
||||||
|
'filename' => null,
|
||||||
|
'curlHandle' => curl_init(),
|
||||||
|
'resolve' => static function (): void {
|
||||||
|
},
|
||||||
|
'reject' => static function (): void {
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
private function createDownloader(?IOInterface $io = null): CurlDownloader
|
||||||
|
{
|
||||||
|
$config = $this->getMockBuilder('Composer\Config')->getMock();
|
||||||
|
$config->expects($this->any())
|
||||||
|
->method('get')
|
||||||
|
->will($this->returnCallback(static function ($key) {
|
||||||
|
if ($key === 'github-domains' || $key === 'gitlab-domains') {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}));
|
||||||
|
|
||||||
|
return new CurlDownloader($io ?? $this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $config);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param mixed[] $job
|
||||||
|
*/
|
||||||
|
private function scheduleRetry(CurlDownloader $downloader, array $job, ?float $delay, int $statusCode = 429): void
|
||||||
|
{
|
||||||
|
self::callPrivate($downloader, 'restartJobWithDelay', $job, $job['url'], $job['attributes'], $delay, $statusCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function announceRetryAfterWaits(CurlDownloader $downloader): void
|
||||||
|
{
|
||||||
|
self::callPrivate($downloader, 'announceRetryAfterWaits');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param mixed[] $job
|
||||||
|
* @return mixed
|
||||||
|
*/
|
||||||
|
private function decideRetry(array $job, Response $response)
|
||||||
|
{
|
||||||
|
return self::callPrivate($this->createDownloader(), 'isStatusCodeRetryNeeded', $job, $response);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return mixed
|
||||||
|
*/
|
||||||
|
private static function parseRetryAfter(Response $response)
|
||||||
|
{
|
||||||
|
return self::callPrivate(CurlDownloader::class, 'getRetryAfterDelay', $response);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param object|class-string $target
|
||||||
|
* @param mixed ...$args
|
||||||
|
* @return mixed
|
||||||
|
*/
|
||||||
|
private static function callPrivate($target, string $method, ...$args)
|
||||||
|
{
|
||||||
|
$reflection = new \ReflectionMethod($target, $method);
|
||||||
|
if (\PHP_VERSION_ID < 80100) {
|
||||||
|
$reflection->setAccessible(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $reflection->invoke(is_object($target) ? $target : null, ...$args);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return mixed[]
|
||||||
|
*/
|
||||||
|
private function readDelayedJobs(CurlDownloader $downloader): array
|
||||||
|
{
|
||||||
|
$delayedJobs = $this->readProperty($downloader, 'delayedJobs');
|
||||||
|
self::assertIsArray($delayedJobs);
|
||||||
|
|
||||||
|
return $delayedJobs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return mixed
|
||||||
|
*/
|
||||||
|
private function readProperty(CurlDownloader $downloader, string $name)
|
||||||
|
{
|
||||||
|
$property = new \ReflectionProperty($downloader, $name);
|
||||||
|
if (\PHP_VERSION_ID < 80100) {
|
||||||
|
$property->setAccessible(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $property->getValue($downloader);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Moves every delayed retry and origin hold into the past, as if their wait had run out
|
||||||
|
*/
|
||||||
|
private function letWaitsRunOut(CurlDownloader $downloader): void
|
||||||
|
{
|
||||||
|
$delayedJobs = $this->readDelayedJobs($downloader);
|
||||||
|
foreach (array_keys($delayedJobs) as $i) {
|
||||||
|
$delayedJobs[$i]['at'] = 0.0;
|
||||||
|
}
|
||||||
|
$this->writeProperty($downloader, 'delayedJobs', $delayedJobs);
|
||||||
|
|
||||||
|
$origins = $this->readProperty($downloader, 'retryAfterOrigins');
|
||||||
|
self::assertIsArray($origins);
|
||||||
|
foreach (array_keys($origins) as $origin) {
|
||||||
|
$origins[$origin]['until'] = 0.0;
|
||||||
|
}
|
||||||
|
$this->writeProperty($downloader, 'retryAfterOrigins', $origins);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param mixed $value
|
||||||
|
*/
|
||||||
|
private function writeProperty(CurlDownloader $downloader, string $name, $value): void
|
||||||
|
{
|
||||||
|
$property = new \ReflectionProperty($downloader, $name);
|
||||||
|
if (\PHP_VERSION_ID < 80100) {
|
||||||
|
$property->setAccessible(true);
|
||||||
|
}
|
||||||
|
$property->setValue($downloader, $value);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user