diff --git a/install.ps1 b/install.ps1 index ccf3f6e2..b08adde8 100644 --- a/install.ps1 +++ b/install.ps1 @@ -8,7 +8,7 @@ param( $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -$defaultInstallSha256 = "68fa53debdfbe8d16310d037fb50ab9231fa4b51f2df5c4b8984de7a61d01374" +$defaultInstallSha256 = "39f3864183bd67c30c5705fccf39961aac0ad2d40238398ab2a67ef329eff359" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 6c30535a..cc8f99fc 100644 --- a/install.py +++ b/install.py @@ -1528,16 +1528,45 @@ def _validate_targets(data: object) -> dict[str, dict]: return normalized +NETWORK_ADVICE = (" Check the connection, and HTTPS_PROXY if this network needs a " + "proxy, then run the same command again.") + + +def _fetch_json(url: str, limit: int, label: str) -> object: + """A JSON document, tried MAX_RETRIES times like every file download. + + A file list fetched once ended the run on one timeout over flaky Wi-Fi. + A 4xx answer or an invalid document is final: asking again changes + nothing. + """ + for attempt in range(1, MAX_RETRIES + 1): + try: + with urllib.request.urlopen(url, timeout=30) as resp: + return _read_limited_json(resp, limit, label) + except urllib.error.HTTPError as exc: + if exc.code < 500 and exc.code != 429 or attempt == MAX_RETRIES: + raise + delay = _retry_delay(attempt, exc.headers.get("Retry-After") if exc.headers else None) + except (urllib.error.URLError, OSError): + if attempt == MAX_RETRIES: + raise + delay = _retry_delay(attempt, None) + time.sleep(delay) + raise AssertionError("unreachable") + + def fetch_manifest(plat: str) -> dict: """Download platform manifest JSON.""" url = MANIFEST_URL.format(platform=plat) try: - with urllib.request.urlopen(url, timeout=30) as resp: - return _validate_manifest( - _read_limited_json(resp, MAX_MANIFEST_BYTES, "manifest"), plat - ) - except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError) as exc: - print(f" Failed to fetch manifest for {plat}: {exc}", file=sys.stderr) + return _validate_manifest(_fetch_json(url, MAX_MANIFEST_BYTES, "manifest"), plat) + except ValueError as exc: + print(f" The file list for {plat} is not valid: {exc}", file=sys.stderr) + sys.exit(1) + except (urllib.error.URLError, OSError) as exc: + print(f" Could not fetch the file list for {plat} after {MAX_RETRIES} " + f"attempts: {getattr(exc, 'reason', None) or exc}", file=sys.stderr) + print(NETWORK_ADVICE, file=sys.stderr) sys.exit(1) @@ -1550,17 +1579,20 @@ def fetch_targets(plat: str) -> "dict | None": """ url = TARGETS_URL.format(platform=plat) try: - with urllib.request.urlopen(url, timeout=30) as resp: - return _validate_targets( - _read_limited_json(resp, MAX_TARGETS_BYTES, "targets manifest") - ) + return _validate_targets(_fetch_json(url, MAX_TARGETS_BYTES, "targets manifest")) except urllib.error.HTTPError as exc: if exc.code == 404: return {} print(f" Error: failed to fetch targets for {plat}: {exc}", file=sys.stderr) + print(NETWORK_ADVICE, file=sys.stderr) return None - except (urllib.error.URLError, OSError, ValueError) as exc: - print(f" Error: failed to fetch targets for {plat}: {exc}", file=sys.stderr) + except ValueError as exc: + print(f" Error: the target list for {plat} is not valid: {exc}", file=sys.stderr) + return None + except (urllib.error.URLError, OSError) as exc: + print(f" Error: failed to fetch targets for {plat}: " + f"{getattr(exc, 'reason', None) or exc}", file=sys.stderr) + print(NETWORK_ADVICE, file=sys.stderr) return None diff --git a/install.sh b/install.sh index a1e542a2..9f774e88 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="68fa53debdfbe8d16310d037fb50ab9231fa4b51f2df5c4b8984de7a61d01374" +DEFAULT_INSTALL_SHA256="39f3864183bd67c30c5705fccf39961aac0ad2d40238398ab2a67ef329eff359" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 88747002..f13e38c5 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1973,6 +1973,60 @@ class TestAvailablePlatforms(unittest.TestCase): self.assertIn("installer stdin is a terminal: True", proc.stdout) +class FileListsAreRetried(unittest.TestCase): + """The file list was fetched once with a 30 s timeout: one timeout on + flaky Wi-Fi ended the run, while each file got three attempts.""" + + def _fetch(self, outcomes): + calls = [] + + def urlopen(url, timeout=None): + calls.append(url) + outcome = outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return contextlib.nullcontext(io.BytesIO(outcome)) + + with unittest.mock.patch.object(install.urllib.request, "urlopen", urlopen), \ + unittest.mock.patch.object(install.time, "sleep"), \ + unittest.mock.patch.object(install, "_read_limited_json", + lambda resp, limit, label: json.loads(resp.read())): + try: + return install._fetch_json("https://x/list.json", 1000, "list"), calls + except OSError as exc: + return exc, calls + + def test_a_timeout_then_an_answer_succeeds(self): + import urllib.error + + result, calls = self._fetch([urllib.error.URLError("timed out"), b'{"a": 1}']) + self.assertEqual(result, {"a": 1}) + self.assertEqual(len(calls), 2) + + def test_a_404_is_not_retried(self): + import urllib.error + + missing = urllib.error.HTTPError("https://x", 404, "Not Found", None, None) + result, calls = self._fetch([missing]) + self.assertIs(result, missing) + self.assertEqual(len(calls), 1) + + def test_three_failures_end_with_advice(self): + import urllib.error + + failures = [urllib.error.URLError("timed out")] * install.MAX_RETRIES + err = io.StringIO() + with unittest.mock.patch.object(install.urllib.request, "urlopen", + side_effect=failures), \ + unittest.mock.patch.object(install.time, "sleep"), \ + contextlib.redirect_stderr(err): + with self.assertRaises(SystemExit) as caught: + install.fetch_manifest("retroarch") + self.assertEqual(caught.exception.code, 1) + self.assertIn("HTTPS_PROXY", err.getvalue()) + self.assertIn(f"after {install.MAX_RETRIES} attempts", err.getvalue()) + + class TargetListFailureIsNotAnAnswer(unittest.TestCase): """A timeout is not 'no targets': it sent users to the full pack."""