fix: retry the installer's file and target lists

install.ps1

@@ -8,7 +8,7 @@ param(
 
 $ErrorActionPreference = "Stop"
 $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-$defaultInstallSha256 = "68fa53debdfbe8d16310d037fb50ab9231fa4b51f2df5c4b8984de7a61d01374"
+$defaultInstallSha256 = "39f3864183bd67c30c5705fccf39961aac0ad2d40238398ab2a67ef329eff359"
 $maximumInstallerBytes = 2MB
 $installerExit = 0
 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }

install.py

@@ -1528,16 +1528,45 @@ def _validate_targets(data: object) -> dict[str, dict]:
     return normalized
 
 
+NETWORK_ADVICE = ("  Check the connection, and HTTPS_PROXY if this network needs a "
+                  "proxy, then run the same command again.")
+
+
+def _fetch_json(url: str, limit: int, label: str) -> object:
+    """A JSON document, tried MAX_RETRIES times like every file download.
+
+    A file list fetched once ended the run on one timeout over flaky Wi-Fi.
+    A 4xx answer or an invalid document is final: asking again changes
+    nothing.
+    """
+    for attempt in range(1, MAX_RETRIES + 1):
+        try:
+            with urllib.request.urlopen(url, timeout=30) as resp:
+                return _read_limited_json(resp, limit, label)
+        except urllib.error.HTTPError as exc:
+            if exc.code < 500 and exc.code != 429 or attempt == MAX_RETRIES:
+                raise
+            delay = _retry_delay(attempt, exc.headers.get("Retry-After") if exc.headers else None)
+        except (urllib.error.URLError, OSError):
+            if attempt == MAX_RETRIES:
+                raise
+            delay = _retry_delay(attempt, None)
+        time.sleep(delay)
+    raise AssertionError("unreachable")
+
+
 def fetch_manifest(plat: str) -> dict:
     """Download platform manifest JSON."""
     url = MANIFEST_URL.format(platform=plat)
     try:
-        with urllib.request.urlopen(url, timeout=30) as resp:
-            return _validate_manifest(
-                _read_limited_json(resp, MAX_MANIFEST_BYTES, "manifest"), plat
-            )
-    except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError) as exc:
-        print(f"  Failed to fetch manifest for {plat}: {exc}", file=sys.stderr)
+        return _validate_manifest(_fetch_json(url, MAX_MANIFEST_BYTES, "manifest"), plat)
+    except ValueError as exc:
+        print(f"  The file list for {plat} is not valid: {exc}", file=sys.stderr)
+        sys.exit(1)
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"  Could not fetch the file list for {plat} after {MAX_RETRIES} "
+              f"attempts: {getattr(exc, 'reason', None) or exc}", file=sys.stderr)
+        print(NETWORK_ADVICE, file=sys.stderr)
         sys.exit(1)
 
 
@@ -1550,17 +1579,20 @@ def fetch_targets(plat: str) -> "dict | None":
     """
     url = TARGETS_URL.format(platform=plat)
     try:
-        with urllib.request.urlopen(url, timeout=30) as resp:
-            return _validate_targets(
-                _read_limited_json(resp, MAX_TARGETS_BYTES, "targets manifest")
-            )
+        return _validate_targets(_fetch_json(url, MAX_TARGETS_BYTES, "targets manifest"))
     except urllib.error.HTTPError as exc:
         if exc.code == 404:
             return {}
         print(f"  Error: failed to fetch targets for {plat}: {exc}", file=sys.stderr)
+        print(NETWORK_ADVICE, file=sys.stderr)
         return None
-    except (urllib.error.URLError, OSError, ValueError) as exc:
-        print(f"  Error: failed to fetch targets for {plat}: {exc}", file=sys.stderr)
+    except ValueError as exc:
+        print(f"  Error: the target list for {plat} is not valid: {exc}", file=sys.stderr)
+        return None
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"  Error: failed to fetch targets for {plat}: "
+              f"{getattr(exc, 'reason', None) or exc}", file=sys.stderr)
+        print(NETWORK_ADVICE, file=sys.stderr)
         return None
 
 

install.sh

@@ -18,7 +18,7 @@ esac
 TEMP_INSTALLER=""
 TEMP_DIRECTORY=""
 DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-DEFAULT_INSTALL_SHA256="68fa53debdfbe8d16310d037fb50ab9231fa4b51f2df5c4b8984de7a61d01374"
+DEFAULT_INSTALL_SHA256="39f3864183bd67c30c5705fccf39961aac0ad2d40238398ab2a67ef329eff359"
 MAX_INSTALLER_BYTES=2097152
 
 cleanup() {

tests/test_install.py

@@ -1973,6 +1973,60 @@ class TestAvailablePlatforms(unittest.TestCase):
         self.assertIn("installer stdin is a terminal: True", proc.stdout)
 
 
+class FileListsAreRetried(unittest.TestCase):
+    """The file list was fetched once with a 30 s timeout: one timeout on
+    flaky Wi-Fi ended the run, while each file got three attempts."""
+
+    def _fetch(self, outcomes):
+        calls = []
+
+        def urlopen(url, timeout=None):
+            calls.append(url)
+            outcome = outcomes.pop(0)
+            if isinstance(outcome, Exception):
+                raise outcome
+            return contextlib.nullcontext(io.BytesIO(outcome))
+
+        with unittest.mock.patch.object(install.urllib.request, "urlopen", urlopen), \
+                unittest.mock.patch.object(install.time, "sleep"), \
+                unittest.mock.patch.object(install, "_read_limited_json",
+                                           lambda resp, limit, label: json.loads(resp.read())):
+            try:
+                return install._fetch_json("https://x/list.json", 1000, "list"), calls
+            except OSError as exc:
+                return exc, calls
+
+    def test_a_timeout_then_an_answer_succeeds(self):
+        import urllib.error
+
+        result, calls = self._fetch([urllib.error.URLError("timed out"), b'{"a": 1}'])
+        self.assertEqual(result, {"a": 1})
+        self.assertEqual(len(calls), 2)
+
+    def test_a_404_is_not_retried(self):
+        import urllib.error
+
+        missing = urllib.error.HTTPError("https://x", 404, "Not Found", None, None)
+        result, calls = self._fetch([missing])
+        self.assertIs(result, missing)
+        self.assertEqual(len(calls), 1)
+
+    def test_three_failures_end_with_advice(self):
+        import urllib.error
+
+        failures = [urllib.error.URLError("timed out")] * install.MAX_RETRIES
+        err = io.StringIO()
+        with unittest.mock.patch.object(install.urllib.request, "urlopen",
+                                        side_effect=failures), \
+                unittest.mock.patch.object(install.time, "sleep"), \
+                contextlib.redirect_stderr(err):
+            with self.assertRaises(SystemExit) as caught:
+                install.fetch_manifest("retroarch")
+        self.assertEqual(caught.exception.code, 1)
+        self.assertIn("HTTPS_PROXY", err.getvalue())
+        self.assertIn(f"after {install.MAX_RETRIES} attempts", err.getvalue())
+
+
 class TargetListFailureIsNotAnAnswer(unittest.TestCase):
     """A timeout is not 'no targets': it sent users to the full pack."""