From 75a63569590ec36b44535773f9b486524bc0fd82 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:14:08 +0200 Subject: [PATCH] feat: say network failures in plain words Offline, the run printed "[Errno -2] Name or service not known". The common failures now read as words, the raw text kept under --verbose. A refused certificate under a clock set before the installer was written is reported as reason clock, with the date it reads, and no trusted certificate list on the machine as reason no_ca, naming SSL_CERT_FILE. Neither is retried, and verification is never turned off. --- install.ps1 | 2 +- install.py | 131 ++++++++++++++++++++++++++++++++++++---- install.sh | 2 +- tests/test_install.py | 88 ++++++++++++++++++++++++++- wiki/installer.md | 16 ++++- wiki/troubleshooting.md | 2 + 6 files changed, 225 insertions(+), 16 deletions(-) diff --git a/install.ps1 b/install.ps1 index 37aaf354..da599e0c 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "a0b6a103d350e49f2751928c5a5560e239c52df81637a656e0691b9934e438bb" + $defaultInstallSha256 = "ea5d767eeb7612312e6a8fbf5f67b578fdc746f2d34ccc46cf96b978be7424ce" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 81e6386f..39a8d488 100644 --- a/install.py +++ b/install.py @@ -16,6 +16,7 @@ from __future__ import annotations import argparse import concurrent.futures +import datetime import errno import hashlib import http.client @@ -478,8 +479,10 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { "start": ("schema", "installer", "mode"), # platforms: [{"platform", "name", "bios_dir"}], empty when none is found. "detect": ("os", "os_name", "platforms"), - # stage: manifest (reason: network, not_found or invalid) or detect - # (reason: conflict, two systems found on one machine). + # stage: manifest (reason: network, not_found, invalid, clock for a + # refused certificate under a clock set in the past, or no_ca for no + # trusted certificates on the machine) or detect (reason: conflict, two + # systems found on one machine). "error": ("stage", "reason", "platform", "message"), # state: present (hash verified), wrong_hash or missing. "file": ("platform", "dest", "systems", "size", "state"), @@ -2977,6 +2980,102 @@ NETWORK_ADVICE = (" Check the connection, and HTTPS_PROXY if this network needs f" More help: {TROUBLESHOOTING_PAGE}") +# The installer was written after this day. A clock that reads earlier is +# wrong, and TLS then finds every certificate not yet valid: a handheld +# with no clock battery boots on 1970 or 1980. +CLOCK_FLOOR = datetime.date(2026, 1, 1) +# Where systems keep their trusted certificates, for a Python built to look +# elsewhere: Debian, Fedora, openSUSE, the BSDs and macOS, Homebrew, Termux. +CA_BUNDLES = ( + "/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/ca-bundle.pem", "/etc/pki/tls/cacert.pem", "/etc/ssl/cert.pem", + "/usr/local/etc/openssl/cert.pem", "/opt/homebrew/etc/openssl@3/cert.pem", + "/data/data/com.termux/files/usr/etc/tls/cert.pem", +) +# Set from --verbose: the error text Python gives stays beside the words. +_RAW_ERRORS = threading.Event() + + +def _default_ca_present() -> bool: + """Whether the certificates this Python reads by default exist.""" + import ssl # noqa: PLC0415 + + paths = ssl.get_default_verify_paths() + for value in (os.environ.get("SSL_CERT_FILE"), paths.cafile, paths.openssl_cafile): + if value and Path(value).is_file(): + return True + for value in (os.environ.get("SSL_CERT_DIR"), paths.capath, paths.openssl_capath): + try: + if value and any(Path(value).iterdir()): + return True + except OSError: + continue + return False + + +def _known_bundle() -> str: + """A certificate bundle the system keeps where this Python does not look.""" + return next((path for path in CA_BUNDLES if Path(path).is_file()), "") + + +def _ca_bundle_found() -> bool: + """Whether any list of trusted certificates is on this machine. + + Windows verifies against its own store, which Python reads directly. + """ + return os.name == "nt" or _default_ca_present() or bool(_known_bundle()) + + +def network_words( + exc: BaseException, today: datetime.date | None = None, verbose: bool | None = None, +) -> tuple[str, str]: + """The reason a request failed, and the same in words a beginner reads. + + The reason is clock, no_ca or network. Offline, the run printed + "[Errno -2] Name or service not known": the raw text now follows the + words only under --verbose. + """ + reason = getattr(exc, "reason", None) + inner = reason if isinstance(reason, BaseException) else exc + raw = str(reason or exc) + code, words = "network", "" + cert_error = getattr(sys.modules.get("ssl"), "SSLCertVerificationError", None) + if cert_error is not None and isinstance(inner, cert_error): + today = today or datetime.date.today() + if not _ca_bundle_found(): + code = "no_ca" + words = ("This system has no list of trusted certificates to check the " + "server with. Install the ca-certificates package, or name a " + "certificate bundle with SSL_CERT_FILE, then run again") + elif today < CLOCK_FLOOR: + code = "clock" + words = (f"The clock reads {today.isoformat()}: connect to Wi-Fi and let the " + "time sync, or set the date, then run again") + else: + words = "the server's certificate was rejected" + bundle = "" if _default_ca_present() else _known_bundle() + words += (f": this Python reads no certificate list, run again with " + f"SSL_CERT_FILE={bundle}" if bundle + else ", which a proxy or a filter on this network can cause") + elif isinstance(inner, socket.gaierror): + words = "no internet connection, or the server name could not be looked up" + elif isinstance(inner, (socket.timeout, TimeoutError)): + words = "the connection timed out" + elif isinstance(inner, ConnectionRefusedError): + words = "the server refused the connection" + elif isinstance(inner, OSError) and inner.errno in ( + errno.ENETUNREACH, errno.EHOSTUNREACH, errno.ENETDOWN): + words = "no internet connection: the network cannot be reached" + elif isinstance(inner, (ConnectionResetError, ConnectionAbortedError, + http.client.HTTPException)): + words = "the connection was cut" + if not words: + return code, raw + if _RAW_ERRORS.is_set() if verbose is None else verbose: + words += f" [{raw}]" + return code, words + + def _address_lines(url: str) -> list[str]: """The address that failed, and the override that chose it if one did. @@ -3008,12 +3107,13 @@ def _fetch_json(url: str, limit: int, label: str) -> object: delay = _retry_delay(attempt, exc.headers.get("Retry-After") if exc.headers else None) problem = f"HTTP {exc.code}" except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: - if attempt == MAX_RETRIES: + code, problem = network_words(exc) + # A wrong clock or no certificate list fails every attempt alike. + if attempt == MAX_RETRIES or code != "network": if isinstance(exc, http.client.HTTPException): raise ConnectionError(f"connection cut: {exc!r}") from exc raise delay = _retry_delay(attempt, None) - problem = str(getattr(exc, "reason", None) or exc) # Said, or three 30-second attempts read as a frozen program. print(f" The {label} did not arrive ({problem}), trying again " f"({attempt + 1}/{MAX_RETRIES}) in {delay:.0f} s.", file=sys.stderr) @@ -3034,7 +3134,7 @@ def fetch_manifest(plat: str) -> dict: sys.exit(1) url = MANIFEST_URL.format(platform=plat) try: - return _validate_manifest(_fetch_json(url, MAX_MANIFEST_BYTES, "manifest"), plat) + return _validate_manifest(_fetch_json(url, MAX_MANIFEST_BYTES, "file list"), plat) except ValueError as exc: EVENTS.emit("error", stage="manifest", reason="invalid", platform=plat, message=f"the file list is not valid: {exc}") @@ -3068,11 +3168,16 @@ def fetch_manifest(plat: str) -> dict: print(NETWORK_ADVICE, file=sys.stderr) sys.exit(1) except (urllib.error.URLError, OSError) as exc: - EVENTS.emit("error", stage="manifest", reason="network", platform=plat, - message=f"the file list could not be fetched: " - f"{getattr(exc, 'reason', None) or exc}") + code, words = network_words(exc) + EVENTS.emit("error", stage="manifest", reason=code, platform=plat, + message=words if code != "network" + else f"the file list could not be fetched: {words}") + if code != "network": + print(f" Could not fetch the file list for {plat}.", file=sys.stderr) + print(f" {words}.", file=sys.stderr) + sys.exit(1) print(f" Could not fetch the file list for {plat} after {MAX_RETRIES} " - f"attempts: {getattr(exc, 'reason', None) or exc}", file=sys.stderr) + f"attempts: {words}", file=sys.stderr) for line in _address_lines(url): print(line, file=sys.stderr) print(NETWORK_ADVICE, file=sys.stderr) @@ -3088,7 +3193,7 @@ def fetch_targets(plat: str) -> "dict | None": """ url = TARGETS_URL.format(platform=plat) try: - return _validate_targets(_fetch_json(url, MAX_TARGETS_BYTES, "targets manifest")) + return _validate_targets(_fetch_json(url, MAX_TARGETS_BYTES, "target list")) except urllib.error.HTTPError as exc: if exc.code == 404: return {} @@ -3102,7 +3207,7 @@ def fetch_targets(plat: str) -> "dict | None": return None except (urllib.error.URLError, OSError) as exc: print(f" Error: failed to fetch targets for {plat}: " - f"{getattr(exc, 'reason', None) or exc}", file=sys.stderr) + f"{network_words(exc)[1]}", file=sys.stderr) for line in _address_lines(url): print(line, file=sys.stderr) print(NETWORK_ADVICE, file=sys.stderr) @@ -4250,7 +4355,7 @@ def _download_one( reason = READ_ONLY if _read_only_file(dest) else NOT_WRITABLE refused = True elif _transfer_failure(exc): - reason = NetworkReason(getattr(exc, "reason", "") or exc) + reason = NetworkReason(network_words(exc)[1]) else: reason = str(getattr(exc, "reason", "") or exc) if verbose: @@ -6959,6 +7064,8 @@ def main() -> None: ) args = parser.parse_args() + if args.verbose: + _RAW_ERRORS.set() if args.cores_dir: args.installed_cores = True _INVOCATION.record(args) diff --git a/install.sh b/install.sh index 9312862d..5bd706ba 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="a0b6a103d350e49f2751928c5a5560e239c52df81637a656e0691b9934e438bb" +DEFAULT_INSTALL_SHA256="ea5d767eeb7612312e6a8fbf5f67b578fdc746f2d34ccc46cf96b978be7424ce" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 900bc8a2..e0bca74c 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3301,7 +3301,8 @@ class AChunkedCutIsADownloadFailure(unittest.TestCase): unittest.mock.patch.object(install, "_retry_delay", return_value=0): dest, ok, reason = install._download_one(entry, Path(_tempdir(self))) self.assertFalse(ok) - self.assertIn("IncompleteRead", reason) + self.assertEqual(reason, "the connection was cut") + self.assertIsInstance(reason, install.NetworkReason) @@ -5112,6 +5113,91 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): self.assertIn(f"`{key}`", row, f"{kind}.{key}") +class NetworkFailuresInPlainWords(unittest.TestCase): + """Offline, the run printed "[Errno -2] Name or service not known". A + beginner needs "no internet connection", and the raw text only under + --verbose.""" + + def _words(self, reason: BaseException, **kwargs) -> tuple[str, str]: + return install.network_words(urllib.error.URLError(reason), **kwargs) + + def test_the_common_failures(self): + import ssl # noqa: PLC0415 + + cases = ( + (socket.gaierror(-2, "Name or service not known"), "no internet connection"), + (socket.timeout("timed out"), "timed out"), + (ConnectionRefusedError(111, "Connection refused"), "refused the connection"), + (OSError(101, "Network is unreachable"), "no internet connection"), + (ssl.SSLCertVerificationError(1, "certificate verify failed"), + "certificate was rejected"), + ) + with unittest.mock.patch.object(install, "_ca_bundle_found", return_value=True): + for reason, words in cases: + with self.subTest(reason=reason): + code, text = self._words(reason) + self.assertEqual(code, "network") + self.assertIn(words, text) + self.assertNotIn("Errno", text) + + def test_verbose_keeps_the_raw_text(self): + _code, text = self._words(socket.gaierror(-2, "Name or service not known"), + verbose=True) + self.assertIn("Name or service not known", text) + + def test_a_clock_in_the_past_is_named(self): + """A handheld with no clock battery boots in 1980, and every + certificate then reads as not yet valid.""" + import datetime # noqa: PLC0415 + import ssl # noqa: PLC0415 + + error = ssl.SSLCertVerificationError(1, "certificate is not yet valid") + with unittest.mock.patch.object(install, "_ca_bundle_found", return_value=True): + code, text = self._words(error, today=datetime.date(1980, 1, 1)) + self.assertEqual(code, "clock") + self.assertIn("The clock reads 1980-01-01", text) + self.assertIn("set the date", text) + code, _text = self._words(error, today=install.CLOCK_FLOOR) + self.assertEqual(code, "network") + + def test_no_certificate_list_names_ssl_cert_file(self): + import ssl # noqa: PLC0415 + + error = ssl.SSLCertVerificationError(1, "unable to get local issuer certificate") + with unittest.mock.patch.object(install, "_ca_bundle_found", return_value=False): + code, text = self._words(error) + self.assertEqual(code, "no_ca") + self.assertIn("SSL_CERT_FILE", text) + + def test_a_certificate_list_is_looked_for_where_openssl_and_systems_keep_it(self): + root = Path(_tempdir(self)) + bundle = root / "cert.pem" + paths = SimpleNamespace(cafile=str(root / "none.pem"), capath=str(root / "none"), + openssl_cafile=str(root / "no.pem"), + openssl_capath=str(root / "no")) + with unittest.mock.patch("ssl.get_default_verify_paths", return_value=paths), \ + unittest.mock.patch.object(install, "CA_BUNDLES", ()), \ + unittest.mock.patch.dict(os.environ, {"SSL_CERT_FILE": "", "SSL_CERT_DIR": ""}): + self.assertFalse(install._ca_bundle_found()) + with unittest.mock.patch.object(install, "CA_BUNDLES", (str(bundle),)): + self.assertFalse(install._ca_bundle_found()) + bundle.write_text("x") + self.assertTrue(install._ca_bundle_found()) + + def test_a_file_list_out_of_reach_by_name_says_so(self): + env = {**os.environ, "RETROBIOS_BASE_URL": "https://retrobios-nohost.invalid", + "HOME": _tempdir(self)} + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", _tempdir(self), "--check"], + env=env, capture_output=True, text=True, timeout=120, check=False, + stdin=subprocess.DEVNULL, + ) + self.assertEqual(proc.returncode, 1) + self.assertIn("no internet connection", proc.stderr) + self.assertNotIn("Errno", proc.stderr) + + class AFileYouHadIsKept(unittest.TestCase): """A BIOS file whose contents differ was replaced and lost: a personal dump or another revision could not be brought back by deleting what the diff --git a/wiki/installer.md b/wiki/installer.md index 3ad1f64b..f41ad9ad 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1044,7 +1044,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is |------|------|------| | `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` | | `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found | -| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`). `reason` is `network`, `not_found` or `invalid` | +| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`). `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). `message` says it in words | | `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` | | `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id. Files with no system count under `other`. `name` is the platform's own name for it | | `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply | @@ -1069,6 +1069,7 @@ no event at all. A program reads a missing `end` as that case. | `RETROBIOS_INSTALL_SHA256` | The SHA-256 the bootstrap requires of that installer, 64 hex characters. When either of these two is set and the fingerprint does not match, the error names the variable instead of suggesting a retry | | `RETROBIOS_BOOTSTRAP` | Set by `install.sh` (`sh`) and `install.ps1` (`ps1`) for the temporary installer they run, so that its hints give back the one-line command rather than `python3 install.py` | | `NO_COLOR` | Plain text, no colour, even at a terminal. `TERM=dumb` also turns the live progress line off | +| `SSL_CERT_FILE`, `SSL_CERT_DIR` | The trusted certificates to check servers with, for a Python that finds none of its own | | `HTTPS_PROXY`, `NO_PROXY` | Honoured for every download through the standard library's default proxy handling: one names the proxy, the other the hosts to reach directly | ## When something goes wrong @@ -1084,6 +1085,19 @@ The installer fetches its manifest, then downloads only the repository or environment. A file that fails three attempts is counted and reported at the end, and the next run fetches only what is still missing. +The reason is said in words: no internet connection or a name that could not +be looked up, a connection that timed out, was refused or was cut, a +certificate that was rejected. `--verbose` adds the text Python gave. Two +causes are not the network and are not retried: + +- `The clock reads 1980-01-01`: a device without a clock battery starts on an + old date, and every certificate then looks not yet valid. Connect to Wi-Fi + and let the time sync, or set the date, then run again. +- `This system has no list of trusted certificates`: install the + `ca-certificates` package, or name a certificate bundle with `SSL_CERT_FILE`. + When the system keeps one where this Python does not look, the message names + it. Certificate checks are never turned off. + ### Permission denied The installer needs to write to the BIOS folder. On Linux and macOS, check who diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md index 14f4ff73..fdbfaeb7 100644 --- a/wiki/troubleshooting.md +++ b/wiki/troubleshooting.md @@ -67,6 +67,8 @@ version. ## If the command fails { #installation-script-fails } - **Network error:** try again once github.com opens. +- **`The clock reads ...`:** the date is wrong. Let it sync over Wi-Fi, or set + it, then run again. - **Permission denied:** do what the message says. It names what to run. - **No setup found:** name it and its folder: