diff --git a/install.ps1 b/install.ps1 index 2ab5bf18..9edaabee 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "851c1d9e8b2b481adf50d1af2c67d1e44d9b50504fdef3d5b6514332ac4457eb" + $defaultInstallSha256 = "8cfe5cb80b8df7f8c2736cfe5855a768c3eacad6ebf5bfd7336b3b4f5a4d5d76" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 74807803..bf8b2288 100644 --- a/install.py +++ b/install.py @@ -16,6 +16,7 @@ from __future__ import annotations import argparse import concurrent.futures +import datetime import errno import hashlib import http.client @@ -40,7 +41,7 @@ import urllib.request import xml.etree.ElementTree as ET import zipfile import zlib -from collections.abc import Callable, Iterator +from collections.abc import Callable, Collection, Iterator from pathlib import Path, PurePosixPath from typing import BinaryIO, NamedTuple, TextIO @@ -283,6 +284,8 @@ def _os_default_dests(os_type: str) -> dict[str, Path]: # Set when the run is interrupted, so queued downloads end instead of draining # the pool: every future is submitted before the first one completes. _stop = threading.Event() +# Set once a run starts writing: Ctrl+C before that has nothing to carry on. +_writing = threading.Event() class _Interrupted(Exception): """Raised inside a worker once the run has been stopped.""" @@ -322,6 +325,9 @@ class Terminal: """What the output stream can show: a live line, colour, or neither. A pipe or a log gets plain lines, and NO_COLOR and TERM=dumb are honoured. + As no-color.org reads it, an empty NO_COLOR is not set: `NO_COLOR= cmd` + turned colour off. FORCE_COLOR or CLICOLOR_FORCE paints a pipe, for a + log viewer that shows colour, and never makes the line live. """ def __init__(self, stream: TextIO | None = None, quiet: bool = False) -> None: @@ -329,8 +335,10 @@ class Terminal: self.quiet = quiet isatty = getattr(self.stream, "isatty", None) self.live = bool(isatty and isatty()) and os.environ.get("TERM") != "dumb" + forced = any(os.environ.get(name, "0") not in ("", "0") + for name in ("FORCE_COLOR", "CLICOLOR_FORCE")) self.colour = ( - self.live and "NO_COLOR" not in os.environ and _enable_ansi() + (self.live or forced) and not os.environ.get("NO_COLOR") and _enable_ansi() ) if not self.colour: self.live = self.live and os.name != "nt" @@ -377,6 +385,9 @@ class Progress: self.files = 0 self.bytes = 0 self.resumed = 0 + # Files written to their place, which a stopped run counts: the + # queue's length said "118 files installed" over 8 on disk. + self.installed = 0 self.start = time.monotonic() self._drawn = 0.0 self._evented = self.start @@ -473,8 +484,10 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { "start": ("schema", "installer", "mode"), # platforms: [{"platform", "name", "bios_dir"}], empty when none is found. "detect": ("os", "os_name", "platforms"), - # stage: manifest (reason: network, not_found or invalid) or detect - # (reason: conflict, two systems found on one machine). + # stage: manifest (reason: network, not_found, invalid, clock for a + # refused certificate under a clock set in the past, or no_ca for no + # trusted certificates on the machine) or detect (reason: conflict, two + # systems found on one machine). "error": ("stage", "reason", "platform", "message"), # state: present (hash verified), wrong_hash or missing. "file": ("platform", "dest", "systems", "size", "state"), @@ -556,8 +569,14 @@ def _os_name() -> str: def _emit_check( plat: str, manifest: dict, bios_path: Path, files: list[dict], up_to_date: list[dict], mismatched: list[dict], data_missing: int, not_installable: int, + chosen: list[str] | None = None, ) -> None: - """One event per file, per system and per platform after a check.""" + """One event per file, per system and per platform after a check. + + With systems chosen, only those are described: a file of SkyEmu's + carries four systems, and --system nintendo-gba described gb, gbc and + ds by the few files the choice kept of them. + """ present = {id(entry) for entry in up_to_date} wrong = {id(entry) for entry in mismatched} systems: dict[str, dict[str, int]] = {} @@ -578,6 +597,8 @@ def _emit_check( counts["bytes_needed"] += size names = manifest.get("system_names") or {} for system in sorted(systems): + if chosen and system not in chosen: + continue EVENTS.emit("system", platform=plat, system=system, name=", ".join(names.get(system) or [system]), **systems[system]) needed = [entry for entry in files if id(entry) not in present] @@ -1249,8 +1270,9 @@ _EMBEDDED_RULES: tuple[_OsRule, ...] = ( _OsRule("", "TrimUI", "file_exists", ("/usr/trimui",), None), # MinUI, MyMinUI and NextUI keep their system in .system at the card root # (MinUI dbf89435, workspace/all/common/defines.h:14-16) and give each core - # Bios// (workspace/all/minarch/minarch.c:2942), a layout the - # installer does not write. + # Bios// (workspace/all/minarch/minarch.c:2942). Their file lists + # write that layout below the card root, and the card's version.txt + # says which of the three it is (card_platform). _OsRule("", "MinUI", "file_exists", ("/mnt/SDCARD/.system/res",), None, base="TrimUI"), _OsRule("", "MinUI", "file_exists", ("/mnt/sdcard/.system/res",), None, base="TrimUI"), _OsRule("retroarch", "Anbernic stock firmware", "file_exists", @@ -1260,9 +1282,13 @@ _EMBEDDED_RULES: tuple[_OsRule, ...] = ( # Why a system found here gets no folder: said once per run. _NO_FOLDER = { "TrimUI": "its firmware publishes no source naming the folder its RetroArch reads", - "MinUI": "MinUI, MyMinUI and NextUI give each core its own Bios// folder, " - "a layout the installer does not write yet", + "MinUI": f"its card names no release in {CARD_MARKER}, which tells MinUI, NextUI " + "and MyMinUI apart. Name the card's root and the one it runs: " + "--platform minui (or nextui, myminui) --dest /mnt/SDCARD", } +# A system found by its card, with the end of the path that proves it: the +# card root is what comes before. +_CARD_SYSTEMS = {"MinUI": "/.system/res"} # (platform, BIOS folder) -> the name the system found there gives itself. _DETECTED_OS: dict[tuple[str, Path], str] = {} @@ -1332,15 +1358,23 @@ def _detect_embedded() -> list[tuple[str, Path]]: if name in bases: continue rules = [rule for rule in matched if rule.name == name] - folder = _os_system_dir(rules) if rules[0].manifest else None + plat = rules[0].manifest + folder = _os_system_dir(rules) if plat else None + if name in _CARD_SYSTEMS: + marker = next(value for rule in rules for value in rule.values + if value.endswith(_CARD_SYSTEMS[name])) + root = _on_device(marker[:-len(_CARD_SYSTEMS[name])]) + plat = card_platform(root) or "" + if plat: + folder, name = root, display_name(plat) if folder is None: reason = _NO_FOLDER.get( name, "nothing on it names the folder its RetroArch reads") - print(f"{name} found here, but {reason}. Name the folder with --dest.", - file=sys.stderr) + closing = "" if name in _CARD_SYSTEMS else " Name the folder with --dest." + print(f"{name} found here, but {reason}.{closing}", file=sys.stderr) continue - _DETECTED_OS[(rules[0].manifest, folder)] = name - found.append((rules[0].manifest, folder)) + _DETECTED_OS[(plat, folder)] = name + found.append((plat, folder)) return found @@ -1360,7 +1394,10 @@ def _system_lines() -> list[str]: lines = [] for name in dict.fromkeys(rule.name for rule in _EMBEDDED_RULES): plat = next(rule.manifest for rule in _EMBEDDED_RULES if rule.name == name) - if plat: + if name in _CARD_SYSTEMS: + lines.append(f"{name}: minui, nextui or myminui, as the card's {CARD_MARKER} " + "names it (from a computer: --dest )") + elif plat: lines.append(f"{name}: {plat}") else: lines.append(f"{name}: not installed to, {_NO_FOLDER[name]} (use --dest)") @@ -2231,15 +2268,58 @@ def _name_key(name: str) -> str: SHORT_LIST = 12 +# Names people type for a system, each with its id. Only ever suggested: +# --system gba, psx and ps1 were refused with no hint on RetroArch, whose +# list names no system that way, and taking one in silence could install +# another console than the one meant. +SYSTEM_SHORT_NAMES = { + **dict.fromkeys(("psx", "ps1", "ps", "playstation1"), "sony-playstation"), + "ps2": "sony-playstation-2", "ps3": "sony-playstation-3", "psp": "sony-psp", + **dict.fromkeys(("vita", "psvita"), "sony-playstation-vita"), + "gb": "nintendo-gb", "gbc": "nintendo-gbc", "gba": "nintendo-gba", + **dict.fromkeys(("nes", "famicom", "fc"), "nintendo-nes"), "fds": "nintendo-fds", + **dict.fromkeys(("snes", "sfc", "superfamicom"), "nintendo-snes"), + "n64": "nintendo-64", "n64dd": "nintendo-64dd", "sgb": "nintendo-sgb", + **dict.fromkeys(("nds", "ds"), "nintendo-ds"), "dsi": "nintendo-dsi", + "3ds": "nintendo-3ds", "wiiu": "nintendo-wii-u", "switch": "nintendo-switch", + **dict.fromkeys(("gc", "ngc", "gcn", "gamecube"), "nintendo-gamecube"), + "pokemini": "nintendo-pokemon-mini", + **dict.fromkeys(("md", "gen", "genesis", "megadrive"), "sega-mega-drive"), + **dict.fromkeys(("scd", "segacd", "megacd"), "sega-mega-cd"), + "32x": "sega-32x", **dict.fromkeys(("sms", "mastersystem"), "sega-master-system"), + **dict.fromkeys(("gg", "gamegear"), "sega-game-gear"), + **dict.fromkeys(("ss", "saturn"), "sega-saturn"), + **dict.fromkeys(("dc", "dreamcast"), "sega-dreamcast"), + **dict.fromkeys(("naomi", "atomiswave"), "sega-dreamcast-arcade"), + **dict.fromkeys(("pce", "tg16", "turbografx", "turbografx16", "pcengine", + "supergrafx"), "nec-pc-engine"), + **dict.fromkeys(("pcecd", "tgcd", "turbografxcd"), "nec-pc-engine-cd"), + "pcfx": "nec-pc-fx", + **dict.fromkeys(("mvs", "aes"), "snk-neogeo"), + **dict.fromkeys(("neocd", "neogeocd"), "snk-neogeo-cd"), + **dict.fromkeys(("msx", "msx2"), "microsoft-msx"), + "c64": "commodore-c64", "c128": "commodore-c128", + **dict.fromkeys(("amiga", "cd32"), "commodore-amiga"), + "x68k": "sharp-x68000", **dict.fromkeys(("jag", "jaguar"), "atari-jaguar"), + **dict.fromkeys(("st", "atarist"), "atari-st"), "a800": "atari-400-800", + **dict.fromkeys(("coleco", "colecovision"), "coleco-colecovision"), + **dict.fromkeys(("intv", "intellivision"), "mattel-intellivision"), + "o2": "magnavox-odyssey2", "channelf": "fairchild-channel-f", + **dict.fromkeys(("zx", "spectrum", "zxspectrum"), "sinclair-zx-spectrum"), + "cpc": "amstrad-cpc", "bbc": "acorn-bbc-micro", "apple2": "apple-ii", +} + + def did_you_mean( typed: list[str], known: list[str], listing: str, - aliases: dict[str, str] | None = None, + aliases: dict[str, str] | None = None, inside_from: int = 4, ) -> str: """The closest known names to what was typed, and the option listing them all. A typo was answered with every name the platform has, up to 400 on one line, with no hint that playstation is sony-playstation. An alias close - to what was typed suggests the name it stands for. + to what was typed suggests the name it stands for. A name holding what + was typed, inside_from letters or more, is suggested too. """ import difflib # noqa: PLC0415 @@ -2254,9 +2334,12 @@ def did_you_mean( if not key: continue matches = difflib.get_close_matches(key, list(by_key), n=3, cutoff=0.6) - if len(key) >= 4: + if len(key) >= inside_from: matches += sorted(k for k in by_key if key in k and k not in matches) - close += [by_key[k] for k in matches if by_key[k] not in close] + for k in matches: + # Two aliases of one system (psx, ps1) name it once. + if by_key[k] not in close: + close.append(by_key[k]) if close: return f"Did you mean: {', '.join(close[:3])}? {listing} lists all {len(known)}." if len(known) <= SHORT_LIST: @@ -2274,6 +2357,13 @@ def normalize_platform(name: str) -> str: for plat in AVAILABLE_PLATFORMS: if key and key in (_name_key(plat), _name_key(display_name(plat))): return plat + # --list-platforms prints `muOS: retroarch`, and --platform muos was + # refused: a system named there stands for the list it gets. + for rule in _EMBEDDED_RULES: + if key and rule.manifest and key == _name_key(rule.name): + print(f" {rule.name} gets the {display_name(rule.manifest)} file list: " + f"--platform {rule.manifest}.") + return rule.manifest print(f"Unknown platform '{name.strip()}'. " + did_you_mean([name], list(AVAILABLE_PLATFORMS), "--list-platforms"), file=sys.stderr) @@ -2903,6 +2993,102 @@ NETWORK_ADVICE = (" Check the connection, and HTTPS_PROXY if this network needs f" More help: {TROUBLESHOOTING_PAGE}") +# The installer was written after this day. A clock that reads earlier is +# wrong, and TLS then finds every certificate not yet valid: a handheld +# with no clock battery boots on 1970 or 1980. +CLOCK_FLOOR = datetime.date(2026, 1, 1) +# Where systems keep their trusted certificates, for a Python built to look +# elsewhere: Debian, Fedora, openSUSE, the BSDs and macOS, Homebrew, Termux. +CA_BUNDLES = ( + "/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/ca-bundle.pem", "/etc/pki/tls/cacert.pem", "/etc/ssl/cert.pem", + "/usr/local/etc/openssl/cert.pem", "/opt/homebrew/etc/openssl@3/cert.pem", + "/data/data/com.termux/files/usr/etc/tls/cert.pem", +) +# Set from --verbose: the error text Python gives stays beside the words. +_RAW_ERRORS = threading.Event() + + +def _default_ca_present() -> bool: + """Whether the certificates this Python reads by default exist.""" + import ssl # noqa: PLC0415 + + paths = ssl.get_default_verify_paths() + for value in (os.environ.get("SSL_CERT_FILE"), paths.cafile, paths.openssl_cafile): + if value and Path(value).is_file(): + return True + for value in (os.environ.get("SSL_CERT_DIR"), paths.capath, paths.openssl_capath): + try: + if value and any(Path(value).iterdir()): + return True + except OSError: + continue + return False + + +def _known_bundle() -> str: + """A certificate bundle the system keeps where this Python does not look.""" + return next((path for path in CA_BUNDLES if Path(path).is_file()), "") + + +def _ca_bundle_found() -> bool: + """Whether any list of trusted certificates is on this machine. + + Windows verifies against its own store, which Python reads directly. + """ + return os.name == "nt" or _default_ca_present() or bool(_known_bundle()) + + +def network_words( + exc: BaseException, today: datetime.date | None = None, verbose: bool | None = None, +) -> tuple[str, str]: + """The reason a request failed, and the same in words a beginner reads. + + The reason is clock, no_ca or network. Offline, the run printed + "[Errno -2] Name or service not known": the raw text now follows the + words only under --verbose. + """ + reason = getattr(exc, "reason", None) + inner = reason if isinstance(reason, BaseException) else exc + raw = str(reason or exc) + code, words = "network", "" + cert_error = getattr(sys.modules.get("ssl"), "SSLCertVerificationError", None) + if cert_error is not None and isinstance(inner, cert_error): + today = today or datetime.date.today() + if not _ca_bundle_found(): + code = "no_ca" + words = ("This system has no list of trusted certificates to check the " + "server with. Install the ca-certificates package, or name a " + "certificate bundle with SSL_CERT_FILE, then run again") + elif today < CLOCK_FLOOR: + code = "clock" + words = (f"The clock reads {today.isoformat()}: connect to Wi-Fi and let the " + "time sync, or set the date, then run again") + else: + words = "the server's certificate was rejected" + bundle = "" if _default_ca_present() else _known_bundle() + words += (f": this Python reads no certificate list, run again with " + f"SSL_CERT_FILE={bundle}" if bundle + else ", which a proxy or a filter on this network can cause") + elif isinstance(inner, socket.gaierror): + words = "no internet connection, or the server name could not be looked up" + elif isinstance(inner, (socket.timeout, TimeoutError)): + words = "the connection timed out" + elif isinstance(inner, ConnectionRefusedError): + words = "the server refused the connection" + elif isinstance(inner, OSError) and inner.errno in ( + errno.ENETUNREACH, errno.EHOSTUNREACH, errno.ENETDOWN): + words = "no internet connection: the network cannot be reached" + elif isinstance(inner, (ConnectionResetError, ConnectionAbortedError, + http.client.HTTPException)): + words = "the connection was cut" + if not words: + return code, raw + if _RAW_ERRORS.is_set() if verbose is None else verbose: + words += f" [{raw}]" + return code, words + + def _address_lines(url: str) -> list[str]: """The address that failed, and the override that chose it if one did. @@ -2934,12 +3120,13 @@ def _fetch_json(url: str, limit: int, label: str) -> object: delay = _retry_delay(attempt, exc.headers.get("Retry-After") if exc.headers else None) problem = f"HTTP {exc.code}" except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: - if attempt == MAX_RETRIES: + code, problem = network_words(exc) + # A wrong clock or no certificate list fails every attempt alike. + if attempt == MAX_RETRIES or code != "network": if isinstance(exc, http.client.HTTPException): raise ConnectionError(f"connection cut: {exc!r}") from exc raise delay = _retry_delay(attempt, None) - problem = str(getattr(exc, "reason", None) or exc) # Said, or three 30-second attempts read as a frozen program. print(f" The {label} did not arrive ({problem}), trying again " f"({attempt + 1}/{MAX_RETRIES}) in {delay:.0f} s.", file=sys.stderr) @@ -2960,7 +3147,7 @@ def fetch_manifest(plat: str) -> dict: sys.exit(1) url = MANIFEST_URL.format(platform=plat) try: - return _validate_manifest(_fetch_json(url, MAX_MANIFEST_BYTES, "manifest"), plat) + return _validate_manifest(_fetch_json(url, MAX_MANIFEST_BYTES, "file list"), plat) except ValueError as exc: EVENTS.emit("error", stage="manifest", reason="invalid", platform=plat, message=f"the file list is not valid: {exc}") @@ -2994,11 +3181,16 @@ def fetch_manifest(plat: str) -> dict: print(NETWORK_ADVICE, file=sys.stderr) sys.exit(1) except (urllib.error.URLError, OSError) as exc: - EVENTS.emit("error", stage="manifest", reason="network", platform=plat, - message=f"the file list could not be fetched: " - f"{getattr(exc, 'reason', None) or exc}") + code, words = network_words(exc) + EVENTS.emit("error", stage="manifest", reason=code, platform=plat, + message=words if code != "network" + else f"the file list could not be fetched: {words}") + if code != "network": + print(f" Could not fetch the file list for {plat}.", file=sys.stderr) + print(f" {words}.", file=sys.stderr) + sys.exit(1) print(f" Could not fetch the file list for {plat} after {MAX_RETRIES} " - f"attempts: {getattr(exc, 'reason', None) or exc}", file=sys.stderr) + f"attempts: {words}", file=sys.stderr) for line in _address_lines(url): print(line, file=sys.stderr) print(NETWORK_ADVICE, file=sys.stderr) @@ -3014,7 +3206,7 @@ def fetch_targets(plat: str) -> "dict | None": """ url = TARGETS_URL.format(platform=plat) try: - return _validate_targets(_fetch_json(url, MAX_TARGETS_BYTES, "targets manifest")) + return _validate_targets(_fetch_json(url, MAX_TARGETS_BYTES, "target list")) except urllib.error.HTTPError as exc: if exc.code == 404: return {} @@ -3028,7 +3220,7 @@ def fetch_targets(plat: str) -> "dict | None": return None except (urllib.error.URLError, OSError) as exc: print(f" Error: failed to fetch targets for {plat}: " - f"{getattr(exc, 'reason', None) or exc}", file=sys.stderr) + f"{network_words(exc)[1]}", file=sys.stderr) for line in _address_lines(url): print(line, file=sys.stderr) print(NETWORK_ADVICE, file=sys.stderr) @@ -3418,9 +3610,15 @@ def resolve_choices( raise ValueError(f"unknown {label}: {', '.join(unknown)}. This list records none.") listing = {"system": "--list-systems", "emulator": "--list-emulators"}.get( label, "--list-cores") + hints, inside_from = aliases, 4 + if label == "system": + # gba is the end of nintendo-gba: three letters are enough here. + hints = {**{short: name for short, name in SYSTEM_SHORT_NAMES.items() + if name in known}, **aliases} + inside_from = 3 raise ValueError( f"unknown {label}: {', '.join(unknown)}. " - f"{did_you_mean(unknown, known, listing, aliases)}" + f"{did_you_mean(unknown, known, listing, hints, inside_from)}" ) if values and not chosen: raise ValueError(f"--{label} needs at least one name") @@ -4170,7 +4368,7 @@ def _download_one( reason = READ_ONLY if _read_only_file(dest) else NOT_WRITABLE refused = True elif _transfer_failure(exc): - reason = NetworkReason(getattr(exc, "reason", "") or exc) + reason = NetworkReason(network_words(exc)[1]) else: reason = str(getattr(exc, "reason", "") or exc) if verbose: @@ -4841,14 +5039,15 @@ def download_files( """Download files in parallel. Returns the destinations that failed, each with its reason, and whether - the run was interrupted. Every future is submitted up front, so a bare - KeyboardInterrupt would let the pool drain the queue in full: the stop - flag is what makes Ctrl+C end the run rather than only stop the - reporting. + the run was interrupted. progress.installed counts the files written. + Every future is submitted up front, so a bare KeyboardInterrupt would + let the pool drain the queue in full: the stop flag is what makes Ctrl+C + end the run rather than only stop the reporting. """ failed: list[tuple[str, str]] = [] interrupted = False _stop.clear() + _writing.set() term = term or Terminal() if progress is None: progress = Progress(len(files), sum(f.get("size", 0) for f in files), term) @@ -4859,14 +5058,24 @@ def download_files( for f in files } refused = False + refused_events = 0 try: for future in concurrent.futures.as_completed(future_map): dest, success, reason = future.result() if refused: # The folder they all go in refused the first: each - # would fail alike, counted without a line of its own. + # would fail alike, counted without a line of its own, + # though a program reading the events still gets one. if not success: failed.append((dest, NOT_WRITABLE)) + refused_events += 1 + EVENTS.emit( + "downloaded", dest=dest, ok=False, reason=NOT_WRITABLE, + size=future_map[future].get("size", 0), + files_done=progress.files + refused_events, + files_total=progress.total_files, bytes_done=progress.bytes, + bytes_total=progress.expected_bytes, + ) continue if _stop.is_set(): continue @@ -4889,6 +5098,12 @@ def download_files( _stop.set() progress.finish() print("\n Stopping. Files already installed are kept.") + # The pool has run every future by now: a file that landed after Ctrl+C + # is on disk and counts. + progress.installed = sum( + 1 for future in future_map + if not future.cancelled() and future.exception() is None and future.result()[1] + ) progress.finish() return failed, interrupted @@ -4999,6 +5214,7 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict: "not_kept": []} if not needed: return counts + _writing.set() try: root.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=root) as scratch: @@ -5201,7 +5417,7 @@ def _install_standalone_folders( to_download, folder, jobs=args.jobs, verbose=args.verbose, term=term, progress=progress, ) - downloaded += len(to_download) - len(failed) + downloaded += progress.installed errors += len(failed) failures.extend((str(folder / dest), reason) for dest, reason in failed) transferred += progress.transferred() @@ -5374,7 +5590,7 @@ def _choose_platform(title: str, default: str = "") -> str: if not choice and default: return default if choice.lower() == "q": - sys.exit(0) + _quit() if choice.isdigit() and 1 <= int(choice) <= len(AVAILABLE_PLATFORMS): return AVAILABLE_PLATFORMS[int(choice) - 1] key = _name_key(choice) @@ -5387,7 +5603,8 @@ def _choose_platform(title: str, default: str = "") -> str: def _prompt_manual_platform(os_type: str) -> list[tuple[str, Path]]: """Ask for a platform and a BIOS directory when nothing was detected.""" - plat = _choose_platform("Choose the platform to install for:") + plat = _choose_platform("No supported platform found here. Choose the one to " + "install for:", "retroarch") default_dest = _default_dest(os_type, plat) raw = "" while not raw: @@ -5397,10 +5614,24 @@ def _prompt_manual_platform(os_type: str) -> list[tuple[str, Path]]: ) if default_dest is not None: break - dest = Path(raw).expanduser() if raw else default_dest - if os_type == "windows" and raw: - dest = Path(os.path.expandvars(raw)) - return [(plat, dest)] + if raw: + return [(plat, _typed_folder(raw))] + state = "" if default_dest.exists() else ". It does not exist yet and will be created" + print(f" {display_name(plat)}'s default BIOS folder: {default_dest}{state}.") + return [(plat, default_dest)] + + +def _typed_folder(raw: str) -> Path: + """A folder typed at a question, read as a shell reads an argument. + + Explorer's "Copy as path" puts quotes around it, and users quote a path + with spaces: kept, they sent the files into a folder named `"` under + the current directory, and the run said Done. + """ + text = raw.strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"": + text = text[1:-1].strip() + return Path(os.path.expandvars(os.path.expanduser(text))) # Platforms with no install location of their own: the BIOS folder sits @@ -5463,10 +5694,10 @@ def _manual_usage_hint(os_type: str, listing: str = "") -> list[str]: def _prompt_platform_choice( - platforms: list[tuple[str, Path]], + platforms: list[tuple[str, Path]], title: str = "Install for:", ) -> list[tuple[str, Path]]: """Prompt user to choose among detected platforms.""" - print("\nInstall for:") + print(f"\n{title}") for i, (name, path) in enumerate(platforms, 1): system = detected_os(name, path) label = f"{system}, {display_name(name)} files" if system and system != display_name(name) \ @@ -5481,7 +5712,7 @@ def _prompt_platform_choice( while True: choice = _ask("> ").lower() if choice == "q": - sys.exit(0) + _quit() idx = int(choice) if choice.isdigit() else 0 if 1 <= idx <= len(platforms): return [platforms[idx - 1]] @@ -5505,26 +5736,62 @@ def _ask(prompt: str) -> str: raise KeyboardInterrupt from None +def _quit() -> None: + """q at a question ends the run where Ctrl+C does, on 130: with 0, + `install.sh && retroarch` started RetroArch on no BIOS.""" + sys.exit(130) + + def _sized(files: list[dict]) -> str: return f"{_plural(len(files), 'file')}, {format_size(sum(f.get('size', 0) for f in files))}" -def _pick(title: str, rows: list[tuple[str, list[dict]]], ordered: bool = False) -> list[str]: - """Numbered list of names. An empty answer keeps every one.""" +# A core whose files are all the platform's own. "pcsx_rearmed 0 files, +# 0 B" read as a core that gets nothing. +NO_CORE_FILES = "no extra file: it reads the platform's own, always kept" + + +def _core_sized(files: list[dict]) -> str: + return _sized(files) if files else NO_CORE_FILES + + +def _system_sized(files: list[dict]) -> str: + """A system's files, those other systems share counted apart. + + `3do 1704 files, 349.0 MB` read as the 3DO BIOS: 1689 of them are the + MAME hash and plugin files six systems carry. + """ + shared = [f for f in files if len(entry_systems(f)) > 1] + if not shared: + return _sized(files) + if len(shared) == len(files): + return f"{_sized(shared)}, all shared with other systems" + own = [f for f in files if len(entry_systems(f)) <= 1] + return f"{_sized(own)}, and {_sized(shared)} shared with other systems" + + +def _pick( + title: str, rows: list[tuple[str, list[dict]]], ordered: bool = False, + sizer: Callable[[list[dict]], str] = _sized, +) -> list[str]: + """Numbered list of names. An empty answer keeps every one. + + sizer says what each name holds. + """ if not rows: return [] print(f"\n{title}") width = len(str(len(rows))) name_width = min(max(len(name) for name, _files in rows), 32) for index, (name, files) in enumerate(rows, 1): - print(f" {index:>{width}}) {name:<{name_width}} {_sized(files)}") + print(f" {index:>{width}}) {name:<{name_width}} {sizer(files)}") hint = "order of preference, e.g. 2,1" if ordered else "e.g. 1,3-5" while True: text = _ask(f"Numbers ({hint}), Enter for all, q to quit: ") if not text: return [] if text.lower() == "q": - sys.exit(0) + _quit() try: chosen = parse_selection(text, len(rows), ordered=ordered) except ValueError as exc: @@ -5560,7 +5827,7 @@ def _prompt_custom_selection( if answer == "": return None if answer == "q": - sys.exit(0) + _quit() if answer == "i" and installed is not None: return INSTALLED_ONLY if answer == "c": @@ -5575,7 +5842,7 @@ def _prompt_custom_selection( systems = _pick("Systems:", [ (name, [f for f in files if _chosen(f, {name})]) for name in choices["systems"] - ]) + ], sizer=_system_sized) kept, kept_omitted = narrow(files, omitted, systems, [], []) cores: list[str] = [] @@ -5587,7 +5854,7 @@ def _prompt_custom_selection( (name, _filter_by_target( [f for f in kept if f.get("cores") is not None], [name])) for name in core_names - ]) + ], sizer=_core_sized) kept, kept_omitted = narrow(kept, kept_omitted, [], cores, []) regions: list[str] = [] @@ -5648,7 +5915,8 @@ def _print_choices( if systems else _filter_by_target([f for f in files if f.get("cores") is not None], [name]) ) - print(f" {labels[name]:<{width}} {_sized(held)}") + print(f" {labels[name]:<{width}} " + f"{_system_sized(held) if systems else _core_sized(held)}") def emulator_names(manifest: dict, files: list[dict], omitted: list[dict]) -> dict[str, str]: @@ -6564,11 +6832,15 @@ def _bytes(files: list[dict]) -> int: def bulk_hint( to_download: list[dict], manifest: dict, systems_chosen: bool, cores_chosen: bool, + accepted: Collection[str] = (), ) -> list[str]: """Name what pulls most of a download, when narrowing saves over half of it. `--system 3do` fetched 349 MB for RetroArch, and `--core opera` 12.5 MB: the arcade cores that also read 3DO pulled the rest, and nothing said so. + accepted holds the names --core takes. Drawn from every core of the + system, the advice named ares, which loads no file of its own, and + `--core ares` was then refused. """ total = _bytes(to_download) if total < BULK_HINT_FROM: @@ -6576,7 +6848,8 @@ def bulk_hint( systems = {s for entry in to_download for s in entry_systems(entry)} if not cores_chosen and len(systems) == 1: system = next(iter(systems)) - candidates = (manifest.get("system_cores") or {}).get(system) or [] + candidates = [core for core in (manifest.get("system_cores") or {}).get(system) or [] + if core in accepted] by_core: dict[str, int] = {} for entry in to_download: for core in entry.get("cores") or []: @@ -6824,6 +7097,8 @@ def main() -> None: ) args = parser.parse_args() + if args.verbose: + _RAW_ERRORS.set() if args.cores_dir: args.installed_cores = True _INVOCATION.record(args) @@ -6952,7 +7227,7 @@ def _open_local_sources(args: argparse.Namespace) -> None: sys.exit(1) -def _missing_dest(dest: Path, ask: bool) -> None: +def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest") -> None: """Say that a --dest folder does not exist before anything goes there. A typo (~/.config/retroarh/system) was created with every missing @@ -6971,7 +7246,7 @@ def _missing_dest(dest: Path, ask: bool) -> None: if missing > 1 and ask: answer = _ask(f" Create {_plural(missing, 'folder')} under {nearest}? [y/N] ") if not answer.strip().lower().startswith("y"): - print(" Nothing installed. Check the folder named with --dest, then run " + print(f" Nothing installed. Check the folder {named}, then run " "the command again.") sys.exit(2) @@ -7091,7 +7366,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: print("Available platforms:") for p in AVAILABLE_PLATFORMS: print(f" {p}") - print("\nSystems recognized by their own files:") + print("\nSystems recognized by their own files, and the file list each gets " + "(--platform takes either name):") for line in _system_lines(): print(f" {line}") detected = detect_platforms(os_type) @@ -7100,6 +7376,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: for name, path in detected: system = detected_os(name, path) print(f" {name}: {path}" + (f" ({system})" if system else "")) + else: + print("\nDetected on this system: none") return # Platform detection or override. A destination taken from the default @@ -7193,8 +7471,13 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: print(line) sys.exit(1) platforms = _prompt_manual_platform(os_type) - for name, path in platforms: - print(f" Found {found_label(name, path)}") + plat, typed = platforms[0] + if typed != _default_dest(os_type, plat): + _missing_dest(typed, not (args.check or args.dry_run or args.yes), + "typed") + else: + for name, path in platforms: + print(f" Found {found_label(name, path)}") conflict = os_conflict(platforms) if conflict and not interactive and not listing: @@ -7215,7 +7498,9 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: ]) if len(platforms) > 1 and not listing and interactive: - platforms = _prompt_platform_choice(platforms) + # --check installs nothing, and "Install for:" said it would. + platforms = _prompt_platform_choice( + platforms, "Check:" if args.check else "Install for:") if args.dest is not None and not listing: _missing_dest(args.dest, interactive and not (args.check or args.dry_run or args.yes)) @@ -7417,7 +7702,11 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: args.emulator, manifest, plat_name, files, omitted_files ) + # What --core resolves against, for advice that names one. + core_choices = set(available_choices(files, omitted_files)["cores"]) selection = _requested_narrowing(args, plat_name, files, omitted_files, manifest) + # What declining the size question points back at, when it was asked. + smaller = "" if ( selection is None and interactive @@ -7433,6 +7722,13 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: plat_name, files, omitted_files, installed, display_name(plat_name, manifest), ) + if selection is None: + smaller = (" Nothing downloaded. To install less, run the same command " + "again and type c at the first question to choose systems") + if installed is not None: + smaller += (f", or i for only what the installed cores read " + f"({format_size(_bytes(installed[0]))})") + smaller += "." if selection == INSTALLED_ONLY: files, omitted_files = installed print("\n Same choice without questions: " @@ -7529,6 +7825,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: systems_chosen=bool(selection and selection[0]), cores_chosen=bool((selection and selection[1]) or args.emulator or args.installed_cores or target), + accepted=core_choices, ) narrowed = bool( @@ -7544,6 +7841,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: _emit_check( plat_name, manifest, bios_path, files, up_to_date, mismatched, sum(len(state[1]) for _entry, state in data_states), len(omitted_files), + chosen=selection[0] if selection else None, ) total_verified += len(up_to_date) @@ -7608,7 +7906,9 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: f"\nDownload {format_size(dl_size)} to {bios_path}? [Y/n] " ) ): - print(" Nothing downloaded. --system, --core and --region " + # A beginner who just passed the c entry was sent to a + # 91-line --help. + print(smaller or " Nothing downloaded. --system, --core and --region " "install less (see --help).") total_up_to_date += len(up_to_date) total_declined += len(to_download) @@ -7646,10 +7946,9 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: term=term, progress=progress, ) bulk_lines += hint - total_downloaded += len(to_download) - len(failed) - files_pending += len(failed) - if interrupted: - files_pending += max(len(to_download) - progress.files, 0) + total_downloaded += progress.installed + files_pending += len(to_download) - progress.installed if interrupted \ + else len(failed) total_errors += len(failed) failures.extend(failed) total_bytes += progress.transferred() @@ -7782,7 +8081,11 @@ if __name__ == "__main__": except KeyboardInterrupt: # Outside the download phase (fetching the file list, checking the # files present, rebuilding data directories): nothing is half - # written there, and a socket traceback read as a crash. - print("\nStopped. Run the same command again to carry on from here. " - "Nothing is lost.") + # written there, and a socket traceback read as a crash. Stopped at + # a question, nothing has started, and there is nothing to carry on. + if _writing.is_set(): + print("\nStopped. Run the same command again to carry on from here. " + "Nothing is lost.") + else: + print("\nStopped. Nothing was installed.") sys.exit(130) diff --git a/install.sh b/install.sh index b3217604..405fa690 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="851c1d9e8b2b481adf50d1af2c67d1e44d9b50504fdef3d5b6514332ac4457eb" +DEFAULT_INSTALL_SHA256="8cfe5cb80b8df7f8c2736cfe5855a768c3eacad6ebf5bfd7336b3b4f5a4d5d76" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 89e0ca04..025cde83 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -392,7 +392,8 @@ class ExitStatusTable(unittest.TestCase): section = _section(_page("installer.md"), "Exit status") documented = {int(row[0].strip("`")) for row in _table_rows(section)} raised = {int(code) for code in re.findall(r"sys\.exit\((\d+)\)", source)} - self.assertEqual(documented, raised | {2}) + # 0 is a run that returns, 2 the code argparse leaves with. + self.assertEqual(documented, raised | {0, 2}) self.assertIn("install.py: error:", section) for option in set(re.findall(r"(--[a-z][a-z-]+)", section)): self.assertIn(f'"{option}"', source, option) diff --git a/tests/test_install.py b/tests/test_install.py index 4bde3446..14f27fc4 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1784,7 +1784,7 @@ class TestDetectedPlatformPrompt(unittest.TestCase): def test_q_quits_and_the_menu_says_so(self): result, out = self._answer("q") self.assertIsInstance(result, SystemExit) - self.assertEqual(result.code, 0) + self.assertEqual(result.code, 130) self.assertIn("q) quit", out) def test_a_wrong_answer_names_the_choices(self): @@ -1808,6 +1808,16 @@ class TestManualPlatformPrompt(unittest.TestCase): chosen, [("retroarch", Path.home() / ".config" / "retroarch" / "system")] ) + def test_enter_takes_retroarch_as_the_dest_menu_does(self): + """The --dest menu offered `[Enter for RetroArch]` and this one, the + same list, had no default.""" + answers = iter(["", ""]) + out = io.StringIO() + with unittest.mock.patch("builtins.input", lambda _prompt="": next(answers)), \ + contextlib.redirect_stdout(out): + chosen = install._prompt_manual_platform("linux") + self.assertEqual(chosen[0][0], "retroarch") + def test_name_and_explicit_directory(self): answers = iter(["batocera", "~/my bios"]) with unittest.mock.patch("builtins.input", lambda _prompt="": next(answers)): @@ -1820,11 +1830,51 @@ class TestManualPlatformPrompt(unittest.TestCase): chosen = install._prompt_manual_platform("linux") self.assertEqual(chosen, [("batocera", install.DEFAULT_DESTS["batocera"])]) - def test_quit_exits_cleanly(self): + def test_quit_stops_like_ctrl_c(self): + """`install.sh && retroarch` went on to start RetroArch with no BIOS + after a q, which left with 0 where Ctrl+C leaves with 130.""" with unittest.mock.patch("builtins.input", lambda _prompt="": "q"): with self.assertRaises(SystemExit) as raised: install._prompt_manual_platform("linux") - self.assertEqual(raised.exception.code, 0) + self.assertEqual(raised.exception.code, 130) + + def test_quotes_around_a_typed_folder_are_dropped(self): + """Explorer's "Copy as path" quotes the folder: the files went into a + folder named `"` under the current directory, and the run said Done.""" + root = Path(_tempdir(self)) + for typed in (f'"{root}/My Card/bios"', f"'{root}/My Card/bios'", + f' "{root}/My Card/bios" '): + answers = iter(["1", typed]) + with self.subTest(typed=typed), \ + unittest.mock.patch("builtins.input", lambda _prompt="": next(answers)), \ + contextlib.redirect_stdout(io.StringIO()): + chosen = install._prompt_manual_platform("linux") + self.assertEqual(chosen, [("retroarch", root / "My Card" / "bios")]) + + def test_home_and_variables_are_expanded_on_every_system(self): + for os_type in ("linux", "windows", "darwin"): + answers = iter(["1", '"~/$RB_TEST_DIR/bios"']) + with self.subTest(os_type=os_type), \ + unittest.mock.patch.dict(os.environ, {"RB_TEST_DIR": "cards"}), \ + unittest.mock.patch("builtins.input", lambda _prompt="": next(answers)), \ + contextlib.redirect_stdout(io.StringIO()): + chosen = install._prompt_manual_platform(os_type) + self.assertEqual(chosen[0][1], Path.home() / "cards" / "bios") + + def test_a_default_folder_not_there_is_not_found(self): + """With nothing installed, Enter printed "Found RetroArch at" a + folder that did not exist.""" + root = Path(_tempdir(self)) + answers = iter(["1", ""]) + out = io.StringIO() + with unittest.mock.patch.object(install, "_default_dest", + return_value=root / "retroarch" / "system"), \ + unittest.mock.patch("builtins.input", lambda _prompt="": next(answers)), \ + contextlib.redirect_stdout(out): + install._prompt_manual_platform("linux") + self.assertIn("default BIOS folder", out.getvalue()) + self.assertIn("will be created", out.getvalue()) + self.assertNotIn("Found", out.getvalue()) def test_a_wrong_answer_says_what_to_type(self): answers = iter(["99", "1", ""]) @@ -3261,7 +3311,8 @@ class AChunkedCutIsADownloadFailure(unittest.TestCase): unittest.mock.patch.object(install, "_retry_delay", return_value=0): dest, ok, reason = install._download_one(entry, Path(_tempdir(self))) self.assertFalse(ok) - self.assertIn("IncompleteRead", reason) + self.assertEqual(reason, "the connection was cut") + self.assertIsInstance(reason, install.NetworkReason) @@ -3719,6 +3770,21 @@ class ATypoGetsTheClosestName(unittest.TestCase): self.assertEqual(install.normalize_platform("RetroArch "), "retroarch") self.assertEqual(install.normalize_platform("RetroDECK"), "retrodeck") + def test_a_system_name_stands_for_the_list_it_gets(self): + """--list-platforms printed `muOS: retroarch`, and --platform muos + answered "Unknown platform".""" + for typed, plat in (("muos", "retroarch"), ("muOS", "retroarch"), + ("ArkOS", "retroarch"), ("PAN4ELEC", "amberelec"), + ("JELOS", "retroarch")): + out = io.StringIO() + with self.subTest(typed=typed), contextlib.redirect_stdout(out): + self.assertEqual(install.normalize_platform(typed), plat) + self.assertIn(f"--platform {plat}", out.getvalue()) + + def test_a_system_that_gets_no_list_is_still_refused(self): + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + install.normalize_platform("TrimUI") + def test_a_platform_typo_is_answered_with_one_name(self): err = io.StringIO() with contextlib.redirect_stderr(err), self.assertRaises(SystemExit) as raised: @@ -4672,8 +4738,28 @@ class HandheldSystemsByTheirOwnFiles(unittest.TestCase): {"/usr/trimui/": "", "/mnt/SDCARD/.system/res/": ""}) self.assertEqual(found, []) self.assertIn("MinUI found here", err) + self.assertIn("--platform minui", err) + self.assertIn("--dest", err) self.assertNotIn("TrimUI", err) + def test_a_minui_card_on_its_device_gets_its_file_list(self): + """--list-platforms said the MinUI layout was not written yet, while + --from-pack and --dest wrote it: on the device the card root is the + folder, and the card's version.txt names which of the three it is.""" + for release, plat, shown in (("MinUI-20251127-0", "minui", "MinUI"), + ("NextUI-20250601-1", "nextui", "NextUI"), + ("MyMinUI-20260809b-0", "myminui", "MyMinUI")): + with self.subTest(release=release): + root, found, name = self._one({ + "/usr/trimui/": "", "/mnt/SDCARD/.system/res/": "", + "/mnt/SDCARD/.system/version.txt": release + "\n"}) + self.assertEqual((found, name), ((plat, root / "mnt/SDCARD"), shown)) + + def test_the_listing_says_what_the_minui_family_gets(self): + line = next(line for line in install._system_lines() if line.startswith("MinUI:")) + self.assertNotIn("does not write", line) + self.assertIn("minui, nextui or myminui", line) + def test_two_unrelated_systems_are_refused(self): _root, found, names, _err = self._found( {"/mnt/SDCARD/Koriki/version.txt": "1\n"}, 'OS_NAME="JELOS"\n') @@ -4918,6 +5004,31 @@ def _events(proc: subprocess.CompletedProcess) -> list[dict]: return [json.loads(line) for line in proc.stdout.splitlines()] +class SystemEventsFollowTheChoice(unittest.TestCase): + """--system nintendo-gba wrote `system` events for nintendo-ds, gb and + gbc: SkyEmu's files carry four systems, and each was described by the + few of its files the choice kept.""" + + def test_only_the_chosen_systems_are_described(self): + files = [{"dest": "gba_bios.bin", "size": 1, "systems": ["nintendo-gba"]}, + {"dest": "SkyEmu/dmg.bin", "size": 1, + "systems": ["nintendo-gb", "nintendo-gba"]}] + stream = io.StringIO() + install.EVENTS.open(stream) + self.addCleanup(setattr, install.EVENTS, "stream", None) + install._emit_check("retroarch", {}, Path("/b"), files, [], [], 0, 0, + chosen=["nintendo-gba"]) + systems = [json.loads(line)["system"] for line in stream.getvalue().splitlines() + if json.loads(line)["event"] == "system"] + self.assertEqual(systems, ["nintendo-gba"]) + stream.seek(0) + stream.truncate() + install._emit_check("retroarch", {}, Path("/b"), files, [], [], 0, 0) + systems = [json.loads(line)["system"] for line in stream.getvalue().splitlines() + if json.loads(line)["event"] == "system"] + self.assertEqual(systems, ["nintendo-gb", "nintendo-gba"]) + + class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): """--events: the handheld port reads the run as JSON lines on stdout and never parses the text, which goes to stderr.""" @@ -5037,6 +5148,91 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): self.assertIn(f"`{key}`", row, f"{kind}.{key}") +class NetworkFailuresInPlainWords(unittest.TestCase): + """Offline, the run printed "[Errno -2] Name or service not known". A + beginner needs "no internet connection", and the raw text only under + --verbose.""" + + def _words(self, reason: BaseException, **kwargs) -> tuple[str, str]: + return install.network_words(urllib.error.URLError(reason), **kwargs) + + def test_the_common_failures(self): + import ssl # noqa: PLC0415 + + cases = ( + (socket.gaierror(-2, "Name or service not known"), "no internet connection"), + (socket.timeout("timed out"), "timed out"), + (ConnectionRefusedError(111, "Connection refused"), "refused the connection"), + (OSError(101, "Network is unreachable"), "no internet connection"), + (ssl.SSLCertVerificationError(1, "certificate verify failed"), + "certificate was rejected"), + ) + with unittest.mock.patch.object(install, "_ca_bundle_found", return_value=True): + for reason, words in cases: + with self.subTest(reason=reason): + code, text = self._words(reason) + self.assertEqual(code, "network") + self.assertIn(words, text) + self.assertNotIn("Errno", text) + + def test_verbose_keeps_the_raw_text(self): + _code, text = self._words(socket.gaierror(-2, "Name or service not known"), + verbose=True) + self.assertIn("Name or service not known", text) + + def test_a_clock_in_the_past_is_named(self): + """A handheld with no clock battery boots in 1980, and every + certificate then reads as not yet valid.""" + import datetime # noqa: PLC0415 + import ssl # noqa: PLC0415 + + error = ssl.SSLCertVerificationError(1, "certificate is not yet valid") + with unittest.mock.patch.object(install, "_ca_bundle_found", return_value=True): + code, text = self._words(error, today=datetime.date(1980, 1, 1)) + self.assertEqual(code, "clock") + self.assertIn("The clock reads 1980-01-01", text) + self.assertIn("set the date", text) + code, _text = self._words(error, today=install.CLOCK_FLOOR) + self.assertEqual(code, "network") + + def test_no_certificate_list_names_ssl_cert_file(self): + import ssl # noqa: PLC0415 + + error = ssl.SSLCertVerificationError(1, "unable to get local issuer certificate") + with unittest.mock.patch.object(install, "_ca_bundle_found", return_value=False): + code, text = self._words(error) + self.assertEqual(code, "no_ca") + self.assertIn("SSL_CERT_FILE", text) + + def test_a_certificate_list_is_looked_for_where_openssl_and_systems_keep_it(self): + root = Path(_tempdir(self)) + bundle = root / "cert.pem" + paths = SimpleNamespace(cafile=str(root / "none.pem"), capath=str(root / "none"), + openssl_cafile=str(root / "no.pem"), + openssl_capath=str(root / "no")) + with unittest.mock.patch("ssl.get_default_verify_paths", return_value=paths), \ + unittest.mock.patch.object(install, "CA_BUNDLES", ()), \ + unittest.mock.patch.dict(os.environ, {"SSL_CERT_FILE": "", "SSL_CERT_DIR": ""}): + self.assertFalse(install._ca_bundle_found()) + with unittest.mock.patch.object(install, "CA_BUNDLES", (str(bundle),)): + self.assertFalse(install._ca_bundle_found()) + bundle.write_text("x") + self.assertTrue(install._ca_bundle_found()) + + def test_a_file_list_out_of_reach_by_name_says_so(self): + env = {**os.environ, "RETROBIOS_BASE_URL": "https://retrobios-nohost.invalid", + "HOME": _tempdir(self)} + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", _tempdir(self), "--check"], + env=env, capture_output=True, text=True, timeout=120, check=False, + stdin=subprocess.DEVNULL, + ) + self.assertEqual(proc.returncode, 1) + self.assertIn("no internet connection", proc.stderr) + self.assertNotIn("Errno", proc.stderr) + + class AFileYouHadIsKept(unittest.TestCase): """A BIOS file whose contents differ was replaced and lost: a personal dump or another revision could not be brought back by deleting what the diff --git a/tests/test_install_experience.py b/tests/test_install_experience.py index f356a2f3..243a3c55 100644 --- a/tests/test_install_experience.py +++ b/tests/test_install_experience.py @@ -404,6 +404,161 @@ class ReviewCases(unittest.TestCase): progress.add(40) self.assertEqual(progress.transferred(), 40) +class _HeldHandler(http.server.BaseHTTPRequestHandler): + """Serves the manifest and first.bin at once, every other file once released.""" + + files: dict[str, bytes] = {} + manifest = b"" + release = threading.Event() + + def log_message(self, *args) -> None: # noqa: D102 + pass + + def do_GET(self) -> None: # noqa: N802, D102 + if self.path.endswith(".json"): + body = self.manifest + else: + name = self.path.rsplit("/", 1)[-1] + if name != "first.bin": + self.release.wait(30) + body = self.files[name] + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class AStoppedRunCountsWhatLanded(unittest.TestCase): + """Stopped at 8 of 118 files, the summary said "118 files installed": + every queued file was counted, written or not, and a user who read it + did not run the command again.""" + + @unittest.skipIf(os.name == "nt", "SIGINT is sent the POSIX way") + def test_only_the_written_file_is_counted(self): + import signal # noqa: PLC0415 + import time # noqa: PLC0415 + + files = {name: name.encode() * 64 for name in ("first.bin", "second.bin", "third.bin")} + manifest = {"manifest_version": 2, "platform": "retroarch", "files": [ + {"dest": name, "size": len(data), "sha1": hashlib.sha1(data).hexdigest(), + "repo_path": f"bios/{name}", "cores": None} + for name, data in files.items() + ]} + _HeldHandler.files, _HeldHandler.manifest = files, json.dumps(manifest).encode() + _HeldHandler.release = threading.Event() + httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _HeldHandler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + self.addCleanup(httpd.server_close) + self.addCleanup(httpd.shutdown) + self.addCleanup(_HeldHandler.release.set) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + dest = Path(tmp) / "bios" + env = dict(os.environ, + RETROBIOS_BASE_URL=f"http://127.0.0.1:{httpd.server_address[1]}") + proc = subprocess.Popen( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", str(dest), "--no-input"], + env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, + ) + deadline = time.monotonic() + 30 + while not (dest / "first.bin").exists() and time.monotonic() < deadline: + time.sleep(0.05) + time.sleep(0.2) + proc.send_signal(signal.SIGINT) + time.sleep(0.3) + _HeldHandler.release.set() + out, err = proc.communicate(timeout=60) + written = sorted(p.name for p in dest.iterdir() if not p.name.startswith(".")) + self.assertEqual(proc.returncode, 130, out + err) + self.assertEqual(written, ["first.bin"]) + self.assertIn("Stopped. 1 file installed.", out) + + +class ARefusedFolderReportsEachFile(unittest.TestCase): + """A read-only folder ended the run on its first refusal, and --events + wrote one `downloaded` line for seven failed files, while the events + table promises one per finished file.""" + + @unittest.skipIf(os.name == "nt" or os.geteuid() == 0, + "a read-only folder is refused by POSIX permissions") + def test_one_event_per_file_with_the_reason(self): + with FakeRepository() as repo: + repo.dest.mkdir() + os.chmod(repo.dest, 0o555) + try: + proc = repo.run("--events") + finally: + os.chmod(repo.dest, 0o755) + events = [json.loads(line) for line in proc.stdout.splitlines()] + downloaded = [e for e in events if e["event"] == "downloaded"] + self.assertEqual(proc.returncode, 2, proc.stderr) + self.assertEqual(len(downloaded), len(FakeRepository.PAYLOADS)) + self.assertEqual({e["reason"] for e in downloaded}, {install.NOT_WRITABLE}) + self.assertEqual(sorted(e["files_done"] for e in downloaded), + list(range(1, len(downloaded) + 1))) + + +class _Tty(io.StringIO): + def isatty(self) -> bool: + return True + + +class ColourFollowsNoColorOrg(unittest.TestCase): + """no-color.org: only a NO_COLOR that is set and not empty turns colour + off. An empty one, as `NO_COLOR= cmd` leaves, turned it off too.""" + + def _colour(self, **env: str) -> bool: + clean = {k: v for k, v in os.environ.items() + if k not in ("NO_COLOR", "FORCE_COLOR", "CLICOLOR_FORCE", "TERM")} + with mock.patch.dict(os.environ, {**clean, **env}, clear=True), \ + mock.patch.object(install, "_enable_ansi", return_value=True): + return install.Terminal(_Tty()).colour + + def test_an_empty_no_color_leaves_colour_on(self): + self.assertTrue(self._colour()) + self.assertTrue(self._colour(NO_COLOR="")) + self.assertFalse(self._colour(NO_COLOR="1")) + + def test_force_color_paints_a_pipe(self): + with mock.patch.object(install, "_enable_ansi", return_value=True): + for env, expected in (({}, False), ({"FORCE_COLOR": "1"}, True), + ({"CLICOLOR_FORCE": "1"}, True), + ({"FORCE_COLOR": "0"}, False), + ({"FORCE_COLOR": "1", "NO_COLOR": "1"}, False)): + clean = {k: v for k, v in os.environ.items() + if k not in ("NO_COLOR", "FORCE_COLOR", "CLICOLOR_FORCE")} + with self.subTest(env=env), \ + mock.patch.dict(os.environ, {**clean, **env}, clear=True): + term = install.Terminal(io.StringIO()) + self.assertEqual(term.colour, expected) + self.assertFalse(term.live) + + +class SmallWordings(unittest.TestCase): + def test_check_asks_which_to_check(self): + """--check with several platforms asked "Install for:", and + installs nothing.""" + out = io.StringIO() + with mock.patch("builtins.input", lambda _prompt="": "1"), \ + contextlib.redirect_stdout(out): + install._prompt_platform_choice( + [("emudeck", Path("/e")), ("retroarch", Path("/r"))], "Check:") + self.assertIn("Check:", out.getvalue()) + self.assertNotIn("Install for:", out.getvalue()) + + def test_nothing_detected_is_said(self): + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as home: + env = dict(os.environ, HOME=home, USERPROFILE=home, XDG_CONFIG_HOME=home, + RETROBIOS_OS="linux") + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--list-platforms"], + env=env, capture_output=True, text=True, timeout=60, check=False, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("Detected on this system: none", proc.stdout) + + class WindowsMandatoryLock(unittest.TestCase): """Issue 82: on Windows the partial copy's lock is mandatory, so hashing it by reopening its path failed with a sharing violation, and every file diff --git a/tests/test_install_offline.py b/tests/test_install_offline.py index 834e320a..7a614694 100644 --- a/tests/test_install_offline.py +++ b/tests/test_install_offline.py @@ -477,12 +477,33 @@ class TheSummaryNamesTheBulk(unittest.TestCase): _sized("mame/3do_m2.zip", 37 * MB, ["3do"], ["mame"]), ] lines = install.bulk_hint(files, {"system_cores": {"3do": ["mame", "opera"]}}, - systems_chosen=True, cores_chosen=False) + systems_chosen=True, cores_chosen=False, + accepted={"mame", "opera"}) self.assertEqual(lines, [ " mame pulls 337.0 MB of this 349.0 MB download for 3do. --core opera " "keeps 12.0 MB of it and saves 337.0 MB (96%)." ]) + def test_only_a_core_the_option_accepts_is_advised(self): + """`--core ares keeps 11.0 MB` was printed, and `--core ares` then + answered "unknown core": ares emulates the PlayStation and loads no + file of its own, and --core takes only the cores that do.""" + files = [ + _sized("scph5501.bin", 1 * MB, ["sony-playstation"]), + _sized("ps2_bios.bin", 300 * MB, ["sony-playstation"], ["duckstation"]), + _sized("pcsx/extra.bin", 2 * MB, ["sony-playstation"], ["pcsx_rearmed"]), + ] + manifest = {"system_cores": {"sony-playstation": ["ares", "duckstation", + "pcsx_rearmed"]}} + lines = install.bulk_hint(files, manifest, True, False, + accepted={"duckstation", "pcsx_rearmed"}) + self.assertEqual(len(lines), 1) + self.assertIn("--core pcsx_rearmed keeps 3.0 MB", lines[0]) + self.assertNotIn("ares", lines[0]) + self.assertEqual(install.bulk_hint(files, manifest, True, False, + accepted={"duckstation"}), []) + self.assertEqual(install.bulk_hint(files, manifest, True, False), []) + def test_a_system_pulling_most_of_the_download_is_named(self): files = [_sized("a.zip", 400 * MB, ["arcade"]), _sized("b.bin", 50 * MB, ["nes"]), _sized("c.bin", 10 * MB, ["gba"])] diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index b8169e69..667a8d21 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -9,10 +9,12 @@ to one answer. from __future__ import annotations +import contextlib import functools import hashlib import http.server import importlib.util +import io import json import os import random @@ -311,6 +313,24 @@ class CommandLineChoices(unittest.TestCase): self.assertIn("sega-saturn", str(ctx.exception)) self.assertIn("nintendo-gba", str(ctx.exception)) + def test_a_short_name_is_suggested_never_taken(self): + """`--system gba`, `psx` and `ps1` were refused on RetroArch with no + suggestion, though nintendo-gba holds the very letters.""" + known = [f"system-{i}" for i in range(40)] + [ + "nintendo-gb", "nintendo-gba", "nintendo-gbc", "sony-playstation", + "sony-playstation-2", "sega-mega-drive"] + for typed, meant in (("gba", "nintendo-gba"), ("psx", "sony-playstation"), + ("ps1", "sony-playstation"), ("PS2", "sony-playstation-2"), + ("genesis", "sega-mega-drive"), ("gb", "nintendo-gb")): + with self.subTest(typed=typed), self.assertRaises(ValueError) as ctx: + install.resolve_choices([typed], known, "system") + self.assertIn(f"Did you mean: {meant}", str(ctx.exception)) + + def test_a_short_name_for_a_system_the_list_lacks_is_not_suggested(self): + with self.assertRaises(ValueError) as ctx: + install.resolve_choices(["psx"], [f"system-{i}" for i in range(40)], "system") + self.assertNotIn("sony-playstation", str(ctx.exception)) + def test_an_empty_list_is_refused(self): with self.assertRaises(ValueError): install.resolve_regions([" , "]) @@ -334,6 +354,114 @@ class CommandLineChoices(unittest.TestCase): install.parse_selection(bad, 5) +class DecliningTheSizePointsAtTheChoice(unittest.TestCase): + """Saying no to `Download 8.4 GB?` printed "--system, --core and + --region install less (see --help)": a 91-line help for someone who had + just passed the c entry that does it.""" + + def _run(self, answers: list[str], *options: str) -> str: + replies = iter(answers) + out = io.StringIO() + with FakeRepository() as repo, \ + mock.patch.object(sys, "argv", ["install.py", "--platform", "retroarch", + "--dest", str(repo.dest), *options]), \ + mock.patch.object(install, "MANIFEST_URL", + repo.env()["RETROBIOS_BASE_URL"] + "/install/{platform}.json"), \ + mock.patch.object(install, "RAW_FILE_URL", + repo.env()["RETROBIOS_BASE_URL"] + "/{path}"), \ + mock.patch.object(install, "CONFIRM_FROM_BYTES", 1), \ + mock.patch("sys.stdin", mock.Mock(isatty=lambda: True)), \ + mock.patch("builtins.input", lambda _prompt="": next(replies)), \ + contextlib.redirect_stdout(out): + try: + install.main() + except SystemExit: + pass + self.assertEqual(repo.installed(), []) + return out.getvalue() + + def test_install_everything_then_no_names_the_c_entry(self): + out = self._run(["", "n"]) + self.assertIn("type c at the first question", out) + self.assertNotIn("see --help", out) + + def test_options_given_keep_the_option_advice(self): + out = self._run(["n"], "--system", "sony-playstation") + self.assertIn("--system, --core and --region install less", out) + + +class ACoreWithNoFileOfItsOwn(unittest.TestCase): + """pcsx_rearmed, swanstation and pcsx1 showed `0 files, 0 B` in the + cores step and in --list-cores, which reads as "this core gets nothing": + their files are the platform's own, which are always kept.""" + + FILES = [ + {"dest": "scph5501.bin", "size": 5, "cores": None, "systems": ["sony-playstation"], + "declared_choices": [], "slot_choices": [{"cores": ["pcsx_rearmed"]}]}, + {"dest": "ps2.bin", "size": 9, "cores": ["duckstation"], + "systems": ["sony-playstation"]}, + ] + + def _line(self, out: str, core: str) -> str: + return next(line for line in out.splitlines() if line.strip().startswith(core) + or f") {core}" in line) + + def test_the_listing_says_its_files_are_the_platforms(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + install._print_choices("retroarch", self.FILES, [], systems=False) + line = self._line(out.getvalue(), "pcsx_rearmed") + self.assertNotIn("0 files", line) + self.assertIn("always kept", line) + self.assertIn("1 file, 9 B", self._line(out.getvalue(), "duckstation")) + + def test_the_cores_step_says_so_too(self): + answers = iter(["c", "", ""]) + out = io.StringIO() + with mock.patch("builtins.input", lambda _prompt="": next(answers)), \ + contextlib.redirect_stdout(out): + install._prompt_custom_selection("retroarch", self.FILES, []) + line = self._line(out.getvalue(), "pcsx_rearmed") + self.assertNotIn("0 files", line) + self.assertIn("always kept", line) + + +class FilesSharedBySystemsAreCountedApart(unittest.TestCase): + """`3do 1704 files, 349.0 MB`: 1689 of them are MAME's hash and plugin + files, which six systems carry. Picking 3DO, one expects its BIOS.""" + + FILES = [ + {"dest": "panafz10.bin", "size": 4, "cores": None, "systems": ["3do"]}, + {"dest": "mame/hash/a.xml", "size": 100, "cores": None, + "systems": ["3do", "philips-cdi"]}, + {"dest": "mame/hash/b.xml", "size": 50, "cores": None, + "systems": ["3do", "philips-cdi"]}, + {"dest": "gba_bios.bin", "size": 8, "cores": None, "systems": ["nintendo-gba"]}, + ] + + def _listing(self) -> str: + out = io.StringIO() + with contextlib.redirect_stdout(out): + install._print_choices("retroarch", self.FILES, [], systems=True) + return out.getvalue() + + def test_the_listing(self): + lines = {line.split()[0]: line for line in self._listing().splitlines()} + self.assertIn("1 file, 4 B, and 2 files, 150 B shared with other systems", + lines["3do"]) + self.assertIn("2 files, 150 B, all shared with other systems", lines["philips-cdi"]) + self.assertTrue(lines["nintendo-gba"].endswith("1 file, 8 B")) + + def test_the_systems_step(self): + answers = iter(["c", ""]) + out = io.StringIO() + with mock.patch("builtins.input", lambda _prompt="": next(answers)), \ + contextlib.redirect_stdout(out): + install._prompt_custom_selection("retroarch", self.FILES, []) + self.assertIn("1 file, 4 B, and 2 files, 150 B shared with other systems", + out.getvalue()) + + class TheBoundaryChecksTheNewFields(unittest.TestCase): def _manifest(self, **extra): entry = {"dest": "a.bin", "sha1": "a" * 40, "size": 1, @@ -544,6 +672,74 @@ class TheOneLinerNarrows(unittest.TestCase): self.assertIn("--system sony-playstation --region north-america", out) self.assertEqual(repo.installed(), ["pcsx/extra.bin", "scph5501.bin"]) + @unittest.skipUnless(shutil.which("script"), "script(1) provides the terminal") + def test_a_quoted_folder_typed_at_the_prompt(self): + """The quotes "Copy as path" adds were kept: the files went into a + folder named `"` under the current directory, created without the + question --dest asks, and the run said Done.""" + with FakeRepository() as repo: + home = repo.root / "home" + home.mkdir() + # Read for the standalone emulator folders, of which there are none. + (repo.root / "install" / "emudeck.json").write_text(json.dumps( + {"manifest_version": 2, "platform": "emudeck", "files": []})) + card = repo.root / "My Card" / "bios" + env = dict(repo.env(), HOME=str(home), USERPROFILE=str(home), + XDG_CONFIG_HOME=str(home), RETROBIOS_OS="linux") + command = f"{shlex.quote(sys.executable)} {shlex.quote(str(REPO_ROOT / 'install.py'))}" + for answer, code in (("n", 2), ("y", 0)): + with self.subTest(answer=answer): + proc = subprocess.run( + ["script", "-qec", command, "/dev/null"], + input=f'1\n"{card}"\n{answer}\n\n', env=env, cwd=repo.root, + capture_output=True, text=True, timeout=60, check=False, + ) + self.assertEqual(proc.returncode, code, proc.stdout + proc.stderr) + self.assertIn(f"Create 2 folders under {repo.root}?", proc.stdout) + self.assertFalse((repo.root / '"').exists()) + self.assertEqual(len([p for p in card.rglob("*") if p.is_file()]), 5) + + def test_q_at_every_menu_stops_on_130(self): + """`install.sh && retroarch` started RetroArch after a q, which left + with 0 where Ctrl+C and Ctrl+D leave with 130.""" + files = [{"dest": "a.bin", "size": 1, "systems": ["nintendo-gba"]}] + menus = ( + lambda: install._choose_platform("Platform:"), + lambda: install._pick("Systems:", [("nintendo-gba", files)]), + lambda: install._prompt_custom_selection("retroarch", files, []), + ) + for menu in menus: + with self.subTest(menu=menu), \ + mock.patch("builtins.input", lambda _prompt="": "q"), \ + mock.patch("sys.stdout", new_callable=lambda: open(os.devnull, "w")), \ + self.assertRaises(SystemExit) as stop: + menu() + self.assertEqual(stop.exception.code, 130) + + @unittest.skipIf(os.name == "nt", "a pseudo-terminal is a POSIX one") + def test_ctrl_d_at_the_first_question_has_nothing_to_carry_on(self): + import pty # noqa: PLC0415 + + with FakeRepository() as repo: + home = repo.root / "home" + home.mkdir() + (repo.root / "install" / "emudeck.json").write_text(json.dumps( + {"manifest_version": 2, "platform": "emudeck", "files": []})) + env = dict(repo.env(), HOME=str(home), USERPROFILE=str(home), + XDG_CONFIG_HOME=str(home), RETROBIOS_OS="linux") + leader, follower = pty.openpty() + proc = subprocess.Popen( + [sys.executable, str(REPO_ROOT / "install.py")], stdin=follower, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, text=True, + ) + os.close(follower) + os.write(leader, b"\x04") + out, err = proc.communicate(timeout=60) + os.close(leader) + self.assertEqual(proc.returncode, 130, out + err) + self.assertIn("Stopped. Nothing was installed.", out) + self.assertNotIn("carry on", out) + @unittest.skipUnless(shutil.which("script"), "script(1) provides the terminal") def test_enter_installs_everything_at_a_terminal(self): with FakeRepository() as repo: diff --git a/wiki/first-install.md b/wiki/first-install.md index 9b5381b3..f6a62a28 100644 --- a/wiki/first-install.md +++ b/wiki/first-install.md @@ -76,8 +76,13 @@ It shows your setup and its BIOS folder: ``` - **Right folder?** Go on. -- **Wrong folder?** Press Ctrl+C. Run the command again with `--dest` and your - folder. +- **Wrong folder?** Press Ctrl+C. Run the command again with your folder + after `--dest`: + + +```sh +curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --dest ~/path/to/bios +``` ## 3. Choose what to install @@ -115,7 +120,13 @@ Add `--check` to the same command. It changes nothing. curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --check ``` -All good when you see `0 wrong hash`. +All good when every file is present and none is wrong: + +``` + 109/109 present (109 verified, 0 wrong hash) +``` + +An empty folder also shows `0 wrong hash`, with `0/109 present`. ## Undo @@ -127,7 +138,8 @@ BIOS folder. The end of the run prints the line that puts them back. ### No supported platform detected. At a terminal, a menu follows. Pick your setup by its number, then type its -BIOS folder. +BIOS folder, or press Enter for the usual one. A folder pasted with quotes +around it is fine. With `--no-input`, or without a terminal, it stops and prints the command to use. diff --git a/wiki/installer.md b/wiki/installer.md index 94f1f878..daf32cc5 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -493,19 +493,19 @@ what the options name, everything by default. | Option | Effect | |--------|--------| -| `--platform NAME` | Install for this platform instead of the detected one. Unknown names are refused with the available list | +| `--platform NAME` | Install for this platform instead of the detected one. A system `--list-platforms` names under its own files (`muos`, `arkos`) stands for the file list it gets, and the run says which. Unknown names are refused with the closest ones | | `--dest PATH` | Destination directory, overriding detection. Without `--platform`, the file list is the platform detected here: at a terminal the installer asks, Enter taking the one found, and otherwise it uses the one found, RetroArch's when nothing is found, and refuses when several are. The run prints whose list it is. A folder that does not exist is named with the nearest one that does, then created. When more than its last level is missing (a mistyped path), a run at a terminal asks first, and the answer defaults to no | | `--target NAME` | Keep only the files the cores of that hardware target need, and the systems they emulate. An unknown target is refused rather than ignored, since carrying on would install everything. On Batocera, Recalbox, ROCKNIX and Lakka the board is read from the system (`batocera.arch`, `recalbox.arch`, `HW_DEVICE`, `LIBREELEC_DEVICE` or `LIBREELEC_PROJECT`) and applied when no `--target` is given | | `--every-board` | Install the files of every board instead of the one the system names, for a card that moves between devices | | `--installed-cores` | Keep only what the cores already installed read (RetroArch, Lakka, RetroPie). See [Installed cores](#installed-cores) | | `--cores-dir PATH` | The folder RetroArch loads its cores from, for `--installed-cores` when it is not found. Implies `--installed-cores` | | `--save-dir PATH` | The frontend's save directory (RetroArch's `savefile_directory`), for the files some cores read there, when the run cannot find it. The core's own folder is still added below it when the frontend sorts saves by core | -| `--system NAMES` | Install only these systems. Comma-separated or repeated. `--list-systems` prints the names, `other` included | +| `--system NAMES` | Install only these systems. Comma-separated or repeated. `--list-systems` prints the names, `other` included. A short name the platform does not use (`gba`, `psx`) is refused with the ids it may stand for, never taken in their place | | `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names | | `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included | | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | | `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | -| `--list-systems` | Print a platform's systems with the files and size each holds | +| `--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart | | `--list-cores` | Print the cores whose extra files a platform installs | | `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | | `--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) | @@ -581,7 +581,7 @@ rewrites both to `/roms2/bios`: a second card is followed through them. | `/mnt/plumos-user` | plumOS | RetroArch | `/mnt/plumos-user/bios`, which its launcher appends to every RetroArch run | | `/etc/os-release` `ID=plumos` without the folder above | plumOS | none | its builds choose a folder per launch: named, not installed to | | `/usr/trimui` | TrimUI | none | no published source names the stock firmware's folder: named, not installed to | -| `/mnt/SDCARD/.system/res` or `/mnt/sdcard/.system/res` | MinUI | none | MinUI, MyMinUI and NextUI give each core its own `Bios//`, a layout not written yet: named, not installed to | +| `/mnt/SDCARD/.system/res` or `/mnt/sdcard/.system/res` | MinUI, NextUI or MyMinUI, as the first line of the card's `.system/version.txt` names it | MinUI, NextUI or MyMinUI | the card root, below which the list writes `Bios//`. Without `version.txt` the three cannot be told apart: named, not installed to, and the message gives `--platform` and `--dest` to use | | `/mnt/vendor/bin/dmenu.bin` | Anbernic stock firmware | RetroArch | `/mnt/mmc/bios` | AmberELEC, EmuELEC and KNULLI install the BIOS list they publish themselves. TheRA is not detected: no @@ -692,9 +692,9 @@ for EmuDeck (`C:\Emulation\bios` on Windows), `/storage/roms/bios` for ROCKNIX, directory for RetroArch. BizHawk has no such folder: it reads `Firmware` beside its executable wherever the archive was extracted, so a forced `--platform bizhawk` that finds no install stops and asks for `--dest`. MinUI, -NextUI and MyMinUI keep their files on the SD card, which the installer reads -from a computer it is mounted on: a forced `--platform minui` stops and asks -for the card's root as `--dest`, and a `--dest` that is a folder inside such a +NextUI and MyMinUI keep their files on the SD card. Run on the device, the +installer finds the card root. From a computer the card is mounted on, a +forced `--platform minui` stops and asks for the card's root as `--dest`, and a `--dest` that is a folder inside such a card (its `Bios/`) is refused with the root to name instead. On Android a forced `--platform retroarch` falls back to `/RetroArch/system` instead, a home directory there belonging to Termux rather than to the frontend. Pass `--dest` to say where instead of @@ -826,9 +826,10 @@ are PortMaster's own, so the port carries no graphics code. - From PortMaster's catalog, once the port is accepted there: open PortMaster, find RetroBIOS under the Other genre, install. -- `RetroBIOS.zip`, attached to each [release](https://github.com/Abdess/retrobios/releases/latest) - cut since the port was added, copied into PortMaster's - autoinstall folder. Starting PortMaster then installs it, with or without a +- `RetroBIOS.zip`, copied into PortMaster's autoinstall folder. The release + process attaches it to each [release](https://github.com/Abdess/retrobios/releases/latest) + cut after the port was added. Releases cut before carry none: when the + latest one lists no `RetroBIOS.zip`, build it from a clone (below). Starting PortMaster then installs it, with or without a network connection. PortMaster's FAQ names the folder: `/roms/ports/PortMaster/autoinstall/` on AmberELEC and ROCKNIX, `/roms/tools/PortMaster/autoinstall/` on ArkOS, `/mmc/MUOS/PortMaster/autoinstall/` on muOS, @@ -1012,8 +1013,8 @@ fields, the installer refuses to narrow rather than install everything. |------|---------| | `0` | everything asked for is installed or already up to date | | `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, or `--restore` could not put a file back | -| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a `--dest` folder that does not exist was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | -| `130` | stopped with Ctrl+C, or with Ctrl+C or Ctrl+D at a question. Run the same command again to carry on | +| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | +| `130` | stopped with Ctrl+C, or with `q`, Ctrl+C or Ctrl+D at a question. Run the same command again to carry on | | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) | `install.ps1` passes the installer's code on unchanged: run as a file it exits @@ -1044,9 +1045,9 @@ kind, and each kind has exactly the keys below, in this order. The schema is |------|------|------| | `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` | | `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found | -| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`). `reason` is `network`, `not_found` or `invalid` | +| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`). `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). `message` says it in words | | `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` | -| `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id. Files with no system count under `other`. `name` is the platform's own name for it | +| `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id, and with `--system` only the systems named. Files with no system count under `other`. `name` is the platform's own name for it | | `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply | | `download` | `platform`, `files`, `bytes` | Before the downloads start | | `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive | @@ -1068,7 +1069,9 @@ no event at all. A program reads a missing `end` as that case. | `RETROBIOS_INSTALL_URL` | Where the bootstrap fetches `install.py`. HTTPS only | | `RETROBIOS_INSTALL_SHA256` | The SHA-256 the bootstrap requires of that installer, 64 hex characters. When either of these two is set and the fingerprint does not match, the error names the variable instead of suggesting a retry | | `RETROBIOS_BOOTSTRAP` | Set by `install.sh` (`sh`) and `install.ps1` (`ps1`) for the temporary installer they run, so that its hints give back the one-line command rather than `python3 install.py` | -| `NO_COLOR` | Plain text, no colour, even at a terminal. `TERM=dumb` also turns the live progress line off | +| `NO_COLOR` | Plain text, no colour, even at a terminal, when set to any value but an empty one. `TERM=dumb` also turns the live progress line off | +| `FORCE_COLOR`, `CLICOLOR_FORCE` | Colour even when the output is a pipe or a file, set to any value but `0` or an empty one. `NO_COLOR` wins over both, and the progress line stays one line per file | +| `SSL_CERT_FILE`, `SSL_CERT_DIR` | The trusted certificates to check servers with, for a Python that finds none of its own | | `HTTPS_PROXY`, `NO_PROXY` | Honoured for every download through the standard library's default proxy handling: one names the proxy, the other the hosts to reach directly | ## When something goes wrong @@ -1084,6 +1087,19 @@ The installer fetches its manifest, then downloads only the repository or environment. A file that fails three attempts is counted and reported at the end, and the next run fetches only what is still missing. +The reason is said in words: no internet connection or a name that could not +be looked up, a connection that timed out, was refused or was cut, a +certificate that was rejected. `--verbose` adds the text Python gave. Two +causes are not the network and are not retried: + +- `The clock reads 1980-01-01`: a device without a clock battery starts on an + old date, and every certificate then looks not yet valid. Connect to Wi-Fi + and let the time sync, or set the date, then run again. +- `This system has no list of trusted certificates`: install the + `ca-certificates` package, or name a certificate bundle with `SSL_CERT_FILE`. + When the system keeps one where this Python does not look, the message names + it. Certificate checks are never turned off. + ### Permission denied The installer needs to write to the BIOS folder. On Linux and macOS, check who diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md index 14f4ff73..fdbfaeb7 100644 --- a/wiki/troubleshooting.md +++ b/wiki/troubleshooting.md @@ -67,6 +67,8 @@ version. ## If the command fails { #installation-script-fails } - **Network error:** try again once github.com opens. +- **`The clock reads ...`:** the date is wrong. Let it sync over Wi-Fi, or set + it, then run again. - **Permission denied:** do what the message says. It names what to run. - **No setup found:** name it and its folder: