diff --git a/install.py b/install.py index 779a12f13..cbdff8afc 100644 --- a/install.py +++ b/install.py @@ -7905,7 +7905,8 @@ def fold_line(text: str, hang: str = " ") -> str: at /media/moi/ONION/BIOS" broke inside "(RetroArch files)" and a folder inside its name (tui_audit_17 P3-8).""" cols = max(terminal_size()[0] - 1, 8) - if cell_width(text) <= cols: + if cell_width(text) <= cols or not _is_terminal(sys.stdout): + # A pipe or a log folds nothing: the line is kept whole there. return text lead = text[:len(text) - len(text.lstrip(" "))] prefix = lead + hang diff --git a/scripts/generate_installer_doc.py b/scripts/generate_installer_doc.py index a249bfd69..e3f799958 100644 --- a/scripts/generate_installer_doc.py +++ b/scripts/generate_installer_doc.py @@ -14,12 +14,15 @@ install.py gains has no prose yet: the generator names it and writes nothing, and the row is added by hand inside the markers, its first cell holding the option, kind, code or value alone. -The run samples (a dry run, a first install, a later check) are printed by -install.py itself, run in its own process against the committed -install/.json served on the loopback, from a home and a card made -under tmp/ and shown as /home/me and /userdata. Only the transport and the -machine are stood in for, see _child. Each is dated by the `generated` field -of its list, so a rendering is the same from one day to the next. +The run samples (a dry run, a first install, a later check, the reasons a +file is refused, a run copying from an old card) are printed by install.py +itself, run in its own process against the committed install/.json +served on the loopback, from a home and a card made under tmp/ and shown as +/home/me and /userdata. The reasons and the copy are read from a four-file +list laid out in that home (SMALL_LIST), whose files are written there. Only +the transport and the machine are stood in for, see _child. Each is dated by +the `generated` field of its list, so a rendering is the same from one day +to the next. """ from __future__ import annotations @@ -33,10 +36,12 @@ import json import os import re import shutil +import socketserver import subprocess import sys import tempfile import threading +import zipfile from pathlib import Path, PurePosixPath from typing import Callable @@ -282,6 +287,44 @@ class _ListHandler(http.server.SimpleHTTPRequestHandler): return +class _LoopbackServer(http.server.ThreadingHTTPServer): + """The list server, named by its number: HTTPServer.server_bind looks + 127.0.0.1 up with socket.getfqdn, a DNS query wherever /etc/hosts does + not name it (code_audit_24 P3-12).""" + + def server_bind(self) -> None: + socketserver.TCPServer.server_bind(self) + host, port = self.server_address[:2] + self.server_name, self.server_port = host, port + + +# The list the reasons and the copy are shown with: four files, each with +# the contents its name says, read as RetroArch's. +SMALL_LIST = { + "scph5500.bin": (b"J" * 64, ["sony-playstation"], ["japan"], None), + "scph5501.bin": (b"U" * 64, ["sony-playstation"], ["north-america"], + ["beetle_psx", "pcsx_rearmed"]), + "gba_bios.bin": (b"G" * 32, ["nintendo-gba"], None, ["mgba"]), + "pcsx/extra.bin": (b"E" * 8, ["sony-playstation"], None, ["pcsx_rearmed"]), +} + + +def write_small_list(path: Path) -> None: + import hashlib # noqa: PLC0415 + + files = [] + for dest, (data, systems, regions, cores) in SMALL_LIST.items(): + files.append({"dest": dest, "size": len(data), "sha1": hashlib.sha1(data).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), + "md5": hashlib.md5(data).hexdigest(), "systems": systems, + "cores": cores, "repo_path": f"bios/{dest}", + **({"regions": regions} if regions else {})}) + path.write_text(json.dumps({ + "manifest_version": 2, "platform": "retroarch", "display_name": "RetroArch", + "base_destination": "system", "files": files, "total_files": len(files), + "total_size": sum(entry["size"] for entry in files)}), encoding="utf-8") + + def _child(spec: dict) -> int: """install.main() as the sample runs it, in its own process. Only the transport and the machine are stood in for: no connection leaves the @@ -300,8 +343,13 @@ def _child(spec: dict) -> int: raise OSError(f"the sample makes no request to {address[0]}") return real_connect(address, *args, **kwargs) - def landed(f: dict, _bios: Path, _verbose: bool = False, progress: object = None, - _copy: object = None) -> tuple[str, bool, str]: + real_download = _install._download_one + + def landed(f: dict, bios: Path, verbose: bool = False, progress: object = None, + copy: object = None) -> tuple[str, bool, str]: + if copy is not None: + # A copy from this machine is made for real: it is local. + return real_download(f, bios, verbose, progress, copy) if progress is not None: progress.add(f["size"]) return f["dest"], True, "" @@ -353,7 +401,7 @@ class Sandbox: config = self.home / ".config" / "retroarch" / "retroarch.cfg" config.parent.mkdir(parents=True) config.write_text(RETROARCH_CFG, encoding="utf-8") - self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _ListHandler) + self.server = _LoopbackServer(("127.0.0.1", 0), _ListHandler) threading.Thread(target=self.server.serve_forever, daemon=True).start() return self @@ -362,26 +410,55 @@ class Sandbox: self.server.server_close() shutil.rmtree(self.root) + def small(self, kind: str) -> str: + """A run over SMALL_LIST in a home of its own: "refused", a --check + of a folder holding a copy of each kind a check names, or "from", a + run copying from an old card what it holds and downloading the rest.""" + home = self.root / kind + system = home / ".config" / "retroarch" / "system" + system.mkdir(parents=True) + (system.parent / "retroarch.cfg").write_text(RETROARCH_CFG, encoding="utf-8") + listed = home / "list.json" + write_small_list(listed) + data = {dest: entry[0] for dest, entry in SMALL_LIST.items()} + if kind == "refused": + (system / "scph5500.bin").write_bytes(data["scph5500.bin"]) + (system / "scph5501.bin").write_bytes(data["scph5500.bin"]) + (system / "system").mkdir() + (system / "system" / "gba_bios.bin").write_bytes(data["gba_bios.bin"]) + with zipfile.ZipFile(system / "PSX BIOS.zip", "w") as archive: + archive.writestr("extra.bin", data["pcsx/extra.bin"]) + # A check that flags a file ends on 1, as documented. + return self.run("retroarch", "--check", "--manifest", str(listed), home=home, + codes=(1,)) + old = home / "old" + old.mkdir() + for dest in ("scph5500.bin", "scph5501.bin"): + (old / dest).write_bytes(data[dest]) + return self.run("retroarch", "--yes", "--manifest", str(listed), "--from", str(old), + home=home, landed=True) + def run(self, platform: str, *argv: str, landed: bool = False, - held: bool = False) -> str: + held: bool = False, home: Path | None = None, codes: tuple = (0,)) -> str: """What install.py prints for the run, the sandbox read as a user's home and card.""" + home = home or self.home card = self.card / "userdata" / "bios" if platform != "retroarch" else None if card is not None: card.mkdir(parents=True, exist_ok=True) spec = {"platform": platform, "argv": ["--plain", "--platform", platform, *argv], "card": str(card) if card else None, "landed": landed, "held": held} - env = {"PATH": os.environ.get("PATH", ""), "HOME": str(self.home), + env = {"PATH": os.environ.get("PATH", ""), "HOME": str(home), "RETROBIOS_OS": "linux", "LANG": "C.UTF-8", "RETROBIOS_BASE_URL": f"http://127.0.0.1:{self.server.server_address[1]}"} done = subprocess.run( [sys.executable, "-s", "-c", CHILD, json.dumps(spec)], cwd=ROOT, env=env, stdin=subprocess.DEVNULL, capture_output=True, text=True, check=False) - if done.returncode != 0: + if done.returncode not in codes: raise SampleFailed(f"install.py {' '.join(spec['argv'])} exited " f"{done.returncode}: {done.stderr.strip()[-2000:]}") - return done.stdout.replace(str(self.home), SHOWN_HOME).replace(str(self.card), SHOWN_CARD) + return done.stdout.replace(str(home), SHOWN_HOME).replace(str(self.card), SHOWN_CARD) # The runs printed in this process, by what they read: the page is rendered @@ -411,6 +488,14 @@ class _LazySandbox: _RUNS[key] = self.box.run(platform, *argv, **kinds) return _RUNS[key] + def small(self, kind: str) -> str: + key = ("small", kind, _stamp("retroarch")) + if key not in _RUNS: + if self.box is None: + self.box = self.stack.enter_context(Sandbox()) + _RUNS[key] = self.box.small(kind) + return _RUNS[key] + class SampleFailed(Exception): """A run sample install.py could not print.""" @@ -487,9 +572,23 @@ def sample_writers(box: Sandbox) -> dict[str, Callable[[str], str]]: + _fenced([_bulk_line(box)])), "run-check": lambda _body: _fenced( _lines_from(check_run(), "Checking existing files", " Everything is in place")), + "check-refused": lambda _body: _fenced(_refused_lines(box.small("refused"))), + "from-run": lambda _body: _fenced( + _lines_from(box.small("from"), "Done.", count=2)), } +def _refused_lines(text: str) -> list[str]: + """The files a check flags, each with its reason and where its cores read it.""" + lines = text.splitlines() + start = next((i for i, line in enumerate(lines) + if re.fullmatch(r" \d+ files? [a-z ]+:", line)), None) + if start is None: + raise SampleFailed("the check printed no files not in place") + end = max(i for i, line in enumerate(lines) if line.lstrip().startswith("read by ")) + return lines[start:end + 1] + + def render(page: str) -> str: """The page with each marked table and run sample written from install.py.""" writers = { diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index bed3b080a..435cd54ad 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -675,7 +675,7 @@ This executes in sequence: 4. `generate_pack.py --all` - build ZIP packs + install and target manifests 5. Consistency check: verify and the pack agree on counts and on the paths each archive holds 6. Pack integrity - extract ZIPs and verify hashes per platform mode -7. `generate_readme.py` - regenerate README and CONTRIBUTING +7. `generate_readme.py` - regenerate README and CONTRIBUTING, then `generate_installer_doc.py` rewrites the tables and run samples of `wiki/installer.md` from the new lists 8. `generate_site.py` - regenerate the documentation site and its navigation A new platform is archived-by-default nowhere: `status: active` puts it in diff --git a/wiki/installer.md b/wiki/installer.md index 477fae82f..6a8183256 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -480,7 +480,9 @@ On a first install nothing is in place. On a later run the line counts what is already there, and only the rest is downloaded. A download of 1 GB or more is confirmed first, the size and the folder in the question. -The second phase downloads what is missing, one line per file: +The second phase downloads what is missing. At a terminal one progress line +moves with the count and the size. A pipe or a log gets one line per file, +as the log of the run below: @@ -525,9 +527,10 @@ RetroArch with `--system sony-playstation`, from the list of 2026-10-27: `--dry-run` prints the same line under its plan. -When some files came from this machine rather than the network, a line under -`Done` says where each came from (`38 files copied from E:\, 3 copied from -the wrong place in the BIOS folder, 4 downloaded.`). See +When some files came from this machine rather than the network, the run says +where each came from: on the `Done` line when all came from one place +(`Done. 22 files copied from /home/me/old.`), on a line under it otherwise +(`2 files copied from /home/me/old, 2 downloaded.`). See [Copy from an old card](#copy-from-an-old-card). The word `Done` and the BIOS folder are the confirmation. Any other ending @@ -565,20 +568,27 @@ replaces it and keeps the old one, see ### Why a file is refused { #why-a-file-is-refused } Each file `--check` flags comes with the reason it found and the place each -core opens it: +core reads it. A RetroArch folder holding a copy of each kind, checked +against a list of four files: + + ``` 3 files to download: + 2 of these files are in the BIOS folder in the wrong place: a run without --check copies them where the cores open them, and leaves your copies as they are. wrong copy: scph5501.bin another region: the contents of scph5500.bin (japan) - opened at /home/me/.config/retroarch/system/scph5501.bin by beetle_psx, pcsx_rearmed + read by beetle_psx, pcsx_rearmed at /home/me/.config/retroarch/system/scph5501.bin missing: gba_bios.bin nested folder: it is at system/system/gba_bios.bin, a system folder inside system - opened at /home/me/.config/retroarch/system/gba_bios.bin by mgba + a run without --check copies it into place + read by mgba at /home/me/.config/retroarch/system/gba_bios.bin missing: pcsx/extra.bin still zipped: it is inside PSX BIOS.zip - opened at /home/me/.config/retroarch/system/pcsx/extra.bin by pcsx_rearmed + a run without --check copies it into place + read by pcsx_rearmed at /home/me/.config/retroarch/system/pcsx/extra.bin ``` + | Reason | What the folder holds | |--------|-----------------------| @@ -699,12 +709,15 @@ other contents is kept aside first, as a download would replace it (see [Files you already had](#files-you-already-had)). What no folder holds is downloaded as usual, or taken from the pack with `--from-pack`. Several `--from` are read in the order given, and the end of the run says where each -file came from: +file came from. A run of a list of four files, two of them in `old`: + + ``` -Done. 44 files installed. - 38 files copied from E:\, 4 downloaded, 2 still missing. +Done. 4 files installed. + 2 files copied from /home/me/old, 2 downloaded. ``` + `--check --from FOLDER` writes nothing and says how many files the folder would give (`E:\ holds 38 of these files: a run without --check copies @@ -1056,6 +1069,8 @@ gives for the device, below the card's root instead of its mount point: | `.tmp_update/onionVersion/version.txt` | Onion | `BIOS`, RetroArch's list | | `.allium/` | Allium | `BIOS`, RetroArch's list | | `Koriki/version.txt` | Koriki | `BIOS`, RetroArch's list | +| `bgmusic/` beside `bios/`, on a drive that is not a fixed disk | ArkOS | `bios`, RetroArch's list | +| `launchimages/` beside `bios/`, on a drive that is not a fixed disk | ArkOS | `bios`, RetroArch's list | | `gamedata/`, `homebrew/` beside `bios/`, on a drive that is not a fixed disk | AmberELEC | `bios`, AmberELEC's list | | `bezels/default.cfg` beside `bios/`, on a drive that is not a fixed disk | EmuELEC | `bios`, EmuELEC's list | | `system/batocera.conf` holding `system.hostname=BATOCERA` or no hostname line, and no KNULLI file above | Batocera SHARE drive | `bios`, Batocera's list | @@ -1635,14 +1650,20 @@ The build scripts need 3.9 and are not bound by this. Every change to `install.py` recomputes the SHA-256 pinned in `install.sh` and `install.ps1` in the same commit. The tests compare the pins with the file. -The tables of options, events, exit codes and keys on this page sit between -`generated` markers and are written from `install.py` by -`scripts/generate_installer_doc.py`: the options from its parser, the events -from `EVENT_KEYS`, the codes from its `sys.exit` calls, the keys from -`TUI_ROUTES`. The prose cells stay as written. A row the code gains stops the -script until its text is added inside the markers, and `--check`, which a -test runs, fails on a table behind the code. `--page FILE` renders another -copy of the page, which the tests use. +The tables of options, events, exit codes, keys and card markers, the line +menu and the run samples on this page sit between `generated` markers and are +written from `install.py` by `scripts/generate_installer_doc.py`: the options +from its parser, the events from `EVENT_KEYS`, the codes from its `sys.exit` +calls, the keys from `TUI_ROUTES`, the card markers from its card rules. The +run samples (a dry run, a first install, a check, the reasons a file is +refused, a copy from an old card) are printed by `install.py` itself, in a +sandbox under `tmp/` served on the loopback from the committed +`install/*.json`. A new list therefore rewrites this page, and `--check` +fails until the script has run. Pipeline step 7 runs it after the README. +The prose cells stay as written. A row the code gains stops the script until +its text is added inside the markers, and `--check`, which a test runs, fails +on a table behind the code. `--page FILE` renders another copy of the page, +which the tests use. The [handheld](#on-a-handheld-the-retrobios-port) archives each carry `install.py`, the one the one-line command of the same commit downloads: @@ -1714,6 +1735,7 @@ other schemas under `api/v1/schemas/`. | `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered | | `version` | `platform`, `list_version`, `installed_version`, `message` | Before `choices`, when the system here runs another version than the one the list was built for: `list_version` is the list's, `installed_version` the one the system's version file states, and `message` says it and what it means. Not written when either version is unknown. The run goes on | | `choices` | `platform`, `systems` | Before `checking`: the systems the run's list lets one choose among, sorted by id, `other` last for the files no system names. The run's other options apply (`--core`, `--region`, `--no-game-data`, the files left out for an unknown save directory), `--system` does not: these are the systems a screen lists to choose from, and the count a `--system` is out of | +| `set_aside` | `platform`, `dests` | Right after `choices`: the files of the list the run sets aside before any choice, in list order, those read from a save directory whose place is not known among them. A screen chooses its cores among the rest, the cores `--core` accepts | | `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` | | `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id, and with `--system` only the systems named. Files with no system count under `other`. `name` is the platform's own name for it | | `checking` | `files_done`, `files_total` | While the files already in the folder are read and hashed, at most every half second and once at the end. `files_total` counts the files found in place | diff --git a/wiki/tools.md b/wiki/tools.md index 624a58ade..498008b3e 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -49,7 +49,7 @@ Pipeline steps: | 4d | Collected copies only an emulator install provides (`bundled_only.py --strict`, gates the run) | - | | 5 | Consistency check: counts, paths per archive, every emulator pack | if verify or pack skipped | | 6 | Pack integrity (extract + hash) | `--skip-packs` | -| 7 | Generate README | `--skip-docs` | +| 7 | Generate README and CONTRIBUTING, then the installer page's tables and run samples (`generate_installer_doc.py`) | `--skip-docs` | | 8 | Generate site | `--skip-docs` | | 8b | Render the site with `npm run build --prefix website`, then `validate_site.py`, as the deploy does | `--skip-docs`, or Node not set up (`npm ci --prefix website`) | @@ -1301,7 +1301,7 @@ before a release (wiki/release-process.md). Its lines stay in | `validate_pr.py` | Validate BIOS files in pull requests, post markdown report | | `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs | | `generate_event_schema.py` | Write `schemas/install-events.schema.json` from `EVENT_KEYS` in `install.py` (`--check` exits 1 when the file is stale, which `validate_schemas.py` also reports) | -| `generate_installer_doc.py` | Rewrite the generated tables of `wiki/installer.md` (options, events, exit codes, keys per route) from `install.py` (`--check` exits 1 when a table is stale, `--page FILE` writes another copy of the page) | +| `generate_installer_doc.py` | Rewrite the generated blocks of `wiki/installer.md` from `install.py`: the tables (options, events, exit codes, keys per route, card markers), the line menu, and the run samples `install.py` prints in a sandbox against the committed `install/*.json`, so a new list rewrites the page (`--check` exits 1 when a block is stale, `--page FILE` writes another copy of the page). Pipeline step 7 | | `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline). A download is accepted only when every cryptographic hash the entry declares (sha256, sha1, md5) matches. crc32 and size find a file the database already holds, never accept a download. An entry with no cryptographic hash is reported and not searched | | `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones, used by CI) | | `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) |