diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index 2fbee0d48..8f1ca775f 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -32,7 +32,6 @@ from __future__ import annotations import argparse import contextlib -import hashlib import io import json import re @@ -51,6 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import export_native # noqa: E402 +import pin_installer # noqa: E402 import refresh_data_dirs # noqa: E402 import upstream # noqa: E402 from common import ( # noqa: E402 @@ -83,7 +83,6 @@ PYPI_URL = "https://pypi.org/pypi/{name}/json" NPM_URL = "https://registry.npmjs.org/{name}/latest" # The site's npm dependencies, pinned to exact versions and built by Deploy Site. SITE_PACKAGE = REPO_ROOT / "website" / "package.json" -INSTALLERS = ("install.sh", "install.ps1") SHOWN_ITEMS = 4 # items named in a diff summary before ", ..." SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..." @@ -1465,18 +1464,16 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]: def _check_installer_pins() -> Finding: """The bootstraps pin install.py by SHA-256, so a stale pin refuses every install.""" - digest = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest() - stale = [] - for name in INSTALLERS: - text = (REPO_ROOT / name).read_text(encoding="utf-8", errors="replace") - if digest not in text: - stale.append(name) + stale = pin_installer.stale_pins(REPO_ROOT) return Finding( "ci", "install.py pin", STALE if stale else OK, - local=digest[:12], - detail=("pin outdated in " + ", ".join(stale)) if stale else "install.sh and install.ps1 match", + local=pin_installer.installer_sha256(REPO_ROOT)[:12], + detail=( + f"pin outdated in {', '.join(stale)}, {pin_installer.COMMAND} writes it" + if stale else "install.sh and install.ps1 match" + ), ) diff --git a/scripts/pin_installer.py b/scripts/pin_installer.py new file mode 100644 index 000000000..07e366cf0 --- /dev/null +++ b/scripts/pin_installer.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Write the SHA-256 of install.py into the bootstraps that pin it. + +install.sh and install.ps1 refuse an installer whose hash is not the one +they carry, so every change to install.py moves both pins in the same +commit. --check writes nothing and exits 1 naming each stale pin. + +Usage: + python scripts/pin_installer.py + python scripts/pin_installer.py --check +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +INSTALLER = "install.py" +# Each bootstrap and its one pin line, the value between the two groups. +PINS: dict[str, re.Pattern[str]] = { + "install.sh": re.compile(r'^(DEFAULT_INSTALL_SHA256=")[0-9a-f]{64}(")', re.M), + "install.ps1": re.compile(r'^(\s*\$defaultInstallSha256 = ")[0-9a-f]{64}(")', re.M), +} +COMMAND = "python scripts/pin_installer.py" + + +def installer_sha256(root: Path = ROOT) -> str: + return hashlib.sha256((root / INSTALLER).read_bytes()).hexdigest() + + +def stale_pins(root: Path = ROOT) -> list[str]: + """The bootstraps whose pin is not the SHA-256 of install.py.""" + digest = installer_sha256(root) + stale = [] + for name, pin in PINS.items(): + found = pin.search((root / name).read_bytes().decode("utf-8")) + if found is None: + raise ValueError(f"{name}: no installer pin line") + if found.group(0) != f"{found.group(1)}{digest}{found.group(2)}": + stale.append(name) + return stale + + +def write_pins(root: Path = ROOT) -> list[str]: + """Rewrite each stale pin. Returns the bootstraps written.""" + digest = installer_sha256(root) + written = [] + for name in stale_pins(root): + path = root / name + text = path.read_bytes().decode("utf-8") + new, count = PINS[name].subn(rf"\g<1>{digest}\g<2>", text) + if count != 1: + raise ValueError(f"{name}: {count} installer pin lines, expected 1") + path.write_bytes(new.encode("utf-8")) + written.append(name) + return written + + +def main() -> int: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0]) + parser.add_argument("--check", action="store_true", + help="write nothing, exit 1 if a pin is stale") + args = parser.parse_args() + if args.check: + stale = stale_pins() + for name in stale: + print(f"{name}: pin is not the SHA-256 of {INSTALLER}, {COMMAND} writes it", + file=sys.stderr) + return 1 if stale else 0 + for name in write_pins(): + print(f"{name}: pinned {installer_sha256()}") + return 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/scripts/refresh_stale.py b/scripts/refresh_stale.py index 960b2c3da..5ec112efc 100755 --- a/scripts/refresh_stale.py +++ b/scripts/refresh_stale.py @@ -34,7 +34,8 @@ has ended, and its export is rewritten even when no row named it. Surfaced only (no refresh): data rows that are not a _data_dirs.yml key (the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs, -CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync, rom-folder +CI pins (PyPI, pinned actions; the install.py SHA-256 pin is written by +scripts/pin_installer.py), profile_sync, rom-folder pins (moved by hand, then python -m scripts.scraper.romfolders --setup NAME reads them again). Each needs a human read (new .info file, manual DAT download, PyPI version bump). @@ -401,6 +402,10 @@ def _command_for( "--force", ] + if area == "ci" and subject == "install.py pin": + # The pin is the hash of the committed install.py, nothing to judge. + return [sys.executable, "scripts/pin_installer.py"] + if area == "catalogs": if subject == "mame recipes": return [ diff --git a/tests/test_install.py b/tests/test_install.py index f36389a1e..f128216e6 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -36,6 +36,9 @@ _spec.loader.exec_module(install) sys.path.insert(0, str(REPO_ROOT / "tests")) from netguard import module_guard # noqa: E402 +# What a stale pin's failure says: the fix is one command. +PIN_ADVICE = "install.py changed: python scripts/pin_installer.py rewrites both pins" + # No connection off the loopback and no card of the host reaches a test: # the module fails at its end otherwise. setUpModule, tearDownModule = module_guard("test_install") @@ -2201,7 +2204,7 @@ class TestAvailablePlatforms(unittest.TestCase): match = re.search(r'\$defaultInstallSha256 = "([0-9a-f]{64})"', content) self.assertIsNotNone(match, "install.ps1 must embed the installer SHA256") expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest() - self.assertEqual(match.group(1), expected) + self.assertEqual(match.group(1), expected, PIN_ADVICE) def test_powershell_wrapper_rejects_non_https_bootstrap(self): content = (REPO_ROOT / "install.ps1").read_text() @@ -2212,7 +2215,31 @@ class TestAvailablePlatforms(unittest.TestCase): match = re.search(r'DEFAULT_INSTALL_SHA256="([0-9a-f]{64})"', content) self.assertIsNotNone(match, "install.sh must embed the installer SHA256") expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest() - self.assertEqual(match.group(1), expected) + self.assertEqual(match.group(1), expected, PIN_ADVICE) + + def test_pin_script_writes_both_pins(self): + """Every edit to install.py meant a sha256sum pasted by hand into two + files with two syntaxes, and the failing test named no command.""" + sys.path.insert(0, str(REPO_ROOT / "scripts")) + import pin_installer # noqa: PLC0415 + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory: + root = Path(directory) + for name in ("install.py", "install.sh", "install.ps1"): + shutil.copy2(REPO_ROOT / name, root / name) + (root / "install.py").write_text("# changed\n", encoding="utf-8") + sh_mode = (root / "install.sh").stat().st_mode + self.assertEqual(pin_installer.stale_pins(root), ["install.sh", "install.ps1"]) + self.assertEqual(pin_installer.write_pins(root), ["install.sh", "install.ps1"]) + self.assertEqual(pin_installer.stale_pins(root), []) + digest = hashlib.sha256(b"# changed\n").hexdigest() + for name in ("install.sh", "install.ps1"): + before = (REPO_ROOT / name).read_text(encoding="utf-8").splitlines() + after = (root / name).read_text(encoding="utf-8").splitlines() + changed = [b for a, b in zip(before, after) if a != b] + self.assertEqual(len(changed), 1, name) + self.assertIn(digest, changed[0]) + self.assertEqual((root / "install.sh").stat().st_mode, sh_mode) def test_shell_one_liner_downloads_verifies_and_forwards_arguments(self): scratch = REPO_ROOT / "tmp" / "tests" diff --git a/tests/test_refresh_stale.py b/tests/test_refresh_stale.py index a1c463b0c..e92887a66 100644 --- a/tests/test_refresh_stale.py +++ b/tests/test_refresh_stale.py @@ -162,6 +162,11 @@ class Planning(unittest.TestCase): self.assertEqual(jobs, []) self.assertEqual(len(surfaced), 3) + def test_the_installer_pin_is_written_by_its_script(self): + jobs, surfaced, _ = self._plan(finding("ci", "install.py pin")) + self.assertEqual([job.command[1:] for job in jobs], [["scripts/pin_installer.py"]]) + self.assertEqual(surfaced, []) + class Running(unittest.TestCase): def test_a_child_starts_after_its_parent_ends(self): diff --git a/wiki/installer.md b/wiki/installer.md index 73719bf16..d5a930694 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1702,7 +1702,9 @@ a real 3.8 interpreter, which CI installs beside 3.12 so the check cannot skip. The build scripts need 3.9 and are not bound by this. Every change to `install.py` recomputes the SHA-256 pinned in `install.sh` and -`install.ps1` in the same commit. The tests compare the pins with the file. +`install.ps1` in the same commit: `python scripts/pin_installer.py` writes both +(`--check` writes nothing and exits 1 on a stale pin). The tests compare the +pins with the file and name that command when they differ. The tables of options, events, exit codes, keys and card markers, the line menu and the run samples on this page sit between `generated` markers and are diff --git a/wiki/tools.md b/wiki/tools.md index f566efe53..65296bcee 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1346,6 +1346,7 @@ before a release (wiki/release-process.md). Its lines stay in | `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs | | `generate_event_schema.py` | Write `schemas/install-events.schema.json` from `EVENT_KEYS` in `install.py` (`--check` exits 1 when the file is stale, which `validate_schemas.py` also reports) | | `generate_installer_doc.py` | Rewrite the generated blocks of `wiki/installer.md` from `install.py`: the tables (options, events, exit codes, keys per route, card markers), the line menu, and the run samples `install.py` prints in a sandbox against the committed `install/*.json`, so a new list rewrites the page (`--check` exits 1 when a block is stale, `--page FILE` writes another copy of the page). Pipeline step 7 | +| `pin_installer.py` | Write the SHA-256 of `install.py` into the pins of `install.sh` and `install.ps1`, which refuse any other installer (`--check` writes nothing and exits 1 when a pin is stale). `check_freshness.py` reports a stale pin and `refresh_stale.py` runs this | | `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline). A download is accepted only when every cryptographic hash the entry declares (sha256, sha1, md5) matches. crc32 and size find a file the database already holds, never accept a download. An entry with no cryptographic hash is reported and not searched | | `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones) | | `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) |