diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 928bd71f4..965d1a80a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,10 @@ the most useful contribution. hash differs from the one already held under its name goes to that folder's `.variants/`. +A ZIP that no list declares as a whole is read file by file, and the report +says which of them the collection already holds: upload the files +themselves, unzipped. + From a clone, the file goes to `bios/Manufacturer/Console/filename`, and `python scripts/validate_pr.py path/to/file` names that folder for a file still outside `bios/`, and `--place` copies it there under its declared name. Given a diff --git a/install/targets/emudeck.json b/install/targets/emudeck.json index 36f30d211..4ef016714 100644 --- a/install/targets/emudeck.json +++ b/install/targets/emudeck.json @@ -83,6 +83,7 @@ "galaksija", "gam4980", "gambatte", + "gametank", "gearboy", "gearcoleco", "geargrafx", @@ -346,6 +347,7 @@ "galaksija", "gam4980", "gambatte", + "gametank", "gearboy", "gearcoleco", "geargrafx", diff --git a/platforms/targets/_overrides.yml b/platforms/targets/_overrides.yml index 1080067b0..d07226f89 100644 --- a/platforms/targets/_overrides.yml +++ b/platforms/targets/_overrides.yml @@ -121,13 +121,6 @@ retrodeck: # includes the frontend name alongside the actual cores. remove_cores: [retroarch] -emudeck: - targets: - _default: - # EmuDeck's EmuScripts tree carries a template.sh placeholder used - # to scaffold new installers. It is not an emulator. - remove_cores: [template] - lakka: targets: generic: diff --git a/platforms/targets/emudeck.yml b/platforms/targets/emudeck.yml index 49ac73621..c445457fb 100644 --- a/platforms/targets/emudeck.yml +++ b/platforms/targets/emudeck.yml @@ -1,6 +1,6 @@ platform: emudeck -source: https://github.com/dragoonDorise/EmuDeck -scraped_at: '2026-10-17T18:43:57Z' +source: https://github.com/dragoonDorise/EmuDeck/tree/dddb30229adbbd54d5ff7283cfd70e11c4322c26 +scraped_at: '2026-11-06T07:10:22Z' targets: steamos: architecture: x86_64 @@ -88,6 +88,7 @@ targets: - galaksija - gam4980 - gambatte + - gametank - gearboy - gearcoleco - geargrafx @@ -352,6 +353,7 @@ targets: - galaksija - gam4980 - gambatte + - gametank - gearboy - gearcoleco - geargrafx @@ -487,7 +489,6 @@ targets: - supermodel - swanstation - tamalibretro - - template - tgbdual - theodore - thepowdertoy diff --git a/scripts/artifacts.py b/scripts/artifacts.py index 66b752837..79643ae39 100644 --- a/scripts/artifacts.py +++ b/scripts/artifacts.py @@ -19,6 +19,7 @@ import re _TIMESTAMP_PATTERNS = [ re.compile(r'"generated_at":\s*"[^"]*"'), # database.json re.compile(r'"imported_at":\s*"[^"]*"'), # provenance snapshots + re.compile(r'"fetched":\s*"[^"]*"'), # games/catalogs/ re.compile(r"\*Auto-generated on [^*]*\*"), # README.md re.compile(r"\*Generated on [^*]*\*"), # docs site pages # The decorated pages carry the same stamp again as a rendered element, @@ -208,6 +209,10 @@ def _strip_timestamps(text: str) -> str: class ArtifactLockBusy(RuntimeError): """Raised when another process already holds the artifact directory.""" + +class ArtifactDirectoryMissing(FileNotFoundError): + """A reader named a directory that does not exist.""" + # A run that holds artifact directories for its whole duration names them # here, os.pathsep apart, for the steps it spawns: they would otherwise be # refused by it. A pipeline holds the repository root and its --output-dir. @@ -227,6 +232,14 @@ def artifact_lock(directory: str, exclusive: bool = True) -> Iterator[None]: Writers take the lock exclusively, readers share it. On platforms without flock the lock is a no-op. """ + # A reader creating the directory it was pointed at left a typo behind + # as a directory holding a .lock, and the next run with the same typo + # no longer said the path did not exist. + if not exclusive and not os.path.isdir(directory): + raise ArtifactDirectoryMissing( + f"{directory}: no such directory, pass the --output-dir the packs " + "were written to (default dist/)" + ) try: import fcntl except ImportError: diff --git a/scripts/bios_integrity.py b/scripts/bios_integrity.py index 72d1caf71..1b0a69b17 100644 --- a/scripts/bios_integrity.py +++ b/scripts/bios_integrity.py @@ -91,8 +91,19 @@ def sha1_of(path: Path) -> str: return digest.hexdigest() +class GitFailed(RuntimeError): + """A git command failed: its empty output says nothing about the file.""" + + def git(root: Path, *args: str) -> subprocess.CompletedProcess: - return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False) + """Run git in root. A failure raises: an unreadable index read as an empty + status made a session's working change look unmodified, and --apply wrote + HEAD's bytes over it.""" + done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False) + if done.returncode: + message = done.stderr.decode(errors="replace").strip() + raise GitFailed(f"git {' '.join(args)} exited {done.returncode}: {message}") + return done def head_blob(root: Path, path: str) -> str: @@ -197,7 +208,11 @@ def main() -> int: from common import load_database root = Path.cwd() - report = check(root, load_database(args.db), args.apply) + try: + report = check(root, load_database(args.db), args.apply) + except GitFailed as exc: + print(f"ERROR: {exc}. Nothing was restored past this point.", file=sys.stderr) + return 2 if args.json: print(json.dumps(report.as_dict(), indent=2)) else: diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index b8e03f28b..8f1ca775f 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -32,7 +32,6 @@ from __future__ import annotations import argparse import contextlib -import hashlib import io import json import re @@ -51,6 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import export_native # noqa: E402 +import pin_installer # noqa: E402 import refresh_data_dirs # noqa: E402 import upstream # noqa: E402 from common import ( # noqa: E402 @@ -83,7 +83,6 @@ PYPI_URL = "https://pypi.org/pypi/{name}/json" NPM_URL = "https://registry.npmjs.org/{name}/latest" # The site's npm dependencies, pinned to exact versions and built by Deploy Site. SITE_PACKAGE = REPO_ROOT / "website" / "package.json" -INSTALLERS = ("install.sh", "install.ps1") SHOWN_ITEMS = 4 # items named in a diff summary before ", ..." SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..." @@ -509,7 +508,10 @@ def check_native( ) -> list[Finding]: exporters = discover_exporters() index = cache_dir / export_native.SOURCES_INDEX - recorded = export_native.load_sources(cache_dir) + try: + recorded = export_native.load_sources(cache_dir) + except export_native.SourcesIndexUnreadable as exc: + return [Finding("native", index.name, ERROR, detail=str(exc))] findings: list[Finding] = [] for name in list_registered_platforms(str(platforms_dir), include_archived=True): exporter_class = exporters.get(name) @@ -1462,18 +1464,16 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]: def _check_installer_pins() -> Finding: """The bootstraps pin install.py by SHA-256, so a stale pin refuses every install.""" - digest = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest() - stale = [] - for name in INSTALLERS: - text = (REPO_ROOT / name).read_text(encoding="utf-8", errors="replace") - if digest not in text: - stale.append(name) + stale = pin_installer.stale_pins(REPO_ROOT) return Finding( "ci", "install.py pin", STALE if stale else OK, - local=digest[:12], - detail=("pin outdated in " + ", ".join(stale)) if stale else "install.sh and install.ps1 match", + local=pin_installer.installer_sha256(REPO_ROOT)[:12], + detail=( + f"pin outdated in {', '.join(stale)}, {pin_installer.COMMAND} writes it" + if stale else "install.sh and install.ps1 match" + ), ) @@ -1757,15 +1757,18 @@ def render(findings: list[Finding]) -> str: # Columns are padded, never cut: the profile_sync row says in its # local and remote values what to re-read, and lost the end of it. local_width = min(max((len(f.local) for f in rows), default=0), 44) + remote_width = min(max((len(f.remote) for f in rows), default=0), 44) + # The arrow and the remote value, or as many blanks where a row has none. + remote_span = 4 + remote_width if remote_width else 0 for f in rows: cols = f" {f.status:7} {f.subject:{width}}" if f.remote: - cols += f" {f.local:{local_width}} -> {f.remote}" - elif f.local or (f.detail and local_width): - # An arrow pointing at nothing read as a lost value. An empty - # local value still takes its column, or the detail starts - # where the other rows show theirs. - cols += f" {f.local:{local_width}}" + cols += f" {f.local:{local_width}} -> {f.remote:{remote_width}}" + elif f.local or (f.detail and (local_width or remote_span)): + # An arrow pointing at nothing read as a lost value. Empty + # local and remote values still take their columns, or the + # detail starts where the other rows show theirs. + cols += f" {f.local:{local_width}}" + " " * remote_span if f.detail: cols += f" {f.detail}" lines.append(cols.rstrip()) diff --git a/scripts/common.py b/scripts/common.py index 50c90874d..f1890d55c 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -3587,6 +3587,7 @@ from artifacts import ( # noqa: E402,F401 copy_file_atomic, plain_file_mode, set_plain_mode, + ArtifactDirectoryMissing, ArtifactLockBusy, artifact_lock, hold_artifact_lock, @@ -3634,7 +3635,7 @@ def run_cli(main: Callable[[], int | None]) -> None: try: code = main() sys.stdout.flush() - except (InputMissing, ArtifactLockBusy) as exc: + except (InputMissing, ArtifactLockBusy, ArtifactDirectoryMissing) as exc: print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) except BrokenPipeError: diff --git a/scripts/export_native.py b/scripts/export_native.py index 42d529eba..d91e9e783 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -36,6 +36,7 @@ from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_tex import yaml from common import ( + InputMissing, build_zip_contents_index, platform_arg, list_registered_platforms, @@ -62,13 +63,27 @@ _USER_AGENT = "retrobios-exporter/1.0" _MAX_BYTES = 64 * 1024 * 1024 +class SourcesIndexUnreadable(InputMissing): + """The cache's URL index exists and does not read as JSON.""" + + def _load_sources(index: Path | None) -> dict[str, str]: + """The URL each cached file was fetched from. + + An absent index records nothing. An index that does not read raises: + read as empty it let every cached original pass for current, and the + next download rewrote it with that one entry. + """ if index is None or not index.is_file(): return {} try: return json.loads(index.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return {} + except (OSError, json.JSONDecodeError) as exc: + raise SourcesIndexUnreadable( + f"{index} does not read ({exc}): remove it, then" + " python scripts/export_native.py --all --refresh-cache" + " downloads every original again and records its URL" + ) from exc def source_key(destination: Path, index: Path | None) -> str: diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 5418e4889..0a9d8e6d8 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -941,6 +941,10 @@ the most useful contribution. hash differs from the one already held under its name goes to that folder's `.variants/`. +A ZIP that no list declares as a whole is read file by file, and the report +says which of them the collection already holds: upload the files +themselves, unzipped. + From a clone, the file goes to `bios/Manufacturer/Console/filename`, and `python scripts/validate_pr.py path/to/file` names that folder for a file still outside `bios/`, and `--place` copies it there under its declared name. Given a diff --git a/scripts/generate_truth.py b/scripts/generate_truth.py index ac5856667..472ae3ea7 100644 --- a/scripts/generate_truth.py +++ b/scripts/generate_truth.py @@ -14,13 +14,13 @@ import sys sys.path.insert(0, os.path.dirname(__file__)) from common import ( + build_target_cores_cache, platform_arg, list_registered_platforms, load_data_dir_registry, load_database, load_emulator_profiles, load_platform_config, - load_target_config, require_directory, require_platform, require_yaml, @@ -97,6 +97,19 @@ def main(argv: list[str] | None = None) -> None: require_platform(args.platform, args.platforms_dir) platforms = [args.platform] + # A named platform without that target is refused, as verify and + # generate_pack refuse it: skipped, it printed Done. and exited 0 with + # no model written. --all leaves such platforms out and says so. + target_cores_cache: dict[str, set[str] | None] = {} + if args.target: + try: + target_cores_cache, platforms = build_target_cores_cache( + platforms, args.target, args.platforms_dir, is_all=args.all + ) + except (FileNotFoundError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + # A target-narrowed model is a different artifact: written under the # platform's own name it overwrote the full one, and diff_truth then # compared a narrowed model against the whole scrape. Its directory is @@ -109,17 +122,7 @@ def main(argv: list[str] | None = None) -> None: output_dir, [name], args.target, args.platforms_dir ) os.makedirs(output_dir, exist_ok=True) - target_cores: set[str] | None = None - if args.target: - try: - target_cores = load_target_config( - name, - args.target, - args.platforms_dir, - ) - except FileNotFoundError: - print(f" {name}: no target config, skipped") - continue + target_cores = target_cores_cache.get(name) # Load platform config (with inheritance) and registry entry try: diff --git a/scripts/pin_installer.py b/scripts/pin_installer.py new file mode 100644 index 000000000..07e366cf0 --- /dev/null +++ b/scripts/pin_installer.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Write the SHA-256 of install.py into the bootstraps that pin it. + +install.sh and install.ps1 refuse an installer whose hash is not the one +they carry, so every change to install.py moves both pins in the same +commit. --check writes nothing and exits 1 naming each stale pin. + +Usage: + python scripts/pin_installer.py + python scripts/pin_installer.py --check +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +INSTALLER = "install.py" +# Each bootstrap and its one pin line, the value between the two groups. +PINS: dict[str, re.Pattern[str]] = { + "install.sh": re.compile(r'^(DEFAULT_INSTALL_SHA256=")[0-9a-f]{64}(")', re.M), + "install.ps1": re.compile(r'^(\s*\$defaultInstallSha256 = ")[0-9a-f]{64}(")', re.M), +} +COMMAND = "python scripts/pin_installer.py" + + +def installer_sha256(root: Path = ROOT) -> str: + return hashlib.sha256((root / INSTALLER).read_bytes()).hexdigest() + + +def stale_pins(root: Path = ROOT) -> list[str]: + """The bootstraps whose pin is not the SHA-256 of install.py.""" + digest = installer_sha256(root) + stale = [] + for name, pin in PINS.items(): + found = pin.search((root / name).read_bytes().decode("utf-8")) + if found is None: + raise ValueError(f"{name}: no installer pin line") + if found.group(0) != f"{found.group(1)}{digest}{found.group(2)}": + stale.append(name) + return stale + + +def write_pins(root: Path = ROOT) -> list[str]: + """Rewrite each stale pin. Returns the bootstraps written.""" + digest = installer_sha256(root) + written = [] + for name in stale_pins(root): + path = root / name + text = path.read_bytes().decode("utf-8") + new, count = PINS[name].subn(rf"\g<1>{digest}\g<2>", text) + if count != 1: + raise ValueError(f"{name}: {count} installer pin lines, expected 1") + path.write_bytes(new.encode("utf-8")) + written.append(name) + return written + + +def main() -> int: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0]) + parser.add_argument("--check", action="store_true", + help="write nothing, exit 1 if a pin is stale") + args = parser.parse_args() + if args.check: + stale = stale_pins() + for name in stale: + print(f"{name}: pin is not the SHA-256 of {INSTALLER}, {COMMAND} writes it", + file=sys.stderr) + return 1 if stale else 0 + for name in write_pins(): + print(f"{name}: pinned {installer_sha256()}") + return 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/scripts/refresh_data_dirs.py b/scripts/refresh_data_dirs.py index 7b74f7be1..31def57cc 100644 --- a/scripts/refresh_data_dirs.py +++ b/scripts/refresh_data_dirs.py @@ -71,6 +71,24 @@ def cache_lock( return _announced_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared, cache_dir) +def _wait_message(lock_path: Path, shared: bool, cache_dir: Path) -> str: + """Who to wait for, and, for a refresh, how not to. + + The bare "another run holds it" named neither the lock nor its holder, + and the build waited with no limit. + """ + message = ( + f"waiting for {cache_dir}: another run holds {lock_path} " + f"(lsof {lock_path} names it)" + ) + if not shared: + message += ( + ". generate_pack.py --no-refresh-data, or --offline there and in" + " pipeline.py, builds from the cache as it is without waiting" + ) + return message + + @contextlib.contextmanager def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[None]: """file_lock, saying so when it has to wait. @@ -90,7 +108,7 @@ def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[ try: fcntl.flock(handle, mode | fcntl.LOCK_NB) except BlockingIOError: - log.warning("waiting for %s: another run holds it", cache_dir) + log.warning(_wait_message(lock_path, shared, cache_dir)) fcntl.flock(handle, mode) try: yield diff --git a/scripts/refresh_stale.py b/scripts/refresh_stale.py index 4e79e70a5..bf65bb6ac 100755 --- a/scripts/refresh_stale.py +++ b/scripts/refresh_stale.py @@ -34,7 +34,10 @@ has ended, and its export is rewritten even when no row named it. Surfaced only (no refresh): data rows that are not a _data_dirs.yml key (the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs, -CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs +CI pins (PyPI, pinned actions, while scripts/pin_installer.py writes the +install.py SHA-256 pin), profile_sync, rom-folder +pins (moved by hand, then python -m scripts.scraper.romfolders --setup NAME +reads them again). Each needs a human read (new .info file, manual DAT download, PyPI version bump). Usage: @@ -65,7 +68,9 @@ PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml" DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml" AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs") -MANUAL_AREAS = ("coreinfo", "ci", "profiles") +# A rom-folder pin is moved by hand in platforms/romfolders/_sources/, +# then `python -m scripts.scraper.romfolders --setup NAME` reads it again. +MANUAL_AREAS = ("romfolders", "coreinfo", "ci", "profiles") JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10 @@ -397,6 +402,10 @@ def _command_for( "--force", ] + if area == "ci" and subject == "install.py pin": + # The pin is the hash of the committed install.py, nothing to judge. + return [sys.executable, "scripts/pin_installer.py"] + if area == "catalogs": if subject == "mame recipes": return [ @@ -563,7 +572,8 @@ def main() -> int: parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0]) parser.add_argument( "--only", - help="comma-separated areas to check: " + ",".join(AUTO_AREAS), + help="comma-separated areas to check: " + ",".join(AUTO_AREAS) + + " (refreshed), " + ",".join(MANUAL_AREAS) + " (listed for a human)", ) parser.add_argument( "--dry-run", @@ -579,7 +589,7 @@ def main() -> int: parser.add_argument( "--json", action="store_true", - help="machine-readable summary (jobs, surfaced, ignored)", + help="machine-readable summary (jobs, surfaced, ignored), the plan with --dry-run", ) parser.add_argument( "--findings", @@ -618,6 +628,22 @@ def main() -> int: registry = _load_platform_registry() jobs, surfaced, ignored = plan_jobs(findings, registry) + if args.dry_run and args.json: + print(json.dumps({ + "planned": [ + { + "area": job.area, + "subject": job.subject, + "command": job.command, + "after": sorted(job.after), + "then": [list(command) for command in job.then], + } + for job in sorted(jobs, key=lambda j: (j.area, j.subject)) + ], + "surfaced": surfaced, + "ignored_count": len(ignored), + }, indent=2)) + return 0 if args.dry_run: print(f"\n[plan] {len(jobs)} refreshable, {len(surfaced)} manual, " f"{len(ignored)} already clean/skipped") diff --git a/scripts/scraper/arcade_parent_importer.py b/scripts/scraper/arcade_parent_importer.py index ee1eca55a..a884b3188 100644 --- a/scripts/scraper/arcade_parent_importer.py +++ b/scripts/scraper/arcade_parent_importer.py @@ -49,7 +49,7 @@ from collections.abc import Callable, Iterator from dataclasses import dataclass, field from pathlib import Path -from ..artifacts import write_text_atomic +from ..artifacts import write_if_changed from ..common import load_emulator_profiles, run_cli from .romset_dat_importer import ( LISTXML_CHUNK, @@ -1114,14 +1114,6 @@ def dump(doc: dict) -> str: return json.dumps(doc, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" -def write_if_changed(path: Path, text: str) -> bool: - if path.is_file() and path.read_text(encoding="utf-8") == text: - return False - path.parent.mkdir(parents=True, exist_ok=True) - write_text_atomic(str(path), text) - return True - - def main() -> int: parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0]) which = parser.add_mutually_exclusive_group(required=True) @@ -1197,7 +1189,8 @@ def main() -> int: continue if not args.dry_run: target = args.output_dir / f"{profile}.json" - if write_if_changed(target, dump(doc)): + target.parent.mkdir(parents=True, exist_ok=True) + if write_if_changed(str(target), dump(doc)): print(f" wrote {target}") return 1 if failed else 0 diff --git a/scripts/scraper/emudeck_launchers.py b/scripts/scraper/emudeck_launchers.py index cbb7252ba..2723d62d8 100644 --- a/scripts/scraper/emudeck_launchers.py +++ b/scripts/scraper/emudeck_launchers.py @@ -46,14 +46,18 @@ STORAGE = "${storagePath}" FLATPAK_DATA = "${HOME}/.var/app/" # EmuScripts file names that differ from the emulator profile they set up. +# The platform and target scrapers both read them here. SCRIPT_EMULATORS = { "pcsx2qt": "pcsx2", "rpcs3legacy": "rpcs3", "cemuproton": "cemu", "rmg": "mupen64plus_next", } -# The main RetroArch config script installs no emulator. -NOT_EMULATORS = {"retroarch_maincfg"} +# The main RetroArch config script installs no emulator, and emudeck-we's +# emuDeckTemplate.ps1 is the skeleton new installers are copied from. +NOT_EMULATORS = {"retroarch_maincfg", "template"} +# EmuDeck's SteamOS installers are shell scripts, emudeck-we's PowerShell. +SCRIPT_SUFFIXES = (".sh", ".ps1") # What the emulator does with a directory written into each key. DIRECTORY_KINDS = { @@ -66,9 +70,10 @@ DIRECTORY_KINDS = { def emulator_of_script(file_name: str) -> str | None: """The emulator an EmuScripts file sets up, None for a helper script.""" - if not file_name.endswith(".sh"): + stem, dot, suffix = file_name.rpartition(".") + if not dot or f".{suffix}" not in SCRIPT_SUFFIXES: return None - key = re.sub(r"^emuDeck", "", file_name[: -len(".sh")], flags=re.IGNORECASE).lower() + key = re.sub(r"^emuDeck", "", stem, flags=re.IGNORECASE).lower() if not key or key in NOT_EMULATORS: return None return SCRIPT_EMULATORS.get(key, key) diff --git a/scripts/scraper/emudeck_scraper.py b/scripts/scraper/emudeck_scraper.py index 4f72adefb..d4fadacf0 100644 --- a/scripts/scraper/emudeck_scraper.py +++ b/scripts/scraper/emudeck_scraper.py @@ -469,9 +469,10 @@ class Scraper(BaseScraper): version = fetch_github_latest_version("dragoonDorise/EmuDeck") - cores = self._fetch_installed_emulators() + revision = launcher_revision(self.checkbios_url) + cores = self._fetch_installed_emulators(revision) standalone = [core for core in cores if core != "retroarch"] - tree = RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url)) + tree = RepoTree.github(EMUDECK_REPO, revision) libretro = set(installed_libretro_cores(tree.read(RETROARCH_SCRIPT))) return { @@ -501,16 +502,16 @@ class Scraper(BaseScraper): "launchers": extract_launchers(tree), } - def _fetch_installed_emulators(self) -> list[str]: - """Fetch the list of emulators installed by EmuDeck from EmuScripts. - - Returns core names normalized to match emulator profile keys. + def _fetch_installed_emulators(self, revision: str) -> list[str]: + """The emulators EmuDeck's EmuScripts install at the pinned revision, + named as their profiles are: the launcher layer and the SteamOS + target are read at that revision too. """ import json api_url = ( "https://api.github.com/repos/dragoonDorise/EmuDeck/" - "contents/functions/EmuScripts" + f"contents/functions/EmuScripts?ref={revision}" ) try: req = urllib.request.Request(api_url, headers=github_headers()) diff --git a/scripts/scraper/game_dat_importer.py b/scripts/scraper/game_dat_importer.py index e15a57fda..0968008bf 100644 --- a/scripts/scraper/game_dat_importer.py +++ b/scripts/scraper/game_dat_importer.py @@ -38,7 +38,7 @@ _SCRIPTS_DIR = str(Path(__file__).resolve().parents[1]) if _SCRIPTS_DIR not in sys.path: sys.path.insert(0, _SCRIPTS_DIR) -from artifacts import copy_stream_atomic, write_text_atomic # noqa: E402 +from artifacts import copy_stream_atomic, write_if_changed, write_text_atomic # noqa: E402 from common import parse_untrusted_xml # noqa: E402 from region import REGIONS, WORLD # noqa: E402 from region_audit import NOINTRO # noqa: E402 @@ -275,14 +275,6 @@ def dump_catalog(doc: dict) -> str: return f'{text},"games":[\n{games}\n]}}\n' -def write_if_changed(path: Path, text: str) -> bool: - if path.is_file() and path.read_text(encoding="utf-8") == text: - return False - path.parent.mkdir(parents=True, exist_ok=True) - write_text_atomic(str(path), text) - return True - - # Fetching --------------------------------------------------------------- @@ -503,7 +495,9 @@ def main() -> int: failures.append(f"{doc['source']}/{doc['system']}: {cov['games']} games, " f"floor {floor_of(doc)}") if not args.dry_run: - write_if_changed(args.output_dir / doc["source"] / f"{doc['system']}.json", text) + target = args.output_dir / doc["source"] / f"{doc['system']}.json" + target.parent.mkdir(parents=True, exist_ok=True) + write_if_changed(str(target), text) print(f"{len(docs)} catalogs, {raw_total} bytes raw, {gz_total} bytes gzip") for failure in failures: print(f"FAIL {failure}", file=sys.stderr) diff --git a/scripts/scraper/targets/emudeck_targets_scraper.py b/scripts/scraper/targets/emudeck_targets_scraper.py index 0e69d055e..fa8fa75b2 100644 --- a/scripts/scraper/targets/emudeck_targets_scraper.py +++ b/scripts/scraper/targets/emudeck_targets_scraper.py @@ -12,7 +12,6 @@ from __future__ import annotations import argparse import json -import re import sys import urllib.error import urllib.request @@ -22,6 +21,7 @@ import yaml from common import github_token, yaml_load +from ..emudeck_launchers import SCRIPT_SUFFIXES, emulator_of_script from . import BaseTargetScraper PLATFORM_NAME = "emudeck" @@ -33,24 +33,6 @@ WINDOWS_API = ( "https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts" ) -# Map EmuDeck script names to emulator profile keys -# Script naming: emuDeckDolphin.sh -> dolphin -# Some need explicit mapping when names differ -_NAME_OVERRIDES: dict[str, str] = { - "pcsx2qt": "pcsx2", - "rpcs3legacy": "rpcs3", - "cemuproton": "cemu", - "rmg": "mupen64plus_next", - "bigpemu": "bigpemu", - "eden": "eden", - "suyu": "suyu", - "ares": "ares", -} - -# Scripts that are not emulators (config helpers, etc.) -_SKIP = {"retroarch_maincfg", "retroarch"} - - def _fetch(url: str) -> str: headers = {"User-Agent": "retrobios-scraper/1.0"} token = github_token() @@ -69,26 +51,13 @@ def _list_emuscripts(api_url: str) -> list[str]: """List emulator script filenames from GitHub API.""" entries = json.loads(_fetch(api_url)) names = [ - e["name"] for e in entries if e.get("name", "").endswith((".sh", ".ps1")) + e["name"] for e in entries if e.get("name", "").endswith(SCRIPT_SUFFIXES) ] if not names: raise RuntimeError(f"no emulator scripts listed at {api_url}") return names -def _script_to_core(filename: str) -> str | None: - """Convert EmuScripts filename to core profile key.""" - # Strip extension and emuDeck prefix - name = re.sub(r"\.(sh|ps1)$", "", filename, flags=re.IGNORECASE) - name = re.sub(r"^emuDeck", "", name, flags=re.IGNORECASE) - if not name: - return None - key = name.lower() - if key in _SKIP: - return None - return _NAME_OVERRIDES.get(key, key) - - class Scraper(BaseTargetScraper): """Fetches emulator lists for EmuDeck SteamOS and Windows targets.""" @@ -114,14 +83,13 @@ class Scraper(BaseTargetScraper): seen: set[str] = set() has_retroarch = False for script in scripts: - core = _script_to_core(script) - if core and core not in seen: + core = emulator_of_script(script) + # RetroArch stands for its libretro cores, added below. + if core == "retroarch": + has_retroarch = True + elif core and core not in seen: seen.add(core) cores.append(core) - # Detect RetroArch presence (provides all libretro cores) - name = re.sub(r"\.(sh|ps1)$", "", script, flags=re.IGNORECASE) - if name.lower() in ("emudeckretroarch", "retroarch_maincfg"): - has_retroarch = True standalone_count = len(cores) # EmuDeck ships RetroArch = all its libretro cores are available diff --git a/scripts/slots.py b/scripts/slots.py index 8e7f826cd..d08a0b7a4 100644 --- a/scripts/slots.py +++ b/scripts/slots.py @@ -677,12 +677,21 @@ class Collision: resolved: list[str] # Each declaration with the file it resolved to, in the platform's order. declarations: list[tuple[dict, str | None]] = field(default_factory=list) + # The system of each declaration, in the same order. + systems: list[str] = field(default_factory=list) + # The file the pack ships at the destination, as the builder decides it. + shipped: str | None = None def _collision_json(collision: Collision) -> dict: return { "destination": collision.destination, "resolved": list(collision.resolved), + "shipped": collision.shipped, + "declarations": [ + {"system": system, "name": entry.get("name", ""), "file": local} + for system, (entry, local) in zip(collision.systems, collision.declarations) + ], } @@ -702,19 +711,27 @@ def find_collisions( CoCo disk ROM at one bios/disk.rom. """ fold = bool(config.get("case_insensitive_fs", False)) - by_dest: dict[str, list[dict]] = {} - for system in (config.get("systems") or {}).values(): + systems = config.get("systems") or {} + # The builder lets a hash-constrained declaration that resolves by + # content take the path, else the first it reaches walking the systems + # sorted: the same choice says which file ships. + preferred = set( + preferred_entries(systems, db, "bios", "", False, zip_contents, True).values() + ) + by_dest: dict[str, list[tuple[str, dict]]] = {} + for sys_id, system in systems.items(): for entry in system.get("files") or []: if not isinstance(entry, dict): continue dest = entry.get("destination") or entry.get("name") or "" if dest: - by_dest.setdefault(_normalize(dest, fold), []).append(entry) + by_dest.setdefault(_normalize(dest, fold), []).append((sys_id, entry)) collisions = [] - for key, entries in sorted(by_dest.items()): - if len(entries) < 2: + for key, claims in sorted(by_dest.items()): + if len(claims) < 2: continue + entries = [entry for _sys_id, entry in claims] resolved = [] declarations = [] for entry in entries: @@ -734,8 +751,22 @@ def find_collisions( # also names the first one's file. if any(_accepted_by_all(path, entries, db) for path in resolved): continue + reached = sorted( + (sys_id, local) + for (sys_id, _entry), (_e, local) in zip(claims, declarations) if local + ) + shipped = next( + (local for entry, local in declarations if local and id(entry) in preferred), + reached[0][1], + ) collisions.append( - Collision(destination=key, resolved=resolved, declarations=declarations) + Collision( + destination=key, + resolved=resolved, + declarations=declarations, + systems=[sys_id for sys_id, _entry in claims], + shipped=shipped, + ) ) return collisions @@ -863,11 +894,18 @@ def _same_file_family(paths: list[str]) -> bool: def format_collision(collision: Collision) -> str: - """One line naming the destination and the files fighting for it.""" - return ( - f"{collision.destination}: declared for " - + " and for ".join(collision.resolved) - ) + """One line naming the destination, which system declares which file, + and the one the pack ships there.""" + owners: dict[str, list[str]] = {} + for system, (_entry, local) in zip(collision.systems, collision.declarations): + if local and system not in owners.setdefault(local, []): + owners[local].append(system) + parts = [ + f"{path} ({', '.join(owners[path])}) " + + ("packed" if path == collision.shipped else "left out") + for path in collision.resolved + ] + return f"{collision.destination}: " + ", ".join(parts) def format_conflict(conflict: Conflict) -> str: diff --git a/scripts/split_pack.py b/scripts/split_pack.py index 9e830ff72..3df3fe866 100644 --- a/scripts/split_pack.py +++ b/scripts/split_pack.py @@ -338,6 +338,10 @@ def main() -> int: # The packs are rewritten in place by generate_pack under this lock: a # build running beside the split mixes its bytes into the pack being # read, or writes a pack that the split then deletes. + if not args.target.exists(): + print(f"Error: {args.target}: no such pack or directory, pass a pack or the " + "--output-dir generate_pack wrote (default dist/)", file=sys.stderr) + return 1 directory = args.target if args.target.is_dir() else args.target.parent try: with artifact_lock(str(directory)): diff --git a/scripts/validate_pr.py b/scripts/validate_pr.py index e3050f83c..a07140fb6 100644 --- a/scripts/validate_pr.py +++ b/scripts/validate_pr.py @@ -18,6 +18,7 @@ Outputs a structured report suitable for PR comments. from __future__ import annotations import argparse +import hashlib import json import os import re @@ -25,6 +26,7 @@ import shutil import subprocess import sys import urllib.parse +import zipfile from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) @@ -335,6 +337,53 @@ def load_profile_hashes( return known +def _archive_members( + filepath: str, db: dict | None, platform_hashes: dict, profile_hashes: dict +) -> list[str] | None: + """What each file of a ZIP is, None when filepath is not a ZIP. + + Read only for an archive no list declares as a whole: MAME and FBNeo + sets are ZIP BIOS files in their own right. + """ + if not zipfile.is_zipfile(filepath): + return None + held = (db or {}).get("files", {}) + described = [] + try: + with zipfile.ZipFile(filepath) as archive: + for info in archive.infolist(): + if info.is_dir(): + continue + if info.file_size > MAX_FILE_SIZE: + described.append(f"`{info.filename}` too large to read here") + continue + sha1, md5 = hashlib.sha1(), hashlib.md5() + with archive.open(info) as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + sha1.update(chunk) + md5.update(chunk) + sha1_hex, md5_hex = sha1.hexdigest(), md5.hexdigest() + crc = f"{info.CRC:08x}" + if sha1_hex in held: + state = f"already held at `{held[sha1_hex]['path']}`" + elif ( + sha1_hex in platform_hashes.get("sha1", set()) + or md5_hex in platform_hashes.get("md5", set()) + or (crc, info.file_size) in platform_hashes.get("crc32", set()) + or profile_hashes.get("sha1", {}).get(sha1_hex) + or profile_hashes.get("md5", {}).get(md5_hex) + or profile_hashes.get("crc32", {}).get((crc, info.file_size)) + ): + state = "declared and not held" + else: + state = "not recognised" + described.append(f"`{info.filename}` {state}") + except (zipfile.BadZipFile, RuntimeError, OSError) as exc: + # An encrypted member raises RuntimeError. + described.append(f"unreadable ({exc})") + return described + + def validate_file( filepath: str, db: dict | None, @@ -446,6 +495,8 @@ def validate_file( or profile_hashes.get("crc32", {}).get((result.crc32, None)) ) + # The files of a ZIP no list declares whole, when the upload is one. + members: list[str] | None = None if sha1_known: result.add_check(True, "SHA1 matches known platform requirement") elif md5_known: @@ -466,11 +517,26 @@ def validate_file( ) else: result.outcome = "unknown" - result.add_warning( - "Not recognised: no platform list or emulator profile declares this " - f"file. Say in the {'issue' if too_large else 'pull request'} which " - "emulator reads it, for which system, and where the dump comes from" + members = None if too_large else _archive_members( + filepath, db, platform_hashes, profile_hashes ) + if members is not None: + # A folder zipped to upload many files at once was hashed as one + # file and its contributor asked where that dump came from. + shown = "; ".join(members[:10]) + ( + f"; and {len(members) - 10} more" if len(members) > 10 else "" + ) + result.add_warning( + "A ZIP no platform list or emulator profile declares as a whole: " + "add the files it holds, unzipped, not the archive. Inside: " + + (shown or "nothing") + ) + else: + result.add_warning( + "Not recognised: no platform list or emulator profile declares this " + f"file. Say in the {'issue' if too_large else 'pull request'} which " + "emulator reads it, for which system, and where the dump comes from" + ) normalized = os.path.normpath(relative) # A platform's declaration first, then a profile's: most files the gaps @@ -503,7 +569,8 @@ def validate_file( result.add_check(True, f"Correct placement: bios/{parts[1]}/{parts[2]}/") else: result.add_warning("File should be in bios/Manufacturer/Console/ structure") - elif result.outcome == "held": + elif result.outcome == "held" or members is not None: + # Nothing to place: a copy, or an archive whose files go in unzipped. pass elif expected_folders: # The contributor asks before forking: say where it goes. @@ -847,7 +914,8 @@ def main() -> None: { "file": r.filepath, "passed": r.passed, - "outcome": r.outcome or "new", + # A file that failed, unread or refused, is not a new one. + "outcome": r.outcome or ("new" if r.passed else "failed"), "destination": r.destination, "sha1": r.sha1, "md5": r.md5, diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 5380f64c6..6bb3167c0 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -14,7 +14,15 @@ from pathlib import Path, PurePosixPath import yaml from common import source_yaml_paths, yaml_load -from jsonschema import Draft202012Validator, FormatChecker + +try: + from jsonschema import Draft202012Validator, FormatChecker +except ImportError: + # Tests import this module and skip on the ImportError, the command + # names the install pipeline.py names for the same step. + if __name__ != "__main__": + raise + sys.exit('Error: jsonschema required (pip install "jsonschema[format-nongpl]")') ROOT = Path(__file__).resolve().parent.parent SCHEMAS = ROOT / "schemas" diff --git a/scripts/verify.py b/scripts/verify.py index f47c7f011..efbfda178 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -2988,13 +2988,13 @@ def _main(held: contextlib.ExitStack) -> None: print(_overall_summary(group_results)) if args.destinations: - held = {} + destinations = {} for (result, names), (group, _rep) in zip(group_results, groups): for name, key in zip(names, group): - held[name] = { + destinations[name] = { "platform": key, "pack_destinations": result["pack_destinations"], } - Path(args.destinations).write_text(json.dumps(held, indent=1, sort_keys=True)) + Path(args.destinations).write_text(json.dumps(destinations, indent=1, sort_keys=True)) if args.json: for r in all_results.values(): diff --git a/tests/__init__.py b/tests/__init__.py index bbe42a6ec..1540b4305 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -13,6 +13,7 @@ temporary directory instead of naming the missing parent. from __future__ import annotations import os +import tempfile from pathlib import Path os.environ.setdefault("RETROBIOS_OFFLINE", "1") @@ -22,9 +23,16 @@ os.environ.setdefault( str(Path(__file__).resolve().parent.parent / "tmp" / "tests" / "probe.lock"), ) -(Path(__file__).resolve().parent.parent / "tmp" / "tests").mkdir( - parents=True, exist_ok=True -) +_SCRATCH = Path(__file__).resolve().parent.parent / "tmp" / "tests" +_SCRATCH.mkdir(parents=True, exist_ok=True) +# A TemporaryDirectory() without dir= landed in /tmp, a 4 GB tmpfs here: once +# another run filled it, modules unrelated to any change failed with ENOSPC. +# The default is the repository's tmp/, for this process and its children. +os.environ["TMPDIR"] = str(_SCRATCH) +tempfile.tempdir = str(_SCRATCH) +# A scratch directory is not part of this checkout: git run inside one must +# not walk up to the repository, as it could not from /tmp. +os.environ["GIT_CEILING_DIRECTORIES"] = str(_SCRATCH) # The installer's cache (the games index, what reading a card found) is the # suite's own: a run never reads nor fills the one of the person running it. os.environ.setdefault( diff --git a/tests/test_artifact_lock.py b/tests/test_artifact_lock.py index 612bd87d8..72804b1cb 100644 --- a/tests/test_artifact_lock.py +++ b/tests/test_artifact_lock.py @@ -61,6 +61,28 @@ class ArtifactLockTest(unittest.TestCase): with artifact_lock(self.dir, exclusive=False): pass + def test_a_reader_never_creates_the_directory(self): + """A mistyped --output-dir given to --verify-packs or release_record + was created holding a .lock, and the next run with the same typo no + longer said the path did not exist.""" + from common import ArtifactDirectoryMissing # noqa: PLC0415 + + missing = os.path.join(self.dir, "sub", "dsit") + with self.assertRaises(ArtifactDirectoryMissing): + with artifact_lock(missing, exclusive=False): + pass + self.assertFalse(os.path.exists(os.path.join(self.dir, "sub"))) + + def test_split_refuses_a_missing_target_before_locking(self): + missing = Path(self.dir) / "sub" / "dsit" + proc = subprocess.run( + [sys.executable, "scripts/split_pack.py", str(missing)], + cwd=REPO_ROOT, capture_output=True, text=True, check=False, timeout=60, + ) + self.assertEqual(proc.returncode, 1) + self.assertIn("no such pack or directory", proc.stderr) + self.assertFalse(missing.parent.exists()) + def test_lock_is_released_on_exit(self): with artifact_lock(self.dir): pass diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 3df8850fe..35e200ba0 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -206,11 +206,16 @@ class AModuleRunAloneHasItsScratchDirectory(unittest.TestCase): ) proc = subprocess.run( [sys.executable, "-I", "-c", - "import sys; sys.path.insert(0, '.'); import tests"], + "import sys; sys.path.insert(0, '.'); import tests, tempfile, os;" + " print(tempfile.gettempdir()); print(os.environ['TMPDIR'])"], cwd=clone, capture_output=True, text=True, timeout=60, check=False, ) self.assertEqual(proc.returncode, 0, proc.stderr) self.assertTrue((clone / "tmp" / "tests").is_dir()) + # 264 TemporaryDirectory() calls without dir= wrote to the /tmp + # tmpfs, and a full tmpfs failed modules no change touched. + scratch = str((clone / "tmp" / "tests").resolve()) + self.assertEqual(proc.stdout.split(), [scratch, scratch]) class ReleaseSigningRegressions(unittest.TestCase): @@ -2291,7 +2296,9 @@ class EveryManifestEntryIsFetchable(unittest.TestCase): else "gitignored" if repo_path in ignored else "untracked, never committed" ) - if bool(entry.get("release_asset")) != (state == "gitignored"): + if state == "untracked, never committed" or ( + bool(entry.get("release_asset")) != (state == "gitignored") + ): wrong.append((entry["dest"], repo_path, state)) self.assertEqual(wrong, [], f"{name} picks the wrong source: {wrong[:3]}") @@ -2305,6 +2312,36 @@ class EveryManifestEntryIsFetchable(unittest.TestCase): self.skipTest("not a git checkout") self.assertEqual(sorted(untracked)[:20], [], f"{len(untracked)} untracked files under bios/") + def test_every_database_path_is_committed_or_a_release_asset(self): + """A git mv under bios/ committed without regenerating anything left + database.json and the manifests agreeing on the old path: the tests + above compare them with each other, so all passed, and the installer + got a 404 (issue 78, with the database stale too).""" + database = ROOT / "database.json" + if not database.is_file(): + self.skipTest("database.json not generated") + files = json.loads(database.read_text(encoding="utf-8"))["files"] + paths = { + path + for entry in files.values() + for path in [entry.get("path"), *(entry.get("paths") or [])] + if path + } + tracked = _git_paths("ls-files", "-z") + if tracked is None: + self.skipTest("not a git checkout") + outside = sorted(paths - tracked) + ignored = _git_paths("check-ignore", "-z", "--no-index", "--stdin", stdin=outside) + if ignored is None: + self.skipTest("not a git checkout") + stale = [path for path in outside if path not in ignored] + self.assertEqual( + stale[:20], [], + f"{len(stale)} database paths git neither holds nor ignores: " + "python scripts/generate_db.py --force --bios-dir bios --output database.json, " + "then the manifests", + ) + class ReleaseAssetCriterion(unittest.TestCase): """What decides whether the installer fetches from the release is git, not size. diff --git a/tests/test_autorun.py b/tests/test_autorun.py index c8c04e0ad..ad15f7421 100644 --- a/tests/test_autorun.py +++ b/tests/test_autorun.py @@ -891,6 +891,16 @@ class BiosIntegrity(Scratch): self.assertEqual([(f.path, f.verdict) for f in report.findings], [(rel, bib.SKIPPED)]) self.assertEqual((self.repo / rel).read_bytes(), b"\x02" * 80) + def test_an_unreadable_index_stops_before_any_restore(self): + """git status failed on a corrupt index, its empty stdout read as + "unmodified", and --apply wrote HEAD's bytes over the session's.""" + rel = "bios/Sony/edited.bin" + (self.repo / rel).write_bytes(b"\x02" * 80) + (self.repo / ".git" / "index").write_bytes(b"garbage\n") + with self.assertRaises(bib.GitFailed): + bib.check(self.repo, self.db, apply=True) + self.assertEqual((self.repo / rel).read_bytes(), b"\x02" * 80) + def test_a_mismatch_the_second_read_does_not_confirm_is_a_misread(self): reads = {} expected = {e["path"]: e["sha1"] for e in self.db["files"].values()} diff --git a/tests/test_check_freshness.py b/tests/test_check_freshness.py index 72846872f..829062f66 100644 --- a/tests/test_check_freshness.py +++ b/tests/test_check_freshness.py @@ -35,6 +35,14 @@ class NativeCacheTests(unittest.TestCase): self.root.mkdir() self.wanted = {"list.txt": "https://example.invalid/tag-2/list.txt"} + def test_an_unreadable_index_is_an_error_not_unrecorded(self): + """Read as empty, it named "no URL recorded" for every original.""" + self.index.write_text("{", encoding="utf-8") + findings = cf.check_native(REPO_ROOT / "platforms", self.cache, self.cache) + self.assertEqual([(f.subject, f.status) for f in findings], + [(self.index.name, cf.ERROR)]) + self.assertIn("--refresh-cache", findings[0].detail) + def _cached(self, mtime: float) -> None: path = self.root / "list.txt" path.write_text("x", encoding="utf-8") @@ -734,6 +742,19 @@ class RowsSayAllTheyStandFor(unittest.TestCase): rows = [line for line in text.splitlines() if line.startswith(" STALE")] self.assertEqual(rows[0].index("no longer"), rows[1].index("is not what")) + def test_a_remote_value_keeps_the_detail_column(self): + """A row with a remote value pushed its detail past the column the + rows without one used, in [coreinfo] by some fifteen characters.""" + text = cf.render([ + cf.Finding("coreinfo", "anybor", cf.STALE, detail="no profile claims"), + cf.Finding("coreinfo", "amiberry", cf.STALE, local="e5d1f0f9a143", + remote="ae545d85c06f", detail=".info changed"), + cf.Finding("coreinfo", "x", cf.STALE, local="abc", remote="d", detail="third"), + ]) + rows = [line for line in text.splitlines() if line.startswith(" STALE")] + columns = {row.index(word) for row, word in zip(rows, ("no profile", ".info", "third"))} + self.assertEqual(len(columns), 1, rows) + def test_json_carries_every_unresolved_name(self): missing = {f"core{i:02d}": [] for i in range(50)} (row,) = cf._unresolved_findings("retropie", missing) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f84b4e624..391171064 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1005,6 +1005,22 @@ class CachedOriginal(unittest.TestCase): self._upstream("v1.txt", "moved") self.assertEqual(fetch(url, target, self.index), b"one") + def test_an_unreadable_index_serves_nothing(self): + """A truncated index read as empty: the cached batocera-41 file was + served for a 43.1 URL, and the next download erased every other + platform's recorded URL.""" + import export_native + + old = self._upstream("batocera-41.txt", "old") + target = self.cache / "plat" / "list.txt" + fetch(old, target, self.index) + self.index.write_text('{"plat/list.txt": "' + old + '",', encoding="utf-8") + new = self._upstream("batocera-43.txt", "new") + with self.assertRaises(export_native.SourcesIndexUnreadable) as caught: + fetch(new, target, self.index) + self.assertIn("--refresh-cache", str(caught.exception)) + self.assertTrue(self.index.read_text(encoding="utf-8").endswith(",")) + def test_refresh_reads_a_branch_url_again(self): """A branch URL keeps its name while the platform moves.""" url = self._upstream("branch.txt", "one") diff --git a/tests/test_game_catalogs.py b/tests/test_game_catalogs.py index 61ca33b63..306d78844 100644 --- a/tests/test_game_catalogs.py +++ b/tests/test_game_catalogs.py @@ -102,6 +102,17 @@ class FixtureImport(unittest.TestCase): twin = second / path.relative_to(first) self.assertEqual(path.read_bytes(), twin.read_bytes(), path.name) + def test_a_new_fetch_date_alone_rewrites_nothing(self) -> None: + """A refresh rewrote every catalog for the day it ran, and the diff + read before commit no longer showed which DAT had moved.""" + self.assertEqual(run_import("redump", self.out).returncode, 0) + path = self.out / "redump" / "sega-megacd.json" + doc = json.loads(path.read_text()) + doc["fetched"] = "2000-01-01" + path.write_text(importer.dump_catalog(doc)) + self.assertEqual(run_import("redump", self.out).returncode, 0) + self.assertEqual(json.loads(path.read_text())["fetched"], "2000-01-01") + def test_floor_refuses_a_short_dat(self) -> None: result = run_import("no-intro", self.out, "--floor", "--dry-run") self.assertEqual(result.returncode, 1) diff --git a/tests/test_install.py b/tests/test_install.py index f36389a1e..f128216e6 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -36,6 +36,9 @@ _spec.loader.exec_module(install) sys.path.insert(0, str(REPO_ROOT / "tests")) from netguard import module_guard # noqa: E402 +# What a stale pin's failure says: the fix is one command. +PIN_ADVICE = "install.py changed: python scripts/pin_installer.py rewrites both pins" + # No connection off the loopback and no card of the host reaches a test: # the module fails at its end otherwise. setUpModule, tearDownModule = module_guard("test_install") @@ -2201,7 +2204,7 @@ class TestAvailablePlatforms(unittest.TestCase): match = re.search(r'\$defaultInstallSha256 = "([0-9a-f]{64})"', content) self.assertIsNotNone(match, "install.ps1 must embed the installer SHA256") expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest() - self.assertEqual(match.group(1), expected) + self.assertEqual(match.group(1), expected, PIN_ADVICE) def test_powershell_wrapper_rejects_non_https_bootstrap(self): content = (REPO_ROOT / "install.ps1").read_text() @@ -2212,7 +2215,31 @@ class TestAvailablePlatforms(unittest.TestCase): match = re.search(r'DEFAULT_INSTALL_SHA256="([0-9a-f]{64})"', content) self.assertIsNotNone(match, "install.sh must embed the installer SHA256") expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest() - self.assertEqual(match.group(1), expected) + self.assertEqual(match.group(1), expected, PIN_ADVICE) + + def test_pin_script_writes_both_pins(self): + """Every edit to install.py meant a sha256sum pasted by hand into two + files with two syntaxes, and the failing test named no command.""" + sys.path.insert(0, str(REPO_ROOT / "scripts")) + import pin_installer # noqa: PLC0415 + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory: + root = Path(directory) + for name in ("install.py", "install.sh", "install.ps1"): + shutil.copy2(REPO_ROOT / name, root / name) + (root / "install.py").write_text("# changed\n", encoding="utf-8") + sh_mode = (root / "install.sh").stat().st_mode + self.assertEqual(pin_installer.stale_pins(root), ["install.sh", "install.ps1"]) + self.assertEqual(pin_installer.write_pins(root), ["install.sh", "install.ps1"]) + self.assertEqual(pin_installer.stale_pins(root), []) + digest = hashlib.sha256(b"# changed\n").hexdigest() + for name in ("install.sh", "install.ps1"): + before = (REPO_ROOT / name).read_text(encoding="utf-8").splitlines() + after = (root / name).read_text(encoding="utf-8").splitlines() + changed = [b for a, b in zip(before, after) if a != b] + self.assertEqual(len(changed), 1, name) + self.assertIn(digest, changed[0]) + self.assertEqual((root / "install.sh").stat().st_mode, sh_mode) def test_shell_one_liner_downloads_verifies_and_forwards_arguments(self): scratch = REPO_ROOT / "tmp" / "tests" diff --git a/tests/test_launchers_emudeck.py b/tests/test_launchers_emudeck.py index 314994972..33d7596fe 100644 --- a/tests/test_launchers_emudeck.py +++ b/tests/test_launchers_emudeck.py @@ -373,6 +373,8 @@ class Helpers(unittest.TestCase): self.assertEqual(el.emulator_of_script("emuDeckXemu.sh"), "xemu") self.assertIsNone(el.emulator_of_script("RetroArch_maincfg.sh")) self.assertIsNone(el.emulator_of_script("README.md")) + self.assertEqual(el.emulator_of_script("emuDeckPCSX2QT.ps1"), "pcsx2") + self.assertIsNone(el.emulator_of_script("emuDeckTemplate.ps1")) def test_the_revision_comes_from_the_pinned_url(self): url = ( diff --git a/tests/test_refresh_data_dirs.py b/tests/test_refresh_data_dirs.py index 42f21d122..dc38143ac 100644 --- a/tests/test_refresh_data_dirs.py +++ b/tests/test_refresh_data_dirs.py @@ -118,6 +118,35 @@ class RefreshConcurrency(unittest.TestCase): self.assertTrue(finished, "the refresh waited for a reader") self.assertEqual(result, [False]) + def test_a_waiting_refresh_names_the_lock_and_the_way_around(self): + """"waiting for data/dolphin-sys: another run holds it" named + neither the lock nor its holder, and the build waited unbounded.""" + import fcntl # noqa: PLC0415 + + self.cache.mkdir(parents=True) + lock_path = self.cache.with_name(".pak.lock") + entered = threading.Event() + + def refresh() -> None: + with rdd.cache_lock(self.cache): + entered.set() + + with open(lock_path, "a") as reader, self.assertLogs(rdd.log, "WARNING") as logs: + fcntl.flock(reader, fcntl.LOCK_SH) + worker = threading.Thread(target=refresh, daemon=True) + worker.start() + for _ in range(200): + if logs.records: + break + worker.join(timeout=0.05) + self.assertFalse(entered.is_set()) + fcntl.flock(reader, fcntl.LOCK_UN) + worker.join(timeout=20) + self.assertTrue(entered.is_set()) + message = logs.records[0].getMessage() + self.assertIn(f"lsof {lock_path}", message) + self.assertIn("--no-refresh-data", message) + def test_version_file_is_never_left_truncated(self): Path(self.versions).parent.mkdir(parents=True) Path(self.versions).write_text(json.dumps({"keep": {"sha": "1"}})) diff --git a/tests/test_refresh_stale.py b/tests/test_refresh_stale.py index 3310d068e..e92887a66 100644 --- a/tests/test_refresh_stale.py +++ b/tests/test_refresh_stale.py @@ -162,6 +162,11 @@ class Planning(unittest.TestCase): self.assertEqual(jobs, []) self.assertEqual(len(surfaced), 3) + def test_the_installer_pin_is_written_by_its_script(self): + jobs, surfaced, _ = self._plan(finding("ci", "install.py pin")) + self.assertEqual([job.command[1:] for job in jobs], [["scripts/pin_installer.py"]]) + self.assertEqual(surfaced, []) + class Running(unittest.TestCase): def test_a_child_starts_after_its_parent_ends(self): @@ -234,6 +239,38 @@ class SavedFindings(unittest.TestCase): findings = rs.load_findings(path, ("data", "catalogs")) self.assertEqual([f["subject"] for f in findings], ["dolphin-sys"]) + def test_dry_run_json_prints_the_plan_as_json(self): + """--dry-run --json printed the text plan, and a caller parsing it + as JSON failed on the first line.""" + import json # noqa: PLC0415 + import subprocess # noqa: PLC0415 + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory: + path = Path(directory) / "freshness.json" + path.write_text(json.dumps([ + finding("catalogs", "mame recipes"), finding("coreinfo", "anybor"), + ]), encoding="utf-8") + proc = subprocess.run( + [sys.executable, "scripts/refresh_stale.py", "--findings", str(path), + "--dry-run", "--json"], + cwd=REPO_ROOT, capture_output=True, text=True, check=False, timeout=120, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + plan = json.loads(proc.stdout) + self.assertEqual([(j["area"], j["subject"]) for j in plan["planned"]], + [("catalogs", "mame recipes")]) + self.assertEqual([f["subject"] for f in plan["surfaced"]], ["anybor"]) + + def test_every_area_check_freshness_reports_is_known(self): + """romfolders was added to check_freshness and not here: its rows + were dropped from a saved report without a word, and --only + romfolders was refused.""" + import check_freshness # noqa: PLC0415 + + self.assertEqual(set(check_freshness.AREAS), set(rs.AUTO_AREAS + rs.MANUAL_AREAS)) + jobs, surfaced, _ = rs.plan_jobs([finding("romfolders", "muos")], REGISTRY, {}) + self.assertEqual((jobs, [f["subject"] for f in surfaced]), ([], ["muos"])) + def test_a_report_that_is_not_a_list_is_refused(self): with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory: path = Path(directory) / "freshness.json" diff --git a/tests/test_schema_semantic_gate.py b/tests/test_schema_semantic_gate.py index 2710455ac..9ccc9692c 100644 --- a/tests/test_schema_semantic_gate.py +++ b/tests/test_schema_semantic_gate.py @@ -120,5 +120,31 @@ class DataRootIsKeyedBySystem(unittest.TestCase): self.assertTrue(self._errors({})) +class WithoutJsonschema(unittest.TestCase): + """The documented command stopped on a bare ImportError traceback that + named neither the install nor the format-nongpl extra.""" + + def test_the_command_names_the_install(self): + import os + import subprocess + + scratch = REPO_ROOT / "tmp" / "tests" + scratch.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=scratch) as directory: + stub = Path(directory) / "jsonschema" + stub.mkdir() + (stub / "__init__.py").write_text( + 'raise ImportError("No module named jsonschema")\n', encoding="utf-8" + ) + env = {**os.environ, "PYTHONPATH": directory} + proc = subprocess.run( + [sys.executable, "scripts/validate_schemas.py", "--source-only"], + capture_output=True, text=True, check=False, cwd=REPO_ROOT, env=env, + ) + self.assertEqual(proc.returncode, 1) + self.assertNotIn("Traceback", proc.stderr) + self.assertIn('pip install "jsonschema[format-nongpl]"', proc.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_scraper_contract.py b/tests/test_scraper_contract.py index bfe7f1553..cf9e04c20 100644 --- a/tests/test_scraper_contract.py +++ b/tests/test_scraper_contract.py @@ -724,7 +724,7 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase): with mock.patch("urllib.request.urlopen", self._refuse), self.assertRaises( RuntimeError ): - emudeck_scraper.Scraper()._fetch_installed_emulators() + emudeck_scraper.Scraper(checkbios_url="pinned")._fetch_installed_emulators("0" * 40) def test_libretro_core_metadata_raises(self): """libretro dropped every system name and manufacturer when the diff --git a/tests/test_slots.py b/tests/test_slots.py index 44afbe8e0..fb0e33e6c 100644 --- a/tests/test_slots.py +++ b/tests/test_slots.py @@ -509,6 +509,23 @@ class TestSelfContradictingDestinations(unittest.TestCase): self.assertEqual(len(collisions), 1) self.assertEqual(len(collisions[0].resolved), 2) + def test_the_line_names_each_system_and_the_file_packed(self): + """The line named two files and neither the system that declares + each nor the one the pack ships: the reader went to the YAML.""" + config = {"systems": { + "zeta": {"files": [{"name": "IPL.bin", "destination": "disk.rom", + "md5": "m" * 32}]}, + "alpha": {"files": [{"name": "IPL.bin", "destination": "disk.rom", + "md5": "n" * 32}]}, + }} + collision = slots.find_collisions(config, REGIONS_DB)[0] + line = slots.format_collision(collision) + left, packed = collision.resolved + # The builder walks the systems sorted: alpha's declaration ships. + self.assertIn(f"{packed} (alpha) packed", line) + self.assertIn(f"{left} (zeta) left out", line) + self.assertEqual(collision.shipped, packed) + def test_one_archive_named_by_its_inner_roms_is_not_a_collision(self): # The documented zipped_file pattern: several md5s, one archive. config = self._config( diff --git a/tests/test_target_scrapers.py b/tests/test_target_scrapers.py index c2c6626fb..6b05e356c 100644 --- a/tests/test_target_scrapers.py +++ b/tests/test_target_scrapers.py @@ -541,6 +541,59 @@ class TargetsReadThePlatformRevision(unittest.TestCase): self.assertTrue(steamos) self.assertTrue(all(u.endswith(f"?ref={pin}") for u in steamos), steamos) + def test_emudeck_platform_lists_its_installers_at_the_pin(self): + """cores: was read on master while launchers: and the SteamOS target + were read at the pin: the YAML described two revisions.""" + from scripts.scraper import emudeck_scraper + + pin = "d" * 40 + seen: list[str] = [] + + class _Body: + def __init__(self, url): + seen.append(url) + + def __enter__(self): + return self + + def __exit__(self, *_exc): + return False + + def read(self): + return json.dumps([{"name": "emuDeckRMG.sh"}]).encode() + + with mock.patch("urllib.request.urlopen", lambda req, **_kw: _Body(req.full_url)): + cores = emudeck_scraper.Scraper(checkbios_url="pinned")._fetch_installed_emulators(pin) + self.assertEqual(cores, ["mupen64plus_next"]) + self.assertEqual(len(seen), 1) + self.assertTrue(seen[0].endswith(f"?ref={pin}"), seen) + + +class EmuDeckScriptNamesHaveOneRule(unittest.TestCase): + """The target scraper kept its own copy of the EmuScripts-to-profile + table: it published emudeck-we's template skeleton as a core, and a + rename added to one copy only made --target filter that core out.""" + + def test_the_target_scraper_names_scripts_as_the_platform_does(self): + from scripts.scraper.emudeck_launchers import emulator_of_script + + listing = [ + "emuDeckRMG.sh", + "emuDeckPCSX2QT.ps1", + "emuDeckTemplate.ps1", + "emuDeckRetroArch.ps1", + "RetroArch_maincfg.sh", + "emuDeckXemu.ps1", + ] + scraper = emudeck_targets_scraper.Scraper(revision="0" * 40) + with mock.patch.object( + emudeck_targets_scraper, "_list_emuscripts", return_value=listing + ), mock.patch.object(scraper, "_load_retroarch_cores", return_value=["mgba"]): + cores = scraper._fetch_cores_for_target("unused", "Windows") + standalone = {emulator_of_script(name) for name in listing} - {None, "retroarch"} + self.assertEqual(standalone, {"mupen64plus_next", "pcsx2", "xemu"}) + self.assertEqual(cores, sorted(standalone | {"mgba"})) + class HashScrapersRefuseAnUnknownVersion(unittest.TestCase): """mame_hash_scraper wrote core_version "unknown" after a failed API call diff --git a/tests/test_truth_diff.py b/tests/test_truth_diff.py index fa48edf6e..949be7c7d 100644 --- a/tests/test_truth_diff.py +++ b/tests/test_truth_diff.py @@ -60,6 +60,22 @@ class ATargetedModelIsItsOwnArtifact(unittest.TestCase): ) + def test_a_named_platform_without_the_target_is_refused(self): + """bizhawk --target switch printed "skipped", then Done., exit 0, + and wrote no model: the diff after it failed instead.""" + repo = pathlib.Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory(dir=str(repo / "tmp")) as directory: + proc = subprocess.run( + [sys.executable, "scripts/generate_truth.py", "--platform", "bizhawk", + "--target", "switch", "--output-dir", directory], + capture_output=True, check=False, text=True, cwd=str(repo), timeout=400, + ) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertIn("bizhawk has no hardware targets", proc.stderr) + self.assertNotIn("Done.", proc.stdout) + self.assertEqual(list(pathlib.Path(directory).iterdir()), []) + + class ADiffThatComparedNothingFails(unittest.TestCase): """diff_truth --all skipped every platform without a model and exited 0.""" diff --git a/tests/test_validate_pr.py b/tests/test_validate_pr.py index 642420795..68e4f5d3b 100644 --- a/tests/test_validate_pr.py +++ b/tests/test_validate_pr.py @@ -269,6 +269,29 @@ class HashRecognition(_Fixture): self.assertIn("no platform list or emulator profile declares this file", markdown) self.assertIn("which emulator reads it", markdown) + def test_a_zip_of_bios_files_is_read_member_by_member(self): + """A card's BIOS folder zipped to upload it was hashed as one file, + and its contributor was asked where that dump came from.""" + import hashlib # noqa: PLC0415 + import zipfile # noqa: PLC0415 + + held, declared = b"HELD BIOS", b"DECLARED BIOS" + with zipfile.ZipFile(self.tmp / "mybios.zip", "w") as archive: + archive.writestr("scph5501.bin", held) + archive.writestr("sub/declared.bin", declared) + archive.writestr("other.bin", b"noise") + db = {"files": {hashlib.sha1(held).hexdigest(): {"path": "bios/Sony/PS/scph5501.bin"}}} + known = {**EMPTY_HASHES, "md5": {hashlib.md5(declared).hexdigest()}} + result = validate_pr.validate_file("mybios.zip", db, known) + markdown = result.to_markdown() + self.assertEqual(result.outcome, "unknown") + self.assertIn("unzipped, not the archive", markdown) + self.assertIn("`scph5501.bin` already held at `bios/Sony/PS/scph5501.bin`", markdown) + self.assertIn("`sub/declared.bin` declared and not held", markdown) + self.assertIn("`other.bin` not recognised", markdown) + self.assertNotIn("where the dump comes from", markdown) + self.assertNotIn("Place it in", markdown) + def test_a_duplicate_of_a_known_file_is_reported(self): """A copy of a held file passed, and was sent to the folder that already holds it.""" @@ -522,6 +545,15 @@ class FlagsAreAppliedOrRefused(unittest.TestCase): self.assertEqual(result.returncode, 2) self.assertIn("not allowed with", result.stderr) + def test_a_failed_file_is_not_reported_new(self): + """A path that does not exist came out as "outcome": "new", and a + reader listing new contributions announced a BIOS nobody sent.""" + import json # noqa: PLC0415 + + result = self._run("--json", "bios/nope.bin") + report = json.loads(result.stdout) + self.assertEqual([(r["passed"], r["outcome"]) for r in report], [(False, "failed")]) + class AFileOutsideBiosIsToldWhereToGo(unittest.TestCase): diff --git a/tests/test_verify_listing.py b/tests/test_verify_listing.py index c825b52a5..11d05d6ce 100644 --- a/tests/test_verify_listing.py +++ b/tests/test_verify_listing.py @@ -173,6 +173,23 @@ class TheDestinationsFlagNeedsAPack(unittest.TestCase): self.assertEqual(result.returncode, 2, result.stdout[-300:]) self.assertIn("--destinations", result.stderr) + def test_the_lock_stack_is_not_rebound(self): + """The --destinations branch reused `held`, the ExitStack that holds + the data directory locks, for its dict: a later enter_context in + that mode raised AttributeError.""" + import ast + + tree = ast.parse((REPO_ROOT / "scripts" / "verify.py").read_text(encoding="utf-8")) + body = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_main") + stack = body.args.args[0].arg + rebound = [ + node.lineno for node in ast.walk(body) + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)) + for target in (node.targets if isinstance(node, ast.Assign) else [node.target]) + if isinstance(target, ast.Name) and target.id == stack + ] + self.assertEqual(rebound, []) + class EveryGapCountedIsNamed(unittest.TestCase): """verify --platform batocera --verbose counted 3 optional core files and diff --git a/wiki/installer.md b/wiki/installer.md index 73719bf16..d5a930694 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1702,7 +1702,9 @@ a real 3.8 interpreter, which CI installs beside 3.12 so the check cannot skip. The build scripts need 3.9 and are not bound by this. Every change to `install.py` recomputes the SHA-256 pinned in `install.sh` and -`install.ps1` in the same commit. The tests compare the pins with the file. +`install.ps1` in the same commit: `python scripts/pin_installer.py` writes both +(`--check` writes nothing and exits 1 on a stale pin). The tests compare the +pins with the file and name that command when they differ. The tables of options, events, exit codes, keys and card markers, the line menu and the run samples on this page sit between `generated` markers and are diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 3ef512d15..1a9f64b74 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -9,9 +9,11 @@ The tests hold what users and platforms rely on: a pack holds what its platform checks, the site states what the code does, the installer writes only where it should. -No test needs network access. Most build synthetic fixtures under the -repository's `tmp/`, and modules that read real artifacts skip when those are -absent. The test runner prints the count. +No test needs network access. Most build synthetic fixtures, and every +temporary directory lands under the repository's `tmp/tests/`: +`tests/__init__.py` points `tempfile` and `TMPDIR` there, never at the `/tmp` +tmpfs. Modules that read real artifacts skip when those are absent. The test +runner prints the count. ## Running tests diff --git a/wiki/tools.md b/wiki/tools.md index dab60e5b0..65296bcee 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -760,7 +760,10 @@ committed. Rows that need a decision are listed and never run: a buildbot core with no profile, a new `.info` file, the Redump, No-Intro and TOSEC packs, the CI -pins, and `profile_sync`. A failed scraper leaves its cached original +pins, `profile_sync`, and a rom-folder pin (moved by hand in +`platforms/romfolders/_sources/`, then `python -m scripts.scraper.romfolders +--setup NAME` reads it again). `--dry-run --json` prints the plan as JSON. +A failed scraper leaves its cached original untouched. Each job writes its output to `tmp/refresh_stale/`, and a GitHub token is taken from `GITHUB_TOKEN` or from `gh auth token`. @@ -1343,6 +1346,7 @@ before a release (wiki/release-process.md). Its lines stay in | `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs | | `generate_event_schema.py` | Write `schemas/install-events.schema.json` from `EVENT_KEYS` in `install.py` (`--check` exits 1 when the file is stale, which `validate_schemas.py` also reports) | | `generate_installer_doc.py` | Rewrite the generated blocks of `wiki/installer.md` from `install.py`: the tables (options, events, exit codes, keys per route, card markers), the line menu, and the run samples `install.py` prints in a sandbox against the committed `install/*.json`, so a new list rewrites the page (`--check` exits 1 when a block is stale, `--page FILE` writes another copy of the page). Pipeline step 7 | +| `pin_installer.py` | Write the SHA-256 of `install.py` into the pins of `install.sh` and `install.ps1`, which refuse any other installer (`--check` writes nothing and exits 1 when a pin is stale). `check_freshness.py` reports a stale pin and `refresh_stale.py` runs this | | `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline). A download is accepted only when every cryptographic hash the entry declares (sha256, sha1, md5) matches. crc32 and size find a file the database already holds, never accept a download. An entry with no cryptographic hash is reported and not searched | | `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones) | | `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) |