From 8022ef4cb31c9140d195863c7e6299773f73e04b Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:12:43 +0100
Subject: [PATCH 01/22] fix: read the given database on every page
The page-finishing pass reloaded the repository's database.json whatever
--db said, so a card held only by the given collection raised KeyError.
siteinputs holds the inputs generate_site was given.
---
scripts/generate_site.py | 25 +++++++++++-----
scripts/siteinputs.py | 32 +++++++++++++++++++++
tests/test_site_inputs.py | 60 +++++++++++++++++++++++++++++++++++++++
3 files changed, 110 insertions(+), 7 deletions(-)
create mode 100644 scripts/siteinputs.py
create mode 100644 tests/test_site_inputs.py
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index 32f926da0..9b83ff328 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -89,6 +89,7 @@ from generate_readme import (
from provenance_report import build_report, distinct_missing
import sitemarkup as markup
import siteentries
+import siteinputs
import retired_files
import sitestarlight
from siteexports import generate_exports_page, load_export_reports
@@ -3174,7 +3175,9 @@ def _front_matter(markdown: str) -> dict:
return data if isinstance(data, dict) else {}
-_SHA1_BY_PREFIX: dict[str, str] = {}
+@functools.lru_cache(maxsize=None)
+def _sha1_by_prefix(db_path: str) -> dict[str, str]:
+ return {sha1[:12]: sha1 for sha1 in load_database(db_path)["files"]}
def _cards_elsewhere(markdown: str) -> str:
@@ -3185,11 +3188,9 @@ def _cards_elsewhere(markdown: str) -> str:
away = {ident for ident in links if f'id="{ident}"' not in markdown}
if not away:
return markdown
- if not _SHA1_BY_PREFIX:
- db = load_database(str(_REPO_ROOT / "database.json"))
- _SHA1_BY_PREFIX.update({sha1[:12]: sha1 for sha1 in db["files"]})
+ by_prefix = _sha1_by_prefix(str(siteinputs.current().db))
for ident in away:
- full = _SHA1_BY_PREFIX[ident.removeprefix("sha1-")]
+ full = by_prefix[ident.removeprefix("sha1-")]
markdown = markdown.replace(f"](#{ident})", f"]({markup.site_href(f'files/?sha1={full}')})")
return markdown
@@ -3500,10 +3501,14 @@ def _list_pages(
f"[part {i}]({other}.md)" for i, other in enumerate(stems, 1))
-@functools.lru_cache(maxsize=1)
def _collection_sha1_by_path() -> dict[str, str]:
"""The sha1 of each file the collection holds, by its path."""
- with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle:
+ return _sha1_by_path_in(str(siteinputs.current().db))
+
+
+@functools.lru_cache(maxsize=None)
+def _sha1_by_path_in(db_path: str) -> dict[str, str]:
+ with open(db_path, encoding="utf-8") as handle:
files = json.load(handle).get("files", {})
return {str(entry.get("path", "")): sha1 for sha1, entry in files.items()}
@@ -12260,6 +12265,12 @@ def main() -> None:
def _generate(args: argparse.Namespace) -> None:
+ # Every page reads these, never the repository's own files.
+ siteinputs.use(siteinputs.SiteInputs(
+ db=Path(args.db).resolve(),
+ emulators=Path(args.emulators_dir).resolve(),
+ platforms=Path(args.platforms_dir).resolve(),
+ ))
db = load_database(args.db)
docs = Path(args.docs_dir)
# Files the site serves as they are (data, downloads, images), in the
diff --git a/scripts/siteinputs.py b/scripts/siteinputs.py
new file mode 100644
index 000000000..7aa3326ff
--- /dev/null
+++ b/scripts/siteinputs.py
@@ -0,0 +1,32 @@
+"""The collection, profiles and platform files the site describes.
+
+generate_site.py takes them as --db, --emulators-dir and --platforms-dir and
+installs them here before writing a page. Every page module reads its inputs
+from current(), so a run on another collection describes that collection on
+every page. A module used on its own reads the repository's files.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+@dataclass(frozen=True)
+class SiteInputs:
+ db: Path = ROOT / "database.json"
+ emulators: Path = ROOT / "emulators"
+ platforms: Path = ROOT / "platforms"
+
+
+_current = [SiteInputs()]
+
+
+def current() -> SiteInputs:
+ return _current[0]
+
+
+def use(inputs: SiteInputs) -> None:
+ _current[0] = inputs
diff --git a/tests/test_site_inputs.py b/tests/test_site_inputs.py
new file mode 100644
index 000000000..ab75d4be7
--- /dev/null
+++ b/tests/test_site_inputs.py
@@ -0,0 +1,60 @@
+"""generate_site --db reaches every page.
+
+The run loaded the given database, but the page-finishing pass reloaded the
+repository's database.json: a card held only by the given collection stopped
+the whole run on a KeyError.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import shutil
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_site # noqa: E402
+import siteinputs # noqa: E402
+
+SHA1 = "0123456789ab" + "c" * 28
+
+
+class _Inputs(unittest.TestCase):
+ def use(self, **paths: Path) -> None:
+ siteinputs.use(siteinputs.SiteInputs(**paths))
+ self.addCleanup(siteinputs.use, siteinputs.SiteInputs())
+
+
+class TheGivenDatabase(_Inputs):
+ def setUp(self) -> None:
+ tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" if (REPO_ROOT / "tmp").is_dir() else None))
+ self.addCleanup(shutil.rmtree, tmp)
+ self.db = tmp / "database.json"
+ self.db.write_text(json.dumps({"files": {SHA1: {"path": "bios/X/only-here.bin"}}}))
+
+ def test_a_card_held_elsewhere_resolves_in_the_given_collection(self) -> None:
+ self.use(db=self.db)
+ page = generate_site._cards_elsewhere("[x](#sha1-0123456789ab) ")
+ self.assertIn(f"files/?sha1={SHA1}", page)
+
+ def test_paths_are_read_from_the_given_collection(self) -> None:
+ self.use(db=self.db)
+ self.assertEqual(generate_site._collection_sha1_by_path(), {"bios/X/only-here.bin": SHA1})
+
+
+class GenerateInstallsTheInputs(unittest.TestCase):
+ def test_generate_installs_the_inputs_first(self) -> None:
+ source = ast.get_source_segment(
+ (REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8"),
+ next(n for n in ast.parse((REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")).body
+ if isinstance(n, ast.FunctionDef) and n.name == "_generate"))
+ self.assertLess(source.index("siteinputs.use("), source.index("load_database("))
+
+
+if __name__ == "__main__":
+ unittest.main()
From 9b871db83aad31b060758e4825a685a723801927 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:12:50 +0100
Subject: [PATCH 02/22] fix: read the given profiles and platform files
Five generate_site helpers read ./emulators or ./platforms, and
has_release_pack read platforms relative to the working directory: run
from scripts/, a released setup's get page showed the No ZIP line.
siterender's fallbacks and sitediagrams read the repository too. All of
them now read siteinputs. platforms/_guides.yml stays the repository's:
siteguides loads it at import, before any flag is parsed.
---
scripts/generate_site.py | 76 ++++++++++++++++++++++-----------------
scripts/sitediagrams.py | 37 +++++++++++++------
scripts/siterender.py | 13 ++++---
tests/test_site_inputs.py | 68 +++++++++++++++++++++++++++++++++--
wiki/testing-guide.md | 2 +-
5 files changed, 145 insertions(+), 51 deletions(-)
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index 9b83ff328..62f400543 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -663,7 +663,7 @@ def release_line(platform_name: str, record: dict) -> list[str]:
f"[SHA256SUMS.txt]({base}/SHA256SUMS.txt) and its "
f"[signature]({base}/SHA256SUMS.txt.sig).",
]
- config = load_platform_config(platform_name, "platforms")
+ config = load_platform_config(platform_name, str(siteinputs.current().platforms))
page_version = str(config.get("version") or "")
pack_version = _pack_version(pack_name)
if page_version and pack_version and page_version.lstrip("v") != pack_version.lstrip("v"):
@@ -688,7 +688,7 @@ def setup_version(platform_name: str, record: dict | None = None) -> str:
platform YAML was read at. Empty when neither says, or when the
platform has no YAML to read a version from."""
try:
- config = load_platform_config(platform_name, "platforms")
+ config = load_platform_config(platform_name, str(siteinputs.current().platforms))
except FileNotFoundError:
return ""
pack_name = release_record.pack_name_for(platform_name, record or {})
@@ -715,8 +715,7 @@ NO_ZIP_FAQ = "wiki/faq.md#why-is-there-no-zip-for-my-setup"
def has_release_pack(platform_name: str) -> bool:
"""Whether a release publishes a ZIP for the platform (`release_pack`)."""
- entry = load_platform_registry("platforms").get(platform_name) or {}
- return entry.get("release_pack") is True
+ return platform_name in _release_pack_platforms()
def no_zip_line(display: str) -> str:
@@ -735,7 +734,7 @@ def get_page_title(platform_name: str, display: str) -> str:
the system's own download. A setup that is also an emulator (BizHawk)
names it last: "BizHawk BIOS files" is the title of its emulator page."""
kind = "BIOS pack" if has_release_pack(platform_name) else "BIOS files"
- if (_REPO_ROOT / "emulators" / f"{platform_name}.yml").is_file():
+ if (siteinputs.current().emulators / f"{platform_name}.yml").is_file():
return f"{kind} for {display}"
return f"{display} {kind}"
@@ -780,7 +779,7 @@ def _pack_buttons(platform_name: str, label: str = "Pack", prefix: str = "../",
frontend user asks. A setup with no release pack gets the line that
says so, and why."""
if not has_release_pack(platform_name):
- display = display or load_platform_config(platform_name, "platforms").get(
+ display = display or load_platform_config(platform_name, str(siteinputs.current().platforms)).get(
"platform", platform_name)
return f"{no_zip_line(display)} [Why no ZIP?]({prefix}{NO_ZIP_FAQ})"
record = release_record.load_advertised()
@@ -805,7 +804,7 @@ def generate_get_page(platform_name: str) -> str:
the files of the pack the current release holds, read from release.json,
or the command when no release publishes one. The release process
never touches it: committing release.json rebuilds it."""
- config = load_platform_config(platform_name, "platforms")
+ config = load_platform_config(platform_name, str(siteinputs.current().platforms))
display = config.get("platform", platform_name)
record = release_record.load_advertised()
pack = release_record.pack_for(platform_name, record)
@@ -824,7 +823,7 @@ def generate_get_page(platform_name: str) -> str:
if released and pack and pack.get("assets"):
parts = len(pack["assets"])
release = release_line(platform_name, record)
- extract_to = (load_platform_registry("platforms").get(platform_name) or {}).get("extract_to")
+ extract_to = (load_platform_registry(str(siteinputs.current().platforms)).get(platform_name) or {}).get("extract_to")
lines += [
"This address stays the same. It always gives the latest ZIP.",
"",
@@ -873,7 +872,7 @@ def generate_get_page(platform_name: str) -> str:
def generate_get_index() -> str:
"""get/: the permanent page of every setup, grouped as the home tiles
are. A link trimmed to /get/ answered 404."""
- registry = load_platform_registry("platforms")
+ registry = load_platform_registry(str(siteinputs.current().platforms))
lines = [
"---", "search: exclude",
'description: "The BIOS files of every setup, one page each. Each address stays the same."',
@@ -887,7 +886,7 @@ def generate_get_index() -> str:
continue
lines += [f"## {label}", ""]
for name in names:
- display = load_platform_config(name, "platforms").get("platform", name)
+ display = load_platform_config(name, str(siteinputs.current().platforms)).get("platform", name)
lines.append(f"- [{get_page_title(name, display)}]({name}.md)")
lines.append("")
return "\n".join(lines)
@@ -1068,10 +1067,13 @@ def _split_contract(config: dict) -> tuple[dict, dict]:
return contract, registry
-@functools.lru_cache(maxsize=1)
def _release_pack_platforms() -> frozenset[str]:
- return frozenset(list_release_pack_platforms(
- str(_REPO_ROOT / "platforms"), include_archived=True))
+ return _release_packs_in(str(siteinputs.current().platforms))
+
+
+@functools.lru_cache(maxsize=None)
+def _release_packs_in(platforms_dir: str) -> frozenset[str]:
+ return frozenset(list_release_pack_platforms(platforms_dir, include_archived=True))
# Per-entry fields a folder check needs beyond the hashes: an md5 is a ZIP
@@ -2105,7 +2107,7 @@ def _short_sha256(item: dict, catalog: str) -> str:
def _platforms_without_a_list(platforms_dir: Path | None = None) -> list[str]:
"""The names of the platforms whose registry says they keep no list."""
- platforms_dir = platforms_dir or _REPO_ROOT / "platforms"
+ platforms_dir = platforms_dir or siteinputs.current().platforms
names = []
for key, entry in sorted(load_platform_registry(str(platforms_dir)).items()):
if entry.get("native_list") != "none":
@@ -3538,7 +3540,7 @@ def _system_named_link(sys_id: str, prefix: str = "") -> str:
def _emulator_named_link(key: str, prefix: str = "") -> str:
"""An emulator by the name its page carries, linked to that page. A
profile with no page (an alias, a test) is named only."""
- profile = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False).get(key, {})
+ profile = _profiles_on_disk().get(key, {})
label = _escape_tags(str(profile.get("emulator") or key))
if profile.get("type") in ("alias", "test"):
return label
@@ -3928,10 +3930,9 @@ def _platform_data_links(name: str) -> str:
return ", ".join(links)
-@functools.lru_cache(maxsize=1)
def _site_profiles() -> dict[str, dict]:
"""Every emulator profile, read once for the citations of a page."""
- return load_emulator_profiles(str(_REPO_ROOT / "emulators"))
+ return _profiles_in(str(siteinputs.current().emulators), skip_aliases=True)
# Whose code a source reference cites, in one word: a platform page names
@@ -4015,13 +4016,17 @@ def _category_evidence_md(evidence: dict, launched: list[str] | None = None) ->
return f"No reader found ({cores})"
-@functools.lru_cache(maxsize=1)
def _profiles_by_core_name() -> dict[str, list[str]]:
"""Core name as a platform writes it -> the profiles with a page for it.
MinUI writes `mednafen_vb`, the profile and its page are `beetle_vb`.
"""
- profiles = load_emulator_profiles(str(_REPO_ROOT / "emulators"))
+ return _core_names_in(str(siteinputs.current().emulators))
+
+
+@functools.lru_cache(maxsize=None)
+def _core_names_in(emulators_dir: str) -> dict[str, list[str]]:
+ profiles = _profiles_in(emulators_dir, skip_aliases=True)
with_page = {k for k, v in profiles.items() if v.get("type") not in ("alias", "test")}
return {
core: sorted(k for k in keys if k in with_page)
@@ -4191,7 +4196,7 @@ def _card_route(name: str, display: str, guide: Guide) -> list[str]:
sees it is the reader's to name: no guide cites it."""
if guide.on_card or _offline_line(name):
return []
- extract_to = (load_platform_registry("platforms").get(name) or {}).get("extract_to")
+ extract_to = (load_platform_registry(str(siteinputs.current().platforms)).get(name) or {}).get("extract_to")
leaf = (extract_to or "bios").rstrip("/").rsplit("/", 1)[-1].lstrip("~") or "bios"
tabs = guide.card_run or ("windows", "macos", "linux")
args = {key: f"--platform {name} --dest "
@@ -4392,7 +4397,7 @@ def generate_handheld_guide(guide: HandheldGuide) -> str:
A handheld owner starts from the device name. The systems were named
only in a closed note, with a path seen from inside the device."""
uses = GUIDES[guide.uses]
- display = load_platform_config(guide.uses, "platforms").get("platform", guide.uses)
+ display = load_platform_config(guide.uses, str(siteinputs.current().platforms)).get("platform", guide.uses)
args = {"windows": f"--platform {guide.uses} --dest E:\\BIOS",
"macos": f"--platform {guide.uses} --dest /path/to/card/BIOS",
"linux": f"--platform {guide.uses} --dest /path/to/card/BIOS"}
@@ -4447,7 +4452,7 @@ def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list:
lines.append("")
for note in guide.notes:
lines += [note, ""]
- lines += [line for note in extract_notes("platforms", released_only=True)
+ lines += [line for note in extract_notes(str(siteinputs.current().platforms), released_only=True)
if f"The {display} pack " in note
for line in (note, "")]
if guide.more_folders:
@@ -4922,7 +4927,7 @@ def _data_member_digests(
from hashing import compute_hashes # noqa: PLC0415
if data_registry is None:
- data_registry = load_data_dir_registry(str(root / "platforms"))
+ data_registry = load_data_dir_registry(str(siteinputs.current().platforms))
found: dict[str, dict[str, str]] = {}
for ref, tree in sorted(members.items()):
cache = (data_registry.get(ref) or {}).get("local_cache")
@@ -5810,7 +5815,7 @@ def _platform_display(name: str) -> str:
display = _platform_names.get(name)
if display is None:
try:
- config = load_platform_config(name, str(_REPO_ROOT / "platforms"))
+ config = load_platform_config(name, str(siteinputs.current().platforms))
except FileNotFoundError:
config = {}
display = _platform_names[name] = str(config.get("platform") or name)
@@ -6607,11 +6612,13 @@ def _reading_pairs(sha1: str, consumers: FileConsumers) -> list[tuple[str, dict]
return pairs
-@functools.lru_cache(maxsize=1)
def _profiles_on_disk() -> dict[str, dict]:
- from common import load_emulator_profiles # noqa: PLC0415
+ return _profiles_in(str(siteinputs.current().emulators), skip_aliases=False)
- return load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False)
+
+@functools.lru_cache(maxsize=None)
+def _profiles_in(emulators_dir: str, skip_aliases: bool) -> dict[str, dict]:
+ return load_emulator_profiles(emulators_dir, skip_aliases=skip_aliases)
def _entry_systems(emulator: str, entry: dict) -> list[str]:
@@ -8931,10 +8938,14 @@ def emulator_install_route(name: str, profile: dict) -> tuple[str, list[str], tu
return command, setups, oses
-@functools.cache
def _standalone_cores(platform: str) -> frozenset[str]:
"""The emulators a setup launches as apps, by its platform file."""
- config = load_platform_config(platform, str(_REPO_ROOT / "platforms"))
+ return _standalone_cores_in(str(siteinputs.current().platforms), platform)
+
+
+@functools.cache
+def _standalone_cores_in(platforms_dir: str, platform: str) -> frozenset[str]:
+ config = load_platform_config(platform, platforms_dir)
return frozenset(str(c) for c in config.get("standalone_cores") or [])
@@ -11787,7 +11798,7 @@ def _own_root_folders(registry: dict) -> dict[str, str]:
destination carries its root in every path."""
folders = {}
for key, entry in registry.items():
- config = load_platform_config(key, "platforms")
+ config = load_platform_config(key, str(siteinputs.current().platforms))
if config.get("base_destination") == "" and entry.get("extract_to"):
folders[key] = entry["extract_to"]
return folders
@@ -11887,9 +11898,10 @@ def generate_which_pack() -> str:
The tiles are the choice. Open tables under them listed every setup a
second time and made the page 11,800 px tall on a phone."""
- registry = load_platform_registry("platforms")
+ registry = load_platform_registry(str(siteinputs.current().platforms))
record = release_record.load_advertised()
- display = {key: load_platform_config(key, "platforms").get("platform", key) for key in registry}
+ display = {key: load_platform_config(key, str(siteinputs.current().platforms)).get("platform", key)
+ for key in registry}
all_tabs = ("windows", "macos", "linux", "steamdeck", "android")
lines = [
"---",
diff --git a/scripts/sitediagrams.py b/scripts/sitediagrams.py
index 285d2bdea..2171bb371 100644
--- a/scripts/sitediagrams.py
+++ b/scripts/sitediagrams.py
@@ -36,6 +36,7 @@ from xml.sax.saxutils import escape
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
+import siteinputs # noqa: E402
import sitemarkup as markup # noqa: E402
PLACEHOLDER = re.compile(r"(?m)^([ \t]*)[ \t]*$")
@@ -314,21 +315,29 @@ def svg(figure: Figure, layout: str, suffix: str) -> str:
# Data
-@lru_cache(maxsize=None)
def registry() -> dict:
+ return _registry_in(str(siteinputs.current().platforms))
+
+
+@lru_cache(maxsize=None)
+def _registry_in(platforms_dir: str) -> dict:
import yaml
- with open(ROOT / "platforms" / "_registry.yml", encoding="utf-8") as handle:
+ with open(Path(platforms_dir) / "_registry.yml", encoding="utf-8") as handle:
return yaml.safe_load(handle).get("platforms", {})
-@lru_cache(maxsize=None)
def setups() -> tuple[str, ...]:
"""The registered setups, archived ones included: each has its own page,
pack and file list, so every count of setups on the site is this one."""
+ return _setups_in(str(siteinputs.current().platforms))
+
+
+@lru_cache(maxsize=None)
+def _setups_in(platforms_dir: str) -> tuple[str, ...]:
from common import list_registered_platforms
- return tuple(sorted(list_registered_platforms(str(ROOT / "platforms"), include_archived=True)))
+ return tuple(sorted(list_registered_platforms(platforms_dir, include_archived=True)))
def setups_named(named: list[str]) -> str:
@@ -339,15 +348,19 @@ def setups_named(named: list[str]) -> str:
return ", ".join(named) + (f" and {others} more" if others else "")
-@lru_cache(maxsize=None)
def platform_modes() -> dict[str, tuple[str, str, str | None]]:
"""name -> (display name, verification mode, hash rule) for every platform."""
+ return _platform_modes_in(str(siteinputs.current().platforms))
+
+
+@lru_cache(maxsize=None)
+def _platform_modes_in(platforms_dir: str) -> dict[str, tuple[str, str, str | None]]:
from common import load_platform_config
out: dict[str, tuple[str, str, str | None]] = {}
- for name in setups():
- config = load_platform_config(name, str(ROOT / "platforms"))
- rule = registry().get(name, {}).get("hash_rule") or config.get("hash_rule")
+ for name in _setups_in(platforms_dir):
+ config = load_platform_config(name, platforms_dir)
+ rule = _registry_in(platforms_dir).get(name, {}).get("hash_rule") or config.get("hash_rule")
out[name] = (config.get("platform", name), config.get("verification_mode", ""), rule)
return out
@@ -365,13 +378,17 @@ def _folder(name: str) -> str:
CHAIN_EXAMPLES = (("batocera", "duckstation"), ("emudeck", "pcsx2"), ("retroarch", "beetle_psx"))
-@lru_cache(maxsize=None)
def emulator_names() -> dict[str, str]:
+ return _emulator_names_in(str(siteinputs.current().emulators))
+
+
+@lru_cache(maxsize=None)
+def _emulator_names_in(emulators_dir: str) -> dict[str, str]:
import yaml
names: dict[str, str] = {}
for _, emulator in CHAIN_EXAMPLES:
- with open(ROOT / "emulators" / f"{emulator}.yml", encoding="utf-8") as handle:
+ with open(Path(emulators_dir) / f"{emulator}.yml", encoding="utf-8") as handle:
label = str(yaml.safe_load(handle).get("emulator", emulator))
names[emulator] = re.sub(r"\s*\(.*\)$", "", label)
return names
diff --git a/scripts/siterender.py b/scripts/siterender.py
index 9f19a9d02..bca6e60b5 100644
--- a/scripts/siterender.py
+++ b/scripts/siterender.py
@@ -7,6 +7,7 @@ import urllib.parse
from collections.abc import Callable
from pathlib import Path
+import siteinputs
import sitemarkup as markup
@@ -401,7 +402,8 @@ def _build_boards() -> None:
from common import load_emulator_profiles
- database = _REPO_ROOT / "database.json"
+ inputs = siteinputs.current()
+ database = inputs.db
if not database.exists():
_archive_boards.clear()
_board_names.clear()
@@ -409,7 +411,7 @@ def _build_boards() -> None:
return
with open(database, encoding="utf-8") as handle:
db = json.load(handle)
- profiles = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False)
+ profiles = load_emulator_profiles(str(inputs.emulators), skip_aliases=False)
_arcade_source[0] = (profiles, db)
profiles, db = _arcade_source[0]
files = db.get("files", {})
@@ -780,9 +782,10 @@ def _ensure_console_sections() -> None:
from common import load_emulator_profiles
from generate_site import _group_by_manufacturer
- with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle:
+ inputs = siteinputs.current()
+ with open(inputs.db, encoding="utf-8") as handle:
db = json.load(handle)
- profiles = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False)
+ profiles = load_emulator_profiles(str(inputs.emulators), skip_aliases=False)
set_console_sections(_group_by_manufacturer(db), program_names(profiles))
@@ -1431,7 +1434,7 @@ def system_search_aliases(sys_id: str) -> list[str]:
platform_system_names,
)
- platforms_dir = str(_REPO_ROOT / "platforms")
+ platforms_dir = str(siteinputs.current().platforms)
index: dict[str, set[str]] = {}
for platform in list_registered_platforms(platforms_dir, include_archived=True):
config = load_platform_config(platform, platforms_dir)
diff --git a/tests/test_site_inputs.py b/tests/test_site_inputs.py
index ab75d4be7..57ee16fa4 100644
--- a/tests/test_site_inputs.py
+++ b/tests/test_site_inputs.py
@@ -1,14 +1,17 @@
-"""generate_site --db reaches every page.
+"""generate_site --db, --emulators-dir and --platforms-dir reach every page.
The run loaded the given database, but the page-finishing pass reloaded the
repository's database.json: a card held only by the given collection stopped
-the whole run on a KeyError.
+the whole run on a KeyError. The emulator names and engine sections came from
+./emulators, and has_release_pack read `platforms` relative to the working
+directory, so a get/ page said "No ZIP" for a setup the API marked as released.
"""
from __future__ import annotations
import ast
import json
+import os
import shutil
import sys
import tempfile
@@ -19,6 +22,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import generate_site # noqa: E402
+import sitediagrams # noqa: E402
import siteinputs # noqa: E402
SHA1 = "0123456789ab" + "c" * 28
@@ -47,7 +51,65 @@ class TheGivenDatabase(_Inputs):
self.assertEqual(generate_site._collection_sha1_by_path(), {"bios/X/only-here.bin": SHA1})
-class GenerateInstallsTheInputs(unittest.TestCase):
+class TheGivenProfilesAndPlatforms(_Inputs):
+ def setUp(self) -> None:
+ tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" if (REPO_ROOT / "tmp").is_dir() else None))
+ self.addCleanup(shutil.rmtree, tmp)
+ self.emulators = tmp / "emulators"
+ self.emulators.mkdir()
+ (self.emulators / "fixturecore.yml").write_text(
+ "emulator: Fixture Core\ntype: libretro\nsystems: [fixture-system]\nfiles: []\n")
+ self.platforms = tmp / "platforms"
+ shutil.copytree(REPO_ROOT / "platforms", self.platforms)
+
+ def test_emulator_names_come_from_the_given_profiles(self) -> None:
+ self.use(emulators=self.emulators)
+ self.assertEqual(set(generate_site._profiles_on_disk()), {"fixturecore"})
+ self.assertEqual(generate_site._emulator_named_link("fixturecore"),
+ "[Fixture Core](emulators/fixturecore.md)")
+
+ def test_release_pack_does_not_depend_on_the_working_directory(self) -> None:
+ released = generate_site._release_pack_platforms()
+ self.assertIn("retroarch", released)
+ here = os.getcwd()
+ os.chdir(REPO_ROOT / "scripts")
+ self.addCleanup(os.chdir, here)
+ self.assertTrue(generate_site.has_release_pack("retroarch"))
+ self.assertEqual(generate_site.get_page_title("retroarch", "RetroArch"), "RetroArch BIOS pack")
+
+ def test_the_registry_comes_from_the_given_platforms(self) -> None:
+ registry = self.platforms / "_registry.yml"
+ registry.write_text(registry.read_text().replace("release_pack: true", "release_pack: false"))
+ self.use(platforms=self.platforms)
+ self.assertFalse(generate_site.has_release_pack("retroarch"))
+ self.assertEqual(sitediagrams.registry()["retroarch"].get("release_pack"), False)
+
+
+class NoPageModuleReadsTheRepositoryInputs(unittest.TestCase):
+ """A helper that opens the repository's database, profiles or platform
+ files itself bypasses the flags. siteinputs.current() is the one source."""
+
+ MODULES = ("generate_site.py", "siterender.py", "sitediagrams.py")
+ INPUTS = {"database.json", "emulators", "platforms"}
+
+ def test_no_literal_input_path(self) -> None:
+ found = []
+ for module in self.MODULES:
+ tree = ast.parse((REPO_ROOT / "scripts" / module).read_text(encoding="utf-8"))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div) \
+ and getattr(node.left, "id", "") in ("_REPO_ROOT", "ROOT") \
+ and isinstance(node.right, ast.Constant) and node.right.value in self.INPUTS:
+ found.append(f"{module}:{node.lineno}")
+ if isinstance(node, ast.Call) and getattr(node.func, "id", getattr(node.func, "attr", "")) in (
+ "load_platform_config", "load_platform_registry", "load_emulator_profiles",
+ "list_registered_platforms", "load_database", "extract_notes",
+ "load_data_dir_registry"):
+ for arg in node.args:
+ if isinstance(arg, ast.Constant) and arg.value in self.INPUTS:
+ found.append(f"{module}:{node.lineno}")
+ self.assertEqual(found, [])
+
def test_generate_installs_the_inputs_first(self) -> None:
source = ast.get_source_segment(
(REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8"),
diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md
index 3ef512d15..3a106e55c 100644
--- a/wiki/testing-guide.md
+++ b/wiki/testing-guide.md
@@ -65,7 +65,7 @@ Every module in `tests/`, by the area it guards.
| Pack probe | the cores a platform runs for a system, staging grown to a fixed point, runs mapped to pack and manifest entries, the line and its schema, the probe lock per run, that no page shows its lines. One class builds a real pack narrowed to PlayStation and lays it out both ways, from the archive and with `install.py --dest`, and skips without the collection | `test_pack_probe` |
| Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` |
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
-| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` |
+| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_inputs`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` |
| Games on the card | small DATs, listxml and setup sources, a fake fetcher, fixture install lists, and the committed data against its schemas | `test_arcade_parents`, `test_game_catalogs`, `test_game_headers`, `test_game_regions`, `test_games_index`, `test_romfolders_card`, `test_romfolders_es` |
## Test architecture
From 6a79281e07729d3550f9d706d197920f63ec32f2 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:14:45 +0100
Subject: [PATCH 03/22] chore: read every site module in the text guards
Three guards named generate_site.py or a fixed list of four modules, while
page text now lives in twelve site modules and packreadme.py. The list
is a glob. Read that way, the pack README said "all of theirs" twice.
---
scripts/packreadme.py | 6 +++---
tests/sitepages.py | 9 +++++++++
tests/test_site_render.py | 9 +++++----
tests/test_site_validation.py | 18 +++++++++++++-----
4 files changed, 30 insertions(+), 12 deletions(-)
diff --git a/scripts/packreadme.py b/scripts/packreadme.py
index 06ee483a3..ee5e5e26f 100644
--- a/scripts/packreadme.py
+++ b/scripts/packreadme.py
@@ -300,8 +300,8 @@ def _build_readme(
if fallback_systems:
listed = "\n".join(f" {_group_label(s)}" for s in fallback_systems)
region_info += (
- "\n These systems have no BIOS in those regions, so all\n"
- " of theirs were kept:\n"
+ "\n These systems have no BIOS in those regions, so every\n"
+ " BIOS file for them was kept:\n"
f"{listed}\n"
)
region_info += (
@@ -316,7 +316,7 @@ def _build_readme(
slot_info = (
" Where a core declares which BIOS it prefers, only that one was\n"
" kept for each system and region. Systems whose cores declare no\n"
- " order keep all of theirs.\n"
+ " order keep every BIOS file.\n"
)
if undecidable_slots:
slot_info += (
diff --git a/tests/sitepages.py b/tests/sitepages.py
index ee8897a55..f11300dd8 100644
--- a/tests/sitepages.py
+++ b/tests/sitepages.py
@@ -44,3 +44,12 @@ def pages() -> list[Path]:
def name(path: Path) -> str:
"""A page file's docs path, the name tests use for it."""
return path.relative_to(CONTENT).with_suffix(".md").as_posix()
+
+
+def generator_sources() -> list[Path]:
+ """Every module whose strings reach a page: generate_site.py, each
+ scripts/site*.py and the pack README writer. Listed by glob, so text
+ moved out of generate_site.py stays under the guards that read it."""
+ scripts = REPO_ROOT / "scripts"
+ return sorted({*scripts.glob("site*.py"), scripts / "generate_site.py",
+ scripts / "packreadme.py"})
diff --git a/tests/test_site_render.py b/tests/test_site_render.py
index f4b11a4ac..9628bd5ed 100644
--- a/tests/test_site_render.py
+++ b/tests/test_site_render.py
@@ -50,15 +50,16 @@ class EveryPageAddressesPlatformsAsPeers(unittest.TestCase):
text every site generator writes, docstrings and bare identifiers aside."""
WORDS = ("their file", "theirs", "of ours", "corrected", "left out", "drift")
- SOURCES = ("generate_site.py", "siterender.py", "sitesources.py", "siteexports.py")
-
def test_no_page_text_judges_another_project(self):
import ast
import re
+ from tests.sitepages import generator_sources
+
found = []
- for name in self.SOURCES:
- tree = ast.parse((REPO_ROOT / "scripts" / name).read_text(encoding="utf-8"))
+ for source in generator_sources():
+ name = source.name
+ tree = ast.parse(source.read_text(encoding="utf-8"))
docstrings = {
id(node.body[0].value) for node in ast.walk(tree)
if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef,
diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py
index feac2d14a..e7ec40fb4 100644
--- a/tests/test_site_validation.py
+++ b/tests/test_site_validation.py
@@ -349,9 +349,13 @@ class TheSitemapIsAnnouncedWhereCrawlersLook(unittest.TestCase):
config = (root / "website" / "astro.config.mjs").read_text()
self.assertIn("site: SITE", config)
self.assertIn("sitemap.xml", config)
- source = (root / "scripts" / "generate_site.py").read_text()
- self.assertNotIn('"robots.txt"),\n "User-agent', source)
- self.assertNotIn("Sitemap: https://", source)
+ from tests.sitepages import generator_sources
+
+ for path in generator_sources():
+ source = path.read_text(encoding="utf-8")
+ with self.subTest(module=path.name):
+ self.assertNotIn('"robots.txt"),\n "User-agent', source)
+ self.assertNotIn("Sitemap: https://", source)
@@ -409,8 +413,12 @@ class PlainHouseStyle(unittest.TestCase):
def test_headings_do_not_repeat_the_site_name(self):
# The header bar already names the site, and the browser title gets
# it from the theme: "# DuckStation - RetroBIOS" said it twice.
- source = (ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")
- self.assertNotRegex(source, r'"# [^"]* - (?:\{SITE_NAME\}|RetroBIOS)')
+ from tests.sitepages import generator_sources
+
+ for path in generator_sources():
+ with self.subTest(module=path.name):
+ self.assertNotRegex(path.read_text(encoding="utf-8"),
+ r'"# [^"]* - (?:\{SITE_NAME\}|RetroBIOS)')
for page in sorted((ROOT / "wiki").glob("*.md")):
first = page.read_text(encoding="utf-8").splitlines()[0]
with self.subTest(page=page.name):
From 2f721bc640d3ea7425c4dffd4d0e697dea379f17 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:16:44 +0100
Subject: [PATCH 04/22] fix: list unsourceable files as not collected yet
Every card read "Not collectable" and the gaps page listed 259 entries as
acknowledged gaps verified from source code. unsourceable: only keeps a
same-named file from being served in the entry's place, and each entry
stays an acquisition target (wiki/profiling.md). The section now names
the catalog or romset that lists each file, from the provenance report
built with the profiles' needs: XM7's KANJI2.ROM is in No-Intro and in
MAME's fm7740sx.zip. The old anchor still leads to the section.
---
scripts/generate_site.py | 74 +++++++++++++++++++++++----------
tests/test_audit_regressions.py | 28 ++++++++++++-
2 files changed, 79 insertions(+), 23 deletions(-)
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index 62f400543..0eece3cfc 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -86,7 +86,7 @@ from generate_readme import (
extract_targets,
release_totals,
)
-from provenance_report import build_report, distinct_missing
+from provenance_report import build_report, distinct_missing, profile_needs
import sitemarkup as markup
import siteentries
import siteinputs
@@ -8433,12 +8433,7 @@ def _render_emulator_file(
if fnote:
lines.append(f"- {data_root_words(_escape_tags(link_prose_citations(profile, str(fnote))), data_href)}")
if not in_repo and f.get("unsourceable"):
- # The badge alone sent readers after a dump that cannot exist, and the
- # reason was only on the gaps page, which the card did not link.
- lines.append(
- f"- Not collectable: {_escape_tags(str(f['unsourceable']))} "
- "([acknowledged gaps](../gaps.md#acknowledged-gaps))"
- )
+ lines.append(f"- {_unsourceable_line(f)}")
lines.extend(_contents_lines(contents))
lines[body_start:] = _search_skipped_apart(lines[body_start:])
lines.append("")
@@ -8550,12 +8545,7 @@ def _emulator_card_folds(
open_lines.append(data_root_words(
_escape_tags(link_prose_citations(profile, str(fnote))), data_href))
if not in_repo and f.get("unsourceable"):
- # The badge alone sent readers after a dump that cannot exist, and the
- # reason was only on the gaps page, which the card did not link.
- open_lines.append(
- f"Not collectable: {_escape_tags(str(f['unsourceable']))} "
- "([acknowledged gaps](../gaps.md#acknowledged-gaps))"
- )
+ open_lines.append(_unsourceable_line(f))
lines = [f"- {d}" for d in open_lines]
shut_fold = [f"- {d}" for d in hashes + [m for m in more if skipped(m)]]
read_fold = [f"- {m}" for m in more if not skipped(m)]
@@ -10459,6 +10449,33 @@ def _hash_cell(hashes: dict[str, list[str]]) -> str:
)
+def _unsourceable_line(f: dict) -> str:
+ """Why the collection holds no copy of a file a profile marks
+ `unsourceable:`, linked to the list of them. The flag only keeps a
+ same-named file from being served in its place: the file is still
+ wanted, and "Not collectable" sent readers away from KANJI2.ROM, which
+ MAME's fm7740sx.zip carries."""
+ return (f"Not collected yet: {_escape_tags(str(f['unsourceable']))} "
+ "([files not collected yet](../gaps.md#not-collected-yet))")
+
+
+def _catalogued_needs(report: dict | None) -> dict[tuple[str, str], list[str]]:
+ """(profile, file name casefolded) -> where a dump catalog or an
+ imported romset lists that file, from a provenance report built with
+ the profiles' needs."""
+ out: dict[tuple[str, str], list[str]] = {}
+ for source, data in sorted((report or {}).items()):
+ label = _PROVENANCE_LABELS.get(source, source)
+ for entry in data.get("missing", []):
+ where = [f"{label} `{_escape_tags(str(entry.get('name', '')))}`"]
+ where += [f"{_escape_tags(str(hit['source']))} `{_escape_tags(str(hit['archive']))}`"
+ for hit in entry.get("romsets", [])]
+ for need in entry.get("needed_by", []):
+ held = out.setdefault((need["emulator"], str(need["name"]).casefold()), [])
+ held += [w for w in where if w not in held]
+ return out
+
+
def _reason_cell(text: str) -> str:
"""A reason in a gaps table, in the class that keeps its column readable."""
return f'{text}' if text else "-"
@@ -10753,9 +10770,14 @@ def generate_gap_analysis(
registry: dict | None = None,
gap_report: dict | None = None,
moved: dict[str, str] | None = None,
+ catalogued: dict | None = None,
) -> str:
"""Generate a unified gap analysis page.
+ ``catalogued`` is a provenance report built with the profiles' needs:
+ a file the collection does not hold yet names the catalog or romset
+ that lists it.
+
With ``moved``, the list of files to source goes to pages of its own,
keyed by stem: on the page it made 662 KB of HTML.
@@ -11186,7 +11208,7 @@ def generate_gap_analysis(
"",
])
- # ---- Section 4: Acknowledged gaps (unsourceable files) ----
+ # ---- Section 4: files the collection does not hold yet (unsourceable) ----
all_unsourceable: list[dict] = []
for emu_name, data in sorted(report_all.items()):
@@ -11200,19 +11222,26 @@ def generate_gap_analysis(
})
if all_unsourceable:
+ listed = _catalogued_needs(catalogued)
+ # The old heading's address still leads here.
lines.extend([
- "## Acknowledged gaps",
+ '',
"",
- f"{len(all_unsourceable)} files documented as unsourceable "
- "(verified from source code).",
+ "## Not collected yet",
"",
- "| File | Emulator | Reason | Source ref |",
- "|------|----------|--------|-----------|",
+ f"{len(all_unsourceable)} files the collection does not hold. A file "
+ "with the same name is never served in their place. Each is still "
+ "wanted, and the last column names a catalog or romset that lists it.",
+ "",
+ "| File | Emulator | Reason | Source ref | Listed in |",
+ "|------|----------|--------|-----------|-----------|",
])
for u in sorted(all_unsourceable, key=lambda x: x["name"]):
+ where = listed.get((u["emu_key"], str(u["name"]).casefold()), [])
lines.append(
f"| `{u['name']}` | {u['emulator']} | {_reason_cell(u['reason'])} "
- f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} |"
+ f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} "
+ f"| {', '.join(where) or '-'} |"
)
lines.append("")
@@ -12563,12 +12592,16 @@ def _generate(args: argparse.Namespace) -> None:
sitestarlight.MOVED_PAGES.setdefault(
f"cross-reference-{pname}.md", f"platforms/{pname}.md")
+ provenance_report = build_report(
+ db, load_provenance_snapshots(), needs=profile_needs(profiles, db))
+
print("Generating gap analysis page...")
gap_pages: dict[str, str] = {}
write_if_changed(
str(docs / "gaps.md"),
tables_out_of_search(generate_gap_analysis(
profiles, coverages, db, suppl_names, registry, gap_report, gap_pages,
+ catalogued=provenance_report,
)) + reference_next(),
)
for stem, page in gap_pages.items():
@@ -12584,7 +12617,6 @@ def _generate(args: argparse.Namespace) -> None:
)
print("Generating dump provenance page...")
- provenance_report = build_report(db, load_provenance_snapshots())
provenance_pages: dict[str, str] = {}
write_if_changed(
str(docs / "provenance.md"),
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index 3df8850fe..91110a2d5 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -5143,8 +5143,32 @@ class UnsourceableCardSaysWhy(unittest.TestCase):
text = "\n".join(_render_emulator_file(
entry, {"files": [entry]}, None, [entry], lambda f: False, "file-x",
))
- self.assertIn("Not collectable: written by the firmware editor", text)
- self.assertIn("../gaps.md#acknowledged-gaps", text)
+ self.assertIn("Not collected yet: written by the firmware editor", text)
+ self.assertIn("../gaps.md#not-collected-yet", text)
+ self.assertNotIn("collectable", text)
+
+ def test_the_gaps_page_names_where_a_wanted_file_is_listed(self):
+ """The section read "Acknowledged gaps ... (verified from source
+ code)" while No-Intro lists XM7's KANJI2.ROM and MAME's fm7740sx.zip
+ carries it: the flag only keeps a homonym out, the file is wanted."""
+ from scripts.generate_site import generate_gap_analysis
+
+ unsourced = {"name": "KANJI2.ROM", "reason": "no dump distributed", "source_ref": ""}
+ report = {"xm7": {"emulator": "XM7", "gaps": 0, "gap_missing": 0, "gap_bios": 0,
+ "unsourceable": [unsourced], "gap_details": []}}
+ catalogued = {"no-intro": {"missing": [{
+ "name": "KANJI2.rom",
+ "romsets": [{"source": "mame", "archive": "fm7740sx.zip"}],
+ "needed_by": [{"emulator": "xm7", "name": "KANJI2.ROM", "required": True}],
+ }]}}
+ page = generate_gap_analysis(
+ {"xm7": {"emulator": "XM7", "files": []}}, {}, {"files": {}, "indexes": {}},
+ gap_report=report, catalogued=catalogued)
+ self.assertIn("## Not collected yet", page)
+ self.assertIn('', page)
+ self.assertIn("| No-Intro `KANJI2.rom`, mame `fm7740sx.zip` |", page)
+ self.assertNotIn("Acknowledged", page)
+ self.assertNotIn("verified from source code", page)
From 2a0088a0fc03bbe598702aba08da45784a0456c9 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:17:11 +0100
Subject: [PATCH 05/22] docs: name the launcher index in installer changes
install/launchers/index.json records the SHA-256 of install.py that
--add-app checks, and the page listed only the pins of install.sh and
install.ps1 as what a change to install.py rewrites.
---
tests/test_port.py | 7 +++++++
wiki/installer.md | 5 ++++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/tests/test_port.py b/tests/test_port.py
index 269d4023e..5c873e84b 100644
--- a/tests/test_port.py
+++ b/tests/test_port.py
@@ -124,6 +124,13 @@ class LauncherIndex(unittest.TestCase):
"install/launchers/index.json is stale: run "
"python scripts/generate_launcher_index.py")
+ def test_the_installer_page_says_when_to_rewrite_it(self):
+ """The list records install.py's SHA-256, and the page that says
+ what a change to install.py rewrites named only the two pins."""
+ page = (REPO_ROOT / "wiki" / "installer.md").read_text(encoding="utf-8")
+ section = page.split("## Changing the installer", 1)[1].split("\n## ", 1)[0]
+ self.assertIn("scripts/generate_launcher_index.py", section)
+
def test_every_member_of_the_archives_is_listed(self):
import generate_launcher_index # noqa: PLC0415
diff --git a/wiki/installer.md b/wiki/installer.md
index 73719bf16..afcacf4ef 100644
--- a/wiki/installer.md
+++ b/wiki/installer.md
@@ -1702,7 +1702,10 @@ a real 3.8 interpreter, which CI installs beside 3.12 so the check cannot skip.
The build scripts need 3.9 and are not bound by this.
Every change to `install.py` recomputes the SHA-256 pinned in `install.sh` and
-`install.ps1` in the same commit. The tests compare the pins with the file.
+`install.ps1` in the same commit, and runs `scripts/generate_launcher_index.py`:
+`install/launchers/index.json` records the hash of `install.py` that
+`--add-app` checks before it writes a launcher on a card. The tests compare
+the pins and the list with the file.
The tables of options, events, exit codes, keys and card markers, the line
menu and the run samples on this page sit between `generated` markers and are
From 4f930b5573c905880910bc360337460fee20f3b0 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:19:08 +0100
Subject: [PATCH 06/22] feat: find wanted dumps by hash on the files page
files/?hash= read the collection, platforms, emulators and retired files,
never the Wanted list: the SHA1 of Redump's ps2-dvd-212k.bin answered
that nothing the site knows has it. The lookup now carries a wanted
layer from the rows of provenance-wanted.csv, computed once for both,
and the page names the catalog that lists the dump.
---
schemas/site-api-envelope.schema.json | 6 +++--
scripts/generate_site.py | 27 ++++++++++++++++++---
tests/test_site_exports.py | 34 +++++++++++++++++++++++++++
website/src/scripts/site.ts | 17 +++++++++++++-
4 files changed, 78 insertions(+), 6 deletions(-)
diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json
index adbda67c5..4b9e765fd 100644
--- a/schemas/site-api-envelope.schema.json
+++ b/schemas/site-api-envelope.schema.json
@@ -393,7 +393,8 @@
"retired",
"data-directory",
"platform",
- "emulator"
+ "emulator",
+ "wanted"
]
},
"left": {
@@ -433,7 +434,8 @@
"description": "On a platform or emulator record: the algorithm of the value its owner wrote, filed under that value and under the file's other hashes. name: the emulator reads the file by its name and declares no hash that matches it."
},
"owner": {
- "type": "string"
+ "type": "string",
+ "description": "On a platform or emulator record: the platform or profile key. On a wanted record: the dump catalog that lists the value (Redump, No-Intro, TOSEC), the collection holding no file with it."
},
"system_name": {
"type": "string",
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index 0eece3cfc..53e2c875f 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -1620,6 +1620,7 @@ def generate_data_exports(
members = data_directory_members()
locations = file_location_items(db, profiles, members)
+ wanted = _provenance_wanted_rows(db)
envelopes = {
"platforms.json": ("platforms", platform_items),
"emulators.json": ("emulators", emulator_items),
@@ -1628,7 +1629,7 @@ def generate_data_exports(
}
write_lookup_shards(
api, file_lookup_records(db, coverages, profiles, locations, members, consumers,
- retired_files.load(_REPO_ROOT / "retired.json")),
+ retired_files.load(_REPO_ROOT / "retired.json"), wanted),
generated_at, commit,
)
for filename, (kind, items) in envelopes.items():
@@ -1747,7 +1748,7 @@ def generate_data_exports(
_record(downloads / "retrobios.sqlite")
write_if_changed(
str(downloads / "provenance-wanted.csv"),
- _csv_document(list(WANTED_FIELDS), _provenance_wanted_rows(db)),
+ _csv_document(list(WANTED_FIELDS), wanted),
)
assets = [
@@ -5010,12 +5011,14 @@ def file_lookup_records(
members: dict | None = None,
consumers: FileConsumers | None = None,
retired: dict | None = None,
+ wanted: list[dict] | None = None,
) -> list[dict]:
"""Every hash the site knows, one record per value, for files/?hash=.
The collection's own files lead to their card, a platform's declared
value to the system section of its page, a profile's to the entry's
- card. The site search indexed these values as text, 4 MB of the 21 MB
+ card, a wanted dump (``wanted``, the provenance-wanted.csv rows) to
+ its catalog on the provenance page. The site search indexed these values as text, 4 MB of the 21 MB
it made every visitor download: the lookup reads one shard instead.
"""
files = db.get("files", {})
@@ -5088,6 +5091,7 @@ def file_lookup_records(
**({"system_name": entry_label} if entry_label else {}),
})
records += retired_records(retired or {}, files, locations)
+ records += wanted_records(wanted or [])
records += _held_under_every_hash(records, files, db.get("indexes") or {})
for record in records:
if record["layer"] in ("platform", "emulator"):
@@ -5099,6 +5103,23 @@ def file_lookup_records(
r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"]))
+def wanted_records(rows: list[dict]) -> list[dict]:
+ """A record under each hash of a dump a catalog lists and the collection
+ lacks, from the rows of provenance-wanted.csv. A pasted hash of a
+ wanted dump (Redump's ps2-dvd-212k.bin) answered that nothing the site
+ knows has it, and never said the dump is wanted."""
+ records = []
+ for row in rows:
+ label = _PROVENANCE_LABELS.get(row["catalog"], row["catalog"])
+ url = _site_path(f"provenance.md#{_slugify_anchor(label)}")
+ for algorithm in ("sha1", "md5", "crc32"):
+ value = str(row.get(algorithm) or "").lower()
+ if value:
+ records.append({"value": value, "algorithm": algorithm, "layer": "wanted",
+ "owner": label, "name": row["name"], "url": url})
+ return records
+
+
def _read_by_name(records: list[dict], files: dict, consumers: FileConsumers) -> list[dict]:
"""The emulators the file's card names that no declared hash reached.
diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py
index 1d96d3f0a..5455a422b 100644
--- a/tests/test_site_exports.py
+++ b/tests/test_site_exports.py
@@ -832,6 +832,40 @@ class AnArchiveMemberIsFoundByItsCrc(unittest.TestCase):
self.assertIn("byLayer('archive-member')", script)
+class AWantedDumpIsFoundByItsHash(unittest.TestCase):
+ """The SHA1 of a PS2 SCPH-30005R BIOS, which Redump lists as
+ ps2-dvd-212k.bin and the Wanted page asks for, answered that no file
+ the site knows has it."""
+
+ def test_each_hash_of_a_wanted_row_is_a_record(self):
+ import generate_site # noqa: PLC0415
+
+ row = {"catalog": "redump", "dat": "Sony - PlayStation 2 - BIOS Images",
+ "name": "ps2-dvd-212k.bin", "description": "", "size": "4194304",
+ "crc32": "1A2B3C4D", "md5": "", "sha1": "5b346eb9258d710609a9855befd35cab8f3a334a"}
+ records = generate_site.file_lookup_records(
+ {"files": {}, "indexes": {}}, {}, {}, [], wanted=[row, {**row, "dat": "other"}])
+ self.assertEqual(records, [
+ {"value": "1a2b3c4d", "algorithm": "crc32", "layer": "wanted", "owner": "Redump",
+ "name": "ps2-dvd-212k.bin", "url": "/retrobios/provenance/#redump"},
+ {"value": "5b346eb9258d710609a9855befd35cab8f3a334a", "algorithm": "sha1",
+ "layer": "wanted", "owner": "Redump", "name": "ps2-dvd-212k.bin",
+ "url": "/retrobios/provenance/#redump"},
+ ])
+
+ def test_the_schema_and_the_page_carry_the_layer(self):
+ import json as _json # noqa: PLC0415
+
+ import generate_site # noqa: PLC0415
+
+ root = Path(__file__).resolve().parents[1]
+ schema = _json.loads((root / "schemas" / "site-api-envelope.schema.json").read_text())
+ self.assertIn('"wanted"', _json.dumps(schema))
+ self.assertIn("byLayer('wanted')", (root / "website" / "src" / "scripts" / "site.ts").read_text())
+ source = Path(generate_site.__file__).read_text(encoding="utf-8")
+ self.assertIn('retired_files.load(_REPO_ROOT / "retired.json"), wanted)', source)
+
+
class AFileHasAnAddressByContent(unittest.TestCase):
"""mt32_control.rom moved from the Commodore page to the Roland page and
every link to its card broke: the anchor was content-based, the page
diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts
index 09d20902f..263309d34 100644
--- a/website/src/scripts/site.ts
+++ b/website/src/scripts/site.ts
@@ -558,9 +558,12 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched:
const setups = new Set(platforms.map((item) => item.owner));
const members = distinct(byLayer('archive-member'), (item) => `${item.sha1}|${item.name}`);
+ // A dump a catalog lists and the collection lacks: the Wanted list.
+ const wanted = held.length || members.length ? [] : distinct(byLayer('wanted'), (item) => `${item.owner}|${item.name}`);
const counted = [
held.length ? count(held.length, 'file in the collection', 'files in the collection')
: members.length ? count(members.length, 'member of a collection archive', 'members of collection archives')
+ : wanted.length ? 'a wanted dump, not in the collection yet'
: 'not in the collection',
...(platforms.length ? [`${count(platforms.length, 'declaration', 'declarations')} by ${count(setups.size, 'setup', 'setups')}`] : []),
...(owners.length ? [count(owners.length, 'emulator that reads it', 'emulators that read it')] : []),
@@ -617,8 +620,20 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched:
);
});
const retired = held.length ? [] : distinct(byLayer('retired'), (item) => item.sha1 ?? item.value);
- if (!held.length && !members.length && !retired.length) blocks.push(paragraph('No file in the collection has this hash.'));
+ if (!held.length && !members.length && !retired.length && !wanted.length) blocks.push(paragraph('No file in the collection has this hash.'));
blocks.push(...retired.map(retiredParagraph));
+ if (wanted.length) {
+ const list = document.createElement('ul');
+ for (const item of wanted) {
+ const row = document.createElement('li');
+ row.append(link(item.url, item.owner ?? ''), ' lists it as ', code(item.name ?? item.value));
+ list.append(row);
+ }
+ blocks.push(
+ paragraph(strong('Wanted.'), ' The collection does not hold this dump yet. If you dumped it yourself, ', link('contributing/', 'add it'), '.'),
+ list,
+ );
+ }
if (members.length) {
// A ROM the collection holds inside a set: the set's card, by its name.
const list = document.createElement('ul');
From d1535d0787a458caa31c902d7aeba3da15d9ed26 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:20:03 +0100
Subject: [PATCH 07/22] fix: name the file a variant download is saved as
A .variants/ copy downloads as advision.zip.1c086064, its storage name:
raw.githubusercontent.com sends no Content-Disposition and the download
attribute is ignored across origins. A card whose download name differs
from the file name now says what to save it as, at a glance.
---
scripts/generate_site.py | 15 ++++++++++++++-
tests/test_audit_regressions.py | 17 +++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index 53e2c875f..aaae2d7ad 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -5805,6 +5805,13 @@ def _wanted_list(source: str, label: str, data: dict, by_dat: dict[str, list],
_RELEASE_ASSET_NAMES: dict[str, str] | None = None
+def _download_name(path: str) -> str:
+ """The name a browser saves a collection file under: the last segment
+ of its download URL. Neither host sends a Content-Disposition, and the
+ download attribute is ignored on another origin."""
+ return urllib.parse.unquote(_fetch_url(path).rsplit("/", 1)[-1])
+
+
def _fetch_url(path: str) -> str:
"""Where a collection file downloads from: its raw URL, or the
large-files release asset when .gitignore keeps it out of git."""
@@ -6179,6 +6186,12 @@ def _system_file_card(
# Each fact: its label, its values, whether the search reads it. What a
# setup does with the file comes first, the fingerprints last.
facts: list[tuple[str, list[str], bool]] = []
+ saved = _download_name(f["path"]) if f.get("path") else name
+ if saved != name:
+ # A .variants/ copy is stored as advision.zip.1c086064, and the
+ # browser names the download after the URL: copied as it came, the
+ # file a platform pins was not found under its name.
+ facts.append(("Save as", [f"{_code(name)}, the download is named {_code(saved)}"], False))
if plats:
# Each link lands on the platform's system section, and says where
# that platform puts the file. Out of the search: the card's title
@@ -6237,7 +6250,7 @@ def _system_file_card(
# What a system card shows at a glance: who lists the file and who reads it.
# Its fingerprints and the rest open on demand, so a page of 250 cards reads
# as a list of files, not a wall of hashes.
-_CARD_KEY_FACTS = ("On the setup's list", "Packed for its emulators by", "Emulators")
+_CARD_KEY_FACTS = ("Save as", "On the setup's list", "Packed for its emulators by", "Emulators")
_CARD_HASHES = ("SHA1", "MD5", "CRC32", "SHA256")
# The words a card's fold names its other facts by.
_CARD_MORE_WORDS = {"Romset": "romset", "Collection path": "collection path",
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index 91110a2d5..577b0cfff 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -5207,6 +5207,23 @@ class CollectionCardsCarryFullHashes(unittest.TestCase):
self.assertIn(f'{value}', text)
+class AVariantCardSaysWhatToSaveItAs(unittest.TestCase):
+ """The Download button of a .variants/ card saved advision.zip.1c086064,
+ the storage name, and nothing said RetroBat looks for advision.zip."""
+
+ def test_the_name_to_save_it_under(self):
+ from scripts.generate_site import FileConsumers, _system_file_card
+
+ entry = {"name": "advision.zip", "size": 10, "sha1": "a" * 40,
+ "path": "bios/Entex/Adventure Vision/.variants/advision.zip.1c086064"}
+ text = "\n".join(_system_file_card(entry, FileConsumers({}, {})))
+ self.assertIn("Save as", text)
+ self.assertIn("advision.zip.1c086064", text)
+ plain = "\n".join(_system_file_card(
+ {**entry, "path": "bios/Entex/Adventure Vision/advision.zip"}, FileConsumers({}, {})))
+ self.assertNotIn("Save as", plain)
+
+
class NullReadsAsNone(unittest.TestCase):
"""`upstream: ~` printed `| Upstream | None |` and `bios_filename: null`
From c7a30856b0ee20e7f3cd18693bf214f6828a6a39 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:20:46 +0100
Subject: [PATCH 08/22] feat: link the build file that holds a revision
pin_build, in 36 profiles, names the platform repository, revision, file
and variable that hold a pin_frozen revision, and the site read none of
it: the Revision held row printed the file as plain text. A Build file
row links each one at the revision it was read at.
---
scripts/generate_site.py | 20 ++++++++++++++++++++
tests/test_audit_regressions.py | 19 +++++++++++++++++++
2 files changed, 39 insertions(+)
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index aaae2d7ad..fe43620af 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -8196,6 +8196,23 @@ def _exact_size(size: int) -> str:
return _fmt_size(size) if size < 1024 else f"{_fmt_size(size)} ({size:,} bytes)"
+def _build_file_links(pin_build: list[dict]) -> list[str]:
+ """Each platform build file that holds the revision, linked at the
+ revision it was read at. The prose of pin_frozen named the file as
+ plain text, while pin_build carries all a pinned link needs."""
+ from sitesources import _browse_repo, _source_permalink # noqa: PLC0415
+
+ links = []
+ for build in pin_build:
+ repo = _browse_repo(str(build["repo"]))
+ path = str(build["path"])
+ text = f"{_escape_tags(_platform_display(str(build['platform'])))} {_code(path)}"
+ if repo is not None:
+ text = f"[{text}]({_source_permalink(repo, str(build['revision']), path, None, None)})"
+ links.append(f"{text} at {_code(str(build['revision']))} sets {_code(str(build['variable']))}")
+ return links
+
+
def _build_rows(profile: dict) -> list[str]:
"""Meta card rows for who ships the build, how its files are found, why
its revision is held and where its releases are announced."""
@@ -8216,6 +8233,9 @@ def _build_rows(profile: dict) -> list[str]:
if frozen:
held = _escape_tags(' '.join(str(frozen).split())).replace('|', r'\|')
rows.append(f"| {label} | {held} |")
+ builds = _build_file_links(profile.get("pin_build") or [])
+ if builds:
+ rows.append(f"| Build file | {'
'.join(builds)} |")
announced = profile.get("version_source")
if isinstance(announced, dict) and announced.get("url"):
url = str(announced["url"])
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index 577b0cfff..5263d0c54 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -5207,6 +5207,25 @@ class CollectionCardsCarryFullHashes(unittest.TestCase):
self.assertIn(f'{value}', text)
+class AHeldRevisionLinksItsBuildFile(unittest.TestCase):
+ """advanced_drastic's page printed the KNULLI build file that holds its
+ revision as plain text, while pin_build carries the repository, the tag
+ and the path a pinned link needs. 36 profiles carry one."""
+
+ def test_the_build_file_row(self):
+ from scripts.generate_site import _build_rows
+
+ rows = _build_rows({"pin_frozen": "KNULLI builds this revision.", "pin_build": [{
+ "platform": "knulli", "repo": "https://github.com/knulli-cfw/knulli-linux",
+ "revision": "20260511", "path": "package/emulators/advanced_drastic/advanced_drastic.mk",
+ "variable": "ADVANCED_DRASTIC_VERSION", "value": "eaf41ab9"}]})
+ self.assertIn(
+ "| Build file | [KNULLI `package/emulators/advanced_drastic/advanced_drastic.mk`]"
+ "(https://github.com/knulli-cfw/knulli-linux/blob/20260511/package/emulators/"
+ "advanced_drastic/advanced_drastic.mk) at `20260511` sets `ADVANCED_DRASTIC_VERSION` |",
+ rows)
+
+
class AVariantCardSaysWhatToSaveItAs(unittest.TestCase):
"""The Download button of a .variants/ card saved advision.zip.1c086064,
the storage name, and nothing said RetroBat looks for advision.zip."""
From 7876dc09e95b8bdde10d9174556878e3a7db3fdc Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:21:30 +0100
Subject: [PATCH 09/22] fix: send moved part cards to the emulator index
Adding one entry to mame.yml moves 938 cards from one part page to the
next, and a published emulators/mame-files-1/#file-... address landed
on a page without the card. A part page now sends a file- id it does
not hold to the emulator page, whose card index row opens the part that
holds it.
---
tests/test_site_exports.py | 25 +++++++++++++++++++++++++
website/src/scripts/site.ts | 13 +++++++++++--
2 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py
index 5455a422b..4d230afc9 100644
--- a/tests/test_site_exports.py
+++ b/tests/test_site_exports.py
@@ -866,6 +866,31 @@ class AWantedDumpIsFoundByItsHash(unittest.TestCase):
self.assertIn('retired_files.load(_REPO_ROOT / "retired.json"), wanted)', source)
+class AMovedPartCardReachesItsPage(unittest.TestCase):
+ """One entry added to mame.yml moved 938 of its 1,585 cards from one
+ part page to another, and emulators/mame-files-1/#file-mackbd-m0110a-zip
+ led to a page without the card: part pages have no card index, and
+ site.js redirected only sha1- ids."""
+
+ def test_a_part_page_sends_a_missing_card_to_the_emulator_page(self):
+ import re as _re # noqa: PLC0415
+
+ import generate_site # noqa: PLC0415
+
+ script = (Path(__file__).resolve().parents[1] / "website" / "src" / "scripts"
+ / "site.ts").read_text(encoding="utf-8")
+ body = script.split("const followMovedCard = () => {", 1)[1].split("\n};", 1)[0]
+ pattern = _re.search(r"const part = /(.+?)/\.exec\(location\.pathname\)", body).group(1)
+ files = [{"name": f"f{i}.bin", "path": f"f{i}.bin", "required": True} for i in range(600)]
+ stem = generate_site.emulator_parts("mame", files)[1][0]
+ found = _re.search(pattern.replace("\\/", "/"), f"/retrobios/emulators/{stem}/")
+ self.assertEqual(found.group(1), "mame")
+ self.assertIn("emulators/${part[1]}/#", body)
+ self.assertIn("id.startsWith('file-')", body)
+ # The emulator page holds the index whose row leads to the card.
+ self.assertIn("closest('.rb-card-index tr')", body)
+
+
class AFileHasAnAddressByContent(unittest.TestCase):
"""mt32_control.rom moved from the Commodore page to the Roland page and
every link to its card broke: the anchor was content-based, the page
diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts
index 263309d34..daeb78f28 100644
--- a/website/src/scripts/site.ts
+++ b/website/src/scripts/site.ts
@@ -814,7 +814,10 @@ const resolveFileAddress = () => {
// A file card moved to another page keeps its address: an id of the form
// sha1-<12 hex> that this page does not hold is looked up by content, and
-// an emulator card listed in the table of a split page opens its page.
+// an emulator card listed in the table of a split page opens its page. A
+// part of a split emulator page (emulators/mame-files-1/) sends a card id
+// it no longer holds to that table: an entry added to the profile moves
+// hundreds of cards from one part to the next.
const followMovedCard = () => {
const id = decodeURIComponent(location.hash.slice(1));
if (!id) return;
@@ -824,8 +827,14 @@ const followMovedCard = () => {
location.replace(link.href);
return;
}
+ if (target) return;
const match = /^sha1-([0-9a-f]{12})$/.exec(id);
- if (match && !target) location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot()));
+ if (match) {
+ location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot()));
+ return;
+ }
+ const part = /\/emulators\/(.+)-files-\d+\/?$/.exec(location.pathname);
+ if (part && id.startsWith('file-')) location.replace(new URL(`emulators/${part[1]}/#${encodeURIComponent(id)}`, siteRoot()));
};
// A link to a card folded in a table row (emulators with many entries)
From e871e01abfe0b5ecdb45e41ed5edfc0c7751fcfa Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:22:06 +0100
Subject: [PATCH 10/22] docs: list the community tool's fields
Author, Repository and License sat on three lines of one paragraph, which
renders as one line. They are a list, and a guard refuses two bold
labels on consecutive lines of a paragraph.
---
tests/test_prose_style.py | 12 ++++++++++++
wiki/community-tools.md | 6 +++---
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/tests/test_prose_style.py b/tests/test_prose_style.py
index cf439800d..d089e9f45 100644
--- a/tests/test_prose_style.py
+++ b/tests/test_prose_style.py
@@ -386,6 +386,18 @@ class NoClauseSemicolons(unittest.TestCase):
]
self.assertEqual(offenders, [], "\n".join(offenders[:20]))
+ def test_bold_labels_do_not_run_together(self):
+ """community-tools.md wrote Author, Repository and License on three
+ lines of one paragraph: rendered, the three ran on one line."""
+ label = r"\*\*[^*\n]+:\*\*"
+ run = re.compile(rf"^{label}.*\n{label}", re.M)
+ offenders = [
+ f"{p.relative_to(ROOT)}: {m.group(0).splitlines()[0]}"
+ for p in _tracked() if p.suffix == ".md"
+ for m in run.finditer(FENCED.sub("", p.read_text(encoding="utf-8")))
+ ]
+ self.assertEqual(offenders, [], "\n".join(offenders[:20]))
+
def test_wiki_headings_use_sentence_case(self):
"""The gaps page and two wiki pages titled in Title Case ("What Each
Pack Contains") beside sentence-case pages ("Quick install")."""
diff --git a/wiki/community-tools.md b/wiki/community-tools.md
index bf0384e59..f5a56b04c 100644
--- a/wiki/community-tools.md
+++ b/wiki/community-tools.md
@@ -12,9 +12,9 @@ Projects built on RetroBIOS data.
## BIOS preservation tool
-**Author:** [monster-penguin](https://github.com/monster-penguin)
-**Repository:** [github.com/monster-penguin/BIOS-Preservation-Tool](https://github.com/monster-penguin/BIOS-Preservation-Tool)
-**License:** MIT
+- **Author:** [monster-penguin](https://github.com/monster-penguin)
+- **Repository:** [github.com/monster-penguin/BIOS-Preservation-Tool](https://github.com/monster-penguin/BIOS-Preservation-Tool)
+- **License:** MIT
A local tool for managing your own BIOS collection. Point it at your files, it verifies them against RetroBIOS platform YAMLs, stores them in a deduplicated database, and produces ready-to-use directories for each platform. Supports scanning directories, archives, and URLs.
From 1518ca53adb936773e350c539c72345ad8f1b9d4 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:22:27 +0100
Subject: [PATCH 11/22] docs: say the legal answer once
A closing Summary restated the opening lines on the strongest and weakest
ground, and the not-legal-advice line came three times. The disclaimer
stays at the top, and the one sentence the summary added, the risk the
project accepts, moves to the encryption keys section.
---
tests/test_site_wiki.py | 7 +++++++
wiki/legal.md | 10 ++--------
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/tests/test_site_wiki.py b/tests/test_site_wiki.py
index 8ea952643..bf9873856 100644
--- a/tests/test_site_wiki.py
+++ b/tests/test_site_wiki.py
@@ -455,6 +455,13 @@ class LegalOpensInPlainWords(unittest.TestCase):
self.assertEqual(re.findall(r"(?m)^## ", detail), [])
self.assertIn("### Emulation and BIOS redistribution", detail)
+ def test_said_once(self):
+ """A closing Summary restated the opening lines, and "not legal
+ advice" came three times."""
+ self.assertEqual(self.text.count("legal advice"), 1)
+ self.assertNotIn("### Summary", self.text)
+ self.assertIn("The project accepts that risk deliberately", self.text)
+
def test_a_contact_section_names_only_a_channel_that_exists(self):
section = _section(self.text, "Contact")
self.assertIn("https://github.com/Abdess/retrobios/issues", section)
diff --git a/wiki/legal.md b/wiki/legal.md
index 9c0f9c10e..44cd9585a 100644
--- a/wiki/legal.md
+++ b/wiki/legal.md
@@ -18,7 +18,7 @@ It depends on your country. Nothing here is legal advice.
## The reasoning in detail
-Redistributing firmware is not settled law. Below is the reasoning the project acts on, with the strength of each argument, so anyone can weigh it. None of it is legal advice, and none of it has been tested in court.
+Redistributing firmware is not settled law. Below is the reasoning the project acts on, with the strength of each argument, so anyone can weigh it. None of it has been tested in court.
### Emulation and BIOS redistribution
@@ -29,7 +29,7 @@ Redistributing firmware is not settled law. Below is the reasoning the project a
### Encryption keys (Switch prod.keys, 3DS AES keys, Wii U keys)
-This is the most contested area, and the weakest part of the project's position.
+This is the most contested area, and the weakest part of the project's position. The project accepts that risk deliberately, in the belief that these files are worth preserving while they can still be obtained.
- **Keys are unlikely to be copyrightable.** A bare numeric value has no author's creative expression, so copyright is a poor fit. This says nothing about the DMCA, which is the provision that actually applies.
- **DMCA 1201(f) is narrow.** Section 1201(f) allows a person who has lawfully obtained the right to use a copy to circumvent for the sole purpose of achieving interoperability with an independently created program, and 1201(f)(3) allows the means to be made available to others only for that same purpose. Publishing keys openly is not obviously within that allowance, and the trafficking prohibitions in 1201(a)(2) and 1201(b) remain in play. This is the clearest legal risk the project carries.
@@ -43,12 +43,6 @@ This is the most contested area, and the weakest part of the project's position.
- **Functional necessity.** Emulators cannot run the software these systems were built for without this firmware. That is why the files are collected. It is a statement of purpose, not a legal defence in itself.
- **Yuzu context.** The Yuzu settlement (2024) concerned the emulator and its alleged facilitation of piracy. Yuzu settled without admitting liability, so it set no precedent either way on firmware or key redistribution. It does show that rights holders in this space litigate.
-### Summary
-
-This project preserves BIOS files, firmware and keys for personal use, archival and interoperability. The strongest ground is the older, discontinued firmware, where fair use and the absence of a market both point the same way. The weakest is the encryption keys, where DMCA 1201 applies and the interoperability exemption is narrower than the use made of it here. The project accepts that risk deliberately, in the belief that these files are worth preserving while they can still be obtained.
-
-This reflects the project's good-faith understanding, not legal advice.
-
## Asking for a file to be removed
A rights holder, or anyone acting for one, can [open an issue](https://github.com/Abdess/retrobios/issues) identifying the file by path or hash. A file whose removal is requested by its rights holder is removed from the repository and from the next release. No formal notice is required and none will be demanded. State the file and the basis of the claim. The same channel is open to anyone else with a concern about a specific file.
From fd3733fcaa7b87771b6be2266484ba1f58274ec6 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:23:29 +0100
Subject: [PATCH 12/22] docs: give the FAQ console answer a command
The answer said --region us keeps only your region's BIOS, with nothing
to run. On the PlayStation it keeps 44 files, Japanese PS2 images among
them (install.py --system sony-playstation --region us --dry-run). The
answer now gives the --core command the installer itself suggests, in
the tabs of each system, and says what --region drops.
---
tests/test_site_wiki.py | 13 +++++++++++++
wiki/faq.md | 9 ++++++++-
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/tests/test_site_wiki.py b/tests/test_site_wiki.py
index bf9873856..563628d88 100644
--- a/tests/test_site_wiki.py
+++ b/tests/test_site_wiki.py
@@ -426,6 +426,19 @@ def _entry(profile: dict, name: str) -> dict:
raise AssertionError(f"{name} not in the profile")
+class TheFaqAnswersWithACommand(unittest.TestCase):
+ """"Do I need every file of a console?" answered `--region us keeps only
+ your region's BIOS`, with no command to run, and that run kept 44 files
+ of the PlayStation set, Japanese PS2 images among them. `--core pcsx1`
+ is the lever the installer itself names."""
+
+ def test_every_file_of_a_console(self):
+ section = _section(_wiki("faq.md"), "Do I need every file of a console?")
+ self.assertIn("", section)
+ self.assertIn("--system sony-playstation --core pcsx1", section)
+ self.assertNotIn("only your region", section)
+
+
class LegalOpensInPlainWords(unittest.TestCase):
"""C5: Is it legal? is a Start here page and opened on case law."""
diff --git a/wiki/faq.md b/wiki/faq.md
index ff0781a70..70548bfc3 100644
--- a/wiki/faq.md
+++ b/wiki/faq.md
@@ -25,7 +25,14 @@ curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --dry-run
## Do I need every file of a console?
-No. `--region us` keeps only your region's BIOS.
+No. `--core` keeps what your emulator reads, here PCSX for PlayStation:
+
+
+```sh
+curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --system sony-playstation --core pcsx1
+```
+
+`--region us` drops another region's BIOS where a US one exists.
## Does the BIOS set the game's language?
From 556855421aec70eda38a41ea3914b25599fdbd5e Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:25:25 +0100
Subject: [PATCH 13/22] fix: keep release notes to checked claims
The v2026.10.10 notes said a pack holds every file its emulators open, so
a game never stops on a missing file, and that hashes never come from a
forum list: the gap analysis lists files no pack carries, and in md5
mode a pack keeps the platform's own md5 (verification-modes.md). The
generated text and the README now point to the gap analysis, and
release_notes.py refuses a whats-new text holding a judging or
contradicted phrase.
---
README.md | 4 ++--
scripts/generate_readme.py | 2 +-
scripts/release_notes.py | 26 ++++++++++++++++++++++++--
tests/test_pack_counts.py | 31 +++++++++++++++++++++++++++++++
wiki/release-process.md | 5 ++++-
5 files changed, 62 insertions(+), 6 deletions(-)
diff --git a/README.md b/README.md
index ed1f62cc9..3b4d52cbd 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ First time? Follow the [first install, step by step](https://abdess.github.io/re
## Download BIOS packs
-A pack holds everything its platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **Choose systems**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.
+A pack holds its platform's own BIOS list plus the files its emulator cores load. What no pack carries yet is in the [gap analysis](https://abdess.github.io/retrobios/gaps/). Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **Choose systems**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.
A pack over 2 GB comes in several parts (`Pack.part1of2.zip`, `Pack.part2of2.zip`), and every part is needed. Each part is an ordinary ZIP: extract them all into the same folder.
@@ -208,4 +208,4 @@ The scripts and tooling are released under the [MIT License](LICENSE).
The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed.
The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/).
-*Auto-generated on 2026-11-06T05:38:30Z*
+*Auto-generated on 2026-11-06T07:24:52Z*
diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py
index 5418e4889..85b5a9ad1 100644
--- a/scripts/generate_readme.py
+++ b/scripts/generate_readme.py
@@ -608,7 +608,7 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json"
"",
"## Download BIOS packs",
"",
- "A pack holds everything its platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **" + _installer_words().TUI_SYSTEMS_ROW + "**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.",
+ f"A pack holds its platform's own BIOS list plus the files its emulator cores load. What no pack carries yet is in the [gap analysis]({SITE_URL}gaps/). Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **" + _installer_words().TUI_SYSTEMS_ROW + "**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.",
"",
"A pack over 2 GB comes in several parts (`Pack.part1of2.zip`,"
" `Pack.part2of2.zip`), and every part is needed. Each part is an"
diff --git a/scripts/release_notes.py b/scripts/release_notes.py
index cf3903cae..dcf3613f9 100644
--- a/scripts/release_notes.py
+++ b/scripts/release_notes.py
@@ -7,7 +7,8 @@ issue 81 happened. Install commands, the parts sentence and the table come
from the record and the registry. Each row links the platform's page under
get/ on the site, which outlives the release, and the setups the registry
does not mark release_pack are named with the command. The maintainer writes
-what changed, and that text is appended as given.
+what changed, and that text is appended as given once it holds none of the
+REFUSED phrases.
Usage:
python scripts/release_notes.py --whats-new tmp/whats-new.md --output notes.md
@@ -102,6 +103,20 @@ def _link(name: str, pack: dict) -> str:
return f"[{label}]({release_record.get_page(_owner(name, pack), SITE)})"
+# Words the published notes of v2026.10.10 carried by hand: "never from a
+# forum list" judged lists other people keep, and "a game never stops on a
+# missing file" was a claim the gap analysis contradicts. Text about other
+# projects says what changed, not who erred (wiki/release-process.md).
+REFUSED = ("their file", "theirs", "of ours", "corrected", "wrong", "drift",
+ "left out", "never seen", "forum", "never stops", "never fails")
+
+
+def refused_words(text: str) -> list[str]:
+ """The refused phrases the text holds, case ignored."""
+ lowered = text.lower()
+ return [phrase for phrase in REFUSED if phrase in lowered]
+
+
def render(record: dict, platforms_dir: str, whats_new: str) -> str:
packs = record.get("packs", {})
folders = _extract_folders(record, platforms_dir)
@@ -124,7 +139,8 @@ def render(record: dict, platforms_dir: str, whats_new: str) -> str:
"### Download a pack",
"",
("One complete pack for each platform below: the BIOS list the platform"
- " publishes plus every file its emulators open. Each link is a page"
+ " publishes plus the files its emulators open. Files no pack carries yet"
+ f" are listed in the [gap analysis]({SITE}gaps/). Each link is a page"
" that keeps its address from one release to the next. Files is what the pack"
" holds once extracted, the count a file manager shows. Extracted size"
" is the space those files take, and the download is smaller."),
@@ -173,6 +189,12 @@ def main() -> int:
if not whats_new.strip():
print(f"Error: {args.whats_new} is empty", file=sys.stderr)
return 1
+ refused = refused_words(whats_new)
+ if refused:
+ print(f"Error: {args.whats_new} says {', '.join(repr(w) for w in refused)}. State what"
+ " changed and how it was checked, not a verdict on other projects or a"
+ " promise the gap analysis contradicts", file=sys.stderr)
+ return 1
write_text_atomic(str(args.output), render(record, args.platforms_dir, whats_new))
print(f"Wrote {args.output} for {record['tag']}, {len(record['packs'])} packs")
return 0
diff --git a/tests/test_pack_counts.py b/tests/test_pack_counts.py
index 461ed4942..e74ae2d14 100644
--- a/tests/test_pack_counts.py
+++ b/tests/test_pack_counts.py
@@ -11,12 +11,14 @@ installer fetches and leaves out the data directories a pack also carries.
from __future__ import annotations
import hashlib
+import io
import json
import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
+from unittest import mock
import yaml
@@ -552,6 +554,35 @@ class TheReleaseNotesReadTheRecord(unittest.TestCase):
def test_the_notes_never_open_on_the_collection(self):
self.assertNotIn("collection", self.render().split("### What changed")[0])
+ def test_the_notes_claim_only_what_the_gaps_allow(self):
+ """v2026.10.10 said a pack holds every file its emulators open, "so a
+ game never stops on a missing file", and that hashes never come
+ "from a forum list". The gap analysis lists what no pack carries."""
+ text = self.render().split("### What changed")[0]
+ self.assertNotIn("every file its emulators", text)
+ self.assertIn("https://abdess.github.io/retrobios/gaps/", text)
+ self.assertEqual(release_notes.refused_words(text), [])
+
+ def test_a_judging_whats_new_is_refused(self):
+ self.assertEqual(
+ release_notes.refused_words(
+ "- Every hash comes from the emulator's source, never from a forum list.\n"
+ "- So a game never stops on a missing file."),
+ ["forum", "never stops"])
+ with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+ whats_new = Path(tmp) / "whats-new.md"
+ whats_new.write_text("- Hashes never from a forum list\n", encoding="utf-8")
+ record = Path(tmp) / "release.json"
+ record.write_text(json.dumps(self.RECORD), encoding="utf-8")
+ argv = ["release_notes.py", "--record", str(record), "--whats-new", str(whats_new),
+ "--platforms-dir", str(REPO_ROOT / "platforms"),
+ "--output", str(Path(tmp) / "notes.md")]
+ with mock.patch.object(sys, "argv", argv), \
+ mock.patch("sys.stderr", new_callable=io.StringIO) as err:
+ self.assertEqual(release_notes.main(), 1)
+ self.assertIn("'forum'", err.getvalue())
+ self.assertFalse((Path(tmp) / "notes.md").exists())
+
def test_the_release_process_runs_it(self):
page = (REPO_ROOT / "wiki" / "release-process.md").read_text(encoding="utf-8")
self.assertIn("scripts/release_notes.py", page)
diff --git a/wiki/release-process.md b/wiki/release-process.md
index 250ad6ad7..3c031fab7 100644
--- a/wiki/release-process.md
+++ b/wiki/release-process.md
@@ -276,7 +276,10 @@ python scripts/generate_site.py
# 5. Write the notes: install commands, the parts sentence and the pack table
# come from release.json, and what changed since the previous tag is the
-# maintainer's own text in tmp/whats-new.md, appended as written.
+# maintainer's own text in tmp/whats-new.md, appended as written. The
+# script refuses a text that judges another project ("forum list") or
+# promises what the gap analysis contradicts ("never stops"): it says
+# what changed and how it was checked.
python scripts/release_notes.py --whats-new tmp/whats-new.md --output tmp/notes.md
# Create the release as a DRAFT, upload every asset, and only then publish it.
From 5f8205b36b39c46c4806dbef814125f455509c36 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:26:22 +0100
Subject: [PATCH 14/22] docs: date the Redump DATs instead of judging them
The catalog table called redump.org's DATs stale and said Dat-o-Matic
blocks automation. redump.org serves BIOS DATs dated 2010 to 2022
(Content-Disposition of /datfile/{psx,ps2,gc,xbox}-bios/, read
2026-11-06), and the Redump row named /static/bios links the scraper
replaced with one /bios/ route per system.
---
tests/test_doc_pages.py | 17 +++++++++++++++++
wiki/tools.md | 4 ++--
2 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py
index c29301dde..bb0d0d3eb 100644
--- a/tests/test_doc_pages.py
+++ b/tests/test_doc_pages.py
@@ -1986,3 +1986,20 @@ class ArchivedIsWhatTheToolsDo(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class TheCatalogRowsStateFacts(unittest.TestCase):
+ """tools.md said "redump.org serves stale DATs" and "Dat-o-Matic blocks
+ automation", verdicts on two preservation projects, and gave Redump's
+ /static/bios links, which the scraper no longer reads."""
+
+ def test_dates_and_routes_not_verdicts(self):
+ text = (WIKI / "tools.md").read_text(encoding="utf-8")
+ rows = [line for line in text.splitlines()
+ if line.startswith(("| Redump |", "| No-Intro |"))]
+ self.assertEqual(len(rows), 2)
+ for word in ("stale", "blocks"):
+ self.assertNotIn(word, " ".join(rows))
+ scraper = (ROOT / "scripts" / "scraper" / "redump_dat_scraper.py").read_text(encoding="utf-8")
+ self.assertIn("/bios/", scraper)
+ self.assertIn("`/bios/`", rows[0])
diff --git a/wiki/tools.md b/wiki/tools.md
index dab60e5b0..c7dfae1dc 100644
--- a/wiki/tools.md
+++ b/wiki/tools.md
@@ -1643,8 +1643,8 @@ names the profiles read.
| Catalog | Acquisition | Notes |
|---------|------------|-------|
-| Redump | direct fetch from redump.info `/static/bios` | 4 BIOS DATs. redump.org serves stale DATs |
-| No-Intro | local pack, imported with `dat_pack_importer` | Dat-o-Matic blocks automation, and the `hugo19941994/auto-datfile-generator` mirror rebuilds daily |
+| Redump | direct fetch from redump.info, one `/bios/` route per system | 4 BIOS DATs. redump.org serves BIOS DATs dated 2010 to 2022, redump.info the current ones |
+| No-Intro | local pack, imported with `dat_pack_importer` | Dat-o-Matic serves its DATs through a browser session, and the `hugo19941994/auto-datfile-generator` mirror rebuilds daily |
| TOSEC | local pack, imported with `dat_pack_importer` | annual pack from tosecdev.org, 138 Firmware DATs |
The join is by SHA1, then by MD5 + size. A match sets the `provenance` field
From da9e1f568cc0aa0cece1da6ee67e88536edfd153 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:27:32 +0100
Subject: [PATCH 15/22] fix: refuse an offline export with no originals
pipeline.py --offline --with-export, listed as a standard command, passes
no --fetch to step 2e: on a fresh clone the database, verify and truth
were built before the export failed on every platform whose own file it
patches. The run is refused before step 0 when .cache/upstream-native/
holds nothing for such a platform, and tools.md says how to fill it.
---
scripts/pipeline.py | 27 ++++++++++++++++++++++++++
tests/test_tool_messages.py | 38 +++++++++++++++++++++++++++++++++++++
wiki/advanced-usage.md | 3 ++-
wiki/tools.md | 7 +++++++
4 files changed, 74 insertions(+), 1 deletion(-)
diff --git a/scripts/pipeline.py b/scripts/pipeline.py
index 2c2f23ede..90976b050 100644
--- a/scripts/pipeline.py
+++ b/scripts/pipeline.py
@@ -883,6 +883,32 @@ def _refuse_offline_without_caches(
)
+def _refuse_offline_export_without_originals(
+ parser: argparse.ArgumentParser, args: argparse.Namespace,
+ cache: str = ".cache/upstream-native",
+) -> None:
+ """An offline run with --with-export needs each patched file cached.
+
+ Seven formats carry code, so step 2e patches the platform's own file,
+ which --offline does not fetch. On a fresh clone the database, verify
+ and the truth model were built before step 2e failed on every one.
+ """
+ if not (args.offline and args.with_export):
+ return
+ from exporter import discover_exporters # noqa: PLC0415
+
+ lacking = sorted(
+ name for name, exporter in discover_exporters().items()
+ if exporter.needs_original() and not any(Path(cache, name).glob("*"))
+ )
+ if lacking:
+ parser.error(
+ f"--with-export patches each platform's own file, and {cache}/ holds none "
+ f"for {', '.join(lacking)}: run python scripts/export_native.py --all "
+ "--refresh-cache once, or drop --offline"
+ )
+
+
def _truth_steps(args: argparse.Namespace, results: dict) -> bool:
"""Steps 2c to 2e: truth model, its diff, the native export."""
all_ok = True
@@ -1010,6 +1036,7 @@ def main() -> None:
args = parser.parse_args()
_refuse_unapplied_flags(parser, args)
_refuse_offline_without_caches(parser, args)
+ _refuse_offline_export_without_originals(parser, args)
if not args.offline:
# Read once and put in the environment every step inherits: the
# scrapers and refresh_data_dirs went anonymous unless the user had
diff --git a/tests/test_tool_messages.py b/tests/test_tool_messages.py
index 7f043a22b..3f1a8abbd 100644
--- a/tests/test_tool_messages.py
+++ b/tests/test_tool_messages.py
@@ -175,6 +175,44 @@ class AnOfflineRunWithNoCacheStopsFirst(unittest.TestCase):
parser.error.assert_not_called()
+class AnOfflineExportWithNoOriginalsStopsFirst(unittest.TestCase):
+ """pipeline.py --offline --with-export, which tools.md listed as a
+ standard command, built the database, verify and the truth model on a
+ fresh clone, then failed step 2e on every platform whose own file it
+ patches: --offline never passes --fetch."""
+
+ def _args(self, **flags):
+ import argparse
+
+ return argparse.Namespace(**{"offline": True, "with_export": True, **flags})
+
+ def test_refused_before_any_step(self):
+ from exporter import discover_exporters
+
+ patched = sorted(n for n, e in discover_exporters().items() if e.needs_original())
+ self.assertIn("batocera", patched)
+ with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+ parser = mock.Mock()
+ pipeline._refuse_offline_export_without_originals(parser, self._args(), tmp)
+ message = parser.error.call_args.args[0]
+ self.assertIn("export_native.py --all --refresh-cache", message)
+ self.assertIn("batocera", message)
+ for args in (self._args(offline=False), self._args(with_export=False)):
+ parser = mock.Mock()
+ pipeline._refuse_offline_export_without_originals(parser, args, tmp)
+ parser.error.assert_not_called()
+ for name in patched:
+ (Path(tmp) / name).mkdir()
+ (Path(tmp) / name / "original").write_text("x")
+ parser = mock.Mock()
+ pipeline._refuse_offline_export_without_originals(parser, self._args(), tmp)
+ parser.error.assert_not_called()
+
+ def test_the_tools_page_says_so(self):
+ text = (REPO_ROOT / "wiki" / "tools.md").read_text(encoding="utf-8")
+ self.assertIn("export_native.py --all --refresh-cache", text)
+
+
class TheTruthDiffShowsWhereAndAsksNothing(unittest.TestCase):
"""The markdown wrote `**Remove** hiscore.dat (phantom)` beside six
`**Add** hiscore.dat`, none with a path, for files profiled cores read."""
diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md
index c17cc07ac..d54969839 100644
--- a/wiki/advanced-usage.md
+++ b/wiki/advanced-usage.md
@@ -534,7 +534,8 @@ Include truth generation and diffing in the pipeline:
python scripts/pipeline.py --offline --with-truth
```
-Include truth + native format export:
+Include truth + native format export. Offline, the platforms' own files
+must already be cached ([tools](tools.md#pipeline)):
```bash
python scripts/pipeline.py --offline --with-export
diff --git a/wiki/tools.md b/wiki/tools.md
index c7dfae1dc..2c4b4290a 100644
--- a/wiki/tools.md
+++ b/wiki/tools.md
@@ -28,6 +28,13 @@ python scripts/pipeline.py --check-buildbot # online run + buildbot fresh
`python scripts/refresh_data_dirs.py` once first. A run that builds packs with no
data directory cached is refused before step 0.
+`--with-export` online passes `--fetch` to step 2e, which downloads each
+platform's own file and patches it. Offline that file must already be in
+`.cache/upstream-native/`: on a fresh clone run
+`python scripts/export_native.py --all --refresh-cache` once first. An offline
+run with `--with-export` and an empty cache for any platform it patches is
+refused before step 0.
+
Pipeline steps:
| Step | Description | Skipped by |
From f255efca0ffc726dfa371c342595b9ef524fb4db Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:27:59 +0100
Subject: [PATCH 16/22] docs: list pipeline flag refusals in one table
tools.md said --full-narrowings is refused with --skip-packs only, while
pipeline.py also refuses it with --source platform or truth, and
advanced-usage.md kept a second, shorter list. The tools table now
carries every pair, checked against _refuse_unapplied_flags, and
advanced-usage links to it.
---
tests/test_tool_messages.py | 39 +++++++++++++++++++++++++++++++++++++
wiki/advanced-usage.md | 2 +-
wiki/tools.md | 2 +-
3 files changed, 41 insertions(+), 2 deletions(-)
diff --git a/tests/test_tool_messages.py b/tests/test_tool_messages.py
index 3f1a8abbd..1ba332604 100644
--- a/tests/test_tool_messages.py
+++ b/tests/test_tool_messages.py
@@ -213,6 +213,45 @@ class AnOfflineExportWithNoOriginalsStopsFirst(unittest.TestCase):
self.assertIn("export_native.py --all --refresh-cache", text)
+class ThePipelineFlagTableSaysEveryRefusal(unittest.TestCase):
+ """tools.md said --full-narrowings is refused with --skip-packs only, and
+ advanced-usage.md kept its own shorter list: `--source truth
+ --full-narrowings` failed with nothing on either page saying so."""
+
+ PAIRS = {"--source": "platform", "--all-variants": True, "--full-narrowings": True,
+ "--skip-packs": True}
+
+ def _refused(self, flags: dict) -> bool:
+ import argparse
+
+ args = argparse.Namespace(source=None, all_variants=False, full_narrowings=False,
+ skip_packs=False)
+ for flag, value in flags.items():
+ setattr(args, flag.lstrip("-").replace("-", "_"), value)
+ parser = mock.Mock()
+ pipeline._refuse_unapplied_flags(parser, args)
+ return parser.error.called
+
+ def test_each_refused_pair_is_in_the_row_of_a_flag(self):
+ import itertools
+
+ text = (REPO_ROOT / "wiki" / "tools.md").read_text(encoding="utf-8")
+ rows = {line.split("`")[1].split()[0]: line for line in text.splitlines()
+ if line.startswith("| `--")}
+ for (a, va), (b, vb) in itertools.combinations(self.PAIRS.items(), 2):
+ if not self._refused({a: va, b: vb}):
+ continue
+ with self.subTest(pair=(a, b)):
+ self.assertTrue(f"`{b}" in rows.get(a, "") or f"`{a}" in rows.get(b, ""),
+ f"{a} with {b} is refused, and neither row says so")
+ self.assertIn("`--source truth`", rows["--full-narrowings"])
+
+ def test_one_list_of_refusals(self):
+ text = (REPO_ROOT / "wiki" / "advanced-usage.md").read_text(encoding="utf-8")
+ self.assertNotIn("are refused with", text)
+ self.assertIn("tools.md#pipeline", text)
+
+
class TheTruthDiffShowsWhereAndAsksNothing(unittest.TestCase):
"""The markdown wrote `**Remove** hiscore.dat (phantom)` beside six
`**Add** hiscore.dat`, none with a path, for files profiled cores read."""
diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md
index d54969839..9f230fe2e 100644
--- a/wiki/advanced-usage.md
+++ b/wiki/advanced-usage.md
@@ -543,7 +543,7 @@ python scripts/pipeline.py --offline --with-export
### Combining flags
-These flags compose (`--source` and `--all-variants` are refused with `--skip-packs`):
+These flags compose, except the pairs the [pipeline flags](tools.md#pipeline) table marks as refused:
```bash
python scripts/pipeline.py --offline --skip-packs --skip-docs --with-truth --target switch
diff --git a/wiki/tools.md b/wiki/tools.md
index 2c4b4290a..0e62fec66 100644
--- a/wiki/tools.md
+++ b/wiki/tools.md
@@ -76,7 +76,7 @@ Pipeline flags:
| `--target TARGET` | filter verify, packs and truth by hardware target |
| `--source {platform,truth,full}` | pack file source, passed through to `generate_pack.py`. Refused with `--skip-packs` |
| `--all-variants` | build the 6 source x required combinations. Refused with `--skip-packs` and with `--source` |
-| `--full-narrowings` | step 5 compares every region, target and system of each platform instead of three of each. Refused with `--skip-packs` |
+| `--full-narrowings` | step 5 compares every region, target and system of each platform instead of three of each. Refused with `--skip-packs` and with `--source platform` or `--source truth` |
| `--check-buildbot` | run step 2b. Additive, and ignored under `--offline` |
| `--with-truth` | run steps 2c and 2d |
| `--with-export` | run steps 2c, 2d and 2e |
From 328bfbe53b505a5227a71afc574805f6e629b09d Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:37:16 +0100
Subject: [PATCH 17/22] chore: run the catalog row test from its file
The class sat after unittest.main(), so running test_doc_pages.py on its
own skipped it.
---
tests/test_doc_pages.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py
index bb0d0d3eb..5e6eb2c96 100644
--- a/tests/test_doc_pages.py
+++ b/tests/test_doc_pages.py
@@ -1984,9 +1984,6 @@ class ArchivedIsWhatTheToolsDo(unittest.TestCase):
self.assertNotIn("list_platforms", workflow.read_text(encoding="utf-8"))
-if __name__ == "__main__":
- unittest.main()
-
class TheCatalogRowsStateFacts(unittest.TestCase):
"""tools.md said "redump.org serves stale DATs" and "Dat-o-Matic blocks
@@ -2003,3 +2000,8 @@ class TheCatalogRowsStateFacts(unittest.TestCase):
scraper = (ROOT / "scripts" / "scraper" / "redump_dat_scraper.py").read_text(encoding="utf-8")
self.assertIn("/bios/", scraper)
self.assertIn("`/bios/`", rows[0])
+
+
+if __name__ == "__main__":
+ unittest.main()
+
From 23d878c52673eb9080320b2596117cca0919cd82 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 08:37:19 +0100
Subject: [PATCH 18/22] chore: stub the export cache check in a test
test_a_targeted_export_reads_a_full_model runs pipeline.py --offline
--with-export, which is now refused when .cache/upstream-native/ is
empty, as in any fresh checkout. The test is about the truth directory
the export reads.
---
tests/test_audit_regressions.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index 5263d0c54..6776263b8 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -1140,9 +1140,12 @@ class PipelineRegressions(unittest.TestCase):
argv = ["pipeline.py", "--offline", "--skip-packs", "--skip-docs",
"--with-export", "--target", "switch"]
+ # The originals the offline export patches are not cached in a
+ # checkout: the run would be refused before any step.
with mock.patch.object(sys, "argv", argv), mock.patch.object(
pipeline, "run", side_effect=fake_run
- ), contextlib.redirect_stdout(io.StringIO()), contextlib.suppress(SystemExit):
+ ), mock.patch.object(pipeline, "_refuse_offline_export_without_originals"), \
+ contextlib.redirect_stdout(io.StringIO()), contextlib.suppress(SystemExit):
pipeline.main()
truth_runs = [c for c in commands if "scripts/generate_truth.py" in c]
export = next(c for c in commands if "scripts/export_native.py" in c)
From ef6607302d333d2e3b98592062d2fd6bbc21a59c Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 09:11:02 +0100
Subject: [PATCH 19/22] chore: rebuild the site when siteinputs changes
Every page module reads its inputs from scripts/siteinputs.py, so the
deploy workflow names it with the other site modules.
---
.github/workflows/deploy-site.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml
index 145b8dd4b..67130fe8f 100644
--- a/.github/workflows/deploy-site.yml
+++ b/.github/workflows/deploy-site.yml
@@ -49,6 +49,7 @@ on:
- "scripts/sitediagrams.py"
- "scripts/siteentries.py"
- "scripts/siteguides.py"
+ - "scripts/siteinputs.py"
- "scripts/sitemarkup.py"
- "scripts/sitemeasurements.py"
- "scripts/siteexports.py"
From 856cb5618fcbed70c834a74da1b8be521e25bce5 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 09:14:42 +0100
Subject: [PATCH 20/22] fix: link a build file only where it is recorded
Every source link on the site leads to a path the source record saw at
a commit. The Build file row linked pin_build files unchecked, eight of
them at commits the record did not hold. build_source_paths now records
the build files read at a commit, and the row links only those: a tag
can move and stays plain text. The record run also picked up two cited
paths it had not seen before (pom2 CMakeLists.txt, pcem configure.ac).
---
scripts/generate_site.py | 13 ++++++++-----
scripts/sitesources.py | 19 +++++++++++++++++++
tests/test_audit_regressions.py | 27 +++++++++++++++++++++++----
website/src/data/source-paths.json | 14 ++++++++++++++
4 files changed, 64 insertions(+), 9 deletions(-)
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index fe43620af..127c8aa3e 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -8198,17 +8198,20 @@ def _exact_size(size: int) -> str:
def _build_file_links(pin_build: list[dict]) -> list[str]:
"""Each platform build file that holds the revision, linked at the
- revision it was read at. The prose of pin_frozen named the file as
- plain text, while pin_build carries all a pinned link needs."""
- from sitesources import _browse_repo, _source_permalink # noqa: PLC0415
+ commit it was read at when the source record holds it. The prose of
+ pin_frozen named the file as plain text."""
+ from sitesources import _browse_repo, _source_permalink, tree_holds # noqa: PLC0415
links = []
for build in pin_build:
repo = _browse_repo(str(build["repo"]))
path = str(build["path"])
+ revision = str(build["revision"])
text = f"{_escape_tags(_platform_display(str(build['platform'])))} {_code(path)}"
- if repo is not None:
- text = f"[{text}]({_source_permalink(repo, str(build['revision']), path, None, None)})"
+ # Linked where the record saw the file at that commit, as every
+ # source link is: a tag can move, and is never recorded.
+ if repo is not None and tree_holds(repo, revision, path):
+ text = f"[{text}]({_source_permalink(repo, revision, path, None, None)})"
links.append(f"{text} at {_code(str(build['revision']))} sets {_code(str(build['variable']))}")
return links
diff --git a/scripts/sitesources.py b/scripts/sitesources.py
index f42de04ae..c28b7c5f1 100644
--- a/scripts/sitesources.py
+++ b/scripts/sitesources.py
@@ -321,9 +321,28 @@ def build_source_paths(
# Recorded even when it holds no cited path, so a pinned tree
# missing from the record is one no run has read.
held.setdefault(tree_key(repo, pin), set()).update(paths & tree)
+ for repo, pin, path in build_files(profile):
+ tree = (_cached_tree(repo, pin, cache_dirs, fetch)
+ or _listed_directories(repo, pin, {path}, cache_dirs, fetch))
+ if tree is None:
+ unread.add(tree_key(repo, pin))
+ continue
+ held.setdefault(tree_key(repo, pin), set()).update({path} & tree)
return {key: sorted(paths) for key, paths in sorted(held.items())}, sorted(unread)
+def build_files(profile: dict) -> list[tuple[upstream.Repo, str, str]]:
+ """The platform build files `pin_build` names at a commit, as (repo,
+ revision, path). A tag can move, so a build read at a tag is not linked."""
+ found = []
+ for build in profile.get("pin_build") or []:
+ repo = _browse_repo(str(build.get("repo", "")))
+ revision = str(build.get("revision", ""))
+ if repo is not None and re.fullmatch(r"[0-9a-f]{40}", revision):
+ found.append((repo, revision, str(build.get("path", ""))))
+ return found
+
+
def keep_unread(
held: dict[str, list[str]], unread: list[str], previous: dict[str, list[str]]
) -> dict[str, list[str]]:
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index 6776263b8..da75706d6 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -5222,11 +5222,30 @@ class AHeldRevisionLinksItsBuildFile(unittest.TestCase):
"platform": "knulli", "repo": "https://github.com/knulli-cfw/knulli-linux",
"revision": "20260511", "path": "package/emulators/advanced_drastic/advanced_drastic.mk",
"variable": "ADVANCED_DRASTIC_VERSION", "value": "eaf41ab9"}]})
+ # A tag can move: named, never linked.
self.assertIn(
- "| Build file | [KNULLI `package/emulators/advanced_drastic/advanced_drastic.mk`]"
- "(https://github.com/knulli-cfw/knulli-linux/blob/20260511/package/emulators/"
- "advanced_drastic/advanced_drastic.mk) at `20260511` sets `ADVANCED_DRASTIC_VERSION` |",
- rows)
+ "| Build file | KNULLI `package/emulators/advanced_drastic/advanced_drastic.mk` "
+ "at `20260511` sets `ADVANCED_DRASTIC_VERSION` |", rows)
+
+ def test_a_build_read_at_a_recorded_commit_links(self):
+ from scripts.generate_site import _build_rows
+
+ pin = "46128cd252cdecae852a4d0bafa2aaf9d03ab563"
+ rows = _build_rows({"pin_build": [{
+ "platform": "retropie", "repo": "https://github.com/RetroPie/RetroPie-Setup",
+ "revision": pin, "path": "scriptmodules/emulators/advmame-1.4.sh",
+ "variable": "rp_module_repo", "value": "x"}]})
+ self.assertIn(
+ f"[RetroPie `scriptmodules/emulators/advmame-1.4.sh`](https://github.com/RetroPie/"
+ f"RetroPie-Setup/blob/{pin}/scriptmodules/emulators/advmame-1.4.sh)", rows[0])
+
+ def test_the_record_holds_the_build_files(self):
+ import sitesources # noqa: PLC0415
+
+ files = sitesources.build_files({"pin_build": [
+ {"repo": "https://github.com/o/r", "revision": "a" * 40, "path": "p.mk"},
+ {"repo": "https://github.com/o/r", "revision": "v1", "path": "q.mk"}]})
+ self.assertEqual([(f[1], f[2]) for f in files], [("a" * 40, "p.mk")])
class AVariantCardSaysWhatToSaveItAs(unittest.TestCase):
diff --git a/website/src/data/source-paths.json b/website/src/data/source-paths.json
index d591448f7..d0f00e7a2 100644
--- a/website/src/data/source-paths.json
+++ b/website/src/data/source-paths.json
@@ -424,6 +424,11 @@
"crates/wqxemu/src/keypad.rs",
"crates/wqxemu/src/main.rs"
],
+ "github.com/AmberELEC/AmberELEC@a21975bfb1a3af68f66e07a931b994b523e1e6e2": [
+ "packages/games/emulators/lzdoom/package.mk",
+ "packages/games/emulators/piemu/package.mk",
+ "packages/games/libretro/flycast2021le/package.mk"
+ ],
"github.com/Amjad50/plastic@74152adb3acd890d3abee2ff5906408f26a199e2": [
"plastic_core/src/cartridge/mod.rs",
"plastic_core/src/display/color.rs",
@@ -1584,6 +1589,13 @@
"imagine/src/font/FreetypeFont.cc",
"imagine/src/fs/ArchiveFS.cc"
],
+ "github.com/RetroPie/RetroPie-Setup@46128cd252cdecae852a4d0bafa2aaf9d03ab563": [
+ "scriptmodules/emulators/advmame-1.4.sh",
+ "scriptmodules/emulators/ppsspp-1.5.4.sh",
+ "scriptmodules/ports/alephone.sh",
+ "scriptmodules/ports/love-0.10.2.sh",
+ "scriptmodules/ports/lzdoom.sh"
+ ],
"github.com/RetroPie/mame4all-pi@c6bfd2aef331b5a5e157c15d8bf8a77393c85689": [
"readme.txt",
"src/cheat.cpp",
@@ -3475,6 +3487,7 @@
"src/ui/video.rs"
],
"github.com/habib256/pom2@de86e8e35df96fe6ffced56dfb727bbd567f8568": [
+ "CMakeLists.txt",
"packaging/bundle.manifest",
"packaging/fujinet/stage.py",
"src/AppleParallelCard.cpp",
@@ -8930,6 +8943,7 @@
"nes.c"
],
"github.com/sarah-walker-pcem/pcem@10739b9618e388a5bf8c21e752f12b3952fa83e1": [
+ "configure.ac",
"src/mem.c",
"src/nvr.c",
"src/ps1.c",
From adcfa283fb125a2bd853c4517a78a77e9aa0e5c7 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 09:14:47 +0100
Subject: [PATCH 21/22] docs: keep the FAQ under its word ceiling
The --core answer added 14 words to a page held at 450. It now names the
command in as many words as the line it replaced, and the --region line
goes: what --region keeps is on the installer page.
---
wiki/faq.md | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/wiki/faq.md b/wiki/faq.md
index 70548bfc3..f3429d353 100644
--- a/wiki/faq.md
+++ b/wiki/faq.md
@@ -25,15 +25,13 @@ curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --dry-run
## Do I need every file of a console?
-No. `--core` keeps what your emulator reads, here PCSX for PlayStation:
+No. `--core pcsx1` keeps what PCSX reads:
```sh
curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --system sony-playstation --core pcsx1
```
-`--region us` drops another region's BIOS where a US one exists.
-
## Does the BIOS set the game's language?
Rarely. It comes from the game. Some follow the console's setting, like Flycast's **Language**
From 1396350795df5b80bc28b81b60e5e6d28f232baa Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Fri, 6 Nov 2026 10:14:02 +0100
Subject: [PATCH 22/22] fix: count build file trees as pinned
check_source_paths read the trees profiles pin from source and upstream
only, so the two platform trees the record now holds for pin_build
files read as orphaned and the record as stale.
---
scripts/check_freshness.py | 5 +++++
tests/test_site_integration.py | 15 +++++++++++++++
2 files changed, 20 insertions(+)
diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py
index b8e03f28b..075c45641 100644
--- a/scripts/check_freshness.py
+++ b/scripts/check_freshness.py
@@ -1492,6 +1492,11 @@ def check_source_paths(profiles: dict) -> Finding:
sitesources.tree_key(repo, pin)
for profile in profiles.values() if sitesources.cited_paths(profile)
for repo, pin in sitesources.profile_trees(profile)
+ } | {
+ # The platform build files pin_build names, linked from a page too.
+ sitesources.tree_key(repo, pin)
+ for profile in profiles.values()
+ for repo, pin, _path in sitesources.build_files(profile)
}
try:
recorded = set(json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")))
diff --git a/tests/test_site_integration.py b/tests/test_site_integration.py
index 91a3dad0d..a3371662a 100644
--- a/tests/test_site_integration.py
+++ b/tests/test_site_integration.py
@@ -117,6 +117,21 @@ class SourcePathsFreshness(unittest.TestCase):
finding = check_freshness.check_source_paths(profiles)
self.assertEqual(finding.status, check_freshness.OK, finding.detail)
+ def test_a_recorded_build_file_tree_is_pinned(self):
+ """The record holds the platform trees pin_build reads at a commit,
+ and the check called them orphaned."""
+ import sitesources # noqa: PLC0415
+
+ pin = "a" * 40
+ profile = {"pin_build": [{"repo": "https://github.com/o/r", "revision": pin,
+ "path": "p.mk"}]}
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp) / "source-paths.json"
+ path.write_text(json.dumps({f"github.com/o/r@{pin}": ["p.mk"]}), encoding="utf-8")
+ with mock.patch.object(sitesources, "SOURCE_PATHS", path):
+ finding = check_freshness.check_source_paths({"p": profile})
+ self.assertEqual(finding.status, check_freshness.OK, finding.detail)
+
def test_a_moved_pin_is_stale(self):
profile = _profile("geolith")
moved = dict(profile, source_commit="f" * 40, upstream_commit="e" * 40)