chore: merge site and docs fixes, third pass

scripts/check_freshness.py

@@ -80,6 +80,9 @@ TOSEC_DOWNLOADS = "https://www.tosecdev.org/downloads"
 MAME_REPO = "https://github.com/mamedev/mame"
 FBNEO_REPO = "https://github.com/libretro/FBNeo"
 PYPI_URL = "https://pypi.org/pypi/{name}/json"
+NPM_URL = "https://registry.npmjs.org/{name}/latest"
+# The site's npm dependencies, pinned to exact versions and built by Deploy Site.
+SITE_PACKAGE = REPO_ROOT / "website" / "package.json"
 INSTALLERS = ("install.sh", "install.ps1")
 
 SHOWN_ITEMS = 4  # items named in a diff summary before ", ..."
@@ -1356,6 +1359,53 @@ def pypi_latest(name: str) -> str | None:
     return str(info.get("version") or "") or None
 
 
+def parse_npm_pins(text: str) -> dict[str, str]:
+    """Package -> the version a package.json names, dependencies of both kinds."""
+    data = json.loads(text)
+    pins: dict[str, str] = {}
+    for key in ("dependencies", "devDependencies"):
+        pins.update({str(k): str(v) for k, v in (data.get(key) or {}).items()})
+    return pins
+
+
+def npm_latest(name: str) -> str | None:
+    payload = upstream._http_json(NPM_URL.format(name=name))  # noqa: SLF001
+    if not isinstance(payload, dict):
+        return None
+    return str(payload.get("version") or "") or None
+
+
+def _npm_findings() -> list[Finding]:
+    """One row per site dependency: an exact pin is stale once npm's latest
+    release is newer, since nothing else asks npm."""
+    findings = []
+    for name, pinned in sorted(parse_npm_pins(SITE_PACKAGE.read_text(encoding="utf-8")).items()):
+        subject = f"npm:{name}"
+        if not re.fullmatch(r"\d+(\.\d+)*", pinned):
+            findings.append(Finding("ci", subject, UNKNOWN, local=pinned, detail="a range, not an exact pin"))
+            continue
+        try:
+            latest = npm_latest(name)
+        except upstream.UpstreamError as exc:
+            findings.append(Finding("ci", subject, UNKNOWN, local=pinned, detail=str(exc)))
+            continue
+        if latest is None:
+            findings.append(Finding("ci", subject, UNKNOWN, local=pinned, detail="not on npm"))
+            continue
+        newer = _version_tuple(latest) > _version_tuple(pinned)
+        findings.append(
+            Finding(
+                "ci",
+                subject,
+                STALE if newer else OK,
+                local=pinned,
+                remote=latest,
+                detail="a newer release is out" if newer else "",
+            )
+        )
+    return findings
+
+
 def check_ci(cache_dir: str, offline: bool) -> list[Finding]:
     findings = [_check_installer_pins()]
     workflows = sorted((REPO_ROOT / ".github" / "workflows").glob("*.yml"))
@@ -1367,8 +1417,9 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]:
             pins.setdefault(name, spec)
         actions.update(parse_action_pins(text))
     if offline:
-        findings.append(Finding("ci", "workflow pins", SKIPPED, detail="offline"))
+        findings.append(Finding("ci", "workflow and site pins", SKIPPED, detail="offline"))
         return findings
+    findings.extend(_npm_findings())
     for name, spec in sorted(pins.items()):
         try:
             latest = pypi_latest(name)

scripts/generate_db.py

@@ -35,6 +35,7 @@ from common import (
     write_text_atomic,
     yaml_load,
 )
+import retired_files
 
 # Platform and profile aliases are read from YAML: without pyyaml the
 # database lost every one of them and the run still exited 0.
@@ -489,6 +490,19 @@ def _generate(args: argparse.Namespace) -> int:
         "indexes": indexes,
     }
 
+    # A sha1 the rebuild drops keeps its published address: the record
+    # beside the database says when it left and what holds its place.
+    output = Path(args.output)
+    if output.is_file():
+        previous = json.loads(output.read_text(encoding="utf-8"))
+        left = retired_files.update(
+            retired_files.retired_path(output), previous, database,
+            database["generated_at"][:10],
+        )
+        if left:
+            print(f"  Left the collection: {left} files, recorded in "
+                  f"{retired_files.retired_path(output)}")
+
     new_content = json.dumps(database, indent=2)
     written = write_if_changed(args.output, new_content)
 

scripts/generate_pack.py

@@ -3672,9 +3672,9 @@ def _main(held: contextlib.ExitStack) -> None:
     # Platform mode
     if args.all and args.manifest:
         # Install manifests cover every registered platform: the installer
-        # serves each one, with or without a release pack. "Archived" means
-        # upstream is no longer scraped on a schedule, not that the platform
-        # lost its users.
+        # serves each one, with or without a release pack. "Archived" keeps a
+        # platform out of the runs that cover every platform, not out of the
+        # installer: it has not lost its users.
         platforms = list_registered_platforms(args.platforms_dir, include_archived=True)
     elif args.all:
         # A release publishes the packs the registry marks release_pack and

scripts/generate_readme.py

@@ -588,7 +588,7 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json"
         "# Android (Termux: it installs Python and asks for storage access)",
         install_command("sh"),
         "",
-        "# SD card mounted on a PC: a muOS, KNULLI, ROCKNIX, Onion, spruceOS or MinUI card",
+        "# SD card mounted on a PC: a muOS, KNULLI, ROCKNIX, Lakka, MiSTer, Onion, spruceOS or MinUI card",
         "# is found by itself. For another card, --dest is its BIOS folder",
         "# (RetroArch on Switch: retroarch/cores/system, on Android: RetroArch/system)",
         install_command("sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system"),
@@ -602,6 +602,10 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json"
         " files and checks each one. It can also copy the BIOS files from your old SD card.",
         f"Every option, and what the installer detects and writes, is on the [installer page]({SITE_URL}wiki/installer/#passing-options).",
         "",
+        f"First time? Follow the [first install, step by step]({SITE_URL}wiki/first-install/)."
+        f" A game does not start: [what to check]({SITE_URL}wiki/troubleshooting/)."
+        f" Other questions: the [FAQ]({SITE_URL}wiki/faq/).",
+        "",
         "## Download BIOS packs",
         "",
         "A pack holds everything its platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **" + _installer_words().TUI_SYSTEMS_ROW + "**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.",
@@ -647,8 +651,8 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json"
         lines.extend(
             [
                 "",
-                "\\* Archived: the configuration is kept and the installer serves it,"
-                " but upstream is no longer scraped on a schedule.",
+                "\\* Archived: the installer serves it, and checks that cover every"
+                " setup include it only on request.",
             ]
         )
 
@@ -986,7 +990,7 @@ the checks CI runs on a pull request (Python 3.12 or newer):
 pip install pyyaml "jsonschema[format-nongpl]" markdown pyte hypothesis vermin setuptools
 python scripts/validate_pr.py --changed          # the BIOS files the branch adds
 python scripts/validate_schemas.py --source-only # profiles and platform files
-python scripts/profile_sync.py --added --base upstream/main --strict  # each profile the branch adds
+python scripts/profile_sync.py --added --strict  # each profile the branch adds
 python scripts/refresh_data_dirs.py  # data directories the pack-building tests read
 python scripts/generate_site.py       # the pages the site tests read
 npm ci --prefix website && npm run build --prefix website  # the built site, Node 24
@@ -995,10 +999,9 @@ python scripts/validate_site.py --site-dir website/dist  # links and page budget
 vermin --no-tips -t=3.8- --violations install.py         # install.py stays on Python 3.8
 ```
 
-`upstream` is this repository as a remote of your fork
-(`git remote add upstream https://github.com/Abdess/retrobios`, then
-`git fetch upstream`). A profile you only edit is not held to `--strict`: its
-older refs are reviewed by hand.
+`profile_sync.py` compares the branch with this repository's `main`, and
+fetches it in a fork's clone. A profile you only edit is not held to
+`--strict`: its older refs are reviewed by hand.
 
 The full pipeline is not needed for a contribution: it rebuilds every pack
 in `dist/`, which is why maintainers run it before a release.

scripts/generate_site.py

@@ -47,6 +47,7 @@ from common import (
     GAME_DATA_TOPS,
     compute_composition,
     count_catalog_matched,
+    entry_system_ids,
     list_registered_platforms,
     list_release_pack_platforms,
     load_platform_registry,
@@ -88,6 +89,7 @@ from generate_readme import (
 from provenance_report import build_report, distinct_missing
 import sitemarkup as markup
 import siteentries
+import retired_files
 import sitestarlight
 from siteexports import generate_exports_page, load_export_reports
 from siteguides import (
@@ -1622,7 +1624,8 @@ def generate_data_exports(
         "file-locations.json": ("file-locations", locations),
     }
     write_lookup_shards(
-        api, file_lookup_records(db, coverages, profiles, locations, members, consumers),
+        api, file_lookup_records(db, coverages, profiles, locations, members, consumers,
+                                 retired_files.load(_REPO_ROOT / "retired.json")),
         generated_at, commit,
     )
     for filename, (kind, items) in envelopes.items():
@@ -1808,7 +1811,9 @@ def generate_data_exports(
     # entry point is a document of its own, naming where to start.
     entry_points = [
         {"title": "Catalog", "url": "catalog.json",
-         "description": "Every published file with its size, SHA256 and schema."},
+         "description": "Each dataset, manifest, export and download with its size, "
+                        "SHA256 and schema. The one-item documents of emulators/, "
+                        "platforms/, systems/ and lookup/ are not listed."},
         {"title": "System ids", "url": "systems.json",
          "description": "Each system id with its name and the section holding its files."},
         {"title": "Core names", "url": "cores.json",
@@ -2203,11 +2208,15 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         "One hash without the whole list: `api/v1/lookup/<xx>.json`, where `<xx>` "
         "is the first two characters of the hash in lowercase, holds every SHA1, "
         "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` "
-        "(the collection, a data directory, a platform's list or an emulator "
-        "profile), the file `name`, the `url` of the page that shows it and, "
+        "(the collection, a member of a collection archive, a file that left the "
+        "collection, a data directory, a platform's list or an emulator profile), "
+        "the file `name`, the `url` of the page that shows it and, "
         "when one is known, the `system_name` the site gives its system. "
         "[Find or check a file](files/index.md) reads it.",
         "",
+        "`catalog.json` lists the datasets, manifests, exports and downloads. It "
+        "does not list these one-item documents, which hold no SHA256 of their own.",
+        "",
         "In a browser, the SHA1 of a file the reader picks finds its records when "
         "the collection holds that file. A list naming a file the collection lacks "
         "is filed under its MD5 or CRC32 alone, and Web Crypto has no MD5: "
@@ -2225,7 +2234,7 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         "  const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);",
         "  if (!response.ok) return [];",
         "  const shard = await response.json();",
-        "  // layer: collection, data-directory, platform or emulator. owner: which one.",
+        "  // layer: collection, archive-member, retired, data-directory, platform, emulator.",
         "  return shard.items.filter((item) => item.value === sha1);",
         "}",
         "```",
@@ -3350,8 +3359,8 @@ def setup_table(coverages: dict, registry: dict | None = None) -> list[str]:
     lines += ["No ZIP: the setup is served by the command alone. "
               f"[Why?](../{NO_ZIP_FAQ})", ""]
     if archived:
-        lines += ["An archived setup keeps its page and the command. Its list is no longer "
-                  "read again on a schedule.", ""]
+        lines += ["An archived setup keeps its page and the command. Checks that cover "
+                  "every setup include it only on request.", ""]
     return lines
 
 
@@ -4187,7 +4196,11 @@ def _card_route(name: str, display: str, guide: Guide) -> list[str]:
         lead = ("No network here? Run the command on a computer that has one, with a USB "
                 f"drive's folder after `--dest`. Then copy it into {_folder_words(display, extract_to)}.")
         return [lead, "", *terminal_tabs(tabs, args, steps=False)]
-    lead = ("No network? Put the device's card in your computer. After `--dest`, "
+    # A folder on the card's Linux partition: Windows and macOS show the
+    # small boot partition alone, and no folder there fits --dest.
+    put = ("Put the device's card in your computer." if "windows" in tabs
+           else "Only a Linux computer opens the card.")
+    lead = (f"No network? {put} After `--dest`, "
             f"name the folder the device calls `{extract_to}`.")
     return [lead, "", *terminal_tabs(tabs, args, steps=False),
             f"[More on SD cards](../{CARD_ROUTE_FAQ})", ""]
@@ -4729,7 +4742,9 @@ def generate_files_page() -> str:
         "A file's card lives on its manufacturer's page, and a file refiled "
         "between manufacturers moves with it. `files/?sha1=<sha1>` is the address "
         "that does not move: it opens the card wherever it is now. The link icon "
-        "beside each card's name is that address.",
+        "beside each card's name is that address. A file that left the "
+        "collection keeps it: the page says when, and opens the card of the "
+        "file in its place.",
         "",
         "## The index behind it",
         "",
@@ -4763,6 +4778,33 @@ def data_directory_revisions(data_dir: Path = _REPO_ROOT / "data") -> dict[str,
     return json.loads(path.read_text(encoding="utf-8"))
 
 
+_GIT_COMMIT = re.compile(r"[0-9a-f]{40}")
+_GITHUB_REPO = re.compile(r"https://github\.com/([^/]+/[^/]+)/")
+
+
+def data_directory_origin(entry: dict, fetched: dict) -> list[str]:
+    """Where a data directory is fetched from, and the commit the packs hold.
+
+    The URL is written with the `{version}` refresh_data_dirs substitutes:
+    printed raw it led to a 404. A zip source records the server's ETag,
+    which names no revision anyone can fetch, so only a git commit is shown.
+    """
+    lines = []
+    url = str(entry.get("source_url") or "")
+    if url:
+        url = url.replace("{version}", str(entry.get("version", "master")))
+        lines.append(f"Fetched from <{url}>")
+    sha = str(fetched.get("sha", "")).strip('"')
+    if _GIT_COMMIT.fullmatch(sha):
+        repo = _GITHUB_REPO.match(url)
+        commit = f"[{_code(sha[:12])}](https://github.com/{repo.group(1)}/commit/{sha})" \
+            if repo else _code(sha)
+        branch = str(fetched.get("version", ""))
+        lines.append(f"Revision: {commit}" + (
+            f", the head of `{branch}` when fetched" if branch and branch != sha else ""))
+    return lines
+
+
 def page_data_directories(name: str, profiles: dict) -> list[dict]:
     """The data directories an emulator page lists.
 
@@ -4892,6 +4934,64 @@ def _data_member_digests(
     return found
 
 
+def retired_records(retired: dict[str, dict], files: dict, locations: list[dict]) -> list[dict]:
+    """A record for each hash of a file that left the collection.
+
+    Its address (files/?sha1=, #sha1-<12>) was published and answered that
+    no file the site knows has the hash, inviting a contribution of a file
+    the collection held. The record says when it left, and leads to the
+    card of the file holding its place, followed to the one held today.
+    """
+    cards = {item["sha1"]: item["url"] for item in locations}
+    records = []
+    for sha1, record in sorted(retired.items()):
+        if sha1 in files:
+            continue
+        successor, seen = record.get("successor"), {sha1}
+        while successor and successor not in files and successor not in seen:
+            seen.add(successor)
+            successor = (retired.get(successor) or {}).get("successor")
+        url = cards.get(successor) if successor in files else None
+        extra = {"left": str(record["left"])}
+        if url:
+            extra["successor"] = successor
+        else:
+            # The page its card was on: the reader lands among its neighbours.
+            url = _site_path(f"systems/{card_page(str(record.get('path', '')), sha1)}.md")
+        for algorithm in HASH_ALGORITHMS:
+            value = sha1 if algorithm == "sha1" else record.get(algorithm)
+            for one in _hash_values(value):
+                records.append({"value": one, "algorithm": algorithm, "layer": "retired",
+                                "name": str(record.get("name", "")), "sha1": sha1,
+                                "url": url, **extra})
+    return records
+
+
+def _archive_member_records(entry: dict, sha1: str, url: str,
+                            root: Path = _REPO_ROOT) -> list[dict]:
+    """The CRC32 of each member of a collection archive, read from its
+    central directory.
+
+    A pasted CRC of a ROM the collection holds only inside a set (logica2.u2
+    in a500.zip, 8484f426) found nothing: 26,014 of 33,403 members were in
+    no record. The record leads to the archive's card and names it.
+    """
+    path = root / str(entry.get("path", ""))
+    if path.suffix.lower() != ".zip" or not path.is_file():
+        return []
+    try:
+        with zipfile.ZipFile(path) as archive:
+            members = [(info.filename, info.CRC) for info in archive.infolist()
+                       if not info.is_dir()]
+    except (OSError, zipfile.BadZipFile) as exc:
+        print(f"  WARNING: {path}: members unreadable ({exc})", file=sys.stderr)
+        return []
+    return [{"value": f"{crc:08x}", "algorithm": "crc32", "layer": "archive-member",
+             "name": member.rsplit("/", 1)[-1], "archive": str(entry.get("name", "")),
+             "sha1": sha1, "url": url}
+            for member, crc in members]
+
+
 def file_lookup_records(
     db: dict,
     coverages: dict,
@@ -4899,6 +4999,7 @@ def file_lookup_records(
     locations: list[dict],
     members: dict | None = None,
     consumers: FileConsumers | None = None,
+    retired: dict | None = None,
 ) -> list[dict]:
     """Every hash the site knows, one record per value, for files/?hash=.
 
@@ -4936,6 +5037,7 @@ def file_lookup_records(
                     "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"],
                     **({"system_name": held_by} if held_by else {}),
                 })
+        records += _archive_member_records(entry, sha1, item["url"])
     for key, coverage in sorted(coverages.items()):
         for system_id, system in sorted((coverage["config"].get("systems") or {}).items()):
             url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}")
@@ -4975,6 +5077,7 @@ def file_lookup_records(
                             "owner": key, "name": name, "url": url,
                             **({"system_name": entry_label} if entry_label else {}),
                         })
+    records += retired_records(retired or {}, files, locations)
     records += _held_under_every_hash(records, files, db.get("indexes") or {})
     for record in records:
         if record["layer"] in ("platform", "emulator"):
@@ -5999,7 +6102,9 @@ def _system_file_card(
     name = f["name"]
     sha1_full = f.get("sha1", "unknown")
     md5_full = f.get("md5", "unknown")
-    size = _fmt_size(f.get("size", 0))
+    # The size a card names is the file's own, exact: rounded, 44,348,015
+    # and 44,356,548 bytes were one 42.3 MB.
+    size = _card_size(int(f.get("size", 0)))
     # Who reads this file, not who reads a file of this name
     plats = sorted(consumers.platforms.get(sha1_full, ()))
     emus = sorted(consumers.emulators.get(sha1_full, ()))
@@ -6254,7 +6359,7 @@ def search_excluded(lines: list[str]) -> list[str]:
 SEARCH_SKIPPED = (
     "- SHA1:", "- MD5:", "- CRC32:", "- SHA256:", "- Adler32", "- Size:",
     "- Validation", "- Priority:", "- Contents (",
-    "- Platforms:", "- Also installed by:", "- Source:", "- Asked for by:",
+    "- Platforms:", "- Source:", "- Asked for by:",
     "- Collection path:", "- Collection copy:", "- Collection copies:",
     "- Installed by the platform's own package on:",
     "- Not in the platform's package", "- Never opened on these platforms",
@@ -6505,9 +6610,10 @@ def _profiles_on_disk() -> dict[str, dict]:
 
 
 def _entry_systems(emulator: str, entry: dict) -> list[str]:
-    """The system an entry is read for: its own, or its profile's only one."""
-    if entry.get("system"):
-        return [str(entry["system"])]
+    """The systems an entry is read for: its own `system:` or `systems:`
+    (a MAME device set several systems load), else its profile's only one."""
+    if entry.get("system") or entry.get("systems"):
+        return entry_system_ids(entry, {})
     systems = (_profiles_on_disk().get(emulator) or {}).get("systems") or []
     return [str(systems[0])] if len(systems) == 1 else []
 
@@ -7719,20 +7825,17 @@ def _distinct_spellings(names: list) -> list[str]:
     return list(seen.values())
 
 
-# Region slugs whose words are not the slug title-cased.
-_REGION_WORDS = {
-    "uk": "UK",
-    "asia-ntsc": "Asia (NTSC)",
-    "asia-pal": "Asia (PAL)",
-    "world": "every region",
-}
+# Region slugs whose words are not the slug title-cased, beyond region.LABELS.
+_REGION_WORDS = {"world": "every region"}
 
 
 def _region_words(value: object, most: int = 0) -> str:
     """A region: value in words. ``most`` caps the regions named, the rest
     counted: one Saturn BIOS serves seven."""
     raw = value if isinstance(value, list) else [value] if value else []
-    words = [_REGION_WORDS.get(str(r), str(r).replace("-", " ").title()) for r in raw]
+    from region import label  # noqa: PLC0415
+
+    words = [_REGION_WORDS.get(str(r)) or label(str(r)) for r in raw]
     if most and len(words) > most:
         words = words[:most] + [f"{len(words) - most} more"]
     return _said_as_list(words)
@@ -8015,6 +8118,38 @@ def _on_mismatch_lines(value: object) -> list[str]:
     return []
 
 
+def emulator_setups(coverages: dict, profiles: dict) -> dict[str, list[str]]:
+    """Each emulator with the platforms that run it, by display name."""
+    setups: dict[str, list[str]] = {}
+    for platform, cov in coverages.items():
+        for emulator in resolve_platform_cores(cov["config"], profiles):
+            setups.setdefault(emulator, []).append(platform)
+    return {
+        emulator: sorted(names, key=lambda p: _platform_display(p).casefold())
+        for emulator, names in setups.items()
+    }
+
+
+def _validation_names(validation: object) -> set[str]:
+    """The checks a `validation:` names, a list or one list per build."""
+    if isinstance(validation, dict):
+        return {str(v) for checks in validation.values() for v in checks or []}
+    return {str(v) for v in validation or []}
+
+
+def _card_size(size: int) -> str:
+    """A file's size, exact: in whole MB or KB when it is one, else in B."""
+    for unit, scale in (("MB", 1024 * 1024), ("KB", 1024)):
+        if size >= scale and size % scale == 0:
+            return f"{size // scale:,} {unit}"
+    return f"{size:,} B"
+
+
+def _exact_size(size: int) -> str:
+    """A rounded size with its bytes, where the bytes are not the same figure."""
+    return _fmt_size(size) if size < 1024 else f"{_fmt_size(size)} ({size:,} bytes)"
+
+
 def _build_rows(profile: dict) -> list[str]:
     """Meta card rows for who ships the build, how its files are found, why
     its revision is held and where its releases are announced."""
@@ -8148,25 +8283,28 @@ def _render_emulator_file(
         details.append(f"Found by: {found}")
     if fsystem:
         details.append(f"System: {_named_system_link(fsystem, '../')}")
+    # Where the code compares the size, the exact bytes: raze tells three
+    # DUKE3D.GRP apart by size, all three shown as 42.3 MB.
+    sized = _exact_size if "size" in _validation_names(validation) else _fmt_size
     if size:
         if isinstance(size, list):
-            size_str = " / ".join(_fmt_size(s) for s in size)
+            size_str = " / ".join(sized(s) for s in size)
         else:
-            size_str = _fmt_size(size)
+            size_str = sized(size)
         if fmin or fmax:
             bounds = []
             if fmin:
-                bounds.append(f"min {_fmt_size(fmin)}")
+                bounds.append(f"min {sized(fmin)}")
             if fmax:
-                bounds.append(f"max {_fmt_size(fmax)}")
+                bounds.append(f"max {sized(fmax)}")
             size_str += f" ({', '.join(bounds)})"
         details.append(f"Size: {size_str}")
     elif fmin or fmax:
         bounds = []
         if fmin:
-            bounds.append(f"min {_fmt_size(fmin)}")
+            bounds.append(f"min {sized(fmin)}")
         if fmax:
-            bounds.append(f"max {_fmt_size(fmax)}")
+            bounds.append(f"max {sized(fmax)}")
         details.append(f"Size: {', '.join(bounds)}")
     # Starlight folds the hashes and gives each a copy button.
     copyable = markup.starlight()
@@ -9040,6 +9178,21 @@ def _install_args(platform: str, system: str, shell: str) -> str:
     return f"--platform {platform} --system {quoted}"
 
 
+def _default_setup(declared: list[tuple[str, str]]) -> tuple[str, str]:
+    """The setup an install box shows before the reader picks one.
+
+    The first in alphabetical order was AmberELEC on 17 system pages, a
+    handheld system under a Windows tab. The box shows the setup whose
+    command runs under the most of its tabs (the guide's `run`), then on the
+    most systems overall, then one the release publishes a ZIP for.
+    """
+    def reach(pair: tuple[str, str]) -> tuple[int, int, bool]:
+        runs = GUIDES[pair[0]].run if pair[0] in GUIDES else ()
+        return (len(set(runs) & set(INSTALL_PICKER_OSES)), len(runs), has_release_pack(pair[0]))
+
+    return max(declared, key=reach)
+
+
 def _install_picker(declared: list[tuple[str, str]]) -> list[str]:
     """One command, for the setup the reader picks, under the OS tabs.
 
@@ -9054,15 +9207,16 @@ def _install_picker(declared: list[tuple[str, str]]) -> list[str]:
         return html.escape(text, quote=False).replace('"', "&quot;")
 
     names = Counter(platform for platform, _system in declared)
+    lead = _default_setup(declared)
     options = "".join(
-        f'<option value="{attribute(platform)}" '
-        f'data-sh="{attribute(_install_args(platform, system, "sh"))}" '
+        f'<option value="{attribute(platform)}"'
+        + (" selected" if (platform, system) == lead else "")
+        + f' data-sh="{attribute(_install_args(platform, system, "sh"))}" '
         f'data-ps="{attribute(_install_args(platform, system, "powershell"))}">'
         + html.escape(_platform_display(platform) + (f" ({system})" if names[platform] > 1 else ""),
                       quote=False)
         + "</option>"
         for platform, system in declared)
-    lead = declared[0]
     args = {key: _install_args(*lead, TERMINALS[key][2]) for key in INSTALL_PICKER_OSES}
     return [
         "Each platform names the system its own way.",
@@ -9379,6 +9533,8 @@ def _said_as_list(items: list[str]) -> str:
 # Data directory members and revisions, read once per build.
 _DATA_MEMBERS: dict[str, dict[str, dict]] = {}
 _DATA_REVISIONS: dict[str, dict] = {}
+# Emulator -> the platforms that run it, as resolve_platform_cores answers.
+_EMULATOR_SETUPS: dict[str, list[str]] = {}
 # The measurement index items of each emulator, for the proof lines of its page.
 _MEASUREMENT_ITEMS: dict[str, list[dict]] = {}
 # The emulator pages that list each data directory, by ref.
@@ -9974,6 +10130,15 @@ def emulator_pages(
     if revision:
         lines.append(f"| Revision | {revision} |")
     lines.extend(_build_rows(profile))
+    setups = _EMULATOR_SETUPS.get(name)
+    if setups:
+        # The file cards name the setups whose own list holds a file: the
+        # ones that run the emulator and pack its files were said nowhere.
+        # Said once here: a line on each card put MAME's parts and four
+        # other pages over their element budget. Out of the search: setup
+        # names on 530 pages ranked the data page first for "batocera folder".
+        lines.append(f"| Run by | <span {markup.search_exclude()}>" + ", ".join(
+            f"[{_platform_display(p)}](../platforms/{p}.md)" for p in setups) + "</span> |")
     lines.append(f"| Profiled | {profiled} |")
     # The YAML is the reusable document this site renders, and the JSON is the
     # same profile for a program.
@@ -10078,15 +10243,8 @@ def emulator_pages(
             if entry.get("description"):
                 line += f": {entry['description']}"
             lines.append(line)
-            if entry.get("source_url"):
-                lines.append(f"    - Fetched from <{entry['source_url']}>")
-            fetched = _DATA_REVISIONS.get(ref) or {}
-            if fetched.get("version") or fetched.get("sha"):
-                # The URL serves whatever is current: the revision says
-                # which content the packs were built from.
-                revision = str(fetched.get("sha", "")).strip('"')
-                lines.append(f"    - Revision: {fetched.get('version', '')}"
-                             + (f" ({_code(revision)})" if revision else ""))
+            lines.extend(f"    - {line}" for line in data_directory_origin(
+                entry, _DATA_REVISIONS.get(ref) or {}))
             if dd.get("mode"):
                 lines.append(f"    - Read by the {dd['mode']} build only")
             if dd.get("bundled"):
@@ -12311,6 +12469,7 @@ def _generate(args: argparse.Namespace) -> None:
         PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov)
 
     print("Generating emulator pages...")
+    _EMULATOR_SETUPS.update(emulator_setups(coverages, profiles))
     _DATA_MEMBERS.update(data_directory_members())
     _DATA_REVISIONS.update(data_directory_revisions())
     for key in sorted(profiles):

scripts/list_platforms.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
-"""List available platforms for CI matrix strategy.
+"""List the registered platforms.
 
 Respects the `status` field in _registry.yml:
-- active: included in CI releases and automated scraping
-- archived: excluded from CI, user can generate manually
+- active: listed by default
+- archived: listed with --include-archived only
 
 Usage:
-    python scripts/list_platforms.py                # Active platforms (for CI)
+    python scripts/list_platforms.py                # active platforms
     python scripts/list_platforms.py --include-archived  # archived ones too
     python scripts/list_platforms.py >> "$GITHUB_OUTPUT"
 """

scripts/packreadme.py

@@ -5,6 +5,7 @@ goes, and which ones the user has to rename."""
 
 from __future__ import annotations
 
+from region import label as region_label
 from siteguides import INSTALL_PS1, INSTALL_SH, pack_manual
 
 def _detect_systems(detect: list[dict]) -> list[str]:
@@ -290,9 +291,7 @@ def _build_readme(
 
     region_info = ""
     if regions:
-        pretty = ", ".join(
-            " ".join(w.title() for w in slug.split("-")) for slug in regions
-        )
+        pretty = ", ".join(region_label(slug) for slug in regions)
         region_info = (
             "PACK TYPE: Region Filtered\n\n"
             f"  Region priority: {pretty}\n\n"

scripts/region.py

@@ -107,6 +107,19 @@ ALIASES: dict[str, str] = {
 }
 
 
+# Slugs a person does not write title-cased: "Asia Ntsc" read as machine text.
+LABELS: dict[str, str] = {
+    "uk": "UK",
+    "asia-ntsc": "Asia (NTSC)",
+    "asia-pal": "Asia (PAL)",
+}
+
+
+def label(slug: str) -> str:
+    """A region slug in words: north-america as North America."""
+    return LABELS.get(slug) or " ".join(word.title() for word in slug.split("-"))
+
+
 def unknown_region_message(raw: str, aliases: dict[str, str], regions: frozenset[str]) -> str:
     """What to type instead: the closest spellings, then every short form.
 

scripts/retired_files.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Files that left the collection, and the file that took their place.
+
+A file's content address (files/?sha1=, #sha1-<12> on its card) is
+published, and a published address never stops answering. When a rebuild
+of database.json drops a sha1, the file it named is recorded here: its
+name, its last path, its hashes, the day it left and the sha1 the
+collection now holds at that path, or under that name in that folder.
+The site's lookup reads the record and answers with it.
+
+The record is append-only, written by generate_db.py beside the database.
+A sha1 that comes back leaves it. `--backfill` rebuilds it from the
+history of database.json since content addresses were published.
+
+Usage:
+    python scripts/retired_files.py --backfill
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from common import run_cli, write_if_changed  # noqa: E402
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+RETIRED_FILE = REPO_ROOT / "retired.json"
+# The commit that published content addresses: 0d586ef6b, 2026-10-19.
+FIRST_ADDRESSED = "0d586ef6b"
+KEPT_FIELDS = ("name", "path", "size", "md5", "crc32", "sha256")
+
+
+def retired_path(database: str | Path) -> Path:
+    """The record kept beside a database file."""
+    return Path(database).with_name("retired.json")
+
+
+def _successor(entry: dict, by_path: dict[str, str],
+               by_name: dict[str, list[tuple[str, str]]]) -> str | None:
+    """The sha1 the new database holds at the entry's path, else under its
+    name in the same folder, else the one file of that name: a2diskiing.zip
+    was rebuilt from its recipe and filed under Apple II the same day."""
+    path = str(entry.get("path", ""))
+    if path in by_path:
+        return by_path[path]
+    named = by_name.get(str(entry.get("name", "")), [])
+    folder = path.rsplit("/", 1)[0]
+    beside = [sha1 for sha1, other in named if other.rsplit("/", 1)[0] == folder]
+    for found in (beside, [sha1 for sha1, _other in named]):
+        if len(found) == 1:
+            return found[0]
+    return None
+
+
+def retired_between(old: dict, new: dict, day: str) -> dict[str, dict]:
+    """sha1 -> record, for each file of *old* that *new* no longer holds."""
+    old_files = old.get("files") or {}
+    new_files = new.get("files") or {}
+    by_path = {str(e.get("path", "")): sha1 for sha1, e in new_files.items()}
+    by_name: dict[str, list[tuple[str, str]]] = {}
+    for sha1, e in new_files.items():
+        by_name.setdefault(str(e.get("name", "")), []).append((sha1, str(e.get("path", ""))))
+    records = {}
+    for sha1 in sorted(set(old_files) - set(new_files)):
+        entry = old_files[sha1]
+        record = {key: entry[key] for key in KEPT_FIELDS if entry.get(key) not in (None, "")}
+        record["left"] = day
+        successor = _successor(entry, by_path, by_name)
+        if successor:
+            record["successor"] = successor
+        records[sha1] = record
+    return records
+
+
+def merge(record: dict[str, dict], left: dict[str, dict], new: dict) -> dict[str, dict]:
+    """The record with *left* added, first departure kept, and every sha1
+    the new database holds again taken out."""
+    held = set(new.get("files") or {})
+    merged = {sha1: entry for sha1, entry in record.items() if sha1 not in held}
+    for sha1, entry in left.items():
+        merged.setdefault(sha1, entry)
+    return dict(sorted(merged.items()))
+
+
+def load(path: Path) -> dict[str, dict]:
+    if not path.is_file():
+        return {}
+    return json.loads(path.read_text(encoding="utf-8")).get("files") or {}
+
+
+def dump(records: dict[str, dict]) -> str:
+    return json.dumps({"schema_version": 1, "files": records}, indent=1, sort_keys=True) + "\n"
+
+
+def update(path: Path, old: dict, new: dict, day: str) -> int:
+    """Records what *new* dropped from *old*. Returns how many left now."""
+    left = retired_between(old, new, day)
+    record = load(path)
+    merged = merge(record, left, new)
+    if merged != record:
+        write_if_changed(str(path), dump(merged))
+    return len(set(left) - set(record))
+
+
+def _git(*args: str) -> str:
+    return subprocess.run(["git", *args], cwd=REPO_ROOT, capture_output=True, text=True,
+                          check=True).stdout
+
+
+def backfill() -> dict[str, dict]:
+    """Every sha1 that left database.json in the commits since content
+    addresses were published, with the day of the commit that dropped it."""
+    revisions = [line.split() for line in _git(
+        "log", "--reverse", "--format=%H %cs", f"{FIRST_ADDRESSED}^..HEAD", "--", "database.json",
+    ).splitlines() if line.strip()]
+    records: dict[str, dict] = {}
+    previous: dict | None = None
+    for commit, day in revisions:
+        current = json.loads(_git("show", f"{commit}:database.json"))
+        if previous is not None:
+            records = merge(records, retired_between(previous, current, day), current)
+        previous = current
+    return records
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
+    parser.add_argument("--backfill", action="store_true",
+                        help="rebuild the record from the history of database.json")
+    parser.add_argument("--output", default=str(RETIRED_FILE),
+                        help="the record to write (default: retired.json)")
+    args = parser.parse_args()
+    if not args.backfill:
+        parser.print_help()
+        return 2
+    records = backfill()
+    current = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+    records = merge(records, {}, current)
+    write_if_changed(args.output, dump(records))
+    print(f"{len(records)} files left the collection since {FIRST_ADDRESSED}")
+    return 0
+
+
+if __name__ == "__main__":
+    run_cli(main)

scripts/siterender.py

@@ -372,19 +372,21 @@ def _in_arcade(sha1: str, db: dict) -> bool:
     return sha1 in _arcade_suffix_sha1s(db)
 
 
-_suffix_cache: list[tuple[int, frozenset[str]] | None] = [None]
+_suffix_cache: list[tuple[dict, frozenset[str]] | None] = [None]
 
 
 def _arcade_suffix_sha1s(db: dict) -> frozenset[str]:
+    # Holding the database itself, never id(db): a database freed and
+    # replaced at the same address answered with the first one's set.
     cached = _suffix_cache[0]
-    if cached is not None and cached[0] == id(db):
+    if cached is not None and cached[0] is db:
         return cached[1]
     found = frozenset(
         sha1s[0]
         for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items()
         if tail.startswith("Arcade/") and len(sha1s) == 1
     )
-    _suffix_cache[0] = (id(db), found)
+    _suffix_cache[0] = (db, found)
     return found
 
 

scripts/slots.py

@@ -7,11 +7,14 @@ module existed nothing compared the two: the builder resolves the platform
 entry first and drops the profile entry on a bare filename match, so a profile
 that names the right file for a slot could never win and never even be heard.
 
-The index is keyed by destination AND by name, the name carrying the union of
-what every same-named entry claims. Dolphin declares three IPL.bin separated
-only by their path, so a name-only key merges them, and a path-only key answers
-nothing when a candidate is known by name alone. Ambiguity resolves to the
-union and is never discarded, the same rule region.py already applies.
+Claims are keyed by destination, folded to one case only where the platform's
+filesystem folds it. Dolphin declares three IPL.bin separated only by their
+path, and each is its own slot. A profile claim resting on a bare filename
+proves nothing about content and contests no slot.
+
+The pack builder, the install manifest and verify read slot_choices() through
+pack_overrides(). arbitrate() states the same decision for one conflict and
+feeds the report this module prints.
 """
 
 from __future__ import annotations
@@ -126,26 +129,6 @@ def _normalize(destination: str, fold: bool = True) -> str:
     return cleaned.casefold() if fold else cleaned
 
 
-def build_claim_index(claims: list[Claim]) -> dict[str, list[Claim]]:
-    """Index claims by destination and by name, the name carrying the union.
-
-    A lookup by full destination answers for one slot. A lookup by bare name
-    answers with every slot that shares it, which is what a candidate known
-    only by its filename needs in order not to be dropped.
-    """
-    index: dict[str, list[Claim]] = {}
-    for claim in claims:
-        keys = {_normalize(claim.destination)}
-        if claim.name:
-            keys.add(_normalize(claim.name))
-        base = claim.destination.rsplit("/", 1)[-1]
-        if base:
-            keys.add(_normalize(base))
-        for key in keys - {""}:
-            index.setdefault(key, []).append(claim)
-    return index
-
-
 def platform_claims(
     config: dict,
     db: dict,

scripts/validate_pr.py

@@ -24,6 +24,7 @@ import re
 import shutil
 import subprocess
 import sys
+import urllib.parse
 from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
@@ -73,6 +74,22 @@ LARGE_FILE_ISSUE_URL = (
 )
 
 
+def large_file_issue_url(result: ValidationResult) -> str:
+    """The issue form with the values this run computed already filled in.
+
+    GitHub fills an issue form field from the query parameter named by its
+    id: filename, sha1, md5 and size in missing-bios.yml.
+    """
+    fields = {
+        "title": f"[BIOS] Missing: {result.filename}",
+        "filename": result.filename,
+        "sha1": result.sha1,
+        "md5": result.md5,
+        "size": str(result.size),
+    }
+    return f"{LARGE_FILE_ISSUE_URL}&{urllib.parse.urlencode(fields)}"
+
+
 class ValidationResult:
     def __init__(self, filepath: str) -> None:
         self.filepath = filepath
@@ -359,14 +376,15 @@ def validate_file(
     if ext in BLOCKED_EXTENSIONS:
         result.add_check(False, f"Blocked file extension: {ext}")
 
-    if result.size > MAX_FILE_SIZE:
+    too_large = result.size > MAX_FILE_SIZE
+    if too_large:
         # GitHub refuses a pushed file over 100 MiB, so a pull request cannot
         # carry it. Only a maintainer can add an asset to the release.
         result.add_check(
             False,
             f"File too large for git ({result.size:,} > {MAX_FILE_SIZE:,} bytes). "
-            "Leave it out of the pull request and open an issue "
-            f"({LARGE_FILE_ISSUE_URL}) with the hashes above and where the file "
+            "Leave it out of the pull request and open an issue, its hashes "
+            f"filled in ({large_file_issue_url(result)}), saying where the file "
             "can be obtained: a maintainer adds it to the `large-files` release.",
         )
     elif result.size == 0:
@@ -450,8 +468,8 @@ def validate_file(
         result.outcome = "unknown"
         result.add_warning(
             "Not recognised: no platform list or emulator profile declares this "
-            "file. Say in the pull request which emulator reads it, for which "
-            "system, and where the dump comes from"
+            f"file. Say in the {'issue' if too_large else 'pull request'} which "
+            "emulator reads it, for which system, and where the dump comes from"
         )
 
     normalized = os.path.normpath(relative)
@@ -467,7 +485,11 @@ def validate_file(
         set(),
     )
     expected_folders = sorted(expected)
-    if normalized.startswith("bios" + os.sep):
+    if too_large:
+        # The file stays out of the pull request, so no folder of bios/ is
+        # the contributor's to choose: the release asset is placed by hand.
+        pass
+    elif normalized.startswith("bios" + os.sep):
         parts = normalized.split(os.sep)
         here = f"bios/{parts[1]}/{parts[2]}/" if len(parts) >= 4 else ""
         makers = {folder.split("/")[1] for folder in expected}