chore: merge site and docs fixes, fifth pass

scripts/auto_fetch.py

@@ -6,7 +6,8 @@ Pipeline:
 2. Scan old branches (git show origin/branch:path)
 3. Search public BIOS repos on GitHub
 4. Search archive.org collections
-5. Create GitHub Issue for community help
+5. Print an issue body for the files still missing (--create-issues).
+   It posts nothing.
 
 Usage:
     python scripts/auto_fetch.py --platform retroarch [--dry-run]
@@ -459,43 +460,37 @@ def fetch_missing(
             )
             stats["errors"] += 1
             continue
-        print("    [5] Not found - needs community contribution")
+        print("    [5] Not found, open for contributions")
         still_missing.append(entry)
         stats["not_found"] += 1
 
     return {"stats": stats, "still_missing": still_missing}
 
 
+CONTRIBUTING_URL = "https://github.com/Abdess/retrobios/blob/main/CONTRIBUTING.md"
+
+
 def generate_issue_body(missing: list[dict], platform: str) -> str:
-    """Generate a GitHub Issue body for missing BIOS files."""
+    """An issue body for the files a platform still needs.
+
+    The hashes are written in full: a contributor matches a file against
+    them, and a 12-character prefix confirms nothing.
+    """
     lines = [
-        f"## Missing BIOS Files for {platform}",
-        "",
-        "The following BIOS files are required but not available in the repository.",
-        "If you have any of these files, please submit a Pull Request!",
+        f"## Files {platform} still needs",
         "",
         "| File | System | SHA1 | MD5 |",
         "|------|--------|------|-----|",
     ]
-
     for entry in missing:
-        sha1 = ", ".join(sorted(_accepted(entry, "sha1"))) or "N/A"
-        md5 = ", ".join(sorted(_accepted(entry, "md5"))) or "N/A"
-        lines.append(
-            f"| `{entry['name']}` | {entry['system']} | `{sha1[:12]}...` | `{md5[:12]}...` |"
-        )
-
-    lines.extend(
-        [
-            "",
-            "### How to Contribute",
-            "",
-            "1. Fork this repository",
-            "2. Add the BIOS file to `bios/Manufacturer/Console/`",
-            "3. Create a Pull Request - checksums are verified automatically",
-        ]
-    )
-
+        sha1 = ", ".join(f"`{h}`" for h in sorted(_accepted(entry, "sha1"))) or "none declared"
+        md5 = ", ".join(f"`{h}`" for h in sorted(_accepted(entry, "md5"))) or "none declared"
+        lines.append(f"| `{entry['name']}` | {entry['system']} | {sha1} | {md5} |")
+    lines += [
+        "",
+        "A file with one of these hashes can be added through the upload link in"
+        f" [CONTRIBUTING.md]({CONTRIBUTING_URL}). CI checks it on arrival.",
+    ]
     return "\n".join(lines)
 
 
@@ -563,7 +558,7 @@ def main() -> None:
         result = fetch_missing(missing, db, args.bios_dir, args.dry_run)
 
         if result["still_missing"]:
-            all_still_missing[platform] = result["still_missing"]
+            all_still_missing[config.get("platform") or platform] = result["still_missing"]
 
         stats = result["stats"]
         print(

scripts/commit_rules.py

@@ -10,6 +10,7 @@ Usage as a hook: python scripts/commit_rules.py MESSAGE_FILE
 
 from __future__ import annotations
 
+import argparse
 import re
 import sys
 from pathlib import Path
@@ -51,10 +52,11 @@ def problems(message: str) -> list[str]:
 
 
 def main(argv: list[str]) -> int:
-    if len(argv) != 2:
-        print("usage: commit_rules.py MESSAGE_FILE", file=sys.stderr)
-        return 2
-    found = problems(Path(argv[1]).read_text(encoding="utf-8", errors="replace"))
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False, description="Check a commit message against the rules above")
+    parser.add_argument("message_file", type=Path, help="the message file a commit-msg hook receives")
+    args = parser.parse_args(argv[1:])
+    found = problems(args.message_file.read_text(encoding="utf-8", errors="replace"))
     for problem in found:
         print(f"commit-msg: {problem}", file=sys.stderr)
     return 1 if found else 0

scripts/export_native.py

@@ -515,7 +515,7 @@ class LandedChanges:
 
     @property
     def applied(self) -> list[str]:
-        return self._labels("hash_corrected")
+        return self._labels("hash_updated")
 
     @property
     def filled(self) -> list[str]:
@@ -527,7 +527,7 @@ class LandedChanges:
 
     @property
     def resized(self) -> list[str]:
-        return [label.removesuffix(" size") for label in self._labels("size_corrected")]
+        return [label.removesuffix(" size") for label in self._labels("size_updated")]
 
     @property
     def unsized(self) -> list[str]:
@@ -535,7 +535,7 @@ class LandedChanges:
 
     @property
     def requirements(self) -> int:
-        return sum(r["kind"] == "requirement_corrected" for r in self.records)
+        return sum(r["kind"] == "required_aligned" for r in self.records)
 
     @property
     def lists(self) -> int:
@@ -624,14 +624,14 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
                 and exporter.required_value(entry) != bool(entry.platform["required"])
             ):
                 records.append(_change(
-                    "requirement_corrected", entry, "required",
+                    "required_aligned", entry, "required",
                     bool(entry.platform["required"]), exporter.required_value(entry),
                 ))
             for field_name in entry.corrections:
                 if field_name == "required" or not exporter.states(entry, field_name):
                     continue
                 records.append(_change(
-                    "hash_corrected", entry, field_name,
+                    "hash_updated", entry, field_name,
                     _platform_value(entry, field_name),
                     _written_value(exporter, entry, field_name),
                 ))
@@ -648,7 +648,7 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
                     and exporter.states(entry, field_name)
                 ):
                     records.append(_change(
-                        "hash_corrected", entry, field_name,
+                        "hash_updated", entry, field_name,
                         _platform_value(entry, field_name), listed,
                     ))
             records.extend(
@@ -666,7 +666,7 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
                 and exporter.states(entry, "size")
             ):
                 records.append(_change(
-                    "size_corrected", entry, "size", _single_size(theirs), ours
+                    "size_updated", entry, "size", _single_size(theirs), ours
                 ))
             elif (
                 theirs
@@ -690,7 +690,7 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
     # A value the render rewrote though the model reads it as the
     # platform's own (RomM's sha1 with a leading space).
     records.extend(
-        _change("hash_corrected", entry, field_name, old, new)
+        _change("hash_updated", entry, field_name, old, new)
         for entry, field_name, old, new in getattr(exporter, "normalized_values", [])
     )
     return changes
@@ -863,12 +863,12 @@ def export_platform(
             "not_kept": lost,
         },
         "changes": changes.records,
-        "hashes_corrected": applied,
+        "hashes_updated": applied,
         "hashes_filled": filled,
-        "sizes_corrected": resized,
+        "sizes_updated": resized,
         "sizes_removed": unsized,
         "hashes_cleared": cleared,
-        "requirements_corrected": requirements,
+        "required_aligned": requirements,
         "withheld": list(withheld),
         "found_by_content": report.labels,
         "unattributed": report.unattributed,

scripts/fetch_site_images.py

@@ -172,7 +172,8 @@ def fetch_logos() -> None:
 
 
 def fetch_icons() -> None:
-    from generate_site import _group_by_manufacturer, load_database
+    from common import load_database
+    from sitecollection import _group_by_manufacturer
     from siterender import SYSTEM_ICON_BASE, _icon_name
 
     db = load_database(str(ROOT / "database.json"))

scripts/generate_readme.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 import argparse
 import json
 import os
+import re
 import sys
 from collections.abc import Callable
 from datetime import datetime, timezone
@@ -40,6 +41,8 @@ from nativemode import check_label
 import release_record
 from siteguides import install_command
 from validate_pr import LARGE_FILE_ISSUE_URL
+from largefiles import load_registered_paths
+import pr_checks
 from verify import verify_platform
 
 
@@ -447,6 +450,29 @@ REPO = "Abdess/retrobios"
 UPLOAD_URL = f"https://github.com/{REPO}/upload/main/bios"
 
 
+def clone_line(db: dict) -> str:
+    """How to get the clone the commands below run in, and its size.
+
+    Said bare, "clone the repo" started a multi-gigabyte transfer for a user
+    after a 20 MB --required-only pack. The size counts the collection files
+    the repository commits, a floor: release assets are fetched only when a
+    pack needs them.
+    """
+    assets = set(load_registered_paths())
+    committed = sum(
+        entry.get("size", 0) for entry in db.get("files", {}).values()
+        if entry.get("path") not in assets
+    )
+    return (
+        "Packs are built from a clone, and the clone carries the collection:"
+        f" its checkout alone is over {format_size(committed)}."
+        f" `git clone --depth 1 https://github.com/{REPO}.git` leaves the"
+        " history out. Files kept as release assets are downloaded when a"
+        " pack needs them. Then generate packs for any platform, emulator,"
+        " or system:"
+    )
+
+
 def _existing_contributor_block(readme_path: str = "README.md") -> list[str]:
     """The contributors already published, read back from the file.
 
@@ -767,7 +793,7 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json"
             "",
             "## Build your own pack",
             "",
-            "Clone the repo and generate packs for any platform, emulator, or system:",
+            clone_line(db),
             "",
             "```bash",
             "# The pack of a platform, as released",
@@ -891,6 +917,19 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json"
     return "\n".join(lines) + "\n"
 
 
+def ci_install_lines() -> tuple[str, str]:
+    """The pip lines the test job of validate.yml runs, verbatim.
+
+    The guide listed the packages without their versions, so a local run
+    could pass on releases CI never installs. Read from the workflow, the
+    two cannot disagree. The second line is the Python 3.8 one.
+    """
+    text = (_REPO / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8")
+    main = re.search(r'^\s*run: (pip install pyyaml "jsonschema.*markdown.*)$', text, re.M)
+    floor = re.search(r"^\s*run: (python3\.8 -m pip install .*)$", text, re.M)
+    return main.group(1), floor.group(1)
+
+
 def contributing_text(
     link: Callable[[str], str], title: str, site_words: str
 ) -> str:
@@ -900,6 +939,9 @@ def contributing_text(
     site. *link* renders a site page (absolute URL for the README, relative
     .md for the site); *site_words* names the site from where the text is read.
     """
+    pip_line, floor_line = ci_install_lines()
+    width = max(len(step.command) for step in pr_checks.STEPS)
+    steps = "\n".join(f"{step.command:<{width}}  # {step.why}" for step in pr_checks.STEPS)
     return f"""{title}
 
 ## Add a BIOS file
@@ -974,18 +1016,22 @@ validation above is its check. For code, profiles or platform files, these are
 the checks CI runs on a pull request (Python 3.12 or newer):
 
 ```bash
-pip install pyyaml "jsonschema[format-nongpl]" markdown pyte hypothesis vermin setuptools
-python scripts/validate_pr.py --changed          # the BIOS files the branch adds
-python scripts/validate_schemas.py --source-only # profiles and platform files
-python scripts/profile_sync.py --added --strict  # each profile the branch adds
-python scripts/refresh_data_dirs.py  # data directories the pack-building tests read
-python scripts/generate_site.py       # the pages the site tests read
-npm ci --prefix website && npm run build --prefix website  # the built site, Node 24
-python -m unittest discover -s tests -t .
-python scripts/validate_site.py --site-dir website/dist  # links and page budgets
-vermin --no-tips -t=3.8- --violations install.py         # install.py stays on Python 3.8
+{pip_line}
+python scripts/pr_checks.py
 ```
 
+`pr_checks.py` runs these, in CI's order, carries on past a failure as CI
+does, and names at the end each check that failed or could not run:
+
+```bash
+{steps}
+```
+
+The versions are the ones CI pins. Three tools come from the system, not from
+that line, and the tests that need them skip without them: `shellcheck` for the
+handheld launchers, `bwrap` (bubblewrap) for the probe sandbox, and a Python 3.8
+with pyte (`{floor_line}`) for the full screen. CI has all three.
+
 `profile_sync.py` compares the branch with this repository's `main`, and
 fetches it in a fork's clone. A profile you only edit is not held to
 `--strict`: its older refs are reviewed by hand.

scripts/generate_site.py

@@ -38,6 +38,10 @@ from pathlib import Path
 from typing import TypeVar
 
 sys.path.insert(0, os.path.dirname(__file__))
+from sitecollection import (  # noqa: E402
+    _copy_places, _group_by_manufacturer, _profiles_in,
+    _profiles_on_disk, _slug, collection_paths,
+)
 from common import (
     BUCKET_FOLDERS,
     names_manufacturer,
@@ -71,10 +75,6 @@ from cross_reference import FileConsumers, entry_target, held_copy
 from nativemode import check_label, digest_algorithm, reads_file_contents
 
 
-
-
-
-
 yaml = require_yaml()
 from generate_readme import (
     compute_coverage,
@@ -139,8 +139,6 @@ GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads
 PUBLIC_DIRS = ("api", "downloads")
 
 
-
-
 def _content_check_ceiling(profiles: dict) -> str:
     """How far content checking can reach across every profiled entry."""
     hashed = sized = neither = 0
@@ -170,9 +168,6 @@ def _content_check_ceiling(profiles: dict) -> str:
     )
 
 
-
-
-
 CLS_LABELS = {
     "official_port": "Official ports",
     "community_fork": "Community forks",
@@ -202,33 +197,6 @@ CLS_LABEL_ONE = {
 }
 
 
-def _copy_places(db: dict) -> dict[str, set[tuple[str, str, str]]]:
-    """sha1 -> every (manufacturer, console, file name) holding a copy.
-
-    The database keeps one path per content, deduplication having left one
-    copy. by_path_suffix still records the other folders a copy is filed in,
-    under the name that folder gives it.
-    """
-    files = db.get("files", {})
-    bios_dir = Path(str(db.get("bios_dir") or "bios"))
-    places: dict[str, set[tuple[str, str, str]]] = {}
-    for sha1, entry in files.items():
-        parts = str(entry.get("path", "")).split("/")
-        if len(parts) >= 3 and parts[0] == "bios":
-            places.setdefault(sha1, set()).add((parts[1], parts[2], parts[-1]))
-    for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items():
-        parts = tail.split("/")
-        if len(parts) < 3 or parts[1].startswith(".") or len(sha1s) != 1:
-            continue
-        if sha1s[0] in files and (bios_dir / tail).is_file():
-            places.setdefault(sha1s[0], set()).add((parts[0], parts[1], parts[-1]))
-    return places
-
-
-def _slug(manufacturer: str) -> str:
-    return manufacturer.lower().replace(" ", "-")
-
-
 def _copies_by_path(db: dict) -> dict[str, set[tuple[str, str, str]]]:
     """The copy places of each content, keyed by the path the database keeps."""
     files = db.get("files", {})
@@ -414,11 +382,6 @@ def _system_pages_from_data(
     return pages
 
 
-
-
-
-
-
 def _timestamp() -> str:
     """The build time for data files (stats, API records), ISO 8601."""
     return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -437,8 +400,6 @@ def _date_line() -> str:
     return f'<div class="rb-timestamp">{data_as_of()}.</div>'
 
 
-
-
 # Home page
 
 
@@ -2486,10 +2447,16 @@ def _browser_title(relative: Path, title: str) -> str:
     return title
 
 
+_LINK_TEXT = r"(?:`[^`]*`|\[[^\]\[]*\]|[^\]\[])"
+
+
 def _plain_markdown(text: str) -> str:
     text = re.sub(r"<[^>]+>", " ", text)
-    text = re.sub(r"!\[([^]]*)\]\([^)]*\)", r"\1", text)
-    text = re.sub(r"\[([^]]+)\]\([^)]*\)", r"\1", text)
+    # Link text may hold brackets: a code span or a No-Intro name such as
+    # `[BIOS] LaserActive PAC-N1 (Japan) (v1.02).bin` left the whole link,
+    # anchor included, in the Pioneer page's meta description.
+    text = re.sub(rf"!\[({_LINK_TEXT}*)\]\([^)]*\)", r"\1", text)
+    text = re.sub(rf"\[({_LINK_TEXT}+)\]\([^)]*\)", r"\1", text)
     # Code spans keep their text whole: validate_schemas.py read
     # validateschemas.py once every underscore was taken for emphasis.
     pieces = re.split(r"(`[^`]*`)", text)
@@ -5266,122 +5233,6 @@ def write_lookup_shards(
     return len(shards)
 
 
-def collection_paths(db: dict) -> dict[str, set[str]]:
-    """Every path under bios/ that holds each file, by sha1.
-
-    The database keeps one path per content. The others are the full tails
-    by_path_suffix records that name a file on disk: RPG Maker, engine
-    trees and the copies filed under several consoles keep 2,502 of them.
-    """
-    files = db.get("files", {})
-    bios_dir = Path(str(db.get("bios_dir") or "bios"))
-    paths: dict[str, set[str]] = {}
-    for sha1, entry in files.items():
-        path = str(entry.get("path", ""))
-        if path.startswith("bios/") and path.count("/") >= 2:
-            paths.setdefault(sha1, set()).add(path)
-    for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items():
-        if tail.count("/") < 2 or len(sha1s) != 1 or sha1s[0] not in files:
-            continue
-        if (bios_dir / tail).is_file():
-            paths.setdefault(sha1s[0], set()).add(f"bios/{tail}")
-    return paths
-
-
-def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]:
-    """Group files by manufacturer -> console -> files.
-
-    The database keeps one path per content, and a copy filed elsewhere was
-    on no page of its own: the Channel F BIOS sat under Arcade > FBNeo only,
-    and Fairchild had no system page. Every folder holding a copy lists it,
-    read from the full paths by_path_suffix records. The game engines'
-    files are grouped by engine (_engine_sections).
-    """
-    files = db.get("files", {})
-    manufacturers: dict[str, dict[str, list]] = {}
-    for sha1, where in _copy_places(db).items():
-        for manufacturer, console in sorted({(m, c) for m, c, _ in where}):
-            manufacturers.setdefault(manufacturer, {}).setdefault(console, []).append(files[sha1])
-    return _engine_sections(manufacturers)
-
-
-def _engine_sections(manufacturers: dict[str, dict[str, list]]) -> dict[str, dict[str, list]]:
-    """A game engine's files in one section headed with the engine's name.
-
-    The Systems index listed ".variants", "scummvm.zip" and "wildmidi.cfg"
-    as games, and ScummVM's files sat on three pages. A folder of variants
-    joins the section of its primary, a file kept at a top's root joins
-    the section of the one engine that reads it (else of the engine the
-    top is named after), the folder under Other named after a game-engine
-    top joins that top, and a folder named by a profile key is headed with
-    the engine's name (SECTION_LABELS), its address unchanged.
-    """
-    from siterender import _PROGRAM_TOPS, SECTION_LABELS, record_card_move  # noqa: PLC0415
-
-    profiles = {key: profile for key, profile in _profiles_on_disk().items()
-                if profile.get("type") not in ("alias", "test")}
-    readers: dict[str, set[str]] = {}
-    for key, profile in profiles.items():
-        for entry in profile.get("files") or []:
-            if isinstance(entry, dict):
-                for name in [entry.get("name"), *(entry.get("aliases") or [])]:
-                    if name:
-                        readers.setdefault(str(name).rsplit("/", 1)[-1], set()).add(key)
-
-    def engine(key: str) -> str:
-        return str(profiles[key].get("emulator") or key)
-
-    def profile_of(name: str) -> str | None:
-        return next((k for k in sorted(profiles)
-                     if name.casefold() in (k.casefold(), engine(k).casefold())), None)
-
-    out: dict[str, dict[str, list]] = {}
-    for mfr, consoles in manufacturers.items():
-        if mfr not in GAME_DATA_TOPS:
-            out[mfr] = dict(consoles)
-            continue
-        own = profile_of(mfr)
-        regrouped: dict[str, list] = {}
-        for console, files in sorted(consoles.items(), key=lambda item: item[0].startswith(".")):
-            for f in files:
-                parts = str(f.get("path", "")).split("/")
-                target = console
-                if console.startswith(".") or (len(parts) == 3 and console == parts[-1]):
-                    # A variant answers its primary's name: scummvm.zip.a93f1c4b.
-                    base = parts[-1].rsplit(".", 1)[0] if console.startswith(".") else parts[-1]
-                    keys = sorted(readers.get(base, ()))
-                    key = keys[0] if len(keys) == 1 else own
-                    found = next((c for c, held in regrouped.items()
-                                  if any(h.get("name") == base for h in held)), None)
-                    target = found or (next((c for c in consoles if c == key), None)
-                                       or (engine(key) if key else base))
-                if target != console:
-                    record_card_move(mfr, console, str(f.get("sha1", "")), mfr, target)
-                regrouped.setdefault(target, []).append(f)
-        out[mfr] = regrouped
-    # The folder under Other an engine top is named after: ScummVM's
-    # engine data under Other/ScummVM, its themes under ScummVM/.
-    others = out.get("Other", {})
-    for folder in sorted(others):
-        top = next((t for t in out if t in GAME_DATA_TOPS and t.casefold() == folder.casefold()),
-                   None)
-        if top is None or top in _PROGRAM_TOPS:
-            continue
-        sections = out[top]
-        target = next((c for c in sections if c.casefold() == folder.casefold()), folder)
-        held = {str(f.get("sha1", "")) for f in sections.get(target, [])}
-        for f in others[folder]:
-            record_card_move("Other", folder, str(f.get("sha1", "")), top, target)
-        sections[target] = [*sections.get(target, []),
-                            *(f for f in others.pop(folder) if str(f.get("sha1", "")) not in held)]
-    for mfr in GAME_DATA_TOPS:
-        for console in out.get(mfr, {}):
-            key = profiles.get(console) and console
-            if key and engine(key) != console:
-                SECTION_LABELS[(mfr, console)] = engine(key)
-    return {mfr: consoles for mfr, consoles in out.items() if consoles}
-
-
 def _console_display(manufacturer: str, console: str) -> str:
     """A console section's name as the reader looks for it: the heading
     system_name gives its id, or the folder alone under a program's top
@@ -6696,15 +6547,6 @@ def _reading_pairs(sha1: str, consumers: FileConsumers) -> list[tuple[str, dict]
     return pairs
 
 
-def _profiles_on_disk() -> dict[str, dict]:
-    return _profiles_in(str(siteinputs.current().emulators), skip_aliases=False)
-
-
-@functools.lru_cache(maxsize=None)
-def _profiles_in(emulators_dir: str, skip_aliases: bool) -> dict[str, dict]:
-    return load_emulator_profiles(emulators_dir, skip_aliases=skip_aliases)
-
-
 def _entry_systems(emulator: str, entry: dict) -> list[str]:
     """The systems an entry is read for: its own `system:` or `systems:`
     (a MAME device set several systems load), else its profile's only one."""
@@ -11356,8 +11198,6 @@ def generate_gap_analysis(
     return "\n".join(lines) + "\n"
 
 
-
-
 def _core_scope_note(config: dict, profiles: dict) -> str:
     """" (all libretro)" when the platform resolves every libretro core.
 

scripts/make_og_cards.py

@@ -70,7 +70,7 @@ def system_cards() -> dict[str, tuple[str, str]]:
     manufacturer and listing its consoles. Read from the same grouping the
     system pages are written from."""
     from common import GAME_DATA_TOPS, load_database
-    from generate_site import _group_by_manufacturer, _slug
+    from sitecollection import _group_by_manufacturer, _slug
 
     def lead(manufacturer: str) -> str:
         # Other holds one folder per emulator, a game publisher's page its

scripts/pr_checks.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Run the checks validate.yml runs on a pull request, in its order.
+
+Usage:
+    python scripts/pr_checks.py
+    python scripts/pr_checks.py --list
+
+Each step runs even when an earlier one failed, as CI runs them, and the
+summary names every step that failed or could not run. A step whose tool is
+absent (npm, python3.8, vermin) is reported as not run, with what provides it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+@dataclass(frozen=True)
+class Step:
+    command: str
+    why: str
+    needs: str = ""
+    provider: str = ""
+    env: dict[str, str] = field(default_factory=dict)
+
+
+# The order of validate.yml: data fetched, site generated and built, then the
+# suite, its 3.8 run and the built-site check.
+STEPS: tuple[Step, ...] = (
+    Step("python scripts/validate_pr.py --changed", "the BIOS files the branch adds"),
+    Step("python scripts/validate_schemas.py --source-only", "profiles and platform files"),
+    Step("python scripts/profile_sync.py --added --strict", "each profile the branch adds"),
+    Step("vermin --no-tips -t=3.8- --violations install.py", "install.py stays on Python 3.8",
+         needs="vermin", provider="the pip line of CONTRIBUTING.md"),
+    Step("python scripts/export_native.py --all --fetch --output-dir tmp/native",
+         "files the exporter tests compare with"),
+    Step("python scripts/refresh_data_dirs.py", "data directories the pack-building tests read"),
+    Step("python scripts/generate_site.py", "the pages the site tests read"),
+    Step("npm ci --prefix website", "the site's packages, Node 24",
+         needs="npm", provider="Node 24"),
+    Step("npm run build --prefix website", "the built site",
+         needs="npm", provider="Node 24"),
+    Step("python -m unittest discover -s tests -t .", "the suite",
+         env={"RETROBIOS_OFFLINE": "1"}),
+    Step("python3.8 -m unittest tests.test_install_tui", "the full screen on the installer's floor",
+         needs="python3.8", provider="a Python 3.8 with pyte",
+         env={"RETROBIOS_OFFLINE": "1"}),
+    Step("python scripts/validate_site.py --site-dir website/dist", "links and page budgets"),
+)
+
+
+def argv(step: Step) -> list[str]:
+    """The command, with `python` run by the interpreter running this script."""
+    words = step.command.split()
+    return [sys.executable, *words[1:]] if words[0] == "python" else words
+
+
+def run(steps: tuple[Step, ...] = STEPS) -> int:
+    failed: list[str] = []
+    not_run: list[str] = []
+    for step in steps:
+        print(f"\n== {step.command}", flush=True)
+        if step.needs and shutil.which(step.needs) is None:
+            print(f"not run: {step.needs} is not installed ({step.provider})")
+            not_run.append(f"{step.command} ({step.needs}: {step.provider})")
+            continue
+        code = subprocess.run(argv(step), cwd=ROOT, env={**os.environ, **step.env}).returncode
+        if code:
+            failed.append(f"{step.command} (exit {code})")
+    print()
+    for line in failed:
+        print(f"FAILED   {line}")
+    for line in not_run:
+        print(f"NOT RUN  {line}")
+    print(f"{len(steps) - len(failed) - len(not_run)}/{len(steps)} checks passed")
+    return 1 if failed else 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
+    parser.add_argument("--list", action="store_true", help="print the steps and exit")
+    args = parser.parse_args()
+    if args.list:
+        for step in STEPS:
+            print(step.command)
+        return 0
+    return run()
+
+
+if __name__ == "__main__":
+    from common import run_cli
+
+    run_cli(main)

scripts/sitecollection.py

@@ -0,0 +1,169 @@
+"""The collection as the site files it: every folder holding each file.
+
+The database keeps one path per content. The folders a copy is also filed
+in, the system sections and the game-engine sections are read here, by the
+page generator and by the tools that draw or fetch per-system images. The
+bios folder comes from siteinputs, never the working directory.
+"""
+
+from __future__ import annotations
+
+import functools
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import siteinputs  # noqa: E402
+from common import GAME_DATA_TOPS, load_emulator_profiles  # noqa: E402
+
+
+def _copy_places(db: dict) -> dict[str, set[tuple[str, str, str]]]:
+    """sha1 -> every (manufacturer, console, file name) holding a copy.
+
+    The database keeps one path per content, deduplication having left one
+    copy. by_path_suffix still records the other folders a copy is filed in,
+    under the name that folder gives it.
+    """
+    files = db.get("files", {})
+    bios_dir = siteinputs.bios_dir(db)
+    places: dict[str, set[tuple[str, str, str]]] = {}
+    for sha1, entry in files.items():
+        parts = str(entry.get("path", "")).split("/")
+        if len(parts) >= 3 and parts[0] == "bios":
+            places.setdefault(sha1, set()).add((parts[1], parts[2], parts[-1]))
+    for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items():
+        parts = tail.split("/")
+        if len(parts) < 3 or parts[1].startswith(".") or len(sha1s) != 1:
+            continue
+        if sha1s[0] in files and (bios_dir / tail).is_file():
+            places.setdefault(sha1s[0], set()).add((parts[0], parts[1], parts[-1]))
+    return places
+
+
+def _slug(manufacturer: str) -> str:
+    return manufacturer.lower().replace(" ", "-")
+
+
+def collection_paths(db: dict) -> dict[str, set[str]]:
+    """Every path under bios/ that holds each file, by sha1.
+
+    The database keeps one path per content. The others are the full tails
+    by_path_suffix records that name a file on disk: RPG Maker, engine
+    trees and the copies filed under several consoles keep 2,502 of them.
+    """
+    files = db.get("files", {})
+    bios_dir = siteinputs.bios_dir(db)
+    paths: dict[str, set[str]] = {}
+    for sha1, entry in files.items():
+        path = str(entry.get("path", ""))
+        if path.startswith("bios/") and path.count("/") >= 2:
+            paths.setdefault(sha1, set()).add(path)
+    for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items():
+        if tail.count("/") < 2 or len(sha1s) != 1 or sha1s[0] not in files:
+            continue
+        if (bios_dir / tail).is_file():
+            paths.setdefault(sha1s[0], set()).add(f"bios/{tail}")
+    return paths
+
+
+def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]:
+    """Group files by manufacturer -> console -> files.
+
+    The database keeps one path per content, and a copy filed elsewhere was
+    on no page of its own: the Channel F BIOS sat under Arcade > FBNeo only,
+    and Fairchild had no system page. Every folder holding a copy lists it,
+    read from the full paths by_path_suffix records. The game engines'
+    files are grouped by engine (_engine_sections).
+    """
+    files = db.get("files", {})
+    manufacturers: dict[str, dict[str, list]] = {}
+    for sha1, where in _copy_places(db).items():
+        for manufacturer, console in sorted({(m, c) for m, c, _ in where}):
+            manufacturers.setdefault(manufacturer, {}).setdefault(console, []).append(files[sha1])
+    return _engine_sections(manufacturers)
+
+
+def _engine_sections(manufacturers: dict[str, dict[str, list]]) -> dict[str, dict[str, list]]:
+    """A game engine's files in one section headed with the engine's name.
+
+    The Systems index listed ".variants", "scummvm.zip" and "wildmidi.cfg"
+    as games, and ScummVM's files sat on three pages. A folder of variants
+    joins the section of its primary, a file kept at a top's root joins
+    the section of the one engine that reads it (else of the engine the
+    top is named after), the folder under Other named after a game-engine
+    top joins that top, and a folder named by a profile key is headed with
+    the engine's name (SECTION_LABELS), its address unchanged.
+    """
+    from siterender import _PROGRAM_TOPS, SECTION_LABELS, record_card_move  # noqa: PLC0415
+
+    profiles = {key: profile for key, profile in _profiles_on_disk().items()
+                if profile.get("type") not in ("alias", "test")}
+    readers: dict[str, set[str]] = {}
+    for key, profile in profiles.items():
+        for entry in profile.get("files") or []:
+            if isinstance(entry, dict):
+                for name in [entry.get("name"), *(entry.get("aliases") or [])]:
+                    if name:
+                        readers.setdefault(str(name).rsplit("/", 1)[-1], set()).add(key)
+
+    def engine(key: str) -> str:
+        return str(profiles[key].get("emulator") or key)
+
+    def profile_of(name: str) -> str | None:
+        return next((k for k in sorted(profiles)
+                     if name.casefold() in (k.casefold(), engine(k).casefold())), None)
+
+    out: dict[str, dict[str, list]] = {}
+    for mfr, consoles in manufacturers.items():
+        if mfr not in GAME_DATA_TOPS:
+            out[mfr] = dict(consoles)
+            continue
+        own = profile_of(mfr)
+        regrouped: dict[str, list] = {}
+        for console, files in sorted(consoles.items(), key=lambda item: item[0].startswith(".")):
+            for f in files:
+                parts = str(f.get("path", "")).split("/")
+                target = console
+                if console.startswith(".") or (len(parts) == 3 and console == parts[-1]):
+                    # A variant answers its primary's name: scummvm.zip.a93f1c4b.
+                    base = parts[-1].rsplit(".", 1)[0] if console.startswith(".") else parts[-1]
+                    keys = sorted(readers.get(base, ()))
+                    key = keys[0] if len(keys) == 1 else own
+                    found = next((c for c, held in regrouped.items()
+                                  if any(h.get("name") == base for h in held)), None)
+                    target = found or (next((c for c in consoles if c == key), None)
+                                       or (engine(key) if key else base))
+                if target != console:
+                    record_card_move(mfr, console, str(f.get("sha1", "")), mfr, target)
+                regrouped.setdefault(target, []).append(f)
+        out[mfr] = regrouped
+    # The folder under Other an engine top is named after: ScummVM's
+    # engine data under Other/ScummVM, its themes under ScummVM/.
+    others = out.get("Other", {})
+    for folder in sorted(others):
+        top = next((t for t in out if t in GAME_DATA_TOPS and t.casefold() == folder.casefold()),
+                   None)
+        if top is None or top in _PROGRAM_TOPS:
+            continue
+        sections = out[top]
+        target = next((c for c in sections if c.casefold() == folder.casefold()), folder)
+        held = {str(f.get("sha1", "")) for f in sections.get(target, [])}
+        for f in others[folder]:
+            record_card_move("Other", folder, str(f.get("sha1", "")), top, target)
+        sections[target] = [*sections.get(target, []),
+                            *(f for f in others.pop(folder) if str(f.get("sha1", "")) not in held)]
+    for mfr in GAME_DATA_TOPS:
+        for console in out.get(mfr, {}):
+            key = profiles.get(console) and console
+            if key and engine(key) != console:
+                SECTION_LABELS[(mfr, console)] = engine(key)
+    return {mfr: consoles for mfr, consoles in out.items() if consoles}
+
+
+def _profiles_on_disk() -> dict[str, dict]:
+    return _profiles_in(str(siteinputs.current().emulators), skip_aliases=False)
+
+
+@functools.lru_cache(maxsize=None)
+def _profiles_in(emulators_dir: str, skip_aliases: bool) -> dict[str, dict]:
+    return load_emulator_profiles(emulators_dir, skip_aliases=skip_aliases)

scripts/siteexports.py

@@ -22,9 +22,9 @@ API_EXPORTS = "api/v1/exports"
 
 # Report key, then how a line of it reads on the page.
 CHANGE_KINDS = (
-    ("hashes_corrected", "Hashes updated to the emulator source"),
+    ("hashes_updated", "Hashes updated to the emulator source"),
     ("hashes_filled", "Hashes added where the original lists none"),
-    ("sizes_corrected", "Sizes updated to the emulator source"),
+    ("sizes_updated", "Sizes updated to the emulator source"),
     ("sizes_removed", "Sizes no longer stated"),
     ("hashes_cleared", "Hashes no longer listed"),
     ("withheld", "Changes not applied"),
@@ -33,9 +33,9 @@ CHANGE_KINDS = (
 
 # Report key, then the kind each line of `changes` carries.
 CHANGE_RECORDS = {
-    "hashes_corrected": "hash_corrected",
+    "hashes_updated": "hash_updated",
     "hashes_filled": "hash_filled",
-    "sizes_corrected": "size_corrected",
+    "sizes_updated": "size_updated",
     "sizes_removed": "size_removed",
     "hashes_cleared": "hash_cleared",
 }
@@ -52,11 +52,11 @@ def describe_change(change: dict) -> str:
     """One change in words, with the value before and after it."""
     label = f"`{change['system']}/{change['name']}` {change['field']}"
     kind, old, new = change["kind"], change["old"], change["new"]
-    if kind in ("hash_corrected", "size_corrected"):
+    if kind in ("hash_updated", "size_updated"):
         return f"{label} {_value(old)} becomes {_value(new)}"
     if kind == "hash_filled":
         return f"{label} {_value(new)} added"
-    if kind == "requirement_corrected":
+    if kind == "required_aligned":
         return f"`{change['system']}/{change['name']}` becomes {'required' if new else 'optional'}"
     return f"{label} {_value(old)} no longer listed"
 
@@ -76,7 +76,7 @@ def requirement_lines(report: dict) -> list[str]:
     """Each required flag aligned with the code, and what it is now."""
     items = [
         describe_change(c) for c in report.get("changes") or []
-        if c["kind"] == "requirement_corrected"
+        if c["kind"] == "required_aligned"
     ]
     if not items:
         return []
@@ -108,11 +108,11 @@ def change_text(report: dict) -> str:
     parts = [f"{_plural(counts['kept'], 'entry', 'entries')} kept"]
     extra = [
         (counts["added"], "file", "files", "added"),
-        (len(report.get("hashes_corrected") or []), "hash", "hashes", "updated"),
-        (report.get("requirements_corrected") or 0, "required flag", "required flags",
+        (len(report.get("hashes_updated") or []), "hash", "hashes", "updated"),
+        (report.get("required_aligned") or 0, "required flag", "required flags",
          "aligned with the code"),
         (len(report.get("hashes_filled") or []), "hash", "hashes", "added"),
-        (len(report.get("sizes_corrected") or []), "size", "sizes", "updated"),
+        (len(report.get("sizes_updated") or []), "size", "sizes", "updated"),
     ]
     parts += [f"{_plural(n, one, many)} {verb}" for n, one, many, verb in extra if n]
     return ", ".join(parts)
@@ -216,7 +216,7 @@ def _display(platform: str, names: dict[str, str]) -> str:
     return names.get(platform) or platform
 
 
-def _corrected_link(report: dict) -> str:
+def _updated_link(report: dict) -> str:
     platform, files = report["platform"], report["files"]
     if len(files) == 1:
         path = f"{EXPORTS_DIR}/{platform}/{files[0]}"
@@ -286,7 +286,7 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
         platform = report["platform"]
         lines.append(
             f"| [{_display(platform, names)}](#{platform}) | "
-            f"{_original_link(report)} | {_corrected_link(report)} | "
+            f"{_original_link(report)} | {_updated_link(report)} | "
             f"{change_text(report)} |"
         )
     lines.append("")

scripts/siteinputs.py

@@ -30,3 +30,14 @@ def current() -> SiteInputs:
 
 def use(inputs: SiteInputs) -> None:
     _current[0] = inputs
+
+
+def bios_dir(db: dict) -> Path:
+    """The database's bios folder, as the inputs place it.
+
+    database.json names it relative to its own folder ("bios"). Read from the
+    working directory instead, a tool started elsewhere found no copy on disk
+    and dropped five system sections and 2,500 collection paths in silence.
+    """
+    named = Path(str(db.get("bios_dir") or "bios"))
+    return named if named.is_absolute() else current().db.parent / named

scripts/sitemeasurements.py

@@ -733,8 +733,8 @@ def data_page_section(path: Path) -> list[str]:
         "",
         "[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe "
         f"record kept in [`{mr.RECORDS_DIR}/`]({REPO}/tree/main/{mr.RECORDS_DIR}), and "
-        "`downloads/measurements/<emulator>.sqlite` the records of one emulator, linked "
-        f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads "
+        "`downloads/measurements/<emulator>.sqlite` the records of one emulator. The "
+        f"[Measurements]({EXPLORER_PAGE}) page reads "
         "[`data/measurements/index.json`](data/measurements/index.json). A record line is "
         "valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json).",
         "",

scripts/siterender.py

@@ -780,7 +780,7 @@ def _ensure_console_sections() -> None:
     import json
 
     from common import load_emulator_profiles
-    from generate_site import _group_by_manufacturer
+    from sitecollection import _group_by_manufacturer
 
     inputs = siteinputs.current()
     with open(inputs.db, encoding="utf-8") as handle:

scripts/sitesources.py

@@ -16,9 +16,10 @@ from dataclasses import dataclass
 from pathlib import Path
 
 import upstream
-from siterender import _by_mode
+from siterender import _by_mode, _escape_tags
 from sourceref import (
-    _HEX_ADDRESS_RE, _prose_runs, is_binary_citation, ref_role, source_ref_values,
+    _HEX_ADDRESS_RE, _prose_runs, is_binary_citation, ref_annotation, ref_role,
+    source_ref_values,
     split_source_ref,
 )
 
@@ -512,7 +513,7 @@ def source_ref_items(profile: dict, value: str | dict | None) -> list[tuple[str,
         for part in split_source_ref(refs):
             if is_binary_citation(part.path):
                 text, binary = _binary_citation(part.path, binary)
-                rendered.append((text, ""))
+                rendered.append((text + _annotation_markdown(ref_annotation(part.raw)), ""))
                 continue
             binary = ""
             role, bare = ref_role(part.path)
@@ -544,19 +545,31 @@ def source_ref_items(profile: dict, value: str | dict | None) -> list[tuple[str,
 
             role = (role_label(profile, current_role, part.path)
                     if current_role and _two_repositories(profile) else "")
+            note = _annotation_markdown(ref_annotation(part.raw))
             if selected is None:
-                rendered.append((f"`{display}`", role))
+                rendered.append((f"`{display}`{note}", role))
             else:
                 repo, pin = selected
                 url = _source_permalink(
                     repo, pin, real_path, part.start, part.end
                 )
-                rendered.append((f"[`{display}`]({url})", role))
+                rendered.append((f"[`{display}`]({url}){note}", role))
 
         groups.append((label, rendered or [(f"`{refs}`", "")]))
     return groups
 
 
+def _annotation_markdown(text: str) -> str:
+    """A part's annotation after its citation, as written: `(lsf default)`.
+
+    Profile prose, so placeholders keep their brackets and nothing in it
+    opens emphasis or a table cell.
+    """
+    if not text:
+        return ""
+    return " " + _escape_tags(re.sub(r"([*|\\])", r"\\\1", text))
+
+
 def _binary_citation(path: str, previous: str) -> tuple[str, str]:
     """A part citing code inside a shipped binary, and the binary it names.
 

scripts/sourceref.py

@@ -123,6 +123,25 @@ def split_source_ref(ref: str) -> list[RefPart]:
     return parts
 
 
+_LOCATION_RE = re.compile(r":\d+(?:-\d+)?|^\s*\d+(?:-\d+)?(?=\s|$)")
+
+
+def ref_annotation(raw: str) -> str:
+    """The words a part writes beside its location, for a reader.
+
+    split_source_ref drops them because a comparison has no use for them,
+    but `src/loader2.cpp:284 (lsf default), src/netcfg.cpp:204 (lsf
+    override)` says which line wins only through them, and 1,103 of the
+    profiles' annotations were on no page.
+    """
+    paren = raw.find("(")
+    head = raw if paren < 0 else raw[:paren]
+    located = _LOCATION_RE.search(head)
+    if located:
+        return raw[located.end():].strip()
+    return raw[paren:].strip() if paren >= 0 else ""
+
+
 def ref_prose_text(ref: str) -> str:
     """The text of a source_ref its parts do not read, positions kept.