feat: round 4 reference and data

release.json

@@ -1,4 +1,5 @@
 {
+  "schema_version": 1,
   "tag": "v2026.10.10",
   "packs": {
     "Batocera_43.1_BIOS_Pack.zip": {

schemas/install-manifest.schema.json

@@ -23,21 +23,24 @@
   ],
   "properties": {
     "manifest_version": {
-      "const": 2
+      "const": 2,
+      "description": "Format version of the manifest. install.py refuses a version it does not know."
     },
     "source": {
       "enum": [
         "full",
         "platform",
         "truth"
-      ]
+      ],
+      "description": "Which files the pack was built from: full (the platform's list plus the files its cores load), platform (the list alone) or truth (the cores' files alone)."
     },
     "regions": {
       "type": "array",
       "items": {
         "type": "string"
       },
-      "uniqueItems": true
+      "uniqueItems": true,
+      "description": "Region priority the manifest was narrowed to, best first. Empty when it was not narrowed."
     },
     "target": {
       "type": "string",
@@ -79,13 +82,15 @@
       "type": "array",
       "items": {
         "type": "object"
-      }
+      },
+      "description": "Rules install.py follows to find this setup and its BIOS folder on a machine, one per method and operating system."
     },
     "standalone_copies": {
       "type": "array",
       "items": {
         "type": "object"
-      }
+      },
+      "description": "Files install.py also copies into the folders of standalone emulators it finds, or a note it prints when their folder exists."
     },
     "total_files": {
       "type": "integer",
@@ -497,7 +502,7 @@
         },
         "repo_path": {
           "type": "string",
-          "description": "Path in the repository. Fetched from https://raw.githubusercontent.com/Abdess/retrobios/<ref>/<repo_path>, ref being main unless RETROBIOS_REF names a tag."
+          "description": "Path in the repository. Fetched from `https://raw.githubusercontent.com/Abdess/retrobios/<ref>/<repo_path>`, ref being main unless RETROBIOS_REF names a tag."
         },
         "cores": {
           "oneOf": [
@@ -521,7 +526,7 @@
         "release_asset": {
           "type": "string",
           "pattern": "^[^/\\\\]+$",
-          "description": "Asset of the large-files release, fetched from https://github.com/Abdess/retrobios/releases/download/large-files/<asset> instead of repo_path."
+          "description": "Asset of the large-files release, fetched from `https://github.com/Abdess/retrobios/releases/download/large-files/<asset>` instead of repo_path."
         },
         "systems": {
           "description": "Every platform system that declares the file, for the installer's --system.",

schemas/release.schema.json

@@ -5,8 +5,12 @@
   "description": "What the packs of the latest release hold, read from the archives when the release is cut.",
   "type": "object",
   "additionalProperties": false,
-  "required": ["tag", "packs"],
+  "required": ["schema_version", "tag", "packs"],
   "properties": {
+    "schema_version": {
+      "const": 1,
+      "description": "Format version of this record, raised when a field changes meaning or goes away."
+    },
     "tag": { "type": "string", "pattern": "^v[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}$" },
     "packs": {
       "type": "object",

schemas/site-api-envelope.schema.json

@@ -403,7 +403,17 @@
                 },
                 "sha1": {
                   "type": "string",
-                  "pattern": "^[0-9a-f]{40}$"
+                  "pattern": "^[0-9a-f]{40}$",
+                  "description": "SHA1 of the collection file the value designates. A platform or emulator record that designates no collection file has none."
+                },
+                "declared": {
+                  "enum": [
+                    "sha1",
+                    "md5",
+                    "crc32",
+                    "sha256"
+                  ],
+                  "description": "On a copy of a platform or emulator record filed under another hash of its file: the algorithm of the value its owner wrote."
                 },
                 "owner": {
                   "type": "string"

scripts/check_platform_folder.py

@@ -14,49 +14,50 @@ from pathlib import Path
 API = "https://abdess.github.io/retrobios/api/v1/platforms/"
 
 
-# The digest each check mode compares. A mode missing here (existence) only
-# asks that the file be present.
+# The digest each mode compares. Without one (existence) a file need only be present.
 DIGESTS = {"md5": "md5", "sha1": "sha1"}
 
 
 def accepts(entry: dict, path: Path, mode: str) -> bool:
     """Whether the platform's own check passes the file at path."""
-    digest = DIGESTS.get(mode)
-    raw = entry.get(mode) or ""
-    values = raw if isinstance(raw, list) else str(raw).split(",")
-    declared = [value.strip().lower() for value in values if value.strip()]
-    if digest is None or not declared:
+    digest, raw = DIGESTS.get(mode), entry.get(mode) or ""
+    values = [v.strip().lower() for v in (raw if isinstance(raw, list) else str(raw).split(",")) if v.strip()]
+    if digest is None or not values:
         return path.is_file()  # nothing to compare: the platform checks presence
     try:
         if entry.get("zipped_file"):  # the hash of one member, its name casefolded
             with zipfile.ZipFile(path) as archive:
                 names = {name.casefold(): name for name in archive.namelist()}
                 member = names.get(entry["zipped_file"].casefold())
-                return bool(member) and hashlib.new(digest, archive.read(member)).hexdigest() in declared
+                return bool(member) and hashlib.new(digest, archive.read(member)).hexdigest() in values
         actual = hashlib.new(digest, path.read_bytes()).hexdigest()
-        if entry.get("zip_digest") != "composite" and any(actual.startswith(d) for d in declared):
+        if entry.get("zip_digest") != "composite" and any(actual.startswith(v) for v in values):
             return True  # a truncated md5 (Batocera) compares by its prefix
         if digest == DIGESTS["md5"] and path.suffix.lower() == ".zip":  # Recalbox: members in name order
             with zipfile.ZipFile(path) as archive:
                 names = sorted(name for name in archive.namelist() if not name.endswith("/"))
-                return hashlib.md5(b"".join(map(archive.read, names))).hexdigest() in declared
+                return hashlib.md5(b"".join(map(archive.read, names))).hexdigest() in values
     except (OSError, zipfile.BadZipFile):
         return False
     return False
 
 
 def check(platform: str, root: Path, api: str = API) -> list[tuple[str, str]]:
-    """(destination, ok | wrong | missing) for each entry of the list."""
+    """(destination, ok | wrong | missing) for each entry. Entries with no destination
+    (EmuDeck) name no file: their system passes when any file in root has their hash."""
     with urllib.request.urlopen(f"{api}{platform}.json") as response:
         contract = json.load(response)["items"][0]["contract"]
-    mode = contract["verification_mode"]
-    results = []
-    for system in contract["systems"].values():
-        for entry in system.get("files") or []:
-            destination = entry.get("destination") or entry["name"]
-            path = root / destination
+    mode, results = contract["verification_mode"], []
+    loose = [path for path in root.iterdir() if path.is_file()] if root.is_dir() else []
+    for system_id, system in contract["systems"].items():
+        listed = [e for e in system.get("files") or [] if not e.get("destination")]
+        for entry in (e for e in system.get("files") or [] if e.get("destination")):
+            path = root / entry["destination"]
             verdict = "ok" if accepts(entry, path, mode) else "wrong" if path.is_file() else "missing"
-            results.append((destination, verdict))
+            results.append((entry["destination"], verdict))
+        if listed:
+            held = any(accepts(e, path, mode) for e in listed for path in loose)
+            results.append((f"{system_id}: any file", "ok" if held else "missing"))
     return results
 
 

scripts/generate_site.py

@@ -1758,8 +1758,11 @@ def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]:
     api/v1/systems/ is keyed by manufacturer and its consoles by display
     name: a frontend keying by system id (sony-playstation) had no way in.
     The url is the section system_section gives, the one every page links.
+    An id with no section of its own (tyrian, scummvm) that holds files
+    takes the card of its first file, else the page of the emulator naming
+    it: 28 such ids sent their hashes to no page.
     """
-    from siterender import system_ids, system_section
+    from siterender import card_page, system_ids, system_section
 
     sha1_by_path = _sha1_by_path(db)
     files = db.get("files") or {}
@@ -1795,16 +1798,33 @@ def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]:
             name = system_name(sys_id)
         except KeyError:
             name = sys_id
-        items.append({
-            "id": sys_id,
-            "name": name,
-            "manufacturer": manufacturers.get(section[0], "") if section else "",
-            "url": _site_path(console_href(*section)) if section else None,
-            "sha1": sorted(held.get(sys_id, set()) & files.keys()),
-        })
+        sha1s = sorted(held.get(sys_id, set()) & files.keys())
+        manufacturer = manufacturers.get(section[0], "") if section else ""
+        url = _site_path(console_href(*section)) if section else None
+        if url is None and sha1s:
+            manufacturer, url = _first_file_place(sys_id, sha1s, files, profiles, card_page)
+        items.append({"id": sys_id, "name": name, "manufacturer": manufacturer,
+                      "url": url, "sha1": sha1s})
     return items
 
 
+def _first_file_place(
+    sys_id: str, sha1s: list[str], files: dict, profiles: dict, card_page,
+) -> tuple[str, str | None]:
+    """(manufacturer, url) of the card of an id's first collection file,
+    else of the page of the first emulator naming the id."""
+    for sha1 in sha1s:
+        parts = str(files[sha1].get("path", "")).split("/")
+        if len(parts) >= 3 and parts[0] == "bios":
+            page = card_page("/".join(parts), sha1)
+            return parts[1], _site_path(f"systems/{page}.md#sha1-{sha1[:12]}")
+    for key in sorted(profiles):
+        profile = profiles[key]
+        if profile.get("type") not in ("alias", "test") and sys_id in (profile.get("systems") or []):
+            return "", _site_path(f"emulators/{key}.md")
+    return "", None
+
+
 def core_index_items(profiles: dict) -> list[dict]:
     """Every name a core goes by, with the profile it designates.
 
@@ -1834,14 +1854,28 @@ def _is_manifest(item: dict) -> bool:
     return item["url"].startswith("api/v1/manifests/")
 
 
+# A URL template written in prose (https://.../<ref>/<repo_path>): escaped,
+# the autolinker made a link of it, &lt;ref> on screen and a 404 behind it.
+_URL_TEMPLATE = re.compile(r"(?<!`)https?://[^\s`]*<[^\s`]*?(?=[.,;]?(?:\s|$))")
+
+
 def _schema_field_rows(schema: dict, keys: tuple[str, ...]) -> list[str]:
     """One table row per field, its description read from the schema, so the
-    page and the published contract say the same thing."""
+    page and the published contract say the same thing. A URL template is
+    written as code."""
     props = schema.get("properties") or {}
-    return [
-        f"| `{key}` | {_escape_tags(str(props[key]['description']))} |"
-        for key in keys if (props.get(key) or {}).get("description")
-    ]
+    rows = []
+    for key in keys:
+        text = str((props.get(key) or {}).get("description") or "")
+        if text:
+            rows.append(f"| `{key}` | {_escape_tags(_URL_TEMPLATE.sub(r'`\g<0>`', text))} |")
+    return rows
+
+
+def _other_fields(schema: dict, shown: tuple[str, ...]) -> list[str]:
+    """The fields a short table leaves out, every one with its description."""
+    rest = tuple(key for key in schema.get("properties") or {} if key not in shown)
+    return ["| Field | Meaning |", "|-------|---------|", *_schema_field_rows(schema, rest)]
 
 
 # The manifest fields an integrator reads first, in the order they matter.
@@ -1859,12 +1893,34 @@ def _short_sha256(item: dict, catalog: str) -> str:
     return f'<a href="{catalog}"><code title="{value}">{value[:12]}</code></a>'
 
 
+def _platforms_without_a_list(platforms_dir: Path | None = None) -> list[str]:
+    """The names of the platforms whose registry says they keep no list."""
+    platforms_dir = platforms_dir or _REPO_ROOT / "platforms"
+    names = []
+    for key, entry in sorted(load_platform_registry(str(platforms_dir)).items()):
+        if entry.get("native_list") != "none":
+            continue
+        config = platforms_dir / str(entry.get("config") or f"{key}.yml")
+        data = yaml_load(config.read_text(encoding="utf-8")) if config.is_file() else {}
+        names.append(str((data or {}).get("platform") or key))
+    return names
+
+
+def _join_names(names: list[str]) -> str:
+    return names[0] if len(names) == 1 else ", ".join(names[:-1]) + f" and {names[-1]}"
+
+
 def generate_data_page(exports: list[dict]) -> str:
     manifest_schema = json.loads(
         (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8"))
     snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8")
     manifests = [item for item in exports if _is_manifest(item)]
+    no_list = _platforms_without_a_list()
     lines = [
+        "---",
+        "terms: manifest, schema, check a folder, rate limit, api",
+        "---",
+        "",
         "# Data and API",
         "",
         "RetroBIOS publishes the same metadata used by its verifier, pack builder "
@@ -1907,6 +1963,10 @@ def generate_data_page(exports: list[dict]) -> str:
         "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. "
         "Each schema's `$id` is the address it is served at.",
         "",
+        "`database.json`, `stats.json`, `release.json` and the manifests are not "
+        "envelopes. Their revision is the `commit` of `catalog.json` that lists "
+        "their SHA256.",
+        "",
         "One emulator profile without the whole list: "
         "`api/v1/emulators/<id>.json`, the envelope of `emulators.json` holding "
         "that one item, its `profile` valid against "
@@ -1923,7 +1983,10 @@ def generate_data_page(exports: list[dict]) -> str:
         "`url` of the section holding its files and their SHA1. "
         "[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a "
         "platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), "
-        "whose document is `api/v1/emulators/<profile>.json`.",
+        "whose document is `api/v1/emulators/<profile>.json`. A name two profiles "
+        "claim (`pcsx2`) has one item per profile, and `preferred: true` marks the "
+        "one a platform list naming that core gets: the profile with that very id, "
+        "else the first id in alphabetical order.",
         "",
         "One hash without the whole list: `api/v1/lookup/<xx>.json`, where `<xx>` "
         "is the first two characters of the hash in lowercase, holds every SHA1, "
@@ -1931,7 +1994,27 @@ def generate_data_page(exports: list[dict]) -> str:
         "(the collection, a data directory, a platform's list or an emulator "
         "profile), the file `name`, the `url` of the page that shows it and, "
         "when one is known, the `system_name` the site gives its system. "
-        "[Find a file](files/index.md) reads it.",
+        "[Find or check a file](files/index.md) reads it.",
+        "",
+        "In a browser, the SHA1 of a file the reader picks finds its records. Web "
+        "Crypto has no MD5, and none is needed: a setup that lists a file by MD5 "
+        "or CRC32 is filed under the SHA1 of the collection file it names too.",
+        "",
+        "```js",
+        "// The records of the file a reader picked, by its SHA1.",
+        "async function lookUp(file) {",
+        "  const digest = await crypto.subtle.digest(\"SHA-1\", await file.arrayBuffer());",
+        "  const sha1 = [...new Uint8Array(digest)]",
+        "    .map((byte) => byte.toString(16).padStart(2, \"0\"))",
+        "    .join(\"\");",
+        f"  const base = \"{SITE_URL}api/v1/lookup/\";",
+        "  const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);",
+        "  if (!response.ok) return [];",
+        "  const shard = await response.json();",
+        "  // layer: collection, data-directory, platform or emulator. owner: which one.",
+        "  return shard.items.filter((item) => item.value === sha1);",
+        "}",
+        "```",
         "",
         "Two kinds of `url`:",
         "",
@@ -1965,6 +2048,15 @@ def generate_data_page(exports: list[dict]) -> str:
         "|-------|---------|",
         *_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS),
         "",
+        *markup.details("note", "All fields", [
+            "The manifest:",
+            "",
+            *_other_fields(manifest_schema, MANIFEST_TOP_FIELDS),
+            "",
+            "Each entry of `files`:",
+            "",
+            *_other_fields(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS),
+        ]),
         "## Check a folder for one platform",
         "",
         "The job most tools come for: is this BIOS folder what the platform checks? "
@@ -1979,8 +2071,12 @@ def generate_data_page(exports: list[dict]) -> str:
         "Recalbox writes several accepted md5 in one field, comma separated.",
         "- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members "
         "read in name order, never the archive's.",
+        "- An entry with no `destination` (EmuDeck) names no file: the system "
+        "passes when any file in the BIOS folder has one of its md5.",
         "- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. "
         "The script below does not apply it.",
+        *([f"- {_join_names(no_list)} have no list of their own (`native_list: none`). "
+           "An empty list does not mean nothing is needed."] if no_list else []),
         "",
         "```python",
         snippet.rstrip("\n"),
@@ -2057,13 +2153,17 @@ def generate_data_page(exports: list[dict]) -> str:
         "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's "
         "own file does not carry.",
         "",
-        "## Hosting and limits",
+        "## Hosting, rate limits and caching",
         "",
         "- Static files on GitHub Pages, read with a plain GET. No key, no account.",
         "- Any origin may read them (`Access-Control-Allow-Origin: *`), so a "
         "browser tool can call them directly.",
         "- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). "
         "Send the `ETag` back in `If-None-Match` to skip an unchanged download.",
+        "- GitHub may answer HTTP 429 to a client that sends too many requests, and "
+        "a Pages site has a soft limit of 100 GB of traffic a month ([GitHub Pages "
+        "limits](https://docs.github.com/en/pages/getting-started-with-github-pages/"
+        "github-pages-limits)). Fetch the one shard or document a question needs.",
         "- The base is the project's address, "
         f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.",
         "",
@@ -3855,6 +3955,8 @@ def generate_platform_page(
 
     if no_list:
         lines.extend(_core_folders_section(name, config))
+    if _own_list_only(config):
+        lines.extend([f"**Cores:** {OWN_LIST_ONLY}", ""])
 
     # System summary table (quick navigation)
     lines.extend(
@@ -3885,9 +3987,12 @@ def generate_platform_page(
         if len(sys_emus) > 3:
             emu_str += f" [+{len(sys_emus) - 3}](#{anchor})"
         # The name a reader filters on, the id the list writes beside it.
+        # The id and the count are no words a reader searches for: indexed,
+        # every platform page answered a system id before its system's page.
         lines.append(
-            f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) <small>{sys_id}</small> "
-            f"| {ok_count}/{total} | {status} | {emu_str} |"
+            f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) "
+            f"<small data-pagefind-ignore>{sys_id}</small> "
+            f"| <span data-pagefind-ignore>{ok_count}/{total}</span> | {status} | {emu_str} |"
         )
     lines.append("")
 
@@ -3911,7 +4016,13 @@ def generate_files_page() -> str:
     address and opens the card directly.
     """
     return "\n".join([
-        "# Find a file",
+        "---",
+        "terms: check dump, md5 check, verify bios file, check hash",
+        "---",
+        "",
+        "# Find or check a file",
+        "",
+        "Check a BIOS dump: paste its hash or choose the file.",
         "",
         "A hash is a fingerprint of a file's contents ([Glossary](../wiki/glossary.md)). "
         "Tools such as clrmamepro and RomVault print one, and so do emulator logs "
@@ -3944,7 +4055,7 @@ def generate_files_page() -> str:
         "## The index behind it",
         "",
         "The lookup reads one file of [`api/v1/lookup/`](../data.md), named by the "
-        "first two characters of the hash, about 20 KB compressed. "
+        "first two characters of the hash: one of 256, never the whole index. "
         "[`api/v1/file-locations.json`](../api/v1/file-locations.json) lists every "
         "card by SHA1 in one file.",
         "",
@@ -4147,11 +4258,46 @@ def file_lookup_records(
                             "owner": key, "name": name, "url": url,
                             **({"system_name": entry_label} if entry_label else {}),
                         })
+    records += _held_under_every_hash(records, files, db.get("indexes") or {})
     unique = {json.dumps(r, sort_keys=True): r for r in records}
     return sorted(unique.values(), key=lambda r: (
         r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"]))
 
 
+def _collection_sha1(files: dict, indexes: dict, algorithm: str, value: str) -> str | None:
+    """The collection file a declared value designates, None when none does."""
+    if algorithm == "sha1":
+        return value if value in files else None
+    found = indexes.get(f"by_{algorithm}", {}).get(value)
+    if isinstance(found, list):
+        found = found[0] if len(found) == 1 else None
+    return found if found in files else None
+
+
+def _held_under_every_hash(records: list[dict], files: dict, indexes: dict) -> list[dict]:
+    """A platform or emulator record whose value designates a collection
+    file, written under the file's other hashes too.
+
+    Batocera lists scph5501.bin by md5: pasted by its sha1, the file showed
+    no setup at all. Each record now carries the file's sha1, and
+    ``declared`` names the hash its owner wrote when the record is a copy.
+    """
+    copies: list[dict] = []
+    for record in records:
+        if record["layer"] not in ("platform", "emulator"):
+            continue
+        sha1 = _collection_sha1(files, indexes, record["algorithm"], record["value"])
+        if sha1 is None:
+            continue
+        record["sha1"] = sha1
+        for algorithm in HASH_ALGORITHMS:
+            for value in _hash_values(files[sha1].get(algorithm)):
+                if value != record["value"]:
+                    copies.append({**record, "value": value, "algorithm": algorithm,
+                                   "declared": record["algorithm"]})
+    return copies
+
+
 def write_lookup_shards(
     api: Path, records: list[dict], generated_at: str, commit: str = "",
 ) -> int:
@@ -8289,6 +8435,26 @@ def _ratio_cell(done: int, total: int, alarm: int = 0) -> str:
     return cell
 
 
+# GitHub's upload page for a folder of bios/: a browser upload by someone
+# without write access lands on a fork and opens a pull request.
+UPLOAD_URL = f"{REPO_URL}/upload/main/bios"
+# Rows of Files to source on one page: 601 on one page made a phone page
+# 274,509 px long.
+FILES_TO_SOURCE_PART = 100
+
+
+def _upload_folder(profile: dict) -> str:
+    """The bios/ folder a file the emulator loads is uploaded into: the
+    console folder of its first system that has one, else none."""
+    from siterender import _own_section
+
+    for sys_id in profile.get("systems") or []:
+        own = _own_section(str(sys_id))
+        if own is not None:
+            return "/".join(own)
+    return ""
+
+
 def generate_gap_analysis(
     profiles: dict,
     coverages: dict,
@@ -8702,9 +8868,11 @@ def generate_gap_analysis(
                     f"[{_escape_tags(display)}](emulators/{key}.md)"
                     for display, key in zip(m["emulators"], m["emu_keys"])
                 )
+                folder = _upload_folder(profiles[m["emu_keys"][0]])
+                upload = f"{UPLOAD_URL}/{urllib.parse.quote(folder)}" if folder else UPLOAD_URL
                 rows.append(
-                    f"| `{m['name']}` | {destination} | {expected} | {emulators} | {req} | "
-                    f"{plat_badges} | "
+                    f"| `{m['name']}` [Have it? Upload]({upload}) | {destination} | {expected} "
+                    f"| {emulators} | {req} | {plat_badges} | "
                     f"{_source_ref_markdown(profiles[m['emu_keys'][0]], m['source_ref'])} |"
                 )
             lines.extend(["### Files to source", "", counted, ""])
@@ -8716,7 +8884,8 @@ def generate_gap_analysis(
                     f"{counted} Each is a file an emulator loads that no platform lists and "
                     "the collection does not hold. Part of "
                     "[What the packs do not hold](gaps.md#files-to-source).",
-                    head, [[row] for row in rows], moved,
+                    head, [rows[i:i + FILES_TO_SOURCE_PART] for i in range(
+                        0, len(rows), FILES_TO_SOURCE_PART)], moved, budget=1,
                     parent=("What the packs do not hold", "/gaps/"),
                 )
                 lines.extend([f"Listed on {where}.", ""])
@@ -8893,6 +9062,16 @@ def _xref_platform_rows(config: dict, unique: dict) -> tuple[int, list[str]]:
     return len(matched), rows
 
 
+# What a platform that declares no core (cores: []) is told, on its page
+# and in the cross-reference.
+OWN_LIST_ONLY = "None. Its own list is the whole truth: no core profile adds a file."
+
+
+def _own_list_only(config: dict) -> bool:
+    """The platform declares `cores: []`: its own list decides alone."""
+    return isinstance(config.get("cores"), list) and not config["cores"]
+
+
 def _render_xref_by_platform(
     coverages: dict, unique: dict, moved: dict[str, str] | None = None,
 ) -> list[str]:
@@ -8913,6 +9092,12 @@ def _render_xref_by_platform(
         config = cov["config"]
         count, rows = _xref_platform_rows(config, unique)
         stem = f"cross-reference-{pname}"
+        if _own_list_only(config):
+            # cores: [] (MiSTer, BizHawk): no profile describes what they
+            # read, and a page of cores would be an empty table.
+            lines.append(f"| [{_escape_tags(display)}](platforms/{pname}.md) | 0 "
+                         f"| {OWN_LIST_ONLY} |")
+            continue
         lines.append(
             f"| [{_escape_tags(display)}](platforms/{pname}.md) | {count}"
             f"{_core_scope_note(config, unique)} | [Cores on {_escape_tags(display)}]({stem}.md) |"
@@ -10034,6 +10219,12 @@ def _generate(args: argparse.Namespace) -> None:
     )
     for stem, page in xref_pages.items():
         write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page))
+    for pname, cov in coverages.items():
+        if _own_list_only(cov["config"]):
+            # The empty table it had is gone: its address leads to the
+            # platform's page, which says its own list decides alone.
+            sitestarlight.MOVED_PAGES.setdefault(
+                f"cross-reference-{pname}.md", f"platforms/{pname}.md")
 
     print("Generating gap analysis page...")
     gap_pages: dict[str, str] = {}

scripts/release_record.py

@@ -31,6 +31,9 @@ from common import (
 )
 
 RECORD = "release.json"
+# The record's format, which release.schema.json pins: a reader holding the
+# file alone can tell which shape it reads.
+SCHEMA_VERSION = 1
 # Each registered platform has a page at get/<platform>/ on the site: the
 # current pack's files, or the command when no release publishes one. The
 # address outlives every release, so a link written elsewhere keeps working.
@@ -120,7 +123,7 @@ def _read_record(dist: Path, tag: str, owners: dict[str, list[str]]) -> dict:
             "assets": [path.name for path in paths],
             "platforms": owners[name],
         }
-    return {"tag": tag, "packs": packs}
+    return {"schema_version": SCHEMA_VERSION, "tag": tag, "packs": packs}
 
 
 def load_record(path: str | Path = RECORD) -> dict:

scripts/sitesources.py

@@ -5,17 +5,21 @@ was written against, so a permalink is pinned, never guessed."""
 
 from __future__ import annotations
 
+import argparse
 import functools
+import json
 import re
+import sys
 import urllib.parse
+from collections.abc import Iterator
 from dataclasses import dataclass
 from pathlib import Path
-import urllib.request
 
 import upstream
 from siterender import _by_mode
 from sourceref import (
-    _prose_runs, is_binary_citation, ref_role, source_ref_values, split_source_ref,
+    _HEX_ADDRESS_RE, _prose_runs, is_binary_citation, ref_role, source_ref_values,
+    split_source_ref,
 )
 
 
@@ -154,43 +158,159 @@ def _source_permalink(repo: upstream.Repo, pin: str, path: str,
                 url += f"-L{end}"
     return url
 
-# The tree listings profile_sync caches. Read offline only: a page build
-# never calls a forge, and a listing it does not find leaves the link as the
-# profile's declarations alone decide it.
+# The tree listings profile_sync caches, read only by the command that
+# writes SOURCE_PATHS. The pages read that file: the deploy has no tree
+# cache, and a citation it could not check was linked into whichever
+# repository the profile declared, 30 of them to a 404.
 TREE_CACHE = Path(__file__).resolve().parent.parent / upstream.CACHE_DIR
+SOURCE_PATHS = (Path(__file__).resolve().parent.parent
+                / "website" / "src" / "data" / "source-paths.json")
+
+
+def tree_key(repo: upstream.Repo, pin: str) -> str:
+    """How SOURCE_PATHS names one repository at one revision."""
+    return f"{repo.host}/{repo.slug}@{pin}"
 
 
 @functools.lru_cache(maxsize=None)
-def _tree_paths(repo: upstream.Repo, pin: str) -> frozenset[str] | None:
-    """Every file of a repository at a revision, None when not cached or
-    truncated (a missing path would then prove nothing)."""
-    if not repo.api_base:
-        return None  # a browse-only forge: no listing to read
-    paths, truncated = upstream.list_tree(repo, pin, str(TREE_CACHE), offline=True)
-    return frozenset(paths) if paths and not truncated else None
+def _verified() -> dict[str, frozenset[str]]:
+    """The committed record, empty when it has not been written."""
+    try:
+        data = json.loads(SOURCE_PATHS.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return {}
+    return {key: frozenset(paths) for key, paths in data.items()}
 
 
-def tree_holds(repo: upstream.Repo, pin: str, path: str) -> bool | None:
-    """Whether the repository holds ``path`` at ``pin``: None when its tree
-    is not cached. trident cites src/core/loader/ncch.cpp, a Panda3DS file,
-    and the page linked it into 3dsTrident, where it is a 404."""
-    paths = _tree_paths(repo, pin)
-    return None if paths is None else path in paths
+@functools.lru_cache(maxsize=None)
+def _tree_paths(repo: upstream.Repo, pin: str) -> frozenset[str]:
+    """The cited paths of a repository at a revision that its tree holds."""
+    return _verified().get(tree_key(repo, pin), frozenset())
+
+
+def tree_holds(repo: upstream.Repo, pin: str, path: str) -> bool:
+    """Whether ``path`` was found in the repository's tree at ``pin``.
+
+    trident cites src/core/loader/ncch.cpp, a Panda3DS file, and the page
+    linked it into 3dsTrident, where it is a 404. A path no tree confirmed,
+    on a forge whose tree cannot be read or a revision not cached when the
+    record was written, is not linked: no link beats a link to nothing.
+    """
+    return path in _tree_paths(repo, pin)
 
 
 def _holder(
-    candidates: list[tuple[upstream.Repo, str]], path: str, default: tuple | None,
+    candidates: list[tuple[upstream.Repo, str]], path: str,
 ) -> tuple[upstream.Repo, str] | None:
     """The pinned repository that holds ``path``, source before upstream as
-    profile_sync.select_views orders them. Without every tree cached, the
-    declarations decide (``default``). A path no cached tree holds links
-    nowhere: no link beats a link to a file that is not there."""
-    verdicts = [tree_holds(repo, pin, path) for repo, pin in candidates]
-    if not candidates or None in verdicts:
-        if default is not None and tree_holds(*default, path) is False:
+    profile_sync.select_views orders them."""
+    return next((cand for cand in candidates if tree_holds(*cand, path)), None)
+
+
+def _cached_tree(
+    repo: upstream.Repo, pin: str, cache_dirs: list[Path], fetch: bool = False,
+) -> frozenset[str] | None:
+    """Every file of a repository at a revision, None when not cached or
+    truncated (a missing path would then prove nothing). Each cache is read
+    in turn. With *fetch*, a tree none holds is fetched into the last one."""
+    if not repo.api_base:
+        return None  # a browse-only forge: no listing to read
+    for index, cache_dir in enumerate(cache_dirs):
+        online = fetch and index == len(cache_dirs) - 1
+        try:
+            paths, truncated = upstream.list_tree(repo, pin, str(cache_dir), offline=not online)
+        except upstream.UpstreamError as exc:
+            print(f"  {tree_key(repo, pin)}: {exc}", file=sys.stderr)
             return None
-        return default
-    return next((cand for cand, held in zip(candidates, verdicts) if held), None)
+        if paths and not truncated:
+            return frozenset(paths)
+    return None
+
+
+def _walk(value: object) -> Iterator[tuple[str, object]]:
+    """Every (key, value) of a profile, nested lists and mappings included."""
+    if isinstance(value, dict):
+        for key, item in value.items():
+            yield str(key), item
+            yield from _walk(item)
+    elif isinstance(value, list):
+        for item in value:
+            yield from _walk(item)
+
+
+def cited_paths(profile: dict) -> set[str]:
+    """Every path a page of the profile could link: the parts of each
+    source_ref and the file:line citations of its prose."""
+    paths: set[str] = set()
+    for key, value in _walk(profile):
+        if key == "source_ref" and isinstance(value, (str, dict)):
+            for _label, refs in source_ref_values(value):
+                paths.update(ref_role(part.path)[1] for part in split_source_ref(refs))
+        elif isinstance(value, str) and ":" in value:
+            for _n, _span, _spans, parts, _run, _wrapped in _prose_runs(value):
+                paths.update(part.path for part in parts)
+    return {path for path in paths
+            if _PATH_RE.fullmatch(path) and not is_binary_citation(path)}
+
+
+def profile_trees(profile: dict) -> list[tuple[upstream.Repo, str]]:
+    """Every repository and revision a page of the profile can link into."""
+    found: list[tuple[upstream.Repo, str]] = []
+    labels = {""} | {
+        str(mode) for field in ("source", "upstream", "source_commit", "upstream_commit")
+        if isinstance(profile.get(field), dict) for mode in profile[field]}
+    for label in sorted(labels):
+        for pair in _forge_sources(profile, label) + list(role_sources(profile, label).values()):
+            if pair not in found:
+                found.append(pair)
+    return found
+
+
+def _listed_directories(
+    repo: upstream.Repo, pin: str, paths: set[str], cache_dirs: list[Path], fetch: bool,
+) -> frozenset[str] | None:
+    """The cited paths found by listing their directories one by one, for a
+    tree the forge truncates (aps3e vendors RPCS3 whole). A directory that
+    cannot be listed confirms none of its paths. None off GitHub, the one
+    forge with that call."""
+    if repo.family != "github":
+        return None
+    found: set[str] = set()
+    for folder in sorted({path.rpartition("/")[0] for path in paths} - {""}):
+        names = None
+        for index, cache_dir in enumerate(cache_dirs):
+            online = fetch and index == len(cache_dirs) - 1
+            try:
+                names = upstream.list_directory(repo, pin, folder, str(cache_dir), offline=not online)
+            except upstream.UpstreamError:
+                names = None
+            if names is not None:
+                break
+        found.update(f"{folder}/{name}" for name in names or ())
+    return frozenset(found)
+
+
+def build_source_paths(
+    profiles: dict, cache_dirs: list[Path], fetch: bool = False,
+) -> tuple[dict[str, list[str]], list[str]]:
+    """The cited paths each cached tree holds, and the trees not cached."""
+    held: dict[str, set[str]] = {}
+    unread: set[str] = set()
+    for _name, profile in sorted(profiles.items()):
+        paths = cited_paths(profile)
+        if not paths:
+            continue
+        for repo, pin in profile_trees(profile):
+            tree = _cached_tree(repo, pin, cache_dirs, fetch)
+            if tree is None:
+                tree = _listed_directories(repo, pin, paths, cache_dirs, fetch)
+            if tree is None:
+                unread.add(tree_key(repo, pin))
+                continue
+            found = paths & tree
+            if found:
+                held.setdefault(tree_key(repo, pin), set()).update(found)
+    return {key: sorted(paths) for key, paths in sorted(held.items())}, sorted(unread)
 
 
 def _commit_url(repo: upstream.Repo, pin: str) -> str:
@@ -279,11 +399,11 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
     """Render source_ref values as pinned links when their forge is known.
 
     A profile can declare two repositories: the libretro port in ``source`` and
-    the original emulator in ``upstream``. Which of the two carries a given
-    path cannot be decided without reading their trees, and this generator runs
-    offline. Rather than guess, an unattributable path is rendered as plain
+    the original emulator in ``upstream``. A path links into the one whose
+    tree SOURCE_PATHS says holds it. Any other path is rendered as plain
     code: a citation with no link still names the file and the lines, while a
-    link to the wrong repository is a false citation.
+    link to the wrong repository is a false citation. An address inside a
+    shipped binary reads as one (offset 0x8094-0x80ac in libnes.so).
     """
     rendered_groups: list[str] = []
     path_re = re.compile(r"^[A-Za-z0-9_.@/+~-]+$")
@@ -292,7 +412,13 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
         by_role = role_sources(profile, label)
         rendered: list[str] = []
         current_role = ""
+        binary = ""
         for part in split_source_ref(refs):
+            if is_binary_citation(part.path):
+                text, binary = _binary_citation(part.path, binary)
+                rendered.append(text)
+                continue
+            binary = ""
             role, bare = ref_role(part.path)
             if bare != part.path or role:
                 current_role = role
@@ -311,20 +437,14 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
             # broke 470 links. Two repositories and no tree to read: no link.
             selected: tuple[upstream.Repo, str] | None = None
             real_path = part.path
-            # A shipped binary (`DesktopKinect.dll`, `0x10037130-0x100371de`
-            # continuing a BAM.dll range) is no path of either repository:
-            # without a cached tree the single declared one took it.
-            if path_re.fullmatch(part.path) and not is_binary_citation(part.path):
+            if path_re.fullmatch(part.path):
                 if current_role:
                     # Only the repository the part names, or no link at all.
                     selected = by_role.get(current_role)
-                    if selected is not None and tree_holds(*selected, part.path) is False:
+                    if selected is not None and not tree_holds(*selected, part.path):
                         selected = None
                 else:
-                    selected = _holder(
-                        candidates, part.path,
-                        candidates[0] if len(candidates) == 1 else None,
-                    )
+                    selected = _holder(candidates, part.path)
 
             tag = (f" ({role_label(profile, current_role, part.path)})"
                    if current_role and _two_repositories(profile) else "")
@@ -344,6 +464,22 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str:
     return ", ".join(rendered_groups)
 
 
+def _binary_citation(path: str, previous: str) -> tuple[str, str]:
+    """A part citing code inside a shipped binary, and the binary it names.
+
+    `Nesoid 2.5.0 libnes.so 0x455a4-0x45694, 0x8094-0x80ac` cites two
+    ranges of one library: the bare address continues the binary before it.
+    No revision of a repository holds those bytes, so nothing is linked.
+    """
+    found = _HEX_ADDRESS_RE.search(path)
+    if found is None:
+        return f"`{path}`", path
+    binary = path[:found.start()].rstrip(" :") or previous
+    if not binary:
+        return f"`{path}`", ""
+    return f"offset `{path[found.start():]}` in `{binary}`", binary
+
+
 def link_prose_citations(profile: dict, text: str) -> str:
     """Prose file:line citations as pinned links, where the target is certain.
 
@@ -381,7 +517,7 @@ def link_prose_citations(profile: dict, text: str) -> str:
                 continue
             if any(lo <= path_span[0] < hi for lo, hi in guarded):
                 continue
-            if tree_holds(repo, pin, path) is False:
+            if not tree_holds(repo, pin, path):
                 continue
             for index, ((lo, hi), part) in enumerate(zip(spans, parts)):
                 start = path_span[0] if index == 0 else lo
@@ -494,3 +630,44 @@ def launcher_ref_markdown(source: ListSource | None, ref: str) -> str:
         else:
             rendered.append(f"[`{display}`]({source.browse(part.path, part.start, part.end)})")
     return ", ".join(rendered)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        allow_abbrev=False,
+        description="Record which cited source paths the cached upstream trees hold",
+    )
+    parser.add_argument(
+        "--cache-dir", type=Path, action="append",
+        help=f"tree listings profile_sync cached, read in the order given (default: {TREE_CACHE})",
+    )
+    parser.add_argument(
+        "--fetch", action="store_true",
+        help="fetch a tree no cache holds into the last --cache-dir",
+    )
+    parser.add_argument(
+        "--emulators-dir", default="emulators",
+        help="profiles whose citations are checked (default: %(default)s)",
+    )
+    parser.add_argument(
+        "--output", type=Path, default=SOURCE_PATHS,
+        help="record the site reads (default: %(default)s)",
+    )
+    args = parser.parse_args()
+    cache_dirs = args.cache_dir or [TREE_CACHE]
+    if not args.fetch and not any(path.is_dir() for path in cache_dirs):
+        print(f"Error: no tree cache at {', '.join(map(str, cache_dirs))}", file=sys.stderr)
+        return 1
+    from common import load_emulator_profiles, write_if_changed  # noqa: PLC0415
+
+    profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
+    held, unread = build_source_paths(profiles, cache_dirs, args.fetch)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    write_if_changed(str(args.output), json.dumps(held, indent=1, sort_keys=True) + "\n")
+    print(f"{sum(map(len, held.values()))} paths held in {len(held)} trees, "
+          f"{len(unread)} trees not cached: {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_site_exports.py

@@ -8,6 +8,7 @@ import sqlite3
 import sys
 import tempfile
 import unittest
+from unittest import mock
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -138,6 +139,7 @@ class SiteReferenceContracts(unittest.TestCase):
         }
         return db, coverages, profiles, stats
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_source_references_are_revision_pinned_links(self):
         _, _, profiles, _ = self._fixtures()
         rendered = _source_ref_markdown(
@@ -168,6 +170,7 @@ class SiteReferenceContracts(unittest.TestCase):
         prefixed = _source_ref_markdown(profile, "example/jg.c:213-214")
         self.assertEqual(prefixed, "`example/jg.c:213-214`")
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_a_single_repository_links_the_path_as_written(self):
         profile = {"source": "https://github.com/PCSX2/pcsx2", "source_commit": "c" * 40}
         rendered = _source_ref_markdown(profile, "pcsx2/ImGui/FullscreenUI.cpp:755-759")
@@ -590,6 +593,7 @@ class OnlyAPinnedProfileClaimsAPin(unittest.TestCase):
 class ADataDirectorySaysWhatItIs(unittest.TestCase):
     """The page said `dolphin-sys` > `Sys` and nothing of the tree it packs."""
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_description_origin_mode_and_source(self):
         from generate_site import generate_emulator_page  # noqa: PLC0415
 
@@ -773,6 +777,7 @@ class ProseCitationsLinkToThePin(unittest.TestCase):
                "source_commit": "76c7a84f1f7e80f3e6f2bba96fe100cb24e99124",
                "files": [{"name": "x", "source_ref": "libretro.c:10"}]}
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_a_rooted_or_structured_path_is_linked(self):
         from sitesources import link_prose_citations
 
@@ -799,6 +804,7 @@ class ACitationSaysWhoseCodeItIs(unittest.TestCase):
                "upstream": "https://github.com/alekmaul/a5200",
                "source_commit": "a" * 40, "upstream_commit": "b" * 40}
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_each_part_links_to_its_repository_with_its_label(self):
         from sitesources import _source_ref_markdown, requested_by
 

tests/test_site_r2_reference.py

@@ -208,7 +208,7 @@ class PlatformReferenceInWords(unittest.TestCase):
         systems = {"sony-playstation": {"files": [{"name": "f.bin"}]}}
         page = self._page("md5", "md5", systems)
         name = siterender.system_name("sony-playstation")
-        self.assertIn(f"| [{name}](#sony-playstation) <small>sony-playstation</small> |", page)
+        self.assertIn(f"| [{name}](#sony-playstation) <small data-pagefind-ignore>sony-playstation</small> |", page)
         heading = next(line for line in page.splitlines() if line.startswith("### ")
                        and "sony-playstation" in line)
         self.assertIn(name, heading)
@@ -335,7 +335,7 @@ class DatesAndNames(unittest.TestCase):
                          '<time datetime="2026-08-07">7 August 2026</time>')
 
     def test_data_page_title(self):
-        self.assertTrue(generate_site.generate_data_page([]).startswith("# Data and API\n"))
+        self.assertIn("\n---\n\n# Data and API\n", generate_site.generate_data_page([]))
 
 
 @unittest.skipUnless((DIST / "platforms" / "batocera" / "index.html").is_file(),

tests/test_site_r3_reference.py

@@ -296,7 +296,7 @@ class ManifestsArePublished(unittest.TestCase):
             self.assertIn(f"| `{field}` | {generate_site._escape_tags(description)} |", page)
         self.assertIn("## Platform manifests", page)
         self.assertIn("**Licence:**", page)
-        self.assertIn("## Hosting and limits", page)
+        self.assertIn("## Hosting, rate limits and caching", page)
         # The endpoints first, the DAT below them.
         self.assertLess(page.index("## Stable JSON endpoints"), page.index("The DAT is Logiqx XML"))
 
@@ -449,7 +449,7 @@ class SourceLinksGoWhereTheFileIs(unittest.TestCase):
     def tearDown(self):
         sitesources._tree_paths.cache_clear()
 
-    def _render(self, tree: frozenset[str] | None) -> str:
+    def _render(self, tree: frozenset[str]) -> str:
         original = sitesources._tree_paths
         sitesources._tree_paths = lambda repo, pin: tree  # type: ignore[assignment]
         try:
@@ -464,9 +464,12 @@ class SourceLinksGoWhereTheFileIs(unittest.TestCase):
         self.assertIn("`src/core/loader/ncch.cpp:151-164`", text)
         self.assertNotIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader", text)
 
-    def test_without_a_cached_tree_the_declarations_decide(self):
-        text = self._render(None)
-        self.assertIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader/ncch.cpp", text)
+    def test_a_path_no_tree_confirmed_is_not_linked(self):
+        """Round 4: without a tree the declared repository took the path,
+        and the deploy, which has no tree cache, linked 30 paths to a 404."""
+        text = self._render(frozenset())
+        self.assertNotIn("](https://", text)
+        self.assertIn("`src/core/loader/ncch.cpp:151-164`", text)
 
     def test_every_rendered_link_is_in_its_cached_tree(self):
         """Offline: each link the pages render points at a path the cached

tests/test_site_r4_reference.py

@@ -0,0 +1,344 @@
+"""Reference pages and data (round 4, package 8).
+
+The lookup answered a SHA1 without the setups that list the same file by
+MD5, the published folder check failed 134 EmuDeck entries a folder EmuDeck
+accepts, the manifest table printed `<ref>` behind a dead link and showed
+17 of 48 fields, 28 system ids held files and no page, and source links went
+to 404s on the deploy, which has no tree cache. Tests that read the built
+site skip when it has not been generated.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import sys
+import tempfile
+import unittest
+import urllib.parse
+from pathlib import Path
+from unittest import mock
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import check_platform_folder  # noqa: E402
+import generate_site  # noqa: E402
+import release_record  # noqa: E402
+import sitesources  # noqa: E402
+import upstream  # noqa: E402
+
+PUBLIC = REPO_ROOT / "website" / "public"
+API = PUBLIC / "api" / "v1"
+DOCS = REPO_ROOT / "website" / "src" / "content" / "docs"
+DIST = REPO_ROOT / "website" / "dist"
+TMP = REPO_ROOT / "tmp"
+
+
+def _built(path: Path) -> Path:
+    if not path.exists():
+        raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated")
+    return path
+
+
+def _page(stem: str) -> str:
+    for suffix in (".md", ".mdx"):
+        path = DOCS / f"{stem}{suffix}"
+        if path.is_file():
+            return path.read_text(encoding="utf-8")
+    raise unittest.SkipTest(f"{stem} not generated")
+
+
+class LookupRecordsUnderEveryHash(unittest.TestCase):
+    """8.1: a platform listing a file by MD5 is found by its SHA1 too."""
+
+    SHA1 = "a" * 40
+    FILE = {"name": "bios.bin", "path": "bios/Sony/PlayStation/bios.bin", "sha1": SHA1,
+            "md5": "b" * 32, "crc32": "c" * 8, "sha256": "d" * 64}
+
+    def records(self) -> list[dict]:
+        db = {"files": {self.SHA1: self.FILE}, "indexes": {
+            "by_md5": {"b" * 32: self.SHA1}, "by_crc32": {"c" * 8: self.SHA1},
+            "by_sha256": {"d" * 64: self.SHA1}}}
+        coverages = {"setup": {"config": {"systems": {"sony-playstation": {"files": [
+            {"name": "bios.bin", "md5": "b" * 32}]}}}}}
+        profiles = {"core": {"type": "libretro", "systems": ["sony-playstation"],
+                             "files": [{"name": "bios.bin", "crc32": "c" * 8}]}}
+        with mock.patch.object(generate_site, "_console_system_names", return_value={}), \
+                mock.patch.object(generate_site, "_lookup_system_name", return_value=None), \
+                mock.patch.object(generate_site, "emulator_card_page", side_effect=lambda k, a: k):
+            return generate_site.file_lookup_records(
+                db, coverages, profiles, [{"sha1": self.SHA1, "url": "/x"}])
+
+    def test_each_hash_names_the_setup_and_the_emulator(self):
+        records = self.records()
+        for value in (self.SHA1, "b" * 32, "c" * 8, "d" * 64):
+            owners = {(r["layer"], r.get("owner")) for r in records if r["value"] == value}
+            self.assertIn(("platform", "setup"), owners, value)
+            self.assertIn(("emulator", "core"), owners, value)
+
+    def test_records_carry_the_sha1_and_say_what_was_declared(self):
+        for record in self.records():
+            if record["layer"] in ("platform", "emulator"):
+                self.assertEqual(record["sha1"], self.SHA1)
+        copy = next(r for r in self.records()
+                    if r["layer"] == "platform" and r["value"] == self.SHA1)
+        self.assertEqual(copy["declared"], "md5")
+
+    def test_scph5501_gives_one_answer_for_its_four_hashes(self):
+        db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+        sha1 = "0555c6fae8906f3f09baf5988f00e55f88e9f30b"
+        entry = db["files"][sha1]
+        _built(API / "lookup")
+        answers = []
+        for algorithm in ("sha1", "md5", "crc32", "sha256"):
+            value = str(entry[algorithm]).lower()
+            shard = json.loads((API / "lookup" / f"{value[:2]}.json").read_text())
+            found = [r for r in shard["items"] if r["value"] == value]
+            answers.append(tuple(sorted({(r["layer"], r.get("owner", "")) for r in found})))
+        self.assertEqual(len(set(answers)), 1, answers)
+        setups = {owner for layer, owner in answers[0] if layer == "platform"}
+        emulators = {owner for layer, owner in answers[0] if layer == "emulator"}
+        self.assertGreaterEqual(len(setups), 10)
+        self.assertGreaterEqual(len(emulators), 8)
+
+
+class TheFolderCheckReadsEmuDecksWhitelist(unittest.TestCase):
+    """8.2: an entry with no destination names no file. EmuDeck's check
+    passes the system when any top-level file of the folder has its md5."""
+
+    def check(self, files: dict[str, bytes]) -> list[tuple[str, str]]:
+        good = b"ps1 bios bytes"
+        contract = {"verification_mode": "md5", "systems": {"sony-playstation": {"files": [
+            {"name": f"sony-playstation:{hashlib.md5(good).hexdigest()}", "destination": "",
+             "md5": hashlib.md5(good).hexdigest()},
+            {"name": "sony-playstation:" + "0" * 32, "destination": "", "md5": "0" * 32},
+        ]}}}
+        with tempfile.TemporaryDirectory(dir=TMP) as tmp:
+            root = Path(tmp) / "bios"
+            (root / "sub").mkdir(parents=True)
+            for name, data in files.items():
+                (root / name).write_bytes(data)
+            api = Path(tmp) / "api"
+            api.mkdir()
+            (api / "emudeck.json").write_text(json.dumps({"items": [{"contract": contract}]}))
+            found = check_platform_folder.check("emudeck", root, api.as_uri() + "/")
+            # The verifier's verdict on the matching entry, file by file.
+            from verify import verify_entry_md5  # noqa: PLC0415
+
+            verdicts = {name: verify_entry_md5(contract["systems"]["sony-playstation"]["files"][0],
+                                               str(root / name))["status"]
+                        for name in files if (root / name).is_file()}
+        return found, verdicts
+
+    def test_any_name_passes(self):
+        found, verdicts = self.check({"my-ps1-dump.bin": b"ps1 bios bytes"})
+        self.assertEqual(found, [("sony-playstation: any file", "ok")])
+        self.assertEqual(verdicts, {"my-ps1-dump.bin": "ok"})
+
+    def test_a_folder_without_it_is_missing(self):
+        found, _verdicts = self.check({"other.bin": b"x", "sub/deep.bin": b"ps1 bios bytes"})
+        self.assertEqual(found, [("sony-playstation: any file", "missing")])
+
+    def test_the_data_page_states_the_rule(self):
+        page = generate_site.generate_data_page([])
+        self.assertIn("An entry with no `destination` (EmuDeck) names no file", page)
+        self.assertLess(len((REPO_ROOT / "scripts" / "check_platform_folder.py")
+                            .read_text().splitlines()), 70)
+
+
+class ManifestFieldsAreComplete(unittest.TestCase):
+    """8.3 and 8.4."""
+
+    SCHEMA = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text())
+
+    def test_a_url_template_is_code(self):
+        rows = generate_site._schema_field_rows(self.SCHEMA["$defs"]["file"], ("repo_path",))
+        self.assertIn("`https://raw.githubusercontent.com/Abdess/retrobios/<ref>/<repo_path>`",
+                      rows[0])
+        rows = generate_site._schema_field_rows(
+            {"properties": {"a": {"description": "From https://x.test/<asset>, then."}}}, ("a",))
+        self.assertEqual(rows, ["| `a` | From `https://x.test/<asset>`, then. |"])
+
+    def test_every_field_is_described_and_shown(self):
+        page = generate_site.generate_data_page([])
+        for schema in (self.SCHEMA, self.SCHEMA["$defs"]["file"]):
+            for key, spec in schema["properties"].items():
+                self.assertTrue(spec.get("description"), key)
+                self.assertIn(f"| `{key}` |", page)
+
+    def test_no_link_carries_an_escaped_bracket(self):
+        html = _built(DIST / "data" / "index.html").read_text(encoding="utf-8")
+        self.assertNotRegex(html, r'href="[^"]*&amp;lt;')
+        self.assertNotRegex(html, r'href="[^"]*&lt;')
+
+
+class EverySystemIdWithFilesHasAPage(unittest.TestCase):
+    """8.5: tyrian, scummvm, dos and 25 more had files and a null url."""
+
+    def test_built_index(self):
+        items = json.loads(_built(API / "systems.json").read_text())["items"]
+        orphans = [item["id"] for item in items if item["sha1"] and not item["url"]]
+        self.assertEqual(orphans, [])
+
+    def test_first_file_then_emulator(self):
+        files = {"f" * 40: {"path": "bios/DOS/DOS/a.bin"}}
+        with mock.patch("siterender._card_pages", {}):
+            place = generate_site._first_file_place(
+                "dos", ["f" * 40], files, {}, lambda path, sha1: "dos")
+        self.assertEqual(place, ("DOS", "/retrobios/systems/dos/#sha1-ffffffffffff"))
+        place = generate_site._first_file_place(
+            "tyrian", [], files, {"opentyrian": {"systems": ["tyrian"]}}, None)
+        self.assertEqual(place, ("", "/retrobios/emulators/opentyrian/"))
+
+
+# A source permalink of each forge family the pages write.
+_PERMALINK = re.compile(
+    r"\]\(https://(?P<host>[^/]+)/(?P<slug>[^/]+/[^/]+)/"
+    r"(?:blob/|-/blob/|src/commit/)(?P<pin>[0-9a-f]{40})/(?P<path>[^#)]+)")
+
+
+class SourceLinksOnlyWhereTheFileIs(unittest.TestCase):
+    """8.6: the deploy reads the committed record, never a tree cache."""
+
+    def test_an_address_in_a_binary_reads_as_one(self):
+        profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40}
+        text = sitesources._source_ref_markdown(
+            profile, "Nesoid 2.5.0 libnes.so 0x455a4-0x45694 (setter), 0x8094-0x80ac")
+        self.assertEqual(text, "offset `0x455a4-0x45694` in `Nesoid 2.5.0 libnes.so`, "
+                               "offset `0x8094-0x80ac` in `Nesoid 2.5.0 libnes.so`")
+
+    def test_only_a_recorded_path_links(self):
+        profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40}
+        repo = upstream.parse_repo("https://github.com/o/r")
+        record = {sitesources.tree_key(repo, "e" * 40): frozenset({"src/a.c"})}
+        with mock.patch.object(sitesources, "_verified", return_value=record):
+            sitesources._tree_paths.cache_clear()
+            try:
+                text = sitesources._source_ref_markdown(profile, "src/a.c:5, src/b.c:7")
+            finally:
+                sitesources._tree_paths.cache_clear()
+        self.assertIn("[`src/a.c:5`](https://github.com/o/r/blob/" + "e" * 40 + "/src/a.c#L5)", text)
+        self.assertIn("`src/b.c:7`", text)
+        self.assertNotIn("src/b.c#", text)
+
+    def test_the_writer_keeps_held_paths_only(self):
+        profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40,
+                   "notes": "see src/c.c:3", "files": [{"source_ref": "src/a.c:5, gone.c:1"}]}
+        with mock.patch.object(sitesources, "_cached_tree",
+                               return_value=frozenset({"src/a.c", "src/c.c", "x.c"})):
+            held, unread = sitesources.build_source_paths({"p": profile}, [TMP])
+        self.assertEqual(held, {"github.com/o/r@" + "e" * 40: ["src/a.c", "src/c.c"]})
+        self.assertEqual(unread, [])
+
+    def test_every_rendered_link_is_recorded(self):
+        record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8"))
+        pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*"))
+        checked, stray = 0, []
+        for page in pages:
+            for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")):
+                if "/Abdess/retrobios/" in found[0]:
+                    continue
+                key = f"{found['host']}/{found['slug']}@{found['pin']}"
+                path = urllib.parse.unquote(found["path"])
+                checked += 1
+                if path not in record.get(key, ()):
+                    stray.append((page.name, key, path))
+        self.assertGreater(checked, 1000)
+        self.assertEqual(stray[:20], [])
+
+    def test_the_record_is_read_by_the_deploy(self):
+        """The record is committed where the build reads it, sorted."""
+        record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8"))
+        self.assertGreater(len(record), 300)
+        for key, paths in record.items():
+            self.assertRegex(key, r"^[a-z0-9.-]+/.+@[0-9a-f]{40}$")
+            self.assertEqual(paths, sorted(set(paths)), key)
+
+
+class TheReferencePagesAnswerTheirWords(unittest.TestCase):
+    """8.7, 8.8, 8.10 and 8.13."""
+
+    def test_find_or_check_a_file(self):
+        page = generate_site.generate_files_page()
+        self.assertTrue(page.startswith(
+            "---\nterms: check dump, md5 check, verify bios file, check hash\n---\n"))
+        self.assertIn("\n# Find or check a file\n\nCheck a BIOS dump: paste its hash "
+                      "or choose the file.\n", page)
+
+    def test_the_data_page(self):
+        page = generate_site.generate_data_page([])
+        self.assertTrue(page.startswith(
+            "---\nterms: manifest, schema, check a folder, rate limit, api\n---\n"))
+        self.assertIn("## Hosting, rate limits and caching", page)
+        self.assertIn("HTTP 429", page)
+        self.assertIn("crypto.subtle.digest(\"SHA-1\"", page)
+        self.assertIn("api/v1/lookup/", page)
+        self.assertIn("`preferred: true`", page)
+        self.assertIn("`database.json`, `stats.json`, `release.json` and the manifests are not "
+                      "envelopes", page)
+        self.assertIn("- MinUI, MyMinUI and NextUI have no list of their own "
+                      "(`native_list: none`). An empty list does not mean nothing is needed.", page)
+
+    def test_overview_ids_and_counts_are_not_indexed(self):
+        html = _built(DIST / "platforms" / "batocera" / "index.html").read_text(encoding="utf-8")
+        self.assertRegex(html, r"<small data-pagefind-ignore(=\"true\")?>sony-playstation</small>")
+        self.assertRegex(html, r"<span data-pagefind-ignore(=\"true\")?>\d+/\d+</span>")
+
+
+class TheReleaseRecordHasAVersion(unittest.TestCase):
+    """8.9."""
+
+    def test_the_writer_and_the_committed_record(self):
+        self.assertEqual(release_record.SCHEMA_VERSION, 1)
+        source = (REPO_ROOT / "scripts" / "release_record.py").read_text(encoding="utf-8")
+        self.assertIn('{"schema_version": SCHEMA_VERSION, "tag": tag, "packs": packs}', source)
+        schema = json.loads((REPO_ROOT / "schemas" / "release.schema.json").read_text())
+        self.assertIn("schema_version", schema["required"])
+        record = json.loads((REPO_ROOT / "release.json").read_text())
+        self.assertEqual(record["schema_version"], 1)
+
+
+class NoEmptyCoreTable(unittest.TestCase):
+    """8.11: MiSTer and BizHawk declare cores: [] and had a page of nothing."""
+
+    def test_own_list_platforms_get_a_sentence(self):
+        coverages = {"misterfpga": {"platform": "MiSTer FPGA", "config": {"cores": [], "systems": {}}}}
+        moved: dict[str, str] = {}
+        page = generate_site.generate_cross_reference(coverages, {}, moved)
+        self.assertEqual(moved, {})
+        self.assertIn(generate_site.OWN_LIST_ONLY, page)
+
+    def test_built_pages(self):
+        _built(DOCS / "cross-reference.md")
+        for name in ("misterfpga", "bizhawk"):
+            self.assertFalse((DOCS / f"cross-reference-{name}.md").exists())
+            self.assertIn(generate_site.OWN_LIST_ONLY, _page(f"platforms/{name}"))
+        redirects = json.loads((REPO_ROOT / "website" / "src" / "generated" /
+                                "redirects.json").read_text())
+        self.assertEqual(redirects.get("/cross-reference-misterfpga/"), "/platforms/misterfpga/")
+
+
+class FilesToSourceOnAPhone(unittest.TestCase):
+    """8.12: 601 rows on one page, no way to give a file."""
+
+    def test_parts_of_a_hundred_rows_each_with_an_upload(self):
+        _built(DOCS / "gaps-files-to-source.md")
+        parts = sorted(DOCS.glob("gaps-files-to-source*.md"))
+        self.assertGreater(len(parts), 1)
+        for part in parts:
+            rows = [line for line in part.read_text(encoding="utf-8").splitlines()
+                    if line.startswith("| `")]
+            self.assertLessEqual(len(rows), generate_site.FILES_TO_SOURCE_PART, part.name)
+            for row in rows:
+                self.assertIn(f"[Have it? Upload]({generate_site.UPLOAD_URL}", row)
+
+    def test_the_upload_lands_in_the_console_folder(self):
+        with mock.patch("siterender._own_section", return_value=("Sony", "PlayStation")):
+            self.assertEqual(generate_site._upload_folder({"systems": ["sony-playstation"]}),
+                             "Sony/PlayStation")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_site_render.py

@@ -113,6 +113,7 @@ class StructuredBlocks(unittest.TestCase):
 
         return "\n".join(generate_site._render_structured_blocks(profile))
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def test_a_source_ref_in_a_block_is_a_pinned_link(self):
         """186 citations in analysis and firmware blocks were plain code."""
         page = self.render(self.profile("analysis", {"tmss": {
@@ -237,6 +238,7 @@ class ForgesUpstreamDoesNotRead(unittest.TestCase):
     """Profiles pinned on SourceForge, voidpoint.io, Bitbucket or Software
     Heritage showed no revision, and their citations were plain code."""
 
+    @mock.patch("sitesources.tree_holds", lambda *_args: True)  # a path the record confirms
     def links(self, url: str, ref: str, **extra: str) -> tuple[str, str]:
         import sitesources
 

tests/test_source_ref_targets.py

@@ -41,7 +41,7 @@ class BinaryContinuationsAreNotLinked(unittest.TestCase):
 
     def _render(self, value: str) -> str:
         original = sitesources._tree_paths
-        sitesources._tree_paths = lambda repo, pin: None  # type: ignore[assignment]
+        sitesources._tree_paths = lambda repo, pin: frozenset({"FPLoader.cpp"})  # type: ignore[assignment]
         try:
             return sitesources._source_ref_markdown(self.PROFILE, value)
         finally: