chore: merge the site weight and speed work

docs_assets/site.js

@@ -148,28 +148,88 @@
     });
   };
 
-  // files/?sha1=<sha1> is a file's content address: the card it names may
-  // sit on another manufacturer's page today than when the link was cited.
+  // files/?hash= reads one shard of the lookup, named by the first two hex
+  // characters of the value: 20 KB where the whole index was 675 KB.
+  const HASH_LENGTHS = { 8: "CRC32", 32: "MD5", 40: "SHA1", 64: "SHA256" };
+  const LAYER_LABELS = {
+    collection: "In the collection",
+    "data-directory": "In a data directory the packs ship",
+    platform: "Declared by a platform",
+    emulator: "Read by an emulator",
+  };
+  const siteRoot = () => {
+    const script = document.querySelector('script[src$="javascripts/site.js"]');
+    return script ? new URL("../", script.src) : new URL("../", window.location.href);
+  };
+  const lookupShard = (value) =>
+    fetch(new URL(`api/v1/lookup/${value.slice(0, 2)}.json`, siteRoot()))
+      .then((response) => (response.ok ? response.json() : { items: [] }))
+      .then((shard) => shard.items || []);
+  const normalizeHash = (raw) => raw.trim().toLowerCase().replace(/^0x/, "").replace(/\s+/g, "");
+
+  const renderMatches = (status, value, items) => {
+    status.textContent = "";
+    const kind = HASH_LENGTHS[value.length];
+    const heading = document.createElement("p");
+    heading.textContent = `${items.length} ${items.length === 1 ? "match" : "matches"} for ${kind} ${value}:`;
+    status.append(heading);
+    Object.entries(LAYER_LABELS).forEach(([layer, label]) => {
+      const found = items.filter((item) => item.layer === layer);
+      if (!found.length) return;
+      const title = document.createElement("p");
+      const strong = document.createElement("strong");
+      strong.textContent = label;
+      title.append(strong);
+      const list = document.createElement("ul");
+      found.forEach((item) => {
+        const row = document.createElement("li");
+        const link = document.createElement("a");
+        link.href = new URL(item.url, siteRoot()).href;
+        const code = document.createElement("code");
+        code.textContent = item.name || item.value;
+        link.append(code);
+        row.append(link);
+        if (item.owner) row.append(` (${item.owner})`);
+        list.append(row);
+      });
+      status.append(title, list);
+    });
+  };
+
+  // files/?sha1=<sha1> is a file's content address and opens its card
+  // wherever it is now; files/?hash= also takes an MD5, CRC32 or SHA256 and
+  // lists every page that names the value.
   const resolveFileAddress = () => {
     const status = document.getElementById("rb-file-lookup-status");
     if (!status) return;
-    const sha1 = (new URLSearchParams(window.location.search).get("sha1") || "")
-      .trim().toLowerCase();
-    if (!sha1) return;
-    if (!/^[0-9a-f]{40}$/.test(sha1)) {
-      status.textContent = "A SHA1 is 40 hexadecimal characters.";
+    const params = new URLSearchParams(window.location.search);
+    const permalink = params.has("sha1");
+    const value = normalizeHash(params.get("sha1") || params.get("hash") || "");
+    if (!value) return;
+    const input = document.getElementById("rb-hash");
+    if (input) input.value = value;
+    const prefix = permalink && /^[0-9a-f]{12}$/.test(value);
+    if (!prefix && !(/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length])) {
+      status.textContent =
+        "A hash is 8 (CRC32), 32 (MD5), 40 (SHA1) or 64 (SHA256) hexadecimal characters.";
       return;
     }
     status.textContent = "Looking it up...";
-    const index = new URL("../api/v1/file-locations.json", window.location.href);
-    fetch(index)
-      .then((response) => response.json())
-      .then((document_) => {
-        const item = (document_.items || []).find((entry) => entry.sha1 === sha1);
-        if (item) {
-          window.location.replace(new URL("../" + item.url, window.location.href));
+    lookupShard(value)
+      .then((items) => {
+        const matches = items.filter((item) =>
+          prefix
+            ? item.algorithm === "sha1" && item.value.startsWith(value)
+            : item.value === value
+        );
+        const cards = matches.filter((item) => item.layer === "collection" || item.layer === "data-directory");
+        const targets = new Set(cards.map((item) => item.url));
+        if (permalink && targets.size === 1) {
+          window.location.replace(new URL(cards[0].url, siteRoot()));
+        } else if (matches.length) {
+          renderMatches(status, prefix ? matches[0].value : value, prefix ? cards : matches);
         } else {
-          status.textContent = "The collection holds no file with this SHA1.";
+          status.textContent = "No file the site knows has this hash.";
         }
       })
       .catch(() => {
@@ -177,10 +237,45 @@
       });
   };
 
+  // A file card moved to another page keeps its address: an id of the form
+  // sha1-<12 hex> that this page does not hold is looked up by content, and
+  // an emulator card listed in the table of a split page opens its page.
+  const followMovedCard = () => {
+    const id = decodeURIComponent(window.location.hash.slice(1));
+    if (!id) return;
+    const target = document.getElementById(id);
+    const row = target?.closest(".rb-card-index tr");
+    const link = row?.querySelector("a[href]");
+    if (link) {
+      window.location.replace(link.href);
+      return;
+    }
+    const match = /^sha1-([0-9a-f]{12})$/.exec(id);
+    if (!match || target) return;
+    window.location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot()));
+  };
+
+  // The site search no longer holds hashes: Enter on a hash opens the lookup.
+  const hashSearch = () => {
+    const query = document.querySelector('input[data-md-component="search-query"]');
+    if (!query || query.dataset.rbHash === "true") return;
+    query.dataset.rbHash = "true";
+    query.addEventListener("keydown", (event) => {
+      if (event.key !== "Enter") return;
+      const value = normalizeHash(query.value);
+      if (/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length]) {
+        event.preventDefault();
+        window.location.assign(new URL(`files/?hash=${value}`, siteRoot()));
+      }
+    });
+  };
+
   const enhance = () => {
     enhanceTables();
     nameMaterialWidgets();
     resolveFileAddress();
+    followMovedCard();
+    hashSearch();
   };
 
   let resizeTimer = 0;

schemas/site-api-envelope.schema.json

@@ -27,7 +27,8 @@
         "emulators",
         "systems",
         "verification-and-coverage-gaps",
-        "file-locations"
+        "file-locations",
+        "file-lookup"
       ]
     },
     "count": {
@@ -332,6 +333,66 @@
           }
         }
       }
+    },
+    {
+      "if": {
+        "properties": {
+          "kind": {
+            "const": "file-lookup"
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "items": {
+            "items": {
+              "type": "object",
+              "required": [
+                "value",
+                "algorithm",
+                "layer",
+                "name",
+                "url"
+              ],
+              "properties": {
+                "value": {
+                  "type": "string",
+                  "pattern": "^[0-9a-f]+$"
+                },
+                "algorithm": {
+                  "enum": [
+                    "sha1",
+                    "md5",
+                    "crc32",
+                    "sha256"
+                  ]
+                },
+                "layer": {
+                  "enum": [
+                    "collection",
+                    "data-directory",
+                    "platform",
+                    "emulator"
+                  ]
+                },
+                "name": {
+                  "type": "string"
+                },
+                "url": {
+                  "type": "string"
+                },
+                "sha1": {
+                  "type": "string",
+                  "pattern": "^[0-9a-f]{40}$"
+                },
+                "owner": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   ]
 }

scripts/exporter/candidates.py

@@ -12,6 +12,7 @@ the platform names that no profile describes leaves the verdict open.
 from __future__ import annotations
 
 import sys
+from collections.abc import Iterable
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -136,6 +137,81 @@ class ReadEvidence:
         return None
 
 
+# The order ReadEvidence.proof tries: a reader's proof is the first that holds.
+_PROOF_ORDER = ("path", "launcher", "name", "hash", "directory", "data_directory")
+
+
+class ProofIndex:
+    """ReadEvidence.proof for many readers at once, by lookup.
+
+    Asking every core in turn cost one proof per core and entry: 3.8 million
+    calls to categorize 7,430 entries. Each key a reader can match is indexed
+    once, and an entry's own keys find the readers it concerns. The proof of
+    each reader is the one ReadEvidence.proof returns.
+    """
+
+    def __init__(self, readers: list[ReadEvidence]) -> None:
+        self._index: dict[str, dict[str, set[int]]] = {
+            kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member")
+        }
+        for position, evidence in enumerate(readers):
+            for kind, keys in (
+                ("tail", evidence.tails), ("launched", evidence.launched),
+                ("name", evidence.names), ("hash", evidence.hashes),
+                ("dir", evidence.directories), ("tree", evidence.trees),
+                ("member", evidence.tree_members),
+            ):
+                for key in keys:
+                    self._index[kind].setdefault(key, set()).add(position)
+            # A cut md5 names any hash it is the start of.
+            for value in evidence.hashes:
+                for length in range(_CUT_MD5, _FULL_MD5):
+                    if len(value) >= length:
+                        self._index["cut"].setdefault(value[:length], set()).add(position)
+
+    def _hits(self, kind: str, keys: Iterable[str]) -> set[int]:
+        table = self._index[kind]
+        found: set[int] = set()
+        for key in keys:
+            found |= table.get(key, set())
+        return found
+
+    def proofs(self, entry: dict, destination: str) -> dict[int, str]:
+        """Reader position -> its proof, for the readers that have one."""
+        names = {
+            str(n).casefold()
+            for n in (entry.get("name"), *(entry.get("aliases") or []))
+            if n
+        }
+        tails = _tails(destination)
+        last = {tails[-1]} if tails else set()
+        hashes = {
+            v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
+        }
+        cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5}
+        # "/" + d in "/" + path: d is a run of whole segments ending in "/",
+        # so it is a run of the path's segments before the last one.
+        parts = tails[0].split("/") if tails else []
+        runs = {
+            "/".join(parts[i:j]) + "/"
+            for i in range(len(parts)) for j in range(i + 1, len(parts))
+        }
+        found = {
+            "path": self._hits("tail", tails[:-1]),
+            "launcher": self._hits("launched", names | last),
+            "name": self._hits("name", names | last),
+            "hash": self._hits("hash", hashes) | self._hits("cut", cut),
+            "directory": self._hits("dir", runs),
+            "data_directory": self._hits("tree", runs)
+            | self._index["member"].get(tails[0] if tails else "", set()),
+        }
+        result: dict[int, str] = {}
+        for kind in _PROOF_ORDER:
+            for position in found[kind]:
+                result.setdefault(position, kind)
+        return result
+
+
 def _loads_romsets(profile: dict) -> bool:
     """A core that opens ROM sets by name: MAME pulls a device's set from
     device_t::searchpath(), and no profile lists every device set."""
@@ -275,6 +351,8 @@ class EntryCategories:
                 self._readers.append((kind, key, evidence))
         self._launchers = ReadEvidence()
         self._launchers.add_launchers(config.get("launchers") or [])
+        self._proofs = ProofIndex([evidence for _kind, _key, evidence in self._readers]
+                                  + [self._launchers])
         self._by_system: dict[str, list[str]] = {}
         for key in sorted(examined):
             for sid in profiles[key].get("systems") or []:
@@ -313,14 +391,14 @@ class EntryCategories:
     def explain(self, entry: dict, destination: str, system: str) -> dict:
         """The category of an entry and what establishes it."""
         readers: dict[str, list[dict]] = {BIOS: [], GAME_DATA: []}
-        for kind, key, evidence in self._readers:
-            proof = evidence.proof(entry, destination)
-            if proof is not None:
-                readers[kind].append({
-                    "emulator": key, "proof": proof,
-                    "entries": self._matching(key, kind, entry),
-                })
-        if self._launchers.proof(entry, destination) is not None:
+        proofs = self._proofs.proofs(entry, destination)
+        for position in sorted(p for p in proofs if p < len(self._readers)):
+            kind, key, _evidence = self._readers[position]
+            readers[kind].append({
+                "emulator": key, "proof": proofs[position],
+                "entries": self._matching(key, kind, entry),
+            })
+        if len(self._readers) in proofs:
             readers[BIOS].append({"emulator": "launcher", "proof": "launcher", "entries": []})
         if system in self._system_runs:
             system_cores, unresolved = self._system_runs[system]

scripts/generate_readme.py

@@ -111,6 +111,105 @@ def compute_coverage(
     }
 
 
+_REPO = Path(__file__).resolve().parent.parent
+# The site and the README read the same coverage: the first generator to run
+# stores it here and the second reads it back, instead of verifying every
+# platform twice (about a minute of the site build).
+COVERAGE_CACHE_DIR = _REPO / "tmp" / "coverage-cache"
+_COVERAGE_CACHE_FORMAT = "1"
+
+
+def _coverage_inputs(db_path: Path, platforms_dir: str, emulators_dir: str) -> str:
+    """A digest of everything coverage is computed from.
+
+    The database, the platform and emulator files, the code, the install
+    manifests, the data directory revisions and the state of bios/ on disk:
+    a change to any of them computes the coverage again.
+    """
+    import hashlib
+
+    digest = hashlib.sha256(_COVERAGE_CACHE_FORMAT.encode())
+
+    def add_file(path: Path) -> None:
+        digest.update(str(path).encode() + b"\0")
+        digest.update(path.read_bytes() if path.is_file() else b"<absent>")
+
+    add_file(db_path.resolve())
+    for root, pattern in ((Path(platforms_dir), "*"), (Path(emulators_dir), "*.yml"),
+                          (_REPO / "scripts", "*.py"), (_REPO / "install", "*.json"),
+                          (_REPO / "recipes", "*.json")):
+        for path in sorted(p for p in root.resolve().rglob(pattern) if p.is_file()):
+            add_file(path)
+    for path in (_REPO / ".gitignore", _REPO / "_mame_clones.json",
+                 _REPO / "data" / ".versions.json"):
+        add_file(path)
+    bios = _REPO / "bios"
+    for path in sorted(bios.rglob("*")):
+        if path.is_file():
+            stat = path.stat()
+            digest.update(f"{path.relative_to(bios)}\0{stat.st_size}\0{stat.st_mtime_ns}\n".encode())
+    return digest.hexdigest()
+
+
+def platform_coverages(
+    platform_names: list[str],
+    platforms_dir: str,
+    db: dict,
+    data_registry: dict | None = None,
+    emulators_dir: str = _EMULATORS_DIR,
+    db_path: str = "database.json",
+    report: Callable[[str, dict], None] | None = None,
+) -> dict[str, dict]:
+    """compute_coverage for each platform, read from the cache when its
+    inputs have not changed since a generator last stored it.
+
+    A platform with no YAML is left out, as compute_coverage raises for it.
+    ``report`` is called with each platform and its coverage.
+    """
+    # pickle, not JSON: the coverage holds sets and enums that JSON would
+    # turn into lists and strings. The file is written by this function into
+    # the repository's own tmp/, never downloaded.
+    import pickle
+
+    key = _coverage_inputs(Path(db_path), platforms_dir, emulators_dir)
+    names = sorted(platform_names)
+    cache = COVERAGE_CACHE_DIR / f"{key}.pickle"
+    coverages: dict[str, dict] | None = None
+    if cache.is_file():
+        try:
+            with cache.open("rb") as handle:
+                stored = pickle.load(handle)
+        except (OSError, EOFError, pickle.UnpicklingError) as exc:
+            print(f"  coverage cache unreadable, computing again: {exc}", file=sys.stderr)
+        else:
+            if stored.get("platforms") == names:
+                coverages = stored["coverages"]
+                print(f"  coverage read from {cache.relative_to(_REPO)}")
+    if coverages is None:
+        coverages = {}
+        for name in names:
+            try:
+                coverages[name] = compute_coverage(
+                    name, platforms_dir, db, data_registry, emulators_dir=emulators_dir,
+                )
+            except FileNotFoundError as exc:
+                print(f"  {name}: skipped ({exc})", file=sys.stderr)
+        COVERAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        for stale in COVERAGE_CACHE_DIR.glob("*.pickle"):
+            stale.unlink()
+        scratch = cache.with_suffix(f".{os.getpid()}.tmp")
+        with scratch.open("wb") as handle:
+            pickle.dump({"platforms": names, "coverages": coverages}, handle,
+                        protocol=pickle.HIGHEST_PROTOCOL)
+        scratch.replace(cache)
+    ordered = {name: coverages[name] for name in platform_names if name in coverages}
+    if report:
+        for name in names:
+            if name in ordered:
+                report(name, ordered[name])
+    return ordered
+
+
 def release_totals(platform_name: str, record: dict) -> tuple[int | None, int | None]:
     """Files and bytes of the pack the latest release serves for a platform.
 
@@ -347,7 +446,7 @@ def _catalog_matched_line(db: dict) -> list[str]:
     ]
 
 
-def generate_readme(db: dict, platforms_dir: str) -> str:
+def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json") -> str:
     total_files = db.get("total_files", 0)
     total_size = db.get("total_size", 0)
     ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -358,14 +457,8 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
 
     data_registry = load_data_dir_registry(platforms_dir)
 
-    coverages = {}
-    for name in platform_names:
-        try:
-            coverages[name] = compute_coverage(
-                name, platforms_dir, db, data_registry
-            )
-        except FileNotFoundError:
-            pass
+    coverages = platform_coverages(platform_names, platforms_dir, db, data_registry,
+                                   db_path=db_path)
 
     profiles = unique_emulator_profiles(
         load_emulator_profiles("emulators", skip_aliases=False)
@@ -853,7 +946,7 @@ def main() -> None:
     # published counts taken from half of it.
     with contextlib.ExitStack() as held:
         hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
-        readme = generate_readme(db, args.platforms_dir)
+        readme = generate_readme(db, args.platforms_dir, args.db)
     status = "Generated" if write_if_changed("README.md", readme) else "Unchanged"
     print(f"{status} ./README.md")
 

scripts/mkdocs_hooks.py

@@ -10,8 +10,28 @@ from __future__ import annotations
 
 from typing import Any
 
+# Pages made of file cards, level 3 headings by the hundred: their table of
+# contents stops at level 2, the consoles of a system page, the Files of an
+# emulator, the sections of a platform page (which listed every system id).
+CARD_PAGES = ("systems/", "emulators/", "platforms/")
+
 
 def on_page_markdown(markdown: str, page: Any, config: Any, files: Any) -> str:  # noqa: ARG001
     """Take lastmod from the page's own date, or leave it out."""
     page.update_date = str(page.meta.get("date_modified") or "")
     return markdown
+
+
+def _prune(items: list, depth: int) -> None:
+    for item in items:
+        if item.level >= depth:
+            item.children = []
+        else:
+            _prune(item.children, depth)
+
+
+def on_page_content(html: str, page: Any, config: Any, files: Any) -> str:  # noqa: ARG001
+    """Keep the table of contents of a page of cards to level 2."""
+    if page.file.src_uri.startswith(CARD_PAGES):
+        _prune(page.toc.items, 2)
+    return html

scripts/siterender.py

@@ -139,13 +139,75 @@ def _page_by_norm(sys_id: str) -> tuple[str, str] | None:
     return _norm_page_map.get(_norm_system_id(sys_id))
 
 
+# A manufacturer page too large for one page is a summary, each console on
+# pages of its own: (slug, console) -> its page stems under systems/, and
+# (slug, console, sha1) -> the stem holding that card. A console of a page
+# kept whole has no entry, its section is on the manufacturer's page.
+_console_pages: dict[tuple[str, str], list[str]] = {}
+_card_pages: dict[tuple[str, str, str], str] = {}
+
+
+def reset_console_pages() -> None:
+    _console_pages.clear()
+    _card_pages.clear()
+
+
+def record_console_pages(
+    slug: str, console: str, stems: list[str], sha1s: list[list[str]]
+) -> None:
+    """Where a split manufacturer page put one console: its page stems, in
+    order, and the sha1 of each card on each of them."""
+    _console_pages[(slug, console)] = list(stems)
+    for stem, part in zip(stems, sha1s):
+        for sha1 in part:
+            _card_pages.setdefault((slug, console, sha1), stem)
+
+
+def split_system_stems() -> set[str]:
+    """Every page stem of a console moved to a page of its own."""
+    return {stem for stems in _console_pages.values() for stem in stems}
+
+
+def console_href(slug: str, console: str, prefix: str = "") -> str:
+    """Where a console's files are: its own page, or its section of the
+    manufacturer's page (also where a console cut into parts lists them)."""
+    stems = _console_pages.get((slug, console))
+    if stems and len(stems) == 1:
+        return f"{prefix}systems/{urllib.parse.quote(stems[0])}.md"
+    return f"{prefix}systems/{urllib.parse.quote(slug)}.md#{_slugify_anchor(console)}"
+
+
+def card_page(path: str, sha1: str) -> str:
+    """The systems/ page stem holding the card of the copy at ``path``
+    (bios/<Manufacturer>/<Console>/...)."""
+    parts = path.split("/")
+    slug = parts[1].lower().replace(" ", "-")
+    console = parts[2] if len(parts) > 2 else ""
+    return _card_pages.get((slug, console, sha1), slug)
+
+
+# Emulator cards moved off a page too long for one: (emulator, anchor) ->
+# the page stem under emulators/ that holds the card.
+_emulator_card_pages: dict[tuple[str, str], str] = {}
+
+
+def record_emulator_cards(emulator: str, stem: str, anchors: list[str]) -> None:
+    for anchor in anchors:
+        _emulator_card_pages[(emulator, anchor)] = stem
+
+
+def emulator_card_page(emulator: str, anchor: str) -> str:
+    """The emulators/ page stem holding a card: the emulator's own page, or
+    the page its cards went to."""
+    return _emulator_card_pages.get((emulator, anchor), emulator)
+
+
 def _system_link(sys_id: str, prefix: str = "") -> str:
     """Generate a markdown link to a system page with anchor."""
     page = _system_page_map.get(sys_id) or _page_by_norm(sys_id)
     if page:
         slug, console = page
-        anchor = _slugify_anchor(console)
-        return f"[{sys_id}]({prefix}systems/{slug}.md#{anchor})"
+        return f"[{sys_id}]({console_href(slug, console, prefix)})"
     return sys_id
 
 def _field_text(key: object, value: object) -> str:

scripts/validate_schemas.py

@@ -446,7 +446,7 @@ def main() -> int:
             if (api_dir / name).exists()
         ] + sorted((api_dir / "platforms").glob("*.json")) + sorted(
             (api_dir / "systems").glob("*.json")
-        )
+        ) + sorted((api_dir / "lookup").glob("*.json"))
         errors.extend(
             _validate_json_files(envelope_paths, "site-api-envelope.schema.json")
         )

scripts/validate_site.py

@@ -34,7 +34,25 @@ class Page:
     jsonld: list[str] = field(default_factory=list)
     unrendered: list[str] = field(default_factory=list)
     remote_images: list[str] = field(default_factory=list)
+    remote_scripts: list[str] = field(default_factory=list)
     placeholders: list[str] = field(default_factory=list)
+    headings: list[int] = field(default_factory=list)
+    empty_headers: int = 0
+    runtime_diagrams: int = 0
+    size: int = 0
+
+
+# Budgets the site was brought under on 2026-10-22, with room to grow. A
+# change that passes one is a regression of what a visitor downloads, not a
+# matter of taste: raise the number in the same change that explains why.
+#
+# The largest page was 10.5 MB (systems/game-engines/) and took 21 s of
+# main-thread work on a throttled phone. Split pages now top out at 0.9 MB.
+MAX_PAGE_BYTES = 1_000_000
+# The search index is downloaded by every visitor who opens the search:
+# 21.0 MB raw and 4.88 MB gzip with hashes in it, 7.3 MB and 1.44 MB without.
+MAX_SEARCH_INDEX_BYTES = 8_500_000
+MAX_SEARCH_INDEX_GZIP_BYTES = 1_700_000
 
 
 # Text inside these is shown as written, so markup there is not a leftover.
@@ -50,11 +68,21 @@ class _PageParser(HTMLParser):
         self._jsonld_depth = 0
         self._jsonld_parts: list[str] = []
         self._literal_depth = 0
+        self._header_parts: list[str] | None = None
 
     def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
-        data = dict(attrs)
+        data = {key: value or "" for key, value in attrs}
         if tag in _LITERAL_TAGS:
             self._literal_depth += 1
+        if re.fullmatch(r"h[1-6]", tag):
+            self.page.headings.append(int(tag[1]))
+        if tag == "th":
+            self._header_parts = []
+        if "mermaid" in data.get("class", "").split():
+            # Material renders it in the browser with a script from a CDN.
+            self.page.runtime_diagrams += 1
+        if tag == "script" and re.match(r"\s*(?:https?:)?//", data.get("src", "")):
+            self.page.remote_scripts.append(data["src"])
         if tag == "html":
             self.page.lang = data.get("lang", "")
         elif tag == "title":
@@ -97,6 +125,10 @@ class _PageParser(HTMLParser):
     def handle_endtag(self, tag: str) -> None:
         if tag in _LITERAL_TAGS and self._literal_depth:
             self._literal_depth -= 1
+        if tag == "th" and self._header_parts is not None:
+            if not "".join(self._header_parts).strip():
+                self.page.empty_headers += 1
+            self._header_parts = None
         if tag == "title":
             self._title_depth -= 1
         elif tag == "script" and self._jsonld_depth:
@@ -105,6 +137,8 @@ class _PageParser(HTMLParser):
             self._jsonld_parts = []
 
     def handle_data(self, data: str) -> None:
+        if self._header_parts is not None:
+            self._header_parts.append(data)
         if self._title_depth:
             self._title_parts.append(data)
         if self._jsonld_depth:
@@ -119,11 +153,39 @@ class _PageParser(HTMLParser):
 
 def _parse_page(path: Path) -> Page:
     parser = _PageParser(path)
-    parser.feed(path.read_text(encoding="utf-8", errors="replace"))
+    text = path.read_text(encoding="utf-8", errors="replace")
+    parser.feed(text)
     parser.close()
+    parser.page.size = len(text.encode("utf-8"))
     return parser.page
 
 
+def _heading_jumps(levels: list[int]) -> list[str]:
+    """Each heading more than one level below the one before it: a screen
+    reader user moving by level finds nothing at the level skipped."""
+    return [f"h{before} to h{after}" for before, after in zip(levels, levels[1:])
+            if after > before + 1]
+
+
+def _check_search_index(site: Path) -> list[str]:
+    """The index every visitor who opens the search downloads, against its budget."""
+    import gzip
+
+    path = site / "search" / "search_index.json"
+    if not path.is_file():
+        return []
+    raw = path.read_bytes()
+    packed = len(gzip.compress(raw, 6))
+    issues = []
+    if len(raw) > MAX_SEARCH_INDEX_BYTES:
+        issues.append(f"search/search_index.json: {len(raw):,} bytes, "
+                      f"over the {MAX_SEARCH_INDEX_BYTES:,} budget")
+    if packed > MAX_SEARCH_INDEX_GZIP_BYTES:
+        issues.append(f"search/search_index.json: {packed:,} bytes gzip, "
+                      f"over the {MAX_SEARCH_INDEX_GZIP_BYTES:,} budget")
+    return issues
+
+
 def _base_path(config_path: Path) -> str:
     """Read site_url without constructing executable YAML extension tags."""
     config = config_path.read_text(encoding="utf-8")
@@ -220,6 +282,17 @@ def _check_pages(
             report(path, f"images missing alt: {page.missing_alt[:5]}")
         if page.remote_images:
             report(path, f"images loaded from another host: {page.remote_images[:3]}")
+        if page.remote_scripts:
+            report(path, f"scripts loaded from another host: {page.remote_scripts[:3]}")
+        if page.runtime_diagrams:
+            report(path, f"{page.runtime_diagrams} diagram(s) drawn by a script at runtime")
+        if page.size > MAX_PAGE_BYTES:
+            report(path, f"{page.size:,} bytes of HTML, over the {MAX_PAGE_BYTES:,} budget")
+        jumps = _heading_jumps(page.headings)
+        if jumps:
+            report(path, f"heading levels skipped: {sorted(set(jumps))}")
+        if page.empty_headers:
+            report(path, f"{page.empty_headers} table header cell(s) without text")
         if page.placeholders:
             # scripts/sitediagrams.py knows no diagram of this name.
             report(path, f"image placeholders left unresolved: {page.placeholders[:3]}")
@@ -294,6 +367,7 @@ def validate_site(site: Path, config_path: Path) -> list[str]:
                 issues.append(f"duplicate {label} {value!r}: {rendered}")
 
     issues.extend(_check_links(pages, site, base_path))
+    issues.extend(_check_search_index(site))
 
     return issues
 

tests/test_audit_regressions.py

@@ -779,9 +779,11 @@ class FaqRegressions(unittest.TestCase):
 
     @staticmethod
     def _faq() -> str:
-        # The questions about how files are identified and checked moved to
-        # the technical FAQ when the site was laid out by level.
-        return (ROOT / "wiki" / "technical-faq.md").read_text(encoding="utf-8")
+        # The questions about hashes and checks moved to the technical FAQ.
+        return "\n\n".join(
+            (ROOT / "wiki" / name).read_text(encoding="utf-8")
+            for name in ("faq.md", "technical-faq.md")
+        )
 
     @staticmethod
     def _profile(key: str) -> dict:

tests/test_coverage_cache.py

@@ -0,0 +1,100 @@
+"""The site and the README share one coverage computation.
+
+generate_site.py and generate_readme.py each verified every platform, a
+minute of the site build spent twice on the same answer. The first to run
+stores the coverage, keyed by its inputs, and the second reads it. A cache
+that outlived a change to its inputs would publish stale counts, so these
+tests change an input and expect a fresh computation.
+"""
+
+from __future__ import annotations
+
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_readme  # noqa: E402
+
+
+class CoverageCache(unittest.TestCase):
+    def setUp(self) -> None:
+        self.work = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests"))
+        self.platforms = self.work / "platforms"
+        self.platforms.mkdir()
+        (self.platforms / "alpha.yml").write_text("platform: Alpha\n")
+        self.emulators = self.work / "emulators"
+        self.emulators.mkdir()
+        self.db = self.work / "database.json"
+        self.db.write_text("{}")
+        self.calls: list[str] = []
+        patches = [
+            mock.patch.object(generate_readme, "COVERAGE_CACHE_DIR", self.work / "cache"),
+            mock.patch.object(generate_readme, "compute_coverage", self._compute),
+        ]
+        for patch in patches:
+            patch.start()
+            self.addCleanup(patch.stop)
+
+    def tearDown(self) -> None:
+        import shutil
+
+        shutil.rmtree(self.work, ignore_errors=True)
+
+    def _compute(self, name, platforms_dir, db, data_registry=None, emulators_dir=""):
+        self.calls.append(name)
+        if name == "ghost":
+            raise FileNotFoundError(name)
+        return {"platform": name.title(), "missing_names": {"a.bin"}}
+
+    def _run(self, names=("beta", "alpha")) -> dict:
+        return generate_readme.platform_coverages(
+            list(names), str(self.platforms), {}, {},
+            emulators_dir=str(self.emulators), db_path=str(self.db),
+        )
+
+    def test_second_run_reads_the_first(self):
+        first = self._run()
+        self.assertEqual(self.calls, ["alpha", "beta"])
+        second = self._run()
+        self.assertEqual(self.calls, ["alpha", "beta"])
+        self.assertEqual(first, second)
+        # The caller's order, not the computation's.
+        self.assertEqual(list(second), ["beta", "alpha"])
+        self.assertEqual(second["alpha"]["missing_names"], {"a.bin"})
+
+    def test_a_changed_input_computes_again(self):
+        self._run()
+        for change in (
+            lambda: (self.platforms / "alpha.yml").write_text("platform: Alpha 2\n"),
+            lambda: self.db.write_text('{"files": {}}'),
+            lambda: (self.emulators / "core.yml").write_text("emulator: core\n"),
+        ):
+            self.calls.clear()
+            change()
+            self._run()
+            self.assertEqual(self.calls, ["alpha", "beta"])
+
+    def test_another_platform_list_computes_again(self):
+        self._run()
+        self.calls.clear()
+        self._run(("alpha",))
+        self.assertEqual(self.calls, ["alpha"])
+
+    def test_a_platform_without_a_file_is_left_out(self):
+        self.assertEqual(list(self._run(("ghost", "alpha"))), ["alpha"])
+        self.assertEqual(list(self._run(("ghost", "alpha"))), ["alpha"])
+
+    def test_one_cache_file_is_kept(self):
+        self._run()
+        (self.platforms / "alpha.yml").write_text("platform: Alpha 3\n")
+        self._run()
+        self.assertEqual(len(list((self.work / "cache").glob("*.pickle"))), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_entry_categories.py

@@ -13,7 +13,14 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
 
 import verify  # noqa: E402
-from exporter.candidates import BIOS, GAME_DATA, UNREAD, EntryCategories  # noqa: E402
+from exporter.candidates import (  # noqa: E402
+    BIOS,
+    GAME_DATA,
+    UNREAD,
+    EntryCategories,
+    ProofIndex,
+    ReadEvidence,
+)
 
 PROFILES = {
     "bluemsx": {
@@ -144,5 +151,49 @@ class StrictCountsFirmwareOnly(unittest.TestCase):
         self.assertEqual(evidence["diabdat.mpq"]["read_by"][0]["emulator"], "devilutionx")
 
 
+class IndexedProof(unittest.TestCase):
+    """ProofIndex answers what ReadEvidence.proof answers, reader by reader."""
+
+    def test_every_kind_matches_the_direct_proof(self):
+        md5 = "490f666e1afb15b7362b406ed1cea246"
+        readers = []
+        for profile in (
+            {"files": [{"name": "a.bin", "path": "sys/dir/a.bin"}]},
+            {"files": [{"name": "other.rom", "aliases": ["B.ROM"]}]},
+            {"files": [{"name": "c.bin", "md5": md5}]},
+            {"files": [{"name": "tree/", "type": "directory", "path": "data/tree/"}]},
+            {"files": [], "data_directories": [{"ref": "x", "destination": "assets/sys"}]},
+            {"files": [{"name": "d.bin", "sha1": "ab" * 20, "crc32": "12345678"}]},
+        ):
+            evidence = ReadEvidence()
+            evidence.add_profile(profile, {})
+            readers.append(evidence)
+        launcher = ReadEvidence()
+        launcher.add_launchers([{"names": ["scph101.bin"]}])
+        readers.append(launcher)
+        index = ProofIndex(readers)
+        cases = [
+            ({"name": "a.bin"}, "bios/sys/dir/a.bin"),
+            ({"name": "a.bin"}, "elsewhere/a.bin"),
+            ({"name": "x", "aliases": ["b.rom"]}, "x"),
+            ({"name": "y", "md5": md5[:29]}, "y"),
+            ({"name": "y", "md5": md5.upper()}, "y"),
+            ({"name": "z", "md5": md5[:28]}, "z"),
+            ({"name": "z"}, "root/data/tree/deep/z"),
+            ({"name": "z"}, "data/tree"),
+            ({"name": "z"}, "a/assets/sys/z.png"),
+            ({"name": "scph101.bin"}, "bios/scph101.bin"),
+            ({"name": "q", "crc32": "12345678, 9"}, "q"),
+            ({"name": "q"}, ""),
+        ]
+        for entry, destination in cases:
+            expected = {
+                position: proof for position, evidence in enumerate(readers)
+                if (proof := evidence.proof(entry, destination)) is not None
+            }
+            with self.subTest(entry=entry, destination=destination):
+                self.assertEqual(index.proofs(entry, destination), expected)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_site_exports.py

@@ -651,6 +651,62 @@ class APerCoreDocumentValidates(unittest.TestCase):
         jsonschema.validate(document["items"][0]["profile"], schema)
 
 
+class AHashFindsEveryPageThatNamesIt(unittest.TestCase):
+    """Hashes left the site search, 4 MB of the index every visitor
+    downloaded. files/?hash= reads one shard instead, and it must answer at
+    least what the search answered: the card, the platforms, the profiles."""
+
+    def test_records_cover_every_layer_and_shard_by_prefix(self):
+        import json as _json
+        import tempfile
+
+        import generate_site  # noqa: PLC0415
+
+        sha1 = "ab" * 20
+        md5 = "490f666e1afb15b7362b406ed1cea246"
+        db = {"files": {sha1: {"name": "scph5501.bin", "md5": md5, "crc32": "8D8CB7E4",
+                               "path": "bios/Sony/PlayStation/scph5501.bin"}}}
+        coverages = {"batocera": {"config": {"systems": {"sony-playstation": {"files": [
+            {"name": "scph5501.bin", "md5": f"{md5}, 0123456789abcdef0123456789abcdef"},
+            {"name": "typo.bin", "md5": "not-a-hash"},
+        ]}}}}}
+        profiles = {
+            "duckstation": {"files": [{"name": "scph5501.bin", "md5": md5},
+                                      {"name": "set.zip", "contents": [
+                                          {"name": "m.rom", "crc32": "11223344"}]}]},
+            "an-alias": {"type": "alias", "files": [{"name": "x", "md5": md5}]},
+        }
+        locations = generate_site.file_location_items(db) + [
+            {"sha1": "cd" * 20, "url": "emulators/dolphin/#sha1-" + "cd" * 6}]
+        members = {"dolphin-sys": {"Sys/GC/font.bin": {"sha1": "cd" * 20}}}
+        records = generate_site.file_lookup_records(db, coverages, profiles, locations, members)
+        found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records}
+        self.assertEqual(found[("collection", "", "md5", md5)]["url"],
+                         "systems/sony/#sha1-" + "ab" * 6)
+        self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found)
+        self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin")
+        self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"],
+                         "platforms/batocera/#sony-playstation")
+        self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found)
+        self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"],
+                         "emulators/duckstation/#file-scph5501-bin-490f666e")
+        self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom")
+        self.assertNotIn(("emulator", "an-alias", "md5", md5), found)
+
+        with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp:
+            shards = generate_site.write_lookup_shards(Path(tmp), records, "2026-01-01T00:00:00Z")
+            folder = Path(tmp) / "lookup"
+            names = sorted(path.name for path in folder.glob("*.json"))
+            self.assertEqual(len(names), shards)
+            for path in folder.glob("*.json"):
+                document = _json.loads(path.read_text())
+                self.assertEqual(document["kind"], "file-lookup")
+                self.assertEqual(document["count"], len(document["items"]))
+                for item in document["items"]:
+                    self.assertTrue(item["value"].startswith(path.stem))
+            self.assertNotIn("no.json", names)
+
+
 class AFileHasAnAddressByContent(unittest.TestCase):
     """mt32_control.rom moved from the Commodore page to the Roland page and
     every link to its card broke: the anchor was content-based, the page
@@ -669,7 +725,8 @@ class AFileHasAnAddressByContent(unittest.TestCase):
         )
         self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page())
         script = (Path(__file__).resolve().parents[1] / "docs_assets" / "site.js").read_text()
-        self.assertIn("file-locations.json", script)
+        self.assertIn("api/v1/lookup/", script)
+        self.assertIn('params.has("sha1")', script)
         from cross_reference import FileConsumers  # noqa: PLC0415
 
         card = "\n".join(generate_site._system_file_card(

tests/test_site_render.py

@@ -323,7 +323,7 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
         entry = {"name": "scph5501.bin", "required": True}
         lines = generate_site._render_emulator_file(
             entry, {"files": [entry]}, None, [entry], lambda f: True, "file-scph5501-bin")
-        self.assertTrue(lines[2].startswith("#### `scph5501.bin` "))
+        self.assertTrue(lines[2].startswith("### `scph5501.bin` "))
         self.assertTrue(lines[2].endswith("{ #file-scph5501-bin .rb-file-title }"))
 
     def test_a_system_card_is_a_heading_with_its_anchor(self):
@@ -332,16 +332,17 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
 
         entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3}
         lines = generate_site._system_file_card(entry, FileConsumers({"files": {}}))
-        self.assertTrue(lines[2].startswith("#### `a.bin` "))
+        self.assertTrue(lines[2].startswith("### `a.bin` "))
         self.assertTrue(lines[2].endswith("{ #sha1-aaaaaaaaaaaa .rb-file-title }"))
 
 
 class TheSearchSkipsLinkLists(unittest.TestCase):
     """The search index was 24.7 MB, read and indexed before the first
     result. Lists of platforms, copies and source citations on every card
-    made 6.6 MB of it, names already indexed on their own pages."""
+    made 6.6 MB of it, names already indexed on their own pages, and the
+    hashes 4 MB more: files/?hash= finds those."""
 
-    def test_a_system_card_keeps_its_hashes_and_skips_its_platforms(self):
+    def test_a_system_card_keeps_its_name_and_skips_its_body(self):
         import generate_site
         from cross_reference import FileConsumers
 
@@ -352,26 +353,48 @@ class TheSearchSkipsLinkLists(unittest.TestCase):
         consumers.emulators = {sha1: {"duckstation"}}
         entry = {"name": "a.bin", "sha1": sha1, "md5": "b" * 32, "size": 3}
         text = "\n".join(generate_site._system_file_card(entry, consumers))
-        searched, skipped = text.split('<div class="rb-search-skip" data-search-exclude markdown>')
-        self.assertIn("- MD5:", searched)
-        self.assertIn("- Emulators:", searched)
-        self.assertIn("- Platforms:", skipped)
-        self.assertIn("- Also installed by:", skipped)
-        self.assertEqual(text.count("</div>"), 2)
+        searched, skipped = text.split("<section data-search-exclude markdown>")
+        self.assertIn("`a.bin`", searched)
+        self.assertNotIn("- MD5:", searched)
+        for line in ("- MD5:", "- Emulators:", "- Platforms:", "- Also installed by:"):
+            self.assertIn(line, skipped)
+
+    def test_a_game_data_card_skips_its_name_too(self):
+        import generate_site
+        from cross_reference import FileConsumers
+
+        entry = {"name": "0.ogg", "sha1": "a" * 40, "size": 3}
+        lines = generate_site._system_file_card(
+            entry, FileConsumers({"files": {}}), searchable=False)
+        self.assertTrue(lines[2].endswith("data-search-exclude }"))
 
     def test_runs_alternate_in_card_order(self):
         import generate_site
 
-        lines = ["- SHA1: x", "- Source: `a.c:1`", "- Platforms: p",
-                 "- Destination: `d`", "- Contents (1 members):", "    - `m`"]
+        lines = ["- Path: `p`", "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p",
+                 "- Destination: `d`", "- Contents (1 members):", "    - `m`",
+                 "- Note"]
         out = [line for line in generate_site._search_skipped_apart(lines) if line]
         self.assertEqual(out, [
-            "- SHA1: x",
+            "- Path: `p`",
             '<div class="rb-search-skip" data-search-exclude markdown>',
-            "- Source: `a.c:1`", "- Platforms: p", "</div>",
-            "- Destination: `d`", "- Contents (1 members):", "    - `m`",
+            "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", "</div>",
+            "- Destination: `d`",
+            '<div class="rb-search-skip" data-search-exclude markdown>',
+            "- Contents (1 members):", "    - `m`", "</div>",
+            "- Note",
         ])
 
+    def test_long_tables_leave_the_search(self):
+        import generate_site
+
+        short = "| a |\n|---|\n| 1 |"
+        long = "    | a |\n    |---|\n" + "\n".join(f"    | {'x' * 50} |" for _ in range(80))
+        text = generate_site.tables_out_of_search(f"# T\n\n{short}\n\n{long}\n")
+        self.assertEqual(text.count("data-search-exclude"), 1)
+        self.assertIn("    <section data-search-exclude markdown>\n\n    | a |", text)
+        self.assertIn(short, text)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_site_validation.py

@@ -166,6 +166,67 @@ class RenderedSiteValidation(unittest.TestCase):
 
 
 
+class TheSiteStaysWithinItsBudgets(unittest.TestCase):
+    """The 10.5 MB page and the 21 MB search index passed every check:
+    validate_site now holds the gains, each limit with its reason."""
+
+    def setUp(self) -> None:
+        self.temp = tempfile.TemporaryDirectory(dir=TMP_ROOT)
+        self.site = Path(self.temp.name) / "site"
+        self.site.mkdir()
+        self.config = Path(self.temp.name) / "mkdocs.yml"
+        self.config.write_text("site_url: https://example.test/retrobios/\n", encoding="utf-8")
+
+    def tearDown(self) -> None:
+        self.temp.cleanup()
+
+    def _issues(self, body: str) -> list[str]:
+        (self.site / "index.html").write_text(_page("Home", "Home page.", body), encoding="utf-8")
+        return validate_site(self.site, self.config)
+
+    def test_a_clean_page_passes(self):
+        (self.site / "javascripts").mkdir()
+        (self.site / "javascripts" / "site.js").write_text("", encoding="utf-8")
+        self.assertEqual(self._issues(
+            "<h2 id='a'>A</h2><h3 id='b'>B</h3><h2 id='c'>C</h2>"
+            "<table><thead><tr><th>Detail</th><th>Value</th></tr></thead></table>"
+            '<script src="javascripts/site.js"></script>'), [])
+
+    def test_a_page_over_its_size(self):
+        import validate_site as module
+
+        issues = self._issues("<p>" + "x" * module.MAX_PAGE_BYTES + "</p>")
+        self.assertTrue(any("over the" in issue and "HTML" in issue for issue in issues), issues)
+
+    def test_a_skipped_heading_level(self):
+        issues = self._issues("<h2 id='a'>A</h2><h4 id='b'>B</h4>")
+        self.assertTrue(any("h2 to h4" in issue for issue in issues), issues)
+
+    def test_an_empty_table_header(self):
+        issues = self._issues("<table><thead><tr><th> </th><th></th></tr></thead></table>")
+        self.assertTrue(any("2 table header cell(s) without text" in issue for issue in issues))
+
+    def test_a_script_or_a_diagram_from_another_host(self):
+        issues = self._issues(
+            '<script src="https://unpkg.com/mermaid@11/dist/mermaid.min.js"></script>'
+            '<pre class="mermaid">graph LR</pre>')
+        self.assertTrue(any("scripts loaded from another host" in issue for issue in issues))
+        self.assertTrue(any("drawn by a script at runtime" in issue for issue in issues))
+
+    def test_a_search_index_over_its_budget(self):
+        import random
+
+        import validate_site as module
+
+        (self.site / "search").mkdir()
+        rng = random.Random(1)
+        noise = "".join(rng.choice("0123456789abcdef") for _ in range(module.MAX_SEARCH_INDEX_BYTES + 1))
+        (self.site / "search" / "search_index.json").write_text(noise, encoding="utf-8")
+        issues = self._issues("")
+        self.assertTrue(any("search_index.json" in issue and "gzip" not in issue for issue in issues))
+        self.assertTrue(any("search_index.json" in issue and "gzip" in issue for issue in issues))
+
+
 class UnrenderedMarkdown(unittest.TestCase):
     """The clownmdemu analysis block reached readers as "- **bios_hle:**
     True": a list indented one level too deep, read as code."""

tests/test_site_weight.py

@@ -0,0 +1,168 @@
+"""Pages a phone can open: split pages, light navigation, short tables of contents.
+
+systems/game-engines/ was one 10.5 MB page, emulators/mame/ 3 MB, and each
+of 600 emulator pages carried 65 KB of navigation. A large page is now a
+summary with pages of its own, and every address cited before the split
+still lands: a console's heading stays on the summary, a card's sha1 anchor
+is looked up by content, an emulator card's anchor sits on its row.
+"""
+
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_site  # noqa: E402
+import mkdocs_hooks  # noqa: E402
+import siterender  # noqa: E402
+from cross_reference import FileConsumers  # noqa: E402
+
+
+def _file(name: str, index: int, console: str, maker: str = "Maker") -> dict:
+    sha1 = f"{index:012x}" + "0" * 28
+    return {"name": name, "sha1": sha1, "md5": f"{index:032x}", "size": 10,
+            "path": f"bios/{maker}/{console}/{name}"}
+
+
+class ASystemPageOverItsBudgetIsSplit(unittest.TestCase):
+    def setUp(self) -> None:
+        siterender.reset_console_pages()
+        self.addCleanup(siterender.reset_console_pages)
+        self.consoles = {
+            "Big": [_file(f"big{i:03d}.bin", i, "Big") for i in range(60)],
+            "Small": [_file("small.bin", 900, "Small")],
+        }
+
+    def _pages(self, budget: int) -> dict:
+        return generate_site.system_pages(
+            "Maker", self.consoles, FileConsumers({"files": {}}), budget=budget)
+
+    def test_a_page_within_its_budget_stays_whole(self):
+        pages = self._pages(10_000_000)
+        self.assertEqual(list(pages), ["maker"])
+        self.assertIn("## Big", pages["maker"][0])
+
+    def test_the_largest_section_moves_and_is_cut_in_parts(self):
+        pages = self._pages(20_000)
+        summary = pages["maker"][0]
+        # Both headings stay on the summary: systems/maker/#big still lands.
+        self.assertIn("## Big", summary)
+        self.assertIn("## Small", summary)
+        # The small section keeps its cards in place.
+        self.assertIn("{ #sha1-" + f"{900:012x}", summary)
+        parts = [stem for stem in pages if stem.startswith("maker-big")]
+        self.assertGreater(len(parts), 1)
+        self.assertEqual(parts[0], "maker-big")
+        for stem in parts:
+            self.assertIn(f"]({stem}.md)", summary)
+            self.assertLessEqual(len(pages[stem][0]), 20_000 + 5_000)
+            self.assertIn("## Files", pages[stem][0])
+        # Each card is on exactly one part, and the links follow it.
+        held = [f["sha1"] for stem in parts for f in pages[stem][1]["Big"]]
+        self.assertEqual(sorted(held), sorted(f["sha1"] for f in self.consoles["Big"]))
+        last = self.consoles["Big"][-1]
+        self.assertEqual(siterender.card_page(last["path"], last["sha1"]), parts[-1])
+        small = self.consoles["Small"][0]
+        self.assertEqual(siterender.card_page(small["path"], small["sha1"]), "maker")
+        self.assertEqual(siterender.console_href("maker", "Big"), "systems/maker.md#big")
+
+    def test_a_console_on_one_page_is_linked_directly(self):
+        self.consoles["Big"] = self.consoles["Big"][:12]
+        whole = self._pages(10**9)["maker"][0]
+        pages = self._pages(len(whole) - 1)
+        self.assertIn("maker-big", pages)
+        self.assertNotIn("maker-big-2", pages)
+        self.assertEqual(siterender.console_href("maker", "Big", "../"),
+                         "../systems/maker-big.md")
+
+
+class AnEmulatorWithTooManyCardsIsSplit(unittest.TestCase):
+    def test_parts_are_balanced_and_rows_keep_the_anchors(self):
+        files = [{"name": f"set{i:04d}.zip", "required": i % 2 == 0, "category": "bios_zip"}
+                 for i in range(generate_site.EMULATOR_CARDS_PER_PAGE * 2 + 1)]
+        parts = generate_site.emulator_parts("big", files)
+        self.assertEqual([stem for stem, _ in parts], ["big-files-1", "big-files-2", "big-files-3"])
+        sizes = [len(entries) for _, entries in parts]
+        self.assertLessEqual(max(sizes) - min(sizes), 1)
+        self.assertEqual(generate_site.emulator_parts("small", files[:3]), [])
+
+        pages = generate_site.emulator_pages(
+            "big", {"emulator": "Big", "type": "standalone", "files": files}, {"files": {}})
+        self.assertEqual(sorted(pages), ["big", "big-files-1", "big-files-2", "big-files-3"])
+        main = pages["big"]
+        anchors = generate_site.emulator_file_anchors(files)
+        for anchor in anchors:
+            self.assertIn(f'<span id="{anchor}"></span>', main)
+        self.assertIn('class="rb-card-index"', main)
+        self.assertNotIn(".rb-file-title", main)
+        self.assertIn(f"{{ #{anchors[0]} .rb-file-title }}", pages["big-files-1"])
+
+    def test_cards_are_linked_on_the_page_that_holds_them(self):
+        files = [{"name": f"set{i:04d}.zip"} for i in range(generate_site.EMULATOR_CARDS_PER_PAGE + 1)]
+        generate_site.plan_emulator_pages({"big": {"files": files}})
+        anchors = generate_site.emulator_file_anchors(files)
+        self.assertEqual(siterender.emulator_card_page("big", anchors[-1]), "big-files-2")
+        self.assertEqual(siterender.emulator_card_page("other", "file-x"), "other")
+
+
+class APlatformSystemOverItsBudgetMoves(unittest.TestCase):
+    def test_the_heading_stays_and_the_list_moves(self):
+        files = [{"name": f"f{i:04d}.zip", "status": "ok"} for i in range(40)]
+        config = {("arcade", f["name"]): [{"name": f["name"], "md5": "a" * 32}] for f in files}
+        moved: dict[str, str] = {}
+        budget = generate_site.PLATFORM_SECTION_BUDGET
+        generate_site.PLATFORM_SECTION_BUDGET = 500
+        try:
+            lines = generate_site._render_platform_systems(
+                {"arcade": files}, config, {}, ("deck", "Deck"), moved)
+        finally:
+            generate_site.PLATFORM_SECTION_BUDGET = budget
+        text = "\n".join(lines)
+        self.assertIn("{ #arcade }", text)
+        self.assertIn("](deck-arcade.md)", text)
+        self.assertNotIn("f0001.zip", text)
+        self.assertIn("f0001.zip", moved["deck-arcade"])
+        self.assertIn("[Deck page](deck.md#arcade)", moved["deck-arcade"])
+
+
+class TheNavigationOpensOneInitial(unittest.TestCase):
+    def test_a_group_of_several_initials_is_nested(self):
+        profiles = {
+            key: {"emulator": name, "systems": ["s"]}
+            for key, name in (("a1", "Alpha"), ("b1", "Beta"), ("b2", "bravo"), ("z", "Zed"))
+        }
+        nav = generate_site._emulator_nav(profiles)
+        groups = {next(iter(g)): next(iter(g.values())) for g in nav[1:]}
+        self.assertEqual(groups["A to C"], [
+            {"A": [{"Alpha": "emulators/a1.md"}]},
+            {"B": [{"Beta": "emulators/b1.md"}, {"bravo": "emulators/b2.md"}]},
+        ])
+        self.assertEqual(groups["V to Z"], [{"Zed": "emulators/z.md"}])
+
+
+class TheTableOfContentsOfACardPageStopsAtLevelTwo(unittest.TestCase):
+    @staticmethod
+    def _item(level: int, children: list | None = None) -> SimpleNamespace:
+        return SimpleNamespace(level=level, children=children or [])
+
+    def test_cards_leave_the_toc_and_wiki_pages_keep_level_three(self):
+        def page(uri: str) -> SimpleNamespace:
+            h2 = self._item(2, [self._item(3)])
+            return SimpleNamespace(file=SimpleNamespace(src_uri=uri),
+                                   toc=SimpleNamespace(items=[self._item(1, [h2])]))
+
+        for uri, kept in (("systems/sony.md", 0), ("platforms/retroarch.md", 0),
+                          ("emulators/mame.md", 0), ("wiki/faq.md", 1)):
+            current = page(uri)
+            mkdocs_hooks.on_page_content("", current, None, None)
+            with self.subTest(uri=uri):
+                self.assertEqual(len(current.toc.items[0].children[0].children), kept)
+
+
+if __name__ == "__main__":
+    unittest.main()

wiki/architecture.md

@@ -71,14 +71,32 @@ dist/                    generated packs (gitignored)
 Documentation changes belong in `wiki/` or in the generator.
 
 The site search splits each page at its headings and sends a result to the
-heading above the match. Every file card is therefore a level 4 heading with
-the card's anchor, and every platform system a heading of its own: a hash
-pasted in the search opens that card or that system, not the top of the page.
-Level 4 stays out of the table of contents (`toc_depth: 3`). The lines that
-list other pages (platforms, packs, source citations, collection paths) sit
-in a `data-search-exclude` block, since the browser downloads and indexes the
-whole search index before it shows a first result. File names, hashes, the
-paths an emulator reads and the notes stay searchable.
+heading above the match. Every file card is therefore a level 3 heading with
+the card's anchor, under its console or a Files heading, and every platform
+system a heading of its own: a file name typed in the search opens that card,
+not the top of the page. On pages of cards (systems, emulators, platforms) the
+table of contents stops at level 2 (`scripts/mkdocs_hooks.py`).
+
+The browser downloads and indexes the whole search index before it shows a
+first result, so the index holds names and prose only. Hashes, lists of other
+pages, install commands, member tables and long reference tables sit in
+`data-search-exclude` blocks. Those blocks are `section` elements: the search
+plugin compares elements by tag, and a `div` stopped excluding at the first
+`div` closed inside it. A hash goes to [Find a file](../files/index.md), which
+reads one shard of `api/v1/lookup/` named by the first two characters of the
+value.
+
+A page has a budget. A manufacturer page over 500 KB of Markdown becomes a
+summary: its largest consoles move to pages of their own, cut into parts when
+one console alone is over the budget, and every console keeps its heading on
+the summary. An emulator with more than 400 entries (MAME) lists them in a
+table on its page, each row holding the old anchor of the card, and the cards
+move to pages of 400. A platform system whose list is over 200 KB moves to a
+page of its own. A card anchor (`#sha1-...`) that a page no longer holds is
+looked up by content, so a link cited before a split still opens the card.
+`validate_site.py` fails a page over 1 MB of HTML, a search index over 8.5 MB
+(1.7 MB compressed), a skipped heading level, a table header without text, a
+script from another host and a diagram drawn at runtime.
 
 ## Data flow
 

wiki/index.md

@@ -20,7 +20,7 @@ toolchain works. To set up BIOS files on a device, start with
 ## Data
 
 - **[Data and API](../data.md)** - JSON, CSV and SQLite exports with their checksums
-- **[Find a file by its SHA1](../files/index.md)** - the permanent address of a file by its contents
+- **[Find a file](../files/index.md)** - any SHA1, MD5, CRC32 or SHA256: the file, the platforms that declare it, the emulators that check it
 - **[Data model](data-model.md)** - database.json structure, indexes, file resolution order, YAML formats
 
 ## The toolchain

wiki/release-process.md

@@ -54,7 +54,8 @@ is missing.
 6. Run `generate_readme.py` (rebuilds README.md and CONTRIBUTING.md)
 7. `mkdocs build --strict` to produce the static site
 8. Run `validate_site.py` on the rendered HTML (metadata, headings, image
-   alternatives, duplicate ids, local links and fragments)
+   alternatives, duplicate ids, local links and fragments, page and search
+   index budgets)
 9. Require the committed README and CONTRIBUTING to match what the generator
    just produced. `write_if_changed()` compares content with the timestamp line
    stripped, so a run that only moves the clock leaves the files untouched and

wiki/testing-guide.md

@@ -51,7 +51,7 @@ Every module in `tests/`, by the area it guards.
 | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
 | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
-| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
 
 ## Test architecture
 

wiki/tools.md

@@ -692,7 +692,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6
 | `find_pinned_revision.py` | Walk the upstream commits that touched a file and return the revision whose bytes match a platform's pinned md5 (`--write` saves them) |
 | `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database |
 | `generate_site.py` | Generate all MkDocs site pages (this documentation) |
-| `validate_site.py` | Validate rendered metadata, headings, image alternatives, JSON-LD, local resources, links and fragments |
+| `validate_site.py` | Validate rendered metadata, heading order, image alternatives, JSON-LD, local resources, links and fragments, and the page size and search index budgets |
 | `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held |
 | `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch <revision>` reads FBNeo at that tag, branch or commit |
 | `deterministic_zip.py` | Rebuild MAME BIOS ZIPs deterministically (same ROMs = same hash) |