chore: merge the site weight and speed work

scripts/exporter/candidates.py

@@ -12,6 +12,7 @@ the platform names that no profile describes leaves the verdict open.
 from __future__ import annotations
 
 import sys
+from collections.abc import Iterable
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -136,6 +137,81 @@ class ReadEvidence:
         return None
 
 
+# The order ReadEvidence.proof tries: a reader's proof is the first that holds.
+_PROOF_ORDER = ("path", "launcher", "name", "hash", "directory", "data_directory")
+
+
+class ProofIndex:
+    """ReadEvidence.proof for many readers at once, by lookup.
+
+    Asking every core in turn cost one proof per core and entry: 3.8 million
+    calls to categorize 7,430 entries. Each key a reader can match is indexed
+    once, and an entry's own keys find the readers it concerns. The proof of
+    each reader is the one ReadEvidence.proof returns.
+    """
+
+    def __init__(self, readers: list[ReadEvidence]) -> None:
+        self._index: dict[str, dict[str, set[int]]] = {
+            kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member")
+        }
+        for position, evidence in enumerate(readers):
+            for kind, keys in (
+                ("tail", evidence.tails), ("launched", evidence.launched),
+                ("name", evidence.names), ("hash", evidence.hashes),
+                ("dir", evidence.directories), ("tree", evidence.trees),
+                ("member", evidence.tree_members),
+            ):
+                for key in keys:
+                    self._index[kind].setdefault(key, set()).add(position)
+            # A cut md5 names any hash it is the start of.
+            for value in evidence.hashes:
+                for length in range(_CUT_MD5, _FULL_MD5):
+                    if len(value) >= length:
+                        self._index["cut"].setdefault(value[:length], set()).add(position)
+
+    def _hits(self, kind: str, keys: Iterable[str]) -> set[int]:
+        table = self._index[kind]
+        found: set[int] = set()
+        for key in keys:
+            found |= table.get(key, set())
+        return found
+
+    def proofs(self, entry: dict, destination: str) -> dict[int, str]:
+        """Reader position -> its proof, for the readers that have one."""
+        names = {
+            str(n).casefold()
+            for n in (entry.get("name"), *(entry.get("aliases") or []))
+            if n
+        }
+        tails = _tails(destination)
+        last = {tails[-1]} if tails else set()
+        hashes = {
+            v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
+        }
+        cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5}
+        # "/" + d in "/" + path: d is a run of whole segments ending in "/",
+        # so it is a run of the path's segments before the last one.
+        parts = tails[0].split("/") if tails else []
+        runs = {
+            "/".join(parts[i:j]) + "/"
+            for i in range(len(parts)) for j in range(i + 1, len(parts))
+        }
+        found = {
+            "path": self._hits("tail", tails[:-1]),
+            "launcher": self._hits("launched", names | last),
+            "name": self._hits("name", names | last),
+            "hash": self._hits("hash", hashes) | self._hits("cut", cut),
+            "directory": self._hits("dir", runs),
+            "data_directory": self._hits("tree", runs)
+            | self._index["member"].get(tails[0] if tails else "", set()),
+        }
+        result: dict[int, str] = {}
+        for kind in _PROOF_ORDER:
+            for position in found[kind]:
+                result.setdefault(position, kind)
+        return result
+
+
 def _loads_romsets(profile: dict) -> bool:
     """A core that opens ROM sets by name: MAME pulls a device's set from
     device_t::searchpath(), and no profile lists every device set."""
@@ -275,6 +351,8 @@ class EntryCategories:
                 self._readers.append((kind, key, evidence))
         self._launchers = ReadEvidence()
         self._launchers.add_launchers(config.get("launchers") or [])
+        self._proofs = ProofIndex([evidence for _kind, _key, evidence in self._readers]
+                                  + [self._launchers])
         self._by_system: dict[str, list[str]] = {}
         for key in sorted(examined):
             for sid in profiles[key].get("systems") or []:
@@ -313,14 +391,14 @@ class EntryCategories:
     def explain(self, entry: dict, destination: str, system: str) -> dict:
         """The category of an entry and what establishes it."""
         readers: dict[str, list[dict]] = {BIOS: [], GAME_DATA: []}
-        for kind, key, evidence in self._readers:
-            proof = evidence.proof(entry, destination)
-            if proof is not None:
-                readers[kind].append({
-                    "emulator": key, "proof": proof,
-                    "entries": self._matching(key, kind, entry),
-                })
-        if self._launchers.proof(entry, destination) is not None:
+        proofs = self._proofs.proofs(entry, destination)
+        for position in sorted(p for p in proofs if p < len(self._readers)):
+            kind, key, _evidence = self._readers[position]
+            readers[kind].append({
+                "emulator": key, "proof": proofs[position],
+                "entries": self._matching(key, kind, entry),
+            })
+        if len(self._readers) in proofs:
             readers[BIOS].append({"emulator": "launcher", "proof": "launcher", "entries": []})
         if system in self._system_runs:
             system_cores, unresolved = self._system_runs[system]

scripts/generate_readme.py

@@ -111,6 +111,105 @@ def compute_coverage(
     }
 
 
+_REPO = Path(__file__).resolve().parent.parent
+# The site and the README read the same coverage: the first generator to run
+# stores it here and the second reads it back, instead of verifying every
+# platform twice (about a minute of the site build).
+COVERAGE_CACHE_DIR = _REPO / "tmp" / "coverage-cache"
+_COVERAGE_CACHE_FORMAT = "1"
+
+
+def _coverage_inputs(db_path: Path, platforms_dir: str, emulators_dir: str) -> str:
+    """A digest of everything coverage is computed from.
+
+    The database, the platform and emulator files, the code, the install
+    manifests, the data directory revisions and the state of bios/ on disk:
+    a change to any of them computes the coverage again.
+    """
+    import hashlib
+
+    digest = hashlib.sha256(_COVERAGE_CACHE_FORMAT.encode())
+
+    def add_file(path: Path) -> None:
+        digest.update(str(path).encode() + b"\0")
+        digest.update(path.read_bytes() if path.is_file() else b"<absent>")
+
+    add_file(db_path.resolve())
+    for root, pattern in ((Path(platforms_dir), "*"), (Path(emulators_dir), "*.yml"),
+                          (_REPO / "scripts", "*.py"), (_REPO / "install", "*.json"),
+                          (_REPO / "recipes", "*.json")):
+        for path in sorted(p for p in root.resolve().rglob(pattern) if p.is_file()):
+            add_file(path)
+    for path in (_REPO / ".gitignore", _REPO / "_mame_clones.json",
+                 _REPO / "data" / ".versions.json"):
+        add_file(path)
+    bios = _REPO / "bios"
+    for path in sorted(bios.rglob("*")):
+        if path.is_file():
+            stat = path.stat()
+            digest.update(f"{path.relative_to(bios)}\0{stat.st_size}\0{stat.st_mtime_ns}\n".encode())
+    return digest.hexdigest()
+
+
+def platform_coverages(
+    platform_names: list[str],
+    platforms_dir: str,
+    db: dict,
+    data_registry: dict | None = None,
+    emulators_dir: str = _EMULATORS_DIR,
+    db_path: str = "database.json",
+    report: Callable[[str, dict], None] | None = None,
+) -> dict[str, dict]:
+    """compute_coverage for each platform, read from the cache when its
+    inputs have not changed since a generator last stored it.
+
+    A platform with no YAML is left out, as compute_coverage raises for it.
+    ``report`` is called with each platform and its coverage.
+    """
+    # pickle, not JSON: the coverage holds sets and enums that JSON would
+    # turn into lists and strings. The file is written by this function into
+    # the repository's own tmp/, never downloaded.
+    import pickle
+
+    key = _coverage_inputs(Path(db_path), platforms_dir, emulators_dir)
+    names = sorted(platform_names)
+    cache = COVERAGE_CACHE_DIR / f"{key}.pickle"
+    coverages: dict[str, dict] | None = None
+    if cache.is_file():
+        try:
+            with cache.open("rb") as handle:
+                stored = pickle.load(handle)
+        except (OSError, EOFError, pickle.UnpicklingError) as exc:
+            print(f"  coverage cache unreadable, computing again: {exc}", file=sys.stderr)
+        else:
+            if stored.get("platforms") == names:
+                coverages = stored["coverages"]
+                print(f"  coverage read from {cache.relative_to(_REPO)}")
+    if coverages is None:
+        coverages = {}
+        for name in names:
+            try:
+                coverages[name] = compute_coverage(
+                    name, platforms_dir, db, data_registry, emulators_dir=emulators_dir,
+                )
+            except FileNotFoundError as exc:
+                print(f"  {name}: skipped ({exc})", file=sys.stderr)
+        COVERAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        for stale in COVERAGE_CACHE_DIR.glob("*.pickle"):
+            stale.unlink()
+        scratch = cache.with_suffix(f".{os.getpid()}.tmp")
+        with scratch.open("wb") as handle:
+            pickle.dump({"platforms": names, "coverages": coverages}, handle,
+                        protocol=pickle.HIGHEST_PROTOCOL)
+        scratch.replace(cache)
+    ordered = {name: coverages[name] for name in platform_names if name in coverages}
+    if report:
+        for name in names:
+            if name in ordered:
+                report(name, ordered[name])
+    return ordered
+
+
 def release_totals(platform_name: str, record: dict) -> tuple[int | None, int | None]:
     """Files and bytes of the pack the latest release serves for a platform.
 
@@ -347,7 +446,7 @@ def _catalog_matched_line(db: dict) -> list[str]:
     ]
 
 
-def generate_readme(db: dict, platforms_dir: str) -> str:
+def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json") -> str:
     total_files = db.get("total_files", 0)
     total_size = db.get("total_size", 0)
     ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -358,14 +457,8 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
 
     data_registry = load_data_dir_registry(platforms_dir)
 
-    coverages = {}
-    for name in platform_names:
-        try:
-            coverages[name] = compute_coverage(
-                name, platforms_dir, db, data_registry
-            )
-        except FileNotFoundError:
-            pass
+    coverages = platform_coverages(platform_names, platforms_dir, db, data_registry,
+                                   db_path=db_path)
 
     profiles = unique_emulator_profiles(
         load_emulator_profiles("emulators", skip_aliases=False)
@@ -853,7 +946,7 @@ def main() -> None:
     # published counts taken from half of it.
     with contextlib.ExitStack() as held:
         hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
-        readme = generate_readme(db, args.platforms_dir)
+        readme = generate_readme(db, args.platforms_dir, args.db)
     status = "Generated" if write_if_changed("README.md", readme) else "Unchanged"
     print(f"{status} ./README.md")
 

scripts/mkdocs_hooks.py

@@ -10,8 +10,28 @@ from __future__ import annotations
 
 from typing import Any
 
+# Pages made of file cards, level 3 headings by the hundred: their table of
+# contents stops at level 2, the consoles of a system page, the Files of an
+# emulator, the sections of a platform page (which listed every system id).
+CARD_PAGES = ("systems/", "emulators/", "platforms/")
+
 
 def on_page_markdown(markdown: str, page: Any, config: Any, files: Any) -> str:  # noqa: ARG001
     """Take lastmod from the page's own date, or leave it out."""
     page.update_date = str(page.meta.get("date_modified") or "")
     return markdown
+
+
+def _prune(items: list, depth: int) -> None:
+    for item in items:
+        if item.level >= depth:
+            item.children = []
+        else:
+            _prune(item.children, depth)
+
+
+def on_page_content(html: str, page: Any, config: Any, files: Any) -> str:  # noqa: ARG001
+    """Keep the table of contents of a page of cards to level 2."""
+    if page.file.src_uri.startswith(CARD_PAGES):
+        _prune(page.toc.items, 2)
+    return html

scripts/siterender.py

@@ -139,13 +139,75 @@ def _page_by_norm(sys_id: str) -> tuple[str, str] | None:
     return _norm_page_map.get(_norm_system_id(sys_id))
 
 
+# A manufacturer page too large for one page is a summary, each console on
+# pages of its own: (slug, console) -> its page stems under systems/, and
+# (slug, console, sha1) -> the stem holding that card. A console of a page
+# kept whole has no entry, its section is on the manufacturer's page.
+_console_pages: dict[tuple[str, str], list[str]] = {}
+_card_pages: dict[tuple[str, str, str], str] = {}
+
+
+def reset_console_pages() -> None:
+    _console_pages.clear()
+    _card_pages.clear()
+
+
+def record_console_pages(
+    slug: str, console: str, stems: list[str], sha1s: list[list[str]]
+) -> None:
+    """Where a split manufacturer page put one console: its page stems, in
+    order, and the sha1 of each card on each of them."""
+    _console_pages[(slug, console)] = list(stems)
+    for stem, part in zip(stems, sha1s):
+        for sha1 in part:
+            _card_pages.setdefault((slug, console, sha1), stem)
+
+
+def split_system_stems() -> set[str]:
+    """Every page stem of a console moved to a page of its own."""
+    return {stem for stems in _console_pages.values() for stem in stems}
+
+
+def console_href(slug: str, console: str, prefix: str = "") -> str:
+    """Where a console's files are: its own page, or its section of the
+    manufacturer's page (also where a console cut into parts lists them)."""
+    stems = _console_pages.get((slug, console))
+    if stems and len(stems) == 1:
+        return f"{prefix}systems/{urllib.parse.quote(stems[0])}.md"
+    return f"{prefix}systems/{urllib.parse.quote(slug)}.md#{_slugify_anchor(console)}"
+
+
+def card_page(path: str, sha1: str) -> str:
+    """The systems/ page stem holding the card of the copy at ``path``
+    (bios/<Manufacturer>/<Console>/...)."""
+    parts = path.split("/")
+    slug = parts[1].lower().replace(" ", "-")
+    console = parts[2] if len(parts) > 2 else ""
+    return _card_pages.get((slug, console, sha1), slug)
+
+
+# Emulator cards moved off a page too long for one: (emulator, anchor) ->
+# the page stem under emulators/ that holds the card.
+_emulator_card_pages: dict[tuple[str, str], str] = {}
+
+
+def record_emulator_cards(emulator: str, stem: str, anchors: list[str]) -> None:
+    for anchor in anchors:
+        _emulator_card_pages[(emulator, anchor)] = stem
+
+
+def emulator_card_page(emulator: str, anchor: str) -> str:
+    """The emulators/ page stem holding a card: the emulator's own page, or
+    the page its cards went to."""
+    return _emulator_card_pages.get((emulator, anchor), emulator)
+
+
 def _system_link(sys_id: str, prefix: str = "") -> str:
     """Generate a markdown link to a system page with anchor."""
     page = _system_page_map.get(sys_id) or _page_by_norm(sys_id)
     if page:
         slug, console = page
-        anchor = _slugify_anchor(console)
-        return f"[{sys_id}]({prefix}systems/{slug}.md#{anchor})"
+        return f"[{sys_id}]({console_href(slug, console, prefix)})"
     return sys_id
 
 def _field_text(key: object, value: object) -> str:

scripts/validate_schemas.py

@@ -446,7 +446,7 @@ def main() -> int:
             if (api_dir / name).exists()
         ] + sorted((api_dir / "platforms").glob("*.json")) + sorted(
             (api_dir / "systems").glob("*.json")
-        )
+        ) + sorted((api_dir / "lookup").glob("*.json"))
         errors.extend(
             _validate_json_files(envelope_paths, "site-api-envelope.schema.json")
         )

scripts/validate_site.py

@@ -34,7 +34,25 @@ class Page:
     jsonld: list[str] = field(default_factory=list)
     unrendered: list[str] = field(default_factory=list)
     remote_images: list[str] = field(default_factory=list)
+    remote_scripts: list[str] = field(default_factory=list)
     placeholders: list[str] = field(default_factory=list)
+    headings: list[int] = field(default_factory=list)
+    empty_headers: int = 0
+    runtime_diagrams: int = 0
+    size: int = 0
+
+
+# Budgets the site was brought under on 2026-10-22, with room to grow. A
+# change that passes one is a regression of what a visitor downloads, not a
+# matter of taste: raise the number in the same change that explains why.
+#
+# The largest page was 10.5 MB (systems/game-engines/) and took 21 s of
+# main-thread work on a throttled phone. Split pages now top out at 0.9 MB.
+MAX_PAGE_BYTES = 1_000_000
+# The search index is downloaded by every visitor who opens the search:
+# 21.0 MB raw and 4.88 MB gzip with hashes in it, 7.3 MB and 1.44 MB without.
+MAX_SEARCH_INDEX_BYTES = 8_500_000
+MAX_SEARCH_INDEX_GZIP_BYTES = 1_700_000
 
 
 # Text inside these is shown as written, so markup there is not a leftover.
@@ -50,11 +68,21 @@ class _PageParser(HTMLParser):
         self._jsonld_depth = 0
         self._jsonld_parts: list[str] = []
         self._literal_depth = 0
+        self._header_parts: list[str] | None = None
 
     def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
-        data = dict(attrs)
+        data = {key: value or "" for key, value in attrs}
         if tag in _LITERAL_TAGS:
             self._literal_depth += 1
+        if re.fullmatch(r"h[1-6]", tag):
+            self.page.headings.append(int(tag[1]))
+        if tag == "th":
+            self._header_parts = []
+        if "mermaid" in data.get("class", "").split():
+            # Material renders it in the browser with a script from a CDN.
+            self.page.runtime_diagrams += 1
+        if tag == "script" and re.match(r"\s*(?:https?:)?//", data.get("src", "")):
+            self.page.remote_scripts.append(data["src"])
         if tag == "html":
             self.page.lang = data.get("lang", "")
         elif tag == "title":
@@ -97,6 +125,10 @@ class _PageParser(HTMLParser):
     def handle_endtag(self, tag: str) -> None:
         if tag in _LITERAL_TAGS and self._literal_depth:
             self._literal_depth -= 1
+        if tag == "th" and self._header_parts is not None:
+            if not "".join(self._header_parts).strip():
+                self.page.empty_headers += 1
+            self._header_parts = None
         if tag == "title":
             self._title_depth -= 1
         elif tag == "script" and self._jsonld_depth:
@@ -105,6 +137,8 @@ class _PageParser(HTMLParser):
             self._jsonld_parts = []
 
     def handle_data(self, data: str) -> None:
+        if self._header_parts is not None:
+            self._header_parts.append(data)
         if self._title_depth:
             self._title_parts.append(data)
         if self._jsonld_depth:
@@ -119,11 +153,39 @@ class _PageParser(HTMLParser):
 
 def _parse_page(path: Path) -> Page:
     parser = _PageParser(path)
-    parser.feed(path.read_text(encoding="utf-8", errors="replace"))
+    text = path.read_text(encoding="utf-8", errors="replace")
+    parser.feed(text)
     parser.close()
+    parser.page.size = len(text.encode("utf-8"))
     return parser.page
 
 
+def _heading_jumps(levels: list[int]) -> list[str]:
+    """Each heading more than one level below the one before it: a screen
+    reader user moving by level finds nothing at the level skipped."""
+    return [f"h{before} to h{after}" for before, after in zip(levels, levels[1:])
+            if after > before + 1]
+
+
+def _check_search_index(site: Path) -> list[str]:
+    """The index every visitor who opens the search downloads, against its budget."""
+    import gzip
+
+    path = site / "search" / "search_index.json"
+    if not path.is_file():
+        return []
+    raw = path.read_bytes()
+    packed = len(gzip.compress(raw, 6))
+    issues = []
+    if len(raw) > MAX_SEARCH_INDEX_BYTES:
+        issues.append(f"search/search_index.json: {len(raw):,} bytes, "
+                      f"over the {MAX_SEARCH_INDEX_BYTES:,} budget")
+    if packed > MAX_SEARCH_INDEX_GZIP_BYTES:
+        issues.append(f"search/search_index.json: {packed:,} bytes gzip, "
+                      f"over the {MAX_SEARCH_INDEX_GZIP_BYTES:,} budget")
+    return issues
+
+
 def _base_path(config_path: Path) -> str:
     """Read site_url without constructing executable YAML extension tags."""
     config = config_path.read_text(encoding="utf-8")
@@ -220,6 +282,17 @@ def _check_pages(
             report(path, f"images missing alt: {page.missing_alt[:5]}")
         if page.remote_images:
             report(path, f"images loaded from another host: {page.remote_images[:3]}")
+        if page.remote_scripts:
+            report(path, f"scripts loaded from another host: {page.remote_scripts[:3]}")
+        if page.runtime_diagrams:
+            report(path, f"{page.runtime_diagrams} diagram(s) drawn by a script at runtime")
+        if page.size > MAX_PAGE_BYTES:
+            report(path, f"{page.size:,} bytes of HTML, over the {MAX_PAGE_BYTES:,} budget")
+        jumps = _heading_jumps(page.headings)
+        if jumps:
+            report(path, f"heading levels skipped: {sorted(set(jumps))}")
+        if page.empty_headers:
+            report(path, f"{page.empty_headers} table header cell(s) without text")
         if page.placeholders:
             # scripts/sitediagrams.py knows no diagram of this name.
             report(path, f"image placeholders left unresolved: {page.placeholders[:3]}")
@@ -294,6 +367,7 @@ def validate_site(site: Path, config_path: Path) -> list[str]:
                 issues.append(f"duplicate {label} {value!r}: {rendered}")
 
     issues.extend(_check_links(pages, site, base_path))
+    issues.extend(_check_search_index(site))
 
     return issues