refactor: compute coverage once, index proofs

EntryCategories asked every core in turn: 3.8 million proofs for 7,430
entries. ProofIndex looks each key up once, and verify --all --json is
byte-identical (24 s to 17 s). generate_site and generate_readme now share
one coverage, cached in tmp/coverage-cache keyed by its inputs, so the
README step drops from 25 s to 3 s.

scripts/exporter/candidates.py

@@ -136,6 +136,81 @@ class ReadEvidence:
         return None
 
 
+# The order ReadEvidence.proof tries: a reader's proof is the first that holds.
+_PROOF_ORDER = ("path", "launcher", "name", "hash", "directory", "data_directory")
+
+
+class ProofIndex:
+    """ReadEvidence.proof for many readers at once, by lookup.
+
+    Asking every core in turn cost one proof per core and entry: 3.8 million
+    calls to categorize 7,430 entries. Each key a reader can match is indexed
+    once, and an entry's own keys find the readers it concerns. The proof of
+    each reader is the one ReadEvidence.proof returns.
+    """
+
+    def __init__(self, readers: list[ReadEvidence]) -> None:
+        self._index: dict[str, dict[str, set[int]]] = {
+            kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member")
+        }
+        for position, evidence in enumerate(readers):
+            for kind, keys in (
+                ("tail", evidence.tails), ("launched", evidence.launched),
+                ("name", evidence.names), ("hash", evidence.hashes),
+                ("dir", evidence.directories), ("tree", evidence.trees),
+                ("member", evidence.tree_members),
+            ):
+                for key in keys:
+                    self._index[kind].setdefault(key, set()).add(position)
+            # A cut md5 names any hash it is the start of.
+            for value in evidence.hashes:
+                for length in range(_CUT_MD5, _FULL_MD5):
+                    if len(value) >= length:
+                        self._index["cut"].setdefault(value[:length], set()).add(position)
+
+    def _hits(self, kind: str, keys) -> set[int]:
+        table = self._index[kind]
+        found: set[int] = set()
+        for key in keys:
+            found |= table.get(key, set())
+        return found
+
+    def proofs(self, entry: dict, destination: str) -> dict[int, str]:
+        """Reader position -> its proof, for the readers that have one."""
+        names = {
+            str(n).casefold()
+            for n in (entry.get("name"), *(entry.get("aliases") or []))
+            if n
+        }
+        tails = _tails(destination)
+        last = {tails[-1]} if tails else set()
+        hashes = {
+            v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
+        }
+        cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5}
+        # "/" + d in "/" + path: d is a run of whole segments ending in "/",
+        # so it is a run of the path's segments before the last one.
+        parts = tails[0].split("/") if tails else []
+        runs = {
+            "/".join(parts[i:j]) + "/"
+            for i in range(len(parts)) for j in range(i + 1, len(parts))
+        }
+        found = {
+            "path": self._hits("tail", tails[:-1]),
+            "launcher": self._hits("launched", names | last),
+            "name": self._hits("name", names | last),
+            "hash": self._hits("hash", hashes) | self._hits("cut", cut),
+            "directory": self._hits("dir", runs),
+            "data_directory": self._hits("tree", runs)
+            | self._index["member"].get(tails[0] if tails else "", set()),
+        }
+        result: dict[int, str] = {}
+        for kind in _PROOF_ORDER:
+            for position in found[kind]:
+                result.setdefault(position, kind)
+        return result
+
+
 def _loads_romsets(profile: dict) -> bool:
     """A core that opens ROM sets by name: MAME pulls a device's set from
     device_t::searchpath(), and no profile lists every device set."""
@@ -275,6 +350,8 @@ class EntryCategories:
                 self._readers.append((kind, key, evidence))
         self._launchers = ReadEvidence()
         self._launchers.add_launchers(config.get("launchers") or [])
+        self._proofs = ProofIndex([evidence for _kind, _key, evidence in self._readers]
+                                  + [self._launchers])
         self._by_system: dict[str, list[str]] = {}
         for key in sorted(examined):
             for sid in profiles[key].get("systems") or []:
@@ -313,14 +390,14 @@ class EntryCategories:
     def explain(self, entry: dict, destination: str, system: str) -> dict:
         """The category of an entry and what establishes it."""
         readers: dict[str, list[dict]] = {BIOS: [], GAME_DATA: []}
-        for kind, key, evidence in self._readers:
-            proof = evidence.proof(entry, destination)
-            if proof is not None:
-                readers[kind].append({
-                    "emulator": key, "proof": proof,
-                    "entries": self._matching(key, kind, entry),
-                })
-        if self._launchers.proof(entry, destination) is not None:
+        proofs = self._proofs.proofs(entry, destination)
+        for position in sorted(p for p in proofs if p < len(self._readers)):
+            kind, key, _evidence = self._readers[position]
+            readers[kind].append({
+                "emulator": key, "proof": proofs[position],
+                "entries": self._matching(key, kind, entry),
+            })
+        if len(self._readers) in proofs:
             readers[BIOS].append({"emulator": "launcher", "proof": "launcher", "entries": []})
         if system in self._system_runs:
             system_cores, unresolved = self._system_runs[system]

scripts/generate_readme.py

@@ -111,6 +111,105 @@ def compute_coverage(
     }
 
 
+_REPO = Path(__file__).resolve().parent.parent
+# The site and the README read the same coverage: the first generator to run
+# stores it here and the second reads it back, instead of verifying every
+# platform twice (about a minute of the site build).
+COVERAGE_CACHE_DIR = _REPO / "tmp" / "coverage-cache"
+_COVERAGE_CACHE_FORMAT = "1"
+
+
+def _coverage_inputs(db_path: Path, platforms_dir: str, emulators_dir: str) -> str:
+    """A digest of everything coverage is computed from.
+
+    The database, the platform and emulator files, the code, the install
+    manifests, the data directory revisions and the state of bios/ on disk:
+    a change to any of them computes the coverage again.
+    """
+    import hashlib
+
+    digest = hashlib.sha256(_COVERAGE_CACHE_FORMAT.encode())
+
+    def add_file(path: Path) -> None:
+        digest.update(str(path).encode() + b"\0")
+        digest.update(path.read_bytes() if path.is_file() else b"<absent>")
+
+    add_file(db_path.resolve())
+    for root, pattern in ((Path(platforms_dir), "*"), (Path(emulators_dir), "*.yml"),
+                          (_REPO / "scripts", "*.py"), (_REPO / "install", "*.json"),
+                          (_REPO / "recipes", "*.json")):
+        for path in sorted(p for p in root.resolve().rglob(pattern) if p.is_file()):
+            add_file(path)
+    for path in (_REPO / ".gitignore", _REPO / "_mame_clones.json",
+                 _REPO / "data" / ".versions.json"):
+        add_file(path)
+    bios = _REPO / "bios"
+    for path in sorted(bios.rglob("*")):
+        if path.is_file():
+            stat = path.stat()
+            digest.update(f"{path.relative_to(bios)}\0{stat.st_size}\0{stat.st_mtime_ns}\n".encode())
+    return digest.hexdigest()
+
+
+def platform_coverages(
+    platform_names: list[str],
+    platforms_dir: str,
+    db: dict,
+    data_registry: dict | None = None,
+    emulators_dir: str = _EMULATORS_DIR,
+    db_path: str = "database.json",
+    report: Callable[[str, dict], None] | None = None,
+) -> dict[str, dict]:
+    """compute_coverage for each platform, read from the cache when its
+    inputs have not changed since a generator last stored it.
+
+    A platform with no YAML is left out, as compute_coverage raises for it.
+    ``report`` is called with each platform and its coverage.
+    """
+    # pickle, not JSON: the coverage holds sets and enums that JSON would
+    # turn into lists and strings. The file is written by this function into
+    # the repository's own tmp/, never downloaded.
+    import pickle
+
+    key = _coverage_inputs(Path(db_path), platforms_dir, emulators_dir)
+    names = sorted(platform_names)
+    cache = COVERAGE_CACHE_DIR / f"{key}.pickle"
+    coverages: dict[str, dict] | None = None
+    if cache.is_file():
+        try:
+            with cache.open("rb") as handle:
+                stored = pickle.load(handle)
+        except (OSError, EOFError, pickle.UnpicklingError) as exc:
+            print(f"  coverage cache unreadable, computing again: {exc}", file=sys.stderr)
+        else:
+            if stored.get("platforms") == names:
+                coverages = stored["coverages"]
+                print(f"  coverage read from {cache.relative_to(_REPO)}")
+    if coverages is None:
+        coverages = {}
+        for name in names:
+            try:
+                coverages[name] = compute_coverage(
+                    name, platforms_dir, db, data_registry, emulators_dir=emulators_dir,
+                )
+            except FileNotFoundError as exc:
+                print(f"  {name}: skipped ({exc})", file=sys.stderr)
+        COVERAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        for stale in COVERAGE_CACHE_DIR.glob("*.pickle"):
+            stale.unlink()
+        scratch = cache.with_suffix(f".{os.getpid()}.tmp")
+        with scratch.open("wb") as handle:
+            pickle.dump({"platforms": names, "coverages": coverages}, handle,
+                        protocol=pickle.HIGHEST_PROTOCOL)
+        scratch.replace(cache)
+    ordered = {name: coverages[name] for name in platform_names if name in coverages}
+    if report:
+        for name in names:
+            if name in ordered:
+                report(name, ordered[name])
+    return ordered
+
+
 def release_totals(platform_name: str, record: dict) -> tuple[int | None, int | None]:
     """Files and bytes of the pack the latest release serves for a platform.
 
@@ -347,7 +446,7 @@ def _catalog_matched_line(db: dict) -> list[str]:
     ]
 
 
-def generate_readme(db: dict, platforms_dir: str) -> str:
+def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json") -> str:
     total_files = db.get("total_files", 0)
     total_size = db.get("total_size", 0)
     ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -358,14 +457,8 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
 
     data_registry = load_data_dir_registry(platforms_dir)
 
-    coverages = {}
-    for name in platform_names:
-        try:
-            coverages[name] = compute_coverage(
-                name, platforms_dir, db, data_registry
-            )
-        except FileNotFoundError:
-            pass
+    coverages = platform_coverages(platform_names, platforms_dir, db, data_registry,
+                                   db_path=db_path)
 
     profiles = unique_emulator_profiles(
         load_emulator_profiles("emulators", skip_aliases=False)
@@ -853,7 +946,7 @@ def main() -> None:
     # published counts taken from half of it.
     with contextlib.ExitStack() as held:
         hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
-        readme = generate_readme(db, args.platforms_dir)
+        readme = generate_readme(db, args.platforms_dir, args.db)
     status = "Generated" if write_if_changed("README.md", readme) else "Unchanged"
     print(f"{status} ./README.md")
 

scripts/generate_site.py

@@ -65,6 +65,7 @@ from nativemode import digest_algorithm, reads_file_contents
 yaml = require_yaml()
 from generate_readme import (
     compute_coverage,
+    platform_coverages,
     contributing_text,
     download_links,
     extract_notes,
@@ -5955,19 +5956,13 @@ def _generate(args: argparse.Namespace) -> None:
     suppl_names = _build_supplemental_index()
 
     print("Computing platform coverage...")
-    coverages = {}
-    for name in sorted(platform_names):
-        try:
-            cov = compute_coverage(
-                name, args.platforms_dir, db, data_registry,
-                emulators_dir=args.emulators_dir,
-            )
-            coverages[name] = cov
-            print(
-                f"  {cov['platform']}: {cov['present']}/{cov['total']} ({_pct(cov['present'], cov['total'])})"
-            )
-        except FileNotFoundError as e:
-            print(f"  {name}: skipped ({e})", file=sys.stderr)
+    coverages = platform_coverages(
+        sorted(platform_names), args.platforms_dir, db, data_registry,
+        emulators_dir=args.emulators_dir, db_path=args.db,
+        report=lambda _name, cov: print(
+            f"  {cov['platform']}: {cov['present']}/{cov['total']} ({_pct(cov['present'], cov['total'])})"
+        ),
+    )
 
     print("Loading emulator profiles...")
     profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)