feat: find a file by any hash, out of the search

Hashes, link lists, install commands, member tables and long reference
tables leave the site search: the index drops from 21.0 MB (4.88 MB
gzip) to 6.8 MB (1.33 MB gzip). Names and prose stay searchable.

files/?hash= takes a SHA1, MD5, CRC32 or SHA256 and reads one shard of
api/v1/lookup/ (about 22 KB gzip, was 675 KB) listing the collection
card, the platforms that declare the value and the profiles that check
it. files/?sha1= still opens the card, and a sha1- anchor missing from
a page is looked up there, so a card can move between pages.

Excluded blocks are section elements: the search plugin compares
elements by tag, so a div ended at the first nested div.

docs_assets/site.js

@@ -148,28 +148,88 @@
     });
   };
 
-  // files/?sha1=<sha1> is a file's content address: the card it names may
-  // sit on another manufacturer's page today than when the link was cited.
+  // files/?hash= reads one shard of the lookup, named by the first two hex
+  // characters of the value: 20 KB where the whole index was 675 KB.
+  const HASH_LENGTHS = { 8: "CRC32", 32: "MD5", 40: "SHA1", 64: "SHA256" };
+  const LAYER_LABELS = {
+    collection: "In the collection",
+    "data-directory": "In a data directory the packs ship",
+    platform: "Declared by a platform",
+    emulator: "Read by an emulator",
+  };
+  const siteRoot = () => {
+    const script = document.querySelector('script[src$="javascripts/site.js"]');
+    return script ? new URL("../", script.src) : new URL("../", window.location.href);
+  };
+  const lookupShard = (value) =>
+    fetch(new URL(`api/v1/lookup/${value.slice(0, 2)}.json`, siteRoot()))
+      .then((response) => (response.ok ? response.json() : { items: [] }))
+      .then((shard) => shard.items || []);
+  const normalizeHash = (raw) => raw.trim().toLowerCase().replace(/^0x/, "").replace(/\s+/g, "");
+
+  const renderMatches = (status, value, items) => {
+    status.textContent = "";
+    const kind = HASH_LENGTHS[value.length];
+    const heading = document.createElement("p");
+    heading.textContent = `${items.length} ${items.length === 1 ? "match" : "matches"} for ${kind} ${value}:`;
+    status.append(heading);
+    Object.entries(LAYER_LABELS).forEach(([layer, label]) => {
+      const found = items.filter((item) => item.layer === layer);
+      if (!found.length) return;
+      const title = document.createElement("p");
+      const strong = document.createElement("strong");
+      strong.textContent = label;
+      title.append(strong);
+      const list = document.createElement("ul");
+      found.forEach((item) => {
+        const row = document.createElement("li");
+        const link = document.createElement("a");
+        link.href = new URL(item.url, siteRoot()).href;
+        const code = document.createElement("code");
+        code.textContent = item.name || item.value;
+        link.append(code);
+        row.append(link);
+        if (item.owner) row.append(` (${item.owner})`);
+        list.append(row);
+      });
+      status.append(title, list);
+    });
+  };
+
+  // files/?sha1=<sha1> is a file's content address and opens its card
+  // wherever it is now; files/?hash= also takes an MD5, CRC32 or SHA256 and
+  // lists every page that names the value.
   const resolveFileAddress = () => {
     const status = document.getElementById("rb-file-lookup-status");
     if (!status) return;
-    const sha1 = (new URLSearchParams(window.location.search).get("sha1") || "")
-      .trim().toLowerCase();
-    if (!sha1) return;
-    if (!/^[0-9a-f]{40}$/.test(sha1)) {
-      status.textContent = "A SHA1 is 40 hexadecimal characters.";
+    const params = new URLSearchParams(window.location.search);
+    const permalink = params.has("sha1");
+    const value = normalizeHash(params.get("sha1") || params.get("hash") || "");
+    if (!value) return;
+    const input = document.getElementById("rb-hash");
+    if (input) input.value = value;
+    const prefix = permalink && /^[0-9a-f]{12}$/.test(value);
+    if (!prefix && !(/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length])) {
+      status.textContent =
+        "A hash is 8 (CRC32), 32 (MD5), 40 (SHA1) or 64 (SHA256) hexadecimal characters.";
       return;
     }
     status.textContent = "Looking it up...";
-    const index = new URL("../api/v1/file-locations.json", window.location.href);
-    fetch(index)
-      .then((response) => response.json())
-      .then((document_) => {
-        const item = (document_.items || []).find((entry) => entry.sha1 === sha1);
-        if (item) {
-          window.location.replace(new URL("../" + item.url, window.location.href));
+    lookupShard(value)
+      .then((items) => {
+        const matches = items.filter((item) =>
+          prefix
+            ? item.algorithm === "sha1" && item.value.startsWith(value)
+            : item.value === value
+        );
+        const cards = matches.filter((item) => item.layer === "collection" || item.layer === "data-directory");
+        const targets = new Set(cards.map((item) => item.url));
+        if (permalink && targets.size === 1) {
+          window.location.replace(new URL(cards[0].url, siteRoot()));
+        } else if (matches.length) {
+          renderMatches(status, prefix ? matches[0].value : value, prefix ? cards : matches);
         } else {
-          status.textContent = "The collection holds no file with this SHA1.";
+          status.textContent = "No file the site knows has this hash.";
         }
       })
       .catch(() => {
@@ -177,10 +237,35 @@
       });
   };
 
+  // A file card moved to another page keeps its address: an id of the form
+  // sha1-<12 hex> that this page does not hold is looked up by content.
+  const followMovedCard = () => {
+    const match = /^#sha1-([0-9a-f]{12})$/.exec(window.location.hash);
+    if (!match || document.getElementById(`sha1-${match[1]}`)) return;
+    window.location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot()));
+  };
+
+  // The site search no longer holds hashes: Enter on a hash opens the lookup.
+  const hashSearch = () => {
+    const query = document.querySelector('input[data-md-component="search-query"]');
+    if (!query || query.dataset.rbHash === "true") return;
+    query.dataset.rbHash = "true";
+    query.addEventListener("keydown", (event) => {
+      if (event.key !== "Enter") return;
+      const value = normalizeHash(query.value);
+      if (/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length]) {
+        event.preventDefault();
+        window.location.assign(new URL(`files/?hash=${value}`, siteRoot()));
+      }
+    });
+  };
+
   const enhance = () => {
     enhanceTables();
     nameMaterialWidgets();
     resolveFileAddress();
+    followMovedCard();
+    hashSearch();
   };
 
   let resizeTimer = 0;

mkdocs.yml

@@ -835,7 +835,7 @@ nav:
   - Dump provenance: provenance.md
   - Updated platform files: exports.md
   - Data and API: data.md
-  - Find a file by its SHA1: files/index.md
+  - Find a file: files/index.md
   - Data model: wiki/data-model.md
   - Architecture: wiki/architecture.md
   - Build your own packs: wiki/advanced-usage.md

schemas/site-api-envelope.schema.json

@@ -27,7 +27,8 @@
         "emulators",
         "systems",
         "verification-and-coverage-gaps",
-        "file-locations"
+        "file-locations",
+        "file-lookup"
       ]
     },
     "count": {
@@ -332,6 +333,66 @@
           }
         }
       }
+    },
+    {
+      "if": {
+        "properties": {
+          "kind": {
+            "const": "file-lookup"
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "items": {
+            "items": {
+              "type": "object",
+              "required": [
+                "value",
+                "algorithm",
+                "layer",
+                "name",
+                "url"
+              ],
+              "properties": {
+                "value": {
+                  "type": "string",
+                  "pattern": "^[0-9a-f]+$"
+                },
+                "algorithm": {
+                  "enum": [
+                    "sha1",
+                    "md5",
+                    "crc32",
+                    "sha256"
+                  ]
+                },
+                "layer": {
+                  "enum": [
+                    "collection",
+                    "data-directory",
+                    "platform",
+                    "emulator"
+                  ]
+                },
+                "name": {
+                  "type": "string"
+                },
+                "url": {
+                  "type": "string"
+                },
+                "sha1": {
+                  "type": "string",
+                  "pattern": "^[0-9a-f]{40}$"
+                },
+                "owner": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   ]
 }

scripts/exporter/candidates.py

@@ -12,6 +12,7 @@ the platform names that no profile describes leaves the verdict open.
 from __future__ import annotations
 
 import sys
+from collections.abc import Iterable
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -168,7 +169,7 @@ class ProofIndex:
                     if len(value) >= length:
                         self._index["cut"].setdefault(value[:length], set()).add(position)
 
-    def _hits(self, kind: str, keys) -> set[int]:
+    def _hits(self, kind: str, keys: Iterable[str]) -> set[int]:
         table = self._index[kind]
         found: set[int] = set()
         for key in keys:

scripts/generate_site.py

@@ -79,6 +79,7 @@ from siteguides import (
     DEVICE_GROUPS,
     GROUP_ANCHORS,
     GUIDES,
+    Guide,
     INSTALL_BASE,
     install_command,
     platform_args,
@@ -1130,13 +1131,17 @@ def generate_data_exports(
     gap_rows = _gap_export_rows(coverages, gap_report)
     cross_rows = _cross_reference_export_rows(coverages, profiles)
 
+    members = data_directory_members()
+    locations = file_location_items(db, profiles, members)
     envelopes = {
         "platforms.json": ("platforms", platform_items),
         "emulators.json": ("emulators", emulator_items),
         "gaps.json": ("verification-and-coverage-gaps", gap_rows),
-        "file-locations.json": ("file-locations", file_location_items(
-            db, profiles, data_directory_members())),
+        "file-locations.json": ("file-locations", locations),
     }
+    write_lookup_shards(
+        api, file_lookup_records(db, coverages, profiles, locations, members), generated_at,
+    )
     for filename, (kind, items) in envelopes.items():
         document = {
             "schema_version": 1,
@@ -1338,6 +1343,13 @@ def generate_data_page(exports: list[dict]) -> str:
         "platform of `platforms.json`, and `api/v1/systems/<slug>.json` the files "
         "of one system page by console. Each platform and system page links its own.",
         "",
+        "One hash without the whole list: `api/v1/lookup/<xx>.json`, where `<xx>` "
+        "is the first two characters of the hash in lowercase, holds every SHA1, "
+        "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` "
+        "(the collection, a data directory, a platform's list or an emulator "
+        "profile), the file `name` and the `url` of the page that shows it. "
+        "[Find a file](files/index.md) reads it.",
+        "",
         "## Bulk downloads",
         "",
         "| Export | Format | Size | SHA256 |",
@@ -2059,6 +2071,7 @@ def _render_platform_systems(
         # emulator cards link here, and the site search splits a page at its
         # headings. Folded into blocks under one heading, every hash pasted
         # in the search led to the systems overview of a 100-block page.
+        block_start = len(lines) + 2
         if use_collapsible:
             status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue")
             lines.append(f"### {sys_link} {{ #{anchor} }}")
@@ -2141,7 +2154,9 @@ def _render_platform_systems(
             if f.get("category_evidence"):
                 lines.append(f"{pad}    {_category_evidence_md(f['category_evidence'])}")
 
-        lines.append("")
+        # The section's heading stays searchable, its file list with every
+        # hash is the lookup's: retrodeck's arcade list was 495 KB of index.
+        lines[block_start:] = search_excluded(lines[block_start:])
 
     return lines
 
@@ -2572,28 +2587,44 @@ def generate_platform_page(
 
 
 def generate_files_page() -> str:
-    """The content address of every held file: files/?sha1=<sha1>.
+    """Find a file by any of its hashes: files/?hash=<value>.
 
-    site.js reads api/v1/file-locations.json and goes to the card. Without
-    a sha1, or with one the collection does not hold, the page says so.
+    site.js reads the one shard of api/v1/lookup/ the value names and lists
+    every page that cites it. files/?sha1=<sha1> is the card's permanent
+    address and opens the card directly.
     """
     return "\n".join([
-        "# Find a file by its SHA1",
+        "# Find a file",
         "",
-        "A file's card lives on its manufacturer's page, and a file refiled "
-        "between manufacturers moves with it. `files/?sha1=<sha1>` is the "
-        "address that does not move: it opens the card wherever it is now. "
-        "Each card's **permalink** is that address.",
+        "Paste a hash from clrmamepro, RomVault, an emulator log or a platform's "
+        "BIOS list. The page lists the file in the collection that has it, the "
+        "platforms that declare it and the emulators that check for it.",
         "",
         '<form class="rb-file-lookup" action="./" method="get">',
-        '<label for="rb-sha1">SHA1</label> ',
-        '<input id="rb-sha1" name="sha1" size="44" pattern="[0-9a-fA-F]{40}" required> ',
-        '<button type="submit">Open</button>',
+        '<label for="rb-hash">SHA1, MD5, CRC32 or SHA256</label> ',
+        '<input id="rb-hash" name="hash" size="66" autocomplete="off" spellcheck="false" '
+        'pattern="\\s*(0x)?([0-9a-fA-F]{8}|[0-9a-fA-F]{32}|[0-9a-fA-F]{40}|[0-9a-fA-F]{64})\\s*" '
+        'required> ',
+        '<button type="submit">Find</button>',
         "</form>",
         "",
         '<p id="rb-file-lookup-status" role="status"></p>',
         "",
-        "The index behind it is [`api/v1/file-locations.json`](../api/v1/file-locations.json).",
+        "To look a file up by its name, use the search at the top of the page.",
+        "",
+        "## Permanent links",
+        "",
+        "A file's card lives on its manufacturer's page, and a file refiled "
+        "between manufacturers moves with it. `files/?sha1=<sha1>` is the address "
+        "that does not move: it opens the card wherever it is now. Each card's "
+        "**permalink** is that address.",
+        "",
+        "## The index behind it",
+        "",
+        "The lookup reads one file of [`api/v1/lookup/`](../data.md), named by the "
+        "first two characters of the hash, about 20 KB compressed. "
+        "[`api/v1/file-locations.json`](../api/v1/file-locations.json) lists every "
+        "card by SHA1 in one file.",
         "",
     ])
 
@@ -2688,6 +2719,99 @@ def file_location_items(
     return sorted(items, key=lambda item: item["sha1"])
 
 
+LOOKUP_PREFIX = 2  # hex characters naming a lookup shard: 256 files
+
+
+def file_lookup_records(
+    db: dict,
+    coverages: dict,
+    profiles: dict,
+    locations: list[dict],
+    members: dict | None = None,
+) -> list[dict]:
+    """Every hash the site knows, one record per value, for files/?hash=.
+
+    The collection's own files lead to their card, a platform's declared
+    value to the system section of its page, a profile's to the entry's
+    card. The site search indexed these values as text, 4 MB of the 21 MB
+    it made every visitor download: the lookup reads one shard instead.
+    """
+    files = db.get("files", {})
+    member_names = {
+        str(info.get("sha1", "")): member.rsplit("/", 1)[-1]
+        for tree in (members or {}).values() for member, info in sorted(tree.items())
+    }
+    records: list[dict] = []
+    for item in locations:
+        sha1 = item["sha1"]
+        entry = files.get(sha1)
+        if entry is None:
+            # A data directory member: the manifests record its sha1 only.
+            records.append({"value": sha1, "algorithm": "sha1", "layer": "data-directory",
+                            "name": member_names.get(sha1, ""), "sha1": sha1,
+                            "url": item["url"]})
+            continue
+        for algorithm in HASH_ALGORITHMS:
+            for value in _hash_values(entry.get(algorithm)):
+                records.append({
+                    "value": value, "algorithm": algorithm, "layer": "collection",
+                    "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"],
+                })
+    for key, coverage in sorted(coverages.items()):
+        for system_id, system in sorted((coverage["config"].get("systems") or {}).items()):
+            url = f"platforms/{key}/#{system_id.replace(' ', '-')}"
+            for entry in system.get("files") or []:
+                for algorithm in HASH_ALGORITHMS:
+                    for value in _hash_values(entry.get(algorithm)):
+                        records.append({
+                            "value": value, "algorithm": algorithm, "layer": "platform",
+                            "owner": key, "name": str(entry.get("name", "")), "url": url,
+                        })
+    for key, profile in sorted(profiles.items()):
+        if profile.get("type") in ("alias", "test"):
+            continue
+        entries = profile.get("files") or []
+        for entry, anchor in zip(entries, emulator_file_anchors(entries)):
+            url = f"emulators/{key}/#{anchor}"
+            members = [(str(entry.get("name", "")), entry)] + [
+                (str(member["name"]), member) for member in entry.get("contents") or []
+                if isinstance(member, dict) and member.get("name")
+            ]
+            for name, holder in members:
+                for algorithm in HASH_ALGORITHMS:
+                    for value in _hash_values(holder.get(algorithm)):
+                        records.append({
+                            "value": value, "algorithm": algorithm, "layer": "emulator",
+                            "owner": key, "name": name, "url": url,
+                        })
+    unique = {json.dumps(r, sort_keys=True): r for r in records}
+    return sorted(unique.values(), key=lambda r: (
+        r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"]))
+
+
+def write_lookup_shards(api: Path, records: list[dict], generated_at: str) -> int:
+    """One envelope per leading pair of hex characters, as the lookup asks.
+
+    A value that is not hexadecimal (a typo in a profile) has no shard the
+    lookup would read, and is left out. Returns the number of shards.
+    """
+    shards: dict[str, list[dict]] = {}
+    for record in records:
+        if re.fullmatch(r"[0-9a-f]{8,}", record["value"]):
+            shards.setdefault(record["value"][:LOOKUP_PREFIX], []).append(record)
+    folder = api / "lookup"
+    folder.mkdir(parents=True, exist_ok=True)
+    for prefix, items in sorted(shards.items()):
+        write_if_changed(str(folder / f"{prefix}.json"), json.dumps({
+            "schema_version": 1,
+            "generated_at": generated_at,
+            "kind": "file-lookup",
+            "count": len(items),
+            "items": items,
+        }, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n")
+    return len(shards)
+
+
 def collection_paths(db: dict) -> dict[str, set[str]]:
     """Every path under bios/ that holds each file, by sha1.
 
@@ -3101,9 +3225,14 @@ def _installed_links(sha1: str, consumers: FileConsumers, declared: set[str]) ->
 
 
 def _system_file_card(
-    f: dict, consumers: FileConsumers, anchored: set[str] | None = None
+    f: dict, consumers: FileConsumers, anchored: set[str] | None = None,
+    searchable: bool = True,
 ) -> list[str]:
-    """One file of a system page: hashes, who reads it, its provenance."""
+    """One file of a system page: hashes, who reads it, its provenance.
+
+    ``searchable`` false keeps even the name out of the site search: the
+    asset trees of game engines hold thousands of 0.ogg and tile.png.
+    """
     name = f["name"]
     sha1_full = f.get("sha1", "unknown")
     md5_full = f.get("md5", "unknown")
@@ -3129,7 +3258,7 @@ def _system_file_card(
     ) if anchor else ""
     lines.append(_card_title(
         f'{_code(name)} <span class="rb-badge rb-badge-muted">{size}</span>{permalink}',
-        anchor,
+        anchor, searchable,
     ))
     lines.append("")
     # Every hash in full: the site search indexes text, not a title
@@ -3160,17 +3289,68 @@ def _system_file_card(
         # looking up scph5501.bin found every platform but RetroBat, whose
         # pack installs it for six cores.
         lines.append(f"- Also installed by: {', '.join(installed)}")
-    lines[2:] = _search_skipped_apart(lines[2:])
+    # The title stays searchable: the file name is what a reader types.
+    lines[4:] = search_excluded(lines[4:])
     lines.extend(["", "</div>", ""])
     return lines
 
 
+SEARCH_TABLE_BYTES = 3000
+
+
+def tables_out_of_search(markdown: str, threshold: int = SEARCH_TABLE_BYTES) -> str:
+    """Each table longer than ``threshold`` bytes, wrapped out of the search.
+
+    A search result shows the start of a section: a 600-row table as one
+    snippet helps nobody, and the gaps, provenance and cross-reference
+    tables were 900 KB of the index. The headings above them stay.
+    """
+    lines = markdown.split("\n")
+    out: list[str] = []
+    index = 0
+    while index < len(lines):
+        match = re.match(r"( *)\|", lines[index])
+        if not match:
+            out.append(lines[index])
+            index += 1
+            continue
+        indent = match.group(1)
+        end = index
+        while end < len(lines) and re.match(rf"{indent}\|", lines[end]):
+            end += 1
+        table = lines[index:end]
+        if sum(len(line) for line in table) > threshold:
+            out += ["", f"{indent}<section data-search-exclude markdown>", "", *table,
+                    "", f"{indent}</section>", ""]
+        else:
+            out += table
+        index = end
+    return "\n".join(out)
+
+
+def search_excluded(lines: list[str]) -> list[str]:
+    """Lines the site search leaves out, as a block of their own.
+
+    The browser downloads and indexes the whole search index before the
+    first result: hashes, link lists and install commands repeated on 600
+    pages made it 21 MB. Hashes are found by files/?hash=, names and prose
+    stay searchable.
+
+    A section, not a div: the search plugin compares elements by tag, so
+    the first div closed inside an excluded div (a code block's) ended the
+    exclusion and the rest of the block was indexed.
+    """
+    return ["", "<section data-search-exclude markdown>", "", *lines, "", "</section>", ""]
+
+
 # Card lines the site search leaves out. They list where a file goes and
 # who cites it, links to other pages that index the same names, and they
 # were 6.6 of the 24.7 MB the browser downloads and indexes before the first
 # result. The name, the hashes, the entry's path and the notes stay
 # searchable.
 SEARCH_SKIPPED = (
+    "- SHA1:", "- MD5:", "- CRC32:", "- SHA256:", "- Adler32", "- Size:",
+    "- Validation", "- Priority:", "- Contents (",
     "- Platforms:", "- Also installed by:", "- Source:", "- Asked for by:",
     "- Collection path:", "- Collection copy:", "- Collection copies:",
     "- Installed by the platform's own package on:",
@@ -3195,7 +3375,7 @@ def _search_skipped_apart(lines: list[str]) -> list[str]:
     return out
 
 
-def _card_title(text: str, anchor: str) -> str:
+def _card_title(text: str, anchor: str, searchable: bool = True) -> str:
     """A file card's heading, so the site search lands on the card.
 
     The search splits a page at its headings: a card titled in bold sent a
@@ -3204,6 +3384,8 @@ def _card_title(text: str, anchor: str) -> str:
     keeps the id the toc gives it.
     """
     attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title"
+    if not searchable:
+        attributes += " data-search-exclude"
     return f"#### {text} {{ {attributes} }}"
 
 
@@ -3256,6 +3438,7 @@ def generate_system_page(
         "",
     ]
     anchored: set[str] = set()
+    searchable = manufacturer not in GAME_DATA_TOPS
 
     for console_name in sorted(consoles.keys()):
         files = consoles[console_name]
@@ -3267,7 +3450,7 @@ def generate_system_page(
         variant_files = [f for f in files if "/.variants/" in f["path"]]
 
         for f in sorted(main_files, key=lambda x: x["name"]):
-            lines.extend(_system_file_card(f, consumers, anchored))
+            lines.extend(_system_file_card(f, consumers, anchored, searchable))
 
         if variant_files:
             # A variant gets the same card as a primary: the revision a
@@ -3278,7 +3461,7 @@ def generate_system_page(
             for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])):
                 lines.extend(
                     f"    {line}" if line else ""
-                    for line in _system_file_card(v, consumers, anchored)
+                    for line in _system_file_card(v, consumers, anchored, searchable)
                 )
             lines.append("")
 
@@ -3907,14 +4090,14 @@ def _install_block(name: str, profile: dict) -> list[str]:
     """
     if profile.get("type") in ("alias", "launcher", "test"):
         return []
-    return [
+    return search_excluded([
         '??? tip "Install this emulator\'s files"',
         "",
         "    The installer finds your platform and its BIOS folder, then downloads "
         "only the files this emulator reads there and checks each one.",
         "",
         *install_commands(f"--emulator {name}", "    "),
-    ]
+    ])
 
 
 def _platform_install_block(name: str, display: str, on_card: bool = False) -> list[str]:
@@ -3975,7 +4158,7 @@ def _console_install_block(console: str, declared: list[tuple[str, str]]) -> lis
         f"    {install_command('sh', f'--platform {platform} --system {shlex.quote(system)}')}"
         for platform, system in declared
     ]
-    return lines + ["    ```", ""]
+    return search_excluded(lines + ["    ```", ""])
 
 
 def _render_platform_details(profile: dict) -> list[str]:
@@ -4351,7 +4534,9 @@ def generate_emulator_page(
             )
 
     if data_dirs:
-        lines.extend(["## Data directories", ""])
+        # The member tables are a file list with hashes: out of the search,
+        # dolphin, primehack and ishiiruka each put 210 KB of it there.
+        lines.extend(["## Data directories { data-search-exclude }", ""])
         member_anchors: set[str] = set()
         # Each tree with what it holds, where it comes from, which build reads
         # it and the code that does: these trees put thousands of files in
@@ -5536,7 +5721,7 @@ def _mkdocs_config(docs: Path, text: str) -> tuple[Path, str]:
 _OS_NAMES = {"windows": "Windows", "darwin": "macOS", "linux": "Linux"}
 
 
-def _runs_on(entry: dict, guide) -> str:
+def _runs_on(entry: dict, guide: Guide) -> str:
     """Where a setup runs, read from the systems the installer detects it on."""
     if guide.on_card:
         return "SD card, filled from a computer"
@@ -5553,7 +5738,7 @@ def _runs_on(entry: dict, guide) -> str:
     return ", ".join(names)
 
 
-def _folder_cell(name: str, guide, prefix: str = "") -> str:
+def _folder_cell(name: str, guide: Guide, prefix: str = "") -> str:
     """The BIOS folder in a table cell, or a link where it depends on the system."""
     if len(guide.folders) > 3:
         return f"Depends on the system: [listed on its page]({prefix}platforms/{name}.md#where-the-files-go)"
@@ -5840,7 +6025,7 @@ def generate_mkdocs_nav(
             {"Dump provenance": "provenance.md"},
             {"Updated platform files": "exports.md"},
             {"Data and API": "data.md"},
-            {"Find a file by its SHA1": "files/index.md"},
+            {"Find a file": "files/index.md"},
             {"Data model": "wiki/data-model.md"},
             {"Architecture": "wiki/architecture.md"},
             {"Build your own packs": "wiki/advanced-usage.md"},
@@ -6089,15 +6274,15 @@ def _generate(args: argparse.Namespace) -> None:
     print("Generating cross-reference page...")
     write_if_changed(
         str(docs / "cross-reference.md"),
-        generate_cross_reference(coverages, profiles) + REFERENCE_NEXT,
+        tables_out_of_search(generate_cross_reference(coverages, profiles)) + REFERENCE_NEXT,
     )
 
     print("Generating gap analysis page...")
     write_if_changed(
         str(docs / "gaps.md"),
-        generate_gap_analysis(
+        tables_out_of_search(generate_gap_analysis(
             profiles, coverages, db, suppl_names, registry, gap_report
-        ) + REFERENCE_NEXT,
+        )) + REFERENCE_NEXT,
     )
 
     print("Generating updated platform files page...")
@@ -6113,7 +6298,9 @@ def _generate(args: argparse.Namespace) -> None:
     provenance_report = build_report(db, load_provenance_snapshots())
     write_if_changed(
         str(docs / "provenance.md"),
-        generate_provenance_page(db, provenance_report, load_derivable()) + REFERENCE_NEXT,
+        tables_out_of_search(
+            generate_provenance_page(db, provenance_report, load_derivable())
+        ) + REFERENCE_NEXT,
     )
 
     # Wiki pages: copy manually maintained sources + generate dynamic ones

scripts/validate_schemas.py

@@ -446,7 +446,7 @@ def main() -> int:
             if (api_dir / name).exists()
         ] + sorted((api_dir / "platforms").glob("*.json")) + sorted(
             (api_dir / "systems").glob("*.json")
-        )
+        ) + sorted((api_dir / "lookup").glob("*.json"))
         errors.extend(
             _validate_json_files(envelope_paths, "site-api-envelope.schema.json")
         )

tests/test_site_exports.py

@@ -651,6 +651,62 @@ class APerCoreDocumentValidates(unittest.TestCase):
         jsonschema.validate(document["items"][0]["profile"], schema)
 
 
+class AHashFindsEveryPageThatNamesIt(unittest.TestCase):
+    """Hashes left the site search, 4 MB of the index every visitor
+    downloaded. files/?hash= reads one shard instead, and it must answer at
+    least what the search answered: the card, the platforms, the profiles."""
+
+    def test_records_cover_every_layer_and_shard_by_prefix(self):
+        import json as _json
+        import tempfile
+
+        import generate_site  # noqa: PLC0415
+
+        sha1 = "ab" * 20
+        md5 = "490f666e1afb15b7362b406ed1cea246"
+        db = {"files": {sha1: {"name": "scph5501.bin", "md5": md5, "crc32": "8D8CB7E4",
+                               "path": "bios/Sony/PlayStation/scph5501.bin"}}}
+        coverages = {"batocera": {"config": {"systems": {"sony-playstation": {"files": [
+            {"name": "scph5501.bin", "md5": f"{md5}, 0123456789abcdef0123456789abcdef"},
+            {"name": "typo.bin", "md5": "not-a-hash"},
+        ]}}}}}
+        profiles = {
+            "duckstation": {"files": [{"name": "scph5501.bin", "md5": md5},
+                                      {"name": "set.zip", "contents": [
+                                          {"name": "m.rom", "crc32": "11223344"}]}]},
+            "an-alias": {"type": "alias", "files": [{"name": "x", "md5": md5}]},
+        }
+        locations = generate_site.file_location_items(db) + [
+            {"sha1": "cd" * 20, "url": "emulators/dolphin/#sha1-" + "cd" * 6}]
+        members = {"dolphin-sys": {"Sys/GC/font.bin": {"sha1": "cd" * 20}}}
+        records = generate_site.file_lookup_records(db, coverages, profiles, locations, members)
+        found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records}
+        self.assertEqual(found[("collection", "", "md5", md5)]["url"],
+                         "systems/sony/#sha1-" + "ab" * 6)
+        self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found)
+        self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin")
+        self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"],
+                         "platforms/batocera/#sony-playstation")
+        self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found)
+        self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"],
+                         "emulators/duckstation/#file-scph5501-bin-490f666e")
+        self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom")
+        self.assertNotIn(("emulator", "an-alias", "md5", md5), found)
+
+        with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp:
+            shards = generate_site.write_lookup_shards(Path(tmp), records, "2026-01-01T00:00:00Z")
+            folder = Path(tmp) / "lookup"
+            names = sorted(path.name for path in folder.glob("*.json"))
+            self.assertEqual(len(names), shards)
+            for path in folder.glob("*.json"):
+                document = _json.loads(path.read_text())
+                self.assertEqual(document["kind"], "file-lookup")
+                self.assertEqual(document["count"], len(document["items"]))
+                for item in document["items"]:
+                    self.assertTrue(item["value"].startswith(path.stem))
+            self.assertNotIn("no.json", names)
+
+
 class AFileHasAnAddressByContent(unittest.TestCase):
     """mt32_control.rom moved from the Commodore page to the Roland page and
     every link to its card broke: the anchor was content-based, the page
@@ -669,7 +725,8 @@ class AFileHasAnAddressByContent(unittest.TestCase):
         )
         self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page())
         script = (Path(__file__).resolve().parents[1] / "docs_assets" / "site.js").read_text()
-        self.assertIn("file-locations.json", script)
+        self.assertIn("api/v1/lookup/", script)
+        self.assertIn('params.has("sha1")', script)
         from cross_reference import FileConsumers  # noqa: PLC0415
 
         card = "\n".join(generate_site._system_file_card(

tests/test_site_render.py

@@ -339,9 +339,10 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
 class TheSearchSkipsLinkLists(unittest.TestCase):
     """The search index was 24.7 MB, read and indexed before the first
     result. Lists of platforms, copies and source citations on every card
-    made 6.6 MB of it, names already indexed on their own pages."""
+    made 6.6 MB of it, names already indexed on their own pages, and the
+    hashes 4 MB more: files/?hash= finds those."""
 
-    def test_a_system_card_keeps_its_hashes_and_skips_its_platforms(self):
+    def test_a_system_card_keeps_its_name_and_skips_its_body(self):
         import generate_site
         from cross_reference import FileConsumers
 
@@ -352,26 +353,48 @@ class TheSearchSkipsLinkLists(unittest.TestCase):
         consumers.emulators = {sha1: {"duckstation"}}
         entry = {"name": "a.bin", "sha1": sha1, "md5": "b" * 32, "size": 3}
         text = "\n".join(generate_site._system_file_card(entry, consumers))
-        searched, skipped = text.split('<div class="rb-search-skip" data-search-exclude markdown>')
-        self.assertIn("- MD5:", searched)
-        self.assertIn("- Emulators:", searched)
-        self.assertIn("- Platforms:", skipped)
-        self.assertIn("- Also installed by:", skipped)
-        self.assertEqual(text.count("</div>"), 2)
+        searched, skipped = text.split("<section data-search-exclude markdown>")
+        self.assertIn("`a.bin`", searched)
+        self.assertNotIn("- MD5:", searched)
+        for line in ("- MD5:", "- Emulators:", "- Platforms:", "- Also installed by:"):
+            self.assertIn(line, skipped)
+
+    def test_a_game_data_card_skips_its_name_too(self):
+        import generate_site
+        from cross_reference import FileConsumers
+
+        entry = {"name": "0.ogg", "sha1": "a" * 40, "size": 3}
+        lines = generate_site._system_file_card(
+            entry, FileConsumers({"files": {}}), searchable=False)
+        self.assertTrue(lines[2].endswith("data-search-exclude }"))
 
     def test_runs_alternate_in_card_order(self):
         import generate_site
 
-        lines = ["- SHA1: x", "- Source: `a.c:1`", "- Platforms: p",
-                 "- Destination: `d`", "- Contents (1 members):", "    - `m`"]
+        lines = ["- Path: `p`", "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p",
+                 "- Destination: `d`", "- Contents (1 members):", "    - `m`",
+                 "- Note"]
         out = [line for line in generate_site._search_skipped_apart(lines) if line]
         self.assertEqual(out, [
-            "- SHA1: x",
+            "- Path: `p`",
             '<div class="rb-search-skip" data-search-exclude markdown>',
-            "- Source: `a.c:1`", "- Platforms: p", "</div>",
-            "- Destination: `d`", "- Contents (1 members):", "    - `m`",
+            "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", "</div>",
+            "- Destination: `d`",
+            '<div class="rb-search-skip" data-search-exclude markdown>',
+            "- Contents (1 members):", "    - `m`", "</div>",
+            "- Note",
         ])
 
+    def test_long_tables_leave_the_search(self):
+        import generate_site
+
+        short = "| a |\n|---|\n| 1 |"
+        long = "    | a |\n    |---|\n" + "\n".join(f"    | {'x' * 50} |" for _ in range(80))
+        text = generate_site.tables_out_of_search(f"# T\n\n{short}\n\n{long}\n")
+        self.assertEqual(text.count("data-search-exclude"), 1)
+        self.assertIn("    <section data-search-exclude markdown>\n\n    | a |", text)
+        self.assertIn(short, text)
+
 
 if __name__ == "__main__":
     unittest.main()

wiki/index.md

@@ -20,7 +20,7 @@ toolchain works. To set up BIOS files on a device, start with
 ## Data
 
 - **[Data and API](../data.md)** - JSON, CSV and SQLite exports with their checksums
-- **[Find a file by its SHA1](../files/index.md)** - the permanent address of a file by its contents
+- **[Find a file](../files/index.md)** - any SHA1, MD5, CRC32 or SHA256: the file, the platforms that declare it, the emulators that check it
 - **[Data model](data-model.md)** - database.json structure, indexes, file resolution order, YAML formats
 
 ## The toolchain