diff --git a/docs_assets/site.js b/docs_assets/site.js index db4d72e1..aacaaa24 100644 --- a/docs_assets/site.js +++ b/docs_assets/site.js @@ -148,28 +148,88 @@ }); }; - // files/?sha1= is a file's content address: the card it names may - // sit on another manufacturer's page today than when the link was cited. + // files/?hash= reads one shard of the lookup, named by the first two hex + // characters of the value: 20 KB where the whole index was 675 KB. + const HASH_LENGTHS = { 8: "CRC32", 32: "MD5", 40: "SHA1", 64: "SHA256" }; + const LAYER_LABELS = { + collection: "In the collection", + "data-directory": "In a data directory the packs ship", + platform: "Declared by a platform", + emulator: "Read by an emulator", + }; + const siteRoot = () => { + const script = document.querySelector('script[src$="javascripts/site.js"]'); + return script ? new URL("../", script.src) : new URL("../", window.location.href); + }; + const lookupShard = (value) => + fetch(new URL(`api/v1/lookup/${value.slice(0, 2)}.json`, siteRoot())) + .then((response) => (response.ok ? response.json() : { items: [] })) + .then((shard) => shard.items || []); + const normalizeHash = (raw) => raw.trim().toLowerCase().replace(/^0x/, "").replace(/\s+/g, ""); + + const renderMatches = (status, value, items) => { + status.textContent = ""; + const kind = HASH_LENGTHS[value.length]; + const heading = document.createElement("p"); + heading.textContent = `${items.length} ${items.length === 1 ? "match" : "matches"} for ${kind} ${value}:`; + status.append(heading); + Object.entries(LAYER_LABELS).forEach(([layer, label]) => { + const found = items.filter((item) => item.layer === layer); + if (!found.length) return; + const title = document.createElement("p"); + const strong = document.createElement("strong"); + strong.textContent = label; + title.append(strong); + const list = document.createElement("ul"); + found.forEach((item) => { + const row = document.createElement("li"); + const link = document.createElement("a"); + link.href = new URL(item.url, siteRoot()).href; + const code = document.createElement("code"); + code.textContent = item.name || item.value; + link.append(code); + row.append(link); + if (item.owner) row.append(` (${item.owner})`); + list.append(row); + }); + status.append(title, list); + }); + }; + + // files/?sha1= is a file's content address and opens its card + // wherever it is now; files/?hash= also takes an MD5, CRC32 or SHA256 and + // lists every page that names the value. const resolveFileAddress = () => { const status = document.getElementById("rb-file-lookup-status"); if (!status) return; - const sha1 = (new URLSearchParams(window.location.search).get("sha1") || "") - .trim().toLowerCase(); - if (!sha1) return; - if (!/^[0-9a-f]{40}$/.test(sha1)) { - status.textContent = "A SHA1 is 40 hexadecimal characters."; + const params = new URLSearchParams(window.location.search); + const permalink = params.has("sha1"); + const value = normalizeHash(params.get("sha1") || params.get("hash") || ""); + if (!value) return; + const input = document.getElementById("rb-hash"); + if (input) input.value = value; + const prefix = permalink && /^[0-9a-f]{12}$/.test(value); + if (!prefix && !(/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length])) { + status.textContent = + "A hash is 8 (CRC32), 32 (MD5), 40 (SHA1) or 64 (SHA256) hexadecimal characters."; return; } status.textContent = "Looking it up..."; - const index = new URL("../api/v1/file-locations.json", window.location.href); - fetch(index) - .then((response) => response.json()) - .then((document_) => { - const item = (document_.items || []).find((entry) => entry.sha1 === sha1); - if (item) { - window.location.replace(new URL("../" + item.url, window.location.href)); + lookupShard(value) + .then((items) => { + const matches = items.filter((item) => + prefix + ? item.algorithm === "sha1" && item.value.startsWith(value) + : item.value === value + ); + const cards = matches.filter((item) => item.layer === "collection" || item.layer === "data-directory"); + const targets = new Set(cards.map((item) => item.url)); + if (permalink && targets.size === 1) { + window.location.replace(new URL(cards[0].url, siteRoot())); + } else if (matches.length) { + renderMatches(status, prefix ? matches[0].value : value, prefix ? cards : matches); } else { - status.textContent = "The collection holds no file with this SHA1."; + status.textContent = "No file the site knows has this hash."; } }) .catch(() => { @@ -177,10 +237,35 @@ }); }; + // A file card moved to another page keeps its address: an id of the form + // sha1-<12 hex> that this page does not hold is looked up by content. + const followMovedCard = () => { + const match = /^#sha1-([0-9a-f]{12})$/.exec(window.location.hash); + if (!match || document.getElementById(`sha1-${match[1]}`)) return; + window.location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot())); + }; + + // The site search no longer holds hashes: Enter on a hash opens the lookup. + const hashSearch = () => { + const query = document.querySelector('input[data-md-component="search-query"]'); + if (!query || query.dataset.rbHash === "true") return; + query.dataset.rbHash = "true"; + query.addEventListener("keydown", (event) => { + if (event.key !== "Enter") return; + const value = normalizeHash(query.value); + if (/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length]) { + event.preventDefault(); + window.location.assign(new URL(`files/?hash=${value}`, siteRoot())); + } + }); + }; + const enhance = () => { enhanceTables(); nameMaterialWidgets(); resolveFileAddress(); + followMovedCard(); + hashSearch(); }; let resizeTimer = 0; diff --git a/mkdocs.yml b/mkdocs.yml index 74e26649..2bbc5ee9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -835,7 +835,7 @@ nav: - Dump provenance: provenance.md - Updated platform files: exports.md - Data and API: data.md - - Find a file by its SHA1: files/index.md + - Find a file: files/index.md - Data model: wiki/data-model.md - Architecture: wiki/architecture.md - Build your own packs: wiki/advanced-usage.md diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index 18bf24ff..dcd422b2 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -27,7 +27,8 @@ "emulators", "systems", "verification-and-coverage-gaps", - "file-locations" + "file-locations", + "file-lookup" ] }, "count": { @@ -332,6 +333,66 @@ } } } + }, + { + "if": { + "properties": { + "kind": { + "const": "file-lookup" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "value", + "algorithm", + "layer", + "name", + "url" + ], + "properties": { + "value": { + "type": "string", + "pattern": "^[0-9a-f]+$" + }, + "algorithm": { + "enum": [ + "sha1", + "md5", + "crc32", + "sha256" + ] + }, + "layer": { + "enum": [ + "collection", + "data-directory", + "platform", + "emulator" + ] + }, + "name": { + "type": "string" + }, + "url": { + "type": "string" + }, + "sha1": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "owner": { + "type": "string" + } + } + } + } + } + } } ] } diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py index e4fe619f..7af1656c 100644 --- a/scripts/exporter/candidates.py +++ b/scripts/exporter/candidates.py @@ -12,6 +12,7 @@ the platform names that no profile describes leaves the verdict open. from __future__ import annotations import sys +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path @@ -168,7 +169,7 @@ class ProofIndex: if len(value) >= length: self._index["cut"].setdefault(value[:length], set()).add(position) - def _hits(self, kind: str, keys) -> set[int]: + def _hits(self, kind: str, keys: Iterable[str]) -> set[int]: table = self._index[kind] found: set[int] = set() for key in keys: diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 8350170e..b94d3779 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -79,6 +79,7 @@ from siteguides import ( DEVICE_GROUPS, GROUP_ANCHORS, GUIDES, + Guide, INSTALL_BASE, install_command, platform_args, @@ -1130,13 +1131,17 @@ def generate_data_exports( gap_rows = _gap_export_rows(coverages, gap_report) cross_rows = _cross_reference_export_rows(coverages, profiles) + members = data_directory_members() + locations = file_location_items(db, profiles, members) envelopes = { "platforms.json": ("platforms", platform_items), "emulators.json": ("emulators", emulator_items), "gaps.json": ("verification-and-coverage-gaps", gap_rows), - "file-locations.json": ("file-locations", file_location_items( - db, profiles, data_directory_members())), + "file-locations.json": ("file-locations", locations), } + write_lookup_shards( + api, file_lookup_records(db, coverages, profiles, locations, members), generated_at, + ) for filename, (kind, items) in envelopes.items(): document = { "schema_version": 1, @@ -1338,6 +1343,13 @@ def generate_data_page(exports: list[dict]) -> str: "platform of `platforms.json`, and `api/v1/systems/.json` the files " "of one system page by console. Each platform and system page links its own.", "", + "One hash without the whole list: `api/v1/lookup/.json`, where `` " + "is the first two characters of the hash in lowercase, holds every SHA1, " + "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` " + "(the collection, a data directory, a platform's list or an emulator " + "profile), the file `name` and the `url` of the page that shows it. " + "[Find a file](files/index.md) reads it.", + "", "## Bulk downloads", "", "| Export | Format | Size | SHA256 |", @@ -2059,6 +2071,7 @@ def _render_platform_systems( # emulator cards link here, and the site search splits a page at its # headings. Folded into blocks under one heading, every hash pasted # in the search led to the systems overview of a 100-block page. + block_start = len(lines) + 2 if use_collapsible: status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue") lines.append(f"### {sys_link} {{ #{anchor} }}") @@ -2141,7 +2154,9 @@ def _render_platform_systems( if f.get("category_evidence"): lines.append(f"{pad} {_category_evidence_md(f['category_evidence'])}") - lines.append("") + # The section's heading stays searchable, its file list with every + # hash is the lookup's: retrodeck's arcade list was 495 KB of index. + lines[block_start:] = search_excluded(lines[block_start:]) return lines @@ -2572,28 +2587,44 @@ def generate_platform_page( def generate_files_page() -> str: - """The content address of every held file: files/?sha1=. + """Find a file by any of its hashes: files/?hash=. - site.js reads api/v1/file-locations.json and goes to the card. Without - a sha1, or with one the collection does not hold, the page says so. + site.js reads the one shard of api/v1/lookup/ the value names and lists + every page that cites it. files/?sha1= is the card's permanent + address and opens the card directly. """ return "\n".join([ - "# Find a file by its SHA1", + "# Find a file", "", - "A file's card lives on its manufacturer's page, and a file refiled " - "between manufacturers moves with it. `files/?sha1=` is the " - "address that does not move: it opens the card wherever it is now. " - "Each card's **permalink** is that address.", + "Paste a hash from clrmamepro, RomVault, an emulator log or a platform's " + "BIOS list. The page lists the file in the collection that has it, the " + "platforms that declare it and the emulators that check for it.", "", '
', - ' ', - ' ', - '', + ' ', + ' ', + '', "
", "", '

', "", - "The index behind it is [`api/v1/file-locations.json`](../api/v1/file-locations.json).", + "To look a file up by its name, use the search at the top of the page.", + "", + "## Permanent links", + "", + "A file's card lives on its manufacturer's page, and a file refiled " + "between manufacturers moves with it. `files/?sha1=` is the address " + "that does not move: it opens the card wherever it is now. Each card's " + "**permalink** is that address.", + "", + "## The index behind it", + "", + "The lookup reads one file of [`api/v1/lookup/`](../data.md), named by the " + "first two characters of the hash, about 20 KB compressed. " + "[`api/v1/file-locations.json`](../api/v1/file-locations.json) lists every " + "card by SHA1 in one file.", "", ]) @@ -2688,6 +2719,99 @@ def file_location_items( return sorted(items, key=lambda item: item["sha1"]) +LOOKUP_PREFIX = 2 # hex characters naming a lookup shard: 256 files + + +def file_lookup_records( + db: dict, + coverages: dict, + profiles: dict, + locations: list[dict], + members: dict | None = None, +) -> list[dict]: + """Every hash the site knows, one record per value, for files/?hash=. + + The collection's own files lead to their card, a platform's declared + value to the system section of its page, a profile's to the entry's + card. The site search indexed these values as text, 4 MB of the 21 MB + it made every visitor download: the lookup reads one shard instead. + """ + files = db.get("files", {}) + member_names = { + str(info.get("sha1", "")): member.rsplit("/", 1)[-1] + for tree in (members or {}).values() for member, info in sorted(tree.items()) + } + records: list[dict] = [] + for item in locations: + sha1 = item["sha1"] + entry = files.get(sha1) + if entry is None: + # A data directory member: the manifests record its sha1 only. + records.append({"value": sha1, "algorithm": "sha1", "layer": "data-directory", + "name": member_names.get(sha1, ""), "sha1": sha1, + "url": item["url"]}) + continue + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(entry.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "collection", + "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"], + }) + for key, coverage in sorted(coverages.items()): + for system_id, system in sorted((coverage["config"].get("systems") or {}).items()): + url = f"platforms/{key}/#{system_id.replace(' ', '-')}" + for entry in system.get("files") or []: + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(entry.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "platform", + "owner": key, "name": str(entry.get("name", "")), "url": url, + }) + for key, profile in sorted(profiles.items()): + if profile.get("type") in ("alias", "test"): + continue + entries = profile.get("files") or [] + for entry, anchor in zip(entries, emulator_file_anchors(entries)): + url = f"emulators/{key}/#{anchor}" + members = [(str(entry.get("name", "")), entry)] + [ + (str(member["name"]), member) for member in entry.get("contents") or [] + if isinstance(member, dict) and member.get("name") + ] + for name, holder in members: + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(holder.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "emulator", + "owner": key, "name": name, "url": url, + }) + unique = {json.dumps(r, sort_keys=True): r for r in records} + return sorted(unique.values(), key=lambda r: ( + r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"])) + + +def write_lookup_shards(api: Path, records: list[dict], generated_at: str) -> int: + """One envelope per leading pair of hex characters, as the lookup asks. + + A value that is not hexadecimal (a typo in a profile) has no shard the + lookup would read, and is left out. Returns the number of shards. + """ + shards: dict[str, list[dict]] = {} + for record in records: + if re.fullmatch(r"[0-9a-f]{8,}", record["value"]): + shards.setdefault(record["value"][:LOOKUP_PREFIX], []).append(record) + folder = api / "lookup" + folder.mkdir(parents=True, exist_ok=True) + for prefix, items in sorted(shards.items()): + write_if_changed(str(folder / f"{prefix}.json"), json.dumps({ + "schema_version": 1, + "generated_at": generated_at, + "kind": "file-lookup", + "count": len(items), + "items": items, + }, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n") + return len(shards) + + def collection_paths(db: dict) -> dict[str, set[str]]: """Every path under bios/ that holds each file, by sha1. @@ -3101,9 +3225,14 @@ def _installed_links(sha1: str, consumers: FileConsumers, declared: set[str]) -> def _system_file_card( - f: dict, consumers: FileConsumers, anchored: set[str] | None = None + f: dict, consumers: FileConsumers, anchored: set[str] | None = None, + searchable: bool = True, ) -> list[str]: - """One file of a system page: hashes, who reads it, its provenance.""" + """One file of a system page: hashes, who reads it, its provenance. + + ``searchable`` false keeps even the name out of the site search: the + asset trees of game engines hold thousands of 0.ogg and tile.png. + """ name = f["name"] sha1_full = f.get("sha1", "unknown") md5_full = f.get("md5", "unknown") @@ -3129,7 +3258,7 @@ def _system_file_card( ) if anchor else "" lines.append(_card_title( f'{_code(name)} {size}{permalink}', - anchor, + anchor, searchable, )) lines.append("") # Every hash in full: the site search indexes text, not a title @@ -3160,17 +3289,68 @@ def _system_file_card( # looking up scph5501.bin found every platform but RetroBat, whose # pack installs it for six cores. lines.append(f"- Also installed by: {', '.join(installed)}") - lines[2:] = _search_skipped_apart(lines[2:]) + # The title stays searchable: the file name is what a reader types. + lines[4:] = search_excluded(lines[4:]) lines.extend(["", "", ""]) return lines +SEARCH_TABLE_BYTES = 3000 + + +def tables_out_of_search(markdown: str, threshold: int = SEARCH_TABLE_BYTES) -> str: + """Each table longer than ``threshold`` bytes, wrapped out of the search. + + A search result shows the start of a section: a 600-row table as one + snippet helps nobody, and the gaps, provenance and cross-reference + tables were 900 KB of the index. The headings above them stay. + """ + lines = markdown.split("\n") + out: list[str] = [] + index = 0 + while index < len(lines): + match = re.match(r"( *)\|", lines[index]) + if not match: + out.append(lines[index]) + index += 1 + continue + indent = match.group(1) + end = index + while end < len(lines) and re.match(rf"{indent}\|", lines[end]): + end += 1 + table = lines[index:end] + if sum(len(line) for line in table) > threshold: + out += ["", f"{indent}
", "", *table, + "", f"{indent}
", ""] + else: + out += table + index = end + return "\n".join(out) + + +def search_excluded(lines: list[str]) -> list[str]: + """Lines the site search leaves out, as a block of their own. + + The browser downloads and indexes the whole search index before the + first result: hashes, link lists and install commands repeated on 600 + pages made it 21 MB. Hashes are found by files/?hash=, names and prose + stay searchable. + + A section, not a div: the search plugin compares elements by tag, so + the first div closed inside an excluded div (a code block's) ended the + exclusion and the rest of the block was indexed. + """ + return ["", "
", "", *lines, "", "
", ""] + + # Card lines the site search leaves out. They list where a file goes and # who cites it, links to other pages that index the same names, and they # were 6.6 of the 24.7 MB the browser downloads and indexes before the first # result. The name, the hashes, the entry's path and the notes stay # searchable. SEARCH_SKIPPED = ( + "- SHA1:", "- MD5:", "- CRC32:", "- SHA256:", "- Adler32", "- Size:", + "- Validation", "- Priority:", "- Contents (", "- Platforms:", "- Also installed by:", "- Source:", "- Asked for by:", "- Collection path:", "- Collection copy:", "- Collection copies:", "- Installed by the platform's own package on:", @@ -3195,7 +3375,7 @@ def _search_skipped_apart(lines: list[str]) -> list[str]: return out -def _card_title(text: str, anchor: str) -> str: +def _card_title(text: str, anchor: str, searchable: bool = True) -> str: """A file card's heading, so the site search lands on the card. The search splits a page at its headings: a card titled in bold sent a @@ -3204,6 +3384,8 @@ def _card_title(text: str, anchor: str) -> str: keeps the id the toc gives it. """ attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title" + if not searchable: + attributes += " data-search-exclude" return f"#### {text} {{ {attributes} }}" @@ -3256,6 +3438,7 @@ def generate_system_page( "", ] anchored: set[str] = set() + searchable = manufacturer not in GAME_DATA_TOPS for console_name in sorted(consoles.keys()): files = consoles[console_name] @@ -3267,7 +3450,7 @@ def generate_system_page( variant_files = [f for f in files if "/.variants/" in f["path"]] for f in sorted(main_files, key=lambda x: x["name"]): - lines.extend(_system_file_card(f, consumers, anchored)) + lines.extend(_system_file_card(f, consumers, anchored, searchable)) if variant_files: # A variant gets the same card as a primary: the revision a @@ -3278,7 +3461,7 @@ def generate_system_page( for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])): lines.extend( f" {line}" if line else "" - for line in _system_file_card(v, consumers, anchored) + for line in _system_file_card(v, consumers, anchored, searchable) ) lines.append("") @@ -3907,14 +4090,14 @@ def _install_block(name: str, profile: dict) -> list[str]: """ if profile.get("type") in ("alias", "launcher", "test"): return [] - return [ + return search_excluded([ '??? tip "Install this emulator\'s files"', "", " The installer finds your platform and its BIOS folder, then downloads " "only the files this emulator reads there and checks each one.", "", *install_commands(f"--emulator {name}", " "), - ] + ]) def _platform_install_block(name: str, display: str, on_card: bool = False) -> list[str]: @@ -3975,7 +4158,7 @@ def _console_install_block(console: str, declared: list[tuple[str, str]]) -> lis f" {install_command('sh', f'--platform {platform} --system {shlex.quote(system)}')}" for platform, system in declared ] - return lines + [" ```", ""] + return search_excluded(lines + [" ```", ""]) def _render_platform_details(profile: dict) -> list[str]: @@ -4351,7 +4534,9 @@ def generate_emulator_page( ) if data_dirs: - lines.extend(["## Data directories", ""]) + # The member tables are a file list with hashes: out of the search, + # dolphin, primehack and ishiiruka each put 210 KB of it there. + lines.extend(["## Data directories { data-search-exclude }", ""]) member_anchors: set[str] = set() # Each tree with what it holds, where it comes from, which build reads # it and the code that does: these trees put thousands of files in @@ -5536,7 +5721,7 @@ def _mkdocs_config(docs: Path, text: str) -> tuple[Path, str]: _OS_NAMES = {"windows": "Windows", "darwin": "macOS", "linux": "Linux"} -def _runs_on(entry: dict, guide) -> str: +def _runs_on(entry: dict, guide: Guide) -> str: """Where a setup runs, read from the systems the installer detects it on.""" if guide.on_card: return "SD card, filled from a computer" @@ -5553,7 +5738,7 @@ def _runs_on(entry: dict, guide) -> str: return ", ".join(names) -def _folder_cell(name: str, guide, prefix: str = "") -> str: +def _folder_cell(name: str, guide: Guide, prefix: str = "") -> str: """The BIOS folder in a table cell, or a link where it depends on the system.""" if len(guide.folders) > 3: return f"Depends on the system: [listed on its page]({prefix}platforms/{name}.md#where-the-files-go)" @@ -5840,7 +6025,7 @@ def generate_mkdocs_nav( {"Dump provenance": "provenance.md"}, {"Updated platform files": "exports.md"}, {"Data and API": "data.md"}, - {"Find a file by its SHA1": "files/index.md"}, + {"Find a file": "files/index.md"}, {"Data model": "wiki/data-model.md"}, {"Architecture": "wiki/architecture.md"}, {"Build your own packs": "wiki/advanced-usage.md"}, @@ -6089,15 +6274,15 @@ def _generate(args: argparse.Namespace) -> None: print("Generating cross-reference page...") write_if_changed( str(docs / "cross-reference.md"), - generate_cross_reference(coverages, profiles) + REFERENCE_NEXT, + tables_out_of_search(generate_cross_reference(coverages, profiles)) + REFERENCE_NEXT, ) print("Generating gap analysis page...") write_if_changed( str(docs / "gaps.md"), - generate_gap_analysis( + tables_out_of_search(generate_gap_analysis( profiles, coverages, db, suppl_names, registry, gap_report - ) + REFERENCE_NEXT, + )) + REFERENCE_NEXT, ) print("Generating updated platform files page...") @@ -6113,7 +6298,9 @@ def _generate(args: argparse.Namespace) -> None: provenance_report = build_report(db, load_provenance_snapshots()) write_if_changed( str(docs / "provenance.md"), - generate_provenance_page(db, provenance_report, load_derivable()) + REFERENCE_NEXT, + tables_out_of_search( + generate_provenance_page(db, provenance_report, load_derivable()) + ) + REFERENCE_NEXT, ) # Wiki pages: copy manually maintained sources + generate dynamic ones diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index c968d30b..80ed62e6 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -446,7 +446,7 @@ def main() -> int: if (api_dir / name).exists() ] + sorted((api_dir / "platforms").glob("*.json")) + sorted( (api_dir / "systems").glob("*.json") - ) + ) + sorted((api_dir / "lookup").glob("*.json")) errors.extend( _validate_json_files(envelope_paths, "site-api-envelope.schema.json") ) diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 07a31239..be861d5c 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -651,6 +651,62 @@ class APerCoreDocumentValidates(unittest.TestCase): jsonschema.validate(document["items"][0]["profile"], schema) +class AHashFindsEveryPageThatNamesIt(unittest.TestCase): + """Hashes left the site search, 4 MB of the index every visitor + downloaded. files/?hash= reads one shard instead, and it must answer at + least what the search answered: the card, the platforms, the profiles.""" + + def test_records_cover_every_layer_and_shard_by_prefix(self): + import json as _json + import tempfile + + import generate_site # noqa: PLC0415 + + sha1 = "ab" * 20 + md5 = "490f666e1afb15b7362b406ed1cea246" + db = {"files": {sha1: {"name": "scph5501.bin", "md5": md5, "crc32": "8D8CB7E4", + "path": "bios/Sony/PlayStation/scph5501.bin"}}} + coverages = {"batocera": {"config": {"systems": {"sony-playstation": {"files": [ + {"name": "scph5501.bin", "md5": f"{md5}, 0123456789abcdef0123456789abcdef"}, + {"name": "typo.bin", "md5": "not-a-hash"}, + ]}}}}} + profiles = { + "duckstation": {"files": [{"name": "scph5501.bin", "md5": md5}, + {"name": "set.zip", "contents": [ + {"name": "m.rom", "crc32": "11223344"}]}]}, + "an-alias": {"type": "alias", "files": [{"name": "x", "md5": md5}]}, + } + locations = generate_site.file_location_items(db) + [ + {"sha1": "cd" * 20, "url": "emulators/dolphin/#sha1-" + "cd" * 6}] + members = {"dolphin-sys": {"Sys/GC/font.bin": {"sha1": "cd" * 20}}} + records = generate_site.file_lookup_records(db, coverages, profiles, locations, members) + found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records} + self.assertEqual(found[("collection", "", "md5", md5)]["url"], + "systems/sony/#sha1-" + "ab" * 6) + self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found) + self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin") + self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"], + "platforms/batocera/#sony-playstation") + self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found) + self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"], + "emulators/duckstation/#file-scph5501-bin-490f666e") + self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom") + self.assertNotIn(("emulator", "an-alias", "md5", md5), found) + + with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp: + shards = generate_site.write_lookup_shards(Path(tmp), records, "2026-01-01T00:00:00Z") + folder = Path(tmp) / "lookup" + names = sorted(path.name for path in folder.glob("*.json")) + self.assertEqual(len(names), shards) + for path in folder.glob("*.json"): + document = _json.loads(path.read_text()) + self.assertEqual(document["kind"], "file-lookup") + self.assertEqual(document["count"], len(document["items"])) + for item in document["items"]: + self.assertTrue(item["value"].startswith(path.stem)) + self.assertNotIn("no.json", names) + + class AFileHasAnAddressByContent(unittest.TestCase): """mt32_control.rom moved from the Commodore page to the Roland page and every link to its card broke: the anchor was content-based, the page @@ -669,7 +725,8 @@ class AFileHasAnAddressByContent(unittest.TestCase): ) self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page()) script = (Path(__file__).resolve().parents[1] / "docs_assets" / "site.js").read_text() - self.assertIn("file-locations.json", script) + self.assertIn("api/v1/lookup/", script) + self.assertIn('params.has("sha1")', script) from cross_reference import FileConsumers # noqa: PLC0415 card = "\n".join(generate_site._system_file_card( diff --git a/tests/test_site_render.py b/tests/test_site_render.py index 78d9aeb5..8188ece8 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -339,9 +339,10 @@ class TheSearchLandsOnTheCard(unittest.TestCase): class TheSearchSkipsLinkLists(unittest.TestCase): """The search index was 24.7 MB, read and indexed before the first result. Lists of platforms, copies and source citations on every card - made 6.6 MB of it, names already indexed on their own pages.""" + made 6.6 MB of it, names already indexed on their own pages, and the + hashes 4 MB more: files/?hash= finds those.""" - def test_a_system_card_keeps_its_hashes_and_skips_its_platforms(self): + def test_a_system_card_keeps_its_name_and_skips_its_body(self): import generate_site from cross_reference import FileConsumers @@ -352,26 +353,48 @@ class TheSearchSkipsLinkLists(unittest.TestCase): consumers.emulators = {sha1: {"duckstation"}} entry = {"name": "a.bin", "sha1": sha1, "md5": "b" * 32, "size": 3} text = "\n".join(generate_site._system_file_card(entry, consumers)) - searched, skipped = text.split('
') - self.assertIn("- MD5:", searched) - self.assertIn("- Emulators:", searched) - self.assertIn("- Platforms:", skipped) - self.assertIn("- Also installed by:", skipped) - self.assertEqual(text.count("
"), 2) + searched, skipped = text.split("
") + self.assertIn("`a.bin`", searched) + self.assertNotIn("- MD5:", searched) + for line in ("- MD5:", "- Emulators:", "- Platforms:", "- Also installed by:"): + self.assertIn(line, skipped) + + def test_a_game_data_card_skips_its_name_too(self): + import generate_site + from cross_reference import FileConsumers + + entry = {"name": "0.ogg", "sha1": "a" * 40, "size": 3} + lines = generate_site._system_file_card( + entry, FileConsumers({"files": {}}), searchable=False) + self.assertTrue(lines[2].endswith("data-search-exclude }")) def test_runs_alternate_in_card_order(self): import generate_site - lines = ["- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", - "- Destination: `d`", "- Contents (1 members):", " - `m`"] + lines = ["- Path: `p`", "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", + "- Destination: `d`", "- Contents (1 members):", " - `m`", + "- Note"] out = [line for line in generate_site._search_skipped_apart(lines) if line] self.assertEqual(out, [ - "- SHA1: x", + "- Path: `p`", '
', - "- Source: `a.c:1`", "- Platforms: p", "
", - "- Destination: `d`", "- Contents (1 members):", " - `m`", + "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", "", + "- Destination: `d`", + '
', + "- Contents (1 members):", " - `m`", "
", + "- Note", ]) + def test_long_tables_leave_the_search(self): + import generate_site + + short = "| a |\n|---|\n| 1 |" + long = " | a |\n |---|\n" + "\n".join(f" | {'x' * 50} |" for _ in range(80)) + text = generate_site.tables_out_of_search(f"# T\n\n{short}\n\n{long}\n") + self.assertEqual(text.count("data-search-exclude"), 1) + self.assertIn("
\n\n | a |", text) + self.assertIn(short, text) + if __name__ == "__main__": unittest.main() diff --git a/wiki/index.md b/wiki/index.md index 755d1ee3..dfe00c8a 100644 --- a/wiki/index.md +++ b/wiki/index.md @@ -20,7 +20,7 @@ toolchain works. To set up BIOS files on a device, start with ## Data - **[Data and API](../data.md)** - JSON, CSV and SQLite exports with their checksums -- **[Find a file by its SHA1](../files/index.md)** - the permanent address of a file by its contents +- **[Find a file](../files/index.md)** - any SHA1, MD5, CRC32 or SHA256: the file, the platforms that declare it, the emulators that check it - **[Data model](data-model.md)** - database.json structure, indexes, file resolution order, YAML formats ## The toolchain