From baf38ffbeeb8d649d61a1395104d6762869e9cf9 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:23:26 +0200 Subject: [PATCH 1/5] refactor: compute coverage once, index proofs EntryCategories asked every core in turn: 3.8 million proofs for 7,430 entries. ProofIndex looks each key up once, and verify --all --json is byte-identical (24 s to 17 s). generate_site and generate_readme now share one coverage, cached in tmp/coverage-cache keyed by its inputs, so the README step drops from 25 s to 3 s. --- scripts/exporter/candidates.py | 93 ++++++++++++++++++++++++--- scripts/generate_readme.py | 113 ++++++++++++++++++++++++++++++--- scripts/generate_site.py | 21 +++--- tests/test_coverage_cache.py | 100 +++++++++++++++++++++++++++++ tests/test_entry_categories.py | 53 +++++++++++++++- wiki/testing-guide.md | 2 +- 6 files changed, 349 insertions(+), 33 deletions(-) create mode 100644 tests/test_coverage_cache.py diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py index 8b8c2b66..e4fe619f 100644 --- a/scripts/exporter/candidates.py +++ b/scripts/exporter/candidates.py @@ -136,6 +136,81 @@ class ReadEvidence: return None +# The order ReadEvidence.proof tries: a reader's proof is the first that holds. +_PROOF_ORDER = ("path", "launcher", "name", "hash", "directory", "data_directory") + + +class ProofIndex: + """ReadEvidence.proof for many readers at once, by lookup. + + Asking every core in turn cost one proof per core and entry: 3.8 million + calls to categorize 7,430 entries. Each key a reader can match is indexed + once, and an entry's own keys find the readers it concerns. The proof of + each reader is the one ReadEvidence.proof returns. + """ + + def __init__(self, readers: list[ReadEvidence]) -> None: + self._index: dict[str, dict[str, set[int]]] = { + kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member") + } + for position, evidence in enumerate(readers): + for kind, keys in ( + ("tail", evidence.tails), ("launched", evidence.launched), + ("name", evidence.names), ("hash", evidence.hashes), + ("dir", evidence.directories), ("tree", evidence.trees), + ("member", evidence.tree_members), + ): + for key in keys: + self._index[kind].setdefault(key, set()).add(position) + # A cut md5 names any hash it is the start of. + for value in evidence.hashes: + for length in range(_CUT_MD5, _FULL_MD5): + if len(value) >= length: + self._index["cut"].setdefault(value[:length], set()).add(position) + + def _hits(self, kind: str, keys) -> set[int]: + table = self._index[kind] + found: set[int] = set() + for key in keys: + found |= table.get(key, set()) + return found + + def proofs(self, entry: dict, destination: str) -> dict[int, str]: + """Reader position -> its proof, for the readers that have one.""" + names = { + str(n).casefold() + for n in (entry.get("name"), *(entry.get("aliases") or [])) + if n + } + tails = _tails(destination) + last = {tails[-1]} if tails else set() + hashes = { + v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f) + } + cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5} + # "/" + d in "/" + path: d is a run of whole segments ending in "/", + # so it is a run of the path's segments before the last one. + parts = tails[0].split("/") if tails else [] + runs = { + "/".join(parts[i:j]) + "/" + for i in range(len(parts)) for j in range(i + 1, len(parts)) + } + found = { + "path": self._hits("tail", tails[:-1]), + "launcher": self._hits("launched", names | last), + "name": self._hits("name", names | last), + "hash": self._hits("hash", hashes) | self._hits("cut", cut), + "directory": self._hits("dir", runs), + "data_directory": self._hits("tree", runs) + | self._index["member"].get(tails[0] if tails else "", set()), + } + result: dict[int, str] = {} + for kind in _PROOF_ORDER: + for position in found[kind]: + result.setdefault(position, kind) + return result + + def _loads_romsets(profile: dict) -> bool: """A core that opens ROM sets by name: MAME pulls a device's set from device_t::searchpath(), and no profile lists every device set.""" @@ -275,6 +350,8 @@ class EntryCategories: self._readers.append((kind, key, evidence)) self._launchers = ReadEvidence() self._launchers.add_launchers(config.get("launchers") or []) + self._proofs = ProofIndex([evidence for _kind, _key, evidence in self._readers] + + [self._launchers]) self._by_system: dict[str, list[str]] = {} for key in sorted(examined): for sid in profiles[key].get("systems") or []: @@ -313,14 +390,14 @@ class EntryCategories: def explain(self, entry: dict, destination: str, system: str) -> dict: """The category of an entry and what establishes it.""" readers: dict[str, list[dict]] = {BIOS: [], GAME_DATA: []} - for kind, key, evidence in self._readers: - proof = evidence.proof(entry, destination) - if proof is not None: - readers[kind].append({ - "emulator": key, "proof": proof, - "entries": self._matching(key, kind, entry), - }) - if self._launchers.proof(entry, destination) is not None: + proofs = self._proofs.proofs(entry, destination) + for position in sorted(p for p in proofs if p < len(self._readers)): + kind, key, _evidence = self._readers[position] + readers[kind].append({ + "emulator": key, "proof": proofs[position], + "entries": self._matching(key, kind, entry), + }) + if len(self._readers) in proofs: readers[BIOS].append({"emulator": "launcher", "proof": "launcher", "entries": []}) if system in self._system_runs: system_cores, unresolved = self._system_runs[system] diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 1a8ab504..519d4202 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -111,6 +111,105 @@ def compute_coverage( } +_REPO = Path(__file__).resolve().parent.parent +# The site and the README read the same coverage: the first generator to run +# stores it here and the second reads it back, instead of verifying every +# platform twice (about a minute of the site build). +COVERAGE_CACHE_DIR = _REPO / "tmp" / "coverage-cache" +_COVERAGE_CACHE_FORMAT = "1" + + +def _coverage_inputs(db_path: Path, platforms_dir: str, emulators_dir: str) -> str: + """A digest of everything coverage is computed from. + + The database, the platform and emulator files, the code, the install + manifests, the data directory revisions and the state of bios/ on disk: + a change to any of them computes the coverage again. + """ + import hashlib + + digest = hashlib.sha256(_COVERAGE_CACHE_FORMAT.encode()) + + def add_file(path: Path) -> None: + digest.update(str(path).encode() + b"\0") + digest.update(path.read_bytes() if path.is_file() else b"") + + add_file(db_path.resolve()) + for root, pattern in ((Path(platforms_dir), "*"), (Path(emulators_dir), "*.yml"), + (_REPO / "scripts", "*.py"), (_REPO / "install", "*.json"), + (_REPO / "recipes", "*.json")): + for path in sorted(p for p in root.resolve().rglob(pattern) if p.is_file()): + add_file(path) + for path in (_REPO / ".gitignore", _REPO / "_mame_clones.json", + _REPO / "data" / ".versions.json"): + add_file(path) + bios = _REPO / "bios" + for path in sorted(bios.rglob("*")): + if path.is_file(): + stat = path.stat() + digest.update(f"{path.relative_to(bios)}\0{stat.st_size}\0{stat.st_mtime_ns}\n".encode()) + return digest.hexdigest() + + +def platform_coverages( + platform_names: list[str], + platforms_dir: str, + db: dict, + data_registry: dict | None = None, + emulators_dir: str = _EMULATORS_DIR, + db_path: str = "database.json", + report: Callable[[str, dict], None] | None = None, +) -> dict[str, dict]: + """compute_coverage for each platform, read from the cache when its + inputs have not changed since a generator last stored it. + + A platform with no YAML is left out, as compute_coverage raises for it. + ``report`` is called with each platform and its coverage. + """ + # pickle, not JSON: the coverage holds sets and enums that JSON would + # turn into lists and strings. The file is written by this function into + # the repository's own tmp/, never downloaded. + import pickle + + key = _coverage_inputs(Path(db_path), platforms_dir, emulators_dir) + names = sorted(platform_names) + cache = COVERAGE_CACHE_DIR / f"{key}.pickle" + coverages: dict[str, dict] | None = None + if cache.is_file(): + try: + with cache.open("rb") as handle: + stored = pickle.load(handle) + except (OSError, EOFError, pickle.UnpicklingError) as exc: + print(f" coverage cache unreadable, computing again: {exc}", file=sys.stderr) + else: + if stored.get("platforms") == names: + coverages = stored["coverages"] + print(f" coverage read from {cache.relative_to(_REPO)}") + if coverages is None: + coverages = {} + for name in names: + try: + coverages[name] = compute_coverage( + name, platforms_dir, db, data_registry, emulators_dir=emulators_dir, + ) + except FileNotFoundError as exc: + print(f" {name}: skipped ({exc})", file=sys.stderr) + COVERAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True) + for stale in COVERAGE_CACHE_DIR.glob("*.pickle"): + stale.unlink() + scratch = cache.with_suffix(f".{os.getpid()}.tmp") + with scratch.open("wb") as handle: + pickle.dump({"platforms": names, "coverages": coverages}, handle, + protocol=pickle.HIGHEST_PROTOCOL) + scratch.replace(cache) + ordered = {name: coverages[name] for name in platform_names if name in coverages} + if report: + for name in names: + if name in ordered: + report(name, ordered[name]) + return ordered + + def release_totals(platform_name: str, record: dict) -> tuple[int | None, int | None]: """Files and bytes of the pack the latest release serves for a platform. @@ -347,7 +446,7 @@ def _catalog_matched_line(db: dict) -> list[str]: ] -def generate_readme(db: dict, platforms_dir: str) -> str: +def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json") -> str: total_files = db.get("total_files", 0) total_size = db.get("total_size", 0) ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -358,14 +457,8 @@ def generate_readme(db: dict, platforms_dir: str) -> str: data_registry = load_data_dir_registry(platforms_dir) - coverages = {} - for name in platform_names: - try: - coverages[name] = compute_coverage( - name, platforms_dir, db, data_registry - ) - except FileNotFoundError: - pass + coverages = platform_coverages(platform_names, platforms_dir, db, data_registry, + db_path=db_path) profiles = unique_emulator_profiles( load_emulator_profiles("emulators", skip_aliases=False) @@ -853,7 +946,7 @@ def main() -> None: # published counts taken from half of it. with contextlib.ExitStack() as held: hold_data_dirs(held, load_data_dir_registry(args.platforms_dir)) - readme = generate_readme(db, args.platforms_dir) + readme = generate_readme(db, args.platforms_dir, args.db) status = "Generated" if write_if_changed("README.md", readme) else "Unchanged" print(f"{status} ./README.md") diff --git a/scripts/generate_site.py b/scripts/generate_site.py index ac529ac8..8350170e 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -65,6 +65,7 @@ from nativemode import digest_algorithm, reads_file_contents yaml = require_yaml() from generate_readme import ( compute_coverage, + platform_coverages, contributing_text, download_links, extract_notes, @@ -5955,19 +5956,13 @@ def _generate(args: argparse.Namespace) -> None: suppl_names = _build_supplemental_index() print("Computing platform coverage...") - coverages = {} - for name in sorted(platform_names): - try: - cov = compute_coverage( - name, args.platforms_dir, db, data_registry, - emulators_dir=args.emulators_dir, - ) - coverages[name] = cov - print( - f" {cov['platform']}: {cov['present']}/{cov['total']} ({_pct(cov['present'], cov['total'])})" - ) - except FileNotFoundError as e: - print(f" {name}: skipped ({e})", file=sys.stderr) + coverages = platform_coverages( + sorted(platform_names), args.platforms_dir, db, data_registry, + emulators_dir=args.emulators_dir, db_path=args.db, + report=lambda _name, cov: print( + f" {cov['platform']}: {cov['present']}/{cov['total']} ({_pct(cov['present'], cov['total'])})" + ), + ) print("Loading emulator profiles...") profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False) diff --git a/tests/test_coverage_cache.py b/tests/test_coverage_cache.py new file mode 100644 index 00000000..b9b38398 --- /dev/null +++ b/tests/test_coverage_cache.py @@ -0,0 +1,100 @@ +"""The site and the README share one coverage computation. + +generate_site.py and generate_readme.py each verified every platform, a +minute of the site build spent twice on the same answer. The first to run +stores the coverage, keyed by its inputs, and the second reads it. A cache +that outlived a change to its inputs would publish stale counts, so these +tests change an input and expect a fresh computation. +""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_readme # noqa: E402 + + +class CoverageCache(unittest.TestCase): + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests")) + self.platforms = self.work / "platforms" + self.platforms.mkdir() + (self.platforms / "alpha.yml").write_text("platform: Alpha\n") + self.emulators = self.work / "emulators" + self.emulators.mkdir() + self.db = self.work / "database.json" + self.db.write_text("{}") + self.calls: list[str] = [] + patches = [ + mock.patch.object(generate_readme, "COVERAGE_CACHE_DIR", self.work / "cache"), + mock.patch.object(generate_readme, "compute_coverage", self._compute), + ] + for patch in patches: + patch.start() + self.addCleanup(patch.stop) + + def tearDown(self) -> None: + import shutil + + shutil.rmtree(self.work, ignore_errors=True) + + def _compute(self, name, platforms_dir, db, data_registry=None, emulators_dir=""): + self.calls.append(name) + if name == "ghost": + raise FileNotFoundError(name) + return {"platform": name.title(), "missing_names": {"a.bin"}} + + def _run(self, names=("beta", "alpha")) -> dict: + return generate_readme.platform_coverages( + list(names), str(self.platforms), {}, {}, + emulators_dir=str(self.emulators), db_path=str(self.db), + ) + + def test_second_run_reads_the_first(self): + first = self._run() + self.assertEqual(self.calls, ["alpha", "beta"]) + second = self._run() + self.assertEqual(self.calls, ["alpha", "beta"]) + self.assertEqual(first, second) + # The caller's order, not the computation's. + self.assertEqual(list(second), ["beta", "alpha"]) + self.assertEqual(second["alpha"]["missing_names"], {"a.bin"}) + + def test_a_changed_input_computes_again(self): + self._run() + for change in ( + lambda: (self.platforms / "alpha.yml").write_text("platform: Alpha 2\n"), + lambda: self.db.write_text('{"files": {}}'), + lambda: (self.emulators / "core.yml").write_text("emulator: core\n"), + ): + self.calls.clear() + change() + self._run() + self.assertEqual(self.calls, ["alpha", "beta"]) + + def test_another_platform_list_computes_again(self): + self._run() + self.calls.clear() + self._run(("alpha",)) + self.assertEqual(self.calls, ["alpha"]) + + def test_a_platform_without_a_file_is_left_out(self): + self.assertEqual(list(self._run(("ghost", "alpha"))), ["alpha"]) + self.assertEqual(list(self._run(("ghost", "alpha"))), ["alpha"]) + + def test_one_cache_file_is_kept(self): + self._run() + (self.platforms / "alpha.yml").write_text("platform: Alpha 3\n") + self._run() + self.assertEqual(len(list((self.work / "cache").glob("*.pickle"))), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_entry_categories.py b/tests/test_entry_categories.py index d6cbb324..3d2791a9 100644 --- a/tests/test_entry_categories.py +++ b/tests/test_entry_categories.py @@ -13,7 +13,14 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) import verify # noqa: E402 -from exporter.candidates import BIOS, GAME_DATA, UNREAD, EntryCategories # noqa: E402 +from exporter.candidates import ( # noqa: E402 + BIOS, + GAME_DATA, + UNREAD, + EntryCategories, + ProofIndex, + ReadEvidence, +) PROFILES = { "bluemsx": { @@ -144,5 +151,49 @@ class StrictCountsFirmwareOnly(unittest.TestCase): self.assertEqual(evidence["diabdat.mpq"]["read_by"][0]["emulator"], "devilutionx") +class IndexedProof(unittest.TestCase): + """ProofIndex answers what ReadEvidence.proof answers, reader by reader.""" + + def test_every_kind_matches_the_direct_proof(self): + md5 = "490f666e1afb15b7362b406ed1cea246" + readers = [] + for profile in ( + {"files": [{"name": "a.bin", "path": "sys/dir/a.bin"}]}, + {"files": [{"name": "other.rom", "aliases": ["B.ROM"]}]}, + {"files": [{"name": "c.bin", "md5": md5}]}, + {"files": [{"name": "tree/", "type": "directory", "path": "data/tree/"}]}, + {"files": [], "data_directories": [{"ref": "x", "destination": "assets/sys"}]}, + {"files": [{"name": "d.bin", "sha1": "ab" * 20, "crc32": "12345678"}]}, + ): + evidence = ReadEvidence() + evidence.add_profile(profile, {}) + readers.append(evidence) + launcher = ReadEvidence() + launcher.add_launchers([{"names": ["scph101.bin"]}]) + readers.append(launcher) + index = ProofIndex(readers) + cases = [ + ({"name": "a.bin"}, "bios/sys/dir/a.bin"), + ({"name": "a.bin"}, "elsewhere/a.bin"), + ({"name": "x", "aliases": ["b.rom"]}, "x"), + ({"name": "y", "md5": md5[:29]}, "y"), + ({"name": "y", "md5": md5.upper()}, "y"), + ({"name": "z", "md5": md5[:28]}, "z"), + ({"name": "z"}, "root/data/tree/deep/z"), + ({"name": "z"}, "data/tree"), + ({"name": "z"}, "a/assets/sys/z.png"), + ({"name": "scph101.bin"}, "bios/scph101.bin"), + ({"name": "q", "crc32": "12345678, 9"}, "q"), + ({"name": "q"}, ""), + ] + for entry, destination in cases: + expected = { + position: proof for position, evidence in enumerate(readers) + if (proof := evidence.proof(entry, destination)) is not None + } + with self.subTest(entry=entry, destination=destination): + self.assertEqual(index.proofs(entry, destination), expected) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 5cf5e803..51bda600 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -51,7 +51,7 @@ Every module in `tests/`, by the area it guards. | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture From 42a0498d2060e643d113a25ebdac9aed522b3269 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:46:34 +0200 Subject: [PATCH 2/5] chore: run site tools through run_cli The image and diagram scripts accepted abbreviated options and bypassed run_cli. The FAQ checks read faq.md only, while the questions on hashes and checks moved to technical-faq.md. --- scripts/fetch_site_images.py | 6 ++++-- scripts/make_og_cards.py | 4 +++- scripts/sitediagrams.py | 6 ++++-- tests/test_audit_regressions.py | 6 +++++- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/fetch_site_images.py b/scripts/fetch_site_images.py index ddaeaaba..deee4fb1 100644 --- a/scripts/fetch_site_images.py +++ b/scripts/fetch_site_images.py @@ -172,7 +172,7 @@ def fetch_icons() -> None: def main() -> None: - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0]) parser.add_argument("--only", choices=("logos", "icons"), help="fetch one kind only") args = parser.parse_args() if args.only in (None, "logos"): @@ -184,4 +184,6 @@ def main() -> None: if __name__ == "__main__": - main() + from common import run_cli + + run_cli(main) diff --git a/scripts/make_og_cards.py b/scripts/make_og_cards.py index ec472fb5..e25f5731 100644 --- a/scripts/make_og_cards.py +++ b/scripts/make_og_cards.py @@ -45,4 +45,6 @@ def main() -> None: if __name__ == "__main__": - main() + from common import run_cli + + run_cli(main) diff --git a/scripts/sitediagrams.py b/scripts/sitediagrams.py index d06cacb0..6df4d2f8 100644 --- a/scripts/sitediagrams.py +++ b/scripts/sitediagrams.py @@ -744,7 +744,7 @@ def write_svgs(directory: Path) -> list[Path]: def main() -> None: import argparse - parser = argparse.ArgumentParser(description="Write the site diagrams as SVG files.") + parser = argparse.ArgumentParser(allow_abbrev=False, description="Write the site diagrams as SVG files.") parser.add_argument("--output-dir", default="tmp/diagrams", help="default: %(default)s") args = parser.parse_args() for path in write_svgs(Path(args.output_dir)): @@ -752,4 +752,6 @@ def main() -> None: if __name__ == "__main__": - main() + from common import run_cli + + run_cli(main) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 3d872ef4..58b44745 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -779,7 +779,11 @@ class FaqRegressions(unittest.TestCase): @staticmethod def _faq() -> str: - return (ROOT / "wiki" / "faq.md").read_text(encoding="utf-8") + # The questions about hashes and checks moved to the technical FAQ. + return "\n\n".join( + (ROOT / "wiki" / name).read_text(encoding="utf-8") + for name in ("faq.md", "technical-faq.md") + ) @staticmethod def _profile(key: str) -> dict: From 33041b6bd1d7294d8df75fa13538e4d972c55033 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:46:40 +0200 Subject: [PATCH 3/5] feat: find a file by any hash, out of the search Hashes, link lists, install commands, member tables and long reference tables leave the site search: the index drops from 21.0 MB (4.88 MB gzip) to 6.8 MB (1.33 MB gzip). Names and prose stay searchable. files/?hash= takes a SHA1, MD5, CRC32 or SHA256 and reads one shard of api/v1/lookup/ (about 22 KB gzip, was 675 KB) listing the collection card, the platforms that declare the value and the profiles that check it. files/?sha1= still opens the card, and a sha1- anchor missing from a page is looked up there, so a card can move between pages. Excluded blocks are section elements: the search plugin compares elements by tag, so a div ended at the first nested div. --- docs_assets/site.js | 115 ++++++++++-- mkdocs.yml | 2 +- schemas/site-api-envelope.schema.json | 63 ++++++- scripts/exporter/candidates.py | 3 +- scripts/generate_site.py | 253 ++++++++++++++++++++++---- scripts/validate_schemas.py | 2 +- tests/test_site_exports.py | 59 +++++- tests/test_site_render.py | 49 +++-- wiki/index.md | 2 +- 9 files changed, 481 insertions(+), 67 deletions(-) diff --git a/docs_assets/site.js b/docs_assets/site.js index db4d72e1..aacaaa24 100644 --- a/docs_assets/site.js +++ b/docs_assets/site.js @@ -148,28 +148,88 @@ }); }; - // files/?sha1= is a file's content address: the card it names may - // sit on another manufacturer's page today than when the link was cited. + // files/?hash= reads one shard of the lookup, named by the first two hex + // characters of the value: 20 KB where the whole index was 675 KB. + const HASH_LENGTHS = { 8: "CRC32", 32: "MD5", 40: "SHA1", 64: "SHA256" }; + const LAYER_LABELS = { + collection: "In the collection", + "data-directory": "In a data directory the packs ship", + platform: "Declared by a platform", + emulator: "Read by an emulator", + }; + const siteRoot = () => { + const script = document.querySelector('script[src$="javascripts/site.js"]'); + return script ? new URL("../", script.src) : new URL("../", window.location.href); + }; + const lookupShard = (value) => + fetch(new URL(`api/v1/lookup/${value.slice(0, 2)}.json`, siteRoot())) + .then((response) => (response.ok ? response.json() : { items: [] })) + .then((shard) => shard.items || []); + const normalizeHash = (raw) => raw.trim().toLowerCase().replace(/^0x/, "").replace(/\s+/g, ""); + + const renderMatches = (status, value, items) => { + status.textContent = ""; + const kind = HASH_LENGTHS[value.length]; + const heading = document.createElement("p"); + heading.textContent = `${items.length} ${items.length === 1 ? "match" : "matches"} for ${kind} ${value}:`; + status.append(heading); + Object.entries(LAYER_LABELS).forEach(([layer, label]) => { + const found = items.filter((item) => item.layer === layer); + if (!found.length) return; + const title = document.createElement("p"); + const strong = document.createElement("strong"); + strong.textContent = label; + title.append(strong); + const list = document.createElement("ul"); + found.forEach((item) => { + const row = document.createElement("li"); + const link = document.createElement("a"); + link.href = new URL(item.url, siteRoot()).href; + const code = document.createElement("code"); + code.textContent = item.name || item.value; + link.append(code); + row.append(link); + if (item.owner) row.append(` (${item.owner})`); + list.append(row); + }); + status.append(title, list); + }); + }; + + // files/?sha1= is a file's content address and opens its card + // wherever it is now; files/?hash= also takes an MD5, CRC32 or SHA256 and + // lists every page that names the value. const resolveFileAddress = () => { const status = document.getElementById("rb-file-lookup-status"); if (!status) return; - const sha1 = (new URLSearchParams(window.location.search).get("sha1") || "") - .trim().toLowerCase(); - if (!sha1) return; - if (!/^[0-9a-f]{40}$/.test(sha1)) { - status.textContent = "A SHA1 is 40 hexadecimal characters."; + const params = new URLSearchParams(window.location.search); + const permalink = params.has("sha1"); + const value = normalizeHash(params.get("sha1") || params.get("hash") || ""); + if (!value) return; + const input = document.getElementById("rb-hash"); + if (input) input.value = value; + const prefix = permalink && /^[0-9a-f]{12}$/.test(value); + if (!prefix && !(/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length])) { + status.textContent = + "A hash is 8 (CRC32), 32 (MD5), 40 (SHA1) or 64 (SHA256) hexadecimal characters."; return; } status.textContent = "Looking it up..."; - const index = new URL("../api/v1/file-locations.json", window.location.href); - fetch(index) - .then((response) => response.json()) - .then((document_) => { - const item = (document_.items || []).find((entry) => entry.sha1 === sha1); - if (item) { - window.location.replace(new URL("../" + item.url, window.location.href)); + lookupShard(value) + .then((items) => { + const matches = items.filter((item) => + prefix + ? item.algorithm === "sha1" && item.value.startsWith(value) + : item.value === value + ); + const cards = matches.filter((item) => item.layer === "collection" || item.layer === "data-directory"); + const targets = new Set(cards.map((item) => item.url)); + if (permalink && targets.size === 1) { + window.location.replace(new URL(cards[0].url, siteRoot())); + } else if (matches.length) { + renderMatches(status, prefix ? matches[0].value : value, prefix ? cards : matches); } else { - status.textContent = "The collection holds no file with this SHA1."; + status.textContent = "No file the site knows has this hash."; } }) .catch(() => { @@ -177,10 +237,35 @@ }); }; + // A file card moved to another page keeps its address: an id of the form + // sha1-<12 hex> that this page does not hold is looked up by content. + const followMovedCard = () => { + const match = /^#sha1-([0-9a-f]{12})$/.exec(window.location.hash); + if (!match || document.getElementById(`sha1-${match[1]}`)) return; + window.location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot())); + }; + + // The site search no longer holds hashes: Enter on a hash opens the lookup. + const hashSearch = () => { + const query = document.querySelector('input[data-md-component="search-query"]'); + if (!query || query.dataset.rbHash === "true") return; + query.dataset.rbHash = "true"; + query.addEventListener("keydown", (event) => { + if (event.key !== "Enter") return; + const value = normalizeHash(query.value); + if (/^[0-9a-f]+$/.test(value) && HASH_LENGTHS[value.length]) { + event.preventDefault(); + window.location.assign(new URL(`files/?hash=${value}`, siteRoot())); + } + }); + }; + const enhance = () => { enhanceTables(); nameMaterialWidgets(); resolveFileAddress(); + followMovedCard(); + hashSearch(); }; let resizeTimer = 0; diff --git a/mkdocs.yml b/mkdocs.yml index 74e26649..2bbc5ee9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -835,7 +835,7 @@ nav: - Dump provenance: provenance.md - Updated platform files: exports.md - Data and API: data.md - - Find a file by its SHA1: files/index.md + - Find a file: files/index.md - Data model: wiki/data-model.md - Architecture: wiki/architecture.md - Build your own packs: wiki/advanced-usage.md diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index 18bf24ff..dcd422b2 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -27,7 +27,8 @@ "emulators", "systems", "verification-and-coverage-gaps", - "file-locations" + "file-locations", + "file-lookup" ] }, "count": { @@ -332,6 +333,66 @@ } } } + }, + { + "if": { + "properties": { + "kind": { + "const": "file-lookup" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "value", + "algorithm", + "layer", + "name", + "url" + ], + "properties": { + "value": { + "type": "string", + "pattern": "^[0-9a-f]+$" + }, + "algorithm": { + "enum": [ + "sha1", + "md5", + "crc32", + "sha256" + ] + }, + "layer": { + "enum": [ + "collection", + "data-directory", + "platform", + "emulator" + ] + }, + "name": { + "type": "string" + }, + "url": { + "type": "string" + }, + "sha1": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "owner": { + "type": "string" + } + } + } + } + } + } } ] } diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py index e4fe619f..7af1656c 100644 --- a/scripts/exporter/candidates.py +++ b/scripts/exporter/candidates.py @@ -12,6 +12,7 @@ the platform names that no profile describes leaves the verdict open. from __future__ import annotations import sys +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path @@ -168,7 +169,7 @@ class ProofIndex: if len(value) >= length: self._index["cut"].setdefault(value[:length], set()).add(position) - def _hits(self, kind: str, keys) -> set[int]: + def _hits(self, kind: str, keys: Iterable[str]) -> set[int]: table = self._index[kind] found: set[int] = set() for key in keys: diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 8350170e..b94d3779 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -79,6 +79,7 @@ from siteguides import ( DEVICE_GROUPS, GROUP_ANCHORS, GUIDES, + Guide, INSTALL_BASE, install_command, platform_args, @@ -1130,13 +1131,17 @@ def generate_data_exports( gap_rows = _gap_export_rows(coverages, gap_report) cross_rows = _cross_reference_export_rows(coverages, profiles) + members = data_directory_members() + locations = file_location_items(db, profiles, members) envelopes = { "platforms.json": ("platforms", platform_items), "emulators.json": ("emulators", emulator_items), "gaps.json": ("verification-and-coverage-gaps", gap_rows), - "file-locations.json": ("file-locations", file_location_items( - db, profiles, data_directory_members())), + "file-locations.json": ("file-locations", locations), } + write_lookup_shards( + api, file_lookup_records(db, coverages, profiles, locations, members), generated_at, + ) for filename, (kind, items) in envelopes.items(): document = { "schema_version": 1, @@ -1338,6 +1343,13 @@ def generate_data_page(exports: list[dict]) -> str: "platform of `platforms.json`, and `api/v1/systems/.json` the files " "of one system page by console. Each platform and system page links its own.", "", + "One hash without the whole list: `api/v1/lookup/.json`, where `` " + "is the first two characters of the hash in lowercase, holds every SHA1, " + "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` " + "(the collection, a data directory, a platform's list or an emulator " + "profile), the file `name` and the `url` of the page that shows it. " + "[Find a file](files/index.md) reads it.", + "", "## Bulk downloads", "", "| Export | Format | Size | SHA256 |", @@ -2059,6 +2071,7 @@ def _render_platform_systems( # emulator cards link here, and the site search splits a page at its # headings. Folded into blocks under one heading, every hash pasted # in the search led to the systems overview of a 100-block page. + block_start = len(lines) + 2 if use_collapsible: status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue") lines.append(f"### {sys_link} {{ #{anchor} }}") @@ -2141,7 +2154,9 @@ def _render_platform_systems( if f.get("category_evidence"): lines.append(f"{pad} {_category_evidence_md(f['category_evidence'])}") - lines.append("") + # The section's heading stays searchable, its file list with every + # hash is the lookup's: retrodeck's arcade list was 495 KB of index. + lines[block_start:] = search_excluded(lines[block_start:]) return lines @@ -2572,28 +2587,44 @@ def generate_platform_page( def generate_files_page() -> str: - """The content address of every held file: files/?sha1=. + """Find a file by any of its hashes: files/?hash=. - site.js reads api/v1/file-locations.json and goes to the card. Without - a sha1, or with one the collection does not hold, the page says so. + site.js reads the one shard of api/v1/lookup/ the value names and lists + every page that cites it. files/?sha1= is the card's permanent + address and opens the card directly. """ return "\n".join([ - "# Find a file by its SHA1", + "# Find a file", "", - "A file's card lives on its manufacturer's page, and a file refiled " - "between manufacturers moves with it. `files/?sha1=` is the " - "address that does not move: it opens the card wherever it is now. " - "Each card's **permalink** is that address.", + "Paste a hash from clrmamepro, RomVault, an emulator log or a platform's " + "BIOS list. The page lists the file in the collection that has it, the " + "platforms that declare it and the emulators that check for it.", "", '
', - ' ', - ' ', - '', + ' ', + ' ', + '', "
", "", '

', "", - "The index behind it is [`api/v1/file-locations.json`](../api/v1/file-locations.json).", + "To look a file up by its name, use the search at the top of the page.", + "", + "## Permanent links", + "", + "A file's card lives on its manufacturer's page, and a file refiled " + "between manufacturers moves with it. `files/?sha1=` is the address " + "that does not move: it opens the card wherever it is now. Each card's " + "**permalink** is that address.", + "", + "## The index behind it", + "", + "The lookup reads one file of [`api/v1/lookup/`](../data.md), named by the " + "first two characters of the hash, about 20 KB compressed. " + "[`api/v1/file-locations.json`](../api/v1/file-locations.json) lists every " + "card by SHA1 in one file.", "", ]) @@ -2688,6 +2719,99 @@ def file_location_items( return sorted(items, key=lambda item: item["sha1"]) +LOOKUP_PREFIX = 2 # hex characters naming a lookup shard: 256 files + + +def file_lookup_records( + db: dict, + coverages: dict, + profiles: dict, + locations: list[dict], + members: dict | None = None, +) -> list[dict]: + """Every hash the site knows, one record per value, for files/?hash=. + + The collection's own files lead to their card, a platform's declared + value to the system section of its page, a profile's to the entry's + card. The site search indexed these values as text, 4 MB of the 21 MB + it made every visitor download: the lookup reads one shard instead. + """ + files = db.get("files", {}) + member_names = { + str(info.get("sha1", "")): member.rsplit("/", 1)[-1] + for tree in (members or {}).values() for member, info in sorted(tree.items()) + } + records: list[dict] = [] + for item in locations: + sha1 = item["sha1"] + entry = files.get(sha1) + if entry is None: + # A data directory member: the manifests record its sha1 only. + records.append({"value": sha1, "algorithm": "sha1", "layer": "data-directory", + "name": member_names.get(sha1, ""), "sha1": sha1, + "url": item["url"]}) + continue + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(entry.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "collection", + "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"], + }) + for key, coverage in sorted(coverages.items()): + for system_id, system in sorted((coverage["config"].get("systems") or {}).items()): + url = f"platforms/{key}/#{system_id.replace(' ', '-')}" + for entry in system.get("files") or []: + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(entry.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "platform", + "owner": key, "name": str(entry.get("name", "")), "url": url, + }) + for key, profile in sorted(profiles.items()): + if profile.get("type") in ("alias", "test"): + continue + entries = profile.get("files") or [] + for entry, anchor in zip(entries, emulator_file_anchors(entries)): + url = f"emulators/{key}/#{anchor}" + members = [(str(entry.get("name", "")), entry)] + [ + (str(member["name"]), member) for member in entry.get("contents") or [] + if isinstance(member, dict) and member.get("name") + ] + for name, holder in members: + for algorithm in HASH_ALGORITHMS: + for value in _hash_values(holder.get(algorithm)): + records.append({ + "value": value, "algorithm": algorithm, "layer": "emulator", + "owner": key, "name": name, "url": url, + }) + unique = {json.dumps(r, sort_keys=True): r for r in records} + return sorted(unique.values(), key=lambda r: ( + r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"])) + + +def write_lookup_shards(api: Path, records: list[dict], generated_at: str) -> int: + """One envelope per leading pair of hex characters, as the lookup asks. + + A value that is not hexadecimal (a typo in a profile) has no shard the + lookup would read, and is left out. Returns the number of shards. + """ + shards: dict[str, list[dict]] = {} + for record in records: + if re.fullmatch(r"[0-9a-f]{8,}", record["value"]): + shards.setdefault(record["value"][:LOOKUP_PREFIX], []).append(record) + folder = api / "lookup" + folder.mkdir(parents=True, exist_ok=True) + for prefix, items in sorted(shards.items()): + write_if_changed(str(folder / f"{prefix}.json"), json.dumps({ + "schema_version": 1, + "generated_at": generated_at, + "kind": "file-lookup", + "count": len(items), + "items": items, + }, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n") + return len(shards) + + def collection_paths(db: dict) -> dict[str, set[str]]: """Every path under bios/ that holds each file, by sha1. @@ -3101,9 +3225,14 @@ def _installed_links(sha1: str, consumers: FileConsumers, declared: set[str]) -> def _system_file_card( - f: dict, consumers: FileConsumers, anchored: set[str] | None = None + f: dict, consumers: FileConsumers, anchored: set[str] | None = None, + searchable: bool = True, ) -> list[str]: - """One file of a system page: hashes, who reads it, its provenance.""" + """One file of a system page: hashes, who reads it, its provenance. + + ``searchable`` false keeps even the name out of the site search: the + asset trees of game engines hold thousands of 0.ogg and tile.png. + """ name = f["name"] sha1_full = f.get("sha1", "unknown") md5_full = f.get("md5", "unknown") @@ -3129,7 +3258,7 @@ def _system_file_card( ) if anchor else "" lines.append(_card_title( f'{_code(name)} {size}{permalink}', - anchor, + anchor, searchable, )) lines.append("") # Every hash in full: the site search indexes text, not a title @@ -3160,17 +3289,68 @@ def _system_file_card( # looking up scph5501.bin found every platform but RetroBat, whose # pack installs it for six cores. lines.append(f"- Also installed by: {', '.join(installed)}") - lines[2:] = _search_skipped_apart(lines[2:]) + # The title stays searchable: the file name is what a reader types. + lines[4:] = search_excluded(lines[4:]) lines.extend(["", "", ""]) return lines +SEARCH_TABLE_BYTES = 3000 + + +def tables_out_of_search(markdown: str, threshold: int = SEARCH_TABLE_BYTES) -> str: + """Each table longer than ``threshold`` bytes, wrapped out of the search. + + A search result shows the start of a section: a 600-row table as one + snippet helps nobody, and the gaps, provenance and cross-reference + tables were 900 KB of the index. The headings above them stay. + """ + lines = markdown.split("\n") + out: list[str] = [] + index = 0 + while index < len(lines): + match = re.match(r"( *)\|", lines[index]) + if not match: + out.append(lines[index]) + index += 1 + continue + indent = match.group(1) + end = index + while end < len(lines) and re.match(rf"{indent}\|", lines[end]): + end += 1 + table = lines[index:end] + if sum(len(line) for line in table) > threshold: + out += ["", f"{indent}
", "", *table, + "", f"{indent}
", ""] + else: + out += table + index = end + return "\n".join(out) + + +def search_excluded(lines: list[str]) -> list[str]: + """Lines the site search leaves out, as a block of their own. + + The browser downloads and indexes the whole search index before the + first result: hashes, link lists and install commands repeated on 600 + pages made it 21 MB. Hashes are found by files/?hash=, names and prose + stay searchable. + + A section, not a div: the search plugin compares elements by tag, so + the first div closed inside an excluded div (a code block's) ended the + exclusion and the rest of the block was indexed. + """ + return ["", "
", "", *lines, "", "
", ""] + + # Card lines the site search leaves out. They list where a file goes and # who cites it, links to other pages that index the same names, and they # were 6.6 of the 24.7 MB the browser downloads and indexes before the first # result. The name, the hashes, the entry's path and the notes stay # searchable. SEARCH_SKIPPED = ( + "- SHA1:", "- MD5:", "- CRC32:", "- SHA256:", "- Adler32", "- Size:", + "- Validation", "- Priority:", "- Contents (", "- Platforms:", "- Also installed by:", "- Source:", "- Asked for by:", "- Collection path:", "- Collection copy:", "- Collection copies:", "- Installed by the platform's own package on:", @@ -3195,7 +3375,7 @@ def _search_skipped_apart(lines: list[str]) -> list[str]: return out -def _card_title(text: str, anchor: str) -> str: +def _card_title(text: str, anchor: str, searchable: bool = True) -> str: """A file card's heading, so the site search lands on the card. The search splits a page at its headings: a card titled in bold sent a @@ -3204,6 +3384,8 @@ def _card_title(text: str, anchor: str) -> str: keeps the id the toc gives it. """ attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title" + if not searchable: + attributes += " data-search-exclude" return f"#### {text} {{ {attributes} }}" @@ -3256,6 +3438,7 @@ def generate_system_page( "", ] anchored: set[str] = set() + searchable = manufacturer not in GAME_DATA_TOPS for console_name in sorted(consoles.keys()): files = consoles[console_name] @@ -3267,7 +3450,7 @@ def generate_system_page( variant_files = [f for f in files if "/.variants/" in f["path"]] for f in sorted(main_files, key=lambda x: x["name"]): - lines.extend(_system_file_card(f, consumers, anchored)) + lines.extend(_system_file_card(f, consumers, anchored, searchable)) if variant_files: # A variant gets the same card as a primary: the revision a @@ -3278,7 +3461,7 @@ def generate_system_page( for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])): lines.extend( f" {line}" if line else "" - for line in _system_file_card(v, consumers, anchored) + for line in _system_file_card(v, consumers, anchored, searchable) ) lines.append("") @@ -3907,14 +4090,14 @@ def _install_block(name: str, profile: dict) -> list[str]: """ if profile.get("type") in ("alias", "launcher", "test"): return [] - return [ + return search_excluded([ '??? tip "Install this emulator\'s files"', "", " The installer finds your platform and its BIOS folder, then downloads " "only the files this emulator reads there and checks each one.", "", *install_commands(f"--emulator {name}", " "), - ] + ]) def _platform_install_block(name: str, display: str, on_card: bool = False) -> list[str]: @@ -3975,7 +4158,7 @@ def _console_install_block(console: str, declared: list[tuple[str, str]]) -> lis f" {install_command('sh', f'--platform {platform} --system {shlex.quote(system)}')}" for platform, system in declared ] - return lines + [" ```", ""] + return search_excluded(lines + [" ```", ""]) def _render_platform_details(profile: dict) -> list[str]: @@ -4351,7 +4534,9 @@ def generate_emulator_page( ) if data_dirs: - lines.extend(["## Data directories", ""]) + # The member tables are a file list with hashes: out of the search, + # dolphin, primehack and ishiiruka each put 210 KB of it there. + lines.extend(["## Data directories { data-search-exclude }", ""]) member_anchors: set[str] = set() # Each tree with what it holds, where it comes from, which build reads # it and the code that does: these trees put thousands of files in @@ -5536,7 +5721,7 @@ def _mkdocs_config(docs: Path, text: str) -> tuple[Path, str]: _OS_NAMES = {"windows": "Windows", "darwin": "macOS", "linux": "Linux"} -def _runs_on(entry: dict, guide) -> str: +def _runs_on(entry: dict, guide: Guide) -> str: """Where a setup runs, read from the systems the installer detects it on.""" if guide.on_card: return "SD card, filled from a computer" @@ -5553,7 +5738,7 @@ def _runs_on(entry: dict, guide) -> str: return ", ".join(names) -def _folder_cell(name: str, guide, prefix: str = "") -> str: +def _folder_cell(name: str, guide: Guide, prefix: str = "") -> str: """The BIOS folder in a table cell, or a link where it depends on the system.""" if len(guide.folders) > 3: return f"Depends on the system: [listed on its page]({prefix}platforms/{name}.md#where-the-files-go)" @@ -5840,7 +6025,7 @@ def generate_mkdocs_nav( {"Dump provenance": "provenance.md"}, {"Updated platform files": "exports.md"}, {"Data and API": "data.md"}, - {"Find a file by its SHA1": "files/index.md"}, + {"Find a file": "files/index.md"}, {"Data model": "wiki/data-model.md"}, {"Architecture": "wiki/architecture.md"}, {"Build your own packs": "wiki/advanced-usage.md"}, @@ -6089,15 +6274,15 @@ def _generate(args: argparse.Namespace) -> None: print("Generating cross-reference page...") write_if_changed( str(docs / "cross-reference.md"), - generate_cross_reference(coverages, profiles) + REFERENCE_NEXT, + tables_out_of_search(generate_cross_reference(coverages, profiles)) + REFERENCE_NEXT, ) print("Generating gap analysis page...") write_if_changed( str(docs / "gaps.md"), - generate_gap_analysis( + tables_out_of_search(generate_gap_analysis( profiles, coverages, db, suppl_names, registry, gap_report - ) + REFERENCE_NEXT, + )) + REFERENCE_NEXT, ) print("Generating updated platform files page...") @@ -6113,7 +6298,9 @@ def _generate(args: argparse.Namespace) -> None: provenance_report = build_report(db, load_provenance_snapshots()) write_if_changed( str(docs / "provenance.md"), - generate_provenance_page(db, provenance_report, load_derivable()) + REFERENCE_NEXT, + tables_out_of_search( + generate_provenance_page(db, provenance_report, load_derivable()) + ) + REFERENCE_NEXT, ) # Wiki pages: copy manually maintained sources + generate dynamic ones diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index c968d30b..80ed62e6 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -446,7 +446,7 @@ def main() -> int: if (api_dir / name).exists() ] + sorted((api_dir / "platforms").glob("*.json")) + sorted( (api_dir / "systems").glob("*.json") - ) + ) + sorted((api_dir / "lookup").glob("*.json")) errors.extend( _validate_json_files(envelope_paths, "site-api-envelope.schema.json") ) diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 07a31239..be861d5c 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -651,6 +651,62 @@ class APerCoreDocumentValidates(unittest.TestCase): jsonschema.validate(document["items"][0]["profile"], schema) +class AHashFindsEveryPageThatNamesIt(unittest.TestCase): + """Hashes left the site search, 4 MB of the index every visitor + downloaded. files/?hash= reads one shard instead, and it must answer at + least what the search answered: the card, the platforms, the profiles.""" + + def test_records_cover_every_layer_and_shard_by_prefix(self): + import json as _json + import tempfile + + import generate_site # noqa: PLC0415 + + sha1 = "ab" * 20 + md5 = "490f666e1afb15b7362b406ed1cea246" + db = {"files": {sha1: {"name": "scph5501.bin", "md5": md5, "crc32": "8D8CB7E4", + "path": "bios/Sony/PlayStation/scph5501.bin"}}} + coverages = {"batocera": {"config": {"systems": {"sony-playstation": {"files": [ + {"name": "scph5501.bin", "md5": f"{md5}, 0123456789abcdef0123456789abcdef"}, + {"name": "typo.bin", "md5": "not-a-hash"}, + ]}}}}} + profiles = { + "duckstation": {"files": [{"name": "scph5501.bin", "md5": md5}, + {"name": "set.zip", "contents": [ + {"name": "m.rom", "crc32": "11223344"}]}]}, + "an-alias": {"type": "alias", "files": [{"name": "x", "md5": md5}]}, + } + locations = generate_site.file_location_items(db) + [ + {"sha1": "cd" * 20, "url": "emulators/dolphin/#sha1-" + "cd" * 6}] + members = {"dolphin-sys": {"Sys/GC/font.bin": {"sha1": "cd" * 20}}} + records = generate_site.file_lookup_records(db, coverages, profiles, locations, members) + found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records} + self.assertEqual(found[("collection", "", "md5", md5)]["url"], + "systems/sony/#sha1-" + "ab" * 6) + self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found) + self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin") + self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"], + "platforms/batocera/#sony-playstation") + self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found) + self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"], + "emulators/duckstation/#file-scph5501-bin-490f666e") + self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom") + self.assertNotIn(("emulator", "an-alias", "md5", md5), found) + + with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp: + shards = generate_site.write_lookup_shards(Path(tmp), records, "2026-01-01T00:00:00Z") + folder = Path(tmp) / "lookup" + names = sorted(path.name for path in folder.glob("*.json")) + self.assertEqual(len(names), shards) + for path in folder.glob("*.json"): + document = _json.loads(path.read_text()) + self.assertEqual(document["kind"], "file-lookup") + self.assertEqual(document["count"], len(document["items"])) + for item in document["items"]: + self.assertTrue(item["value"].startswith(path.stem)) + self.assertNotIn("no.json", names) + + class AFileHasAnAddressByContent(unittest.TestCase): """mt32_control.rom moved from the Commodore page to the Roland page and every link to its card broke: the anchor was content-based, the page @@ -669,7 +725,8 @@ class AFileHasAnAddressByContent(unittest.TestCase): ) self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page()) script = (Path(__file__).resolve().parents[1] / "docs_assets" / "site.js").read_text() - self.assertIn("file-locations.json", script) + self.assertIn("api/v1/lookup/", script) + self.assertIn('params.has("sha1")', script) from cross_reference import FileConsumers # noqa: PLC0415 card = "\n".join(generate_site._system_file_card( diff --git a/tests/test_site_render.py b/tests/test_site_render.py index 78d9aeb5..8188ece8 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -339,9 +339,10 @@ class TheSearchLandsOnTheCard(unittest.TestCase): class TheSearchSkipsLinkLists(unittest.TestCase): """The search index was 24.7 MB, read and indexed before the first result. Lists of platforms, copies and source citations on every card - made 6.6 MB of it, names already indexed on their own pages.""" + made 6.6 MB of it, names already indexed on their own pages, and the + hashes 4 MB more: files/?hash= finds those.""" - def test_a_system_card_keeps_its_hashes_and_skips_its_platforms(self): + def test_a_system_card_keeps_its_name_and_skips_its_body(self): import generate_site from cross_reference import FileConsumers @@ -352,26 +353,48 @@ class TheSearchSkipsLinkLists(unittest.TestCase): consumers.emulators = {sha1: {"duckstation"}} entry = {"name": "a.bin", "sha1": sha1, "md5": "b" * 32, "size": 3} text = "\n".join(generate_site._system_file_card(entry, consumers)) - searched, skipped = text.split('
') - self.assertIn("- MD5:", searched) - self.assertIn("- Emulators:", searched) - self.assertIn("- Platforms:", skipped) - self.assertIn("- Also installed by:", skipped) - self.assertEqual(text.count("
"), 2) + searched, skipped = text.split("
") + self.assertIn("`a.bin`", searched) + self.assertNotIn("- MD5:", searched) + for line in ("- MD5:", "- Emulators:", "- Platforms:", "- Also installed by:"): + self.assertIn(line, skipped) + + def test_a_game_data_card_skips_its_name_too(self): + import generate_site + from cross_reference import FileConsumers + + entry = {"name": "0.ogg", "sha1": "a" * 40, "size": 3} + lines = generate_site._system_file_card( + entry, FileConsumers({"files": {}}), searchable=False) + self.assertTrue(lines[2].endswith("data-search-exclude }")) def test_runs_alternate_in_card_order(self): import generate_site - lines = ["- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", - "- Destination: `d`", "- Contents (1 members):", " - `m`"] + lines = ["- Path: `p`", "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", + "- Destination: `d`", "- Contents (1 members):", " - `m`", + "- Note"] out = [line for line in generate_site._search_skipped_apart(lines) if line] self.assertEqual(out, [ - "- SHA1: x", + "- Path: `p`", '
', - "- Source: `a.c:1`", "- Platforms: p", "
", - "- Destination: `d`", "- Contents (1 members):", " - `m`", + "- SHA1: x", "- Source: `a.c:1`", "- Platforms: p", "", + "- Destination: `d`", + '
', + "- Contents (1 members):", " - `m`", "
", + "- Note", ]) + def test_long_tables_leave_the_search(self): + import generate_site + + short = "| a |\n|---|\n| 1 |" + long = " | a |\n |---|\n" + "\n".join(f" | {'x' * 50} |" for _ in range(80)) + text = generate_site.tables_out_of_search(f"# T\n\n{short}\n\n{long}\n") + self.assertEqual(text.count("data-search-exclude"), 1) + self.assertIn("
\n\n | a |", text) + self.assertIn(short, text) + if __name__ == "__main__": unittest.main() diff --git a/wiki/index.md b/wiki/index.md index 755d1ee3..dfe00c8a 100644 --- a/wiki/index.md +++ b/wiki/index.md @@ -20,7 +20,7 @@ toolchain works. To set up BIOS files on a device, start with ## Data - **[Data and API](../data.md)** - JSON, CSV and SQLite exports with their checksums -- **[Find a file by its SHA1](../files/index.md)** - the permanent address of a file by its contents +- **[Find a file](../files/index.md)** - any SHA1, MD5, CRC32 or SHA256: the file, the platforms that declare it, the emulators that check it - **[Data model](data-model.md)** - database.json structure, indexes, file resolution order, YAML formats ## The toolchain From 36ac7b0cd6d71487a75b276a0a1a606bcda0514d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:13:52 +0200 Subject: [PATCH 4/5] feat: split the heavy site pages A manufacturer page over 500 KB of Markdown becomes a summary: its largest consoles move to pages of their own, cut into parts when one console is still too large, and every console keeps its heading there. MAME's 1,564 cards move to four pages, its page keeping a row per entry with the card's old anchor. RetroDECK's arcade list gets its own page. site.js follows a moved card by content or by its row. The largest page drops from 10.5 MB to 0.88 MB. Emulator groups of the navigation open by initial (65 KB to 41 KB before the content). File cards are level 3 under a level 2, and the hook keeps the table of contents of card pages to level 2: platform pages no longer list every system id. Key/value cards name their columns. The sweep runs before the decoration, which kept every page no generator wrote any more. --- docs_assets/site.js | 16 +- mkdocs.yml | 1188 +++++++++++++++++++------------------ scripts/generate_site.py | 627 +++++++++++++++----- scripts/mkdocs_hooks.py | 20 + scripts/siterender.py | 66 ++- tests/test_site_render.py | 4 +- tests/test_site_weight.py | 168 ++++++ wiki/testing-guide.md | 2 +- 8 files changed, 1362 insertions(+), 729 deletions(-) create mode 100644 tests/test_site_weight.py diff --git a/docs_assets/site.js b/docs_assets/site.js index aacaaa24..33548400 100644 --- a/docs_assets/site.js +++ b/docs_assets/site.js @@ -238,10 +238,20 @@ }; // A file card moved to another page keeps its address: an id of the form - // sha1-<12 hex> that this page does not hold is looked up by content. + // sha1-<12 hex> that this page does not hold is looked up by content, and + // an emulator card listed in the table of a split page opens its page. const followMovedCard = () => { - const match = /^#sha1-([0-9a-f]{12})$/.exec(window.location.hash); - if (!match || document.getElementById(`sha1-${match[1]}`)) return; + const id = decodeURIComponent(window.location.hash.slice(1)); + if (!id) return; + const target = document.getElementById(id); + const row = target?.closest(".rb-card-index tr"); + const link = row?.querySelector("a[href]"); + if (link) { + window.location.replace(link.href); + return; + } + const match = /^sha1-([0-9a-f]{12})$/.exec(id); + if (!match || target) return; window.location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot())); }; diff --git a/mkdocs.yml b/mkdocs.yml index 2bbc5ee9..f38f336f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,19 +84,17 @@ markdown_extensions: - tables - toc: permalink: true - # File cards are level 4 headings: the search lands on them, and a - # page of a hundred cards keeps a readable table of contents. + # Level 3 is the wiki's subsections. On pages of file cards, also level + # 3, the hook keeps the table of contents to level 2. toc_depth: 3 - pymdownx.details - pymdownx.highlight: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.keys -- pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format +# No mermaid fence: Material drew it in the browser with a script from a +# CDN. Diagrams are SVG files written by scripts/sitediagrams.py. +- pymdownx.superfences - pymdownx.tabbed: alternate_style: true plugins: @@ -245,587 +243,613 @@ nav: - 2600.emu: emulators/2600-emu.md - 3dSen: emulators/3dsen.md - A to C: - - a5200: emulators/a5200.md - - A7800: emulators/a7800.md - - Abuse: emulators/abuse.md - - ACE-DL: emulators/ace-dl.md - - advanced_drastic: emulators/advanced_drastic.md - - AdvanceMAME: emulators/advancemame.md - - AdvanceMAME 0.94: emulators/advmame-0.94.md - - AdvanceMAME 1.4: emulators/advmame-1.4.md - - Adventure Game Studio: emulators/ags.md - - AetherSX2: emulators/aethersx2.md - - Aleph One: emulators/alephone.md - - Altirra: emulators/altirra.md - - amiarcadia: emulators/amiarcadia.md - - Amiberry: emulators/amiberry.md - - AppleWin: emulators/applewin.md - - aPS3e: emulators/aps3e.md - - ArcadeDuck: emulators/arcadeduck.md - - ArcadeFlashWeb: emulators/arcadeflashweb.md - - Ardens: emulators/ardens.md - - Arduous: emulators/arduous.md - - ares: emulators/ares.md - - ARMSX1: emulators/armsx1.md - - ARMSX2: emulators/armsx2.md - - Atari800: emulators/atari800.md - - aX360e: emulators/ax360e.md - - Azahar: emulators/azahar.md - - AzaharPlus: emulators/azaharplus.md - - b2: emulators/b2.md - - Basilisk II: emulators/basiliskii.md - - BBKEmu: emulators/bbkemu.md - - beebem: emulators/beebem.md - - Beetle GBA (Mednafen): emulators/beetle_gba.md - - Beetle Lynx (Mednafen Lynx): emulators/beetle_lynx.md - - Beetle NGP (Mednafen Neo Geo Pocket): emulators/beetle_ngp.md - - Beetle PC-FX (Mednafen): emulators/beetle_pcfx.md - - Beetle PCE (Mednafen PCE): emulators/beetle_pce.md - - Beetle PSX (Mednafen PSX): emulators/beetle_psx.md - - Beetle VB (Mednafen Virtual Boy): emulators/beetle_vb.md - - Beetle WonderSwan (Mednafen WonderSwan): emulators/beetle_wswan.md - - beetle_saturn: emulators/beetle_saturn.md - - BennuGD: emulators/bennugd.md - - BennuGD monolithic (bgdi): emulators/bennugd-monolithic.md - - Bermuda Syndrome: emulators/bsyndrome.md - - Bermuda Syndrome (bs): emulators/bermuda.md - - BigPEmu: emulators/bigpemu.md - - BizHawk: emulators/bizhawk.md - - bk-emulator: emulators/bk.md - - BlastEm: emulators/blastem.md - - blueMSX: emulators/bluemsx.md - - bnes: emulators/bnes.md - - boom3: emulators/boom3.md - - Boytacean: emulators/boytacean.md - - bsnes: emulators/bsnes.md - - bsnes 2014: emulators/bsnes2014.md - - bsnes C++98: emulators/bsnes_cplusplus98.md - - bsnes-hd beta: emulators/bsnes_hd_beta.md - - bsnes-jg: emulators/bsnes-jg.md - - bsnes-mercury: emulators/bsnes_mercury.md - - BStone: emulators/bstone.md - - C-Dogs SDL: emulators/cdogs.md - - C64.emu: emulators/c64-emu.md - - Cannonball: emulators/cannonball.md - - CaPriCe Forever: emulators/capriceforever.md - - Caprice32: emulators/cap32.md - - CapriceRPI: emulators/capricerpi.md - - CatacombGL: emulators/catacombgl.md - - ccleste: emulators/ccleste.md - - CDi 2015: emulators/cdi2015.md - - Cemu: emulators/cemu.md - - Chihiro-DS: emulators/chihiro-ds.md - - ChimeraSNES: emulators/chimerasnes.md - - Chocolate Doom: emulators/chocolate-doom.md - - Citra: emulators/citra.md - - Citra 2018: emulators/citra2018.md - - Citra Canary: emulators/citra_canary.md - - Citra MMJ: emulators/citra-mmj.md - - Citron: emulators/citron.md - - Clock Signal (CLK): emulators/clk.md - - ClownMDEmu: emulators/clownmdemu.md - - ColEm: emulators/colem.md - - Commander Genius: emulators/cgenius.md - - CoolbarEmu: emulators/coolbaremu.md - - CoolCV: emulators/coolcv.md - - corsixth: emulators/corsixth.md - - CPCemu: emulators/cpcemu.md - - CrocoDS: emulators/crocods.md - - Cruzes: emulators/cruzes.md - - CSpect: emulators/cspect.md - - Cxbx-Reloaded: emulators/cxbx-reloaded.md - - Cxbx-Reloaded (31ff15b): emulators/cxbx-reloaded-2023.md + - A: + - a5200: emulators/a5200.md + - A7800: emulators/a7800.md + - Abuse: emulators/abuse.md + - ACE-DL: emulators/ace-dl.md + - advanced_drastic: emulators/advanced_drastic.md + - AdvanceMAME: emulators/advancemame.md + - AdvanceMAME 0.94: emulators/advmame-0.94.md + - AdvanceMAME 1.4: emulators/advmame-1.4.md + - Adventure Game Studio: emulators/ags.md + - AetherSX2: emulators/aethersx2.md + - Aleph One: emulators/alephone.md + - Altirra: emulators/altirra.md + - amiarcadia: emulators/amiarcadia.md + - Amiberry: emulators/amiberry.md + - AppleWin: emulators/applewin.md + - aPS3e: emulators/aps3e.md + - ArcadeDuck: emulators/arcadeduck.md + - ArcadeFlashWeb: emulators/arcadeflashweb.md + - Ardens: emulators/ardens.md + - Arduous: emulators/arduous.md + - ares: emulators/ares.md + - ARMSX1: emulators/armsx1.md + - ARMSX2: emulators/armsx2.md + - Atari800: emulators/atari800.md + - aX360e: emulators/ax360e.md + - Azahar: emulators/azahar.md + - AzaharPlus: emulators/azaharplus.md + - B: + - b2: emulators/b2.md + - Basilisk II: emulators/basiliskii.md + - BBKEmu: emulators/bbkemu.md + - beebem: emulators/beebem.md + - Beetle GBA (Mednafen): emulators/beetle_gba.md + - Beetle Lynx (Mednafen Lynx): emulators/beetle_lynx.md + - Beetle NGP (Mednafen Neo Geo Pocket): emulators/beetle_ngp.md + - Beetle PC-FX (Mednafen): emulators/beetle_pcfx.md + - Beetle PCE (Mednafen PCE): emulators/beetle_pce.md + - Beetle PSX (Mednafen PSX): emulators/beetle_psx.md + - Beetle VB (Mednafen Virtual Boy): emulators/beetle_vb.md + - Beetle WonderSwan (Mednafen WonderSwan): emulators/beetle_wswan.md + - beetle_saturn: emulators/beetle_saturn.md + - BennuGD: emulators/bennugd.md + - BennuGD monolithic (bgdi): emulators/bennugd-monolithic.md + - Bermuda Syndrome: emulators/bsyndrome.md + - Bermuda Syndrome (bs): emulators/bermuda.md + - BigPEmu: emulators/bigpemu.md + - BizHawk: emulators/bizhawk.md + - bk-emulator: emulators/bk.md + - BlastEm: emulators/blastem.md + - blueMSX: emulators/bluemsx.md + - bnes: emulators/bnes.md + - boom3: emulators/boom3.md + - Boytacean: emulators/boytacean.md + - bsnes: emulators/bsnes.md + - bsnes 2014: emulators/bsnes2014.md + - bsnes C++98: emulators/bsnes_cplusplus98.md + - bsnes-hd beta: emulators/bsnes_hd_beta.md + - bsnes-jg: emulators/bsnes-jg.md + - bsnes-mercury: emulators/bsnes_mercury.md + - BStone: emulators/bstone.md + - C: + - C-Dogs SDL: emulators/cdogs.md + - C64.emu: emulators/c64-emu.md + - Cannonball: emulators/cannonball.md + - CaPriCe Forever: emulators/capriceforever.md + - Caprice32: emulators/cap32.md + - CapriceRPI: emulators/capricerpi.md + - CatacombGL: emulators/catacombgl.md + - ccleste: emulators/ccleste.md + - CDi 2015: emulators/cdi2015.md + - Cemu: emulators/cemu.md + - Chihiro-DS: emulators/chihiro-ds.md + - ChimeraSNES: emulators/chimerasnes.md + - Chocolate Doom: emulators/chocolate-doom.md + - Citra: emulators/citra.md + - Citra 2018: emulators/citra2018.md + - Citra Canary: emulators/citra_canary.md + - Citra MMJ: emulators/citra-mmj.md + - Citron: emulators/citron.md + - Clock Signal (CLK): emulators/clk.md + - ClownMDEmu: emulators/clownmdemu.md + - ColEm: emulators/colem.md + - Commander Genius: emulators/cgenius.md + - CoolbarEmu: emulators/coolbaremu.md + - CoolCV: emulators/coolcv.md + - corsixth: emulators/corsixth.md + - CPCemu: emulators/cpcemu.md + - CrocoDS: emulators/crocods.md + - Cruzes: emulators/cruzes.md + - CSpect: emulators/cspect.md + - Cxbx-Reloaded: emulators/cxbx-reloaded.md + - Cxbx-Reloaded (31ff15b): emulators/cxbx-reloaded-2023.md - D to F: - - D.Smile: emulators/dsmile.md - - DaedalusX64: emulators/daedalusx64.md - - Daphne: emulators/daphne.md - - DarkPlaces: emulators/darkplaces.md - - Demul: emulators/demul.md - - DeSmuME: emulators/desmume.md - - DeSmuME 2015: emulators/desmume2015.md - - devilutionx: emulators/devilutionx.md - - DGen/SDL: emulators/dgen.md - - dhewm3: emulators/dhewm3.md - - DICE: emulators/dice.md - - DingooEmu: emulators/dingooemu.md - - Dinothawr: emulators/dinothawr.md - - DirectXBox: emulators/directxbox.md - - DirkSimple: emulators/dirksimple.md - - Dolphin: emulators/dolphin.md - - Dolphin Launcher: emulators/dolphin_launcher.md - - Dolphin MMJR: emulators/dolphin-mmjr.md - - Dolphin MMJR2: emulators/dolphin-mmjr2.md - - DOSBox: emulators/dosbox.md - - DOSBox (SDL2 fork): emulators/dosbox-sdl2.md - - DOSBox Pure: emulators/dosbox_pure.md - - DOSBox Pure Unleashed: emulators/dosbox-pure-unleashed.md - - DOSBox Staging: emulators/dosbox-staging.md - - DOSBox-core: emulators/dosbox_core.md - - DOSBox-SVN: emulators/dosbox_svn.md - - DOSBox-SVN CE: emulators/dosbox_svn_ce.md - - DOSBox-X: emulators/dosbox-x.md - - DoubleCherryGB: emulators/doublecherrygb.md - - doukutsu-rs: emulators/doukutsu_rs.md - - DraStic: emulators/drastic.md - - DREAMM: emulators/dreamm.md - - DroidArcadia: emulators/droidarcadia.md - - dsda-doom: emulators/dsda-doom.md - - DSperate: emulators/dsperate.md - - DuckStation: emulators/duckstation.md - - DuckStation (legacy): emulators/duckstation-legacy.md - - Dusklight: emulators/dusklight.md - - dxx-rebirth: emulators/dxx-rebirth.md - - EasyRPG Player: emulators/easyrpg.md - - ECWolf: emulators/ecwolf.md - - Eden: emulators/eden.md - - EDuke32: emulators/eduke32.md - - EightyOne: emulators/81.md - - EightyOne (Recalbox): emulators/eightyone.md - - eka2l1: emulators/eka2l1.md - - EmuCoreV: emulators/emucorev.md - - EmuCoreX: emulators/emucorex.md - - EmuCV: emulators/emucv.md - - EmuSCV: emulators/emuscv.md - - emux (CHIP-8): emulators/emux_chip8.md - - emux (Game Boy): emulators/emux_gb.md - - emux (NES): emulators/emux_nes.md - - emux (SMS): emulators/emux_sms.md - - ep128emu-core: emulators/ep128emu.md - - ep128emu_core: emulators/ep128emu_core.md - - ePSXe: emulators/epsxe.md - - ET Legacy: emulators/etlegacy.md - - Eternal Computer: emulators/eternal.md - - Exel-Hence: emulators/exel-hence.md - - FAKE-08: emulators/fake08.md - - fallout1-ce: emulators/fallout1-ce.md - - fallout2-ce: emulators/fallout2-ce.md - - FB Alpha 2012: emulators/fbalpha2012.md - - FB Alpha 2012 CPS-1: emulators/fbalpha2012_cps1.md - - FB Alpha 2012 CPS-2: emulators/fbalpha2012_cps2.md - - FB Alpha 2012 CPS-3: emulators/fbalpha2012_cps3.md - - FB Alpha 2012 Neo Geo: emulators/fbalpha2012_neogeo.md - - FB Alpha 2019: emulators/fbalpha2019.md - - FBZX: emulators/fbzx.md - - FCEUmm: emulators/fceumm.md - - FCEUmm Mod: emulators/fceumm_mod.md - - fheroes2: emulators/fheroes2.md - - FinalBurn Neo: emulators/fbneo.md - - FinalBurn Neo (CPS-1/CPS-2): emulators/fbneo_cps12.md - - FinalBurn Neo (Neo Geo): emulators/fbneo_neogeo.md - - fixGB: emulators/fixgb.md - - fixNES: emulators/fixnes.md - - Flycast: emulators/flycast.md - - Flycast 2021: emulators/flycast2021.md - - Flycast 2022 Low-End: emulators/flycast2021le.md - - Flycast Dojo: emulators/flycast-dojo.md - - fMSX: emulators/fmsx.md - - FPse: emulators/fpse.md - - FPseNG: emulators/fpse-ng.md - - FreeChaF: emulators/freechaf.md - - FreeIntv: emulators/freeintv.md - - FreeJ2ME: emulators/freej2me.md - - FreeJ2ME-Plus: emulators/freej2me_plus.md - - Frodo: emulators/frodo.md - - Frotz: emulators/frotz.md - - FS-UAE: emulators/fsuae.md - - FS-UAE 3.1: emulators/fs-uae.md - - Fuse: emulators/fuse.md - - future-pinball: emulators/future-pinball.md - - future-pinball-fploader: emulators/future-pinball-fploader.md + - D: + - D.Smile: emulators/dsmile.md + - DaedalusX64: emulators/daedalusx64.md + - Daphne: emulators/daphne.md + - DarkPlaces: emulators/darkplaces.md + - Demul: emulators/demul.md + - DeSmuME: emulators/desmume.md + - DeSmuME 2015: emulators/desmume2015.md + - devilutionx: emulators/devilutionx.md + - DGen/SDL: emulators/dgen.md + - dhewm3: emulators/dhewm3.md + - DICE: emulators/dice.md + - DingooEmu: emulators/dingooemu.md + - Dinothawr: emulators/dinothawr.md + - DirectXBox: emulators/directxbox.md + - DirkSimple: emulators/dirksimple.md + - Dolphin: emulators/dolphin.md + - Dolphin Launcher: emulators/dolphin_launcher.md + - Dolphin MMJR: emulators/dolphin-mmjr.md + - Dolphin MMJR2: emulators/dolphin-mmjr2.md + - DOSBox: emulators/dosbox.md + - DOSBox (SDL2 fork): emulators/dosbox-sdl2.md + - DOSBox Pure: emulators/dosbox_pure.md + - DOSBox Pure Unleashed: emulators/dosbox-pure-unleashed.md + - DOSBox Staging: emulators/dosbox-staging.md + - DOSBox-core: emulators/dosbox_core.md + - DOSBox-SVN: emulators/dosbox_svn.md + - DOSBox-SVN CE: emulators/dosbox_svn_ce.md + - DOSBox-X: emulators/dosbox-x.md + - DoubleCherryGB: emulators/doublecherrygb.md + - doukutsu-rs: emulators/doukutsu_rs.md + - DraStic: emulators/drastic.md + - DREAMM: emulators/dreamm.md + - DroidArcadia: emulators/droidarcadia.md + - dsda-doom: emulators/dsda-doom.md + - DSperate: emulators/dsperate.md + - DuckStation: emulators/duckstation.md + - DuckStation (legacy): emulators/duckstation-legacy.md + - Dusklight: emulators/dusklight.md + - dxx-rebirth: emulators/dxx-rebirth.md + - E: + - EasyRPG Player: emulators/easyrpg.md + - ECWolf: emulators/ecwolf.md + - Eden: emulators/eden.md + - EDuke32: emulators/eduke32.md + - EightyOne: emulators/81.md + - EightyOne (Recalbox): emulators/eightyone.md + - eka2l1: emulators/eka2l1.md + - EmuCoreV: emulators/emucorev.md + - EmuCoreX: emulators/emucorex.md + - EmuCV: emulators/emucv.md + - EmuSCV: emulators/emuscv.md + - emux (CHIP-8): emulators/emux_chip8.md + - emux (Game Boy): emulators/emux_gb.md + - emux (NES): emulators/emux_nes.md + - emux (SMS): emulators/emux_sms.md + - ep128emu-core: emulators/ep128emu.md + - ep128emu_core: emulators/ep128emu_core.md + - ePSXe: emulators/epsxe.md + - ET Legacy: emulators/etlegacy.md + - Eternal Computer: emulators/eternal.md + - Exel-Hence: emulators/exel-hence.md + - F: + - FAKE-08: emulators/fake08.md + - fallout1-ce: emulators/fallout1-ce.md + - fallout2-ce: emulators/fallout2-ce.md + - FB Alpha 2012: emulators/fbalpha2012.md + - FB Alpha 2012 CPS-1: emulators/fbalpha2012_cps1.md + - FB Alpha 2012 CPS-2: emulators/fbalpha2012_cps2.md + - FB Alpha 2012 CPS-3: emulators/fbalpha2012_cps3.md + - FB Alpha 2012 Neo Geo: emulators/fbalpha2012_neogeo.md + - FB Alpha 2019: emulators/fbalpha2019.md + - FBZX: emulators/fbzx.md + - FCEUmm: emulators/fceumm.md + - FCEUmm Mod: emulators/fceumm_mod.md + - fheroes2: emulators/fheroes2.md + - FinalBurn Neo: emulators/fbneo.md + - FinalBurn Neo (CPS-1/CPS-2): emulators/fbneo_cps12.md + - FinalBurn Neo (Neo Geo): emulators/fbneo_neogeo.md + - fixGB: emulators/fixgb.md + - fixNES: emulators/fixnes.md + - Flycast: emulators/flycast.md + - Flycast 2021: emulators/flycast2021.md + - Flycast 2022 Low-End: emulators/flycast2021le.md + - Flycast Dojo: emulators/flycast-dojo.md + - fMSX: emulators/fmsx.md + - FPse: emulators/fpse.md + - FPseNG: emulators/fpse-ng.md + - FreeChaF: emulators/freechaf.md + - FreeIntv: emulators/freeintv.md + - FreeJ2ME: emulators/freej2me.md + - FreeJ2ME-Plus: emulators/freej2me_plus.md + - Frodo: emulators/frodo.md + - Frotz: emulators/frotz.md + - FS-UAE: emulators/fsuae.md + - FS-UAE 3.1: emulators/fs-uae.md + - Fuse: emulators/fuse.md + - future-pinball: emulators/future-pinball.md + - future-pinball-fploader: emulators/future-pinball-fploader.md - G to L: - - galaksija: emulators/galaksija.md - - GAM4980: emulators/gam4980.md - - Gambatte: emulators/gambatte.md - - Game & Watch: emulators/gw.md - - GameTank: emulators/gametank.md - - Gargoyle: emulators/gargoyle.md - - GBA.emu: emulators/gba-emu.md - - GBC.emu: emulators/gbc-emu.md - - Gearboy: emulators/gearboy.md - - Gearcoleco: emulators/gearcoleco.md - - Geargrafx: emulators/geargrafx.md - - Gearlynx: emulators/gearlynx.md - - Gearsystem: emulators/gearsystem.md - - Geartowns: emulators/geartowns.md - - GemRB: emulators/gemrb.md - - Genesis Plus GX: emulators/genesis_plus_gx.md - - Genesis Plus GX Expanded: emulators/genesis_plus_gx_expanded.md - - Genesis Plus GX Wide: emulators/genesis_plus_gx_wide.md - - Geolith: emulators/geolith.md - - Ghostship: emulators/ghostship.md - - GnGeo-Pi: emulators/gngeopi.md - - Gopher2600: emulators/gopher2600.md - - gopher64: emulators/gopher64.md - - gpSP: emulators/gpsp.md - - GroovyMAME: emulators/groovymame.md - - GSplus: emulators/gsplus.md - - GSplus (legacy): emulators/gsplus-legacy.md - - GZDoom: emulators/gzdoom.md - - hakuX: emulators/hakux.md - - Handy: emulators/handy.md - - Hatari: emulators/hatari.md - - Hatari 2014: emulators/hatari2014.md - - hatariB: emulators/hatarib.md - - HBMAME (Homebrew MAME): emulators/hbmame.md - - higan (SFC Accuracy): emulators/higan_sfc.md - - hode: emulators/hode.md - - Holani: emulators/holani.md - - Hurrican: emulators/hurrican.md - - Hydra Castle Labyrinth: emulators/hcl.md - - Hypseus Singe: emulators/hypseus-singe.md - - idTech4A++: emulators/idtech4a++.md - - Ikemen GO: emulators/ikemen.md - - iNES: emulators/ines.md - - Infinity: emulators/infinity.md - - Ion Fury: emulators/fury.md - - ioquake3: emulators/ioquake3.md - - ioRTCW: emulators/iortcw.md - - IrataJaguar: emulators/iratajaguar.md - - IroGB: emulators/irogb.md - - Ishiiruka: emulators/ishiiruka.md - - izapple2: emulators/izapple2.md - - J2ME Loader: emulators/j2me-loader.md - - JAXE: emulators/jaxe.md - - Jazz² Resurrection: emulators/jazz2-native.md - - jgenesis: emulators/jgenesis.md - - JL-Mod: emulators/jl-mod.md - - JollyCV: emulators/jollycv.md - - Julius: emulators/julius.md - - Jynx: emulators/jynx.md - - jzIntv: emulators/jzintv.md - - Kega Fusion: emulators/kega-fusion.md - - KEGS: emulators/kegs.md - - KEmulator nnmod: emulators/kemulator.md - - Kenji-NX: emulators/kenji-nx.md - - Kronos: emulators/kronos.md - - Lexaloffle: emulators/lexaloffle.md - - lightspark: emulators/lightspark.md - - Lime3DS: emulators/lime3ds.md - - LinApple: emulators/linapple.md - - Lindbergh Loader: emulators/lindbergh-loader.md - - Linkboy: emulators/linkboy.md - - Linux Loader: emulators/linuxloader.md - - LowRes NX: emulators/lowresnx.md - - LRPS2: emulators/lrps2.md - - Lynx.emu: emulators/lynx-emu.md - - LZDoom: emulators/lzdoom.md + - G: + - galaksija: emulators/galaksija.md + - GAM4980: emulators/gam4980.md + - Gambatte: emulators/gambatte.md + - Game & Watch: emulators/gw.md + - GameTank: emulators/gametank.md + - Gargoyle: emulators/gargoyle.md + - GBA.emu: emulators/gba-emu.md + - GBC.emu: emulators/gbc-emu.md + - Gearboy: emulators/gearboy.md + - Gearcoleco: emulators/gearcoleco.md + - Geargrafx: emulators/geargrafx.md + - Gearlynx: emulators/gearlynx.md + - Gearsystem: emulators/gearsystem.md + - Geartowns: emulators/geartowns.md + - GemRB: emulators/gemrb.md + - Genesis Plus GX: emulators/genesis_plus_gx.md + - Genesis Plus GX Expanded: emulators/genesis_plus_gx_expanded.md + - Genesis Plus GX Wide: emulators/genesis_plus_gx_wide.md + - Geolith: emulators/geolith.md + - Ghostship: emulators/ghostship.md + - GnGeo-Pi: emulators/gngeopi.md + - Gopher2600: emulators/gopher2600.md + - gopher64: emulators/gopher64.md + - gpSP: emulators/gpsp.md + - GroovyMAME: emulators/groovymame.md + - GSplus: emulators/gsplus.md + - GSplus (legacy): emulators/gsplus-legacy.md + - GZDoom: emulators/gzdoom.md + - H: + - hakuX: emulators/hakux.md + - Handy: emulators/handy.md + - Hatari: emulators/hatari.md + - Hatari 2014: emulators/hatari2014.md + - hatariB: emulators/hatarib.md + - HBMAME (Homebrew MAME): emulators/hbmame.md + - higan (SFC Accuracy): emulators/higan_sfc.md + - hode: emulators/hode.md + - Holani: emulators/holani.md + - Hurrican: emulators/hurrican.md + - Hydra Castle Labyrinth: emulators/hcl.md + - Hypseus Singe: emulators/hypseus-singe.md + - I: + - idTech4A++: emulators/idtech4a++.md + - Ikemen GO: emulators/ikemen.md + - iNES: emulators/ines.md + - Infinity: emulators/infinity.md + - Ion Fury: emulators/fury.md + - ioquake3: emulators/ioquake3.md + - ioRTCW: emulators/iortcw.md + - IrataJaguar: emulators/iratajaguar.md + - IroGB: emulators/irogb.md + - Ishiiruka: emulators/ishiiruka.md + - izapple2: emulators/izapple2.md + - J: + - J2ME Loader: emulators/j2me-loader.md + - JAXE: emulators/jaxe.md + - Jazz² Resurrection: emulators/jazz2-native.md + - jgenesis: emulators/jgenesis.md + - JL-Mod: emulators/jl-mod.md + - JollyCV: emulators/jollycv.md + - Julius: emulators/julius.md + - Jynx: emulators/jynx.md + - jzIntv: emulators/jzintv.md + - K: + - Kega Fusion: emulators/kega-fusion.md + - KEGS: emulators/kegs.md + - KEmulator nnmod: emulators/kemulator.md + - Kenji-NX: emulators/kenji-nx.md + - Kronos: emulators/kronos.md + - L: + - Lexaloffle: emulators/lexaloffle.md + - lightspark: emulators/lightspark.md + - Lime3DS: emulators/lime3ds.md + - LinApple: emulators/linapple.md + - Lindbergh Loader: emulators/lindbergh-loader.md + - Linkboy: emulators/linkboy.md + - Linux Loader: emulators/linuxloader.md + - LowRes NX: emulators/lowresnx.md + - LRPS2: emulators/lrps2.md + - Lynx.emu: emulators/lynx-emu.md + - LZDoom: emulators/lzdoom.md - M to O: - - M2000: emulators/m2000.md - - M64Plus FZ: emulators/m64plus-fz.md - - MagicEngine: emulators/magicengine.md - - MAME: emulators/mame.md - - MAME 0.258: emulators/mame0258.md - - MAME 0.278: emulators/mame0278.md - - MAME 2000: emulators/mame2000.md - - MAME 2003: emulators/mame2003.md - - MAME 2003 Midway: emulators/mame2003_midway.md - - MAME 2003-Plus: emulators/mame2003_plus.md - - MAME 2009: emulators/mame2009.md - - MAME 2010: emulators/mame2010.md - - MAME 2014: emulators/mame2014.md - - MAME 2015: emulators/mame2015.md - - MAME 2016: emulators/mame2016.md - - MAME Arcade: emulators/mamearcade.md - - MAME MESS: emulators/mamemess.md - - MAME4ALL-Pi: emulators/mame4all.md - - MAME4droid (0.139u1): emulators/mame4droid.md - - MAME4droid 2024: emulators/mame4droid-2024.md - - Mandarine: emulators/mandarine.md - - Mariani: emulators/mariani.md - - MasterGear: emulators/mastergear.md - - MD.emu: emulators/md-emu.md - - Mednafen: emulators/mednafen.md - - Mednafen STV: emulators/mednafen_stv.md - - mednafen_gba: emulators/mednafen_gba.md - - mednafen_pce: emulators/mednafen_pce.md - - mednafen_pce_fast: emulators/mednafen_pce_fast.md - - mednafen_snes: emulators/mednafen_snes.md - - mednafen_supafaust: emulators/mednafen_supafaust.md - - mednafen_supergrafx: emulators/mednafen_supergrafx.md - - melonDS: emulators/melonds.md - - melonDS DS: emulators/melonds_ds.md - - melonDS Nightly: emulators/melonds-nightly.md - - Mesen: emulators/mesen.md - - mesen-s: emulators/mesen-s.md - - MesenCE: emulators/mesence.md - - MESS 2015: emulators/mess2015.md - - Meteor GBA: emulators/meteor.md - - MFME: emulators/mfme.md - - mGBA: emulators/mgba.md - - mGBA Splitscreen: emulators/mgba_splitscreen.md - - Mini vMac: emulators/minivmac.md - - Mini vMac 37: emulators/minivmac37.md - - mkxp-z: emulators/mkxp_z.md - - Model 2 Emulator: emulators/model2.md - - MojoZork: emulators/mojozork.md - - MSX.emu: emulators/msx-emu.md - - Mu: emulators/mu.md - - Mupen64Plus: emulators/mupen64plus.md - - Mupen64Plus (libretro, 2019): emulators/mupen64plus-libretro.md - - Mupen64Plus AE: emulators/mupen64plus-ae.md - - Mupen64Plus-Next: emulators/mupen64plus_next.md - - My Boy!: emulators/my-boy.md - - My OldBoy!: emulators/my-oldboy.md - - NanoBoyAdvance: emulators/nanoboyadvance.md - - Native32Emu: emulators/native32emu.md - - nekop2: emulators/nekop2.md - - NEO.emu: emulators/neo-emu.md - - NeoCD: emulators/neocd.md - - nes: emulators/nes.md - - NES.emu: emulators/nes-emu.md - - Nesoid: emulators/nesoid.md - - Nestopia UE: emulators/nestopia.md - - NetherSX2: emulators/nethersx2.md - - NetherSX2-Turnip: emulators/nethersx2-turnip.md - - NetherSX2-Turnip Classic: emulators/nethersx2-turnip-classic.md - - NGP.emu: emulators/ngp-emu.md - - NicaiEmu: emulators/nicaiemu.md - - no$gba: emulators/nosgba.md - - NooDS: emulators/noods.md - - NP2kai: emulators/np2kai.md - - np2pi: emulators/np2pi.md - - nSide (SFC Balanced): emulators/higan_sfc_balanced.md - - Nuance: emulators/nuance.md - - Numero: emulators/numero.md - - NXEngine: emulators/nxengine.md - - O2EM: emulators/o2em.md - - Oberon: emulators/oberon.md - - Omicron: emulators/omicron.md - - ONScripter: emulators/onscripter.md - - ONScripter Yuri: emulators/onsyuri.md - - openbor: emulators/openbor.md - - OpenGOAL: emulators/opengoal.md - - OpenJazz: emulators/openjazz.md - - openjk: emulators/openjk.md - - OpenJKDF2: emulators/openjkdf2.md - - OpenLara: emulators/openlara.md - - OpenMoHAA: emulators/openmohaa.md - - openMSX: emulators/openmsx.md - - OpenTTD: emulators/openttd.md - - OpenTyrian: emulators/opentyrian.md - - Opera (4DO): emulators/opera.md - - Oricutron: emulators/oricutron.md - - Osmose (RetroPie): emulators/osmose.md + - M: + - M2000: emulators/m2000.md + - M64Plus FZ: emulators/m64plus-fz.md + - MagicEngine: emulators/magicengine.md + - MAME: emulators/mame.md + - MAME 0.258: emulators/mame0258.md + - MAME 0.278: emulators/mame0278.md + - MAME 2000: emulators/mame2000.md + - MAME 2003: emulators/mame2003.md + - MAME 2003 Midway: emulators/mame2003_midway.md + - MAME 2003-Plus: emulators/mame2003_plus.md + - MAME 2009: emulators/mame2009.md + - MAME 2010: emulators/mame2010.md + - MAME 2014: emulators/mame2014.md + - MAME 2015: emulators/mame2015.md + - MAME 2016: emulators/mame2016.md + - MAME Arcade: emulators/mamearcade.md + - MAME MESS: emulators/mamemess.md + - MAME4ALL-Pi: emulators/mame4all.md + - MAME4droid (0.139u1): emulators/mame4droid.md + - MAME4droid 2024: emulators/mame4droid-2024.md + - Mandarine: emulators/mandarine.md + - Mariani: emulators/mariani.md + - MasterGear: emulators/mastergear.md + - MD.emu: emulators/md-emu.md + - Mednafen: emulators/mednafen.md + - Mednafen STV: emulators/mednafen_stv.md + - mednafen_gba: emulators/mednafen_gba.md + - mednafen_pce: emulators/mednafen_pce.md + - mednafen_pce_fast: emulators/mednafen_pce_fast.md + - mednafen_snes: emulators/mednafen_snes.md + - mednafen_supafaust: emulators/mednafen_supafaust.md + - mednafen_supergrafx: emulators/mednafen_supergrafx.md + - melonDS: emulators/melonds.md + - melonDS DS: emulators/melonds_ds.md + - melonDS Nightly: emulators/melonds-nightly.md + - Mesen: emulators/mesen.md + - mesen-s: emulators/mesen-s.md + - MesenCE: emulators/mesence.md + - MESS 2015: emulators/mess2015.md + - Meteor GBA: emulators/meteor.md + - MFME: emulators/mfme.md + - mGBA: emulators/mgba.md + - mGBA Splitscreen: emulators/mgba_splitscreen.md + - Mini vMac: emulators/minivmac.md + - Mini vMac 37: emulators/minivmac37.md + - mkxp-z: emulators/mkxp_z.md + - Model 2 Emulator: emulators/model2.md + - MojoZork: emulators/mojozork.md + - MSX.emu: emulators/msx-emu.md + - Mu: emulators/mu.md + - Mupen64Plus: emulators/mupen64plus.md + - Mupen64Plus (libretro, 2019): emulators/mupen64plus-libretro.md + - Mupen64Plus AE: emulators/mupen64plus-ae.md + - Mupen64Plus-Next: emulators/mupen64plus_next.md + - My Boy!: emulators/my-boy.md + - My OldBoy!: emulators/my-oldboy.md + - N: + - NanoBoyAdvance: emulators/nanoboyadvance.md + - Native32Emu: emulators/native32emu.md + - nekop2: emulators/nekop2.md + - NEO.emu: emulators/neo-emu.md + - NeoCD: emulators/neocd.md + - nes: emulators/nes.md + - NES.emu: emulators/nes-emu.md + - Nesoid: emulators/nesoid.md + - Nestopia UE: emulators/nestopia.md + - NetherSX2: emulators/nethersx2.md + - NetherSX2-Turnip: emulators/nethersx2-turnip.md + - NetherSX2-Turnip Classic: emulators/nethersx2-turnip-classic.md + - NGP.emu: emulators/ngp-emu.md + - NicaiEmu: emulators/nicaiemu.md + - no$gba: emulators/nosgba.md + - NooDS: emulators/noods.md + - NP2kai: emulators/np2kai.md + - np2pi: emulators/np2pi.md + - nSide (SFC Balanced): emulators/higan_sfc_balanced.md + - Nuance: emulators/nuance.md + - Numero: emulators/numero.md + - NXEngine: emulators/nxengine.md + - O: + - O2EM: emulators/o2em.md + - Oberon: emulators/oberon.md + - Omicron: emulators/omicron.md + - ONScripter: emulators/onscripter.md + - ONScripter Yuri: emulators/onsyuri.md + - openbor: emulators/openbor.md + - OpenGOAL: emulators/opengoal.md + - OpenJazz: emulators/openjazz.md + - openjk: emulators/openjk.md + - OpenJKDF2: emulators/openjkdf2.md + - OpenLara: emulators/openlara.md + - OpenMoHAA: emulators/openmohaa.md + - openMSX: emulators/openmsx.md + - OpenTTD: emulators/openttd.md + - OpenTyrian: emulators/opentyrian.md + - Opera (4DO): emulators/opera.md + - Oricutron: emulators/oricutron.md + - Osmose (RetroPie): emulators/osmose.md - P to R: - - P/EMU/SDL: emulators/piemu.md - - Panda3DS: emulators/panda3ds.md - - Parallel Launcher: emulators/parallel-launcher.md - - ParaLLEl N64: emulators/parallel_n64.md - - pce-emu: emulators/pce-emu.md - - PCEE2: emulators/pcee2.md - - PCem: emulators/pcem.md - - PCSX-ReARMed: emulators/pcsx_rearmed.md - - PCSX-ReARMed (standalone): emulators/pcsx_rearmed_standalone.md - - PCSX1: emulators/pcsx1.md - - PCSX2: emulators/pcsx2.md - - PCSX2 Legacy: emulators/pcsx2-legacy.md - - PCSX2x6: emulators/pcsx2x6.md - - PD777: emulators/pd777.md - - Perfect Dark (PC port): emulators/pdark.md - - Phoenix: emulators/phoenix.md - - Picocomputer 6502: emulators/rp6502.md - - PicoDrive: emulators/picodrive.md - - piece-emu: emulators/piece.md - - piFBA (fba2x): emulators/fba2x.md - - PiSNES: emulators/pisnes.md - - Pizza Boy GBA: emulators/pizza-boy-gba.md - - Pizza Boy GBC: emulators/pizza-boy-gbc.md - - Pizza Boy SC: emulators/pizza-boy-sc.md - - Plastic: emulators/plastic.md - - play: emulators/play.md - - PlaydiaEmu: emulators/playdiaemu.md - - PokeMini: emulators/pokemini.md - - pokketstation: emulators/pokketstation.md - - POM2: emulators/pom2.md - - Potator: emulators/potator.md - - PPSSPP: emulators/ppsspp.md - - PPSSPP 1.5.4: emulators/ppsspp-1.5.4.md - - PrBoom: emulators/prboom.md - - PrBoom+: emulators/prboom-plus.md - - PrimeHack: emulators/primehack.md - - project64: emulators/project64.md - - ProSystem: emulators/prosystem.md - - pSxMAME: emulators/psxmame.md - - PUAE (P-UAE): emulators/puae.md - - PUAE 2021: emulators/puae2021.md - - puNES: emulators/punes.md - - px68k: emulators/px68k.md - - px68k (SDL): emulators/px68k-sdl.md - - QEMU: emulators/qemu.md - - Quake 3 (Raspberry Pi): emulators/quake3.md - - QUASI88: emulators/quasi88.md - - QUASI88kai: emulators/quasi88kai.md - - QuickNES: emulators/quicknes.md - - RACE (Neo Geo Pocket): emulators/race.md - - radio: emulators/radio.md - - RAINE: emulators/raine.md - - Raze: emulators/raze.md - - RB5000: emulators/rb5000.md - - Real3DOPlayer: emulators/real3doplayer.md - - RealRTCW: emulators/realrtcw.md - - Redream: emulators/redream.md - - Reicast: emulators/reicast.md - - REminiscence: emulators/reminiscence.md - - Retro8: emulators/retro8.md - - RetroDream: emulators/retrodream.md - - RigelEngine: emulators/rigelengine.md - - Rosalie's Mupen GUI: emulators/rosalies-mupen-gui.md - - RPCS3: emulators/rpcs3.md - - rpix86: emulators/rpix86.md - - ruffle: emulators/ruffle.md - - Rust-DOS: emulators/rust_dos.md - - Rustation: emulators/rustation.md - - RustyNES: emulators/rustynes.md - - RVVM: emulators/rvvm.md - - Ryujinx: emulators/ryujinx.md + - P: + - P/EMU/SDL: emulators/piemu.md + - Panda3DS: emulators/panda3ds.md + - Parallel Launcher: emulators/parallel-launcher.md + - ParaLLEl N64: emulators/parallel_n64.md + - pce-emu: emulators/pce-emu.md + - PCEE2: emulators/pcee2.md + - PCem: emulators/pcem.md + - PCSX-ReARMed: emulators/pcsx_rearmed.md + - PCSX-ReARMed (standalone): emulators/pcsx_rearmed_standalone.md + - PCSX1: emulators/pcsx1.md + - PCSX2: emulators/pcsx2.md + - PCSX2 Legacy: emulators/pcsx2-legacy.md + - PCSX2x6: emulators/pcsx2x6.md + - PD777: emulators/pd777.md + - Perfect Dark (PC port): emulators/pdark.md + - Phoenix: emulators/phoenix.md + - Picocomputer 6502: emulators/rp6502.md + - PicoDrive: emulators/picodrive.md + - piece-emu: emulators/piece.md + - piFBA (fba2x): emulators/fba2x.md + - PiSNES: emulators/pisnes.md + - Pizza Boy GBA: emulators/pizza-boy-gba.md + - Pizza Boy GBC: emulators/pizza-boy-gbc.md + - Pizza Boy SC: emulators/pizza-boy-sc.md + - Plastic: emulators/plastic.md + - play: emulators/play.md + - PlaydiaEmu: emulators/playdiaemu.md + - PokeMini: emulators/pokemini.md + - pokketstation: emulators/pokketstation.md + - POM2: emulators/pom2.md + - Potator: emulators/potator.md + - PPSSPP: emulators/ppsspp.md + - PPSSPP 1.5.4: emulators/ppsspp-1.5.4.md + - PrBoom: emulators/prboom.md + - PrBoom+: emulators/prboom-plus.md + - PrimeHack: emulators/primehack.md + - project64: emulators/project64.md + - ProSystem: emulators/prosystem.md + - pSxMAME: emulators/psxmame.md + - PUAE (P-UAE): emulators/puae.md + - PUAE 2021: emulators/puae2021.md + - puNES: emulators/punes.md + - px68k: emulators/px68k.md + - px68k (SDL): emulators/px68k-sdl.md + - Q: + - QEMU: emulators/qemu.md + - Quake 3 (Raspberry Pi): emulators/quake3.md + - QUASI88: emulators/quasi88.md + - QUASI88kai: emulators/quasi88kai.md + - QuickNES: emulators/quicknes.md + - R: + - RACE (Neo Geo Pocket): emulators/race.md + - radio: emulators/radio.md + - RAINE: emulators/raine.md + - Raze: emulators/raze.md + - RB5000: emulators/rb5000.md + - Real3DOPlayer: emulators/real3doplayer.md + - RealRTCW: emulators/realrtcw.md + - Redream: emulators/redream.md + - Reicast: emulators/reicast.md + - REminiscence: emulators/reminiscence.md + - Retro8: emulators/retro8.md + - RetroDream: emulators/retrodream.md + - RigelEngine: emulators/rigelengine.md + - Rosalie's Mupen GUI: emulators/rosalies-mupen-gui.md + - RPCS3: emulators/rpcs3.md + - rpix86: emulators/rpix86.md + - ruffle: emulators/ruffle.md + - Rust-DOS: emulators/rust_dos.md + - Rustation: emulators/rustation.md + - RustyNES: emulators/rustynes.md + - RVVM: emulators/rvvm.md + - Ryujinx: emulators/ryujinx.md - S to U: - - SAME CDi: emulators/same_cdi.md - - SameBoy: emulators/sameboy.md - - sameduck: emulators/sameduck.md - - Saturn.emu: emulators/saturn-emu.md - - ScummVM: emulators/scummvm.md - - ScummVM (SDL1): emulators/scummvm-sdl1.md - - SDL2TRS: emulators/sdl2trs.md - - SDLPAL: emulators/sdlpal.md - - sdlpop: emulators/sdlpop.md - - shadps4: emulators/shadps4.md - - SheepShaver: emulators/sheepshaver.md - - Ship of Harkinian: emulators/soh.md - - SimCoupe: emulators/simcp.md - - SimCoupe (standalone): emulators/samcoupe.md - - simple64: emulators/simple64.md - - Singe 2: emulators/singe2.md - - sixtyforce: emulators/sixtyforce.md - - skyemu: emulators/skyemu.md - - Skyline: emulators/skyline.md - - SMS Plus GX: emulators/smsplus.md - - snes9x: emulators/snes9x.md - - Snes9x 2002: emulators/snes9x2002.md - - Snes9x 2005: emulators/snes9x2005.md - - Snes9x 2005 Plus: emulators/snes9x2005_plus.md - - Snes9x 2010: emulators/snes9x2010.md - - Snes9x EX+: emulators/snes9x-explus.md - - solarus: emulators/solarus.md - - Solarus 2.0.0: emulators/solarus-2.0.0.md - - Sonic 1 & 2 (2013) Decompilation: emulators/sonic2013.md - - Sonic 3 A.I.R.: emulators/sonic3-air.md - - Sonic CD Decompilation: emulators/soniccd.md - - Sonic Mania Decompilation: emulators/sonic-mania.md - - Sonic Robo Blast 2: emulators/srb2.md - - Speccy: emulators/speccy.md - - SplitWolf: emulators/splitwolf.md - - SPMP8000Emu: emulators/spmp8000emu.md - - SquirrelJME: emulators/squirreljme.md - - SSF: emulators/ssf.md - - Starboard: emulators/starboard.md - - Starship: emulators/starship.md - - Stella: emulators/stella.md - - Stella 2014: emulators/stella2014.md - - Stella 2023: emulators/stella2023.md - - Sudachi: emulators/sudachi.md - - Sugarbox2: emulators/sugarbox2.md - - Super Mario War: emulators/supermariowar.md - - SUPER ZSNES: emulators/superzsnes.md - - SUPER3: emulators/super3.md - - Supermodel: emulators/supermodel.md - - Supermodel (Sinden fork): emulators/supermodel3.md - - Supermodel Dojo: emulators/supermodel-dojo.md - - Supermodel Legacy: emulators/supermodel-legacy.md - - SuperSnes9x: emulators/supersnes9x.md - - SuperTux: emulators/supertux.md - - SuperTuxKart: emulators/supertuxkart.md - - Suyu: emulators/suyu.md - - Swan.emu: emulators/swan-emu.md - - swanstation: emulators/swanstation.md - - SWF Player: emulators/swf-player.md - - Syobon Action: emulators/syobonaction.md - - TamaLIBretro: emulators/tamalibretro.md - - Taradino: emulators/taradino.md - - TeknoParrot: emulators/teknoparrot.md - - TempGBA: emulators/tempgba.md - - TGB Dual: emulators/tgbdual.md - - The Force Engine: emulators/theforceengine.md - - Theodore: emulators/theodore.md - - TheXTech: emulators/thextech.md - - ti99sim: emulators/ti99sim.md - - ti99sim-sdl1: emulators/ti99sim-sdl1.md - - Tia: emulators/tia.md - - TIC-80: emulators/tic80.md - - touchHLE: emulators/touchhle.md - - TR1X: emulators/tr1x.md - - TR2X: emulators/tr2x.md - - trident: emulators/trident.md - - Triforce: emulators/triforce.md - - tsugaru: emulators/tsugaru.md - - TyrQuake: emulators/tyrquake.md - - UAE4ALL (libretro, Chips-fr): emulators/uae4all-rpi.md - - UAE4All2: emulators/uae4all.md - - uae4arm: emulators/uae4arm.md - - UME 2015: emulators/ume2015.md - - uqm: emulators/uqm.md - - UZDoom: emulators/uzdoom.md - - uzem: emulators/uzem.md + - S: + - SAME CDi: emulators/same_cdi.md + - SameBoy: emulators/sameboy.md + - sameduck: emulators/sameduck.md + - Saturn.emu: emulators/saturn-emu.md + - ScummVM: emulators/scummvm.md + - ScummVM (SDL1): emulators/scummvm-sdl1.md + - SDL2TRS: emulators/sdl2trs.md + - SDLPAL: emulators/sdlpal.md + - sdlpop: emulators/sdlpop.md + - shadps4: emulators/shadps4.md + - SheepShaver: emulators/sheepshaver.md + - Ship of Harkinian: emulators/soh.md + - SimCoupe: emulators/simcp.md + - SimCoupe (standalone): emulators/samcoupe.md + - simple64: emulators/simple64.md + - Singe 2: emulators/singe2.md + - sixtyforce: emulators/sixtyforce.md + - skyemu: emulators/skyemu.md + - Skyline: emulators/skyline.md + - SMS Plus GX: emulators/smsplus.md + - snes9x: emulators/snes9x.md + - Snes9x 2002: emulators/snes9x2002.md + - Snes9x 2005: emulators/snes9x2005.md + - Snes9x 2005 Plus: emulators/snes9x2005_plus.md + - Snes9x 2010: emulators/snes9x2010.md + - Snes9x EX+: emulators/snes9x-explus.md + - solarus: emulators/solarus.md + - Solarus 2.0.0: emulators/solarus-2.0.0.md + - Sonic 1 & 2 (2013) Decompilation: emulators/sonic2013.md + - Sonic 3 A.I.R.: emulators/sonic3-air.md + - Sonic CD Decompilation: emulators/soniccd.md + - Sonic Mania Decompilation: emulators/sonic-mania.md + - Sonic Robo Blast 2: emulators/srb2.md + - Speccy: emulators/speccy.md + - SplitWolf: emulators/splitwolf.md + - SPMP8000Emu: emulators/spmp8000emu.md + - SquirrelJME: emulators/squirreljme.md + - SSF: emulators/ssf.md + - Starboard: emulators/starboard.md + - Starship: emulators/starship.md + - Stella: emulators/stella.md + - Stella 2014: emulators/stella2014.md + - Stella 2023: emulators/stella2023.md + - Sudachi: emulators/sudachi.md + - Sugarbox2: emulators/sugarbox2.md + - Super Mario War: emulators/supermariowar.md + - SUPER ZSNES: emulators/superzsnes.md + - SUPER3: emulators/super3.md + - Supermodel: emulators/supermodel.md + - Supermodel (Sinden fork): emulators/supermodel3.md + - Supermodel Dojo: emulators/supermodel-dojo.md + - Supermodel Legacy: emulators/supermodel-legacy.md + - SuperSnes9x: emulators/supersnes9x.md + - SuperTux: emulators/supertux.md + - SuperTuxKart: emulators/supertuxkart.md + - Suyu: emulators/suyu.md + - Swan.emu: emulators/swan-emu.md + - swanstation: emulators/swanstation.md + - SWF Player: emulators/swf-player.md + - Syobon Action: emulators/syobonaction.md + - T: + - TamaLIBretro: emulators/tamalibretro.md + - Taradino: emulators/taradino.md + - TeknoParrot: emulators/teknoparrot.md + - TempGBA: emulators/tempgba.md + - TGB Dual: emulators/tgbdual.md + - The Force Engine: emulators/theforceengine.md + - Theodore: emulators/theodore.md + - TheXTech: emulators/thextech.md + - ti99sim: emulators/ti99sim.md + - ti99sim-sdl1: emulators/ti99sim-sdl1.md + - Tia: emulators/tia.md + - TIC-80: emulators/tic80.md + - touchHLE: emulators/touchhle.md + - TR1X: emulators/tr1x.md + - TR2X: emulators/tr2x.md + - trident: emulators/trident.md + - Triforce: emulators/triforce.md + - tsugaru: emulators/tsugaru.md + - TyrQuake: emulators/tyrquake.md + - U: + - UAE4ALL (libretro, Chips-fr): emulators/uae4all-rpi.md + - UAE4All2: emulators/uae4all.md + - uae4arm: emulators/uae4arm.md + - UME 2015: emulators/ume2015.md + - uqm: emulators/uqm.md + - UZDoom: emulators/uzdoom.md + - uzem: emulators/uzem.md - V to Z: - - VaporSpec: emulators/vaporspec.md - - VBA-M: emulators/vbam.md - - VBA-Next: emulators/vba_next.md - - vecx: emulators/vecx.md - - veesem: emulators/veesem.md - - VeMUlator: emulators/vemulator.md - - VICE: emulators/vice.md - - VICE x128: emulators/vice_x128.md - - VICE x64: emulators/vice_x64.md - - VICE x64dtv: emulators/vice_x64dtv.md - - VICE x64sc: emulators/vice_x64sc.md - - VICE xcbm2: emulators/vice_xcbm2.md - - VICE xcbm5x0: emulators/vice_xcbm5x0.md - - VICE xpet: emulators/vice_xpet.md - - VICE xplus4: emulators/vice_xplus4.md - - VICE xscpu64: emulators/vice_xscpu64.md - - VICE xvic: emulators/vice_xvic.md - - Vircon32: emulators/vircon32.md - - Virtual Jaguar: emulators/virtualjaguar.md - - Virtual Virtual Boy: emulators/virtual-virtual-boy.md - - VirtualXT: emulators/virtualxt.md - - visual-pinball: emulators/visual-pinball.md - - Vita3K: emulators/vita3k.md - - vitaQuakeII: emulators/vitaquake2.md - - vitaQuakeIII: emulators/vitaquake3.md - - vitavoyager: emulators/vitavoyager.md - - vkQuake: emulators/vkquake.md - - vkQuake2: emulators/vkquake2.md - - vkQuake3: emulators/vkquake3.md - - VVVVVV: emulators/vvvvvv.md - - WinUAE: emulators/winuae.md - - Wolf4SDL: emulators/wolf4sdl.md - - WQXEmu: emulators/wqxemu.md - - X Millennium: emulators/x1.md - - X1 BOX: emulators/x1-box.md - - x16emu: emulators/x16emu.md - - x64sdl: emulators/x64sdl.md - - Xash3D FWGS: emulators/xash3d_fwgs.md - - xemu: emulators/xemu.md - - xemu Chihiro: emulators/xemu-chihiro.md - - XenDroid: emulators/xendroid.md - - Xenia Canary: emulators/xenia.md - - Xenia Edge: emulators/xenia-edge.md - - XeniOS: emulators/xenios.md - - xm6pro68k: emulators/xm6pro68k.md - - xm6typeg: emulators/xm6typeg.md - - XM7: emulators/xm7.md - - XRick: emulators/xrick.md - - XRoar: emulators/xroar.md - - yabasanshiro: emulators/yabasanshiro.md - - yabause: emulators/yabause.md - - Yamagi Quake II: emulators/yquake2.md - - yaps2: emulators/yaps2.md - - ymir: emulators/ymir.md - - Yuzu: emulators/yuzu.md - - Zelda Classic 2.55 Prototype: emulators/zc255.md - - Zelda Classic v2.10: emulators/zc210.md - - ZEsarUX: emulators/zesarux.md - - ZiNc: emulators/zinc.md - - ZQuest Classic: emulators/zquestclassic.md + - V: + - VaporSpec: emulators/vaporspec.md + - VBA-M: emulators/vbam.md + - VBA-Next: emulators/vba_next.md + - vecx: emulators/vecx.md + - veesem: emulators/veesem.md + - VeMUlator: emulators/vemulator.md + - VICE: emulators/vice.md + - VICE x128: emulators/vice_x128.md + - VICE x64: emulators/vice_x64.md + - VICE x64dtv: emulators/vice_x64dtv.md + - VICE x64sc: emulators/vice_x64sc.md + - VICE xcbm2: emulators/vice_xcbm2.md + - VICE xcbm5x0: emulators/vice_xcbm5x0.md + - VICE xpet: emulators/vice_xpet.md + - VICE xplus4: emulators/vice_xplus4.md + - VICE xscpu64: emulators/vice_xscpu64.md + - VICE xvic: emulators/vice_xvic.md + - Vircon32: emulators/vircon32.md + - Virtual Jaguar: emulators/virtualjaguar.md + - Virtual Virtual Boy: emulators/virtual-virtual-boy.md + - VirtualXT: emulators/virtualxt.md + - visual-pinball: emulators/visual-pinball.md + - Vita3K: emulators/vita3k.md + - vitaQuakeII: emulators/vitaquake2.md + - vitaQuakeIII: emulators/vitaquake3.md + - vitavoyager: emulators/vitavoyager.md + - vkQuake: emulators/vkquake.md + - vkQuake2: emulators/vkquake2.md + - vkQuake3: emulators/vkquake3.md + - VVVVVV: emulators/vvvvvv.md + - W: + - WinUAE: emulators/winuae.md + - Wolf4SDL: emulators/wolf4sdl.md + - WQXEmu: emulators/wqxemu.md + - X: + - X Millennium: emulators/x1.md + - X1 BOX: emulators/x1-box.md + - x16emu: emulators/x16emu.md + - x64sdl: emulators/x64sdl.md + - Xash3D FWGS: emulators/xash3d_fwgs.md + - xemu: emulators/xemu.md + - xemu Chihiro: emulators/xemu-chihiro.md + - XenDroid: emulators/xendroid.md + - Xenia Canary: emulators/xenia.md + - Xenia Edge: emulators/xenia-edge.md + - XeniOS: emulators/xenios.md + - xm6pro68k: emulators/xm6pro68k.md + - xm6typeg: emulators/xm6typeg.md + - XM7: emulators/xm7.md + - XRick: emulators/xrick.md + - XRoar: emulators/xroar.md + - Y: + - yabasanshiro: emulators/yabasanshiro.md + - yabause: emulators/yabause.md + - Yamagi Quake II: emulators/yquake2.md + - yaps2: emulators/yaps2.md + - ymir: emulators/ymir.md + - Yuzu: emulators/yuzu.md + - Z: + - Zelda Classic 2.55 Prototype: emulators/zc255.md + - Zelda Classic v2.10: emulators/zc210.md + - ZEsarUX: emulators/zesarux.md + - ZiNc: emulators/zinc.md + - ZQuest Classic: emulators/zquestclassic.md - All installer options: wiki/installer.md - Reference: - wiki/index.md diff --git a/scripts/generate_site.py b/scripts/generate_site.py index b94d3779..d12f9e1d 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1430,6 +1430,8 @@ def _browser_title(relative: Path, title: str) -> str: } if key in index_titles: return index_titles[key] + if key in PART_PAGES: + return title if key.startswith("emulators/"): return f"{title} emulator firmware" if key.startswith("systems/"): @@ -1474,6 +1476,9 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: } if key in specific: return specific[key] + if key in PART_PAGES: + return (f"{title}: every file with its hashes, size, source references " + "and who reads it.") if key.startswith("platforms/") and relative.stem != "index": return (f"{title}: where the BIOS files go, the command that installs them, the pack, " "and every file by system with its hashes.") @@ -1510,9 +1515,13 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: # Structured data a page generator knows beyond its text, by page path # relative to docs/: read by decorate_markdown_pages. PAGE_DATA: dict[str, dict] = {} +# Pages that hold part of a page too long for one (a console of a split +# manufacturer, a run of an emulator's cards), by path relative to docs/: +# their H1 already says what they are. +PART_PAGES: set[str] = set() -SYSTEM_PARTS_IN_FULL = 1000 +SYSTEM_PARTS_IN_FULL = 250 def collection_dat(db: dict, generated_at: str) -> str: @@ -1601,8 +1610,9 @@ def system_structured_data(slug: str, consoles: dict[str, list]) -> dict: part["citation"] = citations parts.append(part) if len(parts) > SYSTEM_PARTS_IN_FULL: - # game-engines would carry 2.6 MB of JSON-LD: past the cap a page - # lists its catalogue-verified files and points at the full table. + # A page of cards carried up to 2.6 MB of JSON-LD (game-engines), a + # quarter of its weight: past the cap a page lists its + # catalogue-verified files and points at the full table. return { "hasPart": [part for part in parts if part.get("citation")], "isBasedOn": urllib.parse.urljoin(SITE_URL, "downloads/files.csv"), @@ -1824,6 +1834,11 @@ def emulator_structured_data(profile: dict) -> dict: if members: part["hasPart"] = members parts.append(part) + if len(parts) > SYSTEM_PARTS_IN_FULL: + # MAME's 1,564 entries and their members were 614 KB of JSON-LD: past + # the cap the members are left to the profile's JSON. + for part in parts: + part.pop("hasPart", None) if parts: data["hasPart"] = parts return data @@ -2044,14 +2059,23 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st return "\n".join(lines) + "\n" +# A platform system whose file list passes this much Markdown gets a page +# of its own: RetroDECK's arcade list was 750 KB of a 1.1 MB page. +PLATFORM_SECTION_BUDGET = 200_000 + + def _render_platform_systems( - by_system: dict, config_files: dict, emulator_files: dict + by_system: dict, config_files: dict, emulator_files: dict, + platform: tuple[str, str] | None = None, moved: dict[str, str] | None = None, ) -> list[str]: """One collapsible block per system, listing every file it needs. Split out of generate_platform_page: the block is most of that function's branching, since each file renders a different set of hashes, sizes and provenance depending on what the platform declares. + With ``platform`` (its key and name) and ``moved``, a list longer than + PLATFORM_SECTION_BUDGET goes to a page of its own, written into + ``moved`` by stem, and the system's heading stays with a link to it. """ lines: list[str] = [] # Per-system detail sections (collapsible for large platforms) @@ -2154,6 +2178,26 @@ def _render_platform_systems( if f.get("category_evidence"): lines.append(f"{pad} {_category_evidence_md(f['category_evidence'])}") + block = lines[block_start:] + if moved is not None and platform and sum(len(line) + 1 for line in block) > PLATFORM_SECTION_BUDGET: + key, display = platform + stem = f"{key}-{anchor}" + body = block[2:] if use_collapsible else block + moved[stem] = "\n".join([ + f"# {display}: {sys_id}", "", + f"Part of the [{display} page]({key}.md#{anchor}), whose other systems " + f"are listed there. Each file below is as {display} declares it, with " + "its status in the pack.", + "", "## Files", "", + *search_excluded( + [line[len(pad):] if line.startswith(pad) else line for line in body]), + "", f"*Generated on {_timestamp()}*", + ]) + "\n" + lines[block_start:] = [ + f"{_count(total, 'file')}, {ok_count} verified: listed on " + f"[a page of their own]({stem}.md).", "", + ] + continue # The section's heading stays searchable, its file list with every # hash is the lookup's: retrodeck's arcade list was 495 KB of index. lines[block_start:] = search_excluded(lines[block_start:]) @@ -2386,9 +2430,13 @@ def generate_platform_page( cov: dict, registry: dict | None = None, emulator_files: dict | None = None, + moved: dict[str, str] | None = None, ) -> str: """A guide first (folder, command, pack, check), then every file the - platform's list and its emulators need, under a reference heading.""" + platform's list and its emulators need, under a reference heading. + + ``moved`` collects the pages of systems whose list is too long for it. + """ registry = registry or {} config = cov["config"] display = cov["platform"] @@ -2485,7 +2533,8 @@ def generate_platform_page( "", "", ] - lines.extend([*band, "| | |", "|---|---|"]) + lines.extend([*band, '
', "", + "| Detail | Value |", "|---|---|"]) if hash_type: lines.append(f"| Hash type | {hash_type} |") if version: @@ -2519,6 +2568,7 @@ def generate_platform_page( f"| Contributed by | [@{username}](https://github.com/{username}):" f" {contribution}{pr_link} |" ) + lines.extend(["", "
"]) if name not in GUIDES: detect = (registry.get(name, {}).get("install") or {}).get("detect") or [] on_card = any(rule.get("method") == "card_marker" for rule in detect) @@ -2576,7 +2626,8 @@ def generate_platform_page( lines.append("") lines.extend( - _render_platform_systems(by_system, config_files, emulator_files) + _render_platform_systems(by_system, config_files, emulator_files, + (name, display), moved) ) lines.append(f"*Generated on {_timestamp()}*") @@ -2701,8 +2752,8 @@ def file_location_items( parts = entry.get("path", "").split("/") if len(parts) < 3 or parts[0] != "bios": continue - slug = parts[1].lower().replace(" ", "-") - items.append({"sha1": sha1, "url": f"systems/{slug}/#sha1-{sha1[:12]}"}) + page = card_page(entry.get("path", ""), sha1) + items.append({"sha1": sha1, "url": f"systems/{page}/#sha1-{sha1[:12]}"}) # A data directory's files are listed on the page of an emulator reading # the tree: 2,534 hashes the packs ship were found nowhere on the site. owners = _data_member_owners(profiles or {}) @@ -2772,7 +2823,7 @@ def file_lookup_records( continue entries = profile.get("files") or [] for entry, anchor in zip(entries, emulator_file_anchors(entries)): - url = f"emulators/{key}/#{anchor}" + url = f"emulators/{emulator_card_page(key, anchor)}/#{anchor}" members = [(str(entry.get("name", "")), entry)] + [ (str(member["name"]), member) for member in entry.get("contents") or [] if isinstance(member, dict) and member.get("name") @@ -3207,8 +3258,9 @@ def _emulator_links(sha1: str, consumers: FileConsumers) -> list[str]: index = entries.get(emulator) anchor = (_emulator_card_anchor(emulator, files_of[emulator], index) if index is not None and emulator in files_of else "") + page = emulator_card_page(emulator, anchor) if anchor else emulator fragment = f"#{anchor}" if anchor else "" - links.append(f"[{emulator}](../emulators/{emulator}.md{fragment})") + links.append(f"[{emulator}](../emulators/{page}.md{fragment})") return links @@ -3379,14 +3431,16 @@ def _card_title(text: str, anchor: str, searchable: bool = True) -> str: """A file card's heading, so the site search lands on the card. The search splits a page at its headings: a card titled in bold sent a - pasted md5 to the top of a page of a hundred cards. Level 4 stays out - of the table of contents (toc_depth 3), and a card without an anchor - keeps the id the toc gives it. + pasted md5 to the top of a page of a hundred cards. Level 3, under + the console or the Files heading: a level 4 after a level 2 skipped + one for screen readers on 551 pages. The table of contents of a page + of cards stops above it (mkdocs_hooks.py), and a card without an + anchor keeps the id the toc gives it. """ attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title" if not searchable: attributes += " data-search-exclude" - return f"#### {text} {{ {attributes} }}" + return f"### {text} {{ {attributes} }}" def system_install_choices(coverages: dict) -> dict[str, dict[str, list[tuple[str, str]]]]: @@ -3418,6 +3472,197 @@ def system_install_choices(coverages: dict) -> dict[str, dict[str, list[tuple[st return choices +# A manufacturer page above this much Markdown becomes a summary with one +# page per console, and a console above it is cut into parts. 500 KB of +# Markdown renders to about 750 KB of HTML: game-engines was one 10.5 MB +# page that took 21 s of main-thread work on a phone. +SYSTEM_PAGE_BUDGET = 500_000 + +# A system card: whether it is a variant, its lines, its sha1. +Card = tuple[bool, list[str], str] + + +def _console_cards( + manufacturer: str, + files: list[dict], + consumers: FileConsumers, + anchored: set[str], +) -> list[Card]: + """Each card of one console, primaries by name then variants, with + whether it is a variant.""" + searchable = manufacturer not in GAME_DATA_TOPS + main_files = [f for f in files if "/.variants/" not in f["path"]] + variant_files = [f for f in files if "/.variants/" in f["path"]] + cards = [(False, _system_file_card(f, consumers, anchored, searchable), str(f.get("sha1", ""))) + for f in sorted(main_files, key=lambda x: x["name"])] + cards += [(True, _system_file_card(v, consumers, anchored, searchable), str(v.get("sha1", ""))) + for v in sorted(variant_files, key=lambda x: (x["name"], x["path"]))] + return cards + + +def _cards_markdown(cards: list[Card]) -> list[str]: + """Primaries, then the variants in a closed block.""" + lines = [line for variant, card, _sha1 in cards if not variant for line in card] + variants = [card for variant, card, _sha1 in cards if variant] + if variants: + # A variant gets the same card as a primary: the revision a + # platform pins is the one a reader comes to find, and showed as + # a name and a truncated MD5, nothing said who pins it. + lines.append(f'??? note "Variants ({len(variants)})"') + lines.append("") + for card in variants: + lines.extend(f" {line}" if line else "" for line in card) + lines.append("") + return lines + + +def _parts(cards: list[Card], budget: int) -> list[list[Card]]: + """Cards in runs of at most ``budget`` bytes of Markdown, in order.""" + parts: list[list[Card]] = [[]] + size = 0 + for card in cards: + weight = sum(len(line) + 1 for line in card[1]) + if parts[-1] and size + weight > budget: + parts.append([]) + size = 0 + parts[-1].append(card) + size += weight + return parts + + +_NEXT_FROM_SYSTEMS = [ + "[Get your files](../which-pack.md), for the setup you play on", + "[When a game does not start](../wiki/troubleshooting.md)", + "[Every emulator](../emulators/index.md), and the files each one reads", +] + + +def system_pages( + manufacturer: str, + consoles: dict[str, list], + consumers: FileConsumers, + installs: dict[str, list[tuple[str, str]]] | None = None, + budget: int = SYSTEM_PAGE_BUDGET, +) -> dict[str, tuple[str, dict[str, list]]]: + """A manufacturer's pages by stem under systems/: one page, or a summary + and a page per console when the whole would pass ``budget``. Each page + comes with the files whose cards it holds, by console. + + Every console keeps its heading on the summary, so a link to + systems//# still lands, and site.js sends a card + anchor the page no longer holds to the card's new place. The pages + each card and console went to are recorded for every other link + (siterender.record_console_pages). + """ + slug = _slug(manufacturer) + json_link = (f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json)") + whole_anchors: set[str] = set() + sections = { + console: _console_cards(manufacturer, consoles[console], consumers, whole_anchors) + for console in sorted(consoles) + } + lines = [ + f"# {manufacturer}", "", + f"The files below by console, with every hash, are also published as {json_link}.", + "", + ] + for console, cards in sections.items(): + lines.append(f"## {system_icon_markdown(manufacturer, console)}{console}") + lines.append("") + lines.extend(_console_install_block(console, (installs or {}).get(console, []))) + lines.extend(_cards_markdown(cards)) + lines.append("") + lines.extend(_next_block(_NEXT_FROM_SYSTEMS)) + lines.append(f'
Generated on {_timestamp()}.
') + whole = "\n".join(lines) + "\n" + if len(whole) <= budget: + return {slug: (whole, consoles)} + + # The heaviest sections move to pages of their own until what stays on + # the manufacturer's page is half the budget: a section of one file + # keeps its place, and a reader does not open 91 pages to browse Other. + weights = { + console: sum(len(line) + 1 for _variant, card, _sha1 in cards for line in card) + for console, cards in sections.items() + } + moved: set[str] = set() + for console in sorted(weights, key=lambda c: (-weights[c], c)): + if sum(w for c, w in weights.items() if c not in moved) <= budget // 2: + break + moved.add(console) + + pages: dict[str, tuple[str, dict[str, list]]] = {} + summary = [ + f"# {manufacturer}", "", + f"The files below by console, with every hash, are also published as {json_link}. " + f"The largest sections have a page of their own, linked from their heading.", + "", + ] + summary_anchors: set[str] = set() + for console in sorted(consoles): + summary.append(f"## {system_icon_markdown(manufacturer, console)}{console}") + summary.append("") + if console not in moved: + summary.extend(_console_install_block(console, (installs or {}).get(console, []))) + summary.extend(_cards_markdown( + _console_cards(manufacturer, consoles[console], consumers, summary_anchors))) + summary.append("") + continue + # Cards are rendered again for their own page: the anchors a page + # holds are its own. + cards = _console_cards(manufacturer, consoles[console], consumers, set()) + parts = _parts(cards, budget) + base = f"{slug}-{_slugify_anchor(console) or 'files'}" + stems = [base if index == 0 else f"{base}-{index + 1}" for index in range(len(parts))] + record_console_pages(slug, console, stems, [ + [sha1 for _variant, _card, sha1 in part] for part in parts + ]) + title = f"{manufacturer}: {console}" + count = _count(len(consoles[console]), "file") + if len(stems) == 1: + summary.append(f"{count}, on [their own page]({stems[0]}.md).") + else: + summary.append(f"{count}, on {len(stems)} pages by file name:") + summary.append("") + for index, (stem, part) in enumerate(zip(stems, parts)): + summary.append(f"- [Part {index + 1}]({stem}.md): {_part_range(part)}") + summary.append("") + for index, (stem, part) in enumerate(zip(stems, parts)): + heading = title if len(stems) == 1 else f"{title}, part {index + 1} of {len(stems)}" + lines = [ + f"# {heading}", "", + f"Part of [{manufacturer}]({slug}.md), in [Systems](index.md). " + f"Every hash of these files is also published as {json_link}.", + "", + ] + if len(stems) > 1: + others = ", ".join( + f"[part {i + 1}]({other}.md)" for i, other in enumerate(stems) if i != index) + lines.extend([f"This part holds {_part_range(part)}. The others: {others}.", ""]) + lines.extend(_console_install_block(console, (installs or {}).get(console, []))) + lines.extend(["## Files", ""]) + lines.extend(_cards_markdown(part)) + lines.append("") + lines.extend(_next_block([f"[{manufacturer}]({slug}.md), every section"] + + _NEXT_FROM_SYSTEMS)) + lines.append(f'
Generated on {_timestamp()}.
') + held = {sha1 for _variant, _card, sha1 in part} + pages[stem] = ("\n".join(lines) + "\n", {console: [ + f for f in consoles[console] if str(f.get("sha1", "")) in held]}) + summary.extend(_next_block(_NEXT_FROM_SYSTEMS)) + summary.append(f'
Generated on {_timestamp()}.
') + kept = {console: files for console, files in consoles.items() if console not in moved} + return {slug: ("\n".join(summary) + "\n", kept), **pages} + + +def _part_range(part: list[Card]) -> str: + """The first and last file names of a part, as the cards show them.""" + names = [re.search(r"`([^`]*)`", card[2]).group(1) for _variant, card, _sha1 in part] + if len(names) == 1: + return f"`{names[0]}`" + return f"{_count(len(names), 'file')}, `{names[0]}` to `{names[-1]}`" + + def generate_system_page( manufacturer: str, consoles: dict[str, list], @@ -3427,53 +3672,10 @@ def generate_system_page( """One manufacturer's page, a section per console. ``installs`` gives, per console, the (platform, system id) pairs whose - install command the section offers. + install command the section offers. A page over the budget is the + summary system_pages writes, its consoles on pages of their own. """ - slug = manufacturer.lower().replace(" ", "-") - lines = [ - f"# {manufacturer}", - "", - f"The files below by console, with every hash, are also published as " - f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json).", - "", - ] - anchored: set[str] = set() - searchable = manufacturer not in GAME_DATA_TOPS - - for console_name in sorted(consoles.keys()): - files = consoles[console_name] - icon_md = system_icon_markdown(manufacturer, console_name) - lines.append(f"## {icon_md}{console_name}") - lines.append("") - lines.extend(_console_install_block(console_name, (installs or {}).get(console_name, []))) - main_files = [f for f in files if "/.variants/" not in f["path"]] - variant_files = [f for f in files if "/.variants/" in f["path"]] - - for f in sorted(main_files, key=lambda x: x["name"]): - lines.extend(_system_file_card(f, consumers, anchored, searchable)) - - if variant_files: - # A variant gets the same card as a primary: the revision a - # platform pins is the one a reader comes to find, and showed as - # a name and a truncated MD5, nothing said who pins it. - lines.append(f'??? note "Variants ({len(variant_files)})"') - lines.append("") - for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])): - lines.extend( - f" {line}" if line else "" - for line in _system_file_card(v, consumers, anchored, searchable) - ) - lines.append("") - - lines.append("") - - lines.extend(_next_block([ - "[Get your files](../which-pack.md), for the setup you play on", - "[When a game does not start](../wiki/troubleshooting.md)", - "[Every emulator](../emulators/index.md), and the files each one reads", - ])) - lines.append(f'
Generated on {_timestamp()}.
') - return "\n".join(lines) + "\n" + return system_pages(manufacturer, consoles, consumers, installs)[_slug(manufacturer)][0] # Emulator pages @@ -3914,7 +4116,7 @@ def _render_emulator_file( # Which copy answers this entry, linked to its collection card # (provenance, every hash, download): an entry declaring no hash # said "in repo" without naming the file. - page = parts[1].lower().replace(" ", "-") + page = card_page(held, sha1) details.append( f"Collection copy: [{_code(held)}]" f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" @@ -3988,7 +4190,7 @@ def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str: archive = _code(path) zsha1 = consumers.sha1_at(path) if zsha1 and len(parts) >= 3 and parts[0] == "bios": - page = urllib.parse.quote(parts[1].lower().replace(" ", "-")) + page = urllib.parse.quote(card_page(path, zsha1)) archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})" return f"Collection copy: member {_code(member)} of {archive}" if parts[0] == "data" and len(parts) >= 2: @@ -4020,8 +4222,7 @@ def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]: parts = folder.split("/") where = _code(folder + "/") if len(parts) >= 3 and parts[0] == "bios": - page = urllib.parse.quote(parts[1].lower().replace(" ", "-")) - where = f"[{where}](../systems/{page}.md#{_slugify_anchor(parts[2])})" + where = f"[{where}]({console_href(_slug(parts[1]), parts[2], '../')})" tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(folder)}" return [f"Collection copies: {_count(len(members), 'file')} under {where} " f"([browse]({tree}))"] @@ -4369,6 +4570,41 @@ _DATA_MEMBERS: dict[str, dict[str, dict]] = {} _DATA_REVISIONS: dict[str, dict] = {} +# An emulator page lists at most this many file cards: MAME's 1,564 made a +# 3 MB page. Past it, the cards go to pages of their own and the emulator's +# page keeps a table of every entry, each row holding the card's old +# anchor. A count, not a size: system cards link to these cards and are +# written before them. +EMULATOR_CARDS_PER_PAGE = 400 + + +def emulator_parts(name: str, files: list) -> list[tuple[str, range]]: + """The pages an emulator's cards go to, with the entries each holds, + or none when they fit on the emulator's page. Balanced, in profile order.""" + if len(files) <= EMULATOR_CARDS_PER_PAGE: + return [] + count = -(-len(files) // EMULATOR_CARDS_PER_PAGE) + size = -(-len(files) // count) + return [(f"{name}-files-{index + 1}", range(start, min(start + size, len(files)))) + for index, start in enumerate(range(0, len(files), size))] + + +def plan_emulator_pages(profiles: dict) -> None: + """Record which page holds each emulator card, before any page links one.""" + stems = set() + for name, profile in sorted(profiles.items()): + if profile.get("type") in ("alias", "test"): + continue + files = profile.get("files") or [] + anchors = emulator_file_anchors(files) + for stem, entries in emulator_parts(name, files): + stems.add(stem) + record_emulator_cards(name, stem, [anchors[i] for i in entries]) + clash = stems & set(profiles) + if clash: + raise RuntimeError(f"an emulator's card page takes a profile's address: {sorted(clash)}") + + def generate_emulator_page( name: str, profile: dict, @@ -4379,13 +4615,28 @@ def generate_emulator_page( data_dirs: list[dict] | None = None, ) -> str: """One emulator's page. ``data_dirs`` defaults to the profile's own.""" + return emulator_pages( + name, profile, db, consumers, data_names, data_registry, data_dirs)[name] + + +def emulator_pages( + name: str, + profile: dict, + db: dict, + consumers: FileConsumers | None = None, + data_names: set[str] | None = None, + data_registry: dict | None = None, + data_dirs: list[dict] | None = None, +) -> dict[str, str]: + """An emulator's page by stem under emulators/, and the pages its cards + went to when there are more than EMULATOR_CARDS_PER_PAGE.""" if profile.get("type") == "alias": parent = profile.get("alias_of", profile.get("bios_identical_to", "unknown")) - return ( + return {name: ( f"# {name}\n\n" f"This core uses the same firmware as **{parent}**.\n\n" f"See [{parent}]({parent}.md) for details.\n" - ) + )} emu_name = profile.get("emulator", name) emu_type = profile.get("type", "unknown") @@ -4419,12 +4670,13 @@ def generate_emulator_page( if data_dirs is None: data_dirs = profile.get("data_directories", []) + part_pages: dict[str, str] = {} lines = [ f"# {emu_name}", "", '
', "", - "| | |", + "| Detail | Value |", "|---|---|", f"| Type | {emu_type} |", ] @@ -4522,16 +4774,25 @@ def generate_emulator_page( else: _file_available = _availability_check(db, data_names) + lines.extend(["## Files", ""]) lines.extend(_emulator_file_summary(files, _file_available)) anchors = emulator_file_anchors(files) - for f, anchor, alias in zip(files, anchors, emulator_file_aliases(files, anchors)): - lines.extend( - _render_emulator_file( - f, profile, consumers, files, _file_available, anchor, name, - alias, - ) - ) + aliases = emulator_file_aliases(files, anchors) + cards = [ + _render_emulator_file(f, profile, consumers, files, _file_available, anchor, name, alias) + for f, anchor, alias in zip(files, anchors, aliases) + ] + parts = emulator_parts(name, files) + if not parts: + for card in cards: + lines.extend(card) + else: + lines.extend(_card_index(emu_name, files, anchors, aliases, parts)) + for number, (stem, entries) in enumerate(parts, 1): + part_pages[stem] = _emulator_part_page( + name, emu_name, number, parts, [cards[i] for i in entries], + [files[i] for i in entries]) if data_dirs: # The member tables are a file list with hashes: out of the search, @@ -4593,6 +4854,63 @@ def generate_emulator_page( "[When a game does not start](../wiki/troubleshooting.md)", ])) lines.extend([f"*Generated on {_timestamp()}*"]) + return {name: "\n".join(lines) + "\n", **part_pages} + + +def _card_index( + emu_name: str, files: list[dict], anchors: list[str], aliases: list[str], + parts: list[tuple[str, range]], +) -> list[str]: + """Every entry of a split emulator page, one row each, linked to its card. + + Each row holds the ids the card had on this page, so a link cited + before the split lands on the row, and site.js follows it to the card. + """ + page_of = {index: stem for stem, entries in parts for index in entries} + lines = [ + f"{emu_name} reads more files than one page holds: each entry below " + f"links to its card, on {_count(len(parts), 'page')} by profile order " + + ", ".join(f"[{number}]({stem}.md)" for number, (stem, _e) in enumerate(parts, 1)) + + ".", + "", + '
', + "", + "| File | Required | Category |", + "|------|----------|----------|", + ] + for index, (f, anchor, alias) in enumerate(zip(files, anchors, aliases)): + ids = "".join(f'' for a in (anchor, alias) if a) + shown = str(f.get("path") or f.get("name", "")) + lines.append( + f"| {ids}[{_code(shown)}]({page_of[index]}.md#{anchor}) " + f"| {'yes' if f.get('required') else 'no'} | {f.get('category', 'bios')} |" + ) + lines.extend(["", "
", ""]) + return lines + + +def _emulator_part_page( + name: str, emu_name: str, number: int, parts: list[tuple[str, range]], + cards: list[list[str]], files: list[dict], +) -> str: + """One page of a split emulator's cards.""" + first = str(files[0].get("path") or files[0].get("name", "")) + last = str(files[-1].get("path") or files[-1].get("name", "")) + others = ", ".join( + f"[{n}]({stem}.md)" for n, (stem, _e) in enumerate(parts, 1) if n != number) + lines = [ + f"# {emu_name} files, part {number} of {len(parts)}", + "", + f"Part of [{emu_name}]({name}.md), whose page has the profile, the install " + f"command and every entry. This part holds {_count(len(files), 'entry', 'entries')}, " + f"{_code(first)} to {_code(last)}, in the profile's order. The other parts: {others}.", + "", + "## Files", + "", + *(line for card in cards for line in card), + ] + lines.extend(_next_block([f"[{emu_name}]({name}.md), the profile and every entry"])) + lines.extend([f"*Generated on {_timestamp()}*"]) return "\n".join(lines) + "\n" @@ -5947,19 +6265,35 @@ def nav_emulators(profiles: dict) -> dict[str, dict]: def _emulator_nav(profiles: dict) -> list: - """Emulators in alphabetical groups ("A to C"), by the name a user knows.""" - groups: dict[str, list[tuple[str, str]]] = {} + """Emulators in alphabetical groups ("A to C"), then by initial. + + Material writes the open section of the navigation into every page: a + group of 96 emulators put 65 KB of links before each emulator page's + content. One initial is at most 60. + """ + groups: dict[str, dict[str, list[tuple[str, str]]]] = {} for key, profile in nav_emulators(profiles).items(): display = str(profile.get("emulator", key)) initial = display[:1].upper() letters = next((g for g in EMULATOR_NAV_GROUPS if initial in g), "") label = f"{letters[0]} to {letters[-1]}" if letters else "0 to 9" - groups.setdefault(label, []).append((display, f"emulators/{key}.md")) + sub = initial if letters else label + groups.setdefault(label, {}).setdefault(sub, []).append((display, f"emulators/{key}.md")) + + def entries(items: list[tuple[str, str]]) -> list[dict]: + return [{d: path} for d, path in sorted(items, key=lambda e: e[0].casefold())] + order = ["0 to 9"] + [f"{g[0]} to {g[-1]}" for g in EMULATOR_NAV_GROUPS] - return ["emulators/index.md"] + [ - {label: [{d: path} for d, path in sorted(groups[label], key=lambda e: e[0].casefold())]} - for label in order if label in groups - ] + nav: list = ["emulators/index.md"] + for label in order: + if label not in groups: + continue + subs = groups[label] + if len(subs) == 1: + nav.append({label: entries(next(iter(subs.values())))}) + else: + nav.append({label: [{sub: entries(subs[sub])} for sub in sorted(subs)]}) + return nav def _system_nav(manufacturers: dict) -> list: @@ -6174,14 +6508,8 @@ def _generate(args: argparse.Namespace) -> None: # report the same files. gap_report = build_emulator_gap_report(profiles, coverages, db, suppl_names) - print("Generating static API and bulk data exports...") - exports = generate_data_exports( - docs, db, coverages, profiles, stats, gap_report - ) - write_if_changed(str(docs / "data.md"), generate_data_page(exports) + REFERENCE_NEXT) - PAGE_DATA["data.md"] = dataset_structured_data(exports, db) - # Build system_id -> manufacturer page map (needed by all generators) + plan_emulator_pages(profiles) print("Building system cross-reference map...") manufacturers = _group_by_manufacturer(db) set_system_page_map( @@ -6189,25 +6517,6 @@ def _generate(args: argparse.Namespace) -> None: ) print(f" {len(_system_page_map)} system IDs mapped to pages") - export_reports = load_export_reports() - _EXPORTED.update(report["platform"] for report in export_reports) - - print("Generating platform pages...") - write_if_changed( - str(docs / "platforms" / "index.md"), - generate_platform_index(coverages, registry), - ) - for name, cov in coverages.items(): - # The emulators this platform runs, not every one of the system. - runs = resolve_platform_cores(cov["config"], profiles) - write_if_changed( - str(docs / "platforms" / f"{name}.md"), - generate_platform_page(name, cov, registry, { - emu: data for emu, data in emulator_files.items() if emu in runs - }), - ) - PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov) - print("Generating system pages...") prime_system_icons({ @@ -6222,16 +6531,55 @@ def _generate(args: argparse.Namespace) -> None: console_installs = system_install_choices(coverages) (docs / "files").mkdir(parents=True, exist_ok=True) write_if_changed(str(docs / "files" / "index.md"), generate_files_page()) + reset_console_pages() for mfr, consoles in manufacturers.items(): slug = mfr.lower().replace(" ", "-") - page = generate_system_page(mfr, consoles, consumers, console_installs.get(slug)) - PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles) + pages = system_pages(mfr, consoles, consumers, console_installs.get(slug)) + for stem, (page, held) in pages.items(): + PAGE_DATA[f"systems/{stem}.md"] = system_structured_data(stem, held) + if stem != slug: + PART_PAGES.add(f"systems/{stem}.md") + write_if_changed(str(docs / "systems" / f"{stem}.md"), page) write_single_item( docs / "api" / "v1" / "systems" / f"{slug}.json", "systems", system_export_item(slug, mfr, consoles), str(db.get("generated_at") or _timestamp()), ) - write_if_changed(str(docs / "systems" / f"{slug}.md"), page) + clash = split_system_stems() & {_slug(m) for m in manufacturers} + if clash: + raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}") + + print("Generating static API and bulk data exports...") + exports = generate_data_exports( + docs, db, coverages, profiles, stats, gap_report + ) + write_if_changed(str(docs / "data.md"), generate_data_page(exports) + REFERENCE_NEXT) + PAGE_DATA["data.md"] = dataset_structured_data(exports, db) + + export_reports = load_export_reports() + _EXPORTED.update(report["platform"] for report in export_reports) + + print("Generating platform pages...") + write_if_changed( + str(docs / "platforms" / "index.md"), + generate_platform_index(coverages, registry), + ) + for name, cov in coverages.items(): + # The emulators this platform runs, not every one of the system. + runs = resolve_platform_cores(cov["config"], profiles) + moved: dict[str, str] = {} + write_if_changed( + str(docs / "platforms" / f"{name}.md"), + generate_platform_page(name, cov, registry, { + emu: data for emu, data in emulator_files.items() if emu in runs + }, moved), + ) + for stem, page in moved.items(): + if stem in coverages: + raise RuntimeError(f"a system page takes the platform {stem}'s address") + PART_PAGES.add(f"platforms/{stem}.md") + write_if_changed(str(docs / "platforms" / f"{stem}.md"), page) + PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov) print("Generating emulator pages...") _DATA_MEMBERS.update(data_directory_members()) @@ -6264,12 +6612,18 @@ def _generate(args: argparse.Namespace) -> None: indent=2, sort_keys=True, ensure_ascii=False, default=str, ) + "\n", ) - page = generate_emulator_page( + pages = emulator_pages( name, profile, db, consumers, suppl_names, data_registry, data_dirs=page_data_directories(name, profiles), ) PAGE_DATA[f"emulators/{name}.md"] = emulator_structured_data(profile) - write_if_changed(str(docs / "emulators" / f"{name}.md"), page) + files = profile.get("files") or [] + for stem, entries in emulator_parts(name, files): + PAGE_DATA[f"emulators/{stem}.md"] = emulator_structured_data( + {**profile, "files": [files[i] for i in entries]}) + PART_PAGES.add(f"emulators/{stem}.md") + for stem, page in pages.items(): + write_if_changed(str(docs / "emulators" / f"{stem}.md"), page) print("Generating cross-reference page...") write_if_changed( @@ -6323,6 +6677,11 @@ def _generate(args: argparse.Namespace) -> None: print("Generating contributing page...") write_if_changed(str(docs / "contributing.md"), generate_contributing()) + # Swept before the decoration, which rewrites every page it finds: a + # page no generator produced any more (a console moved back onto its + # manufacturer's page) was decorated, recorded, and kept for ever. + stale = _sweep_generated(docs, GENERATED_DIRS) + print("Adding page metadata and structured data...") decorate_markdown_pages(docs) @@ -6421,19 +6780,17 @@ markdown_extensions: - tables - toc: permalink: true - # File cards are level 4 headings: the search lands on them, and a - # page of a hundred cards keeps a readable table of contents. + # Level 3 is the wiki's subsections. On pages of file cards, also level + # 3, the hook keeps the table of contents to level 2. toc_depth: 3 - pymdownx.details - pymdownx.highlight: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.keys -- pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format +# No mermaid fence: Material drew it in the browser with a script from a +# CDN. Diagrams are SVG files written by scripts/sitediagrams.py. +- pymdownx.superfences - pymdownx.tabbed: alternate_style: true plugins: @@ -6449,26 +6806,11 @@ validation: config_path, config_text = _mkdocs_config(Path(docs), mkdocs_static + nav_yaml) write_if_changed(str(config_path), config_text) - total_pages = ( - 1 # home - + 1 - + len(coverages) # platform index + detail - + 1 - + len(manufacturers) # system index + detail - + 1 # cross-reference - + 1 - + sum( - 1 for profile in profiles.values() - if profile.get("type") not in ("alias", "test") - ) # emulator detail pages (aliases/tests remain metadata-only) - + 1 # gap analysis - + 1 # which-pack - + len(list(Path("wiki").glob("*.md"))) # wiki pages copied verbatim - + 1 # generated wiki/data-model - + 1 # contributing - + 1 # data and API + # Counted on disk: the split pages made a sum of page kinds wrong. + total_pages = sum( + 1 for path in docs.rglob("*.md") + if not any(part in LOCAL_NOTE_DIRS for part in path.relative_to(docs).parts[:-1]) ) - stale = _sweep_generated(docs, GENERATED_DIRS) print(f"\nGenerated {total_pages} pages in {args.docs_dir}/") if stale: print(f"Removed {stale} page(s) nothing produced this run") @@ -6500,6 +6842,13 @@ from siterender import ( # noqa: E402,F401 _system_page_map, _system_link, set_system_page_map, + card_page, + console_href, + record_console_pages, + record_emulator_cards, + emulator_card_page, + reset_console_pages, + split_system_stems, _platform_link, _emulator_link, _fmt_size, diff --git a/scripts/mkdocs_hooks.py b/scripts/mkdocs_hooks.py index 0028a3e1..11cef35d 100644 --- a/scripts/mkdocs_hooks.py +++ b/scripts/mkdocs_hooks.py @@ -10,8 +10,28 @@ from __future__ import annotations from typing import Any +# Pages made of file cards, level 3 headings by the hundred: their table of +# contents stops at level 2, the consoles of a system page, the Files of an +# emulator, the sections of a platform page (which listed every system id). +CARD_PAGES = ("systems/", "emulators/", "platforms/") + def on_page_markdown(markdown: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001 """Take lastmod from the page's own date, or leave it out.""" page.update_date = str(page.meta.get("date_modified") or "") return markdown + + +def _prune(items: list, depth: int) -> None: + for item in items: + if item.level >= depth: + item.children = [] + else: + _prune(item.children, depth) + + +def on_page_content(html: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001 + """Keep the table of contents of a page of cards to level 2.""" + if page.file.src_uri.startswith(CARD_PAGES): + _prune(page.toc.items, 2) + return html diff --git a/scripts/siterender.py b/scripts/siterender.py index f0b1cbc1..0683ad43 100644 --- a/scripts/siterender.py +++ b/scripts/siterender.py @@ -139,13 +139,75 @@ def _page_by_norm(sys_id: str) -> tuple[str, str] | None: return _norm_page_map.get(_norm_system_id(sys_id)) +# A manufacturer page too large for one page is a summary, each console on +# pages of its own: (slug, console) -> its page stems under systems/, and +# (slug, console, sha1) -> the stem holding that card. A console of a page +# kept whole has no entry, its section is on the manufacturer's page. +_console_pages: dict[tuple[str, str], list[str]] = {} +_card_pages: dict[tuple[str, str, str], str] = {} + + +def reset_console_pages() -> None: + _console_pages.clear() + _card_pages.clear() + + +def record_console_pages( + slug: str, console: str, stems: list[str], sha1s: list[list[str]] +) -> None: + """Where a split manufacturer page put one console: its page stems, in + order, and the sha1 of each card on each of them.""" + _console_pages[(slug, console)] = list(stems) + for stem, part in zip(stems, sha1s): + for sha1 in part: + _card_pages.setdefault((slug, console, sha1), stem) + + +def split_system_stems() -> set[str]: + """Every page stem of a console moved to a page of its own.""" + return {stem for stems in _console_pages.values() for stem in stems} + + +def console_href(slug: str, console: str, prefix: str = "") -> str: + """Where a console's files are: its own page, or its section of the + manufacturer's page (also where a console cut into parts lists them).""" + stems = _console_pages.get((slug, console)) + if stems and len(stems) == 1: + return f"{prefix}systems/{urllib.parse.quote(stems[0])}.md" + return f"{prefix}systems/{urllib.parse.quote(slug)}.md#{_slugify_anchor(console)}" + + +def card_page(path: str, sha1: str) -> str: + """The systems/ page stem holding the card of the copy at ``path`` + (bios///...).""" + parts = path.split("/") + slug = parts[1].lower().replace(" ", "-") + console = parts[2] if len(parts) > 2 else "" + return _card_pages.get((slug, console, sha1), slug) + + +# Emulator cards moved off a page too long for one: (emulator, anchor) -> +# the page stem under emulators/ that holds the card. +_emulator_card_pages: dict[tuple[str, str], str] = {} + + +def record_emulator_cards(emulator: str, stem: str, anchors: list[str]) -> None: + for anchor in anchors: + _emulator_card_pages[(emulator, anchor)] = stem + + +def emulator_card_page(emulator: str, anchor: str) -> str: + """The emulators/ page stem holding a card: the emulator's own page, or + the page its cards went to.""" + return _emulator_card_pages.get((emulator, anchor), emulator) + + def _system_link(sys_id: str, prefix: str = "") -> str: """Generate a markdown link to a system page with anchor.""" page = _system_page_map.get(sys_id) or _page_by_norm(sys_id) if page: slug, console = page - anchor = _slugify_anchor(console) - return f"[{sys_id}]({prefix}systems/{slug}.md#{anchor})" + return f"[{sys_id}]({console_href(slug, console, prefix)})" return sys_id def _field_text(key: object, value: object) -> str: diff --git a/tests/test_site_render.py b/tests/test_site_render.py index 8188ece8..a5b4d6f0 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -323,7 +323,7 @@ class TheSearchLandsOnTheCard(unittest.TestCase): entry = {"name": "scph5501.bin", "required": True} lines = generate_site._render_emulator_file( entry, {"files": [entry]}, None, [entry], lambda f: True, "file-scph5501-bin") - self.assertTrue(lines[2].startswith("#### `scph5501.bin` ")) + self.assertTrue(lines[2].startswith("### `scph5501.bin` ")) self.assertTrue(lines[2].endswith("{ #file-scph5501-bin .rb-file-title }")) def test_a_system_card_is_a_heading_with_its_anchor(self): @@ -332,7 +332,7 @@ class TheSearchLandsOnTheCard(unittest.TestCase): entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3} lines = generate_site._system_file_card(entry, FileConsumers({"files": {}})) - self.assertTrue(lines[2].startswith("#### `a.bin` ")) + self.assertTrue(lines[2].startswith("### `a.bin` ")) self.assertTrue(lines[2].endswith("{ #sha1-aaaaaaaaaaaa .rb-file-title }")) diff --git a/tests/test_site_weight.py b/tests/test_site_weight.py new file mode 100644 index 00000000..a524d7ed --- /dev/null +++ b/tests/test_site_weight.py @@ -0,0 +1,168 @@ +"""Pages a phone can open: split pages, light navigation, short tables of contents. + +systems/game-engines/ was one 10.5 MB page, emulators/mame/ 3 MB, and each +of 600 emulator pages carried 65 KB of navigation. A large page is now a +summary with pages of its own, and every address cited before the split +still lands: a console's heading stays on the summary, a card's sha1 anchor +is looked up by content, an emulator card's anchor sits on its row. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site # noqa: E402 +import mkdocs_hooks # noqa: E402 +import siterender # noqa: E402 +from cross_reference import FileConsumers # noqa: E402 + + +def _file(name: str, index: int, console: str, maker: str = "Maker") -> dict: + sha1 = f"{index:012x}" + "0" * 28 + return {"name": name, "sha1": sha1, "md5": f"{index:032x}", "size": 10, + "path": f"bios/{maker}/{console}/{name}"} + + +class ASystemPageOverItsBudgetIsSplit(unittest.TestCase): + def setUp(self) -> None: + siterender.reset_console_pages() + self.addCleanup(siterender.reset_console_pages) + self.consoles = { + "Big": [_file(f"big{i:03d}.bin", i, "Big") for i in range(60)], + "Small": [_file("small.bin", 900, "Small")], + } + + def _pages(self, budget: int) -> dict: + return generate_site.system_pages( + "Maker", self.consoles, FileConsumers({"files": {}}), budget=budget) + + def test_a_page_within_its_budget_stays_whole(self): + pages = self._pages(10_000_000) + self.assertEqual(list(pages), ["maker"]) + self.assertIn("## Big", pages["maker"][0]) + + def test_the_largest_section_moves_and_is_cut_in_parts(self): + pages = self._pages(20_000) + summary = pages["maker"][0] + # Both headings stay on the summary: systems/maker/#big still lands. + self.assertIn("## Big", summary) + self.assertIn("## Small", summary) + # The small section keeps its cards in place. + self.assertIn("{ #sha1-" + f"{900:012x}", summary) + parts = [stem for stem in pages if stem.startswith("maker-big")] + self.assertGreater(len(parts), 1) + self.assertEqual(parts[0], "maker-big") + for stem in parts: + self.assertIn(f"]({stem}.md)", summary) + self.assertLessEqual(len(pages[stem][0]), 20_000 + 5_000) + self.assertIn("## Files", pages[stem][0]) + # Each card is on exactly one part, and the links follow it. + held = [f["sha1"] for stem in parts for f in pages[stem][1]["Big"]] + self.assertEqual(sorted(held), sorted(f["sha1"] for f in self.consoles["Big"])) + last = self.consoles["Big"][-1] + self.assertEqual(siterender.card_page(last["path"], last["sha1"]), parts[-1]) + small = self.consoles["Small"][0] + self.assertEqual(siterender.card_page(small["path"], small["sha1"]), "maker") + self.assertEqual(siterender.console_href("maker", "Big"), "systems/maker.md#big") + + def test_a_console_on_one_page_is_linked_directly(self): + self.consoles["Big"] = self.consoles["Big"][:12] + whole = self._pages(10**9)["maker"][0] + pages = self._pages(len(whole) - 1) + self.assertIn("maker-big", pages) + self.assertNotIn("maker-big-2", pages) + self.assertEqual(siterender.console_href("maker", "Big", "../"), + "../systems/maker-big.md") + + +class AnEmulatorWithTooManyCardsIsSplit(unittest.TestCase): + def test_parts_are_balanced_and_rows_keep_the_anchors(self): + files = [{"name": f"set{i:04d}.zip", "required": i % 2 == 0, "category": "bios_zip"} + for i in range(generate_site.EMULATOR_CARDS_PER_PAGE * 2 + 1)] + parts = generate_site.emulator_parts("big", files) + self.assertEqual([stem for stem, _ in parts], ["big-files-1", "big-files-2", "big-files-3"]) + sizes = [len(entries) for _, entries in parts] + self.assertLessEqual(max(sizes) - min(sizes), 1) + self.assertEqual(generate_site.emulator_parts("small", files[:3]), []) + + pages = generate_site.emulator_pages( + "big", {"emulator": "Big", "type": "standalone", "files": files}, {"files": {}}) + self.assertEqual(sorted(pages), ["big", "big-files-1", "big-files-2", "big-files-3"]) + main = pages["big"] + anchors = generate_site.emulator_file_anchors(files) + for anchor in anchors: + self.assertIn(f'', main) + self.assertIn('class="rb-card-index"', main) + self.assertNotIn(".rb-file-title", main) + self.assertIn(f"{{ #{anchors[0]} .rb-file-title }}", pages["big-files-1"]) + + def test_cards_are_linked_on_the_page_that_holds_them(self): + files = [{"name": f"set{i:04d}.zip"} for i in range(generate_site.EMULATOR_CARDS_PER_PAGE + 1)] + generate_site.plan_emulator_pages({"big": {"files": files}}) + anchors = generate_site.emulator_file_anchors(files) + self.assertEqual(siterender.emulator_card_page("big", anchors[-1]), "big-files-2") + self.assertEqual(siterender.emulator_card_page("other", "file-x"), "other") + + +class APlatformSystemOverItsBudgetMoves(unittest.TestCase): + def test_the_heading_stays_and_the_list_moves(self): + files = [{"name": f"f{i:04d}.zip", "status": "ok"} for i in range(40)] + config = {("arcade", f["name"]): [{"name": f["name"], "md5": "a" * 32}] for f in files} + moved: dict[str, str] = {} + budget = generate_site.PLATFORM_SECTION_BUDGET + generate_site.PLATFORM_SECTION_BUDGET = 500 + try: + lines = generate_site._render_platform_systems( + {"arcade": files}, config, {}, ("deck", "Deck"), moved) + finally: + generate_site.PLATFORM_SECTION_BUDGET = budget + text = "\n".join(lines) + self.assertIn("{ #arcade }", text) + self.assertIn("](deck-arcade.md)", text) + self.assertNotIn("f0001.zip", text) + self.assertIn("f0001.zip", moved["deck-arcade"]) + self.assertIn("[Deck page](deck.md#arcade)", moved["deck-arcade"]) + + +class TheNavigationOpensOneInitial(unittest.TestCase): + def test_a_group_of_several_initials_is_nested(self): + profiles = { + key: {"emulator": name, "systems": ["s"]} + for key, name in (("a1", "Alpha"), ("b1", "Beta"), ("b2", "bravo"), ("z", "Zed")) + } + nav = generate_site._emulator_nav(profiles) + groups = {next(iter(g)): next(iter(g.values())) for g in nav[1:]} + self.assertEqual(groups["A to C"], [ + {"A": [{"Alpha": "emulators/a1.md"}]}, + {"B": [{"Beta": "emulators/b1.md"}, {"bravo": "emulators/b2.md"}]}, + ]) + self.assertEqual(groups["V to Z"], [{"Zed": "emulators/z.md"}]) + + +class TheTableOfContentsOfACardPageStopsAtLevelTwo(unittest.TestCase): + @staticmethod + def _item(level: int, children: list | None = None) -> SimpleNamespace: + return SimpleNamespace(level=level, children=children or []) + + def test_cards_leave_the_toc_and_wiki_pages_keep_level_three(self): + def page(uri: str) -> SimpleNamespace: + h2 = self._item(2, [self._item(3)]) + return SimpleNamespace(file=SimpleNamespace(src_uri=uri), + toc=SimpleNamespace(items=[self._item(1, [h2])])) + + for uri, kept in (("systems/sony.md", 0), ("platforms/retroarch.md", 0), + ("emulators/mame.md", 0), ("wiki/faq.md", 1)): + current = page(uri) + mkdocs_hooks.on_page_content("", current, None, None) + with self.subTest(uri=uri): + self.assertEqual(len(current.toc.items[0].children[0].children), kept) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 51bda600..f3ee044c 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -51,7 +51,7 @@ Every module in `tests/`, by the area it guards. | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture From be5be996dbd7b3af6db1578def99b622eb080f8f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:13:55 +0200 Subject: [PATCH 5/5] feat: hold the site to page and index budgets validate_site.py fails a page over 1 MB of HTML, a search index over 8.5 MB (1.7 MB gzip), a skipped heading level, a table header without text, a script from another host and a diagram drawn at runtime. The limits sit above what the site now measures (0.88 MB, 7.2 MB, 1.43 MB), each with its reason, and each has a test. --- scripts/validate_site.py | 78 ++++++++++++++++++++++++++++++++++- tests/test_site_validation.py | 61 +++++++++++++++++++++++++++ wiki/architecture.md | 34 +++++++++++---- wiki/release-process.md | 3 +- wiki/tools.md | 2 +- 5 files changed, 166 insertions(+), 12 deletions(-) diff --git a/scripts/validate_site.py b/scripts/validate_site.py index d7f95b7a..b39bf9ee 100644 --- a/scripts/validate_site.py +++ b/scripts/validate_site.py @@ -34,7 +34,25 @@ class Page: jsonld: list[str] = field(default_factory=list) unrendered: list[str] = field(default_factory=list) remote_images: list[str] = field(default_factory=list) + remote_scripts: list[str] = field(default_factory=list) placeholders: list[str] = field(default_factory=list) + headings: list[int] = field(default_factory=list) + empty_headers: int = 0 + runtime_diagrams: int = 0 + size: int = 0 + + +# Budgets the site was brought under on 2026-10-22, with room to grow. A +# change that passes one is a regression of what a visitor downloads, not a +# matter of taste: raise the number in the same change that explains why. +# +# The largest page was 10.5 MB (systems/game-engines/) and took 21 s of +# main-thread work on a throttled phone. Split pages now top out at 0.9 MB. +MAX_PAGE_BYTES = 1_000_000 +# The search index is downloaded by every visitor who opens the search: +# 21.0 MB raw and 4.88 MB gzip with hashes in it, 7.3 MB and 1.44 MB without. +MAX_SEARCH_INDEX_BYTES = 8_500_000 +MAX_SEARCH_INDEX_GZIP_BYTES = 1_700_000 # Text inside these is shown as written, so markup there is not a leftover. @@ -50,11 +68,21 @@ class _PageParser(HTMLParser): self._jsonld_depth = 0 self._jsonld_parts: list[str] = [] self._literal_depth = 0 + self._header_parts: list[str] | None = None def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - data = dict(attrs) + data = {key: value or "" for key, value in attrs} if tag in _LITERAL_TAGS: self._literal_depth += 1 + if re.fullmatch(r"h[1-6]", tag): + self.page.headings.append(int(tag[1])) + if tag == "th": + self._header_parts = [] + if "mermaid" in data.get("class", "").split(): + # Material renders it in the browser with a script from a CDN. + self.page.runtime_diagrams += 1 + if tag == "script" and re.match(r"\s*(?:https?:)?//", data.get("src", "")): + self.page.remote_scripts.append(data["src"]) if tag == "html": self.page.lang = data.get("lang", "") elif tag == "title": @@ -97,6 +125,10 @@ class _PageParser(HTMLParser): def handle_endtag(self, tag: str) -> None: if tag in _LITERAL_TAGS and self._literal_depth: self._literal_depth -= 1 + if tag == "th" and self._header_parts is not None: + if not "".join(self._header_parts).strip(): + self.page.empty_headers += 1 + self._header_parts = None if tag == "title": self._title_depth -= 1 elif tag == "script" and self._jsonld_depth: @@ -105,6 +137,8 @@ class _PageParser(HTMLParser): self._jsonld_parts = [] def handle_data(self, data: str) -> None: + if self._header_parts is not None: + self._header_parts.append(data) if self._title_depth: self._title_parts.append(data) if self._jsonld_depth: @@ -119,11 +153,39 @@ class _PageParser(HTMLParser): def _parse_page(path: Path) -> Page: parser = _PageParser(path) - parser.feed(path.read_text(encoding="utf-8", errors="replace")) + text = path.read_text(encoding="utf-8", errors="replace") + parser.feed(text) parser.close() + parser.page.size = len(text.encode("utf-8")) return parser.page +def _heading_jumps(levels: list[int]) -> list[str]: + """Each heading more than one level below the one before it: a screen + reader user moving by level finds nothing at the level skipped.""" + return [f"h{before} to h{after}" for before, after in zip(levels, levels[1:]) + if after > before + 1] + + +def _check_search_index(site: Path) -> list[str]: + """The index every visitor who opens the search downloads, against its budget.""" + import gzip + + path = site / "search" / "search_index.json" + if not path.is_file(): + return [] + raw = path.read_bytes() + packed = len(gzip.compress(raw, 6)) + issues = [] + if len(raw) > MAX_SEARCH_INDEX_BYTES: + issues.append(f"search/search_index.json: {len(raw):,} bytes, " + f"over the {MAX_SEARCH_INDEX_BYTES:,} budget") + if packed > MAX_SEARCH_INDEX_GZIP_BYTES: + issues.append(f"search/search_index.json: {packed:,} bytes gzip, " + f"over the {MAX_SEARCH_INDEX_GZIP_BYTES:,} budget") + return issues + + def _base_path(config_path: Path) -> str: """Read site_url without constructing executable YAML extension tags.""" config = config_path.read_text(encoding="utf-8") @@ -220,6 +282,17 @@ def _check_pages( report(path, f"images missing alt: {page.missing_alt[:5]}") if page.remote_images: report(path, f"images loaded from another host: {page.remote_images[:3]}") + if page.remote_scripts: + report(path, f"scripts loaded from another host: {page.remote_scripts[:3]}") + if page.runtime_diagrams: + report(path, f"{page.runtime_diagrams} diagram(s) drawn by a script at runtime") + if page.size > MAX_PAGE_BYTES: + report(path, f"{page.size:,} bytes of HTML, over the {MAX_PAGE_BYTES:,} budget") + jumps = _heading_jumps(page.headings) + if jumps: + report(path, f"heading levels skipped: {sorted(set(jumps))}") + if page.empty_headers: + report(path, f"{page.empty_headers} table header cell(s) without text") if page.placeholders: # scripts/sitediagrams.py knows no diagram of this name. report(path, f"image placeholders left unresolved: {page.placeholders[:3]}") @@ -294,6 +367,7 @@ def validate_site(site: Path, config_path: Path) -> list[str]: issues.append(f"duplicate {label} {value!r}: {rendered}") issues.extend(_check_links(pages, site, base_path)) + issues.extend(_check_search_index(site)) return issues diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py index 7ebe6ca5..4c897af3 100644 --- a/tests/test_site_validation.py +++ b/tests/test_site_validation.py @@ -166,6 +166,67 @@ class RenderedSiteValidation(unittest.TestCase): +class TheSiteStaysWithinItsBudgets(unittest.TestCase): + """The 10.5 MB page and the 21 MB search index passed every check: + validate_site now holds the gains, each limit with its reason.""" + + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory(dir=TMP_ROOT) + self.site = Path(self.temp.name) / "site" + self.site.mkdir() + self.config = Path(self.temp.name) / "mkdocs.yml" + self.config.write_text("site_url: https://example.test/retrobios/\n", encoding="utf-8") + + def tearDown(self) -> None: + self.temp.cleanup() + + def _issues(self, body: str) -> list[str]: + (self.site / "index.html").write_text(_page("Home", "Home page.", body), encoding="utf-8") + return validate_site(self.site, self.config) + + def test_a_clean_page_passes(self): + (self.site / "javascripts").mkdir() + (self.site / "javascripts" / "site.js").write_text("", encoding="utf-8") + self.assertEqual(self._issues( + "

A

B

C

" + "
DetailValue
" + ''), []) + + def test_a_page_over_its_size(self): + import validate_site as module + + issues = self._issues("

" + "x" * module.MAX_PAGE_BYTES + "

") + self.assertTrue(any("over the" in issue and "HTML" in issue for issue in issues), issues) + + def test_a_skipped_heading_level(self): + issues = self._issues("

A

B

") + self.assertTrue(any("h2 to h4" in issue for issue in issues), issues) + + def test_an_empty_table_header(self): + issues = self._issues("
") + self.assertTrue(any("2 table header cell(s) without text" in issue for issue in issues)) + + def test_a_script_or_a_diagram_from_another_host(self): + issues = self._issues( + '' + '
graph LR
') + self.assertTrue(any("scripts loaded from another host" in issue for issue in issues)) + self.assertTrue(any("drawn by a script at runtime" in issue for issue in issues)) + + def test_a_search_index_over_its_budget(self): + import random + + import validate_site as module + + (self.site / "search").mkdir() + rng = random.Random(1) + noise = "".join(rng.choice("0123456789abcdef") for _ in range(module.MAX_SEARCH_INDEX_BYTES + 1)) + (self.site / "search" / "search_index.json").write_text(noise, encoding="utf-8") + issues = self._issues("") + self.assertTrue(any("search_index.json" in issue and "gzip" not in issue for issue in issues)) + self.assertTrue(any("search_index.json" in issue and "gzip" in issue for issue in issues)) + + class UnrenderedMarkdown(unittest.TestCase): """The clownmdemu analysis block reached readers as "- **bios_hle:** True": a list indented one level too deep, read as code.""" diff --git a/wiki/architecture.md b/wiki/architecture.md index c88d6554..3ab42b66 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -71,14 +71,32 @@ dist/ generated packs (gitignored) Documentation changes belong in `wiki/` or in the generator. The site search splits each page at its headings and sends a result to the -heading above the match. Every file card is therefore a level 4 heading with -the card's anchor, and every platform system a heading of its own: a hash -pasted in the search opens that card or that system, not the top of the page. -Level 4 stays out of the table of contents (`toc_depth: 3`). The lines that -list other pages (platforms, packs, source citations, collection paths) sit -in a `data-search-exclude` block, since the browser downloads and indexes the -whole search index before it shows a first result. File names, hashes, the -paths an emulator reads and the notes stay searchable. +heading above the match. Every file card is therefore a level 3 heading with +the card's anchor, under its console or a Files heading, and every platform +system a heading of its own: a file name typed in the search opens that card, +not the top of the page. On pages of cards (systems, emulators, platforms) the +table of contents stops at level 2 (`scripts/mkdocs_hooks.py`). + +The browser downloads and indexes the whole search index before it shows a +first result, so the index holds names and prose only. Hashes, lists of other +pages, install commands, member tables and long reference tables sit in +`data-search-exclude` blocks. Those blocks are `section` elements: the search +plugin compares elements by tag, and a `div` stopped excluding at the first +`div` closed inside it. A hash goes to [Find a file](../files/index.md), which +reads one shard of `api/v1/lookup/` named by the first two characters of the +value. + +A page has a budget. A manufacturer page over 500 KB of Markdown becomes a +summary: its largest consoles move to pages of their own, cut into parts when +one console alone is over the budget, and every console keeps its heading on +the summary. An emulator with more than 400 entries (MAME) lists them in a +table on its page, each row holding the old anchor of the card, and the cards +move to pages of 400. A platform system whose list is over 200 KB moves to a +page of its own. A card anchor (`#sha1-...`) that a page no longer holds is +looked up by content, so a link cited before a split still opens the card. +`validate_site.py` fails a page over 1 MB of HTML, a search index over 8.5 MB +(1.7 MB compressed), a skipped heading level, a table header without text, a +script from another host and a diagram drawn at runtime. ## Data flow diff --git a/wiki/release-process.md b/wiki/release-process.md index 68c81fb6..cfe47134 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -54,7 +54,8 @@ is missing. 6. Run `generate_readme.py` (rebuilds README.md and CONTRIBUTING.md) 7. `mkdocs build --strict` to produce the static site 8. Run `validate_site.py` on the rendered HTML (metadata, headings, image - alternatives, duplicate ids, local links and fragments) + alternatives, duplicate ids, local links and fragments, page and search + index budgets) 9. Require the committed README and CONTRIBUTING to match what the generator just produced. `write_if_changed()` compares content with the timestamp line stripped, so a run that only moves the clock leaves the files untouched and diff --git a/wiki/tools.md b/wiki/tools.md index 7b9fab1d..793a6f2e 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -692,7 +692,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6 | `find_pinned_revision.py` | Walk the upstream commits that touched a file and return the revision whose bytes match a platform's pinned md5 (`--write` saves them) | | `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database | | `generate_site.py` | Generate all MkDocs site pages (this documentation) | -| `validate_site.py` | Validate rendered metadata, headings, image alternatives, JSON-LD, local resources, links and fragments | +| `validate_site.py` | Validate rendered metadata, heading order, image alternatives, JSON-LD, local resources, links and fragments, and the page size and search index budgets | | `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held | | `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch ` reads FBNeo at that tag, branch or commit | | `deterministic_zip.py` | Rebuild MAME BIOS ZIPs deterministically (same ROMs = same hash) |