From 8b26a8fcac62a75e14089d9911d568da83e1a0ad Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:58:51 +0200 Subject: [PATCH] feat: cite sources and files on reference pages Platform pages link each reader's source line at its pinned revision, name the collection file that serves each entry, show launcher records with their lines, name systems with the id beside them and the list they were read from in words. The largest system lists move to pages of their own so a platform page stays under 500 KB. Gaps: plain numbers with a bar, a badge only for missing required firmware, Files to source as a heading. Cross-reference: counts per platform, one page per platform, one row per upstream project. Exports: gaps listed once, status, category and the serving file in the CSVs and SQLite, the commit in each API envelope, the provenance Wanted list as CSV. Catalog dates in words on the provenance page. --- schemas/site-api-envelope.schema.json | 5 + scripts/generate_site.py | 832 +++++++++++++++++++------- scripts/sitesources.py | 96 +++ tests/test_site_cores.py | 11 +- tests/test_site_exports.py | 4 +- tests/test_site_platform_hashes.py | 6 +- tests/test_site_r2_reference.py | 357 +++++++++++ tests/test_site_render.py | 4 +- tests/test_site_validation.py | 8 +- wiki/testing-guide.md | 2 +- 10 files changed, 1114 insertions(+), 211 deletions(-) create mode 100644 tests/test_site_r2_reference.py diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index dcd422b2..584ee44a 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -19,6 +19,11 @@ "type": "string", "format": "date-time" }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "description": "The repository commit the data was read at." + }, "kind": { "type": "string", "enum": [ diff --git a/scripts/generate_site.py b/scripts/generate_site.py index f60c3d43..905edbc6 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -679,12 +679,47 @@ def _sha256_path(path: Path) -> str: return digest.hexdigest() -def _platform_export_rows(coverages: dict) -> tuple[list[dict], list[dict]]: +def _verify_outcomes(coverage: dict, sha1_by_path: dict[str, str]): + """For each declared entry, in declaration order: what verify found. + + Returns a function of (system, name) giving the next outcome for that + pair: status, category, the collection path that answers the entry and + that file's sha1. The CSV gave the declaration and nothing about it. + """ + details: dict[tuple[str, str], list[dict]] = {} + for detail in coverage.get("details") or []: + details.setdefault((detail.get("system", ""), detail.get("name", "")), []).append(detail) + taken: dict[tuple[str, str], int] = {} + + def outcome(system: str, name: str) -> dict: + found = details.get((system, name)) or [{}] + nth = taken.get((system, name), 0) + taken[(system, name)] = nth + 1 + detail = found[min(nth, len(found) - 1)] + path = str(detail.get("path") or "") if detail.get("status") != "missing" else "" + return { + "status": detail.get("status", ""), + "category": detail.get("category", ""), + "collection_path": path, + "resolved_sha1": sha1_by_path.get(path, "") if path else "", + } + return outcome + + +def _sha1_by_path(db: dict | None) -> dict[str, str]: + return {str(e.get("path", "")): sha1 for sha1, e in ((db or {}).get("files") or {}).items()} + + +def _platform_export_rows( + coverages: dict, db: dict | None = None, +) -> tuple[list[dict], list[dict]]: """Normalized platform summaries and one row per declared file.""" items: list[dict] = [] file_rows: list[dict] = [] + sha1_by_path = _sha1_by_path(db) for key, coverage in sorted(coverages.items()): config = coverage["config"] + outcome = _verify_outcomes(coverage, sha1_by_path) items.append({ "id": key, "name": coverage["platform"], @@ -715,6 +750,7 @@ def _platform_export_rows(coverages: dict) -> tuple[list[dict], list[dict]]: "sha256": _json_text(entry.get("sha256")), "md5": _json_text(entry.get("md5")), "crc32": _json_text(entry.get("crc32")), + **outcome(system_id, entry.get("name", "")), }) return items, file_rows @@ -778,7 +814,10 @@ def build_emulator_gap_report( ) -def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[dict]: +def _gap_export_rows( + coverages: dict, gap_report: dict | None = None, db: dict | None = None, + profiles: dict | None = None, +) -> list[dict]: """Every gap the site reports, both layers, in one table. ``platform`` rows are anomalies against a platform's own BIOS list. @@ -787,11 +826,16 @@ def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[di A `layer` column keeps the two apart instead of publishing only one. """ rows: list[dict] = [] + sha1_by_path = _sha1_by_path(db) for key, coverage in sorted(coverages.items()): for detail in coverage.get("details", []): if detail.get("status") == "ok" and not detail.get("discrepancy"): continue + path = str(detail.get("path") or "") if detail.get("status") != "missing" else "" rows.append({ + "category": detail.get("category", "bios"), + "collection_path": path, + "resolved_sha1": sha1_by_path.get(path, "") if path else "", "layer": "platform", "platform_id": key, "platform": coverage["platform"], @@ -808,8 +852,13 @@ def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[di }) for emu_key, data in sorted((gap_report or {}).items()): systems = ";".join(str(s) for s in data.get("systems", []) or []) + categories = _profile_categories((profiles or {}).get(emu_key) or {}) for gap in data.get("gap_details", []) or []: rows.append({ + "category": categories.get( + (gap.get("name", ""), gap.get("path") or gap.get("name", "")), ""), + "collection_path": "", + "resolved_sha1": "", "layer": "emulator", "platform_id": "", "platform": "", @@ -833,6 +882,10 @@ def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[di }) for entry in data.get("unsourceable", []) or []: rows.append({ + "category": categories.get( + (entry.get("name", ""), entry.get("path") or entry.get("name", "")), ""), + "collection_path": "", + "resolved_sha1": "", "layer": "emulator", "platform_id": "", "platform": "", @@ -847,7 +900,23 @@ def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[di "destination": entry.get("path") or entry.get("name", ""), "expected": "", }) - return rows + # Two declarations that differ only in what the row does not carry are + # one row: AmberELEC's hardwareconfig.xml was printed twice. + unique: dict[tuple, dict] = {} + for row in rows: + unique.setdefault(tuple(sorted((k, str(v)) for k, v in row.items())), row) + return list(unique.values()) + + +def _profile_categories(profile: dict) -> dict[tuple[str, str], str]: + """(name, path) -> the category a profile gives the entry, bios by default.""" + found: dict[tuple[str, str], str] = {} + for entry in profile.get("files") or []: + if isinstance(entry, dict) and entry.get("name"): + name = str(entry["name"]) + found.setdefault((name, str(entry.get("path") or name)), + str(entry.get("category") or "bios")) + return found def _cross_reference_export_rows(coverages: dict, profiles: dict) -> list[dict]: @@ -923,7 +992,8 @@ def _write_sqlite_export( platform_id TEXT NOT NULL, system TEXT NOT NULL, name TEXT NOT NULL, destination TEXT NOT NULL, required INTEGER NOT NULL, region TEXT, variant_group TEXT, size INTEGER, - sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT + sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT, + status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT ); CREATE TABLE emulators ( id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT, @@ -945,7 +1015,8 @@ def _write_sqlite_export( emulator TEXT, system TEXT, name TEXT NOT NULL, status TEXT NOT NULL, required INTEGER NOT NULL, in_repo TEXT, reason TEXT, discrepancy TEXT, - destination TEXT, expected TEXT + destination TEXT, expected TEXT, category TEXT, + collection_path TEXT, resolved_sha1 TEXT ); CREATE TABLE accepted_hashes ( layer TEXT NOT NULL, owner TEXT NOT NULL, system TEXT, @@ -960,9 +1031,11 @@ def _write_sqlite_export( CREATE INDEX emulator_files_name_idx ON emulator_files(name); CREATE INDEX gaps_status_idx ON gaps(layer, status); """) + commit = _source_commit() metadata = { "schema_version": "1", "generated_at": str(db.get("generated_at") or _timestamp()), + **({"commit": commit} if commit else {}), "source": REPO_URL, "scope": "metadata only, no BIOS or firmware payload bytes", } @@ -998,7 +1071,8 @@ def _write_sqlite_export( connection.executemany( "INSERT INTO platform_files VALUES " "(:platform_id, :system, :name, :destination, :required, :region, " - ":variant_group, :size, :sha1, :sha256, :md5, :crc32)", + ":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, " + ":collection_path, :resolved_sha1)", platform_files, ) for item in emulator_items: @@ -1045,7 +1119,8 @@ def _write_sqlite_export( connection.executemany( "INSERT INTO gaps VALUES " "(:layer, :platform_id, :platform, :emulator, :system, :name, " - ":status, :required, :in_repo, :reason, :discrepancy, :destination, :expected)", + ":status, :required, :in_repo, :reason, :discrepancy, :destination, :expected, " + ":category, :collection_path, :resolved_sha1)", gap_rows, ) connection.commit() @@ -1055,13 +1130,16 @@ def _write_sqlite_export( os.replace(temp_path, destination) -def write_single_item(path: Path, kind: str, item: dict, generated_at: str) -> None: +def write_single_item( + path: Path, kind: str, item: dict, generated_at: str, commit: str = "", +) -> None: """One item in the envelope of its collection, for a page to link.""" path.parent.mkdir(parents=True, exist_ok=True) write_if_changed( str(path), json.dumps( - {"schema_version": 1, "generated_at": generated_at, "kind": kind, + {"schema_version": 1, "generated_at": generated_at, + **({"commit": commit} if commit else {}), "kind": kind, "count": 1, "items": [item]}, indent=2, sort_keys=True, ensure_ascii=False, default=str, ) + "\n", @@ -1084,6 +1162,46 @@ def system_export_item(slug: str, manufacturer: str, consoles: dict[str, list]) } +@functools.lru_cache(maxsize=1) +def _source_commit() -> str: + """The commit of the checkout the site is built from, empty outside one.""" + import subprocess + + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=_REPO_ROOT, + capture_output=True, text=True, check=True, + ) + except (OSError, subprocess.CalledProcessError) as error: + print(f" no commit recorded in the exports: {error}", file=sys.stderr) + return "" + commit = result.stdout.strip() + return commit if re.fullmatch(r"[0-9a-f]{40}", commit) else "" + + +# One row per dump a catalog describes and the collection lacks: the list +# the provenance page publishes under "Wanted". +WANTED_FIELDS = ("catalog", "dat", "name", "description", "size", "crc32", "md5", "sha1") + + +def _provenance_wanted_rows(db: dict) -> list[dict]: + """The Wanted list of the provenance page, as rows. A dump the + collection already produces (derivable.json) is not wanted.""" + derivable = load_derivable() + rows: list[dict] = [] + for catalog, data in build_report(db, load_provenance_snapshots()).items(): + for entry in data["missing"]: + if (entry.get("sha1") or "").lower() in derivable: + continue + for dat in entry.get("dats") or [entry.get("dat", "")]: + rows.append({ + "catalog": catalog, "dat": dat, + **{f: _json_text(entry.get(f)) for f in WANTED_FIELDS[2:]}, + }) + rows.sort(key=lambda r: (r["catalog"], r["dat"], r["name"], r["sha1"])) + return rows + + def generate_data_exports( docs: Path, db: dict, @@ -1100,9 +1218,13 @@ def generate_data_exports( directory.mkdir(parents=True, exist_ok=True) generated_at = str(db.get("generated_at") or _timestamp()) - platform_items, platform_files = _platform_export_rows(coverages) + # The revision the data was read at: a tool comparing two downloads + # could not tell which commit each came from. + commit = _source_commit() + stamp = {"commit": commit} if commit else {} + platform_items, platform_files = _platform_export_rows(coverages, db) emulator_items = _emulator_export_items(profiles) - gap_rows = _gap_export_rows(coverages, gap_report) + gap_rows = _gap_export_rows(coverages, gap_report, db, profiles) cross_rows = _cross_reference_export_rows(coverages, profiles) members = data_directory_members() @@ -1115,11 +1237,13 @@ def generate_data_exports( } write_lookup_shards( api, file_lookup_records(db, coverages, profiles, locations, members), generated_at, + commit, ) for filename, (kind, items) in envelopes.items(): document = { "schema_version": 1, "generated_at": generated_at, + **stamp, "kind": kind, "count": len(items), "items": items, @@ -1132,7 +1256,7 @@ def generate_data_exports( # profile: platforms.json is 2.8 MB for twelve platforms. for item in platform_items: write_single_item(api / "platforms" / f"{item['id']}.json", - "platforms", item, generated_at) + "platforms", item, generated_at, commit) write_if_changed(str(api / "stats.json"), generate_stats(stats)) write_if_changed( str(api / "database.json"), @@ -1179,7 +1303,8 @@ def generate_data_exports( [ "platform_id", "platform", "system", "name", "destination", "required", "region", "variant_group", "size", "sha1", - "sha256", "md5", "crc32", + "sha256", "md5", "crc32", "status", "category", + "collection_path", "resolved_sha1", ], platform_files, ), @@ -1201,7 +1326,8 @@ def generate_data_exports( [ "layer", "platform_id", "platform", "emulator", "system", "name", "status", "required", "in_repo", "reason", - "discrepancy", "destination", "expected", + "discrepancy", "destination", "expected", "category", + "collection_path", "resolved_sha1", ], gap_rows, ), @@ -1218,6 +1344,10 @@ def generate_data_exports( emulator_items, gap_rows, ) _record(downloads / "retrobios.sqlite") + write_if_changed( + str(downloads / "provenance-wanted.csv"), + _csv_document(list(WANTED_FIELDS), _provenance_wanted_rows(db)), + ) assets = [ (api / "database.json", "Content database", "application/json", "schemas/database.schema.json"), @@ -1233,6 +1363,7 @@ def generate_data_exports( (downloads / "cross-reference.csv", "Cross-reference CSV", "text/csv", None), (downloads / "gaps.csv", "Verification and coverage gaps CSV", "text/csv", None), (downloads / "accepted-hashes.csv", "Accepted hashes, one row per value", "text/csv", None), + (downloads / "provenance-wanted.csv", "Catalogued dumps the collection lacks", "text/csv", None), (downloads / "retrobios.sqlite", "SQLite snapshot", "application/vnd.sqlite3", None), ] catalog_items: list[dict] = [] @@ -1259,6 +1390,7 @@ def generate_data_exports( catalog = { "schema_version": 1, "generated_at": generated_at, + **stamp, "kind": "catalog", "count": len(published), "items": published, @@ -1272,7 +1404,7 @@ def generate_data_exports( def generate_data_page(exports: list[dict]) -> str: lines = [ - "# Data & API", + "# Data and API", "", "RetroBIOS publishes the same metadata used by its verifier, pack builder " "and website as versioned static files. No account or API key is required.", @@ -1308,7 +1440,8 @@ def generate_data_page(exports: list[dict]) -> str: ) lines.extend([ "", - "Every JSON document carries `schema_version`. Breaking changes use a new " + "Every JSON document carries `schema_version`, and `commit`, the revision " + "of this repository the data was read at. Breaking changes use a new " "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1.", "", "One emulator profile without the whole list: " @@ -1363,7 +1496,14 @@ def generate_data_page(exports: list[dict]) -> str: "`platform` rows are anomalies against a platform's own BIOS list, " "`emulator` rows are files a profiled core loads that no platform " "declares. The two answer different questions and are not comparable " - "totals.", + "totals. Each row is listed once.", + "", + "`platform-files.csv` and the gaps add what the check found: `status` " + "(`ok`, `untested`, `missing`), `category` (`bios`, `game_data`, " + "`unread`), and for a held file `collection_path` and `resolved_sha1`, " + "the collection file that answers the declaration. " + "`provenance-wanted.csv` is the Wanted list of [Dump provenance]" + "(provenance.md): each dump a catalog describes and the collection lacks.", "", "## Semantics and limits", "", @@ -2070,11 +2210,119 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st # A platform system whose file list passes this much Markdown gets a page # of its own: RetroDECK's arcade list was 750 KB of a 1.1 MB page. PLATFORM_SECTION_BUDGET = 200_000 +# A system's own page in runs of at most this much Markdown: RetroDECK's +# arcade list, each file with its served copy and pinned readers, rendered +# 1.4 MB of HTML on one page. +PLATFORM_PART_BUDGET = 350_000 +# The Markdown of a platform page's file lists, past which the largest +# systems move to pages of their own. +PLATFORM_REFERENCE_BUDGET = 250_000 + + +def _file_runs(body: list[str], budget: int) -> list[list[str]]: + """A system's file list in runs of at most ``budget`` characters. + + The lines before the first file (counts, emulators) open every run. A + file is never cut from its hash and evidence lines. + """ + first = next((i for i, line in enumerate(body) if line.startswith("- ")), len(body)) + head, items = body[:first], [] + for line in body[first:]: + if line.startswith("- ") or not items: + items.append([line]) + else: + items[-1].append(line) + runs: list[list[str]] = [[]] + size = 0 + for item in items: + weight = sum(len(line) + 1 for line in item) + if runs[-1] and size + weight > budget: + runs.append([]) + size = 0 + runs[-1].extend(item) + size += weight + return [head + run for run in runs] + + +@functools.lru_cache(maxsize=1) +def _collection_sha1_by_path() -> dict[str, str]: + """The sha1 of each file the collection holds, by its path.""" + with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle: + files = json.load(handle).get("files", {}) + return {str(entry.get("path", "")): sha1 for sha1, entry in files.items()} + + +def _system_heading_link(sys_id: str, prefix: str = "") -> str: + """A system's name, linked to its own console section when it has one. + + The id stays beside it in small text: it is what the platform's list + writes. A system with no section of its own is named, never sent to the + page a storage folder chose. + """ + from siterender import _own_section + + name = _escape_tags(system_name(sys_id)) + section = _own_section(sys_id) + if section is not None: + mfr, console = section + name = f"[{name}]({console_href(_slug(mfr), console, prefix)})" + return f"{name} {_escape_tags(sys_id)}" + + +def _served_line(f: dict) -> str: + """Which collection file answers the entry, from the resolver's result. + + "OK" with no path gave an expert nothing to check: Batocera's + scph1001.bin is served by bios/Sony/PlayStation/scph1001_v20.bin. + """ + path = str(f.get("path") or "") + if f.get("status") == "missing": + return "Not held" + if not path: + # verify reports no path in existence mode: nothing to name. + return "" + sha1 = _collection_sha1_by_path().get(path, "") + label = "Served by" if f.get("status") == "ok" else "Held, not the declared hash" + parts = path.split("/") + if not sha1 or len(parts) < 3 or parts[0] != "bios": + return f"{label}: {_code(path)}" + # The card of that copy: every hash, provenance and the download. + page = urllib.parse.quote(card_page(path, sha1)) + return (f"{label}: [{_code(path)}](../systems/{page}.md#sha1-{sha1[:12]}) " + f"(sha1 `{sha1[:8]}`)") + + +# Platforms whose launcher records are read from the repository and revision +# of their BIOS list: batocera_scraper reads configgen at its tag, +# retrobat_scraper reads emulatorLauncher at the list's commit +# (launcher_revision). The others read another repository. +_LAUNCHERS_IN_LIST_REPO = frozenset({"batocera", "retrobat"}) + + +def _launcher_refs(platform: str, config: dict) -> dict[str, list[str]]: + """Entry name, casefolded -> each launcher record that opens it, rendered.""" + from launchers import _names as launcher_names + from sitesources import launcher_ref_markdown, list_source + + source = (list_source(str(config.get("source") or "")) + if platform in _LAUNCHERS_IN_LIST_REPO else None) + found: dict[str, list[str]] = {} + for record in config.get("launchers") or []: + if not isinstance(record, dict) or not record.get("source_ref"): + continue + text = (f"{_escape_tags(str(record.get('emulator', '')))} at " + f"{launcher_ref_markdown(source, str(record['source_ref']))}") + for name in launcher_names(record): + names = found.setdefault(name.rsplit("/", 1)[-1].casefold(), []) + if text not in names: + names.append(text) + return found def _render_platform_systems( by_system: dict, config_files: dict, emulator_files: dict, platform: tuple[str, str] | None = None, moved: dict[str, str] | None = None, + config: dict | None = None, ) -> list[str]: """One collapsible block per system, listing every file it needs. @@ -2089,26 +2337,19 @@ def _render_platform_systems( # Per-system detail sections (collapsible for large platforms) use_collapsible = len(by_system) > 10 by_system_declared = _declared_by_system(config_files) + launched = _launcher_refs(platform[0] if platform else "", config or {}) + sections: list[dict] = [] for sys_id, files in sorted(by_system.items()): ok_count = sum(1 for f in files if f["status"] == "ok") total = len(files) sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id)) - - sys_link = _system_link(sys_id, "../") - anchor = sys_id.replace(" ", "-") - # Every system is a heading with the same anchor in both layouts: - # emulator cards link here, and the site search splits a page at its - # headings. Folded into blocks under one heading, every hash pasted - # in the search led to the systems overview of a 100-block page. - lines.append(f"### {sys_link} {{ #{anchor} }}") - lines.append("") - block_start = len(lines) body = [f"{ok_count}/{total} files verified"] if sys_emus: emu_links = ", ".join(_emulator_link(e, "../") for e in sorted(sys_emus)) + body[-1] += "\\" body.append(f"Emulators: {emu_links}") body.append("") @@ -2152,6 +2393,8 @@ def _render_platform_systems( destination = str(cfg_entry.get("destination") or "") where = f" at `{destination}`" if destination and destination != fname else "" body.append(f"- `{fname}`{where} - {', '.join(details)}") + item_line = len(body) - 1 + facts: list[str] = [] # Every hash the platform declares, whole: the crc32 is what # System.dat and clrmamepro print, and pasting it found nothing # for a pinned file the collection holds under other bytes. @@ -2172,9 +2415,19 @@ def _render_platform_systems( hash_parts.append(f"CRC32: `{crc32}`") if sha256: hash_parts.append(f"SHA256: `{sha256}`") - body.append(f" {' | '.join(hash_parts)}") + facts.append(" | ".join(hash_parts)) + served = _served_line(f) + if served: + facts.append(served) if f.get("category_evidence"): - body.append(f" {_category_evidence_md(f['category_evidence'])}") + facts.append(_category_evidence_md( + f["category_evidence"], launched.get(fname.rsplit("/", 1)[-1].casefold()))) + # One line each, hashes, the copy that serves it, its readers: as + # continuation lines they ran into one paragraph. + if facts: + body[item_line] += "\\" + body.extend(f" {fact}" + ("\\" if n < len(facts) - 1 else "") + for n, fact in enumerate(facts)) # Measured as the Markdown the split was tuned on, a collapsible # block indented by four, so both engines split the same systems. @@ -2188,26 +2441,62 @@ def _render_platform_systems( block.pop() else: block = body - if moved is not None and platform and size > PLATFORM_SECTION_BUDGET: + sections.append({"sys_id": sys_id, "anchor": anchor, "body": body, "block": block, + "size": size, "total": total, "ok": ok_count}) + + # A list longer than its budget goes to a page of its own, then the + # largest lists until the page's reference fits PLATFORM_REFERENCE_BUDGET: + # with each file's served copy and pinned readers, Batocera's 97 systems + # rendered 785 KB of HTML on one page. + own_page: set[str] = set() + if moved is not None and platform: + own_page = {sec["sys_id"] for sec in sections if sec["size"] > PLATFORM_SECTION_BUDGET} + kept = sum(sec["size"] for sec in sections if sec["sys_id"] not in own_page) + for sec in sorted(sections, key=lambda sec: -sec["size"]): + if kept <= PLATFORM_REFERENCE_BUDGET: + break + if sec["sys_id"] not in own_page: + own_page.add(sec["sys_id"]) + kept -= sec["size"] + + for sec in sections: + sys_id, anchor, body = sec["sys_id"], sec["anchor"], sec["body"] + total, ok_count = sec["total"], sec["ok"] + # Every system is a heading with the same anchor in both layouts: + # emulator cards link here, and the site search splits a page at its + # headings. Folded into blocks under one heading, every hash pasted + # in the search led to the systems overview of a 100-block page. + lines.append(f"### {_system_heading_link(sys_id, '../')} {{ #{anchor} }}") + lines.append("") + if moved is not None and platform and sys_id in own_page: key, display = platform - stem = f"{key}-{anchor}" - moved[stem] = "\n".join([ - f"# {display}: {sys_id}", "", - f"Part of the [{display} page]({key}.md#{anchor}), whose other systems " - f"are listed there. Each file below is as {display} declares it, with " - "its status in the pack.", - "", "## Files", "", - *search_excluded(body), - "", f"*Generated on {_timestamp()}*", - ]) + "\n" - lines[block_start:] = [ - f"{_count(total, 'file')}, {ok_count} verified: listed on " - f"[a page of their own]({stem}.md).", "", - ] + runs = _file_runs(body, PLATFORM_PART_BUDGET) + stems = [f"{key}-{anchor}" + (f"-{n}" if n > 1 else "") + for n in range(1, len(runs) + 1)] + for n, (stem, run) in enumerate(zip(stems, runs), 1): + part = f" (part {n} of {len(runs)})" if len(runs) > 1 else "" + moved[stem] = "\n".join([ + f"# {display}: {system_name(sys_id)}{part}", "", + f"Part of the [{display} page]({key}.md#{anchor}), whose other systems " + f"are listed there. Each file below is as {display} declares it, with " + "its status in the pack.", + *([""] + [", ".join( + f"[part {i}]({other}.md)" if other != stem else f"part {i}" + for i, other in enumerate(stems, 1))] if len(runs) > 1 else []), + "", "## Files", "", + *search_excluded(run), + "", f"*Generated on {_timestamp()}*", + ]) + "\n" + where = (f"[a page of their own]({stems[0]}.md)" if len(stems) == 1 else + "pages of their own: " + ", ".join( + f"[part {i}]({stem}.md)" for i, stem in enumerate(stems, 1))) + lines.extend([ + f"{_count(total, 'file')}, {ok_count} verified: listed on {where}.", "", + ]) continue # The section's heading stays searchable, its file list with every # hash is the lookup's: retrodeck's arcade list was 495 KB of index. - lines[block_start:] = search_excluded(block) + lines.extend(search_excluded(sec["block"])) return lines @@ -2232,24 +2521,50 @@ def _platform_data_links(name: str) -> str: return ", ".join(links) -def _category_evidence_md(evidence: dict) -> str: - """What establishes a missing entry's category, as the page shows it.""" +@functools.lru_cache(maxsize=1) +def _site_profiles() -> dict[str, dict]: + """Every emulator profile, read once for the citations of a page.""" + return load_emulator_profiles(str(_REPO_ROOT / "emulators")) + + +def _category_evidence_md(evidence: dict, launched: list[str] | None = None) -> str: + """What establishes an entry's category, as the page shows it. + + ``launched`` holds the platform's launcher records that open the entry, + already rendered: "(by launcher)" alone named no record and no line. + """ if evidence.get("read_by"): groups: dict[bool, list[str]] = {True: [], False: []} + bare: list[str] = [] + profiles = _site_profiles() for reader in evidence["read_by"]: + # Pinned links, the repository named in words: the refs were + # plain code with the role word ("source") left in front. + profile = profiles.get(reader["emulator"], {}) refs = [ - f"`{e['name']}` at `{e['source_ref']}`" if e.get("source_ref") else f"`{e['name']}`" + (f"`{e['name']}` at {_source_ref_markdown(profile, e['source_ref'])}" + if e.get("source_ref") else f"`{e['name']}`") + + (" (by name)" if e.get("by") == "name" else "") for e in reader.get("entries", []) ] - name = (_emulator_link(reader["emulator"], "../") - if reader["emulator"] != "launcher" else "the platform's launcher") + if reader["emulator"] == "launcher": + name = "the platform's launcher" + refs = list(launched or []) + else: + name = _emulator_link(reader["emulator"], "../") groups[bool(reader.get("runs_system"))].append( f"{name} ({', '.join(refs)})" if refs else f"{name} (by {reader['proof']})") + if not reader.get("runs_system"): + # A launcher record is this platform's own code: kept whole. + bare.append(f"{name} ({', '.join(refs)})" + if reader["emulator"] == "launcher" and refs else name) if not groups[True]: return "Read by " + ", ".join(groups[False]) text = "Read by " + ", ".join(groups[True]) - if groups[False]: - text += ", also named by " + ", ".join(groups[False]) + # Cores of other systems that share the name: named, not cited, as + # their lines describe another machine's file. + if bare: + text += ", also named by " + ", ".join(bare) return text examined = evidence.get("cores_examined") or [] cores = ("cores examined: " + ", ".join(_emulator_link(c, "../") for c in examined) @@ -2427,6 +2742,11 @@ def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list: return lines +# How a platform checks a file, in words. +_CHECK_WORDS = {"existence": "name only", "md5": "MD5", "sha1": "SHA-1", "sha256": "SHA-256", + "crc32": "CRC32"} + + def generate_platform_page( name: str, cov: dict, @@ -2439,16 +2759,17 @@ def generate_platform_page( ``moved`` collects the pages of systems whose list is too long for it. """ + from sitesources import list_source + registry = registry or {} config = cov["config"] display = cov["platform"] mode = cov["mode"] pct = _pct(cov["present"], cov["total"]) + # The mark is drawn beside the title from the front matter: floated + # beside the first line at 48 px it read as a broken image. logo_url = registry.get(name, {}).get("logo", "") - logo_md = ( - markup.image(display, logo_url, width="48", align="right") + "\n\n" if logo_url else "" - ) homepage = config.get("homepage", "") version = config.get("version", "") @@ -2460,9 +2781,9 @@ def generate_platform_page( ) lines = [ + *(["---", f"logo: {json.dumps(logo_url)}", "---", ""] if logo_url else []), f"# {display}", "", - logo_md, ] if registry.get(name, {}).get("status") == "archived": @@ -2504,14 +2825,17 @@ def generate_platform_page( *_stat(f"{cov['present']}/{cov['total']}", f"Coverage ({pct})"), *_stat(str(cov["verified"]), "Verified"), *_stat(str(cov["missing"]), "Missing"), - *_stat(f'{mode}', "Checked by"), + *_stat(f'{_CHECK_WORDS.get(mode, mode)}', + "Checked by"), "", "", ] lines.extend([*band, markup.block_open("div", 'class="rb-meta-card"'), "", "| Detail | Value |", "|---|---|"]) - if hash_type: - lines.append(f"| Hash type | {hash_type} |") + # The hash the list writes, when it is not the one the platform checks: + # RetroArch checks presence and its System.dat writes sha1. + if hash_type and hash_type != mode: + lines.append(f"| Hashes in the list | {_CHECK_WORDS.get(hash_type, hash_type)} |") if version: lines.append(f"| Version | {version} |") # base_destination is only the root the platform's own list writes its @@ -2524,11 +2848,15 @@ def generate_platform_page( if transcribed: # The file every declaration on this page was read from: without it # a hash could not be traced to the revision that states it. - moving = re.search(r"/(?:master|main|db)(?:/|$)|refs/heads/", transcribed) or ( - re.fullmatch(r"https://(?:github|gitlab)\.com/[^/]+/[^/]+/?", transcribed) - ) + listed = list_source(transcribed) + moving = (listed.moving if listed else bool( + re.search(r"/(?:master|main|db)(?:/|$)|refs/heads/", transcribed))) note = " (a branch: the revision read is not recorded)" if moving else "" - lines.append(f"| Transcribed from | <{transcribed}>{note} |") + # Named in words: the raw URL broke into 25 lines of 8 characters + # on a phone. + link = (f"[{_escape_tags(listed.label)}]({listed.browse()})" if listed + else f"<{transcribed}>") + lines.append(f"| Transcribed from | {link}{note} |") if name in _EXPORTED: lines.append(f"| Updated file | [Download it and see the changes](../exports.md#{name}) |") lines.append(f"| Data | {_platform_data_links(name)} |") @@ -2595,14 +2923,16 @@ def generate_platform_page( if len(sys_emus) > 3: emu_str += f" +{len(sys_emus) - 3}" anchor = sys_id.replace(" ", "-") + # The name a reader filters on, the id the list writes beside it. lines.append( - f"| [{sys_id}](#{anchor}) | {ok_count}/{total} | {status} | {emu_str} |" + f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) {sys_id} " + f"| {ok_count}/{total} | {status} | {emu_str} |" ) lines.append("") lines.extend( _render_platform_systems(by_system, config_files, emulator_files, - (name, display), moved) + (name, display), moved, config) ) lines.append(f"*Generated on {_timestamp()}*") @@ -2817,7 +3147,9 @@ def file_lookup_records( r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"])) -def write_lookup_shards(api: Path, records: list[dict], generated_at: str) -> int: +def write_lookup_shards( + api: Path, records: list[dict], generated_at: str, commit: str = "", +) -> int: """One envelope per leading pair of hex characters, as the lookup asks. A value that is not hexadecimal (a typo in a profile) has no shard the @@ -2833,6 +3165,7 @@ def write_lookup_shards(api: Path, records: list[dict], generated_at: str) -> in write_if_changed(str(folder / f"{prefix}.json"), json.dumps({ "schema_version": 1, "generated_at": generated_at, + **({"commit": commit} if commit else {}), "kind": "file-lookup", "count": len(items), "items": items, @@ -3001,16 +3334,31 @@ def load_derivable(path: str = "derivable.json") -> dict[str, dict]: _ROMSET_SOURCES = {"mame": "MAME", "fbneo": "FBNeo"} +def _date_in_words(stamp: str) -> str: + """`2026-08-07T...` as ``.""" + try: + day = datetime.strptime(str(stamp)[:10], "%Y-%m-%d") + except ValueError: + return _escape_tags(str(stamp)) + return f'' + + def generate_provenance_page( db: dict, report: dict, derivable: dict[str, dict] | None = None ) -> str: """Page explaining the verified dump badges and listing catalog gaps.""" matched_files = count_catalog_matched(db) total_files = compute_composition(db)["systems"]["files"] + # When each catalog was read: a 76-day-old snapshot showed no age. + as_of = ", ".join( + f"{_PROVENANCE_LABELS.get(source, source)} {_date_in_words(data['imported_at'])}" + for source, data in report.items() if data.get("imported_at") + ) lines = [ "# Dump provenance", "", + *([f'

Catalogs as of: {as_of}.

', ""] if as_of else []), "Provenance says whether a file is, byte for byte, a dump that Redump, " "No-Intro or TOSEC catalogues. It tells you where a dump is known from, " "not whether an emulator accepts it: that is the emulator's code, read " @@ -3067,7 +3415,7 @@ def generate_provenance_page( name = f"[{label}]({home})" if home else label lines.append( f"| {name} | {data['matched']:,}/{in_scope:,} ({pct:.0f}%) | " - f"{len(data['covered_dats'])} | {data['imported_at']} |" + f"{len(data['covered_dats'])} | {_date_in_words(data['imported_at'])} |" ) lines.extend( @@ -5234,6 +5582,32 @@ def _platform_lists_sentence( ) +def _missing_required_firmware(cov: dict) -> int: + """Entries of a platform's list that are missing, required and firmware. + + Only those stop a console from starting. Game data and files no core + opens are counted apart, and red on every short row carried no signal. + """ + return len({ + (d.get("system", ""), d.get("name", "")) for d in cov.get("details") or [] + if d.get("status") == "missing" and d.get("required", True) + and d.get("category", "bios") == "bios" and not d.get("settled_by") + }) + + +def _ratio_cell(done: int, total: int, alarm: int = 0) -> str: + """A held-over-needed cell: the numbers, a bar, and a badge only when + required firmware is missing.""" + if not total: + return "-" + share = done / total * 100 + cell = (f'' + f"{done:,}/{total:,}") + if alarm: + cell += f' {alarm:,} required missing' + return cell + + def generate_gap_analysis( profiles: dict, coverages: dict, @@ -5299,7 +5673,7 @@ def generate_gap_analysis( }) lines = [ - "# Gap analysis", + "# What the packs do not hold", "", "What each pack still cannot give you, and why. Most gaps are not " "firmware: a platform's list can also name game files, such as " @@ -5330,7 +5704,7 @@ def generate_gap_analysis( lines.extend([ "## What each pack contains", "", - "| Platform | On its BIOS list | Files its emulators load | Checked by |", + "| Platform | On its BIOS list (destinations) | Files its emulators load | Checked by |", "|----------|-----------------:|-------------------------:|------------|", ]) @@ -5340,15 +5714,6 @@ def generate_gap_analysis( "existence": "file presence", } - def _ratio(done: int, total: int) -> str: - """A collected-over-needed cell, flagged when short.""" - if not total: - return "-" - cell = f"{done:,}/{total:,}" - if done < total: - return f'{cell}' - return cell - for pname, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): display = cov["platform"] core_total = ( @@ -5356,18 +5721,20 @@ def generate_gap_analysis( ) lines.append( f"| [{display}](platforms/{pname}.md) " - f"| {_ratio(cov['present'], cov['total'])} " - f"| {_ratio(cov['core_present'], core_total)} " + f"| {_ratio_cell(cov['present'], cov['total'], _missing_required_firmware(cov))} " + f"| {_ratio_cell(cov['core_present'], core_total)} " f"| {mode_labels.get(cov['mode'], cov['mode'])} |" ) lines.extend([ "", "Each fraction is what the pack has over what is needed, counting " "required and optional files alike since both ship. The first column " - "is the BIOS list the platform publishes. The second counts files its " + "is the BIOS list the platform publishes, counted by destination: a " + "path the list declares twice is one file to place. A red badge marks " + "a list that is short of required firmware. The second counts files its " "emulators load that this list never mentions, found by reading their " - "source code, and it is often several times larger. A short " - "fraction is flagged and named in the sections below. That second " + "source code, and it is often several times larger. What is short " + "is named in the sections below. That second " "number is a floor, not a ceiling: an emulator that accepts any file " "handed to it names none in its code, so nothing there can be counted. " "Checked by is the test the platform runs on its own, replicated here " @@ -5615,12 +5982,13 @@ def generate_gap_analysis( if src_missing_details: req_src = [m for m in src_missing_details if m["required"]] lines.extend([ - f"### Files to source ({_count(len(src_missing_details), 'file')}, " - f"{len(req_src)} required" + "### Files to source", + "", + f"{_count(len(src_missing_details), 'file')}, {len(req_src)} required." + ( - f", {_count(entries, 'entry', 'entries')}: " - "a file declared with the same hash at several places is listed once)" - if entries != len(src_missing_details) else ")" + f" They come from {_count(entries, 'entry', 'entries')}: a file " + "declared with the same hash at several places is listed once." + if entries != len(src_missing_details) else "" ), "", "| File | Destination | Expected | Emulator | Required | Affects platforms | Source ref |", @@ -5735,154 +6103,210 @@ def _gap_sources(data: dict) -> dict[str, int]: return counts +def _xref_upstream(profile: dict) -> str: + """The project a core comes from: its upstream, else its own source.""" + raw = profile.get("upstream") or profile.get("source", "") + return raw if isinstance(raw, str) else "" + + +_FORGES = frozenset({ + "github.com", "gitlab.com", "codeberg.org", "bitbucket.org", "git.eden-emu.dev", + "git.ryujinx.app", +}) + + +def upstream_project(url: str) -> tuple[str, str]: + """(project key, revision or path inside it) for an upstream URL. + + MAME was twelve rows: mamedev/mame and one row per release tree + (mamedev/mame/tree/mame0289 and the rest). On a forge the project is + owner/repo, a GitLab project every group before `/-/`, and the rest is + a revision. Elsewhere the address without its scheme, `www.` and + trailing slash is the project. A note written after the address + (`... (sys/src/games/nes/)`) is a path inside it. + """ + url, _, note = url.strip().partition(" ") + note = note.strip().strip("()") + if not url.startswith(("http://", "https://")): + return " ".join(filter(None, (url, note))), "" + parsed = urllib.parse.urlsplit(url) + host = parsed.netloc.lower().removeprefix("www.") + parts = [p for p in parsed.path.split("/") if p] + if host == "gitlab.com" and len(parts) >= 2: + cut = parts.index("-") if "-" in parts else len(parts) + rest = parts[cut + 1:] + if rest[:1] in (["tree"], ["blob"]): + rest = rest[1:] + return f"{host}/{'/'.join(parts[:cut])}", "/".join(filter(None, ("/".join(rest), note))) + if host in _FORGES and len(parts) >= 2: + repo = parts[1].removesuffix(".git") + rest = parts[2:] + if rest[:1] in (["tree"], ["blob"], ["src"]): + rest = rest[1:] + return f"{host}/{parts[0]}/{repo}", "/".join(filter(None, ("/".join(rest), note))) + return "/".join([host, *parts]), note + + +def _upstream_label(key: str) -> str: + """owner/repo on GitHub, host/owner/repo on another forge, and a + Software Heritage revision by its short id.""" + swh = re.fullmatch(r"archive\.softwareheritage\.org/browse/revision/([0-9a-f]{12})[0-9a-f]*", key) + if swh: + return f"Software Heritage revision {swh[1]}" + return key.removeprefix("github.com/") + + +def _breakable(label: str) -> str: + """A project address that may wrap after each slash: unbroken, the + longest pushed three of five columns off a 1,440 px screen.""" + return "/".join(_escape_tags(part) for part in label.split("/")) + + +def _xref_platform_rows(config: dict, unique: dict) -> tuple[int, list[str]]: + """The cores a platform pulls in, as table rows, with their count.""" + # The cores the builder and verify resolve: a hand copy here dropped + # the libretro set a list naming retroarch pulls in, and the system + # fallback EmuDeck relies on. + matched = sorted(key for key in resolve_platform_cores(config, unique) if key in unique) + rows = [] + for emu_name in matched: + p = unique[emu_name] + cls_raw = p.get("core_classification", "-") + systems = p.get("systems", []) or [] + sys_str = ", ".join(_escape_tags(system_name(s)) for s in systems[:3]) + if len(systems) > 3: + sys_str += f" +{len(systems) - 3}" + files = p.get("files", []) or [] + libretro_only = sum(1 for f in files if f.get("mode") == "libretro") + standalone_only = sum(1 for f in files if f.get("mode") == "standalone") + file_str = str(len(files)) + parts = [f"{n} {label}" for n, label in ((libretro_only, "libretro-only"), + (standalone_only, "standalone-only")) if n] + if parts: + file_str += f" ({', '.join(parts)})" + upstream = _xref_upstream(p) + key, _rest = upstream_project(upstream) + upstream_display = (f"[{_breakable(_upstream_label(key))}]({upstream.split()[0]})" + if upstream.startswith("http") else (_escape_tags(upstream) or "-")) + rows.append( + f"| [{_escape_tags(p.get('emulator', emu_name))}](emulators/{emu_name}.md) " + f"| {CLS_LABELS.get(cls_raw, cls_raw)} | {sys_str or '-'} | {file_str} " + f"| {upstream_display} |" + ) + return len(matched), rows + + def _render_xref_by_platform( - coverages: dict, unique: dict + coverages: dict, unique: dict, moved: dict[str, str] | None = None, ) -> list[str]: - """Which cores each platform pulls in, and what they need.""" - lines: list[str] = [] - for pname in sorted(coverages.keys(), key=lambda x: coverages[x]["platform"]): + """Which cores each platform pulls in: counts here, a page per platform. + + One page held every platform's table, 30,551 nodes: the overview keeps + the counts and each platform's table moves to ``moved`` by stem. + """ + lines = [ + "## By platform", + "", + "| Platform | Cores | Core list |", + "|----------|------:|-----------|", + ] + for pname in sorted(coverages, key=lambda x: coverages[x]["platform"]): cov = coverages[pname] display = cov["platform"] config = cov["config"] - - body = [markup.button(f"Open {display} platform profile", f"platforms/{pname}.md"), ""] - - # The cores the builder and verify resolve: a hand copy here dropped - # the libretro set a list naming retroarch pulls in, and the system - # fallback EmuDeck relies on. - matched = { - key: unique[key] - for key in resolve_platform_cores(config, unique) - if key in unique - } - - body.append(f"**{len(matched)} cores**{_core_scope_note(config, unique)}") - body.append("") - - body.append("| Core | Classification | Systems | Files | Upstream |") - body.append("|------|---------------|---------|-------|----------|") - - for emu_name in sorted(matched.keys()): - p = matched[emu_name] - emu_display = p.get("emulator", emu_name) - cls_raw = p.get("core_classification", "-") - cls = CLS_LABELS.get(cls_raw, cls_raw) - p.get("type", "") - # Null means the project is its own upstream: show its source. - upstream_raw2 = p.get("upstream", "") - upstream = upstream_raw2 if isinstance(upstream_raw2, str) else "" - source_raw2 = p.get("source", "") - source = str(source_raw2) if not isinstance(source_raw2, dict) else "" - systems = p.get("systems", []) - files = p.get("files", []) - - sys_str = ", ".join(systems[:3]) - if len(systems) > 3: - sys_str += f" +{len(systems) - 3}" - - file_count = len(files) - libretro_only = sum(1 for f in files if f.get("mode") == "libretro") - standalone_only = sum(1 for f in files if f.get("mode") == "standalone") - file_str = str(file_count) - if libretro_only or standalone_only: - parts = [] - if libretro_only: - parts.append(f"{libretro_only} libretro-only") - if standalone_only: - parts.append(f"{standalone_only} standalone-only") - file_str += f" ({', '.join(parts)})" - - upstream_display = "-" - if upstream and upstream.startswith("http"): - upstream_short = upstream.replace("https://github.com/", "") - upstream_display = f"[{upstream_short}]({upstream})" - elif upstream: - upstream_display = upstream - elif source and source.startswith("http"): - source_short = source.replace("https://github.com/", "") - upstream_display = f"[{source_short}]({source})" - elif source: - upstream_display = source - - body.append( - f"| [{emu_display}](emulators/{emu_name}.md) | {cls} | " - f"{sys_str} | {file_str} | {upstream_display} |" - ) - - lines.extend(markup.details("abstract", display, body)) - + count, rows = _xref_platform_rows(config, unique) + stem = f"cross-reference-{pname}" + lines.append( + f"| [{_escape_tags(display)}](platforms/{pname}.md) | {count}" + f"{_core_scope_note(config, unique)} | [Cores on {_escape_tags(display)}]({stem}.md) |" + ) + if moved is not None: + moved[stem] = "\n".join([ + "---", "level: reference", "---", "", + f"# Cores on {display}", + "", + f"{_count(count, 'core')}{_core_scope_note(config, unique)} that " + f"{display} runs, the systems each emulates and the project it comes " + f"from. Part of [Cores by platform](cross-reference.md). The files " + f"are listed on the [{display} page](platforms/{pname}.md).", + "", + "| Core | Classification | Systems | Files | Upstream |", + "|------|---------------|---------|-------|----------|", + *rows, + "", + ]) + "\n" + lines.append("") return lines def _render_xref_by_upstream( coverages: dict, unique: dict ) -> list[str]: - """The same relation read the other way: one row per upstream.""" - lines: list[str] = [] - # Reverse view: by upstream emulator - lines.extend( - [ - "## By upstream emulator", - "", - "| Upstream | Cores | Classification | Platforms |", - "|----------|-------|---------------|-----------|", - ] - ) + """The same relation read the other way: one row per upstream project.""" + lines: list[str] = [ + "## By upstream emulator", + "", + "One row per project: the releases and folders of one repository are " + "listed in the Revisions column. Platforms counts the platforms that run " + "at least one of its cores.", + "", + "| Cores | Upstream | Revisions | Classification | Platforms |", + "|-------|----------|-----------|----------------|----------:|", + ] - by_upstream: dict[str, list[str]] = {} + by_upstream: dict[str, list[tuple[str, str, str]]] = {} for emu_name, p in sorted(unique.items()): # A null upstream is the project's own source: grouping it under # the string "None" put eight unrelated engines in one row. - raw_up = p.get("upstream") or p.get("source", "") - up_str = raw_up if isinstance(raw_up, str) else "" + up_str = _xref_upstream(p) if up_str: - by_upstream.setdefault(up_str, []).append(emu_name) + key, rest = upstream_project(up_str) + by_upstream.setdefault(key, []).append((emu_name, up_str, rest)) platform_membership: dict[str, set[str]] = {} for pname, cov in coverages.items(): for key in resolve_platform_cores(cov["config"], unique): - platform_membership.setdefault(key, set()).add(pname) + platform_membership.setdefault(key, set()).add(cov["platform"]) - for upstream_url in sorted(by_upstream.keys()): - cores = by_upstream[upstream_url] - upstream_short = upstream_url.replace("https://github.com/", "") - classifications = set() - all_plats: set[str] = set() - for c in cores: - raw_cls = unique[c].get("core_classification", "-") - classifications.add(CLS_LABELS.get(raw_cls, raw_cls)) - all_plats.update(platform_membership.get(c, set())) - - cls_str = ", ".join(sorted(classifications)) - plat_str = ", ".join(sorted(all_plats)) if all_plats else "-" - core_links = ", ".join(f"[{c}](emulators/{c}.md)" for c in sorted(cores)) - - if upstream_url.startswith("http"): - upstream_cell = f"[{upstream_short}]({upstream_url})" - else: - upstream_cell = upstream_short + for key in sorted(by_upstream, key=lambda k: _upstream_label(k).casefold()): + members = by_upstream[key] + cores = sorted(c for c, _url, _rest in members) + classifications = sorted({ + CLS_LABELS.get(unique[c].get("core_classification", "-"), + unique[c].get("core_classification", "-")) + for c in cores + }) + plats = sorted(set().union(*(platform_membership.get(c, set()) for c in cores))) + revisions = sorted({rest for _c, _url, rest in members if rest}) + urls = sorted({url.split()[0] for _c, url, _rest in members}) + # The project's own address: the bare one when a core cites it. + home = next((u for u in urls if not upstream_project(u)[1]), urls[0]) + upstream_cell = (f"[{_breakable(_upstream_label(key))}]({home})" if home.startswith("http") + else _escape_tags(key)) lines.append( - f"| {upstream_cell} | {core_links} | " - f"{cls_str} | {plat_str} |" + "| " + ", ".join(f"[{c}](emulators/{c}.md)" for c in cores) + + f" | {upstream_cell} | {', '.join(_breakable(r) for r in revisions) or '-'} " + f"| {', '.join(classifications)} | {len(plats)} |" ) - return lines def generate_cross_reference( coverages: dict, profiles: dict, + moved: dict[str, str] | None = None, ) -> str: - """Generate cross-reference: Platform -> Core -> Systems -> Upstream.""" + """Cross-reference: counts per platform, then cores by upstream project. + + ``moved`` collects one page per platform with its core table. + """ unique = { k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test") } total_cores = len(unique) total_upstreams = len({ - p.get("upstream", p.get("source", "")) - for p in unique.values() - if p.get("upstream") or p.get("source") + upstream_project(_xref_upstream(p))[0] for p in unique.values() if _xref_upstream(p) }) lines = [ @@ -5906,7 +6330,7 @@ def generate_cross_reference( "", ] - lines.extend(_render_xref_by_platform(coverages, unique)) + lines.extend(_render_xref_by_platform(coverages, unique, moved)) lines.extend(_render_xref_by_upstream(coverages, unique)) lines.extend(["", f"*Generated on {_timestamp()}*"]) @@ -6786,10 +7210,14 @@ def _generate(args: argparse.Namespace) -> None: write_if_changed(str(docs / "emulators" / f"{stem}.md"), page) print("Generating cross-reference page...") + xref_pages: dict[str, str] = {} write_if_changed( str(docs / "cross-reference.md"), - tables_out_of_search(generate_cross_reference(coverages, profiles)) + reference_next(), + tables_out_of_search(generate_cross_reference(coverages, profiles, xref_pages)) + + reference_next(), ) + for stem, page in xref_pages.items(): + write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page)) print("Generating gap analysis page...") write_if_changed( diff --git a/scripts/sitesources.py b/scripts/sitesources.py index f1a62110..267fe766 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -7,6 +7,7 @@ from __future__ import annotations import re import urllib.parse +from dataclasses import dataclass import urllib.request import upstream @@ -345,3 +346,98 @@ _PATH_RE = re.compile(r"^[A-Za-z0-9_.@/+~-]+$") def _guarded_spans(line: str) -> list[tuple[int, int]]: """Code spans and links of a markdown line.""" return [m.span() for m in re.finditer(r"`[^`]*`|\[[^\]]*\]\([^)]*\)", line)] + + +# Where a platform's list was read: the forge, the repository, the revision +# and the path inside it, out of the URL its scraper pinned. +_LIST_URL_RES = ( + re.compile(r"^https://raw\.githubusercontent\.com/(?P[^/]+)/(?P[^/]+)/" + r"(?P[^/]+)/(?P.+)$"), + re.compile(r"^https://github\.com/(?P[^/]+)/(?P[^/]+)/(?:blob|tree)/" + r"(?P[^/]+)(?:/(?P.+))?$"), + re.compile(r"^https://codeload\.github\.com/(?P[^/]+)/(?P[^/]+)/" + r"tar\.gz/(?P[^/]+)$"), + re.compile(r"^https://(?Pgitlab\.com)/(?P[^/]+)/(?P[^/]+)/-/" + r"(?:raw|blob)/(?P[^/]+)/(?P.+)$"), +) +_BRANCHES = frozenset({"master", "main", "db"}) + + +@dataclass(frozen=True) +class ListSource: + """A platform's list as its scraper read it.""" + + host: str + owner: str + repo: str + rev: str + path: str + + @property + def moving(self) -> bool: + """The revision is a branch: what was read is not recorded.""" + return not self.rev or self.rev in _BRANCHES + + @property + def label(self) -> str: + """`batocera-systems at tag batocera-43.1`, said in words.""" + name = self.path.rsplit("/", 1)[-1] if self.path else f"{self.repo} repository" + if not self.rev: + return f"{name}, default branch" + if re.fullmatch(r"[0-9a-f]{40}", self.rev): + return f"{name} at commit {self.rev[:12]}" + if self.moving: + return f"{name} on branch {self.rev}" + return f"{name} at tag {self.rev}" + + def browse(self, path: str = "", start: int | None = None, + end: int | None = None) -> str: + """The page a reader opens: the file, or the tree, at the revision.""" + path = path or self.path + if self.host == "gitlab.com": + url = f"https://gitlab.com/{self.owner}/{self.repo}/-/blob/{self.rev}/{path}" + anchor = f"#L{start}" + (f"-{end}" if end and end != start else "") + else: + base = f"https://github.com/{self.owner}/{self.repo}" + if not self.rev: + return base + kind = "blob" if path else "tree" + url = f"{base}/{kind}/{self.rev}" + (f"/{path}" if path else "") + anchor = f"#L{start}" + (f"-L{end}" if end and end != start else "") + return url + (anchor if start else "") + + +def list_source(url: str) -> ListSource | None: + """The forge, repository, revision and path of a platform's `source:`.""" + url = str(url or "").strip() + for pattern in _LIST_URL_RES: + found = pattern.match(url) + if found: + parts = found.groupdict() + return ListSource(parts.get("host") or "github.com", parts["owner"], + parts["repo"], parts["rev"], parts.get("path") or "") + bare = re.match(r"^https://github\.com/([^/]+)/([^/]+?)/?$", url) + if bare: + return ListSource("github.com", bare[1], bare[2], "", "") + return None + + +def launcher_ref_markdown(source: ListSource | None, ref: str) -> str: + """A launcher record's source_ref, each part a pinned link. + + Linked only where the caller knows the launcher code sits in the + repository and revision of the platform's list. Otherwise each part is + plain code: a link to another repository is a false citation. + """ + rendered: list[str] = [] + for part in split_source_ref(ref): + display = part.path.rsplit("/", 1)[-1] + if part.start is not None: + display += f":{part.start}" + if part.end is not None and part.end != part.start: + display += f"-{part.end}" + if source is None or not source.rev or not _PATH_RE.fullmatch(part.path): + rendered.append(f"`{display}`") + else: + rendered.append(f"[`{display}`]({source.browse(part.path, part.start, part.end)})") + return ", ".join(rendered) diff --git a/tests/test_site_cores.py b/tests/test_site_cores.py index 0133e397..11b14d87 100644 --- a/tests/test_site_cores.py +++ b/tests/test_site_cores.py @@ -33,12 +33,17 @@ class SiteCoreCounts(unittest.TestCase): for name in list_registered_platforms(platforms, include_archived=True): config = load_platform_config(name, platforms) coverages[name] = {"platform": config.get("platform", name), "config": config} - page = generate_site.generate_cross_reference(coverages, profiles) + moved: dict[str, str] = {} + page = generate_site.generate_cross_reference(coverages, profiles, moved) shown = { m.group(1): int(m.group(2)) - for m in re.finditer( - r'([^<]+)\n\n.*\n\n\*\*(\d+) cores', page) + for m in re.finditer(r"^\| \[([^\]]+)\]\(platforms/[^)]+\) \| (\d+)", page, re.M) } + for name, cov in coverages.items(): + with self.subTest(page=name): + rows = [line for line in moved[f"cross-reference-{name}"].splitlines() + if line.startswith("| [")] + self.assertEqual(len(rows), shown[cov["platform"]]) for name, cov in coverages.items(): with self.subTest(platform=name): expected = len(resolve_platform_cores(cov["config"], unique)) diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 2357731f..9960d9c6 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -621,7 +621,9 @@ class APlatformPageNamesTheFileItTranscribes(unittest.TestCase): return generate_site.generate_platform_page("p", cov) tag = "https://raw.githubusercontent.com/o/r/batocera-43.1/x/batocera-systems" - self.assertIn(f"| Transcribed from | <{tag}> |", page(tag)) + self.assertIn( + "| Transcribed from | [batocera-systems at tag batocera-43.1]" + "(https://github.com/o/r/blob/batocera-43.1/x/batocera-systems) |", page(tag)) branch = "https://raw.githubusercontent.com/o/r/master/x.json" self.assertIn("revision read is not recorded", page(branch)) self.assertIn("revision read is not recorded", page("https://github.com/RetroDECK/components")) diff --git a/tests/test_site_platform_hashes.py b/tests/test_site_platform_hashes.py index 27b52a8a..15e5df32 100644 --- a/tests/test_site_platform_hashes.py +++ b/tests/test_site_platform_hashes.py @@ -10,11 +10,13 @@ from __future__ import annotations import sys import unittest +from unittest import mock from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "scripts")) +import siterender # noqa: E402 from generate_site import _render_platform_systems # noqa: E402 A, B, C = "a" * 32, "b" * 32, "c" * 32 @@ -36,7 +38,9 @@ class EachRowItsDeclaration(unittest.TestCase): {"name": "boot.rom", "status": "ok", "system": "saturn"}, ], } - text = "\n".join(_render_platform_systems(by_system, config_files, {})) + names = {"psx": "Sony PlayStation", "saturn": "Sega Saturn"} + with mock.patch.dict(siterender.SYSTEM_NAMES, names): + text = "\n".join(_render_platform_systems(by_system, config_files, {})) psx, saturn = text.split("saturn", 1)[0], text.split("saturn", 1)[1] self.assertIn(A, psx) self.assertNotIn(B, psx) diff --git a/tests/test_site_r2_reference.py b/tests/test_site_r2_reference.py new file mode 100644 index 00000000..9234398c --- /dev/null +++ b/tests/test_site_r2_reference.py @@ -0,0 +1,357 @@ +"""Reference pages (round 2, package 8). + +A reference page cites a hash, a source line or a count, and each citation +has to hold when an expert checks it: the reader of a pinned dump is the +profile entry accepting its hash, a source line is a pinned link, an entry +names the file that serves it, and the exports carry what the page shows. +""" + +from __future__ import annotations + +import csv +import io +import json +import re +import sys +import unittest +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site # noqa: E402 +import siterender # noqa: E402 +from common import ( # noqa: E402 + list_registered_platforms, + load_emulator_profiles, + load_platform_config, +) +from exporter.baseline import HASH_FIELDS, _hash_values # noqa: E402 +from exporter.candidates import EntryCategories # noqa: E402 +from sitesources import launcher_ref_markdown, list_source # noqa: E402 + +PLATFORMS = str(REPO_ROOT / "platforms") +DIST = REPO_ROOT / "website" / "dist" + + +def _accepted(profile_entry: dict) -> set[str]: + values = {v for f in HASH_FIELDS for v in _hash_values(profile_entry, f)} + for member in profile_entry.get("contents") or []: + if isinstance(member, dict): + values |= {v for f in HASH_FIELDS for v in _hash_values(member, f)} + return values + + +class ReaderMatchedByHash(unittest.TestCase): + """Batocera's scph1001.bin (md5 dc2b9bf8, SCPH-1001 v2.0) cited + DuckStation's bios.cpp:36, the line of the v2.2 dump: the reader was + matched by name.""" + + @classmethod + def setUpClass(cls): + cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + + def test_batocera_scph1001_cites_the_v20_line(self): + config = load_platform_config("batocera", PLATFORMS) + cats = EntryCategories(config, self.profiles) + entry = next(e for e in config["systems"]["sony-playstation"]["files"] + if e["name"] == "scph1001.bin") + found = cats.explain(entry, entry.get("destination") or entry["name"], + "sony-playstation") + duck = next(r for r in found["read_by"] if r["emulator"] == "duckstation") + self.assertEqual(duck["entries"], [ + {"name": "scph1001_v20.bin", "source_ref": "src/core/bios.cpp:42"}]) + + def test_every_cited_entry_accepts_the_md5_or_says_by_name(self): + by_name = 0 + for platform in list_registered_platforms(PLATFORMS, include_archived=True): + config = load_platform_config(platform, PLATFORMS) + cats = EntryCategories(config, self.profiles) + for sys_id, system in (config.get("systems") or {}).items(): + for entry in system.get("files") or []: + md5s = set(_hash_values(entry, "md5")) | set(_hash_values(entry, "alt_md5")) + if not md5s: + continue + # The same dump, named by whichever hash both sides state. + declared = md5s | {v for f in HASH_FIELDS for v in _hash_values(entry, f)} + found = cats.explain(entry, entry.get("destination") or entry["name"], sys_id) + for reader in found.get("read_by", []): + if reader["emulator"] == "launcher": + continue + files = self.profiles[reader["emulator"]].get("files") or [] + for cited in reader["entries"]: + if cited.get("by") == "name": + by_name += 1 + continue + accepted = set().union(*( + _accepted(f) for f in files + if isinstance(f, dict) and f.get("name") == cited["name"])) + with self.subTest(platform=platform, entry=entry["name"], + emulator=reader["emulator"]): + self.assertTrue(any( + a == m or (29 <= len(m) < 32 and a.startswith(m)) + for m in declared for a in accepted)) + self.assertGreater(by_name, 0) + + def test_a_contradicting_homonym_is_not_cited(self): + profiles = {"core": {"type": "libretro", "systems": ["s"], "files": [ + {"name": "bios.bin", "md5": "1" * 32, "source_ref": "a.c:1"}, + {"name": "bios_v2.bin", "md5": "2" * 32, "source_ref": "a.c:2"}, + ]}} + config = {"cores": ["core"], "systems": {"s": {"files": []}}} + cats = EntryCategories(config, profiles) + found = cats.explain({"name": "bios.bin", "md5": "2" * 32}, "bios.bin", "s") + self.assertEqual(found["read_by"][0]["entries"], + [{"name": "bios_v2.bin", "source_ref": "a.c:2"}]) + found = cats.explain({"name": "bios.bin", "md5": "3" * 32}, "bios.bin", "s") + self.assertEqual(found["read_by"][0]["entries"], []) + + +class EvidenceLinksItsSource(unittest.TestCase): + """Platform pages printed refs as code with the role word in front: + `source frontend/libretro.c:3864-3867`.""" + + def test_pinned_link_and_by_name(self): + evidence = {"read_by": [{ + "emulator": "pcsx_rearmed", "proof": "name", "runs_system": True, + "entries": [{"name": "scph1001.bin", "by": "name", + "source_ref": "source frontend/libretro.c:3864-3867 (listed_bios[6])"}], + }]} + text = generate_site._category_evidence_md(evidence) + self.assertRegex(text, r"\[`frontend/libretro\.c:3864-3867`\]\(https://github\.com/" + r"libretro/pcsx_rearmed/blob/[0-9a-f]{40}/frontend/libretro\.c" + r"#L3864-L3867\)") + self.assertIn("(by name)", text) + self.assertNotIn("`source ", text) + + def test_launcher_record_named_with_its_lines(self): + evidence = {"read_by": [{"emulator": "launcher", "proof": "launcher", + "entries": [], "runs_system": True}]} + config = load_platform_config("batocera", PLATFORMS) + launched = generate_site._launcher_refs("batocera", config)["scph1001.bin"] + text = generate_site._category_evidence_md(evidence, launched) + self.assertNotIn("(by launcher)", text) + self.assertIn("https://github.com/batocera-linux/batocera.linux/blob/batocera-43.1/" + "package/batocera/core/batocera-configgen/", text) + self.assertIn("duckstationGenerator.py:115-136", text) + + def test_a_launcher_in_another_repository_is_not_linked(self): + source = list_source("https://gitlab.com/recalbox/recalbox/-/raw/10.1.1/x/es_bios.xml") + self.assertEqual(launcher_ref_markdown(None, "a/b.py:3"), "`b.py:3`") + self.assertIn("gitlab.com/recalbox/recalbox/-/blob/10.1.1/a/b.py#L3", + launcher_ref_markdown(source, "a/b.py:3")) + + +class EachEntryNamesItsFile(unittest.TestCase): + def test_served_held_and_missing(self): + path, digest = next( + (p, d) for p, d in generate_site._collection_sha1_by_path().items() + if p.startswith("bios/") and p.count("/") >= 3) + served = generate_site._served_line({"status": "ok", "path": path}) + self.assertRegex(served, rf"\]\(\.\./systems/[^)]+\.md#sha1-{digest[:12]}\)") + self.assertTrue(served.startswith("Served by: ")) + self.assertIn(f"`{digest[:8]}`", served) + self.assertEqual(generate_site._served_line({"status": "missing"}), "Not held") + self.assertTrue(generate_site._served_line( + {"status": "untested", "path": path}).startswith("Held, not the declared hash")) + # Existence mode reports no path: nothing is claimed, held or not. + self.assertEqual(generate_site._served_line({"status": "ok"}), "") + + def test_a_long_system_list_is_cut_between_files(self): + body = ["2/2 files verified", ""] + for n in range(4): + body += [f"- `f{n}.bin` - OK", " " + "x" * 100] + runs = generate_site._file_runs(body, 250) + self.assertEqual(len(runs), 2) + for run in runs: + self.assertEqual(run[:2], body[:2]) + self.assertTrue(run[2].startswith("- ")) + self.assertEqual(sum(len(run) - 2 for run in runs), len(body) - 2) + + +class AlsoNamedIsNotCited(unittest.TestCase): + def test_other_system_cores_are_named_only(self): + evidence = {"read_by": [ + {"emulator": "duckstation", "proof": "name", "runs_system": True, + "entries": [{"name": "scph1001_v20.bin", "source_ref": "src/core/bios.cpp:42"}]}, + {"emulator": "mame", "proof": "name", "runs_system": False, + "entries": [{"name": "x.zip", "source_ref": "src/mame/x.cpp:1", "by": "name"}]}, + ]} + text = generate_site._category_evidence_md(evidence) + also = text.split(", also named by ", 1)[1] + self.assertNotIn("x.cpp", also) + self.assertIn("mame", also) + + +class PlatformReferenceInWords(unittest.TestCase): + def _page(self, mode: str, hash_type: str, systems: dict | None = None, + registry: dict | None = None) -> str: + details = [{"name": "f.bin", "status": "ok", "system": s} for s in (systems or {})] + cov = {"platform": "P", "details": details, "total": 1, "verified": 1, + "present": 1, "missing": 0, "untested": 0, "mode": mode, + "config": {"hash_type": hash_type, "systems": systems or {}, + "source": "https://github.com/o/r/tree/" + "a" * 40}} + return generate_site.generate_platform_page("p", cov, registry or {}) + + def test_hash_type_only_when_it_differs(self): + self.assertNotIn("Hash type", self._page("md5", "md5")) + self.assertNotIn("Hashes in the list", self._page("md5", "md5")) + page = self._page("existence", "sha1") + self.assertIn("| Hashes in the list | SHA-1 |", page) + self.assertIn(">name only", page) + self.assertIn(">MD5", self._page("md5", "md5")) + self.assertIn("[r repository at commit aaaaaaaaaaaa](https://github.com/o/r/tree/" + + "a" * 40 + ")", page) + + def test_systems_by_name_with_the_id_in_small_text(self): + systems = {"sony-playstation": {"files": [{"name": "f.bin"}]}} + page = self._page("md5", "md5", systems) + name = siterender.system_name("sony-playstation") + self.assertIn(f"| [{name}](#sony-playstation) sony-playstation |", page) + heading = next(line for line in page.splitlines() if line.startswith("### ") + and "sony-playstation" in line) + self.assertIn(name, heading) + self.assertTrue(heading.endswith("sony-playstation { #sony-playstation }")) + + def test_logo_in_front_matter_not_floated(self): + page = self._page("md5", "md5", registry={"p": {"logo": "https://x.test/l.png"}}) + self.assertTrue(page.startswith('---\nlogo: "https://x.test/l.png"\n---\n')) + self.assertNotIn('align="right"', page) + + +class GapsPageNumbers(unittest.TestCase): + def test_badge_only_for_missing_required_firmware(self): + self.assertNotIn("rb-badge", generate_site._ratio_cell(99, 100)) + self.assertIn('class="rb-bar"', generate_site._ratio_cell(99, 100)) + self.assertIn("rb-badge-danger", generate_site._ratio_cell(99, 100, 1)) + cov = {"details": [ + {"name": "a", "system": "s", "status": "missing", "required": True, "category": "bios"}, + {"name": "b", "system": "s", "status": "missing", "required": True, + "category": "game_data"}, + {"name": "c", "system": "s", "status": "missing", "required": False}, + ]} + self.assertEqual(generate_site._missing_required_firmware(cov), 1) + + def test_headings_and_header(self): + source = Path(generate_site.__file__).read_text(encoding="utf-8") + body = source[source.index("def generate_gap_analysis("):] + body = body[:body.index("\ndef ", 10)] + self.assertIn('"# What the packs do not hold"', body) + self.assertIn("On its BIOS list (destinations)", body) + self.assertIn('"### Files to source"', body) + self.assertNotIn("### Files to source (", body) + + +class UpstreamsGrouped(unittest.TestCase): + def test_mame_trees_are_one_project(self): + keys = {generate_site.upstream_project(u)[0] for u in ( + "https://github.com/mamedev/mame", + "https://github.com/mamedev/mame/tree/mame0289", + "https://github.com/mamedev/mame/tree/mame0135u4", + )} + self.assertEqual(keys, {"github.com/mamedev/mame"}) + self.assertEqual(generate_site.upstream_project( + "https://github.com/mamedev/mame/tree/mame0289")[1], "mame0289") + + def test_nested_groups_notes_and_archives(self): + project = generate_site.upstream_project + self.assertEqual( + project("https://gitlab.com/recalbox/packages/recalbox-cores/libretro-exel-hence"), + ("gitlab.com/recalbox/packages/recalbox-cores/libretro-exel-hence", "")) + self.assertEqual(project("https://gitlab.com/jgemu/geolith/-/tree/v1"), + ("gitlab.com/jgemu/geolith", "v1")) + self.assertEqual( + project("https://git.9front.org/plan9front/plan9front (sys/src/games/nes/)"), + ("git.9front.org/plan9front/plan9front", "sys/src/games/nes/")) + key, _ = project("https://archive.softwareheritage.org/browse/revision/" + "dc20a615275f66bee20a4fd851ef0231daca4f14/") + self.assertEqual(generate_site._upstream_label(key), + "Software Heritage revision dc20a615275f") + + def test_overview_and_a_page_per_platform(self): + profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + coverages = {} + for name in ("batocera", "retroarch"): + config = load_platform_config(name, PLATFORMS) + coverages[name] = {"platform": config.get("platform", name), "config": config} + moved: dict[str, str] = {} + page = generate_site.generate_cross_reference(coverages, profiles, moved) + self.assertEqual(set(moved), {"cross-reference-batocera", "cross-reference-retroarch"}) + self.assertNotIn("mame\]\(", upstream)), 1) + header = next(line for line in upstream.splitlines() if line.startswith("| Cores")) + self.assertTrue(header.startswith("| Cores | Upstream |")) + + +class ExportsComplete(unittest.TestCase): + def test_gap_rows_once_with_their_outcome(self): + path = next(iter(generate_site._collection_sha1_by_path())) + detail = {"name": "x.bin", "system": "s", "status": "untested", "path": path, + "category": "bios"} + coverages = {"p": {"platform": "P", "details": [ + {"name": "h.xml", "system": "s", "status": "missing"}, + {"name": "h.xml", "system": "s", "status": "missing"}, detail]}} + db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8")) + rows = generate_site._gap_export_rows(coverages, {}, db) + self.assertEqual([r["name"] for r in rows], ["h.xml", "x.bin"]) + held = rows[1] + self.assertEqual(held["collection_path"], path) + self.assertEqual(held["resolved_sha1"], generate_site._collection_sha1_by_path()[path]) + self.assertEqual(held["category"], "bios") + + def test_platform_rows_carry_status_and_file(self): + db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8")) + sha1, entry = next(iter(db["files"].items())) + coverages = {"p": {"platform": "P", "config": {"systems": {"s": {"files": [ + {"name": "a.bin"}, {"name": "b.bin"}]}}}, "details": [ + {"name": "a.bin", "system": "s", "status": "ok", "path": entry["path"], + "category": "bios"}, + {"name": "b.bin", "system": "s", "status": "missing", "category": "game_data"}]}} + _items, rows = generate_site._platform_export_rows(coverages, db) + self.assertEqual((rows[0]["status"], rows[0]["resolved_sha1"]), ("ok", sha1)) + self.assertEqual(rows[0]["collection_path"], entry["path"]) + self.assertEqual((rows[1]["status"], rows[1]["category"], rows[1]["collection_path"]), + ("missing", "game_data", "")) + + def test_commit_and_wanted_list(self): + commit = generate_site._source_commit() + self.assertRegex(commit, r"^[0-9a-f]{40}$") + schema = json.loads((REPO_ROOT / "schemas" / "site-api-envelope.schema.json") + .read_text(encoding="utf-8")) + self.assertIn("commit", schema["properties"]) + db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8")) + rows = generate_site._provenance_wanted_rows(db) + self.assertTrue(rows) + self.assertTrue(all(r["catalog"] and r["name"] for r in rows)) + text = generate_site._csv_document(list(generate_site.WANTED_FIELDS), rows) + self.assertEqual(next(csv.reader(io.StringIO(text))), list(generate_site.WANTED_FIELDS)) + + +class DatesAndNames(unittest.TestCase): + def test_snapshot_date_in_words(self): + self.assertEqual(generate_site._date_in_words("2026-08-07T10:00:00Z"), + '') + + def test_data_page_title(self): + self.assertTrue(generate_site.generate_data_page([]).startswith("# Data and API\n")) + + +@unittest.skipUnless((DIST / "platforms" / "batocera" / "index.html").is_file(), + "site not built") +class BuiltBatoceraPage(unittest.TestCase): + def test_served_by_and_pinned_links(self): + main = (DIST / "platforms" / "batocera" / "index.html").read_text(encoding="utf-8") + self.assertTrue("batocera-systems at tag batocera-43.1" in main) + # PlayStation's list is long enough to sit on a page of its own. + pages = [main] + [p.read_text(encoding="utf-8") for p in + (DIST / "platforms").glob("batocera-sony-playstation*/index.html")] + html = "".join(pages) + for text in ("Served by:", "scph1001_v20.bin", "https://github.com/stenzek/duckstation/blob/"): + with self.subTest(text=text): + self.assertTrue(text in html) + self.assertFalse("(by launcher)" in html) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_render.py b/tests/test_site_render.py index 83c4e69b..b9ac119e 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -312,7 +312,9 @@ class TheSearchLandsOnTheCard(unittest.TestCase): by_system = {f"system-{i:02d}": [{"name": f"f{i}.bin", "status": "ok"}] for i in range(11)} - lines = generate_site._render_platform_systems(by_system, {}, {}) + names = {sys_id: f"System {sys_id[-2:]}" for sys_id in by_system} + with mock.patch.dict(siterender.SYSTEM_NAMES, names): + lines = generate_site._render_platform_systems(by_system, {}, {}) headings = [line for line in lines if line.startswith("### ")] self.assertEqual(len(headings), 11) self.assertTrue(headings[0].endswith("{ #system-00 }")) diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py index b553ac43..f78ee010 100644 --- a/tests/test_site_validation.py +++ b/tests/test_site_validation.py @@ -353,8 +353,12 @@ class PlatformPagesPrintTheDeclaredCrc32(unittest.TestCase): from tests import sitepages # noqa: PLC0415 sitepages.require() - page = sitepages.page("platforms/retroarch.md") - self.assertIn("CRC32: `91e93107`", page.read_text(encoding="utf-8")) + # The page and the systems moved to pages of their own. + text = "".join( + path.read_text(encoding="utf-8") for path in sitepages.pages() + if sitepages.name(path) == "platforms/retroarch.md" + or sitepages.name(path).startswith("platforms/retroarch-")) + self.assertIn("CRC32: `91e93107`", text) class PlainHouseStyle(unittest.TestCase): diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 353e260c..14fcf203 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -56,7 +56,7 @@ Every module in `tests/`, by the area it guards. | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture