diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json
index dcd422b2..584ee44a 100644
--- a/schemas/site-api-envelope.schema.json
+++ b/schemas/site-api-envelope.schema.json
@@ -19,6 +19,11 @@
"type": "string",
"format": "date-time"
},
+ "commit": {
+ "type": "string",
+ "pattern": "^[0-9a-f]{40}$",
+ "description": "The repository commit the data was read at."
+ },
"kind": {
"type": "string",
"enum": [
diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py
index 7af1656c..59df9a4e 100644
--- a/scripts/exporter/candidates.py
+++ b/scripts/exporter/candidates.py
@@ -41,6 +41,37 @@ def _entry_hashes(entry: dict) -> set[str]:
return values
+def _declared_hashes(entry: dict) -> dict[str, set[str]]:
+ """Hash field -> the values a platform entry declares, alt_md5 as md5."""
+ found = {f: set(_hash_values(entry, f)) for f in HASH_FIELDS}
+ found["md5"] |= set(_hash_values(entry, "alt_md5"))
+ return {f: values for f, values in found.items() if values}
+
+
+def _same(declared: str, accepted: str) -> bool:
+ """Equal, or a cut md5 that starts the accepted one."""
+ return declared == accepted or (
+ _CUT_MD5 <= len(declared) < _FULL_MD5 and accepted.startswith(declared)
+ )
+
+
+def _accepts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
+ """True when the profile entry, or a member of its archive, accepts a
+ hash the platform entry declares."""
+ accepted = _entry_hashes(profile_entry)
+ return any(_same(d, a) for values in declared.values() for d in values for a in accepted)
+
+
+def _contradicts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
+ """True when both sides state the same kind of hash and no value agrees:
+ the profile entry describes another dump under the same name."""
+ for field_name, values in declared.items():
+ own = set(_hash_values(profile_entry, field_name))
+ if own and not any(_same(d, a) for d in values for a in own):
+ return True
+ return False
+
+
@dataclass
class ReadEvidence:
"""What the examined profiles open, indexed for one lookup per entry."""
@@ -373,20 +404,34 @@ class EntryCategories:
)
def _matching(self, key: str, kind: str, entry: dict) -> list[dict]:
- """The profile entries of `key` that name the platform entry."""
+ """The profile entries of `key` that read the platform entry.
+
+ A platform entry that declares a hash is answered by the profile
+ entries accepting that hash: one name covers several revisions
+ (DuckStation lists scph1001.bin v2.2 at bios.cpp:36 and v2.0 at
+ bios.cpp:42), and a match by name cited the line of another dump.
+ A same-named entry is kept, marked `"by": "name"`, only when no
+ entry accepts the hash and its own hashes do not contradict it.
+ """
names = {
str(n).casefold()
for n in (entry.get("name"), *(entry.get("aliases") or []))
if n
}
- found = []
+ declared = _declared_hashes(entry)
+ by_hash: list[dict] = []
+ by_name: list[dict] = []
for f in self._profiles[key].get("files") or []:
if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA):
continue
+ cited = {k: f[k] for k in ("name", "source_ref") if f.get(k)}
+ if declared and _accepts(f, declared):
+ by_hash.append(cited)
+ continue
own = {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n}
- if own & names:
- found.append({k: f[k] for k in ("name", "source_ref") if f.get(k)})
- return found
+ if own & names and not (declared and _contradicts(f, declared)):
+ by_name.append({**cited, "by": "name"})
+ return by_hash or by_name
def explain(self, entry: dict, destination: str, system: str) -> dict:
"""The category of an entry and what establishes it."""
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index b721b603..ce53ed8c 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -768,12 +768,47 @@ def _sha256_path(path: Path) -> str:
return digest.hexdigest()
-def _platform_export_rows(coverages: dict) -> tuple[list[dict], list[dict]]:
+def _verify_outcomes(coverage: dict, sha1_by_path: dict[str, str]):
+ """For each declared entry, in declaration order: what verify found.
+
+ Returns a function of (system, name) giving the next outcome for that
+ pair: status, category, the collection path that answers the entry and
+ that file's sha1. The CSV gave the declaration and nothing about it.
+ """
+ details: dict[tuple[str, str], list[dict]] = {}
+ for detail in coverage.get("details") or []:
+ details.setdefault((detail.get("system", ""), detail.get("name", "")), []).append(detail)
+ taken: dict[tuple[str, str], int] = {}
+
+ def outcome(system: str, name: str) -> dict:
+ found = details.get((system, name)) or [{}]
+ nth = taken.get((system, name), 0)
+ taken[(system, name)] = nth + 1
+ detail = found[min(nth, len(found) - 1)]
+ path = str(detail.get("path") or "") if detail.get("status") != "missing" else ""
+ return {
+ "status": detail.get("status", ""),
+ "category": detail.get("category", ""),
+ "collection_path": path,
+ "resolved_sha1": sha1_by_path.get(path, "") if path else "",
+ }
+ return outcome
+
+
+def _sha1_by_path(db: dict | None) -> dict[str, str]:
+ return {str(e.get("path", "")): sha1 for sha1, e in ((db or {}).get("files") or {}).items()}
+
+
+def _platform_export_rows(
+ coverages: dict, db: dict | None = None,
+) -> tuple[list[dict], list[dict]]:
"""Normalized platform summaries and one row per declared file."""
items: list[dict] = []
file_rows: list[dict] = []
+ sha1_by_path = _sha1_by_path(db)
for key, coverage in sorted(coverages.items()):
config = coverage["config"]
+ outcome = _verify_outcomes(coverage, sha1_by_path)
items.append({
"id": key,
"name": coverage["platform"],
@@ -804,6 +839,7 @@ def _platform_export_rows(coverages: dict) -> tuple[list[dict], list[dict]]:
"sha256": _json_text(entry.get("sha256")),
"md5": _json_text(entry.get("md5")),
"crc32": _json_text(entry.get("crc32")),
+ **outcome(system_id, entry.get("name", "")),
})
return items, file_rows
@@ -867,7 +903,10 @@ def build_emulator_gap_report(
)
-def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[dict]:
+def _gap_export_rows(
+ coverages: dict, gap_report: dict | None = None, db: dict | None = None,
+ profiles: dict | None = None,
+) -> list[dict]:
"""Every gap the site reports, both layers, in one table.
``platform`` rows are anomalies against a platform's own BIOS list.
@@ -876,11 +915,16 @@ def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[di
A `layer` column keeps the two apart instead of publishing only one.
"""
rows: list[dict] = []
+ sha1_by_path = _sha1_by_path(db)
for key, coverage in sorted(coverages.items()):
for detail in coverage.get("details", []):
if detail.get("status") == "ok" and not detail.get("discrepancy"):
continue
+ path = str(detail.get("path") or "") if detail.get("status") != "missing" else ""
rows.append({
+ "category": detail.get("category", "bios"),
+ "collection_path": path,
+ "resolved_sha1": sha1_by_path.get(path, "") if path else "",
"layer": "platform",
"platform_id": key,
"platform": coverage["platform"],
@@ -897,8 +941,13 @@ def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[di
})
for emu_key, data in sorted((gap_report or {}).items()):
systems = ";".join(str(s) for s in data.get("systems", []) or [])
+ categories = _profile_categories((profiles or {}).get(emu_key) or {})
for gap in data.get("gap_details", []) or []:
rows.append({
+ "category": categories.get(
+ (gap.get("name", ""), gap.get("path") or gap.get("name", "")), ""),
+ "collection_path": "",
+ "resolved_sha1": "",
"layer": "emulator",
"platform_id": "",
"platform": "",
@@ -922,6 +971,10 @@ def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[di
})
for entry in data.get("unsourceable", []) or []:
rows.append({
+ "category": categories.get(
+ (entry.get("name", ""), entry.get("path") or entry.get("name", "")), ""),
+ "collection_path": "",
+ "resolved_sha1": "",
"layer": "emulator",
"platform_id": "",
"platform": "",
@@ -936,7 +989,23 @@ def _gap_export_rows(coverages: dict, gap_report: dict | None = None) -> list[di
"destination": entry.get("path") or entry.get("name", ""),
"expected": "",
})
- return rows
+ # Two declarations that differ only in what the row does not carry are
+ # one row: AmberELEC's hardwareconfig.xml was printed twice.
+ unique: dict[tuple, dict] = {}
+ for row in rows:
+ unique.setdefault(tuple(sorted((k, str(v)) for k, v in row.items())), row)
+ return list(unique.values())
+
+
+def _profile_categories(profile: dict) -> dict[tuple[str, str], str]:
+ """(name, path) -> the category a profile gives the entry, bios by default."""
+ found: dict[tuple[str, str], str] = {}
+ for entry in profile.get("files") or []:
+ if isinstance(entry, dict) and entry.get("name"):
+ name = str(entry["name"])
+ found.setdefault((name, str(entry.get("path") or name)),
+ str(entry.get("category") or "bios"))
+ return found
def _cross_reference_export_rows(coverages: dict, profiles: dict) -> list[dict]:
@@ -1012,7 +1081,8 @@ def _write_sqlite_export(
platform_id TEXT NOT NULL, system TEXT NOT NULL, name TEXT NOT NULL,
destination TEXT NOT NULL, required INTEGER NOT NULL,
region TEXT, variant_group TEXT, size INTEGER,
- sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT
+ sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT,
+ status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT
);
CREATE TABLE emulators (
id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT,
@@ -1034,7 +1104,8 @@ def _write_sqlite_export(
emulator TEXT, system TEXT, name TEXT NOT NULL,
status TEXT NOT NULL, required INTEGER NOT NULL,
in_repo TEXT, reason TEXT, discrepancy TEXT,
- destination TEXT, expected TEXT
+ destination TEXT, expected TEXT, category TEXT,
+ collection_path TEXT, resolved_sha1 TEXT
);
CREATE TABLE accepted_hashes (
layer TEXT NOT NULL, owner TEXT NOT NULL, system TEXT,
@@ -1049,9 +1120,11 @@ def _write_sqlite_export(
CREATE INDEX emulator_files_name_idx ON emulator_files(name);
CREATE INDEX gaps_status_idx ON gaps(layer, status);
""")
+ commit = _source_commit()
metadata = {
"schema_version": "1",
"generated_at": str(db.get("generated_at") or _timestamp()),
+ **({"commit": commit} if commit else {}),
"source": REPO_URL,
"scope": "metadata only, no BIOS or firmware payload bytes",
}
@@ -1087,7 +1160,8 @@ def _write_sqlite_export(
connection.executemany(
"INSERT INTO platform_files VALUES "
"(:platform_id, :system, :name, :destination, :required, :region, "
- ":variant_group, :size, :sha1, :sha256, :md5, :crc32)",
+ ":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, "
+ ":collection_path, :resolved_sha1)",
platform_files,
)
for item in emulator_items:
@@ -1134,7 +1208,8 @@ def _write_sqlite_export(
connection.executemany(
"INSERT INTO gaps VALUES "
"(:layer, :platform_id, :platform, :emulator, :system, :name, "
- ":status, :required, :in_repo, :reason, :discrepancy, :destination, :expected)",
+ ":status, :required, :in_repo, :reason, :discrepancy, :destination, :expected, "
+ ":category, :collection_path, :resolved_sha1)",
gap_rows,
)
connection.commit()
@@ -1144,13 +1219,16 @@ def _write_sqlite_export(
os.replace(temp_path, destination)
-def write_single_item(path: Path, kind: str, item: dict, generated_at: str) -> None:
+def write_single_item(
+ path: Path, kind: str, item: dict, generated_at: str, commit: str = "",
+) -> None:
"""One item in the envelope of its collection, for a page to link."""
path.parent.mkdir(parents=True, exist_ok=True)
write_if_changed(
str(path),
json.dumps(
- {"schema_version": 1, "generated_at": generated_at, "kind": kind,
+ {"schema_version": 1, "generated_at": generated_at,
+ **({"commit": commit} if commit else {}), "kind": kind,
"count": 1, "items": [item]},
indent=2, sort_keys=True, ensure_ascii=False, default=str,
) + "\n",
@@ -1173,6 +1251,46 @@ def system_export_item(slug: str, manufacturer: str, consoles: dict[str, list])
}
+@functools.lru_cache(maxsize=1)
+def _source_commit() -> str:
+ """The commit of the checkout the site is built from, empty outside one."""
+ import subprocess
+
+ try:
+ result = subprocess.run(
+ ["git", "rev-parse", "HEAD"], cwd=_REPO_ROOT,
+ capture_output=True, text=True, check=True,
+ )
+ except (OSError, subprocess.CalledProcessError) as error:
+ print(f" no commit recorded in the exports: {error}", file=sys.stderr)
+ return ""
+ commit = result.stdout.strip()
+ return commit if re.fullmatch(r"[0-9a-f]{40}", commit) else ""
+
+
+# One row per dump a catalog describes and the collection lacks: the list
+# the provenance page publishes under "Wanted".
+WANTED_FIELDS = ("catalog", "dat", "name", "description", "size", "crc32", "md5", "sha1")
+
+
+def _provenance_wanted_rows(db: dict) -> list[dict]:
+ """The Wanted list of the provenance page, as rows. A dump the
+ collection already produces (derivable.json) is not wanted."""
+ derivable = load_derivable()
+ rows: list[dict] = []
+ for catalog, data in build_report(db, load_provenance_snapshots()).items():
+ for entry in data["missing"]:
+ if (entry.get("sha1") or "").lower() in derivable:
+ continue
+ for dat in entry.get("dats") or [entry.get("dat", "")]:
+ rows.append({
+ "catalog": catalog, "dat": dat,
+ **{f: _json_text(entry.get(f)) for f in WANTED_FIELDS[2:]},
+ })
+ rows.sort(key=lambda r: (r["catalog"], r["dat"], r["name"], r["sha1"]))
+ return rows
+
+
def generate_data_exports(
docs: Path,
db: dict,
@@ -1189,9 +1307,13 @@ def generate_data_exports(
directory.mkdir(parents=True, exist_ok=True)
generated_at = str(db.get("generated_at") or _timestamp())
- platform_items, platform_files = _platform_export_rows(coverages)
+ # The revision the data was read at: a tool comparing two downloads
+ # could not tell which commit each came from.
+ commit = _source_commit()
+ stamp = {"commit": commit} if commit else {}
+ platform_items, platform_files = _platform_export_rows(coverages, db)
emulator_items = _emulator_export_items(profiles)
- gap_rows = _gap_export_rows(coverages, gap_report)
+ gap_rows = _gap_export_rows(coverages, gap_report, db, profiles)
cross_rows = _cross_reference_export_rows(coverages, profiles)
members = data_directory_members()
@@ -1204,11 +1326,13 @@ def generate_data_exports(
}
write_lookup_shards(
api, file_lookup_records(db, coverages, profiles, locations, members), generated_at,
+ commit,
)
for filename, (kind, items) in envelopes.items():
document = {
"schema_version": 1,
"generated_at": generated_at,
+ **stamp,
"kind": kind,
"count": len(items),
"items": items,
@@ -1221,7 +1345,7 @@ def generate_data_exports(
# profile: platforms.json is 2.8 MB for twelve platforms.
for item in platform_items:
write_single_item(api / "platforms" / f"{item['id']}.json",
- "platforms", item, generated_at)
+ "platforms", item, generated_at, commit)
write_if_changed(str(api / "stats.json"), generate_stats(stats))
write_if_changed(
str(api / "database.json"),
@@ -1268,7 +1392,8 @@ def generate_data_exports(
[
"platform_id", "platform", "system", "name", "destination",
"required", "region", "variant_group", "size", "sha1",
- "sha256", "md5", "crc32",
+ "sha256", "md5", "crc32", "status", "category",
+ "collection_path", "resolved_sha1",
],
platform_files,
),
@@ -1290,7 +1415,8 @@ def generate_data_exports(
[
"layer", "platform_id", "platform", "emulator", "system",
"name", "status", "required", "in_repo", "reason",
- "discrepancy", "destination", "expected",
+ "discrepancy", "destination", "expected", "category",
+ "collection_path", "resolved_sha1",
],
gap_rows,
),
@@ -1307,6 +1433,10 @@ def generate_data_exports(
emulator_items, gap_rows,
)
_record(downloads / "retrobios.sqlite")
+ write_if_changed(
+ str(downloads / "provenance-wanted.csv"),
+ _csv_document(list(WANTED_FIELDS), _provenance_wanted_rows(db)),
+ )
assets = [
(api / "database.json", "Content database", "application/json", "schemas/database.schema.json"),
@@ -1322,6 +1452,7 @@ def generate_data_exports(
(downloads / "cross-reference.csv", "Cross-reference CSV", "text/csv", None),
(downloads / "gaps.csv", "Verification and coverage gaps CSV", "text/csv", None),
(downloads / "accepted-hashes.csv", "Accepted hashes, one row per value", "text/csv", None),
+ (downloads / "provenance-wanted.csv", "Catalogued dumps the collection lacks", "text/csv", None),
(downloads / "retrobios.sqlite", "SQLite snapshot", "application/vnd.sqlite3", None),
]
catalog_items: list[dict] = []
@@ -1348,6 +1479,7 @@ def generate_data_exports(
catalog = {
"schema_version": 1,
"generated_at": generated_at,
+ **stamp,
"kind": "catalog",
"count": len(published),
"items": published,
@@ -1361,7 +1493,7 @@ def generate_data_exports(
def generate_data_page(exports: list[dict]) -> str:
lines = [
- "# Data & API",
+ "# Data and API",
"",
"RetroBIOS publishes the same metadata used by its verifier, pack builder "
"and website as versioned static files. No account or API key is required.",
@@ -1397,7 +1529,8 @@ def generate_data_page(exports: list[dict]) -> str:
)
lines.extend([
"",
- "Every JSON document carries `schema_version`. Breaking changes use a new "
+ "Every JSON document carries `schema_version`, and `commit`, the revision "
+ "of this repository the data was read at. Breaking changes use a new "
"URL prefix (`/api/v2/`), and fields may only be added compatibly within v1.",
"",
"One emulator profile without the whole list: "
@@ -1452,7 +1585,14 @@ def generate_data_page(exports: list[dict]) -> str:
"`platform` rows are anomalies against a platform's own BIOS list, "
"`emulator` rows are files a profiled core loads that no platform "
"declares. The two answer different questions and are not comparable "
- "totals.",
+ "totals. Each row is listed once.",
+ "",
+ "`platform-files.csv` and the gaps add what the check found: `status` "
+ "(`ok`, `untested`, `missing`), `category` (`bios`, `game_data`, "
+ "`unread`), and for a held file `collection_path` and `resolved_sha1`, "
+ "the collection file that answers the declaration. "
+ "`provenance-wanted.csv` is the Wanted list of [Dump provenance]"
+ "(provenance.md): each dump a catalog describes and the collection lacks.",
"",
"## Semantics and limits",
"",
@@ -2163,11 +2303,119 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st
# A platform system whose file list passes this much Markdown gets a page
# of its own: RetroDECK's arcade list was 750 KB of a 1.1 MB page.
PLATFORM_SECTION_BUDGET = 200_000
+# A system's own page in runs of at most this much Markdown: RetroDECK's
+# arcade list, each file with its served copy and pinned readers, rendered
+# 1.4 MB of HTML on one page.
+PLATFORM_PART_BUDGET = 350_000
+# The Markdown of a platform page's file lists, past which the largest
+# systems move to pages of their own.
+PLATFORM_REFERENCE_BUDGET = 250_000
+
+
+def _file_runs(body: list[str], budget: int) -> list[list[str]]:
+ """A system's file list in runs of at most ``budget`` characters.
+
+ The lines before the first file (counts, emulators) open every run. A
+ file is never cut from its hash and evidence lines.
+ """
+ first = next((i for i, line in enumerate(body) if line.startswith("- ")), len(body))
+ head, items = body[:first], []
+ for line in body[first:]:
+ if line.startswith("- ") or not items:
+ items.append([line])
+ else:
+ items[-1].append(line)
+ runs: list[list[str]] = [[]]
+ size = 0
+ for item in items:
+ weight = sum(len(line) + 1 for line in item)
+ if runs[-1] and size + weight > budget:
+ runs.append([])
+ size = 0
+ runs[-1].extend(item)
+ size += weight
+ return [head + run for run in runs]
+
+
+@functools.lru_cache(maxsize=1)
+def _collection_sha1_by_path() -> dict[str, str]:
+ """The sha1 of each file the collection holds, by its path."""
+ with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle:
+ files = json.load(handle).get("files", {})
+ return {str(entry.get("path", "")): sha1 for sha1, entry in files.items()}
+
+
+def _system_heading_link(sys_id: str, prefix: str = "") -> str:
+ """A system's name, linked to its own console section when it has one.
+
+ The id stays beside it in small text: it is what the platform's list
+ writes. A system with no section of its own is named, never sent to the
+ page a storage folder chose.
+ """
+ from siterender import _own_section
+
+ name = _escape_tags(system_name(sys_id))
+ section = _own_section(sys_id)
+ if section is not None:
+ mfr, console = section
+ name = f"[{name}]({console_href(_slug(mfr), console, prefix)})"
+ return f"{name} {_escape_tags(sys_id)}"
+
+
+def _served_line(f: dict) -> str:
+ """Which collection file answers the entry, from the resolver's result.
+
+ "OK" with no path gave an expert nothing to check: Batocera's
+ scph1001.bin is served by bios/Sony/PlayStation/scph1001_v20.bin.
+ """
+ path = str(f.get("path") or "")
+ if f.get("status") == "missing":
+ return "Not held"
+ if not path:
+ # verify reports no path in existence mode: nothing to name.
+ return ""
+ sha1 = _collection_sha1_by_path().get(path, "")
+ label = "Served by" if f.get("status") == "ok" else "Held, not the declared hash"
+ parts = path.split("/")
+ if not sha1 or len(parts) < 3 or parts[0] != "bios":
+ return f"{label}: {_code(path)}"
+ # The card of that copy: every hash, provenance and the download.
+ page = urllib.parse.quote(card_page(path, sha1))
+ return (f"{label}: [{_code(path)}](../systems/{page}.md#sha1-{sha1[:12]}) "
+ f"(sha1 `{sha1[:8]}`)")
+
+
+# Platforms whose launcher records are read from the repository and revision
+# of their BIOS list: batocera_scraper reads configgen at its tag,
+# retrobat_scraper reads emulatorLauncher at the list's commit
+# (launcher_revision). The others read another repository.
+_LAUNCHERS_IN_LIST_REPO = frozenset({"batocera", "retrobat"})
+
+
+def _launcher_refs(platform: str, config: dict) -> dict[str, list[str]]:
+ """Entry name, casefolded -> each launcher record that opens it, rendered."""
+ from launchers import _names as launcher_names
+ from sitesources import launcher_ref_markdown, list_source
+
+ source = (list_source(str(config.get("source") or ""))
+ if platform in _LAUNCHERS_IN_LIST_REPO else None)
+ found: dict[str, list[str]] = {}
+ for record in config.get("launchers") or []:
+ if not isinstance(record, dict) or not record.get("source_ref"):
+ continue
+ text = (f"{_escape_tags(str(record.get('emulator', '')))} at "
+ f"{launcher_ref_markdown(source, str(record['source_ref']))}")
+ for name in launcher_names(record):
+ names = found.setdefault(name.rsplit("/", 1)[-1].casefold(), [])
+ if text not in names:
+ names.append(text)
+ return found
def _render_platform_systems(
by_system: dict, config_files: dict, emulator_files: dict,
platform: tuple[str, str] | None = None, moved: dict[str, str] | None = None,
+ config: dict | None = None,
) -> list[str]:
"""One collapsible block per system, listing every file it needs.
@@ -2182,26 +2430,19 @@ def _render_platform_systems(
# Per-system detail sections (collapsible for large platforms)
use_collapsible = len(by_system) > 10
by_system_declared = _declared_by_system(config_files)
+ launched = _launcher_refs(platform[0] if platform else "", config or {})
+ sections: list[dict] = []
for sys_id, files in sorted(by_system.items()):
ok_count = sum(1 for f in files if f["status"] == "ok")
total = len(files)
sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id))
-
- sys_link = _system_link(sys_id, "../")
-
anchor = sys_id.replace(" ", "-")
- # Every system is a heading with the same anchor in both layouts:
- # emulator cards link here, and the site search splits a page at its
- # headings. Folded into blocks under one heading, every hash pasted
- # in the search led to the systems overview of a 100-block page.
- lines.append(f"### {sys_link} {{ #{anchor} }}")
- lines.append("")
- block_start = len(lines)
body = [f"{ok_count}/{total} files verified"]
if sys_emus:
emu_links = ", ".join(_emulator_link(e, "../") for e in sorted(sys_emus))
+ body[-1] += "\\"
body.append(f"Emulators: {emu_links}")
body.append("")
@@ -2245,6 +2486,8 @@ def _render_platform_systems(
destination = str(cfg_entry.get("destination") or "")
where = f" at `{destination}`" if destination and destination != fname else ""
body.append(f"- `{fname}`{where} - {', '.join(details)}")
+ item_line = len(body) - 1
+ facts: list[str] = []
# Every hash the platform declares, whole: the crc32 is what
# System.dat and clrmamepro print, and pasting it found nothing
# for a pinned file the collection holds under other bytes.
@@ -2265,9 +2508,19 @@ def _render_platform_systems(
hash_parts.append(f"CRC32: `{crc32}`")
if sha256:
hash_parts.append(f"SHA256: `{sha256}`")
- body.append(f" {' | '.join(hash_parts)}")
+ facts.append(" | ".join(hash_parts))
+ served = _served_line(f)
+ if served:
+ facts.append(served)
if f.get("category_evidence"):
- body.append(f" {_category_evidence_md(f['category_evidence'])}")
+ facts.append(_category_evidence_md(
+ f["category_evidence"], launched.get(fname.rsplit("/", 1)[-1].casefold())))
+ # One line each, hashes, the copy that serves it, its readers: as
+ # continuation lines they ran into one paragraph.
+ if facts:
+ body[item_line] += "\\"
+ body.extend(f" {fact}" + ("\\" if n < len(facts) - 1 else "")
+ for n, fact in enumerate(facts))
# Measured as the Markdown the split was tuned on, a collapsible
# block indented by four, so both engines split the same systems.
@@ -2281,26 +2534,62 @@ def _render_platform_systems(
block.pop()
else:
block = body
- if moved is not None and platform and size > PLATFORM_SECTION_BUDGET:
+ sections.append({"sys_id": sys_id, "anchor": anchor, "body": body, "block": block,
+ "size": size, "total": total, "ok": ok_count})
+
+ # A list longer than its budget goes to a page of its own, then the
+ # largest lists until the page's reference fits PLATFORM_REFERENCE_BUDGET:
+ # with each file's served copy and pinned readers, Batocera's 97 systems
+ # rendered 785 KB of HTML on one page.
+ own_page: set[str] = set()
+ if moved is not None and platform:
+ own_page = {sec["sys_id"] for sec in sections if sec["size"] > PLATFORM_SECTION_BUDGET}
+ kept = sum(sec["size"] for sec in sections if sec["sys_id"] not in own_page)
+ for sec in sorted(sections, key=lambda sec: -sec["size"]):
+ if kept <= PLATFORM_REFERENCE_BUDGET:
+ break
+ if sec["sys_id"] not in own_page:
+ own_page.add(sec["sys_id"])
+ kept -= sec["size"]
+
+ for sec in sections:
+ sys_id, anchor, body = sec["sys_id"], sec["anchor"], sec["body"]
+ total, ok_count = sec["total"], sec["ok"]
+ # Every system is a heading with the same anchor in both layouts:
+ # emulator cards link here, and the site search splits a page at its
+ # headings. Folded into blocks under one heading, every hash pasted
+ # in the search led to the systems overview of a 100-block page.
+ lines.append(f"### {_system_heading_link(sys_id, '../')} {{ #{anchor} }}")
+ lines.append("")
+ if moved is not None and platform and sys_id in own_page:
key, display = platform
- stem = f"{key}-{anchor}"
- moved[stem] = "\n".join([
- f"# {display}: {sys_id}", "",
- f"Part of the [{display} page]({key}.md#{anchor}), whose other systems "
- f"are listed there. Each file below is as {display} declares it, with "
- "its status in the pack.",
- "", "## Files", "",
- *search_excluded(body),
- "", f"*Generated on {_timestamp()}*",
- ]) + "\n"
- lines[block_start:] = [
- f"{_count(total, 'file')}, {ok_count} verified: listed on "
- f"[a page of their own]({stem}.md).", "",
- ]
+ runs = _file_runs(body, PLATFORM_PART_BUDGET)
+ stems = [f"{key}-{anchor}" + (f"-{n}" if n > 1 else "")
+ for n in range(1, len(runs) + 1)]
+ for n, (stem, run) in enumerate(zip(stems, runs), 1):
+ part = f" (part {n} of {len(runs)})" if len(runs) > 1 else ""
+ moved[stem] = "\n".join([
+ f"# {display}: {system_name(sys_id)}{part}", "",
+ f"Part of the [{display} page]({key}.md#{anchor}), whose other systems "
+ f"are listed there. Each file below is as {display} declares it, with "
+ "its status in the pack.",
+ *([""] + [", ".join(
+ f"[part {i}]({other}.md)" if other != stem else f"part {i}"
+ for i, other in enumerate(stems, 1))] if len(runs) > 1 else []),
+ "", "## Files", "",
+ *search_excluded(run),
+ "", f"*Generated on {_timestamp()}*",
+ ]) + "\n"
+ where = (f"[a page of their own]({stems[0]}.md)" if len(stems) == 1 else
+ "pages of their own: " + ", ".join(
+ f"[part {i}]({stem}.md)" for i, stem in enumerate(stems, 1)))
+ lines.extend([
+ f"{_count(total, 'file')}, {ok_count} verified: listed on {where}.", "",
+ ])
continue
# The section's heading stays searchable, its file list with every
# hash is the lookup's: retrodeck's arcade list was 495 KB of index.
- lines[block_start:] = search_excluded(block)
+ lines.extend(search_excluded(sec["block"]))
return lines
@@ -2325,24 +2614,50 @@ def _platform_data_links(name: str) -> str:
return ", ".join(links)
-def _category_evidence_md(evidence: dict) -> str:
- """What establishes a missing entry's category, as the page shows it."""
+@functools.lru_cache(maxsize=1)
+def _site_profiles() -> dict[str, dict]:
+ """Every emulator profile, read once for the citations of a page."""
+ return load_emulator_profiles(str(_REPO_ROOT / "emulators"))
+
+
+def _category_evidence_md(evidence: dict, launched: list[str] | None = None) -> str:
+ """What establishes an entry's category, as the page shows it.
+
+ ``launched`` holds the platform's launcher records that open the entry,
+ already rendered: "(by launcher)" alone named no record and no line.
+ """
if evidence.get("read_by"):
groups: dict[bool, list[str]] = {True: [], False: []}
+ bare: list[str] = []
+ profiles = _site_profiles()
for reader in evidence["read_by"]:
+ # Pinned links, the repository named in words: the refs were
+ # plain code with the role word ("source") left in front.
+ profile = profiles.get(reader["emulator"], {})
refs = [
- f"`{e['name']}` at `{e['source_ref']}`" if e.get("source_ref") else f"`{e['name']}`"
+ (f"`{e['name']}` at {_source_ref_markdown(profile, e['source_ref'])}"
+ if e.get("source_ref") else f"`{e['name']}`")
+ + (" (by name)" if e.get("by") == "name" else "")
for e in reader.get("entries", [])
]
- name = (_emulator_link(reader["emulator"], "../")
- if reader["emulator"] != "launcher" else "the platform's launcher")
+ if reader["emulator"] == "launcher":
+ name = "the platform's launcher"
+ refs = list(launched or [])
+ else:
+ name = _emulator_link(reader["emulator"], "../")
groups[bool(reader.get("runs_system"))].append(
f"{name} ({', '.join(refs)})" if refs else f"{name} (by {reader['proof']})")
+ if not reader.get("runs_system"):
+ # A launcher record is this platform's own code: kept whole.
+ bare.append(f"{name} ({', '.join(refs)})"
+ if reader["emulator"] == "launcher" and refs else name)
if not groups[True]:
return "Read by " + ", ".join(groups[False])
text = "Read by " + ", ".join(groups[True])
- if groups[False]:
- text += ", also named by " + ", ".join(groups[False])
+ # Cores of other systems that share the name: named, not cited, as
+ # their lines describe another machine's file.
+ if bare:
+ text += ", also named by " + ", ".join(bare)
return text
examined = evidence.get("cores_examined") or []
cores = ("cores examined: " + ", ".join(_emulator_link(c, "../") for c in examined)
@@ -2576,6 +2891,11 @@ def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list:
return lines
+# How a platform checks a file, in words.
+_CHECK_WORDS = {"existence": "name only", "md5": "MD5", "sha1": "SHA-1", "sha256": "SHA-256",
+ "crc32": "CRC32"}
+
+
def generate_platform_page(
name: str,
cov: dict,
@@ -2588,16 +2908,17 @@ def generate_platform_page(
``moved`` collects the pages of systems whose list is too long for it.
"""
+ from sitesources import list_source
+
registry = registry or {}
config = cov["config"]
display = cov["platform"]
mode = cov["mode"]
pct = _pct(cov["present"], cov["total"])
+ # The mark is drawn beside the title from the front matter: floated
+ # beside the first line at 48 px it read as a broken image.
logo_url = registry.get(name, {}).get("logo", "")
- logo_md = (
- markup.image(display, logo_url, width="48", align="right") + "\n\n" if logo_url else ""
- )
homepage = config.get("homepage", "")
version = config.get("version", "")
@@ -2609,9 +2930,9 @@ def generate_platform_page(
)
lines = [
+ *(["---", f"logo: {json.dumps(logo_url)}", "---", ""] if logo_url else []),
f"# {display}",
"",
- logo_md,
]
if registry.get(name, {}).get("status") == "archived":
@@ -2653,14 +2974,17 @@ def generate_platform_page(
*_stat(f"{cov['present']}/{cov['total']}", f"Coverage ({pct})"),
*_stat(str(cov["verified"]), "Verified"),
*_stat(str(cov["missing"]), "Missing"),
- *_stat(f'{mode}', "Checked by"),
+ *_stat(f'{_CHECK_WORDS.get(mode, mode)}',
+ "Checked by"),
"",
"",
]
lines.extend([*band, markup.block_open("div", 'class="rb-meta-card"'), "",
"| Detail | Value |", "|---|---|"])
- if hash_type:
- lines.append(f"| Hash type | {hash_type} |")
+ # The hash the list writes, when it is not the one the platform checks:
+ # RetroArch checks presence and its System.dat writes sha1.
+ if hash_type and hash_type != mode:
+ lines.append(f"| Hashes in the list | {_CHECK_WORDS.get(hash_type, hash_type)} |")
if version:
lines.append(f"| Version | {version} |")
# base_destination is only the root the platform's own list writes its
@@ -2673,11 +2997,15 @@ def generate_platform_page(
if transcribed:
# The file every declaration on this page was read from: without it
# a hash could not be traced to the revision that states it.
- moving = re.search(r"/(?:master|main|db)(?:/|$)|refs/heads/", transcribed) or (
- re.fullmatch(r"https://(?:github|gitlab)\.com/[^/]+/[^/]+/?", transcribed)
- )
+ listed = list_source(transcribed)
+ moving = (listed.moving if listed else bool(
+ re.search(r"/(?:master|main|db)(?:/|$)|refs/heads/", transcribed)))
note = " (a branch: the revision read is not recorded)" if moving else ""
- lines.append(f"| Transcribed from | <{transcribed}>{note} |")
+ # Named in words: the raw URL broke into 25 lines of 8 characters
+ # on a phone.
+ link = (f"[{_escape_tags(listed.label)}]({listed.browse()})" if listed
+ else f"<{transcribed}>")
+ lines.append(f"| Transcribed from | {link}{note} |")
if name in _EXPORTED:
lines.append(f"| Updated file | [Download it and see the changes](../exports.md#{name}) |")
lines.append(f"| Data | {_platform_data_links(name)} |")
@@ -2744,14 +3072,16 @@ def generate_platform_page(
if len(sys_emus) > 3:
emu_str += f" +{len(sys_emus) - 3}"
anchor = sys_id.replace(" ", "-")
+ # The name a reader filters on, the id the list writes beside it.
lines.append(
- f"| [{sys_id}](#{anchor}) | {ok_count}/{total} | {status} | {emu_str} |"
+ f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) {sys_id} "
+ f"| {ok_count}/{total} | {status} | {emu_str} |"
)
lines.append("")
lines.extend(
_render_platform_systems(by_system, config_files, emulator_files,
- (name, display), moved)
+ (name, display), moved, config)
)
lines.append(f"*Generated on {_timestamp()}*")
@@ -2969,7 +3299,9 @@ def file_lookup_records(
r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"]))
-def write_lookup_shards(api: Path, records: list[dict], generated_at: str) -> int:
+def write_lookup_shards(
+ api: Path, records: list[dict], generated_at: str, commit: str = "",
+) -> int:
"""One envelope per leading pair of hex characters, as the lookup asks.
A value that is not hexadecimal (a typo in a profile) has no shard the
@@ -2985,6 +3317,7 @@ def write_lookup_shards(api: Path, records: list[dict], generated_at: str) -> in
write_if_changed(str(folder / f"{prefix}.json"), json.dumps({
"schema_version": 1,
"generated_at": generated_at,
+ **({"commit": commit} if commit else {}),
"kind": "file-lookup",
"count": len(items),
"items": items,
@@ -3182,16 +3515,31 @@ def load_derivable(path: str = "derivable.json") -> dict[str, dict]:
_ROMSET_SOURCES = {"mame": "MAME", "fbneo": "FBNeo"}
+def _date_in_words(stamp: str) -> str:
+ """`2026-08-07T...` as ``."""
+ try:
+ day = datetime.strptime(str(stamp)[:10], "%Y-%m-%d")
+ except ValueError:
+ return _escape_tags(str(stamp))
+ return f''
+
+
def generate_provenance_page(
db: dict, report: dict, derivable: dict[str, dict] | None = None
) -> str:
"""Page explaining the verified dump badges and listing catalog gaps."""
matched_files = count_catalog_matched(db)
total_files = compute_composition(db)["systems"]["files"]
+ # When each catalog was read: a 76-day-old snapshot showed no age.
+ as_of = ", ".join(
+ f"{_PROVENANCE_LABELS.get(source, source)} {_date_in_words(data['imported_at'])}"
+ for source, data in report.items() if data.get("imported_at")
+ )
lines = [
"# Dump provenance",
"",
+ *([f'
Catalogs as of: {as_of}.
', ""] if as_of else []),
"Provenance says whether a file is, byte for byte, a dump that Redump, "
"No-Intro or TOSEC catalogues. It tells you where a dump is known from, "
"not whether an emulator accepts it: that is the emulator's code, read "
@@ -3248,7 +3596,7 @@ def generate_provenance_page(
name = f"[{label}]({home})" if home else label
lines.append(
f"| {name} | {data['matched']:,}/{in_scope:,} ({pct:.0f}%) | "
- f"{len(data['covered_dats'])} | {data['imported_at']} |"
+ f"{len(data['covered_dats'])} | {_date_in_words(data['imported_at'])} |"
)
lines.extend(
@@ -5811,6 +6159,32 @@ def _platform_lists_sentence(
)
+def _missing_required_firmware(cov: dict) -> int:
+ """Entries of a platform's list that are missing, required and firmware.
+
+ Only those stop a console from starting. Game data and files no core
+ opens are counted apart, and red on every short row carried no signal.
+ """
+ return len({
+ (d.get("system", ""), d.get("name", "")) for d in cov.get("details") or []
+ if d.get("status") == "missing" and d.get("required", True)
+ and d.get("category", "bios") == "bios" and not d.get("settled_by")
+ })
+
+
+def _ratio_cell(done: int, total: int, alarm: int = 0) -> str:
+ """A held-over-needed cell: the numbers, a bar, and a badge only when
+ required firmware is missing."""
+ if not total:
+ return "-"
+ share = done / total * 100
+ cell = (f''
+ f"{done:,}/{total:,}")
+ if alarm:
+ cell += f' {alarm:,} required missing'
+ return cell
+
+
def generate_gap_analysis(
profiles: dict,
coverages: dict,
@@ -5876,7 +6250,7 @@ def generate_gap_analysis(
})
lines = [
- "# Gap analysis",
+ "# What the packs do not hold",
"",
"What each pack still cannot give you, and why. Most gaps are not "
"firmware: a platform's list can also name game files, such as "
@@ -5907,7 +6281,7 @@ def generate_gap_analysis(
lines.extend([
"## What each pack contains",
"",
- "| Platform | On its BIOS list | Files its emulators load | Checked by |",
+ "| Platform | On its BIOS list (destinations) | Files its emulators load | Checked by |",
"|----------|-----------------:|-------------------------:|------------|",
])
@@ -5917,15 +6291,6 @@ def generate_gap_analysis(
"existence": "file presence",
}
- def _ratio(done: int, total: int) -> str:
- """A collected-over-needed cell, flagged when short."""
- if not total:
- return "-"
- cell = f"{done:,}/{total:,}"
- if done < total:
- return f'{cell}'
- return cell
-
for pname, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]):
display = cov["platform"]
core_total = (
@@ -5933,18 +6298,20 @@ def generate_gap_analysis(
)
lines.append(
f"| [{display}](platforms/{pname}.md) "
- f"| {_ratio(cov['present'], cov['total'])} "
- f"| {_ratio(cov['core_present'], core_total)} "
+ f"| {_ratio_cell(cov['present'], cov['total'], _missing_required_firmware(cov))} "
+ f"| {_ratio_cell(cov['core_present'], core_total)} "
f"| {mode_labels.get(cov['mode'], cov['mode'])} |"
)
lines.extend([
"",
"Each fraction is what the pack has over what is needed, counting "
"required and optional files alike since both ship. The first column "
- "is the BIOS list the platform publishes. The second counts files its "
+ "is the BIOS list the platform publishes, counted by destination: a "
+ "path the list declares twice is one file to place. A red badge marks "
+ "a list that is short of required firmware. The second counts files its "
"emulators load that this list never mentions, found by reading their "
- "source code, and it is often several times larger. A short "
- "fraction is flagged and named in the sections below. That second "
+ "source code, and it is often several times larger. What is short "
+ "is named in the sections below. That second "
"number is a floor, not a ceiling: an emulator that accepts any file "
"handed to it names none in its code, so nothing there can be counted. "
"Checked by is the test the platform runs on its own, replicated here "
@@ -6192,12 +6559,13 @@ def generate_gap_analysis(
if src_missing_details:
req_src = [m for m in src_missing_details if m["required"]]
lines.extend([
- f"### Files to source ({_count(len(src_missing_details), 'file')}, "
- f"{len(req_src)} required"
+ "### Files to source",
+ "",
+ f"{_count(len(src_missing_details), 'file')}, {len(req_src)} required."
+ (
- f", {_count(entries, 'entry', 'entries')}: "
- "a file declared with the same hash at several places is listed once)"
- if entries != len(src_missing_details) else ")"
+ f" They come from {_count(entries, 'entry', 'entries')}: a file "
+ "declared with the same hash at several places is listed once."
+ if entries != len(src_missing_details) else ""
),
"",
"| File | Destination | Expected | Emulator | Required | Affects platforms | Source ref |",
@@ -6312,154 +6680,210 @@ def _gap_sources(data: dict) -> dict[str, int]:
return counts
+def _xref_upstream(profile: dict) -> str:
+ """The project a core comes from: its upstream, else its own source."""
+ raw = profile.get("upstream") or profile.get("source", "")
+ return raw if isinstance(raw, str) else ""
+
+
+_FORGES = frozenset({
+ "github.com", "gitlab.com", "codeberg.org", "bitbucket.org", "git.eden-emu.dev",
+ "git.ryujinx.app",
+})
+
+
+def upstream_project(url: str) -> tuple[str, str]:
+ """(project key, revision or path inside it) for an upstream URL.
+
+ MAME was twelve rows: mamedev/mame and one row per release tree
+ (mamedev/mame/tree/mame0289 and the rest). On a forge the project is
+ owner/repo, a GitLab project every group before `/-/`, and the rest is
+ a revision. Elsewhere the address without its scheme, `www.` and
+ trailing slash is the project. A note written after the address
+ (`... (sys/src/games/nes/)`) is a path inside it.
+ """
+ url, _, note = url.strip().partition(" ")
+ note = note.strip().strip("()")
+ if not url.startswith(("http://", "https://")):
+ return " ".join(filter(None, (url, note))), ""
+ parsed = urllib.parse.urlsplit(url)
+ host = parsed.netloc.lower().removeprefix("www.")
+ parts = [p for p in parsed.path.split("/") if p]
+ if host == "gitlab.com" and len(parts) >= 2:
+ cut = parts.index("-") if "-" in parts else len(parts)
+ rest = parts[cut + 1:]
+ if rest[:1] in (["tree"], ["blob"]):
+ rest = rest[1:]
+ return f"{host}/{'/'.join(parts[:cut])}", "/".join(filter(None, ("/".join(rest), note)))
+ if host in _FORGES and len(parts) >= 2:
+ repo = parts[1].removesuffix(".git")
+ rest = parts[2:]
+ if rest[:1] in (["tree"], ["blob"], ["src"]):
+ rest = rest[1:]
+ return f"{host}/{parts[0]}/{repo}", "/".join(filter(None, ("/".join(rest), note)))
+ return "/".join([host, *parts]), note
+
+
+def _upstream_label(key: str) -> str:
+ """owner/repo on GitHub, host/owner/repo on another forge, and a
+ Software Heritage revision by its short id."""
+ swh = re.fullmatch(r"archive\.softwareheritage\.org/browse/revision/([0-9a-f]{12})[0-9a-f]*", key)
+ if swh:
+ return f"Software Heritage revision {swh[1]}"
+ return key.removeprefix("github.com/")
+
+
+def _breakable(label: str) -> str:
+ """A project address that may wrap after each slash: unbroken, the
+ longest pushed three of five columns off a 1,440 px screen."""
+ return "/".join(_escape_tags(part) for part in label.split("/"))
+
+
+def _xref_platform_rows(config: dict, unique: dict) -> tuple[int, list[str]]:
+ """The cores a platform pulls in, as table rows, with their count."""
+ # The cores the builder and verify resolve: a hand copy here dropped
+ # the libretro set a list naming retroarch pulls in, and the system
+ # fallback EmuDeck relies on.
+ matched = sorted(key for key in resolve_platform_cores(config, unique) if key in unique)
+ rows = []
+ for emu_name in matched:
+ p = unique[emu_name]
+ cls_raw = p.get("core_classification", "-")
+ systems = p.get("systems", []) or []
+ sys_str = ", ".join(_escape_tags(system_name(s)) for s in systems[:3])
+ if len(systems) > 3:
+ sys_str += f" +{len(systems) - 3}"
+ files = p.get("files", []) or []
+ libretro_only = sum(1 for f in files if f.get("mode") == "libretro")
+ standalone_only = sum(1 for f in files if f.get("mode") == "standalone")
+ file_str = str(len(files))
+ parts = [f"{n} {label}" for n, label in ((libretro_only, "libretro-only"),
+ (standalone_only, "standalone-only")) if n]
+ if parts:
+ file_str += f" ({', '.join(parts)})"
+ upstream = _xref_upstream(p)
+ key, _rest = upstream_project(upstream)
+ upstream_display = (f"[{_breakable(_upstream_label(key))}]({upstream.split()[0]})"
+ if upstream.startswith("http") else (_escape_tags(upstream) or "-"))
+ rows.append(
+ f"| [{_escape_tags(p.get('emulator', emu_name))}](emulators/{emu_name}.md) "
+ f"| {CLS_LABELS.get(cls_raw, cls_raw)} | {sys_str or '-'} | {file_str} "
+ f"| {upstream_display} |"
+ )
+ return len(matched), rows
+
+
def _render_xref_by_platform(
- coverages: dict, unique: dict
+ coverages: dict, unique: dict, moved: dict[str, str] | None = None,
) -> list[str]:
- """Which cores each platform pulls in, and what they need."""
- lines: list[str] = []
- for pname in sorted(coverages.keys(), key=lambda x: coverages[x]["platform"]):
+ """Which cores each platform pulls in: counts here, a page per platform.
+
+ One page held every platform's table, 30,551 nodes: the overview keeps
+ the counts and each platform's table moves to ``moved`` by stem.
+ """
+ lines = [
+ "## By platform",
+ "",
+ "| Platform | Cores | Core list |",
+ "|----------|------:|-----------|",
+ ]
+ for pname in sorted(coverages, key=lambda x: coverages[x]["platform"]):
cov = coverages[pname]
display = cov["platform"]
config = cov["config"]
-
- body = [markup.button(f"Open {display} platform profile", f"platforms/{pname}.md"), ""]
-
- # The cores the builder and verify resolve: a hand copy here dropped
- # the libretro set a list naming retroarch pulls in, and the system
- # fallback EmuDeck relies on.
- matched = {
- key: unique[key]
- for key in resolve_platform_cores(config, unique)
- if key in unique
- }
-
- body.append(f"**{len(matched)} cores**{_core_scope_note(config, unique)}")
- body.append("")
-
- body.append("| Core | Classification | Systems | Files | Upstream |")
- body.append("|------|---------------|---------|-------|----------|")
-
- for emu_name in sorted(matched.keys()):
- p = matched[emu_name]
- emu_display = p.get("emulator", emu_name)
- cls_raw = p.get("core_classification", "-")
- cls = CLS_LABELS.get(cls_raw, cls_raw)
- p.get("type", "")
- # Null means the project is its own upstream: show its source.
- upstream_raw2 = p.get("upstream", "")
- upstream = upstream_raw2 if isinstance(upstream_raw2, str) else ""
- source_raw2 = p.get("source", "")
- source = str(source_raw2) if not isinstance(source_raw2, dict) else ""
- systems = p.get("systems", [])
- files = p.get("files", [])
-
- sys_str = ", ".join(systems[:3])
- if len(systems) > 3:
- sys_str += f" +{len(systems) - 3}"
-
- file_count = len(files)
- libretro_only = sum(1 for f in files if f.get("mode") == "libretro")
- standalone_only = sum(1 for f in files if f.get("mode") == "standalone")
- file_str = str(file_count)
- if libretro_only or standalone_only:
- parts = []
- if libretro_only:
- parts.append(f"{libretro_only} libretro-only")
- if standalone_only:
- parts.append(f"{standalone_only} standalone-only")
- file_str += f" ({', '.join(parts)})"
-
- upstream_display = "-"
- if upstream and upstream.startswith("http"):
- upstream_short = upstream.replace("https://github.com/", "")
- upstream_display = f"[{upstream_short}]({upstream})"
- elif upstream:
- upstream_display = upstream
- elif source and source.startswith("http"):
- source_short = source.replace("https://github.com/", "")
- upstream_display = f"[{source_short}]({source})"
- elif source:
- upstream_display = source
-
- body.append(
- f"| [{emu_display}](emulators/{emu_name}.md) | {cls} | "
- f"{sys_str} | {file_str} | {upstream_display} |"
- )
-
- lines.extend(markup.details("abstract", display, body))
-
+ count, rows = _xref_platform_rows(config, unique)
+ stem = f"cross-reference-{pname}"
+ lines.append(
+ f"| [{_escape_tags(display)}](platforms/{pname}.md) | {count}"
+ f"{_core_scope_note(config, unique)} | [Cores on {_escape_tags(display)}]({stem}.md) |"
+ )
+ if moved is not None:
+ moved[stem] = "\n".join([
+ "---", "level: reference", "---", "",
+ f"# Cores on {display}",
+ "",
+ f"{_count(count, 'core')}{_core_scope_note(config, unique)} that "
+ f"{display} runs, the systems each emulates and the project it comes "
+ f"from. Part of [Cores by platform](cross-reference.md). The files "
+ f"are listed on the [{display} page](platforms/{pname}.md).",
+ "",
+ "| Core | Classification | Systems | Files | Upstream |",
+ "|------|---------------|---------|-------|----------|",
+ *rows,
+ "",
+ ]) + "\n"
+ lines.append("")
return lines
def _render_xref_by_upstream(
coverages: dict, unique: dict
) -> list[str]:
- """The same relation read the other way: one row per upstream."""
- lines: list[str] = []
- # Reverse view: by upstream emulator
- lines.extend(
- [
- "## By upstream emulator",
- "",
- "| Upstream | Cores | Classification | Platforms |",
- "|----------|-------|---------------|-----------|",
- ]
- )
+ """The same relation read the other way: one row per upstream project."""
+ lines: list[str] = [
+ "## By upstream emulator",
+ "",
+ "One row per project: the releases and folders of one repository are "
+ "listed in the Revisions column. Platforms counts the platforms that run "
+ "at least one of its cores.",
+ "",
+ "| Cores | Upstream | Revisions | Classification | Platforms |",
+ "|-------|----------|-----------|----------------|----------:|",
+ ]
- by_upstream: dict[str, list[str]] = {}
+ by_upstream: dict[str, list[tuple[str, str, str]]] = {}
for emu_name, p in sorted(unique.items()):
# A null upstream is the project's own source: grouping it under
# the string "None" put eight unrelated engines in one row.
- raw_up = p.get("upstream") or p.get("source", "")
- up_str = raw_up if isinstance(raw_up, str) else ""
+ up_str = _xref_upstream(p)
if up_str:
- by_upstream.setdefault(up_str, []).append(emu_name)
+ key, rest = upstream_project(up_str)
+ by_upstream.setdefault(key, []).append((emu_name, up_str, rest))
platform_membership: dict[str, set[str]] = {}
for pname, cov in coverages.items():
for key in resolve_platform_cores(cov["config"], unique):
- platform_membership.setdefault(key, set()).add(pname)
+ platform_membership.setdefault(key, set()).add(cov["platform"])
- for upstream_url in sorted(by_upstream.keys()):
- cores = by_upstream[upstream_url]
- upstream_short = upstream_url.replace("https://github.com/", "")
- classifications = set()
- all_plats: set[str] = set()
- for c in cores:
- raw_cls = unique[c].get("core_classification", "-")
- classifications.add(CLS_LABELS.get(raw_cls, raw_cls))
- all_plats.update(platform_membership.get(c, set()))
-
- cls_str = ", ".join(sorted(classifications))
- plat_str = ", ".join(sorted(all_plats)) if all_plats else "-"
- core_links = ", ".join(f"[{c}](emulators/{c}.md)" for c in sorted(cores))
-
- if upstream_url.startswith("http"):
- upstream_cell = f"[{upstream_short}]({upstream_url})"
- else:
- upstream_cell = upstream_short
+ for key in sorted(by_upstream, key=lambda k: _upstream_label(k).casefold()):
+ members = by_upstream[key]
+ cores = sorted(c for c, _url, _rest in members)
+ classifications = sorted({
+ CLS_LABELS.get(unique[c].get("core_classification", "-"),
+ unique[c].get("core_classification", "-"))
+ for c in cores
+ })
+ plats = sorted(set().union(*(platform_membership.get(c, set()) for c in cores)))
+ revisions = sorted({rest for _c, _url, rest in members if rest})
+ urls = sorted({url.split()[0] for _c, url, _rest in members})
+ # The project's own address: the bare one when a core cites it.
+ home = next((u for u in urls if not upstream_project(u)[1]), urls[0])
+ upstream_cell = (f"[{_breakable(_upstream_label(key))}]({home})" if home.startswith("http")
+ else _escape_tags(key))
lines.append(
- f"| {upstream_cell} | {core_links} | "
- f"{cls_str} | {plat_str} |"
+ "| " + ", ".join(f"[{c}](emulators/{c}.md)" for c in cores)
+ + f" | {upstream_cell} | {', '.join(_breakable(r) for r in revisions) or '-'} "
+ f"| {', '.join(classifications)} | {len(plats)} |"
)
-
return lines
def generate_cross_reference(
coverages: dict,
profiles: dict,
+ moved: dict[str, str] | None = None,
) -> str:
- """Generate cross-reference: Platform -> Core -> Systems -> Upstream."""
+ """Cross-reference: counts per platform, then cores by upstream project.
+
+ ``moved`` collects one page per platform with its core table.
+ """
unique = {
k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test")
}
total_cores = len(unique)
total_upstreams = len({
- p.get("upstream", p.get("source", ""))
- for p in unique.values()
- if p.get("upstream") or p.get("source")
+ upstream_project(_xref_upstream(p))[0] for p in unique.values() if _xref_upstream(p)
})
lines = [
@@ -6483,7 +6907,7 @@ def generate_cross_reference(
"",
]
- lines.extend(_render_xref_by_platform(coverages, unique))
+ lines.extend(_render_xref_by_platform(coverages, unique, moved))
lines.extend(_render_xref_by_upstream(coverages, unique))
lines.extend(["", f"*Generated on {_timestamp()}*"])
@@ -7374,10 +7798,14 @@ def _generate(args: argparse.Namespace) -> None:
write_if_changed(str(docs / "emulators" / f"{stem}.md"), page)
print("Generating cross-reference page...")
+ xref_pages: dict[str, str] = {}
write_if_changed(
str(docs / "cross-reference.md"),
- tables_out_of_search(generate_cross_reference(coverages, profiles)) + reference_next(),
+ tables_out_of_search(generate_cross_reference(coverages, profiles, xref_pages))
+ + reference_next(),
)
+ for stem, page in xref_pages.items():
+ write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page))
print("Generating gap analysis page...")
write_if_changed(
diff --git a/scripts/sitesources.py b/scripts/sitesources.py
index f1a62110..267fe766 100644
--- a/scripts/sitesources.py
+++ b/scripts/sitesources.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import re
import urllib.parse
+from dataclasses import dataclass
import urllib.request
import upstream
@@ -345,3 +346,98 @@ _PATH_RE = re.compile(r"^[A-Za-z0-9_.@/+~-]+$")
def _guarded_spans(line: str) -> list[tuple[int, int]]:
"""Code spans and links of a markdown line."""
return [m.span() for m in re.finditer(r"`[^`]*`|\[[^\]]*\]\([^)]*\)", line)]
+
+
+# Where a platform's list was read: the forge, the repository, the revision
+# and the path inside it, out of the URL its scraper pinned.
+_LIST_URL_RES = (
+ re.compile(r"^https://raw\.githubusercontent\.com/(?P[^/]+)/(?P[^/]+)/"
+ r"(?P[^/]+)/(?P.+)$"),
+ re.compile(r"^https://github\.com/(?P[^/]+)/(?P[^/]+)/(?:blob|tree)/"
+ r"(?P[^/]+)(?:/(?P.+))?$"),
+ re.compile(r"^https://codeload\.github\.com/(?P[^/]+)/(?P[^/]+)/"
+ r"tar\.gz/(?P[^/]+)$"),
+ re.compile(r"^https://(?Pgitlab\.com)/(?P[^/]+)/(?P[^/]+)/-/"
+ r"(?:raw|blob)/(?P[^/]+)/(?P.+)$"),
+)
+_BRANCHES = frozenset({"master", "main", "db"})
+
+
+@dataclass(frozen=True)
+class ListSource:
+ """A platform's list as its scraper read it."""
+
+ host: str
+ owner: str
+ repo: str
+ rev: str
+ path: str
+
+ @property
+ def moving(self) -> bool:
+ """The revision is a branch: what was read is not recorded."""
+ return not self.rev or self.rev in _BRANCHES
+
+ @property
+ def label(self) -> str:
+ """`batocera-systems at tag batocera-43.1`, said in words."""
+ name = self.path.rsplit("/", 1)[-1] if self.path else f"{self.repo} repository"
+ if not self.rev:
+ return f"{name}, default branch"
+ if re.fullmatch(r"[0-9a-f]{40}", self.rev):
+ return f"{name} at commit {self.rev[:12]}"
+ if self.moving:
+ return f"{name} on branch {self.rev}"
+ return f"{name} at tag {self.rev}"
+
+ def browse(self, path: str = "", start: int | None = None,
+ end: int | None = None) -> str:
+ """The page a reader opens: the file, or the tree, at the revision."""
+ path = path or self.path
+ if self.host == "gitlab.com":
+ url = f"https://gitlab.com/{self.owner}/{self.repo}/-/blob/{self.rev}/{path}"
+ anchor = f"#L{start}" + (f"-{end}" if end and end != start else "")
+ else:
+ base = f"https://github.com/{self.owner}/{self.repo}"
+ if not self.rev:
+ return base
+ kind = "blob" if path else "tree"
+ url = f"{base}/{kind}/{self.rev}" + (f"/{path}" if path else "")
+ anchor = f"#L{start}" + (f"-L{end}" if end and end != start else "")
+ return url + (anchor if start else "")
+
+
+def list_source(url: str) -> ListSource | None:
+ """The forge, repository, revision and path of a platform's `source:`."""
+ url = str(url or "").strip()
+ for pattern in _LIST_URL_RES:
+ found = pattern.match(url)
+ if found:
+ parts = found.groupdict()
+ return ListSource(parts.get("host") or "github.com", parts["owner"],
+ parts["repo"], parts["rev"], parts.get("path") or "")
+ bare = re.match(r"^https://github\.com/([^/]+)/([^/]+?)/?$", url)
+ if bare:
+ return ListSource("github.com", bare[1], bare[2], "", "")
+ return None
+
+
+def launcher_ref_markdown(source: ListSource | None, ref: str) -> str:
+ """A launcher record's source_ref, each part a pinned link.
+
+ Linked only where the caller knows the launcher code sits in the
+ repository and revision of the platform's list. Otherwise each part is
+ plain code: a link to another repository is a false citation.
+ """
+ rendered: list[str] = []
+ for part in split_source_ref(ref):
+ display = part.path.rsplit("/", 1)[-1]
+ if part.start is not None:
+ display += f":{part.start}"
+ if part.end is not None and part.end != part.start:
+ display += f"-{part.end}"
+ if source is None or not source.rev or not _PATH_RE.fullmatch(part.path):
+ rendered.append(f"`{display}`")
+ else:
+ rendered.append(f"[`{display}`]({source.browse(part.path, part.start, part.end)})")
+ return ", ".join(rendered)
diff --git a/tests/test_entry_categories.py b/tests/test_entry_categories.py
index 3d2791a9..e62a3570 100644
--- a/tests/test_entry_categories.py
+++ b/tests/test_entry_categories.py
@@ -88,9 +88,11 @@ class Evidence(unittest.TestCase):
def test_game_data_names_its_reader_and_reference(self):
found = self.cats.explain({"name": "diabdat.mpq"}, "ports/diablo/diabdat.mpq", "diablo")
self.assertEqual(found["category"], GAME_DATA)
+ # The platform entry declares no hash: the reader is matched by name.
self.assertEqual(found["read_by"], [{
"emulator": "devilutionx", "proof": "name", "runs_system": True,
- "entries": [{"name": "diabdat.mpq", "source_ref": "Source/init.cpp:120"}]}])
+ "entries": [{"name": "diabdat.mpq", "source_ref": "Source/init.cpp:120",
+ "by": "name"}]}])
self.assertEqual(verify.category_evidence_text(found),
"read by devilutionx (diabdat.mpq at Source/init.cpp:120)")
diff --git a/tests/test_site_cores.py b/tests/test_site_cores.py
index 0133e397..11b14d87 100644
--- a/tests/test_site_cores.py
+++ b/tests/test_site_cores.py
@@ -33,12 +33,17 @@ class SiteCoreCounts(unittest.TestCase):
for name in list_registered_platforms(platforms, include_archived=True):
config = load_platform_config(name, platforms)
coverages[name] = {"platform": config.get("platform", name), "config": config}
- page = generate_site.generate_cross_reference(coverages, profiles)
+ moved: dict[str, str] = {}
+ page = generate_site.generate_cross_reference(coverages, profiles, moved)
shown = {
m.group(1): int(m.group(2))
- for m in re.finditer(
- r'([^<]+)\n\n.*\n\n\*\*(\d+) cores', page)
+ for m in re.finditer(r"^\| \[([^\]]+)\]\(platforms/[^)]+\) \| (\d+)", page, re.M)
}
+ for name, cov in coverages.items():
+ with self.subTest(page=name):
+ rows = [line for line in moved[f"cross-reference-{name}"].splitlines()
+ if line.startswith("| [")]
+ self.assertEqual(len(rows), shown[cov["platform"]])
for name, cov in coverages.items():
with self.subTest(platform=name):
expected = len(resolve_platform_cores(cov["config"], unique))
diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py
index 466a7290..83989941 100644
--- a/tests/test_site_exports.py
+++ b/tests/test_site_exports.py
@@ -622,7 +622,9 @@ class APlatformPageNamesTheFileItTranscribes(unittest.TestCase):
return generate_site.generate_platform_page("p", cov)
tag = "https://raw.githubusercontent.com/o/r/batocera-43.1/x/batocera-systems"
- self.assertIn(f"| Transcribed from | <{tag}> |", page(tag))
+ self.assertIn(
+ "| Transcribed from | [batocera-systems at tag batocera-43.1]"
+ "(https://github.com/o/r/blob/batocera-43.1/x/batocera-systems) |", page(tag))
branch = "https://raw.githubusercontent.com/o/r/master/x.json"
self.assertIn("revision read is not recorded", page(branch))
self.assertIn("revision read is not recorded", page("https://github.com/RetroDECK/components"))
diff --git a/tests/test_site_platform_hashes.py b/tests/test_site_platform_hashes.py
index 27b52a8a..15e5df32 100644
--- a/tests/test_site_platform_hashes.py
+++ b/tests/test_site_platform_hashes.py
@@ -10,11 +10,13 @@ from __future__ import annotations
import sys
import unittest
+from unittest import mock
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
+import siterender # noqa: E402
from generate_site import _render_platform_systems # noqa: E402
A, B, C = "a" * 32, "b" * 32, "c" * 32
@@ -36,7 +38,9 @@ class EachRowItsDeclaration(unittest.TestCase):
{"name": "boot.rom", "status": "ok", "system": "saturn"},
],
}
- text = "\n".join(_render_platform_systems(by_system, config_files, {}))
+ names = {"psx": "Sony PlayStation", "saturn": "Sega Saturn"}
+ with mock.patch.dict(siterender.SYSTEM_NAMES, names):
+ text = "\n".join(_render_platform_systems(by_system, config_files, {}))
psx, saturn = text.split("saturn", 1)[0], text.split("saturn", 1)[1]
self.assertIn(A, psx)
self.assertNotIn(B, psx)
diff --git a/tests/test_site_r2_reference.py b/tests/test_site_r2_reference.py
new file mode 100644
index 00000000..9234398c
--- /dev/null
+++ b/tests/test_site_r2_reference.py
@@ -0,0 +1,357 @@
+"""Reference pages (round 2, package 8).
+
+A reference page cites a hash, a source line or a count, and each citation
+has to hold when an expert checks it: the reader of a pinned dump is the
+profile entry accepting its hash, a source line is a pinned link, an entry
+names the file that serves it, and the exports carry what the page shows.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import re
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_site # noqa: E402
+import siterender # noqa: E402
+from common import ( # noqa: E402
+ list_registered_platforms,
+ load_emulator_profiles,
+ load_platform_config,
+)
+from exporter.baseline import HASH_FIELDS, _hash_values # noqa: E402
+from exporter.candidates import EntryCategories # noqa: E402
+from sitesources import launcher_ref_markdown, list_source # noqa: E402
+
+PLATFORMS = str(REPO_ROOT / "platforms")
+DIST = REPO_ROOT / "website" / "dist"
+
+
+def _accepted(profile_entry: dict) -> set[str]:
+ values = {v for f in HASH_FIELDS for v in _hash_values(profile_entry, f)}
+ for member in profile_entry.get("contents") or []:
+ if isinstance(member, dict):
+ values |= {v for f in HASH_FIELDS for v in _hash_values(member, f)}
+ return values
+
+
+class ReaderMatchedByHash(unittest.TestCase):
+ """Batocera's scph1001.bin (md5 dc2b9bf8, SCPH-1001 v2.0) cited
+ DuckStation's bios.cpp:36, the line of the v2.2 dump: the reader was
+ matched by name."""
+
+ @classmethod
+ def setUpClass(cls):
+ cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
+
+ def test_batocera_scph1001_cites_the_v20_line(self):
+ config = load_platform_config("batocera", PLATFORMS)
+ cats = EntryCategories(config, self.profiles)
+ entry = next(e for e in config["systems"]["sony-playstation"]["files"]
+ if e["name"] == "scph1001.bin")
+ found = cats.explain(entry, entry.get("destination") or entry["name"],
+ "sony-playstation")
+ duck = next(r for r in found["read_by"] if r["emulator"] == "duckstation")
+ self.assertEqual(duck["entries"], [
+ {"name": "scph1001_v20.bin", "source_ref": "src/core/bios.cpp:42"}])
+
+ def test_every_cited_entry_accepts_the_md5_or_says_by_name(self):
+ by_name = 0
+ for platform in list_registered_platforms(PLATFORMS, include_archived=True):
+ config = load_platform_config(platform, PLATFORMS)
+ cats = EntryCategories(config, self.profiles)
+ for sys_id, system in (config.get("systems") or {}).items():
+ for entry in system.get("files") or []:
+ md5s = set(_hash_values(entry, "md5")) | set(_hash_values(entry, "alt_md5"))
+ if not md5s:
+ continue
+ # The same dump, named by whichever hash both sides state.
+ declared = md5s | {v for f in HASH_FIELDS for v in _hash_values(entry, f)}
+ found = cats.explain(entry, entry.get("destination") or entry["name"], sys_id)
+ for reader in found.get("read_by", []):
+ if reader["emulator"] == "launcher":
+ continue
+ files = self.profiles[reader["emulator"]].get("files") or []
+ for cited in reader["entries"]:
+ if cited.get("by") == "name":
+ by_name += 1
+ continue
+ accepted = set().union(*(
+ _accepted(f) for f in files
+ if isinstance(f, dict) and f.get("name") == cited["name"]))
+ with self.subTest(platform=platform, entry=entry["name"],
+ emulator=reader["emulator"]):
+ self.assertTrue(any(
+ a == m or (29 <= len(m) < 32 and a.startswith(m))
+ for m in declared for a in accepted))
+ self.assertGreater(by_name, 0)
+
+ def test_a_contradicting_homonym_is_not_cited(self):
+ profiles = {"core": {"type": "libretro", "systems": ["s"], "files": [
+ {"name": "bios.bin", "md5": "1" * 32, "source_ref": "a.c:1"},
+ {"name": "bios_v2.bin", "md5": "2" * 32, "source_ref": "a.c:2"},
+ ]}}
+ config = {"cores": ["core"], "systems": {"s": {"files": []}}}
+ cats = EntryCategories(config, profiles)
+ found = cats.explain({"name": "bios.bin", "md5": "2" * 32}, "bios.bin", "s")
+ self.assertEqual(found["read_by"][0]["entries"],
+ [{"name": "bios_v2.bin", "source_ref": "a.c:2"}])
+ found = cats.explain({"name": "bios.bin", "md5": "3" * 32}, "bios.bin", "s")
+ self.assertEqual(found["read_by"][0]["entries"], [])
+
+
+class EvidenceLinksItsSource(unittest.TestCase):
+ """Platform pages printed refs as code with the role word in front:
+ `source frontend/libretro.c:3864-3867`."""
+
+ def test_pinned_link_and_by_name(self):
+ evidence = {"read_by": [{
+ "emulator": "pcsx_rearmed", "proof": "name", "runs_system": True,
+ "entries": [{"name": "scph1001.bin", "by": "name",
+ "source_ref": "source frontend/libretro.c:3864-3867 (listed_bios[6])"}],
+ }]}
+ text = generate_site._category_evidence_md(evidence)
+ self.assertRegex(text, r"\[`frontend/libretro\.c:3864-3867`\]\(https://github\.com/"
+ r"libretro/pcsx_rearmed/blob/[0-9a-f]{40}/frontend/libretro\.c"
+ r"#L3864-L3867\)")
+ self.assertIn("(by name)", text)
+ self.assertNotIn("`source ", text)
+
+ def test_launcher_record_named_with_its_lines(self):
+ evidence = {"read_by": [{"emulator": "launcher", "proof": "launcher",
+ "entries": [], "runs_system": True}]}
+ config = load_platform_config("batocera", PLATFORMS)
+ launched = generate_site._launcher_refs("batocera", config)["scph1001.bin"]
+ text = generate_site._category_evidence_md(evidence, launched)
+ self.assertNotIn("(by launcher)", text)
+ self.assertIn("https://github.com/batocera-linux/batocera.linux/blob/batocera-43.1/"
+ "package/batocera/core/batocera-configgen/", text)
+ self.assertIn("duckstationGenerator.py:115-136", text)
+
+ def test_a_launcher_in_another_repository_is_not_linked(self):
+ source = list_source("https://gitlab.com/recalbox/recalbox/-/raw/10.1.1/x/es_bios.xml")
+ self.assertEqual(launcher_ref_markdown(None, "a/b.py:3"), "`b.py:3`")
+ self.assertIn("gitlab.com/recalbox/recalbox/-/blob/10.1.1/a/b.py#L3",
+ launcher_ref_markdown(source, "a/b.py:3"))
+
+
+class EachEntryNamesItsFile(unittest.TestCase):
+ def test_served_held_and_missing(self):
+ path, digest = next(
+ (p, d) for p, d in generate_site._collection_sha1_by_path().items()
+ if p.startswith("bios/") and p.count("/") >= 3)
+ served = generate_site._served_line({"status": "ok", "path": path})
+ self.assertRegex(served, rf"\]\(\.\./systems/[^)]+\.md#sha1-{digest[:12]}\)")
+ self.assertTrue(served.startswith("Served by: "))
+ self.assertIn(f"`{digest[:8]}`", served)
+ self.assertEqual(generate_site._served_line({"status": "missing"}), "Not held")
+ self.assertTrue(generate_site._served_line(
+ {"status": "untested", "path": path}).startswith("Held, not the declared hash"))
+ # Existence mode reports no path: nothing is claimed, held or not.
+ self.assertEqual(generate_site._served_line({"status": "ok"}), "")
+
+ def test_a_long_system_list_is_cut_between_files(self):
+ body = ["2/2 files verified", ""]
+ for n in range(4):
+ body += [f"- `f{n}.bin` - OK", " " + "x" * 100]
+ runs = generate_site._file_runs(body, 250)
+ self.assertEqual(len(runs), 2)
+ for run in runs:
+ self.assertEqual(run[:2], body[:2])
+ self.assertTrue(run[2].startswith("- "))
+ self.assertEqual(sum(len(run) - 2 for run in runs), len(body) - 2)
+
+
+class AlsoNamedIsNotCited(unittest.TestCase):
+ def test_other_system_cores_are_named_only(self):
+ evidence = {"read_by": [
+ {"emulator": "duckstation", "proof": "name", "runs_system": True,
+ "entries": [{"name": "scph1001_v20.bin", "source_ref": "src/core/bios.cpp:42"}]},
+ {"emulator": "mame", "proof": "name", "runs_system": False,
+ "entries": [{"name": "x.zip", "source_ref": "src/mame/x.cpp:1", "by": "name"}]},
+ ]}
+ text = generate_site._category_evidence_md(evidence)
+ also = text.split(", also named by ", 1)[1]
+ self.assertNotIn("x.cpp", also)
+ self.assertIn("mame", also)
+
+
+class PlatformReferenceInWords(unittest.TestCase):
+ def _page(self, mode: str, hash_type: str, systems: dict | None = None,
+ registry: dict | None = None) -> str:
+ details = [{"name": "f.bin", "status": "ok", "system": s} for s in (systems or {})]
+ cov = {"platform": "P", "details": details, "total": 1, "verified": 1,
+ "present": 1, "missing": 0, "untested": 0, "mode": mode,
+ "config": {"hash_type": hash_type, "systems": systems or {},
+ "source": "https://github.com/o/r/tree/" + "a" * 40}}
+ return generate_site.generate_platform_page("p", cov, registry or {})
+
+ def test_hash_type_only_when_it_differs(self):
+ self.assertNotIn("Hash type", self._page("md5", "md5"))
+ self.assertNotIn("Hashes in the list", self._page("md5", "md5"))
+ page = self._page("existence", "sha1")
+ self.assertIn("| Hashes in the list | SHA-1 |", page)
+ self.assertIn(">name only", page)
+ self.assertIn(">MD5", self._page("md5", "md5"))
+ self.assertIn("[r repository at commit aaaaaaaaaaaa](https://github.com/o/r/tree/"
+ + "a" * 40 + ")", page)
+
+ def test_systems_by_name_with_the_id_in_small_text(self):
+ systems = {"sony-playstation": {"files": [{"name": "f.bin"}]}}
+ page = self._page("md5", "md5", systems)
+ name = siterender.system_name("sony-playstation")
+ self.assertIn(f"| [{name}](#sony-playstation) sony-playstation |", page)
+ heading = next(line for line in page.splitlines() if line.startswith("### ")
+ and "sony-playstation" in line)
+ self.assertIn(name, heading)
+ self.assertTrue(heading.endswith("sony-playstation { #sony-playstation }"))
+
+ def test_logo_in_front_matter_not_floated(self):
+ page = self._page("md5", "md5", registry={"p": {"logo": "https://x.test/l.png"}})
+ self.assertTrue(page.startswith('---\nlogo: "https://x.test/l.png"\n---\n'))
+ self.assertNotIn('align="right"', page)
+
+
+class GapsPageNumbers(unittest.TestCase):
+ def test_badge_only_for_missing_required_firmware(self):
+ self.assertNotIn("rb-badge", generate_site._ratio_cell(99, 100))
+ self.assertIn('class="rb-bar"', generate_site._ratio_cell(99, 100))
+ self.assertIn("rb-badge-danger", generate_site._ratio_cell(99, 100, 1))
+ cov = {"details": [
+ {"name": "a", "system": "s", "status": "missing", "required": True, "category": "bios"},
+ {"name": "b", "system": "s", "status": "missing", "required": True,
+ "category": "game_data"},
+ {"name": "c", "system": "s", "status": "missing", "required": False},
+ ]}
+ self.assertEqual(generate_site._missing_required_firmware(cov), 1)
+
+ def test_headings_and_header(self):
+ source = Path(generate_site.__file__).read_text(encoding="utf-8")
+ body = source[source.index("def generate_gap_analysis("):]
+ body = body[:body.index("\ndef ", 10)]
+ self.assertIn('"# What the packs do not hold"', body)
+ self.assertIn("On its BIOS list (destinations)", body)
+ self.assertIn('"### Files to source"', body)
+ self.assertNotIn("### Files to source (", body)
+
+
+class UpstreamsGrouped(unittest.TestCase):
+ def test_mame_trees_are_one_project(self):
+ keys = {generate_site.upstream_project(u)[0] for u in (
+ "https://github.com/mamedev/mame",
+ "https://github.com/mamedev/mame/tree/mame0289",
+ "https://github.com/mamedev/mame/tree/mame0135u4",
+ )}
+ self.assertEqual(keys, {"github.com/mamedev/mame"})
+ self.assertEqual(generate_site.upstream_project(
+ "https://github.com/mamedev/mame/tree/mame0289")[1], "mame0289")
+
+ def test_nested_groups_notes_and_archives(self):
+ project = generate_site.upstream_project
+ self.assertEqual(
+ project("https://gitlab.com/recalbox/packages/recalbox-cores/libretro-exel-hence"),
+ ("gitlab.com/recalbox/packages/recalbox-cores/libretro-exel-hence", ""))
+ self.assertEqual(project("https://gitlab.com/jgemu/geolith/-/tree/v1"),
+ ("gitlab.com/jgemu/geolith", "v1"))
+ self.assertEqual(
+ project("https://git.9front.org/plan9front/plan9front (sys/src/games/nes/)"),
+ ("git.9front.org/plan9front/plan9front", "sys/src/games/nes/"))
+ key, _ = project("https://archive.softwareheritage.org/browse/revision/"
+ "dc20a615275f66bee20a4fd851ef0231daca4f14/")
+ self.assertEqual(generate_site._upstream_label(key),
+ "Software Heritage revision dc20a615275f")
+
+ def test_overview_and_a_page_per_platform(self):
+ profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
+ coverages = {}
+ for name in ("batocera", "retroarch"):
+ config = load_platform_config(name, PLATFORMS)
+ coverages[name] = {"platform": config.get("platform", name), "config": config}
+ moved: dict[str, str] = {}
+ page = generate_site.generate_cross_reference(coverages, profiles, moved)
+ self.assertEqual(set(moved), {"cross-reference-batocera", "cross-reference-retroarch"})
+ self.assertNotIn("mame\]\(", upstream)), 1)
+ header = next(line for line in upstream.splitlines() if line.startswith("| Cores"))
+ self.assertTrue(header.startswith("| Cores | Upstream |"))
+
+
+class ExportsComplete(unittest.TestCase):
+ def test_gap_rows_once_with_their_outcome(self):
+ path = next(iter(generate_site._collection_sha1_by_path()))
+ detail = {"name": "x.bin", "system": "s", "status": "untested", "path": path,
+ "category": "bios"}
+ coverages = {"p": {"platform": "P", "details": [
+ {"name": "h.xml", "system": "s", "status": "missing"},
+ {"name": "h.xml", "system": "s", "status": "missing"}, detail]}}
+ db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+ rows = generate_site._gap_export_rows(coverages, {}, db)
+ self.assertEqual([r["name"] for r in rows], ["h.xml", "x.bin"])
+ held = rows[1]
+ self.assertEqual(held["collection_path"], path)
+ self.assertEqual(held["resolved_sha1"], generate_site._collection_sha1_by_path()[path])
+ self.assertEqual(held["category"], "bios")
+
+ def test_platform_rows_carry_status_and_file(self):
+ db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+ sha1, entry = next(iter(db["files"].items()))
+ coverages = {"p": {"platform": "P", "config": {"systems": {"s": {"files": [
+ {"name": "a.bin"}, {"name": "b.bin"}]}}}, "details": [
+ {"name": "a.bin", "system": "s", "status": "ok", "path": entry["path"],
+ "category": "bios"},
+ {"name": "b.bin", "system": "s", "status": "missing", "category": "game_data"}]}}
+ _items, rows = generate_site._platform_export_rows(coverages, db)
+ self.assertEqual((rows[0]["status"], rows[0]["resolved_sha1"]), ("ok", sha1))
+ self.assertEqual(rows[0]["collection_path"], entry["path"])
+ self.assertEqual((rows[1]["status"], rows[1]["category"], rows[1]["collection_path"]),
+ ("missing", "game_data", ""))
+
+ def test_commit_and_wanted_list(self):
+ commit = generate_site._source_commit()
+ self.assertRegex(commit, r"^[0-9a-f]{40}$")
+ schema = json.loads((REPO_ROOT / "schemas" / "site-api-envelope.schema.json")
+ .read_text(encoding="utf-8"))
+ self.assertIn("commit", schema["properties"])
+ db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+ rows = generate_site._provenance_wanted_rows(db)
+ self.assertTrue(rows)
+ self.assertTrue(all(r["catalog"] and r["name"] for r in rows))
+ text = generate_site._csv_document(list(generate_site.WANTED_FIELDS), rows)
+ self.assertEqual(next(csv.reader(io.StringIO(text))), list(generate_site.WANTED_FIELDS))
+
+
+class DatesAndNames(unittest.TestCase):
+ def test_snapshot_date_in_words(self):
+ self.assertEqual(generate_site._date_in_words("2026-08-07T10:00:00Z"),
+ '')
+
+ def test_data_page_title(self):
+ self.assertTrue(generate_site.generate_data_page([]).startswith("# Data and API\n"))
+
+
+@unittest.skipUnless((DIST / "platforms" / "batocera" / "index.html").is_file(),
+ "site not built")
+class BuiltBatoceraPage(unittest.TestCase):
+ def test_served_by_and_pinned_links(self):
+ main = (DIST / "platforms" / "batocera" / "index.html").read_text(encoding="utf-8")
+ self.assertTrue("batocera-systems at tag batocera-43.1" in main)
+ # PlayStation's list is long enough to sit on a page of its own.
+ pages = [main] + [p.read_text(encoding="utf-8") for p in
+ (DIST / "platforms").glob("batocera-sony-playstation*/index.html")]
+ html = "".join(pages)
+ for text in ("Served by:", "scph1001_v20.bin", "https://github.com/stenzek/duckstation/blob/"):
+ with self.subTest(text=text):
+ self.assertTrue(text in html)
+ self.assertFalse("(by launcher)" in html)
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_site_render.py b/tests/test_site_render.py
index c957d34c..efbefc53 100644
--- a/tests/test_site_render.py
+++ b/tests/test_site_render.py
@@ -312,7 +312,9 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
by_system = {f"system-{i:02d}": [{"name": f"f{i}.bin", "status": "ok"}]
for i in range(11)}
- lines = generate_site._render_platform_systems(by_system, {}, {})
+ names = {sys_id: f"System {sys_id[-2:]}" for sys_id in by_system}
+ with mock.patch.dict(siterender.SYSTEM_NAMES, names):
+ lines = generate_site._render_platform_systems(by_system, {}, {})
headings = [line for line in lines if line.startswith("### ")]
self.assertEqual(len(headings), 11)
self.assertTrue(headings[0].endswith("{ #system-00 }"))
diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py
index 644b5b35..ad953d90 100644
--- a/tests/test_site_validation.py
+++ b/tests/test_site_validation.py
@@ -372,8 +372,12 @@ class PlatformPagesPrintTheDeclaredCrc32(unittest.TestCase):
from tests import sitepages # noqa: PLC0415
sitepages.require()
- page = sitepages.page("platforms/retroarch.md")
- self.assertIn("CRC32: `91e93107`", page.read_text(encoding="utf-8"))
+ # The page and the systems moved to pages of their own.
+ text = "".join(
+ path.read_text(encoding="utf-8") for path in sitepages.pages()
+ if sitepages.name(path) == "platforms/retroarch.md"
+ or sitepages.name(path).startswith("platforms/retroarch-"))
+ self.assertIn("CRC32: `91e93107`", text)
class PlainHouseStyle(unittest.TestCase):
diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md
index 69dcabf2..abdbeb0d 100644
--- a/wiki/testing-guide.md
+++ b/wiki/testing-guide.md
@@ -55,7 +55,7 @@ Every module in `tests/`, by the area it guards.
| Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
| Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
-| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
## Test architecture