From e3a54d2423ff7b20ef6a403ab76793080871ef12 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:05 +0200 Subject: [PATCH 1/5] feat: serve each schema at its own $id --- schemas/database.schema.json | 2 +- schemas/emulator.schema.json | 1 + schemas/install-manifest.schema.json | 59 ++++++---- schemas/pack-manifest.schema.json | 2 +- schemas/platform.schema.json | 10 +- schemas/release.schema.json | 2 +- schemas/site-api-envelope.schema.json | 152 ++++++++++++++++++++++++-- schemas/stats.schema.json | 2 +- schemas/target-manifest.schema.json | 2 +- 9 files changed, 199 insertions(+), 33 deletions(-) diff --git a/schemas/database.schema.json b/schemas/database.schema.json index 32a0c3a9..8970b58b 100644 --- a/schemas/database.schema.json +++ b/schemas/database.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Abdess/retrobios/schemas/database.schema.json", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/database.schema.json", "title": "RetroBIOS content database", "type": "object", "additionalProperties": false, diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index e4cca6ee..99e8e178 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -1,5 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/emulator.schema.json", "title": "Emulator BIOS Profile", "description": "One YAML per emulator core or engine, documenting every external file the code loads. Field reference: https://abdess.github.io/retrobios/wiki/profiling/", "type": "object", diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index dee48ee1..a2fd94b8 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Abdess/retrobios/schemas/install-manifest.schema.json", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/install-manifest.schema.json", "title": "RetroBIOS installer manifest", "type": "object", "additionalProperties": false, @@ -54,21 +54,26 @@ }, "platform": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Registry key of the platform (batocera)." }, "display_name": { "type": "string", - "minLength": 1 + "minLength": 1, + "description": "Name the site shows for the platform." }, "version": { - "type": "string" + "type": "string", + "description": "Version of the platform's own list this manifest was built from." }, "generated": { "type": "string", - "format": "date-time" + "format": "date-time", + "description": "When the manifest was written." }, "base_destination": { - "type": "string" + "type": "string", + "description": "The BIOS folder entries are written below, as the platform names it. Empty when entries start one level higher (RetroDECK)." }, "detect": { "type": "array", @@ -84,11 +89,13 @@ }, "total_files": { "type": "integer", - "minimum": 0 + "minimum": 0, + "description": "Number of entries in files." }, "total_size": { "type": "integer", - "minimum": 0 + "minimum": 0, + "description": "Bytes of the files in files." }, "pack_files": { "description": "Files the platform's pack holds once extracted: this list, the data directories and the pack's own documents.", @@ -102,7 +109,8 @@ }, "total_omitted": { "type": "integer", - "minimum": 0 + "minimum": 0, + "description": "Number of entries in omitted_files." }, "core_names": { "description": "Each core a file names, with the other names it goes by upstream (the buildbot's mednafen_psx for beetle_psx).", @@ -331,13 +339,15 @@ "pattern": "^[^/\\\\]+$" } } - } + }, + "description": "Files the pack needs and the repository cannot serve, with the reason." }, "files": { "type": "array", "items": { "$ref": "#/$defs/file" - } + }, + "description": "Every file the installer fetches, one entry per destination." }, "case_variants": { "description": "Files the folded list serves under another case, installed only where the target directory keeps case.", @@ -459,22 +469,27 @@ "properties": { "dest": { "type": "string", - "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" + "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$", + "description": "Path below base_destination, or below the save directory when root is save." }, "sha1": { "type": "string", - "pattern": "^[a-f0-9]{40}$" + "pattern": "^[a-f0-9]{40}$", + "description": "SHA1 of the copy the repository serves. The installer checks it after download." }, "sha256": { "type": "string", - "pattern": "^[a-f0-9]{64}$" + "pattern": "^[a-f0-9]{64}$", + "description": "SHA256 of the same copy." }, "size": { "type": "integer", - "minimum": 0 + "minimum": 0, + "description": "Bytes of the same copy." }, "repo_path": { - "type": "string" + "type": "string", + "description": "Path in the repository. Fetched from https://raw.githubusercontent.com/Abdess/retrobios//, ref being main unless RETROBIOS_REF names a tag." }, "cores": { "oneOf": [ @@ -488,14 +503,17 @@ }, "uniqueItems": true } - ] + ], + "description": "Profiles whose core brings the file into the pack. Null for a file the platform's own list declares." }, "storage": { - "const": "release" + "const": "release", + "description": "release when the file is a large-files release asset." }, "release_asset": { "type": "string", - "pattern": "^[^/\\\\]+$" + "pattern": "^[^/\\\\]+$", + "description": "Asset of the large-files release, fetched from https://github.com/Abdess/retrobios/releases/download/large-files/ instead of repo_path." }, "systems": { "description": "Every platform system that declares the file, for the installer's --system.", @@ -929,5 +947,6 @@ } ] } - } + }, + "description": "What a platform's pack installs, one file per entry, as install.py reads it. The platform's list says what the setup checks, this list says what the pack writes: the list's files plus every file the setup's cores load." } diff --git a/schemas/pack-manifest.schema.json b/schemas/pack-manifest.schema.json index 8438a4ad..54a36b23 100644 --- a/schemas/pack-manifest.schema.json +++ b/schemas/pack-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Abdess/retrobios/schemas/pack-manifest.schema.json", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/pack-manifest.schema.json", "title": "RetroBIOS pack integrity manifest", "type": "object", "additionalProperties": false, diff --git a/schemas/platform.schema.json b/schemas/platform.schema.json index 2bb494ad..73d7206d 100644 --- a/schemas/platform.schema.json +++ b/schemas/platform.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Abdess/retrobios/schemas/platform.schema.json", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/platform.schema.json", "title": "Platform Configuration", "description": "Schema for retrogaming platform BIOS configuration files", "type": "object", @@ -506,6 +506,14 @@ "type": "boolean", "description": "Added by our scraper from the cores' source, not declared in the platform's own file" }, + "shared_group": { + "type": "string", + "description": "Set on load, never written in a platform file: the entry joined the system from this group of _shared.yml, named by the system's includes. The platform's own file does not carry it" + }, + "zip_digest": { + "const": "composite", + "description": "Set on load, never written in a platform file: the md5 of this ZIP is the md5 of its members concatenated in name order, as Recalbox computes it, never the md5 of the archive bytes" + }, "native_name": { "type": "string", "description": "Display name the platform gives its own system" diff --git a/schemas/release.schema.json b/schemas/release.schema.json index 34e8bcde..4c3574ca 100644 --- a/schemas/release.schema.json +++ b/schemas/release.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Abdess/retrobios/schemas/release.schema.json", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/release.schema.json", "title": "RetroBIOS release record", "description": "What the packs of the latest release hold, read from the archives when the release is cut.", "type": "object", diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index 584ee44a..da48f568 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Abdess/retrobios/schemas/site-api-envelope.schema.json", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/site-api-envelope.schema.json", "title": "RetroBIOS static API v1 envelope", "type": "object", "additionalProperties": false, @@ -22,7 +22,11 @@ "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$", - "description": "The repository commit the data was read at." + "description": "The repository commit the data was read at. Every envelope carries it when the site is built from a git checkout." + }, + "license": { + "type": "string", + "description": "The terms the metadata is offered under, from LICENSE. They never cover BIOS or firmware files (NOTICE)." }, "kind": { "type": "string", @@ -33,7 +37,10 @@ "systems", "verification-and-coverage-gaps", "file-locations", - "file-lookup" + "file-lookup", + "index", + "system-index", + "core-index" ] }, "count": { @@ -115,7 +122,8 @@ "type": "string" }, "profile": { - "type": "object" + "type": "object", + "description": "The profile as its YAML states it, valid against emulator.schema.json (https://abdess.github.io/retrobios/api/v1/schemas/emulator.schema.json)." } } } @@ -150,10 +158,15 @@ "type": "string" }, "contract": { - "type": "object" + "type": "object", + "description": "The platform's list as the tools load it, valid against platform.schema.json (https://abdess.github.io/retrobios/api/v1/schemas/platform.schema.json). Entries a shared group adds carry shared_group, and a Recalbox ZIP entry carries zip_digest." }, "coverage": { "type": "object" + }, + "registry": { + "type": "object", + "description": "The platform's registry entry beside its list: install detection, logo, scraper and schedule. Not part of the list." } } } @@ -261,7 +274,8 @@ "pattern": "^[0-9a-f]{40}$" }, "url": { - "type": "string" + "type": "string", + "description": "Site-absolute path of the file's card, starting at /retrobios/." } } } @@ -384,7 +398,8 @@ "type": "string" }, "url": { - "type": "string" + "type": "string", + "description": "Site-absolute path of the page that shows the value, starting at /retrobios/. Prefix https://abdess.github.io for a full URL." }, "sha1": { "type": "string", @@ -398,6 +413,129 @@ } } } + }, + { + "if": { + "properties": { + "kind": { + "const": "index" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "title", + "url" + ], + "properties": { + "title": { + "type": "string" + }, + "url": { + "type": "string", + "description": "Relative to this document's URL." + }, + "description": { + "type": "string" + } + } + } + } + } + } + }, + { + "if": { + "properties": { + "kind": { + "const": "system-index" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "id", + "name", + "manufacturer", + "url", + "sha1" + ], + "properties": { + "id": { + "type": "string", + "description": "The system id a platform list or an emulator profile writes." + }, + "name": { + "type": "string", + "description": "The name the site shows for the id." + }, + "manufacturer": { + "type": "string" + }, + "url": { + "type": [ + "string", + "null" + ], + "description": "Site-absolute path of the console section holding the id's files, null when no section holds them." + }, + "sha1": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "description": "SHA1 of each collection file a platform list or an emulator profile names for the id." + } + } + } + } + } + } + }, + { + "if": { + "properties": { + "kind": { + "const": "core-index" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "name", + "profile" + ], + "properties": { + "name": { + "type": "string", + "description": "A core name as the buildbot or a platform writes it (mednafen_psx), a profile id, or an alias profile." + }, + "profile": { + "type": "string", + "description": "The profile id whose api/v1/emulators/.json and page describe it." + }, + "preferred": { + "type": "boolean", + "description": "The profile a platform list naming this core gets: its own key first, else the first claimant." + } + } + } + } + } + } } ] } diff --git a/schemas/stats.schema.json b/schemas/stats.schema.json index 6e0c9a90..8d920437 100644 --- a/schemas/stats.schema.json +++ b/schemas/stats.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Abdess/retrobios/schemas/stats.schema.json", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/stats.schema.json", "title": "RetroBIOS public statistics", "type": "object", "additionalProperties": false, diff --git a/schemas/target-manifest.schema.json b/schemas/target-manifest.schema.json index 6f74ed53..bb635153 100644 --- a/schemas/target-manifest.schema.json +++ b/schemas/target-manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Abdess/retrobios/schemas/target-manifest.schema.json", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/target-manifest.schema.json", "title": "RetroBIOS hardware target manifest", "description": "Per-target core availability. A null core list means the target publishes no inventory: a known target with no filter, not an absent one.", "type": "object", From db22cc37bbbf7cfacb9889baa2b01a9101ebba1a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:08 +0200 Subject: [PATCH 2/5] fix: link a source path its cached tree holds --- scripts/sitesources.py | 52 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/scripts/sitesources.py b/scripts/sitesources.py index 267fe766..adac7dc2 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -5,9 +5,11 @@ was written against, so a permalink is pinned, never guessed.""" from __future__ import annotations +import functools import re import urllib.parse from dataclasses import dataclass +from pathlib import Path import urllib.request import upstream @@ -150,6 +152,45 @@ def _source_permalink(repo: upstream.Repo, pin: str, path: str, url += f"-L{end}" return url +# The tree listings profile_sync caches. Read offline only: a page build +# never calls a forge, and a listing it does not find leaves the link as the +# profile's declarations alone decide it. +TREE_CACHE = Path(__file__).resolve().parent.parent / upstream.CACHE_DIR + + +@functools.lru_cache(maxsize=None) +def _tree_paths(repo: upstream.Repo, pin: str) -> frozenset[str] | None: + """Every file of a repository at a revision, None when not cached or + truncated (a missing path would then prove nothing).""" + if not repo.api_base: + return None # a browse-only forge: no listing to read + paths, truncated = upstream.list_tree(repo, pin, str(TREE_CACHE), offline=True) + return frozenset(paths) if paths and not truncated else None + + +def tree_holds(repo: upstream.Repo, pin: str, path: str) -> bool | None: + """Whether the repository holds ``path`` at ``pin``: None when its tree + is not cached. trident cites src/core/loader/ncch.cpp, a Panda3DS file, + and the page linked it into 3dsTrident, where it is a 404.""" + paths = _tree_paths(repo, pin) + return None if paths is None else path in paths + + +def _holder( + candidates: list[tuple[upstream.Repo, str]], path: str, default: tuple | None, +) -> tuple[upstream.Repo, str] | None: + """The pinned repository that holds ``path``, source before upstream as + profile_sync.select_views orders them. Without every tree cached, the + declarations decide (``default``). A path no cached tree holds links + nowhere: no link beats a link to a file that is not there.""" + verdicts = [tree_holds(repo, pin, path) for repo, pin in candidates] + if not candidates or None in verdicts: + if default is not None and tree_holds(*default, path) is False: + return None + return default + return next((cand for cand, held in zip(candidates, verdicts) if held), None) + + def _commit_url(repo: upstream.Repo, pin: str) -> str: """Forge-specific browser URL for one commit.""" base = f"https://{repo.host}/{repo.owner}/{repo.name}" @@ -272,8 +313,13 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str: if current_role: # Only the repository the part names, or no link at all. selected = by_role.get(current_role) - elif len(candidates) == 1: - selected = candidates[0] + if selected is not None and tree_holds(*selected, part.path) is False: + selected = None + else: + selected = _holder( + candidates, part.path, + candidates[0] if len(candidates) == 1 else None, + ) tag = (f" ({role_label(profile, current_role, part.path)})" if current_role and _two_repositories(profile) else "") @@ -330,6 +376,8 @@ def link_prose_citations(profile: dict, text: str) -> str: continue if any(lo <= path_span[0] < hi for lo, hi in guarded): continue + if tree_holds(repo, pin, path) is False: + continue for index, ((lo, hi), part) in enumerate(zip(spans, parts)): start = path_span[0] if index == 0 else lo url = _source_permalink(repo, pin, part.path, part.start, part.end) From c3b45182ca124641ac175ef7c888c60974549f1f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:08 +0200 Subject: [PATCH 3/5] feat: add a folder check built on the api --- scripts/check_platform_folder.py | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 scripts/check_platform_folder.py diff --git a/scripts/check_platform_folder.py b/scripts/check_platform_folder.py new file mode 100644 index 00000000..b803c6f9 --- /dev/null +++ b/scripts/check_platform_folder.py @@ -0,0 +1,61 @@ +"""Check a BIOS folder against one platform's list, from the published data only. + + python check_platform_folder.py batocera path/to/bios +""" +from __future__ import annotations + +import hashlib +import json +import sys +import urllib.request +import zipfile +from pathlib import Path + +API = "https://abdess.github.io/retrobios/api/v1/platforms/" + + +def accepts(entry: dict, path: Path, mode: str) -> bool: + """Whether the platform's own check passes the file at path.""" + raw = entry.get(mode) or "" + values = raw if isinstance(raw, list) else str(raw).split(",") + declared = [value.strip().lower() for value in values if value.strip()] + if mode == "existence" or not declared: + return path.is_file() # nothing to compare: the platform checks presence + try: + if entry.get("zipped_file"): # the hash of one member, its name casefolded + with zipfile.ZipFile(path) as archive: + names = {name.casefold(): name for name in archive.namelist()} + member = names.get(entry["zipped_file"].casefold()) + return bool(member) and hashlib.new(mode, archive.read(member)).hexdigest() in declared + actual = hashlib.new(mode, path.read_bytes()).hexdigest() + if entry.get("zip_digest") != "composite" and any(actual.startswith(d) for d in declared): + return True # a truncated md5 (Batocera) compares by its prefix + if mode == "md5" and path.suffix.lower() == ".zip": # Recalbox: members in name order + with zipfile.ZipFile(path) as archive: + names = sorted(name for name in archive.namelist() if not name.endswith("/")) + return hashlib.md5(b"".join(map(archive.read, names))).hexdigest() in declared + except (OSError, zipfile.BadZipFile): + return False + return False + + +def check(platform: str, root: Path, api: str = API) -> list[tuple[str, str]]: + """(destination, ok | wrong | missing) for each entry of the list.""" + with urllib.request.urlopen(f"{api}{platform}.json") as response: + contract = json.load(response)["items"][0]["contract"] + mode = contract["verification_mode"] + results = [] + for system in contract["systems"].values(): + for entry in system.get("files") or []: + destination = entry.get("destination") or entry["name"] + path = root / destination + verdict = "ok" if accepts(entry, path, mode) else "wrong" if path.is_file() else "missing" + results.append((destination, verdict)) + return results + + +if __name__ == "__main__": + found = check(sys.argv[1], Path(sys.argv[2])) + for verdict in ("ok", "wrong", "missing"): + names = sorted({dest for dest, seen in found if seen == verdict}) + print(f"{verdict}: {len(names)}" + "".join(f"\n {name}" for name in names if verdict != "ok")) From 2b64eaab9b50c4fd2570cc4fa5bf5d91eacc5112 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:13 +0200 Subject: [PATCH 4/5] feat: publish manifests, id indexes and licence The CSV and SQLite carry zipped_file and the Recalbox flags, contracts leave registry keys in a registry field and validate, every envelope carries commit, lookup URLs are site-absolute, systems.json and cores.json answer an id, the Data page shows the manifest fields and a folder check, and platform overviews link emulators by name. --- scripts/generate_site.py | 479 +++++++++++++++++++++++++---- tests/test_site_exports.py | 11 +- tests/test_site_platform_hashes.py | 5 +- tests/test_site_r2_gate.py | 2 +- tests/test_site_r3_reference.py | 469 ++++++++++++++++++++++++++++ tests/test_site_render.py | 2 +- wiki/testing-guide.md | 2 +- 7 files changed, 907 insertions(+), 63 deletions(-) create mode 100644 tests/test_site_r3_reference.py diff --git a/scripts/generate_site.py b/scripts/generate_site.py index df4b4a93..4319a7f8 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -106,6 +106,9 @@ STARLIGHT_CONTENT = "src/content/docs" SITE_NAME = "RetroBIOS" REPO_URL = "https://github.com/Abdess/retrobios" RELEASE_URL = f"{REPO_URL}/releases/latest" +# What LICENSE grants over the published data: its MIT terms name the +# metadata. The BIOS files themselves are under NOTICE, never this. +DATA_LICENSE = "MIT (LICENSE: scripts, tooling, schemas, documentation and metadata)" SITE_URL = "https://abdess.github.io/retrobios/" GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads"] # Generated directories of files served as they are, not pages. @@ -806,6 +809,36 @@ def _sha1_by_path(db: dict | None) -> dict[str, str]: return {str(e.get("path", "")): sha1 for sha1, e in ((db or {}).get("files") or {}).items()} +@functools.lru_cache(maxsize=1) +def _contract_keys() -> frozenset[str]: + """The top-level keys platform.schema.json allows in a platform's list.""" + schema = json.loads((_REPO_ROOT / "schemas" / "platform.schema.json").read_text(encoding="utf-8")) + return frozenset(schema.get("properties") or {}) + + +def _split_contract(config: dict) -> tuple[dict, dict]: + """The list, valid against platform.schema.json, and the registry entry. + + load_platform_config merges the registry into the list: `install`, + `logo`, `scraper` and `_key` made all 18 contracts fail the published + schema. They stay published, beside the contract. + """ + allowed = _contract_keys() + contract = {key: value for key, value in config.items() if key in allowed} + registry = {key: value for key, value in config.items() + if key not in allowed and key != "_key"} + return contract, registry + + +# Per-entry fields a folder check needs beyond the hashes: an md5 is a ZIP +# member's when zipped_file is set, and a CSV without it judged 106 good +# Batocera files wrong. +ENTRY_EXPORT_FIELDS = ( + "zipped_file", "hash_match_mandatory", "mandatory_declared", "curated", + "shared_group", "mirror_of", "zip_digest", +) + + def _platform_export_rows( coverages: dict, db: dict | None = None, ) -> tuple[list[dict], list[dict]]: @@ -816,6 +849,7 @@ def _platform_export_rows( for key, coverage in sorted(coverages.items()): config = coverage["config"] outcome = _verify_outcomes(coverage, sha1_by_path) + contract, registry = _split_contract(config) items.append({ "id": key, "name": coverage["platform"], @@ -826,7 +860,8 @@ def _platform_export_rows( "core_present", "core_missing", "core_unsourceable", ) }, - "contract": config, + "contract": contract, + "registry": registry, }) for system_id, system in sorted(config.get("systems", {}).items()): for entry in system.get("files", []) or []: @@ -846,11 +881,20 @@ def _platform_export_rows( "sha256": _json_text(entry.get("sha256")), "md5": _json_text(entry.get("md5")), "crc32": _json_text(entry.get("crc32")), + **{field: _export_flag(entry.get(field)) for field in ENTRY_EXPORT_FIELDS}, **outcome(system_id, entry.get("name", "")), }) return items, file_rows +def _export_flag(value: object) -> object: + """A per-entry field as CSV and SQLite hold it: booleans stay booleans, + an absent field is empty.""" + if value is None: + return "" + return value if isinstance(value, bool) else str(value) + + def _emulator_export_items(profiles: dict) -> list[dict]: return [ {"id": key, "profile": profile} @@ -1089,7 +1133,9 @@ def _write_sqlite_export( destination TEXT NOT NULL, required INTEGER NOT NULL, region TEXT, variant_group TEXT, size INTEGER, sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT, - status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT + status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT, + zipped_file TEXT, hash_match_mandatory INTEGER, mandatory_declared INTEGER, + curated INTEGER, shared_group TEXT, mirror_of TEXT, zip_digest TEXT ); CREATE TABLE emulators ( id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT, @@ -1134,6 +1180,7 @@ def _write_sqlite_export( **({"commit": commit} if commit else {}), "source": REPO_URL, "scope": "metadata only, no BIOS or firmware payload bytes", + "license": DATA_LICENSE, } connection.executemany( "INSERT INTO metadata(key, value) VALUES (?, ?)", @@ -1168,7 +1215,8 @@ def _write_sqlite_export( "INSERT INTO platform_files VALUES " "(:platform_id, :system, :name, :destination, :required, :region, " ":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, " - ":collection_path, :resolved_sha1)", + ":collection_path, :resolved_sha1, :zipped_file, :hash_match_mandatory, " + ":mandatory_declared, :curated, :shared_group, :mirror_of, :zip_digest)", platform_files, ) for item in emulator_items: @@ -1359,13 +1407,21 @@ def generate_data_exports( json.dumps(db, ensure_ascii=False, indent=2) + "\n", ) - schema_names = ( - "database.schema.json", "emulator.schema.json", "platform.schema.json", - "site-api-envelope.schema.json", "stats.schema.json", - ) - for schema_name in schema_names: - shutil.copy2(Path("schemas") / schema_name, schemas_dest / schema_name) - _record(schemas_dest / schema_name) + # Every schema the published files are written against, the manifests' + # too: an integrator had the platform's list and not what the pack writes. + for schema_path in sorted((_REPO_ROOT / "schemas").glob("*.schema.json")): + shutil.copy2(schema_path, schemas_dest / schema_path.name) + _record(schemas_dest / schema_path.name) + + manifest_assets = _publish_manifests(api) + for filename, (kind, items) in { + "systems.json": ("system-index", system_index_items(db, coverages, profiles)), + "cores.json": ("core-index", core_index_items(profiles)), + }.items(): + write_if_changed(str(api / filename), json.dumps({ + "schema_version": 1, "generated_at": generated_at, **stamp, + "kind": kind, "count": len(items), "items": items, + }, ensure_ascii=False, indent=2, sort_keys=True) + "\n") file_rows = [] for sha1, entry in sorted(db.get("files", {}).items()): @@ -1400,7 +1456,7 @@ def generate_data_exports( "platform_id", "platform", "system", "name", "destination", "required", "region", "variant_group", "size", "sha1", "sha256", "md5", "crc32", "status", "category", - "collection_path", "resolved_sha1", + "collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS, ], platform_files, ), @@ -1452,6 +1508,11 @@ def generate_data_exports( (api / "gaps.json", "Verification and coverage gaps", "application/json", "schemas/site-api-envelope.schema.json"), (api / "file-locations.json", "Where each file's card is, by SHA1", "application/json", "schemas/site-api-envelope.schema.json"), (api / "stats.json", "Project statistics", "application/json", "schemas/stats.schema.json"), + (api / "systems.json", "System ids, their names and sections", "application/json", + "schemas/site-api-envelope.schema.json"), + (api / "cores.json", "Core names and the profile each one is", "application/json", + "schemas/site-api-envelope.schema.json"), + *manifest_assets, (downloads / "files.csv", "File hashes CSV", "text/csv", None), (downloads / "retrobios.dat", "Collection DAT (Logiqx XML, for clrmamepro and RomVault)", "application/xml", None), @@ -1488,6 +1549,7 @@ def generate_data_exports( "generated_at": generated_at, **stamp, "kind": "catalog", + "license": DATA_LICENSE, "count": len(published), "items": published, } @@ -1495,10 +1557,176 @@ def generate_data_exports( str(api / "catalog.json"), json.dumps(catalog, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) + # api/v1/ itself is a folder GitHub Pages answers with the 404 page: the + # entry point is a document of its own, naming where to start. + entry_points = [ + {"title": "Catalog", "url": "catalog.json", + "description": "Every published file with its size, SHA256 and schema."}, + {"title": "System ids", "url": "systems.json", + "description": "Each system id with its name and the section holding its files."}, + {"title": "Core names", "url": "cores.json", + "description": "Each core name and alias with the profile it designates."}, + {"title": "Schemas", "url": "schemas/site-api-envelope.schema.json", + "description": "The envelope every JSON document of api/v1 shares."}, + ] + write_if_changed(str(api / "index.json"), json.dumps({ + "schema_version": 1, "generated_at": generated_at, **stamp, + "kind": "index", "license": DATA_LICENSE, + "count": len(entry_points), "items": entry_points, + }, ensure_ascii=False, indent=2, sort_keys=True) + "\n") return catalog_items +def _publish_manifests(api: Path) -> list[tuple]: + """The install manifests and the release record, beside the API. + + The platform's list says what a setup checks. The manifest says what its + pack installs: Batocera's list holds 353 files, its pack 14,508. The + only copy was install/.json on the repository's main branch, + which no page named. Copied byte for byte: the SHA256 the catalog gives + is the repository file's. + """ + folder = api / "manifests" + folder.mkdir(parents=True, exist_ok=True) + assets: list[tuple] = [] + for source in sorted((_REPO_ROOT / "install").glob("*.json")): + target = folder / source.name + shutil.copy2(source, target) + _record(target) + assets.append((target, f"Install manifest: {source.stem}", "application/json", + "schemas/install-manifest.schema.json")) + release = _REPO_ROOT / "release.json" + if release.is_file(): + shutil.copy2(release, api / "release.json") + _record(api / "release.json") + assets.append((api / "release.json", "Latest release: packs, sizes and assets", + "application/json", "schemas/release.schema.json")) + return assets + + +def _site_path(markdown_target: str) -> str: + """A page link as generators write it (systems/sony.md#psx) as a + site-absolute path (/retrobios/systems/sony/#psx), which resolves the same + from any document that holds it.""" + page, _, anchor = markdown_target.partition("#") + base = urllib.parse.urlsplit(SITE_URL).path + page = page.removesuffix(".md") + page = "" if page == "index" else page.removesuffix("/index") + "/" + return f"{base}{page}" + (f"#{anchor}" if anchor else "") + + +def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: + """Each system id the platforms and profiles write, reachable by that id. + + api/v1/systems/ is keyed by manufacturer and its consoles by display + name: a frontend keying by system id (sony-playstation) had no way in. + The url is the section system_section gives, the one every page links. + """ + from siterender import system_ids, system_section + + sha1_by_path = _sha1_by_path(db) + files = db.get("files") or {} + by_md5 = (db.get("indexes") or {}).get("by_md5") or {} + held: dict[str, set[str]] = {} + for coverage in coverages.values(): + for detail in coverage.get("details") or []: + path = str(detail.get("path") or "") + sha1 = sha1_by_path.get(path) if detail.get("status") != "missing" else None + if sha1: + held.setdefault(str(detail.get("system", "")), set()).add(sha1) + for profile in profiles.values(): + if profile.get("type") in ("alias", "test"): + continue + own = [str(s) for s in profile.get("systems") or []] + for entry in profile.get("files") or []: + if not isinstance(entry, dict): + continue + ids = ([str(entry["system"])] if entry.get("system") + else [str(s) for s in entry.get("systems") or []] or + (own if len(own) == 1 else [])) + found = {v for v in _hash_values(entry.get("sha1")) if v in files} + for md5 in _hash_values(entry.get("md5")): + match = by_md5.get(md5) + found.update([match] if isinstance(match, str) else match or []) + for sys_id in ids: + held.setdefault(sys_id, set()).update(found) + manufacturers = {_slug(mfr): mfr for mfr in _group_by_manufacturer(db)} + items = [] + for sys_id in sorted(system_ids(profiles, [c["config"] for c in coverages.values()])): + section = system_section(sys_id) + try: + name = system_name(sys_id) + except KeyError: + name = sys_id + items.append({ + "id": sys_id, + "name": name, + "manufacturer": manufacturers.get(section[0], "") if section else "", + "url": _site_path(console_href(*section)) if section else None, + "sha1": sorted(held.get(sys_id, set()) & files.keys()), + }) + return items + + +def core_index_items(profiles: dict) -> list[dict]: + """Every name a core goes by, with the profile it designates. + + A frontend knows mednafen_psx_libretro, and api/v1/emulators/ had only + beetle_psx.json: the reverse index resolve_platform_cores reads, aliases + included, said once. A name two profiles claim (pcsx2) lists both, the + one a platform list designates first. + """ + from common import preferred_profile, upstream_profile_index + + published = {name for name, p in profiles.items() if p.get("type") not in ("alias", "test")} + index = upstream_profile_index(profiles) + for name, profile in profiles.items(): + target = str(profile.get("alias_of") or "") + if profile.get("type") == "alias" and target in published: + index.setdefault(name, set()).add(target) + items = [] + for name in sorted(index): + claimants = sorted(index[name] & published) + preferred = preferred_profile({name: set(claimants)}, name) + for profile in sorted(claimants, key=lambda p: (p != preferred, p)): + items.append({"name": name, "profile": profile, "preferred": profile == preferred}) + return items + + +def _is_manifest(item: dict) -> bool: + return item["url"].startswith("api/v1/manifests/") + + +def _schema_field_rows(schema: dict, keys: tuple[str, ...]) -> list[str]: + """One table row per field, its description read from the schema, so the + page and the published contract say the same thing.""" + props = schema.get("properties") or {} + return [ + f"| `{key}` | {_escape_tags(str(props[key]['description']))} |" + for key in keys if (props.get(key) or {}).get("description") + ] + + +# The manifest fields an integrator reads first, in the order they matter. +MANIFEST_TOP_FIELDS = ("platform", "version", "base_destination", "files", + "omitted_files", "pack_files", "core_names", "system_cores") +MANIFEST_FILE_FIELDS = ("dest", "sha1", "size", "repo_path", "release_asset", "cores", + "systems", "category", "alternatives") + + +def _short_sha256(item: dict, catalog: str) -> str: + """A digest cell that fits its column: 12 characters linking the catalog + that holds it, the whole value in the title. 64 characters pushed the + table to 932 px in a 720 px column.""" + value = str(item["sha256"]) + return f'{value[:12]}' + + def generate_data_page(exports: list[dict]) -> str: + manifest_schema = json.loads( + (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8")) + snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") + manifests = [item for item in exports if _is_manifest(item)] lines = [ "# Data and API", "", @@ -1511,20 +1739,21 @@ def generate_data_page(exports: list[dict]) -> str: markup.button("Collection DAT", "downloads/retrobios.dat"), ]), "", - "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game " - "per `bios///` folder of the collection, named " - "`Manufacturer\\Console` so the tools look in that subfolder, each file a " - "rom named by its path below that folder. It audits a copy of the " - "collection's `bios/` tree, or rebuilds that layout from your own dumps. A " - "platform pack has its own layout and is checked with `install.py --check`.", + f"**Licence:** the metadata is offered under the MIT terms of [LICENSE]({REPO_URL}/blob/main/LICENSE), " + "which cover the scripts, tooling, schemas, documentation and metadata. " + f"They never cover a BIOS or firmware file ([NOTICE]({REPO_URL}/blob/main/NOTICE)). " + "`catalog.json` and the SQLite `metadata` table carry the same `license`.", "", "## Stable JSON endpoints", "", + "Start at [`api/v1/index.json`](api/v1/index.json): it names the catalog, " + "the two id indexes and the envelope schema.", + "", "| Dataset | Endpoint | Schema | Size |", "|---------|----------|--------|-----:|", ] for item in exports: - if item["media_type"] != "application/json": + if item["media_type"] != "application/json" or _is_manifest(item): continue schema = ( f"[JSON Schema](api/v1/{item['schema']})" @@ -1536,17 +1765,28 @@ def generate_data_page(exports: list[dict]) -> str: ) lines.extend([ "", - "Every JSON document carries `schema_version`, and `commit`, the revision " + "Every envelope carries `schema_version`, `kind` and `commit`, the revision " "of this repository the data was read at. Breaking changes use a new " - "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1.", + "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. " + "Each schema's `$id` is the address it is served at.", "", "One emulator profile without the whole list: " "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " "[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each " "emulator page links its own. `api/v1/platforms/.json` holds one " - "platform of `platforms.json`, and `api/v1/systems/.json` the files " - "of one system page by console. Each platform and system page links its own.", + "platform of `platforms.json`, its `contract` valid against " + "[`platform.schema.json`](api/v1/schemas/platform.schema.json) and its " + "`registry` the install and scraper settings beside the list. " + "`api/v1/systems/.json` holds the files of one system page by console. " + "Each platform and system page links its own.", + "", + "By id: [`systems.json`](api/v1/systems.json) gives each system id a platform " + "or a profile writes (`sony-playstation`) with its name, its manufacturer, the " + "`url` of the section holding its files and their SHA1. " + "[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a " + "platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), " + "whose document is `api/v1/emulators/.json`.", "", "One hash without the whole list: `api/v1/lookup/.json`, where `` " "is the first two characters of the hash in lowercase, holds every SHA1, " @@ -1555,6 +1795,65 @@ def generate_data_page(exports: list[dict]) -> str: "profile), the file `name` and the `url` of the page that shows it. " "[Find a file](files/index.md) reads it.", "", + "Two kinds of `url`:", + "", + "- In `catalog.json` and `index.json`, a `url` is relative to the document " + "holding it, as RFC 3986 resolves it.", + "- In the lookup shards, `file-locations.json` and `systems.json`, a `url` " + f"is site-absolute (`/retrobios/systems/sony/#...`). Prefix " + f"`{SITE_URL.removesuffix(urllib.parse.urlsplit(SITE_URL).path)}` for a full address.", + "", + "## Platform manifests", + "", + "A platform's list is what the setup checks. Its manifest is what the pack " + "installs: the list's files plus every file the setup's cores load. " + "`api/v1/manifests/.json` is the file `install.py` reads, byte " + "for byte, valid against " + "[`install-manifest.schema.json`](api/v1/schemas/install-manifest.schema.json). " + "[`api/v1/release.json`](api/v1/release.json) records the packs of the latest " + "release and the files each one holds.", + "", + "Manifests: " + ", ".join( + f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})" + for item in manifests) + "." if manifests else "", + "", + "| Field | Meaning |", + "|-------|---------|", + *_schema_field_rows(manifest_schema, MANIFEST_TOP_FIELDS), + "", + "Each entry of `files`:", + "", + "| Field | Meaning |", + "|-------|---------|", + *_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), + "", + "## Check a folder for one platform", + "", + "The job most tools come for: is this BIOS folder what the platform checks? " + "Read `api/v1/platforms/.json` and apply the platform's own rules:", + "", + "- `verification_mode: existence`: a file at its `destination` passes.", + "- An entry with no hash in the mode's field is checked on presence too.", + "- `zipped_file` set: the md5 is that of this member inside the ZIP, never " + "the ZIP's. Member names compare casefolded. A ZIP declared once per member " + "appears once per member: Batocera lists 692 entries for 353 destinations.", + "- An md5 shorter than 32 characters (Batocera) compares by its prefix. " + "Recalbox writes several accepted md5 in one field, comma separated.", + "- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members " + "read in name order, never the archive's.", + "- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. " + "The script below does not apply it.", + "", + "```python", + snippet.rstrip("\n"), + "```", + "", + "The script is [`scripts/check_platform_folder.py`]" + f"({REPO_URL}/blob/main/scripts/check_platform_folder.py). A test runs it " + "against the verifier on a sample folder, so the two give the same answer. " + "`install.py --check` answers another question: whether the folder holds " + "what the pack installs, the manifest above.", + "", "## Bulk downloads", "", "| Export | Format | Size | SHA256 |", @@ -1565,9 +1864,19 @@ def generate_data_page(exports: list[dict]) -> str: continue lines.append( f"| [{item['title']}]({item['url']}) | `{item['media_type']}` | " - f"{_fmt_size(item['bytes'])} | `{item['sha256']}` |" + f"{_fmt_size(item['bytes'])} | {_short_sha256(item, 'api/v1/catalog.json')} |" ) lines.extend([ + "", + "Each SHA256 is cut to 12 characters. Point at it for the whole value, or " + "read it in [`catalog.json`](api/v1/catalog.json).", + "", + "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game " + "per `bios///` folder of the collection, named " + "`Manufacturer\\Console` so the tools look in that subfolder, each file a " + "rom named by its path below that folder. It audits a copy of the " + "collection's `bios/` tree, or rebuilds that layout from your own dumps. A " + "platform pack has its own layout and is checked with `install.py --check`.", "", "The SQLite file contains indexed tables for content hashes, provenance " "catalog matches, platform declarations, emulator profiles and current " @@ -1601,6 +1910,25 @@ def generate_data_page(exports: list[dict]) -> str: "`provenance-wanted.csv` is the Wanted list of [Dump provenance]" "(provenance.md): each dump a catalog describes and the collection lacks.", "", + "`platform-files.csv` and the `platform_files` table also carry the entry " + "fields a check needs:", + "", + "- `zipped_file` set: the md5 is the md5 of that member inside the ZIP.", + "- `zip_digest` is `composite` where the md5 of a ZIP covers its members.", + "- `hash_match_mandatory` and `mandatory_declared` are Recalbox's flags.", + "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's " + "own file does not carry.", + "", + "## Hosting and limits", + "", + "- Static files on GitHub Pages, read with a plain GET. No key, no account.", + "- Any origin may read them (`Access-Control-Allow-Origin: *`), so a " + "browser tool can call them directly.", + "- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). " + "Send the `ETag` back in `If-None-Match` to skip an unchanged download.", + "- The base is the project's address, " + f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.", + "", "## Semantics and limits", "", "Treat four questions separately: content identity (hashes), presence in " @@ -2365,19 +2693,30 @@ def _blocks_in_runs(blocks: list[list[str]], budget: int) -> list[list[str]]: return runs +def _parent_front_matter(parent: tuple[str, str] | None) -> list[str]: + """The `parent:` front matter of a part page: the page it belongs to, + by label and site path, which the breadcrumb and sidebar read.""" + if parent is None: + return [] + label, link = parent + return ["parent:", f" label: {json.dumps(label)}", f" link: {json.dumps(link)}"] + + def _list_pages( stem: str, title: str, intro: str, head: list[str], blocks: list[list[str]], moved: dict[str, str], budget: int = LIST_PART_BUDGET, + parent: tuple[str, str] | None = None, ) -> str: """Pages of their own for a long list: ``head`` (a table header, say) opens each part, ``blocks`` are cut between parts. Returns where the list - went, as the page it left says it.""" + went, as the page it left says it. ``parent`` (label, site path) is the + page the parts belong to, for the breadcrumb.""" runs = _blocks_in_runs(blocks, budget) stems = [stem + (f"-{n}" if n > 1 else "") for n in range(1, len(runs) + 1)] for n, (here, run) in enumerate(zip(stems, runs), 1): part = f" (part {n} of {len(runs)})" if len(runs) > 1 else "" moved[here] = "\n".join([ - "---", "level: reference", "---", "", + "---", "level: reference", *_parent_front_matter(parent), "---", "", f"# {title}{part}", "", intro, "", *([", ".join(f"[part {i}]({other}.md)" if other != here else f"part {i}" for i, other in enumerate(stems, 1)), ""] if len(runs) > 1 else []), @@ -2404,6 +2743,11 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str: writes. A system with no section of its own is named, never sent to the page a storage folder chose. """ + return f"{_system_named_link(sys_id, prefix)} {_escape_tags(sys_id)}" + + +def _system_named_link(sys_id: str, prefix: str = "") -> str: + """A system's name, linked to the section system_section gives.""" from siterender import system_section name = _escape_tags(system_name(sys_id)) @@ -2411,7 +2755,17 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str: if section is not None: slug, console = section name = f"[{name}]({console_href(slug, console, prefix)})" - return f"{name} {_escape_tags(sys_id)}" + return name + + +def _emulator_named_link(key: str, prefix: str = "") -> str: + """An emulator by the name its page carries, linked to that page. A + profile with no page (an alias, a test) is named only.""" + profile = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False).get(key, {}) + label = _escape_tags(str(profile.get("emulator") or key)) + if profile.get("type") in ("alias", "test"): + return label + return f"[{label}]({prefix}emulators/{key}.md)" def _served_line(f: dict) -> str: @@ -2493,7 +2847,7 @@ def _render_platform_systems( anchor = sys_id.replace(" ", "-") body = [f"{ok_count}/{total} files verified"] if sys_emus: - emu_links = ", ".join(_emulator_link(e, "../") for e in sorted(sys_emus)) + emu_links = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus)) body[-1] += "\\" body.append(f"Emulators: {emu_links}") body.append("") @@ -2620,8 +2974,12 @@ def _render_platform_systems( for n in range(1, len(runs) + 1)] for n, (stem, run) in enumerate(zip(stems, runs), 1): part = f" (part {n} of {len(runs)})" if len(runs) > 1 else "" + # A console query landed here with no way to the console's + # own page: the list says whose it is first, linked. moved[stem] = "\n".join([ + "---", *_parent_front_matter((display, f"/platforms/{key}/")), "---", "", f"# {display}: {system_name(sys_id)}{part}", "", + f"{display}'s list for {_system_named_link(sys_id, '../')}. " f"Part of the [{display} page]({key}.md#{anchor}), whose other systems " f"are listed there. Each file below is as {display} declares it, with " "its status in the pack.", @@ -3121,10 +3479,12 @@ def generate_platform_page( f'{non_ok} issue{"s" if non_ok > 1 else ""}' ) sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id)) - emu_str = ", ".join(sys_emus[:3]) - if len(sys_emus) > 3: - emu_str += f" +{len(sys_emus) - 3}" anchor = sys_id.replace(" ", "-") + # Named and linked as the detail blocks below do: the column held + # profile ids, none of 1,463 linked. + emu_str = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus)[:3]) + if len(sys_emus) > 3: + emu_str += f" [+{len(sys_emus) - 3}](#{anchor})" # The name a reader filters on, the id the list writes beside it. lines.append( f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) {sys_id} " @@ -3257,7 +3617,9 @@ def file_location_items( A card lives on its manufacturer's page, and a file refiled between manufacturers (mt32_control.rom, Commodore to Roland) moved its link with it. files/?sha1= reads this list and goes to the card wherever it - is now: the address cited depends on the content only. + is now: the address cited depends on the content only. The url is + site-absolute (/retrobios/systems/...): written from the site root, it + resolved against the document holding it to a page that does not exist. """ items = [] for sha1, entry in sorted(db.get("files", {}).items()): @@ -3265,7 +3627,7 @@ def file_location_items( if len(parts) < 3 or parts[0] != "bios": continue page = card_page(entry.get("path", ""), sha1) - items.append({"sha1": sha1, "url": f"systems/{page}/#sha1-{sha1[:12]}"}) + items.append({"sha1": sha1, "url": _site_path(f"systems/{page}.md#sha1-{sha1[:12]}")}) # A data directory's files are listed on the page of an emulator reading # the tree: 2,534 hashes the packs ship were found nowhere on the site. owners = _data_member_owners(profiles or {}) @@ -3281,7 +3643,8 @@ def file_location_items( if sha1 and sha1 not in known: known.add(sha1) page = page_of.get(member, owners[ref]) - items.append({"sha1": sha1, "url": f"emulators/{page}/#sha1-{sha1[:12]}"}) + items.append({"sha1": sha1, + "url": _site_path(f"emulators/{page}.md#sha1-{sha1[:12]}")}) return sorted(items, key=lambda item: item["sha1"]) @@ -3325,7 +3688,7 @@ def file_lookup_records( }) for key, coverage in sorted(coverages.items()): for system_id, system in sorted((coverage["config"].get("systems") or {}).items()): - url = f"platforms/{key}/#{system_id.replace(' ', '-')}" + url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}") for entry in system.get("files") or []: for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): @@ -3338,7 +3701,7 @@ def file_lookup_records( continue entries = profile.get("files") or [] for entry, anchor in zip(entries, emulator_file_anchors(entries)): - url = f"emulators/{emulator_card_page(key, anchor)}/#{anchor}" + url = _site_path(f"emulators/{emulator_card_page(key, anchor)}.md#{anchor}") members = [(str(entry.get("name", "")), entry)] + [ (str(member["name"]), member) for member in entry.get("contents") or [] if isinstance(member, dict) and member.get("name") @@ -3745,7 +4108,7 @@ def generate_provenance_page( f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}", f"Dumps {label} catalogues and the collection lacks, by DAT. Part of " f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).", - [], blocks, moved, + [], blocks, moved, parent=("Dump provenance", "/provenance/"), ) lines.extend([ f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: " @@ -6092,12 +6455,27 @@ def _declared_hashes(config: dict, detail: dict) -> dict[str, list[str]]: return {} +def short_hash(value: str) -> str: + """A hash in a table cell: its first 12 characters, the whole value in + the title. Full MD5 and SHA1 values pushed the gaps tables past the + column on every screen.""" + value = str(value) + if len(value) <= 12: + return _code(value) + return f'{html.escape(value[:12])}' + + def _hash_cell(hashes: dict[str, list[str]]) -> str: return "
".join( - f"{algo} " + ", ".join(_code(v) for v in values) for algo, values in hashes.items() + f"{algo} " + ", ".join(short_hash(v) for v in values) for algo, values in hashes.items() ) +def _reason_cell(text: str) -> str: + """A reason in a gaps table, in the class that keeps its column readable.""" + return f'{text}' if text else "-" + + def _copy_cell(path: str, algos: list[str], db: dict | None) -> str: """The file a path holds, with its hashes in the algorithms declared.""" if not path: @@ -6208,7 +6586,7 @@ def _render_problem_files( plats = ", ".join(sorted(set(f["platforms"]))) declared = f["declared"] lines.append( - f"| {_code(f['name'])} | {plats} | {f['reason']} " + f"| {_code(f['name'])} | {plats} | {_reason_cell(f['reason'])} " f"| {_hash_cell(declared) or '-'} " f"| {_copy_cell(f['path'], list(declared) or ['md5'], db)} |" ) @@ -6230,7 +6608,7 @@ def _render_problem_files( plats = ", ".join(sorted(set(f["platforms"]))) held = _held_accepted(f["discrepancy"], db) cell = f"[{_code(held)}]({_fetch_url(held)})" if held else "-" - lines.append(f"| `{fname}` | {plats} | {f['discrepancy']} | {cell} |") + lines.append(f"| `{fname}` | {plats} | {_reason_cell(f['discrepancy'])} | {cell} |") lines.append("") if settled: @@ -6752,6 +7130,7 @@ def generate_gap_analysis( "the collection does not hold. Part of " "[What the packs do not hold](gaps.md#files-to-source).", head, [[row] for row in rows], moved, + parent=("What the packs do not hold", "/gaps/"), ) lines.extend([f"Listed on {where}.", ""]) if bundled_rows: @@ -6953,7 +7332,8 @@ def _render_xref_by_platform( ) if moved is not None: moved[stem] = "\n".join([ - "---", "level: reference", "---", "", + "---", "level: reference", + *_parent_front_matter(("Cores by platform", "/cross-reference/")), "---", "", f"# Cores on {display}", "", f"{_count(count, 'core')}{_core_scope_note(config, unique)} that " @@ -7869,7 +8249,7 @@ def _generate(args: argparse.Namespace) -> None: write_single_item( public / "api" / "v1" / "systems" / f"{slug}.json", "systems", system_export_item(slug, mfr, consoles), - str(db.get("generated_at") or _timestamp()), + str(db.get("generated_at") or _timestamp()), _source_commit(), ) clash = split_system_stems() & {_slug(m) for m in manufacturers} if clash: @@ -7936,18 +8316,11 @@ def _generate(args: argparse.Namespace) -> None: # The envelope and item of emulators.json, one item long: the # profile validates against the published schema, which an "id" # beside its fields broke on all 530, and schema_version is there. - write_if_changed( - str(public / "api" / "v1" / "emulators" / f"{name}.json"), - json.dumps( - { - "schema_version": 1, - "generated_at": str(db.get("generated_at") or _timestamp()), - "kind": "emulators", - "count": 1, - "items": [{"id": name, "profile": profile}], - }, - indent=2, sort_keys=True, ensure_ascii=False, default=str, - ) + "\n", + # The commit too, as every other envelope: 625 files lacked it. + write_single_item( + public / "api" / "v1" / "emulators" / f"{name}.json", "emulators", + {"id": name, "profile": profile}, + str(db.get("generated_at") or _timestamp()), _source_commit(), ) pages = emulator_pages( name, profile, db, consumers, suppl_names, data_registry, diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 83989941..a4a873c1 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -685,14 +685,14 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase): records = generate_site.file_lookup_records(db, coverages, profiles, locations, members) found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records} self.assertEqual(found[("collection", "", "md5", md5)]["url"], - "systems/sony/#sha1-" + "ab" * 6) + "/retrobios/systems/sony/#sha1-" + "ab" * 6) self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found) self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin") self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"], - "platforms/batocera/#sony-playstation") + "/retrobios/platforms/batocera/#sony-playstation") self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found) self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"], - "emulators/duckstation/#file-scph5501-bin-490f666e") + "/retrobios/emulators/duckstation/#file-scph5501-bin-490f666e") self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom") self.assertNotIn(("emulator", "an-alias", "md5", md5), found) @@ -724,7 +724,7 @@ class AFileHasAnAddressByContent(unittest.TestCase): }} self.assertEqual( generate_site.file_location_items(db), - [{"sha1": "ab" * 20, "url": "systems/roland/#sha1-" + "ab" * 6}], + [{"sha1": "ab" * 20, "url": "/retrobios/systems/roland/#sha1-" + "ab" * 6}], ) self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page()) script = (Path(__file__).resolve().parents[1] / "website" / "src" / "scripts" @@ -865,7 +865,8 @@ class ADataDirectoryFileCanBeLookedUp(unittest.TestCase): members = data_directory_members(Path(tmp)) items = file_location_items( {"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}}, members) - self.assertEqual(items, [{"sha1": "c" * 40, "url": "emulators/dolphin/#sha1-cccccccccccc"}]) + self.assertEqual(items, [{"sha1": "c" * 40, + "url": "/retrobios/emulators/dolphin/#sha1-cccccccccccc"}]) class ACardSaysWhyAFileIsLeftOut(unittest.TestCase): diff --git a/tests/test_site_platform_hashes.py b/tests/test_site_platform_hashes.py index 15e5df32..7d131796 100644 --- a/tests/test_site_platform_hashes.py +++ b/tests/test_site_platform_hashes.py @@ -69,8 +69,9 @@ class AnUntestedRowNamesBothHashes(unittest.TestCase): db = {"files": {"f" * 40: {"path": path, "sha256": held}}} table = "\n".join(gs._render_problem_files([problem], db=db)) row = next(line for line in table.splitlines() if "dsifirmware" in line) - self.assertIn(f"sha256 `{declared}`", row) - self.assertIn(f"sha256 `{held}`", row) + # Cut to 12 characters in the cell, the whole value in its title. + self.assertIn(f'sha256 {declared[:12]}', row) + self.assertIn(f'sha256 {held[:12]}', row) self.assertIn(f"`{path}`", row) diff --git a/tests/test_site_r2_gate.py b/tests/test_site_r2_gate.py index 3a08cfa8..d3bb44a1 100644 --- a/tests/test_site_r2_gate.py +++ b/tests/test_site_r2_gate.py @@ -261,7 +261,7 @@ class TheRules(unittest.TestCase): {"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}}, {"dolphin-sys": members}) last = f"{generate_site.DATA_TREE_ROWS_PER_PAGE:040x}" - self.assertIn({"sha1": last, "url": f"emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items) + self.assertIn({"sha1": last, "url": f"/retrobios/emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items) def test_no_float_rule_for_right_aligned_images(self): css = (REPO_ROOT / "website" / "src" / "styles" / "retrobios.css").read_text(encoding="utf-8") diff --git a/tests/test_site_r3_reference.py b/tests/test_site_r3_reference.py new file mode 100644 index 00000000..fad13ffe --- /dev/null +++ b/tests/test_site_r3_reference.py @@ -0,0 +1,469 @@ +"""Reference pages and data (round 3, package 8). + +The CSV and SQLite dropped zipped_file, so a check built from them judged +106 good Batocera files wrong. The manifest a pack installs was nowhere on +the site. Contracts failed their own schema, 716 documents lacked `commit`, +no endpoint answered a system id or a buildbot core name, lookup URLs +resolved to 404s, the data's licence went unstated, platform overviews +named emulators as unlinked ids, gaps tables overflowed with full hashes +and a source link went into a repository that does not hold the file. +Tests that read the built site skip when it has not been generated. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import re +import shutil +import sqlite3 +import sys +import tempfile +import unittest +import urllib.parse +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import check_platform_folder # noqa: E402 +import generate_site # noqa: E402 +import sitesources # noqa: E402 + +PUBLIC = REPO_ROOT / "website" / "public" +API = PUBLIC / "api" / "v1" +SITE = "https://abdess.github.io/retrobios/" +TMP = REPO_ROOT / "tmp" + + +def _built(path: Path) -> Path: + if not path.exists(): + raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated") + return path + + +def _verify_status(entry: dict, path: Path, mode: str) -> str: + """What verify.py says of one entry, the platform's native logic.""" + from verify import verify_entry_existence, verify_entry_md5, verify_entry_sha1 + + local = str(path) if path.is_file() else None + if mode == "existence": + return verify_entry_existence(entry, local)["status"] + if mode == "sha1": + return verify_entry_sha1(entry, local)["status"] + return verify_entry_md5(entry, local)["status"] + + +def _as_snippet(status: str) -> str: + return {"ok": "ok", "untested": "wrong", "missing": "missing"}[status] + + +class TheExportsKeepWhatACheckNeeds(unittest.TestCase): + """8.1: zipped_file and the Recalbox flags reach the CSV and SQLite.""" + + COVERAGE = {"batocera": { + "platform": "Batocera", "details": [], + "config": {"platform": "Batocera", "verification_mode": "md5", + "install": {"detect": []}, "logo": "x.svg", "_key": "batocera", + "systems": {"bbc": {"files": [ + {"name": "bbcb.zip", "destination": "bbcb.zip", + "md5": "521a197d", "zipped_file": "basic1.rom"}, + {"name": "n.rom", "destination": "n.rom", "md5": "ab", + "hash_match_mandatory": True, "mandatory_declared": False, + "shared_group": "group-x"}, + ]}}}, + }} + + def test_rows_carry_the_member_and_the_flags(self): + _items, rows = generate_site._platform_export_rows(self.COVERAGE) + zipped, flagged = rows + self.assertEqual(zipped["zipped_file"], "basic1.rom") + self.assertIs(flagged["hash_match_mandatory"], True) + self.assertIs(flagged["mandatory_declared"], False) + self.assertEqual(flagged["shared_group"], "group-x") + self.assertEqual(flagged["zipped_file"], "") + + def test_the_csv_header_and_the_table_name_them(self): + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('"collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS,', source) + for field in generate_site.ENTRY_EXPORT_FIELDS: + self.assertIn(f":{field}", source) + for field in ("zipped_file", "hash_match_mandatory", "mandatory_declared", "curated"): + self.assertIn(field, generate_site.ENTRY_EXPORT_FIELDS) + + def test_the_built_csv_has_the_column(self): + text = _built(PUBLIC / "downloads" / "platform-files.csv").read_text(encoding="utf-8") + rows = list(csv.DictReader(io.StringIO(text))) + batocera = [row for row in rows if row["platform_id"] == "batocera"] + self.assertGreater(sum(1 for row in batocera if row["zipped_file"]), 300) + with sqlite3.connect(_built(PUBLIC / "downloads" / "retrobios.sqlite")) as db: + count = db.execute("SELECT count(*) FROM platform_files " + "WHERE platform_id = 'batocera' AND zipped_file != ''").fetchone()[0] + license_row = db.execute("SELECT value FROM metadata WHERE key = 'license'").fetchone() + self.assertGreater(count, 300) + self.assertEqual(license_row[0], generate_site.DATA_LICENSE) + + +class AFolderCheckAgreesWithTheVerifier(unittest.TestCase): + """8.1 and 8.3: the published example gives verify.py's verdicts.""" + + def _folder(self, tmp: Path) -> tuple[dict, Path]: + member = b"basic rom bytes" + root = tmp / "bios" + root.mkdir() + with zipfile.ZipFile(root / "bbcb.zip", "w") as archive: + archive.writestr("BASIC1.ROM", member) + (root / "plain.bin").write_bytes(b"plain") + (root / "wrong.bin").write_bytes(b"not it") + (root / "nohash.bin").write_bytes(b"x") + md5 = lambda data: hashlib.md5(data).hexdigest() # noqa: E731 + contract = {"verification_mode": "md5", "systems": {"s": {"files": [ + {"name": "bbcb.zip", "destination": "bbcb.zip", "md5": md5(member), + "zipped_file": "basic1.rom"}, + {"name": "bbcb.zip", "destination": "bbcb.zip", "md5": "0" * 32, + "zipped_file": "basic1.rom"}, + {"name": "plain.bin", "destination": "plain.bin", "md5": md5(b"plain")[:29]}, + {"name": "wrong.bin", "destination": "wrong.bin", "md5": md5(b"plain")}, + {"name": "nohash.bin", "destination": "nohash.bin"}, + {"name": "gone.bin", "destination": "gone.bin", "md5": "1" * 32}, + ]}}} + return contract, root + + def test_each_rule_on_a_sample_folder(self): + with tempfile.TemporaryDirectory(dir=TMP) as tmp: + contract, root = self._folder(Path(tmp)) + platforms = Path(tmp) / "platforms" + platforms.mkdir() + (platforms / "sample.json").write_text( + json.dumps({"items": [{"contract": contract}]}), encoding="utf-8") + found = check_platform_folder.check("sample", root, platforms.as_uri() + "/") + expected = [ + (entry["destination"], _as_snippet(_verify_status(entry, root / entry["destination"], "md5"))) + for entry in contract["systems"]["s"]["files"] + ] + self.assertEqual(found, expected) + self.assertEqual([v for _d, v in found], ["ok", "wrong", "ok", "wrong", "ok", "missing"]) + + def test_batocera_from_the_csv_matches_the_verifier(self): + """A check driven by platform-files.csv, with zipped_file, gives the + verifier's verdict on every Batocera entry of a folder filled from + the collection.""" + text = _built(PUBLIC / "downloads" / "platform-files.csv").read_text(encoding="utf-8") + rows = [row for row in csv.DictReader(io.StringIO(text)) if row["platform_id"] == "batocera"] + with tempfile.TemporaryDirectory(dir=TMP) as tmp: + root = Path(tmp) + for row in rows: + source = REPO_ROOT / row["collection_path"] if row["collection_path"] else None + target = root / row["destination"] + if source and source.is_file() and not target.exists(): + target.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, target) + except OSError: + shutil.copyfile(source, target) + held = [row for row in rows if (root / row["destination"]).is_file()] + if len(held) < 100: + raise unittest.SkipTest("the collection files are not in this checkout") + # The verifier reads the platform's own entries, the check the CSV. + contract = json.loads(_built(API / "platforms" / "batocera.json").read_text())[ + "items"][0]["contract"] + declared = [entry for _sys, system in sorted(contract["systems"].items()) + for entry in system.get("files") or []] + self.assertEqual([e["name"] for e in declared], [row["name"] for row in rows]) + mismatched = [] + zipped_ok = 0 + for row, platform_entry in zip(rows, declared): + entry = {key: value for key, value in row.items() if value} + path = root / row["destination"] + ours = "ok" if check_platform_folder.accepts(entry, path, "md5") else ( + "wrong" if path.is_file() else "missing") + theirs = _as_snippet(_verify_status(platform_entry, path, "md5")) + if ours != theirs: + mismatched.append((row["destination"], row.get("zipped_file"), ours, theirs)) + zipped_ok += ours == "ok" and bool(row["zipped_file"]) + self.assertEqual(mismatched, []) + self.assertGreater(zipped_ok, 100) + + def test_the_data_page_shows_the_script_itself(self): + script = (REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") + page = generate_site.generate_data_page([]) + self.assertIn("```python\n" + script.rstrip("\n") + "\n```", page) + self.assertLess(len(script.splitlines()), 70) + self.assertIn("## Check a folder for one platform", page) + + +class TheContractsValidate(unittest.TestCase): + """8.4: registry keys leave the contract, every $id is its served URL.""" + + def test_every_contract_validates_against_the_platform_schema(self): + from jsonschema import Draft202012Validator # noqa: PLC0415 + + from common import list_registered_platforms, load_platform_config # noqa: PLC0415 + + schema = json.loads((REPO_ROOT / "schemas" / "platform.schema.json").read_text()) + validator = Draft202012Validator(schema) + platforms = str(REPO_ROOT / "platforms") + for name in list_registered_platforms(platforms, include_archived=True): + config = load_platform_config(name, platforms) + contract, registry = generate_site._split_contract(config) + with self.subTest(platform=name): + errors = [error.message for error in validator.iter_errors(contract)] + self.assertEqual(errors, []) + self.assertIn("install", registry) + self.assertNotIn("_key", registry) + self.assertEqual(set(contract) | set(registry) | {"_key"}, set(config) | {"_key"}) + + def test_every_schema_id_is_the_address_it_is_served_at(self): + for path in sorted((REPO_ROOT / "schemas").glob("*.schema.json")): + schema = json.loads(path.read_text()) + with self.subTest(schema=path.name): + self.assertEqual(schema.get("$id"), f"{SITE}api/v1/schemas/{path.name}") + + def test_every_schema_is_published(self): + published = {p.name for p in _built(API / "schemas").glob("*.json")} + self.assertEqual(published, {p.name for p in (REPO_ROOT / "schemas").glob("*.schema.json")}) + + def test_the_built_api_validates(self): + from jsonschema import Draft202012Validator # noqa: PLC0415 + + schemas = {p.name: json.loads(p.read_text()) for p in (REPO_ROOT / "schemas").glob("*.json")} + envelope = Draft202012Validator(schemas["site-api-envelope.schema.json"]) + platform = Draft202012Validator(schemas["platform.schema.json"]) + document = json.loads(_built(API / "platforms.json").read_text()) + self.assertEqual(list(envelope.iter_errors(document)), []) + for item in document["items"]: + with self.subTest(platform=item["id"]): + self.assertEqual([e.message for e in platform.iter_errors(item["contract"])], []) + for name in ("systems.json", "cores.json", "index.json", "catalog.json"): + with self.subTest(document=name): + self.assertEqual( + [e.message for e in envelope.iter_errors(json.loads((API / name).read_text()))], []) + + +class EveryEnvelopeCarriesTheCommit(unittest.TestCase): + """8.5: emulators/ and systems/ lacked `commit` (716 documents).""" + + def test_the_writers_pass_it(self): + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('str(db.get("generated_at") or _timestamp()), _source_commit(),\n )\n' + " clash", source) + block = source[source.index("public_profiles = {"):source.index("pages = emulator_pages(")] + self.assertIn("write_single_item(", block) + self.assertIn("_source_commit()", block) + + def test_every_built_envelope_has_it(self): + _built(API / "catalog.json") + if not generate_site._source_commit(): + raise unittest.SkipTest("not a git checkout") + lacking = [ + path.relative_to(API).as_posix() for path in sorted(API.rglob("*.json")) + if path.parent.name not in ("schemas", "manifests") + and path.name not in ("database.json", "stats.json", "release.json") + and "commit" not in json.loads(path.read_text()) + ] + self.assertEqual(lacking, []) + + +class ManifestsArePublished(unittest.TestCase): + """8.2: the manifest a pack installs, byte for byte, in the catalog.""" + + def test_each_manifest_is_the_repository_file(self): + catalog = json.loads(_built(API / "catalog.json").read_text()) + listed = {item["url"]: item for item in catalog["items"]} + for source in sorted((REPO_ROOT / "install").glob("*.json")): + with self.subTest(manifest=source.name): + item = listed[f"manifests/{source.name}"] + self.assertEqual(item["sha256"], hashlib.sha256(source.read_bytes()).hexdigest()) + self.assertEqual(item["schema"], "schemas/install-manifest.schema.json") + self.assertIn("release.json", listed) + self.assertEqual(catalog["license"], generate_site.DATA_LICENSE) + + def test_the_data_page_explains_the_fields_from_the_schema(self): + page = generate_site.generate_data_page([]) + schema = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text()) + for field in ("dest", "repo_path", "release_asset", "cores"): + description = schema["$defs"]["file"]["properties"][field]["description"] + self.assertIn(f"| `{field}` | {generate_site._escape_tags(description)} |", page) + self.assertIn("## Platform manifests", page) + self.assertIn("**Licence:**", page) + self.assertIn("## Hosting and limits", page) + # The endpoints first, the DAT below them. + self.assertLess(page.index("## Stable JSON endpoints"), page.index("The DAT is Logiqx XML")) + + +class AnIdLeadsToItsData(unittest.TestCase): + """8.6: a system by its id, a core by the name the buildbot gives it.""" + + def test_a_buildbot_name_designates_its_profile(self): + profiles = { + "beetle_psx": {"type": "libretro", "cores": ["mednafen_psx", "mednafen_psx_hw"]}, + "pcsx2": {"type": "standalone"}, + "lrps2": {"type": "libretro", "cores": ["pcsx2"]}, + "mednafen_psx_alias": {"type": "alias", "alias_of": "beetle_psx"}, + "probe": {"type": "test", "cores": ["probe_core"]}, + } + items = generate_site.core_index_items(profiles) + by_name: dict[str, list[tuple[str, bool]]] = {} + for item in items: + by_name.setdefault(item["name"], []).append((item["profile"], item["preferred"])) + self.assertEqual(by_name["mednafen_psx"], [("beetle_psx", True)]) + self.assertEqual(by_name["pcsx2"], [("pcsx2", True), ("lrps2", False)]) + self.assertEqual(by_name["mednafen_psx_alias"], [("beetle_psx", True)]) + self.assertNotIn("probe_core", by_name) + + def test_the_built_index_resolves_ids_to_sections(self): + items = {item["id"]: item for item in + json.loads(_built(API / "systems.json").read_text())["items"]} + color = items["wonderswan-color"] + self.assertEqual(color["name"], "Bandai WonderSwan Color") + self.assertTrue(color["url"].startswith("/retrobios/systems/bandai/"), color["url"]) + self.assertTrue(items["sony-playstation"]["sha1"]) + cores = json.loads((API / "cores.json").read_text())["items"] + self.assertIn({"name": "mednafen_psx", "profile": "beetle_psx", "preferred": True}, cores) + + +class LookupUrlsResolve(unittest.TestCase): + """8.7: a record's url, joined to the shard holding it, is a page.""" + + def test_site_absolute(self): + db = {"files": {"ab" * 20: {"path": "bios/Sony/PlayStation/scph5501.bin"}}} + [item] = generate_site.file_location_items(db) + self.assertTrue(item["url"].startswith("/retrobios/systems/"), item["url"]) + shard = SITE + "api/v1/lookup/ab.json" + self.assertTrue(urllib.parse.urljoin(shard, item["url"]).startswith(SITE + "systems/")) + + def test_built_records_land_on_built_pages(self): + shard = json.loads(_built(API / "lookup" / "05.json").read_text()) + dist = REPO_ROOT / "website" / "dist" + for record in shard["items"][:200]: + with self.subTest(url=record["url"]): + self.assertTrue(record["url"].startswith("/retrobios/")) + if dist.is_dir(): + page = urllib.parse.unquote(record["url"].split("#")[0].removeprefix("/retrobios/")) + self.assertTrue((dist / page / "index.html").is_file(), page) + + +class PlatformPagesLinkTheirSystemsAndEmulators(unittest.TestCase): + """8.9: the overview named emulators as ids, none of 1,463 linked.""" + + def test_an_emulator_is_named_and_linked(self): + link = generate_site._emulator_named_link("beetle_psx", "../") + self.assertRegex(link, r"^\[[^\]]+\]\(\.\./emulators/beetle_psx\.md\)$") + self.assertNotEqual(link, "[beetle_psx](../emulators/beetle_psx.md)") + + def test_the_overview_column_uses_it(self): + source = Path(generate_site.__file__).read_text(encoding="utf-8") + block = source[source.index("### Systems overview"):source.index("_render_platform_systems(by_system")] + self.assertIn("_emulator_named_link(e, \"../\")", block) + self.assertNotIn('", ".join(sys_emus[:3])', block) + + def test_a_moved_list_opens_on_its_console_and_names_its_parent(self): + # Past PLATFORM_SECTION_BUDGET, so the list leaves for a page of its own. + files = [{"name": f"{'f' * 200}{i}.bin", "status": "ok"} for i in range(1200)] + moved: dict[str, str] = {} + generate_site._render_platform_systems( + {"sony-playstation": files}, {}, {}, ("retroarch", "RetroArch"), moved, {}) + self.assertTrue(moved) + page = next(iter(moved.values())) + self.assertTrue(page.startswith('---\nparent:\n label: "RetroArch"\n' + ' link: "/platforms/retroarch/"\n---\n'), page[:120]) + self.assertRegex(page, r"RetroArch's list for \[Sony PlayStation\]\(\.\./systems/") + + def test_part_pages_of_gaps_and_cross_reference_name_their_parent(self): + moved: dict[str, str] = {} + generate_site._list_pages("gaps-x", "T", "intro", [], [["| a |"]], moved, + parent=("What the packs do not hold", "/gaps/")) + self.assertIn('parent:\n label: "What the packs do not hold"\n link: "/gaps/"', moved["gaps-x"]) + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('_parent_front_matter(("Cores by platform", "/cross-reference/"))', source) + self.assertIn('parent=("Dump provenance", "/provenance/")', source) + + +class TablesFitTheirColumn(unittest.TestCase): + """8.12 and 8.13: a hash in a cell is 12 characters, the rest in its title.""" + + def test_the_downloads_table(self): + item = {"title": "SQLite snapshot", "url": "downloads/retrobios.sqlite", + "media_type": "application/vnd.sqlite3", "bytes": 10, "sha256": "a" * 64} + page = generate_site.generate_data_page([item]) + row = next(line for line in page.splitlines() if "SQLite snapshot" in line and "|" in line) + self.assertIn(f'{"a" * 12}', row) + self.assertNotIn("a" * 13, row.replace(f'title="{"a" * 64}"', "")) + + def test_the_gaps_tables(self): + problem = {"platform": "Batocera", "name": "x.bin", "status": "untested", "required": True, + "reason": "expected md5 differs", "declared": {"md5": ["b" * 32]}, "path": ""} + table = "\n".join(generate_site._render_problem_files([problem], db={"files": {}})) + row = next(line for line in table.splitlines() if "x.bin" in line) + self.assertIn(f'{"b" * 12}', row) + self.assertIn('expected md5 differs', row) + + +class SourceLinksGoWhereTheFileIs(unittest.TestCase): + """8.14: trident cited a Panda3DS path and linked it into 3dsTrident.""" + + PROFILE = { + "emulator": "trident", "type": "libretro", + "source": "https://github.com/DanAlexMorton/3dsTrident", + "upstream": "https://github.com/wheremyfoodat/Panda3DS", + "source_commit": "b" * 40, + } + + def tearDown(self): + sitesources._tree_paths.cache_clear() + + def _render(self, tree: frozenset[str] | None) -> str: + original = sitesources._tree_paths + sitesources._tree_paths = lambda repo, pin: tree # type: ignore[assignment] + try: + return sitesources._source_ref_markdown( + self.PROFILE, "src/emulator.cpp:234; src/core/loader/ncch.cpp:151-164") + finally: + sitesources._tree_paths = original + + def test_a_path_the_tree_lacks_is_not_linked(self): + text = self._render(frozenset({"src/emulator.cpp"})) + self.assertIn("[`src/emulator.cpp:234`](https://github.com/DanAlexMorton/3dsTrident/", text) + self.assertIn("`src/core/loader/ncch.cpp:151-164`", text) + self.assertNotIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader", text) + + def test_without_a_cached_tree_the_declarations_decide(self): + text = self._render(None) + self.assertIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader/ncch.cpp", text) + + def test_every_rendered_link_is_in_its_cached_tree(self): + """Offline: each link the pages render points at a path the cached + tree of its repository holds.""" + if not sitesources.TREE_CACHE.is_dir(): + raise unittest.SkipTest("no cached tree listings") + from common import load_emulator_profiles # noqa: PLC0415 + + link = re.compile(r"\(https://github\.com/([^/]+/[^/]+)/blob/([0-9a-f]{40})/([^#)]+)") + checked, wrong = 0, [] + profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"), skip_aliases=False) + for key, profile in sorted(profiles.items()): + for entry in profile.get("files") or []: + if not isinstance(entry, dict) or not entry.get("source_ref"): + continue + for slug, pin, path in link.findall( + sitesources._source_ref_markdown(profile, entry["source_ref"])): + import upstream # noqa: PLC0415 + + repo = upstream.parse_repo(f"https://github.com/{slug}") + held = sitesources.tree_holds(repo, pin, urllib.parse.unquote(path)) + if held is None: + continue + checked += 1 + if not held: + wrong.append((key, slug, path)) + if not checked: + raise unittest.SkipTest("no rendered link has a cached tree") + self.assertEqual(wrong, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_render.py b/tests/test_site_render.py index efbefc53..dabf9d9a 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -224,7 +224,7 @@ class AnAliasTreeIsListedOnItsParentsPage(unittest.TestCase): members = {"freeintv-overlays": {"a.png": {"sha1": "02" * 20}}} items = generate_site.file_location_items({"files": {}}, self.PROFILES, members) self.assertEqual(items, [{"sha1": "02" * 20, - "url": "emulators/freeintv/#sha1-020202020202"}]) + "url": "/retrobios/emulators/freeintv/#sha1-020202020202"}]) def test_the_parent_page_lists_the_tree(self): import generate_site diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index e4deae8a..0d2df849 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -54,7 +54,7 @@ Every module in `tests/`, by the area it guards. | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_offline`, `test_install_selection` | | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture From 94d18bd7e5f585dd1a354272c8e3d76c05a7f01e Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:54:34 +0200 Subject: [PATCH 5/5] feat: name the system in each lookup record --- schemas/site-api-envelope.schema.json | 4 +++ scripts/generate_site.py | 42 +++++++++++++++++++++++++-- tests/test_site_r3_reference.py | 27 +++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index da48f568..5372333e 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -407,6 +407,10 @@ }, "owner": { "type": "string" + }, + "system_name": { + "type": "string", + "description": "The name the site gives the system the record belongs to (Sony PlayStation): the console a collection file is filed under, the system a platform lists it in, the system of an emulator's entry. Absent when none is known." } } } diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 4319a7f8..079387a7 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1792,7 +1792,8 @@ def generate_data_page(exports: list[dict]) -> str: "is the first two characters of the hash in lowercase, holds every SHA1, " "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` " "(the collection, a data directory, a platform's list or an emulator " - "profile), the file `name` and the `url` of the page that shows it. " + "profile), the file `name`, the `url` of the page that shows it and, " + "when one is known, the `system_name` the site gives its system. " "[Find a file](files/index.md) reads it.", "", "Two kinds of `url`:", @@ -3651,6 +3652,30 @@ def file_location_items( LOOKUP_PREFIX = 2 # hex characters naming a lookup shard: 256 files +def _lookup_system_name(sys_id: str) -> str | None: + """The system's name as every page prints it, None for an id the site + cannot name. The Find a file page reads it, and never builds one.""" + try: + return system_name(sys_id) + except KeyError: + return None + + +def _console_system_names(profiles: dict, coverages: dict) -> dict[tuple[str, str], str]: + """(manufacturer slug, console folder) -> the name of the system whose + own section that folder is: a collection file is named after the console + it is filed under. A folder no id owns, or that two names own, is left out. + """ + from siterender import _own_section, system_ids + + found: dict[tuple[str, str], set[str]] = {} + for sys_id in system_ids(profiles, [c["config"] for c in coverages.values()]): + own = _own_section(sys_id) + if own is not None: + found.setdefault((_slug(own[0]), own[1]), set()).add(system_name(sys_id)) + return {key: next(iter(names)) for key, names in found.items() if len(names) == 1} + + def file_lookup_records( db: dict, coverages: dict, @@ -3666,6 +3691,7 @@ def file_lookup_records( it made every visitor download: the lookup reads one shard instead. """ files = db.get("files", {}) + named = _console_system_names(profiles, coverages) member_names = { str(info.get("sha1", "")): member.rsplit("/", 1)[-1] for tree in (members or {}).values() for member, info in sorted(tree.items()) @@ -3680,28 +3706,37 @@ def file_lookup_records( "name": member_names.get(sha1, ""), "sha1": sha1, "url": item["url"]}) continue + parts = str(entry.get("path", "")).split("/") + held_by = named.get((_slug(parts[1]), parts[2])) if len(parts) > 3 else None for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): records.append({ "value": value, "algorithm": algorithm, "layer": "collection", "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"], + **({"system_name": held_by} if held_by else {}), }) for key, coverage in sorted(coverages.items()): for system_id, system in sorted((coverage["config"].get("systems") or {}).items()): url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}") + system_label = _lookup_system_name(system_id) for entry in system.get("files") or []: for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): records.append({ "value": value, "algorithm": algorithm, "layer": "platform", "owner": key, "name": str(entry.get("name", "")), "url": url, + **({"system_name": system_label} if system_label else {}), }) for key, profile in sorted(profiles.items()): if profile.get("type") in ("alias", "test"): continue entries = profile.get("files") or [] + own_systems = [str(s) for s in profile.get("systems") or []] for entry, anchor in zip(entries, emulator_file_anchors(entries)): url = _site_path(f"emulators/{emulator_card_page(key, anchor)}.md#{anchor}") + sys_id = str(entry.get("system") or "") or ( + own_systems[0] if len(own_systems) == 1 else "") + entry_label = _lookup_system_name(sys_id) if sys_id else None members = [(str(entry.get("name", "")), entry)] + [ (str(member["name"]), member) for member in entry.get("contents") or [] if isinstance(member, dict) and member.get("name") @@ -3712,6 +3747,7 @@ def file_lookup_records( records.append({ "value": value, "algorithm": algorithm, "layer": "emulator", "owner": key, "name": name, "url": url, + **({"system_name": entry_label} if entry_label else {}), }) unique = {json.dumps(r, sort_keys=True): r for r in records} return sorted(unique.values(), key=lambda r: ( @@ -7109,7 +7145,7 @@ def generate_gap_analysis( ) req = "yes" if m["required"] else "no" algorithm, _, value = m["identity"].partition(":") - expected = f"{algorithm} {_code(value)}" if value else "-" + expected = f"{algorithm} {short_hash(value)}" if value else "-" destination = _code(m["path"]) if m["path"] != m["name"] else "-" emulators = ", ".join( f"[{_escape_tags(display)}](emulators/{key}.md)" @@ -7167,7 +7203,7 @@ def generate_gap_analysis( ]) for u in sorted(all_unsourceable, key=lambda x: x["name"]): lines.append( - f"| `{u['name']}` | {u['emulator']} | {u['reason']} " + f"| `{u['name']}` | {u['emulator']} | {_reason_cell(u['reason'])} " f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} |" ) lines.append("") diff --git a/tests/test_site_r3_reference.py b/tests/test_site_r3_reference.py index fad13ffe..d631c7ba 100644 --- a/tests/test_site_r3_reference.py +++ b/tests/test_site_r3_reference.py @@ -336,6 +336,30 @@ class LookupUrlsResolve(unittest.TestCase): shard = SITE + "api/v1/lookup/ab.json" self.assertTrue(urllib.parse.urljoin(shard, item["url"]).startswith(SITE + "systems/")) + def test_each_record_names_its_system_as_the_pages_do(self): + """Find a file opens on "a BIOS file for Sony PlayStation": the name + comes from system_name, never built in the browser.""" + from siterender import system_name # noqa: PLC0415 + + db = {"files": {"ab" * 20: {"path": "bios/Sony/PlayStation/scph5501.bin", + "name": "scph5501.bin", "md5": "490f666e1afb15b7362b406ed1cea246"}}} + coverages = {"batocera": {"platform": "Batocera", "config": {"systems": { + "sony-playstation": {"files": [{"name": "scph5501.bin", "md5": "490f666e1afb15b7362b406ed1cea246"}]}}}}} + profiles = {"duckstation": {"systems": ["sony-playstation"], "files": [ + {"name": "scph5501.bin", "md5": "490f666e1afb15b7362b406ed1cea246"}]}} + records = generate_site.file_lookup_records( + db, coverages, profiles, generate_site.file_location_items(db)) + expected = system_name("sony-playstation") + by_layer = {r["layer"]: r for r in records if r["algorithm"] == "md5"} + for layer in ("collection", "platform", "emulator"): + with self.subTest(layer=layer): + self.assertEqual(by_layer[layer]["system_name"], expected) + + def test_built_shards_carry_the_name(self): + shard = json.loads(_built(API / "lookup" / "05.json").read_text()) + record = next(r for r in shard["items"] if r["value"].startswith("0555c6fae890")) + self.assertEqual(record["system_name"], "Sony PlayStation") + def test_built_records_land_on_built_pages(self): shard = json.loads(_built(API / "lookup" / "05.json").read_text()) dist = REPO_ROOT / "website" / "dist" @@ -401,6 +425,9 @@ class TablesFitTheirColumn(unittest.TestCase): row = next(line for line in table.splitlines() if "x.bin" in line) self.assertIn(f'{"b" * 12}', row) self.assertIn('expected md5 differs', row) + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('expected = f"{algorithm} {short_hash(value)}"', source) + self.assertIn("{_reason_cell(u['reason'])}", source) class SourceLinksGoWhereTheFileIs(unittest.TestCase):