From 2b64eaab9b50c4fd2570cc4fa5bf5d91eacc5112 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:13 +0200 Subject: [PATCH] feat: publish manifests, id indexes and licence The CSV and SQLite carry zipped_file and the Recalbox flags, contracts leave registry keys in a registry field and validate, every envelope carries commit, lookup URLs are site-absolute, systems.json and cores.json answer an id, the Data page shows the manifest fields and a folder check, and platform overviews link emulators by name. --- scripts/generate_site.py | 479 +++++++++++++++++++++++++---- tests/test_site_exports.py | 11 +- tests/test_site_platform_hashes.py | 5 +- tests/test_site_r2_gate.py | 2 +- tests/test_site_r3_reference.py | 469 ++++++++++++++++++++++++++++ tests/test_site_render.py | 2 +- wiki/testing-guide.md | 2 +- 7 files changed, 907 insertions(+), 63 deletions(-) create mode 100644 tests/test_site_r3_reference.py diff --git a/scripts/generate_site.py b/scripts/generate_site.py index df4b4a93..4319a7f8 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -106,6 +106,9 @@ STARLIGHT_CONTENT = "src/content/docs" SITE_NAME = "RetroBIOS" REPO_URL = "https://github.com/Abdess/retrobios" RELEASE_URL = f"{REPO_URL}/releases/latest" +# What LICENSE grants over the published data: its MIT terms name the +# metadata. The BIOS files themselves are under NOTICE, never this. +DATA_LICENSE = "MIT (LICENSE: scripts, tooling, schemas, documentation and metadata)" SITE_URL = "https://abdess.github.io/retrobios/" GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads"] # Generated directories of files served as they are, not pages. @@ -806,6 +809,36 @@ def _sha1_by_path(db: dict | None) -> dict[str, str]: return {str(e.get("path", "")): sha1 for sha1, e in ((db or {}).get("files") or {}).items()} +@functools.lru_cache(maxsize=1) +def _contract_keys() -> frozenset[str]: + """The top-level keys platform.schema.json allows in a platform's list.""" + schema = json.loads((_REPO_ROOT / "schemas" / "platform.schema.json").read_text(encoding="utf-8")) + return frozenset(schema.get("properties") or {}) + + +def _split_contract(config: dict) -> tuple[dict, dict]: + """The list, valid against platform.schema.json, and the registry entry. + + load_platform_config merges the registry into the list: `install`, + `logo`, `scraper` and `_key` made all 18 contracts fail the published + schema. They stay published, beside the contract. + """ + allowed = _contract_keys() + contract = {key: value for key, value in config.items() if key in allowed} + registry = {key: value for key, value in config.items() + if key not in allowed and key != "_key"} + return contract, registry + + +# Per-entry fields a folder check needs beyond the hashes: an md5 is a ZIP +# member's when zipped_file is set, and a CSV without it judged 106 good +# Batocera files wrong. +ENTRY_EXPORT_FIELDS = ( + "zipped_file", "hash_match_mandatory", "mandatory_declared", "curated", + "shared_group", "mirror_of", "zip_digest", +) + + def _platform_export_rows( coverages: dict, db: dict | None = None, ) -> tuple[list[dict], list[dict]]: @@ -816,6 +849,7 @@ def _platform_export_rows( for key, coverage in sorted(coverages.items()): config = coverage["config"] outcome = _verify_outcomes(coverage, sha1_by_path) + contract, registry = _split_contract(config) items.append({ "id": key, "name": coverage["platform"], @@ -826,7 +860,8 @@ def _platform_export_rows( "core_present", "core_missing", "core_unsourceable", ) }, - "contract": config, + "contract": contract, + "registry": registry, }) for system_id, system in sorted(config.get("systems", {}).items()): for entry in system.get("files", []) or []: @@ -846,11 +881,20 @@ def _platform_export_rows( "sha256": _json_text(entry.get("sha256")), "md5": _json_text(entry.get("md5")), "crc32": _json_text(entry.get("crc32")), + **{field: _export_flag(entry.get(field)) for field in ENTRY_EXPORT_FIELDS}, **outcome(system_id, entry.get("name", "")), }) return items, file_rows +def _export_flag(value: object) -> object: + """A per-entry field as CSV and SQLite hold it: booleans stay booleans, + an absent field is empty.""" + if value is None: + return "" + return value if isinstance(value, bool) else str(value) + + def _emulator_export_items(profiles: dict) -> list[dict]: return [ {"id": key, "profile": profile} @@ -1089,7 +1133,9 @@ def _write_sqlite_export( destination TEXT NOT NULL, required INTEGER NOT NULL, region TEXT, variant_group TEXT, size INTEGER, sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT, - status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT + status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT, + zipped_file TEXT, hash_match_mandatory INTEGER, mandatory_declared INTEGER, + curated INTEGER, shared_group TEXT, mirror_of TEXT, zip_digest TEXT ); CREATE TABLE emulators ( id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT, @@ -1134,6 +1180,7 @@ def _write_sqlite_export( **({"commit": commit} if commit else {}), "source": REPO_URL, "scope": "metadata only, no BIOS or firmware payload bytes", + "license": DATA_LICENSE, } connection.executemany( "INSERT INTO metadata(key, value) VALUES (?, ?)", @@ -1168,7 +1215,8 @@ def _write_sqlite_export( "INSERT INTO platform_files VALUES " "(:platform_id, :system, :name, :destination, :required, :region, " ":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, " - ":collection_path, :resolved_sha1)", + ":collection_path, :resolved_sha1, :zipped_file, :hash_match_mandatory, " + ":mandatory_declared, :curated, :shared_group, :mirror_of, :zip_digest)", platform_files, ) for item in emulator_items: @@ -1359,13 +1407,21 @@ def generate_data_exports( json.dumps(db, ensure_ascii=False, indent=2) + "\n", ) - schema_names = ( - "database.schema.json", "emulator.schema.json", "platform.schema.json", - "site-api-envelope.schema.json", "stats.schema.json", - ) - for schema_name in schema_names: - shutil.copy2(Path("schemas") / schema_name, schemas_dest / schema_name) - _record(schemas_dest / schema_name) + # Every schema the published files are written against, the manifests' + # too: an integrator had the platform's list and not what the pack writes. + for schema_path in sorted((_REPO_ROOT / "schemas").glob("*.schema.json")): + shutil.copy2(schema_path, schemas_dest / schema_path.name) + _record(schemas_dest / schema_path.name) + + manifest_assets = _publish_manifests(api) + for filename, (kind, items) in { + "systems.json": ("system-index", system_index_items(db, coverages, profiles)), + "cores.json": ("core-index", core_index_items(profiles)), + }.items(): + write_if_changed(str(api / filename), json.dumps({ + "schema_version": 1, "generated_at": generated_at, **stamp, + "kind": kind, "count": len(items), "items": items, + }, ensure_ascii=False, indent=2, sort_keys=True) + "\n") file_rows = [] for sha1, entry in sorted(db.get("files", {}).items()): @@ -1400,7 +1456,7 @@ def generate_data_exports( "platform_id", "platform", "system", "name", "destination", "required", "region", "variant_group", "size", "sha1", "sha256", "md5", "crc32", "status", "category", - "collection_path", "resolved_sha1", + "collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS, ], platform_files, ), @@ -1452,6 +1508,11 @@ def generate_data_exports( (api / "gaps.json", "Verification and coverage gaps", "application/json", "schemas/site-api-envelope.schema.json"), (api / "file-locations.json", "Where each file's card is, by SHA1", "application/json", "schemas/site-api-envelope.schema.json"), (api / "stats.json", "Project statistics", "application/json", "schemas/stats.schema.json"), + (api / "systems.json", "System ids, their names and sections", "application/json", + "schemas/site-api-envelope.schema.json"), + (api / "cores.json", "Core names and the profile each one is", "application/json", + "schemas/site-api-envelope.schema.json"), + *manifest_assets, (downloads / "files.csv", "File hashes CSV", "text/csv", None), (downloads / "retrobios.dat", "Collection DAT (Logiqx XML, for clrmamepro and RomVault)", "application/xml", None), @@ -1488,6 +1549,7 @@ def generate_data_exports( "generated_at": generated_at, **stamp, "kind": "catalog", + "license": DATA_LICENSE, "count": len(published), "items": published, } @@ -1495,10 +1557,176 @@ def generate_data_exports( str(api / "catalog.json"), json.dumps(catalog, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) + # api/v1/ itself is a folder GitHub Pages answers with the 404 page: the + # entry point is a document of its own, naming where to start. + entry_points = [ + {"title": "Catalog", "url": "catalog.json", + "description": "Every published file with its size, SHA256 and schema."}, + {"title": "System ids", "url": "systems.json", + "description": "Each system id with its name and the section holding its files."}, + {"title": "Core names", "url": "cores.json", + "description": "Each core name and alias with the profile it designates."}, + {"title": "Schemas", "url": "schemas/site-api-envelope.schema.json", + "description": "The envelope every JSON document of api/v1 shares."}, + ] + write_if_changed(str(api / "index.json"), json.dumps({ + "schema_version": 1, "generated_at": generated_at, **stamp, + "kind": "index", "license": DATA_LICENSE, + "count": len(entry_points), "items": entry_points, + }, ensure_ascii=False, indent=2, sort_keys=True) + "\n") return catalog_items +def _publish_manifests(api: Path) -> list[tuple]: + """The install manifests and the release record, beside the API. + + The platform's list says what a setup checks. The manifest says what its + pack installs: Batocera's list holds 353 files, its pack 14,508. The + only copy was install/.json on the repository's main branch, + which no page named. Copied byte for byte: the SHA256 the catalog gives + is the repository file's. + """ + folder = api / "manifests" + folder.mkdir(parents=True, exist_ok=True) + assets: list[tuple] = [] + for source in sorted((_REPO_ROOT / "install").glob("*.json")): + target = folder / source.name + shutil.copy2(source, target) + _record(target) + assets.append((target, f"Install manifest: {source.stem}", "application/json", + "schemas/install-manifest.schema.json")) + release = _REPO_ROOT / "release.json" + if release.is_file(): + shutil.copy2(release, api / "release.json") + _record(api / "release.json") + assets.append((api / "release.json", "Latest release: packs, sizes and assets", + "application/json", "schemas/release.schema.json")) + return assets + + +def _site_path(markdown_target: str) -> str: + """A page link as generators write it (systems/sony.md#psx) as a + site-absolute path (/retrobios/systems/sony/#psx), which resolves the same + from any document that holds it.""" + page, _, anchor = markdown_target.partition("#") + base = urllib.parse.urlsplit(SITE_URL).path + page = page.removesuffix(".md") + page = "" if page == "index" else page.removesuffix("/index") + "/" + return f"{base}{page}" + (f"#{anchor}" if anchor else "") + + +def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: + """Each system id the platforms and profiles write, reachable by that id. + + api/v1/systems/ is keyed by manufacturer and its consoles by display + name: a frontend keying by system id (sony-playstation) had no way in. + The url is the section system_section gives, the one every page links. + """ + from siterender import system_ids, system_section + + sha1_by_path = _sha1_by_path(db) + files = db.get("files") or {} + by_md5 = (db.get("indexes") or {}).get("by_md5") or {} + held: dict[str, set[str]] = {} + for coverage in coverages.values(): + for detail in coverage.get("details") or []: + path = str(detail.get("path") or "") + sha1 = sha1_by_path.get(path) if detail.get("status") != "missing" else None + if sha1: + held.setdefault(str(detail.get("system", "")), set()).add(sha1) + for profile in profiles.values(): + if profile.get("type") in ("alias", "test"): + continue + own = [str(s) for s in profile.get("systems") or []] + for entry in profile.get("files") or []: + if not isinstance(entry, dict): + continue + ids = ([str(entry["system"])] if entry.get("system") + else [str(s) for s in entry.get("systems") or []] or + (own if len(own) == 1 else [])) + found = {v for v in _hash_values(entry.get("sha1")) if v in files} + for md5 in _hash_values(entry.get("md5")): + match = by_md5.get(md5) + found.update([match] if isinstance(match, str) else match or []) + for sys_id in ids: + held.setdefault(sys_id, set()).update(found) + manufacturers = {_slug(mfr): mfr for mfr in _group_by_manufacturer(db)} + items = [] + for sys_id in sorted(system_ids(profiles, [c["config"] for c in coverages.values()])): + section = system_section(sys_id) + try: + name = system_name(sys_id) + except KeyError: + name = sys_id + items.append({ + "id": sys_id, + "name": name, + "manufacturer": manufacturers.get(section[0], "") if section else "", + "url": _site_path(console_href(*section)) if section else None, + "sha1": sorted(held.get(sys_id, set()) & files.keys()), + }) + return items + + +def core_index_items(profiles: dict) -> list[dict]: + """Every name a core goes by, with the profile it designates. + + A frontend knows mednafen_psx_libretro, and api/v1/emulators/ had only + beetle_psx.json: the reverse index resolve_platform_cores reads, aliases + included, said once. A name two profiles claim (pcsx2) lists both, the + one a platform list designates first. + """ + from common import preferred_profile, upstream_profile_index + + published = {name for name, p in profiles.items() if p.get("type") not in ("alias", "test")} + index = upstream_profile_index(profiles) + for name, profile in profiles.items(): + target = str(profile.get("alias_of") or "") + if profile.get("type") == "alias" and target in published: + index.setdefault(name, set()).add(target) + items = [] + for name in sorted(index): + claimants = sorted(index[name] & published) + preferred = preferred_profile({name: set(claimants)}, name) + for profile in sorted(claimants, key=lambda p: (p != preferred, p)): + items.append({"name": name, "profile": profile, "preferred": profile == preferred}) + return items + + +def _is_manifest(item: dict) -> bool: + return item["url"].startswith("api/v1/manifests/") + + +def _schema_field_rows(schema: dict, keys: tuple[str, ...]) -> list[str]: + """One table row per field, its description read from the schema, so the + page and the published contract say the same thing.""" + props = schema.get("properties") or {} + return [ + f"| `{key}` | {_escape_tags(str(props[key]['description']))} |" + for key in keys if (props.get(key) or {}).get("description") + ] + + +# The manifest fields an integrator reads first, in the order they matter. +MANIFEST_TOP_FIELDS = ("platform", "version", "base_destination", "files", + "omitted_files", "pack_files", "core_names", "system_cores") +MANIFEST_FILE_FIELDS = ("dest", "sha1", "size", "repo_path", "release_asset", "cores", + "systems", "category", "alternatives") + + +def _short_sha256(item: dict, catalog: str) -> str: + """A digest cell that fits its column: 12 characters linking the catalog + that holds it, the whole value in the title. 64 characters pushed the + table to 932 px in a 720 px column.""" + value = str(item["sha256"]) + return f'{value[:12]}' + + def generate_data_page(exports: list[dict]) -> str: + manifest_schema = json.loads( + (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8")) + snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") + manifests = [item for item in exports if _is_manifest(item)] lines = [ "# Data and API", "", @@ -1511,20 +1739,21 @@ def generate_data_page(exports: list[dict]) -> str: markup.button("Collection DAT", "downloads/retrobios.dat"), ]), "", - "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game " - "per `bios///` folder of the collection, named " - "`Manufacturer\\Console` so the tools look in that subfolder, each file a " - "rom named by its path below that folder. It audits a copy of the " - "collection's `bios/` tree, or rebuilds that layout from your own dumps. A " - "platform pack has its own layout and is checked with `install.py --check`.", + f"**Licence:** the metadata is offered under the MIT terms of [LICENSE]({REPO_URL}/blob/main/LICENSE), " + "which cover the scripts, tooling, schemas, documentation and metadata. " + f"They never cover a BIOS or firmware file ([NOTICE]({REPO_URL}/blob/main/NOTICE)). " + "`catalog.json` and the SQLite `metadata` table carry the same `license`.", "", "## Stable JSON endpoints", "", + "Start at [`api/v1/index.json`](api/v1/index.json): it names the catalog, " + "the two id indexes and the envelope schema.", + "", "| Dataset | Endpoint | Schema | Size |", "|---------|----------|--------|-----:|", ] for item in exports: - if item["media_type"] != "application/json": + if item["media_type"] != "application/json" or _is_manifest(item): continue schema = ( f"[JSON Schema](api/v1/{item['schema']})" @@ -1536,17 +1765,28 @@ def generate_data_page(exports: list[dict]) -> str: ) lines.extend([ "", - "Every JSON document carries `schema_version`, and `commit`, the revision " + "Every envelope carries `schema_version`, `kind` and `commit`, the revision " "of this repository the data was read at. Breaking changes use a new " - "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1.", + "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. " + "Each schema's `$id` is the address it is served at.", "", "One emulator profile without the whole list: " "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " "[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each " "emulator page links its own. `api/v1/platforms/.json` holds one " - "platform of `platforms.json`, and `api/v1/systems/.json` the files " - "of one system page by console. Each platform and system page links its own.", + "platform of `platforms.json`, its `contract` valid against " + "[`platform.schema.json`](api/v1/schemas/platform.schema.json) and its " + "`registry` the install and scraper settings beside the list. " + "`api/v1/systems/.json` holds the files of one system page by console. " + "Each platform and system page links its own.", + "", + "By id: [`systems.json`](api/v1/systems.json) gives each system id a platform " + "or a profile writes (`sony-playstation`) with its name, its manufacturer, the " + "`url` of the section holding its files and their SHA1. " + "[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a " + "platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), " + "whose document is `api/v1/emulators/.json`.", "", "One hash without the whole list: `api/v1/lookup/.json`, where `` " "is the first two characters of the hash in lowercase, holds every SHA1, " @@ -1555,6 +1795,65 @@ def generate_data_page(exports: list[dict]) -> str: "profile), the file `name` and the `url` of the page that shows it. " "[Find a file](files/index.md) reads it.", "", + "Two kinds of `url`:", + "", + "- In `catalog.json` and `index.json`, a `url` is relative to the document " + "holding it, as RFC 3986 resolves it.", + "- In the lookup shards, `file-locations.json` and `systems.json`, a `url` " + f"is site-absolute (`/retrobios/systems/sony/#...`). Prefix " + f"`{SITE_URL.removesuffix(urllib.parse.urlsplit(SITE_URL).path)}` for a full address.", + "", + "## Platform manifests", + "", + "A platform's list is what the setup checks. Its manifest is what the pack " + "installs: the list's files plus every file the setup's cores load. " + "`api/v1/manifests/.json` is the file `install.py` reads, byte " + "for byte, valid against " + "[`install-manifest.schema.json`](api/v1/schemas/install-manifest.schema.json). " + "[`api/v1/release.json`](api/v1/release.json) records the packs of the latest " + "release and the files each one holds.", + "", + "Manifests: " + ", ".join( + f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})" + for item in manifests) + "." if manifests else "", + "", + "| Field | Meaning |", + "|-------|---------|", + *_schema_field_rows(manifest_schema, MANIFEST_TOP_FIELDS), + "", + "Each entry of `files`:", + "", + "| Field | Meaning |", + "|-------|---------|", + *_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), + "", + "## Check a folder for one platform", + "", + "The job most tools come for: is this BIOS folder what the platform checks? " + "Read `api/v1/platforms/.json` and apply the platform's own rules:", + "", + "- `verification_mode: existence`: a file at its `destination` passes.", + "- An entry with no hash in the mode's field is checked on presence too.", + "- `zipped_file` set: the md5 is that of this member inside the ZIP, never " + "the ZIP's. Member names compare casefolded. A ZIP declared once per member " + "appears once per member: Batocera lists 692 entries for 353 destinations.", + "- An md5 shorter than 32 characters (Batocera) compares by its prefix. " + "Recalbox writes several accepted md5 in one field, comma separated.", + "- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members " + "read in name order, never the archive's.", + "- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. " + "The script below does not apply it.", + "", + "```python", + snippet.rstrip("\n"), + "```", + "", + "The script is [`scripts/check_platform_folder.py`]" + f"({REPO_URL}/blob/main/scripts/check_platform_folder.py). A test runs it " + "against the verifier on a sample folder, so the two give the same answer. " + "`install.py --check` answers another question: whether the folder holds " + "what the pack installs, the manifest above.", + "", "## Bulk downloads", "", "| Export | Format | Size | SHA256 |", @@ -1565,9 +1864,19 @@ def generate_data_page(exports: list[dict]) -> str: continue lines.append( f"| [{item['title']}]({item['url']}) | `{item['media_type']}` | " - f"{_fmt_size(item['bytes'])} | `{item['sha256']}` |" + f"{_fmt_size(item['bytes'])} | {_short_sha256(item, 'api/v1/catalog.json')} |" ) lines.extend([ + "", + "Each SHA256 is cut to 12 characters. Point at it for the whole value, or " + "read it in [`catalog.json`](api/v1/catalog.json).", + "", + "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game " + "per `bios///` folder of the collection, named " + "`Manufacturer\\Console` so the tools look in that subfolder, each file a " + "rom named by its path below that folder. It audits a copy of the " + "collection's `bios/` tree, or rebuilds that layout from your own dumps. A " + "platform pack has its own layout and is checked with `install.py --check`.", "", "The SQLite file contains indexed tables for content hashes, provenance " "catalog matches, platform declarations, emulator profiles and current " @@ -1601,6 +1910,25 @@ def generate_data_page(exports: list[dict]) -> str: "`provenance-wanted.csv` is the Wanted list of [Dump provenance]" "(provenance.md): each dump a catalog describes and the collection lacks.", "", + "`platform-files.csv` and the `platform_files` table also carry the entry " + "fields a check needs:", + "", + "- `zipped_file` set: the md5 is the md5 of that member inside the ZIP.", + "- `zip_digest` is `composite` where the md5 of a ZIP covers its members.", + "- `hash_match_mandatory` and `mandatory_declared` are Recalbox's flags.", + "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's " + "own file does not carry.", + "", + "## Hosting and limits", + "", + "- Static files on GitHub Pages, read with a plain GET. No key, no account.", + "- Any origin may read them (`Access-Control-Allow-Origin: *`), so a " + "browser tool can call them directly.", + "- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). " + "Send the `ETag` back in `If-None-Match` to skip an unchanged download.", + "- The base is the project's address, " + f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.", + "", "## Semantics and limits", "", "Treat four questions separately: content identity (hashes), presence in " @@ -2365,19 +2693,30 @@ def _blocks_in_runs(blocks: list[list[str]], budget: int) -> list[list[str]]: return runs +def _parent_front_matter(parent: tuple[str, str] | None) -> list[str]: + """The `parent:` front matter of a part page: the page it belongs to, + by label and site path, which the breadcrumb and sidebar read.""" + if parent is None: + return [] + label, link = parent + return ["parent:", f" label: {json.dumps(label)}", f" link: {json.dumps(link)}"] + + def _list_pages( stem: str, title: str, intro: str, head: list[str], blocks: list[list[str]], moved: dict[str, str], budget: int = LIST_PART_BUDGET, + parent: tuple[str, str] | None = None, ) -> str: """Pages of their own for a long list: ``head`` (a table header, say) opens each part, ``blocks`` are cut between parts. Returns where the list - went, as the page it left says it.""" + went, as the page it left says it. ``parent`` (label, site path) is the + page the parts belong to, for the breadcrumb.""" runs = _blocks_in_runs(blocks, budget) stems = [stem + (f"-{n}" if n > 1 else "") for n in range(1, len(runs) + 1)] for n, (here, run) in enumerate(zip(stems, runs), 1): part = f" (part {n} of {len(runs)})" if len(runs) > 1 else "" moved[here] = "\n".join([ - "---", "level: reference", "---", "", + "---", "level: reference", *_parent_front_matter(parent), "---", "", f"# {title}{part}", "", intro, "", *([", ".join(f"[part {i}]({other}.md)" if other != here else f"part {i}" for i, other in enumerate(stems, 1)), ""] if len(runs) > 1 else []), @@ -2404,6 +2743,11 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str: writes. A system with no section of its own is named, never sent to the page a storage folder chose. """ + return f"{_system_named_link(sys_id, prefix)} {_escape_tags(sys_id)}" + + +def _system_named_link(sys_id: str, prefix: str = "") -> str: + """A system's name, linked to the section system_section gives.""" from siterender import system_section name = _escape_tags(system_name(sys_id)) @@ -2411,7 +2755,17 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str: if section is not None: slug, console = section name = f"[{name}]({console_href(slug, console, prefix)})" - return f"{name} {_escape_tags(sys_id)}" + return name + + +def _emulator_named_link(key: str, prefix: str = "") -> str: + """An emulator by the name its page carries, linked to that page. A + profile with no page (an alias, a test) is named only.""" + profile = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False).get(key, {}) + label = _escape_tags(str(profile.get("emulator") or key)) + if profile.get("type") in ("alias", "test"): + return label + return f"[{label}]({prefix}emulators/{key}.md)" def _served_line(f: dict) -> str: @@ -2493,7 +2847,7 @@ def _render_platform_systems( anchor = sys_id.replace(" ", "-") body = [f"{ok_count}/{total} files verified"] if sys_emus: - emu_links = ", ".join(_emulator_link(e, "../") for e in sorted(sys_emus)) + emu_links = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus)) body[-1] += "\\" body.append(f"Emulators: {emu_links}") body.append("") @@ -2620,8 +2974,12 @@ def _render_platform_systems( for n in range(1, len(runs) + 1)] for n, (stem, run) in enumerate(zip(stems, runs), 1): part = f" (part {n} of {len(runs)})" if len(runs) > 1 else "" + # A console query landed here with no way to the console's + # own page: the list says whose it is first, linked. moved[stem] = "\n".join([ + "---", *_parent_front_matter((display, f"/platforms/{key}/")), "---", "", f"# {display}: {system_name(sys_id)}{part}", "", + f"{display}'s list for {_system_named_link(sys_id, '../')}. " f"Part of the [{display} page]({key}.md#{anchor}), whose other systems " f"are listed there. Each file below is as {display} declares it, with " "its status in the pack.", @@ -3121,10 +3479,12 @@ def generate_platform_page( f'{non_ok} issue{"s" if non_ok > 1 else ""}' ) sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id)) - emu_str = ", ".join(sys_emus[:3]) - if len(sys_emus) > 3: - emu_str += f" +{len(sys_emus) - 3}" anchor = sys_id.replace(" ", "-") + # Named and linked as the detail blocks below do: the column held + # profile ids, none of 1,463 linked. + emu_str = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus)[:3]) + if len(sys_emus) > 3: + emu_str += f" [+{len(sys_emus) - 3}](#{anchor})" # The name a reader filters on, the id the list writes beside it. lines.append( f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) {sys_id} " @@ -3257,7 +3617,9 @@ def file_location_items( A card lives on its manufacturer's page, and a file refiled between manufacturers (mt32_control.rom, Commodore to Roland) moved its link with it. files/?sha1= reads this list and goes to the card wherever it - is now: the address cited depends on the content only. + is now: the address cited depends on the content only. The url is + site-absolute (/retrobios/systems/...): written from the site root, it + resolved against the document holding it to a page that does not exist. """ items = [] for sha1, entry in sorted(db.get("files", {}).items()): @@ -3265,7 +3627,7 @@ def file_location_items( if len(parts) < 3 or parts[0] != "bios": continue page = card_page(entry.get("path", ""), sha1) - items.append({"sha1": sha1, "url": f"systems/{page}/#sha1-{sha1[:12]}"}) + items.append({"sha1": sha1, "url": _site_path(f"systems/{page}.md#sha1-{sha1[:12]}")}) # A data directory's files are listed on the page of an emulator reading # the tree: 2,534 hashes the packs ship were found nowhere on the site. owners = _data_member_owners(profiles or {}) @@ -3281,7 +3643,8 @@ def file_location_items( if sha1 and sha1 not in known: known.add(sha1) page = page_of.get(member, owners[ref]) - items.append({"sha1": sha1, "url": f"emulators/{page}/#sha1-{sha1[:12]}"}) + items.append({"sha1": sha1, + "url": _site_path(f"emulators/{page}.md#sha1-{sha1[:12]}")}) return sorted(items, key=lambda item: item["sha1"]) @@ -3325,7 +3688,7 @@ def file_lookup_records( }) for key, coverage in sorted(coverages.items()): for system_id, system in sorted((coverage["config"].get("systems") or {}).items()): - url = f"platforms/{key}/#{system_id.replace(' ', '-')}" + url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}") for entry in system.get("files") or []: for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): @@ -3338,7 +3701,7 @@ def file_lookup_records( continue entries = profile.get("files") or [] for entry, anchor in zip(entries, emulator_file_anchors(entries)): - url = f"emulators/{emulator_card_page(key, anchor)}/#{anchor}" + url = _site_path(f"emulators/{emulator_card_page(key, anchor)}.md#{anchor}") members = [(str(entry.get("name", "")), entry)] + [ (str(member["name"]), member) for member in entry.get("contents") or [] if isinstance(member, dict) and member.get("name") @@ -3745,7 +4108,7 @@ def generate_provenance_page( f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}", f"Dumps {label} catalogues and the collection lacks, by DAT. Part of " f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).", - [], blocks, moved, + [], blocks, moved, parent=("Dump provenance", "/provenance/"), ) lines.extend([ f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: " @@ -6092,12 +6455,27 @@ def _declared_hashes(config: dict, detail: dict) -> dict[str, list[str]]: return {} +def short_hash(value: str) -> str: + """A hash in a table cell: its first 12 characters, the whole value in + the title. Full MD5 and SHA1 values pushed the gaps tables past the + column on every screen.""" + value = str(value) + if len(value) <= 12: + return _code(value) + return f'{html.escape(value[:12])}' + + def _hash_cell(hashes: dict[str, list[str]]) -> str: return "
".join( - f"{algo} " + ", ".join(_code(v) for v in values) for algo, values in hashes.items() + f"{algo} " + ", ".join(short_hash(v) for v in values) for algo, values in hashes.items() ) +def _reason_cell(text: str) -> str: + """A reason in a gaps table, in the class that keeps its column readable.""" + return f'{text}' if text else "-" + + def _copy_cell(path: str, algos: list[str], db: dict | None) -> str: """The file a path holds, with its hashes in the algorithms declared.""" if not path: @@ -6208,7 +6586,7 @@ def _render_problem_files( plats = ", ".join(sorted(set(f["platforms"]))) declared = f["declared"] lines.append( - f"| {_code(f['name'])} | {plats} | {f['reason']} " + f"| {_code(f['name'])} | {plats} | {_reason_cell(f['reason'])} " f"| {_hash_cell(declared) or '-'} " f"| {_copy_cell(f['path'], list(declared) or ['md5'], db)} |" ) @@ -6230,7 +6608,7 @@ def _render_problem_files( plats = ", ".join(sorted(set(f["platforms"]))) held = _held_accepted(f["discrepancy"], db) cell = f"[{_code(held)}]({_fetch_url(held)})" if held else "-" - lines.append(f"| `{fname}` | {plats} | {f['discrepancy']} | {cell} |") + lines.append(f"| `{fname}` | {plats} | {_reason_cell(f['discrepancy'])} | {cell} |") lines.append("") if settled: @@ -6752,6 +7130,7 @@ def generate_gap_analysis( "the collection does not hold. Part of " "[What the packs do not hold](gaps.md#files-to-source).", head, [[row] for row in rows], moved, + parent=("What the packs do not hold", "/gaps/"), ) lines.extend([f"Listed on {where}.", ""]) if bundled_rows: @@ -6953,7 +7332,8 @@ def _render_xref_by_platform( ) if moved is not None: moved[stem] = "\n".join([ - "---", "level: reference", "---", "", + "---", "level: reference", + *_parent_front_matter(("Cores by platform", "/cross-reference/")), "---", "", f"# Cores on {display}", "", f"{_count(count, 'core')}{_core_scope_note(config, unique)} that " @@ -7869,7 +8249,7 @@ def _generate(args: argparse.Namespace) -> None: write_single_item( public / "api" / "v1" / "systems" / f"{slug}.json", "systems", system_export_item(slug, mfr, consoles), - str(db.get("generated_at") or _timestamp()), + str(db.get("generated_at") or _timestamp()), _source_commit(), ) clash = split_system_stems() & {_slug(m) for m in manufacturers} if clash: @@ -7936,18 +8316,11 @@ def _generate(args: argparse.Namespace) -> None: # The envelope and item of emulators.json, one item long: the # profile validates against the published schema, which an "id" # beside its fields broke on all 530, and schema_version is there. - write_if_changed( - str(public / "api" / "v1" / "emulators" / f"{name}.json"), - json.dumps( - { - "schema_version": 1, - "generated_at": str(db.get("generated_at") or _timestamp()), - "kind": "emulators", - "count": 1, - "items": [{"id": name, "profile": profile}], - }, - indent=2, sort_keys=True, ensure_ascii=False, default=str, - ) + "\n", + # The commit too, as every other envelope: 625 files lacked it. + write_single_item( + public / "api" / "v1" / "emulators" / f"{name}.json", "emulators", + {"id": name, "profile": profile}, + str(db.get("generated_at") or _timestamp()), _source_commit(), ) pages = emulator_pages( name, profile, db, consumers, suppl_names, data_registry, diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 83989941..a4a873c1 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -685,14 +685,14 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase): records = generate_site.file_lookup_records(db, coverages, profiles, locations, members) found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records} self.assertEqual(found[("collection", "", "md5", md5)]["url"], - "systems/sony/#sha1-" + "ab" * 6) + "/retrobios/systems/sony/#sha1-" + "ab" * 6) self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found) self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin") self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"], - "platforms/batocera/#sony-playstation") + "/retrobios/platforms/batocera/#sony-playstation") self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found) self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"], - "emulators/duckstation/#file-scph5501-bin-490f666e") + "/retrobios/emulators/duckstation/#file-scph5501-bin-490f666e") self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom") self.assertNotIn(("emulator", "an-alias", "md5", md5), found) @@ -724,7 +724,7 @@ class AFileHasAnAddressByContent(unittest.TestCase): }} self.assertEqual( generate_site.file_location_items(db), - [{"sha1": "ab" * 20, "url": "systems/roland/#sha1-" + "ab" * 6}], + [{"sha1": "ab" * 20, "url": "/retrobios/systems/roland/#sha1-" + "ab" * 6}], ) self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page()) script = (Path(__file__).resolve().parents[1] / "website" / "src" / "scripts" @@ -865,7 +865,8 @@ class ADataDirectoryFileCanBeLookedUp(unittest.TestCase): members = data_directory_members(Path(tmp)) items = file_location_items( {"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}}, members) - self.assertEqual(items, [{"sha1": "c" * 40, "url": "emulators/dolphin/#sha1-cccccccccccc"}]) + self.assertEqual(items, [{"sha1": "c" * 40, + "url": "/retrobios/emulators/dolphin/#sha1-cccccccccccc"}]) class ACardSaysWhyAFileIsLeftOut(unittest.TestCase): diff --git a/tests/test_site_platform_hashes.py b/tests/test_site_platform_hashes.py index 15e5df32..7d131796 100644 --- a/tests/test_site_platform_hashes.py +++ b/tests/test_site_platform_hashes.py @@ -69,8 +69,9 @@ class AnUntestedRowNamesBothHashes(unittest.TestCase): db = {"files": {"f" * 40: {"path": path, "sha256": held}}} table = "\n".join(gs._render_problem_files([problem], db=db)) row = next(line for line in table.splitlines() if "dsifirmware" in line) - self.assertIn(f"sha256 `{declared}`", row) - self.assertIn(f"sha256 `{held}`", row) + # Cut to 12 characters in the cell, the whole value in its title. + self.assertIn(f'sha256 {declared[:12]}', row) + self.assertIn(f'sha256 {held[:12]}', row) self.assertIn(f"`{path}`", row) diff --git a/tests/test_site_r2_gate.py b/tests/test_site_r2_gate.py index 3a08cfa8..d3bb44a1 100644 --- a/tests/test_site_r2_gate.py +++ b/tests/test_site_r2_gate.py @@ -261,7 +261,7 @@ class TheRules(unittest.TestCase): {"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}}, {"dolphin-sys": members}) last = f"{generate_site.DATA_TREE_ROWS_PER_PAGE:040x}" - self.assertIn({"sha1": last, "url": f"emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items) + self.assertIn({"sha1": last, "url": f"/retrobios/emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items) def test_no_float_rule_for_right_aligned_images(self): css = (REPO_ROOT / "website" / "src" / "styles" / "retrobios.css").read_text(encoding="utf-8") diff --git a/tests/test_site_r3_reference.py b/tests/test_site_r3_reference.py new file mode 100644 index 00000000..fad13ffe --- /dev/null +++ b/tests/test_site_r3_reference.py @@ -0,0 +1,469 @@ +"""Reference pages and data (round 3, package 8). + +The CSV and SQLite dropped zipped_file, so a check built from them judged +106 good Batocera files wrong. The manifest a pack installs was nowhere on +the site. Contracts failed their own schema, 716 documents lacked `commit`, +no endpoint answered a system id or a buildbot core name, lookup URLs +resolved to 404s, the data's licence went unstated, platform overviews +named emulators as unlinked ids, gaps tables overflowed with full hashes +and a source link went into a repository that does not hold the file. +Tests that read the built site skip when it has not been generated. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import re +import shutil +import sqlite3 +import sys +import tempfile +import unittest +import urllib.parse +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import check_platform_folder # noqa: E402 +import generate_site # noqa: E402 +import sitesources # noqa: E402 + +PUBLIC = REPO_ROOT / "website" / "public" +API = PUBLIC / "api" / "v1" +SITE = "https://abdess.github.io/retrobios/" +TMP = REPO_ROOT / "tmp" + + +def _built(path: Path) -> Path: + if not path.exists(): + raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated") + return path + + +def _verify_status(entry: dict, path: Path, mode: str) -> str: + """What verify.py says of one entry, the platform's native logic.""" + from verify import verify_entry_existence, verify_entry_md5, verify_entry_sha1 + + local = str(path) if path.is_file() else None + if mode == "existence": + return verify_entry_existence(entry, local)["status"] + if mode == "sha1": + return verify_entry_sha1(entry, local)["status"] + return verify_entry_md5(entry, local)["status"] + + +def _as_snippet(status: str) -> str: + return {"ok": "ok", "untested": "wrong", "missing": "missing"}[status] + + +class TheExportsKeepWhatACheckNeeds(unittest.TestCase): + """8.1: zipped_file and the Recalbox flags reach the CSV and SQLite.""" + + COVERAGE = {"batocera": { + "platform": "Batocera", "details": [], + "config": {"platform": "Batocera", "verification_mode": "md5", + "install": {"detect": []}, "logo": "x.svg", "_key": "batocera", + "systems": {"bbc": {"files": [ + {"name": "bbcb.zip", "destination": "bbcb.zip", + "md5": "521a197d", "zipped_file": "basic1.rom"}, + {"name": "n.rom", "destination": "n.rom", "md5": "ab", + "hash_match_mandatory": True, "mandatory_declared": False, + "shared_group": "group-x"}, + ]}}}, + }} + + def test_rows_carry_the_member_and_the_flags(self): + _items, rows = generate_site._platform_export_rows(self.COVERAGE) + zipped, flagged = rows + self.assertEqual(zipped["zipped_file"], "basic1.rom") + self.assertIs(flagged["hash_match_mandatory"], True) + self.assertIs(flagged["mandatory_declared"], False) + self.assertEqual(flagged["shared_group"], "group-x") + self.assertEqual(flagged["zipped_file"], "") + + def test_the_csv_header_and_the_table_name_them(self): + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('"collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS,', source) + for field in generate_site.ENTRY_EXPORT_FIELDS: + self.assertIn(f":{field}", source) + for field in ("zipped_file", "hash_match_mandatory", "mandatory_declared", "curated"): + self.assertIn(field, generate_site.ENTRY_EXPORT_FIELDS) + + def test_the_built_csv_has_the_column(self): + text = _built(PUBLIC / "downloads" / "platform-files.csv").read_text(encoding="utf-8") + rows = list(csv.DictReader(io.StringIO(text))) + batocera = [row for row in rows if row["platform_id"] == "batocera"] + self.assertGreater(sum(1 for row in batocera if row["zipped_file"]), 300) + with sqlite3.connect(_built(PUBLIC / "downloads" / "retrobios.sqlite")) as db: + count = db.execute("SELECT count(*) FROM platform_files " + "WHERE platform_id = 'batocera' AND zipped_file != ''").fetchone()[0] + license_row = db.execute("SELECT value FROM metadata WHERE key = 'license'").fetchone() + self.assertGreater(count, 300) + self.assertEqual(license_row[0], generate_site.DATA_LICENSE) + + +class AFolderCheckAgreesWithTheVerifier(unittest.TestCase): + """8.1 and 8.3: the published example gives verify.py's verdicts.""" + + def _folder(self, tmp: Path) -> tuple[dict, Path]: + member = b"basic rom bytes" + root = tmp / "bios" + root.mkdir() + with zipfile.ZipFile(root / "bbcb.zip", "w") as archive: + archive.writestr("BASIC1.ROM", member) + (root / "plain.bin").write_bytes(b"plain") + (root / "wrong.bin").write_bytes(b"not it") + (root / "nohash.bin").write_bytes(b"x") + md5 = lambda data: hashlib.md5(data).hexdigest() # noqa: E731 + contract = {"verification_mode": "md5", "systems": {"s": {"files": [ + {"name": "bbcb.zip", "destination": "bbcb.zip", "md5": md5(member), + "zipped_file": "basic1.rom"}, + {"name": "bbcb.zip", "destination": "bbcb.zip", "md5": "0" * 32, + "zipped_file": "basic1.rom"}, + {"name": "plain.bin", "destination": "plain.bin", "md5": md5(b"plain")[:29]}, + {"name": "wrong.bin", "destination": "wrong.bin", "md5": md5(b"plain")}, + {"name": "nohash.bin", "destination": "nohash.bin"}, + {"name": "gone.bin", "destination": "gone.bin", "md5": "1" * 32}, + ]}}} + return contract, root + + def test_each_rule_on_a_sample_folder(self): + with tempfile.TemporaryDirectory(dir=TMP) as tmp: + contract, root = self._folder(Path(tmp)) + platforms = Path(tmp) / "platforms" + platforms.mkdir() + (platforms / "sample.json").write_text( + json.dumps({"items": [{"contract": contract}]}), encoding="utf-8") + found = check_platform_folder.check("sample", root, platforms.as_uri() + "/") + expected = [ + (entry["destination"], _as_snippet(_verify_status(entry, root / entry["destination"], "md5"))) + for entry in contract["systems"]["s"]["files"] + ] + self.assertEqual(found, expected) + self.assertEqual([v for _d, v in found], ["ok", "wrong", "ok", "wrong", "ok", "missing"]) + + def test_batocera_from_the_csv_matches_the_verifier(self): + """A check driven by platform-files.csv, with zipped_file, gives the + verifier's verdict on every Batocera entry of a folder filled from + the collection.""" + text = _built(PUBLIC / "downloads" / "platform-files.csv").read_text(encoding="utf-8") + rows = [row for row in csv.DictReader(io.StringIO(text)) if row["platform_id"] == "batocera"] + with tempfile.TemporaryDirectory(dir=TMP) as tmp: + root = Path(tmp) + for row in rows: + source = REPO_ROOT / row["collection_path"] if row["collection_path"] else None + target = root / row["destination"] + if source and source.is_file() and not target.exists(): + target.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, target) + except OSError: + shutil.copyfile(source, target) + held = [row for row in rows if (root / row["destination"]).is_file()] + if len(held) < 100: + raise unittest.SkipTest("the collection files are not in this checkout") + # The verifier reads the platform's own entries, the check the CSV. + contract = json.loads(_built(API / "platforms" / "batocera.json").read_text())[ + "items"][0]["contract"] + declared = [entry for _sys, system in sorted(contract["systems"].items()) + for entry in system.get("files") or []] + self.assertEqual([e["name"] for e in declared], [row["name"] for row in rows]) + mismatched = [] + zipped_ok = 0 + for row, platform_entry in zip(rows, declared): + entry = {key: value for key, value in row.items() if value} + path = root / row["destination"] + ours = "ok" if check_platform_folder.accepts(entry, path, "md5") else ( + "wrong" if path.is_file() else "missing") + theirs = _as_snippet(_verify_status(platform_entry, path, "md5")) + if ours != theirs: + mismatched.append((row["destination"], row.get("zipped_file"), ours, theirs)) + zipped_ok += ours == "ok" and bool(row["zipped_file"]) + self.assertEqual(mismatched, []) + self.assertGreater(zipped_ok, 100) + + def test_the_data_page_shows_the_script_itself(self): + script = (REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") + page = generate_site.generate_data_page([]) + self.assertIn("```python\n" + script.rstrip("\n") + "\n```", page) + self.assertLess(len(script.splitlines()), 70) + self.assertIn("## Check a folder for one platform", page) + + +class TheContractsValidate(unittest.TestCase): + """8.4: registry keys leave the contract, every $id is its served URL.""" + + def test_every_contract_validates_against_the_platform_schema(self): + from jsonschema import Draft202012Validator # noqa: PLC0415 + + from common import list_registered_platforms, load_platform_config # noqa: PLC0415 + + schema = json.loads((REPO_ROOT / "schemas" / "platform.schema.json").read_text()) + validator = Draft202012Validator(schema) + platforms = str(REPO_ROOT / "platforms") + for name in list_registered_platforms(platforms, include_archived=True): + config = load_platform_config(name, platforms) + contract, registry = generate_site._split_contract(config) + with self.subTest(platform=name): + errors = [error.message for error in validator.iter_errors(contract)] + self.assertEqual(errors, []) + self.assertIn("install", registry) + self.assertNotIn("_key", registry) + self.assertEqual(set(contract) | set(registry) | {"_key"}, set(config) | {"_key"}) + + def test_every_schema_id_is_the_address_it_is_served_at(self): + for path in sorted((REPO_ROOT / "schemas").glob("*.schema.json")): + schema = json.loads(path.read_text()) + with self.subTest(schema=path.name): + self.assertEqual(schema.get("$id"), f"{SITE}api/v1/schemas/{path.name}") + + def test_every_schema_is_published(self): + published = {p.name for p in _built(API / "schemas").glob("*.json")} + self.assertEqual(published, {p.name for p in (REPO_ROOT / "schemas").glob("*.schema.json")}) + + def test_the_built_api_validates(self): + from jsonschema import Draft202012Validator # noqa: PLC0415 + + schemas = {p.name: json.loads(p.read_text()) for p in (REPO_ROOT / "schemas").glob("*.json")} + envelope = Draft202012Validator(schemas["site-api-envelope.schema.json"]) + platform = Draft202012Validator(schemas["platform.schema.json"]) + document = json.loads(_built(API / "platforms.json").read_text()) + self.assertEqual(list(envelope.iter_errors(document)), []) + for item in document["items"]: + with self.subTest(platform=item["id"]): + self.assertEqual([e.message for e in platform.iter_errors(item["contract"])], []) + for name in ("systems.json", "cores.json", "index.json", "catalog.json"): + with self.subTest(document=name): + self.assertEqual( + [e.message for e in envelope.iter_errors(json.loads((API / name).read_text()))], []) + + +class EveryEnvelopeCarriesTheCommit(unittest.TestCase): + """8.5: emulators/ and systems/ lacked `commit` (716 documents).""" + + def test_the_writers_pass_it(self): + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('str(db.get("generated_at") or _timestamp()), _source_commit(),\n )\n' + " clash", source) + block = source[source.index("public_profiles = {"):source.index("pages = emulator_pages(")] + self.assertIn("write_single_item(", block) + self.assertIn("_source_commit()", block) + + def test_every_built_envelope_has_it(self): + _built(API / "catalog.json") + if not generate_site._source_commit(): + raise unittest.SkipTest("not a git checkout") + lacking = [ + path.relative_to(API).as_posix() for path in sorted(API.rglob("*.json")) + if path.parent.name not in ("schemas", "manifests") + and path.name not in ("database.json", "stats.json", "release.json") + and "commit" not in json.loads(path.read_text()) + ] + self.assertEqual(lacking, []) + + +class ManifestsArePublished(unittest.TestCase): + """8.2: the manifest a pack installs, byte for byte, in the catalog.""" + + def test_each_manifest_is_the_repository_file(self): + catalog = json.loads(_built(API / "catalog.json").read_text()) + listed = {item["url"]: item for item in catalog["items"]} + for source in sorted((REPO_ROOT / "install").glob("*.json")): + with self.subTest(manifest=source.name): + item = listed[f"manifests/{source.name}"] + self.assertEqual(item["sha256"], hashlib.sha256(source.read_bytes()).hexdigest()) + self.assertEqual(item["schema"], "schemas/install-manifest.schema.json") + self.assertIn("release.json", listed) + self.assertEqual(catalog["license"], generate_site.DATA_LICENSE) + + def test_the_data_page_explains_the_fields_from_the_schema(self): + page = generate_site.generate_data_page([]) + schema = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text()) + for field in ("dest", "repo_path", "release_asset", "cores"): + description = schema["$defs"]["file"]["properties"][field]["description"] + self.assertIn(f"| `{field}` | {generate_site._escape_tags(description)} |", page) + self.assertIn("## Platform manifests", page) + self.assertIn("**Licence:**", page) + self.assertIn("## Hosting and limits", page) + # The endpoints first, the DAT below them. + self.assertLess(page.index("## Stable JSON endpoints"), page.index("The DAT is Logiqx XML")) + + +class AnIdLeadsToItsData(unittest.TestCase): + """8.6: a system by its id, a core by the name the buildbot gives it.""" + + def test_a_buildbot_name_designates_its_profile(self): + profiles = { + "beetle_psx": {"type": "libretro", "cores": ["mednafen_psx", "mednafen_psx_hw"]}, + "pcsx2": {"type": "standalone"}, + "lrps2": {"type": "libretro", "cores": ["pcsx2"]}, + "mednafen_psx_alias": {"type": "alias", "alias_of": "beetle_psx"}, + "probe": {"type": "test", "cores": ["probe_core"]}, + } + items = generate_site.core_index_items(profiles) + by_name: dict[str, list[tuple[str, bool]]] = {} + for item in items: + by_name.setdefault(item["name"], []).append((item["profile"], item["preferred"])) + self.assertEqual(by_name["mednafen_psx"], [("beetle_psx", True)]) + self.assertEqual(by_name["pcsx2"], [("pcsx2", True), ("lrps2", False)]) + self.assertEqual(by_name["mednafen_psx_alias"], [("beetle_psx", True)]) + self.assertNotIn("probe_core", by_name) + + def test_the_built_index_resolves_ids_to_sections(self): + items = {item["id"]: item for item in + json.loads(_built(API / "systems.json").read_text())["items"]} + color = items["wonderswan-color"] + self.assertEqual(color["name"], "Bandai WonderSwan Color") + self.assertTrue(color["url"].startswith("/retrobios/systems/bandai/"), color["url"]) + self.assertTrue(items["sony-playstation"]["sha1"]) + cores = json.loads((API / "cores.json").read_text())["items"] + self.assertIn({"name": "mednafen_psx", "profile": "beetle_psx", "preferred": True}, cores) + + +class LookupUrlsResolve(unittest.TestCase): + """8.7: a record's url, joined to the shard holding it, is a page.""" + + def test_site_absolute(self): + db = {"files": {"ab" * 20: {"path": "bios/Sony/PlayStation/scph5501.bin"}}} + [item] = generate_site.file_location_items(db) + self.assertTrue(item["url"].startswith("/retrobios/systems/"), item["url"]) + shard = SITE + "api/v1/lookup/ab.json" + self.assertTrue(urllib.parse.urljoin(shard, item["url"]).startswith(SITE + "systems/")) + + def test_built_records_land_on_built_pages(self): + shard = json.loads(_built(API / "lookup" / "05.json").read_text()) + dist = REPO_ROOT / "website" / "dist" + for record in shard["items"][:200]: + with self.subTest(url=record["url"]): + self.assertTrue(record["url"].startswith("/retrobios/")) + if dist.is_dir(): + page = urllib.parse.unquote(record["url"].split("#")[0].removeprefix("/retrobios/")) + self.assertTrue((dist / page / "index.html").is_file(), page) + + +class PlatformPagesLinkTheirSystemsAndEmulators(unittest.TestCase): + """8.9: the overview named emulators as ids, none of 1,463 linked.""" + + def test_an_emulator_is_named_and_linked(self): + link = generate_site._emulator_named_link("beetle_psx", "../") + self.assertRegex(link, r"^\[[^\]]+\]\(\.\./emulators/beetle_psx\.md\)$") + self.assertNotEqual(link, "[beetle_psx](../emulators/beetle_psx.md)") + + def test_the_overview_column_uses_it(self): + source = Path(generate_site.__file__).read_text(encoding="utf-8") + block = source[source.index("### Systems overview"):source.index("_render_platform_systems(by_system")] + self.assertIn("_emulator_named_link(e, \"../\")", block) + self.assertNotIn('", ".join(sys_emus[:3])', block) + + def test_a_moved_list_opens_on_its_console_and_names_its_parent(self): + # Past PLATFORM_SECTION_BUDGET, so the list leaves for a page of its own. + files = [{"name": f"{'f' * 200}{i}.bin", "status": "ok"} for i in range(1200)] + moved: dict[str, str] = {} + generate_site._render_platform_systems( + {"sony-playstation": files}, {}, {}, ("retroarch", "RetroArch"), moved, {}) + self.assertTrue(moved) + page = next(iter(moved.values())) + self.assertTrue(page.startswith('---\nparent:\n label: "RetroArch"\n' + ' link: "/platforms/retroarch/"\n---\n'), page[:120]) + self.assertRegex(page, r"RetroArch's list for \[Sony PlayStation\]\(\.\./systems/") + + def test_part_pages_of_gaps_and_cross_reference_name_their_parent(self): + moved: dict[str, str] = {} + generate_site._list_pages("gaps-x", "T", "intro", [], [["| a |"]], moved, + parent=("What the packs do not hold", "/gaps/")) + self.assertIn('parent:\n label: "What the packs do not hold"\n link: "/gaps/"', moved["gaps-x"]) + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('_parent_front_matter(("Cores by platform", "/cross-reference/"))', source) + self.assertIn('parent=("Dump provenance", "/provenance/")', source) + + +class TablesFitTheirColumn(unittest.TestCase): + """8.12 and 8.13: a hash in a cell is 12 characters, the rest in its title.""" + + def test_the_downloads_table(self): + item = {"title": "SQLite snapshot", "url": "downloads/retrobios.sqlite", + "media_type": "application/vnd.sqlite3", "bytes": 10, "sha256": "a" * 64} + page = generate_site.generate_data_page([item]) + row = next(line for line in page.splitlines() if "SQLite snapshot" in line and "|" in line) + self.assertIn(f'{"a" * 12}', row) + self.assertNotIn("a" * 13, row.replace(f'title="{"a" * 64}"', "")) + + def test_the_gaps_tables(self): + problem = {"platform": "Batocera", "name": "x.bin", "status": "untested", "required": True, + "reason": "expected md5 differs", "declared": {"md5": ["b" * 32]}, "path": ""} + table = "\n".join(generate_site._render_problem_files([problem], db={"files": {}})) + row = next(line for line in table.splitlines() if "x.bin" in line) + self.assertIn(f'{"b" * 12}', row) + self.assertIn('expected md5 differs', row) + + +class SourceLinksGoWhereTheFileIs(unittest.TestCase): + """8.14: trident cited a Panda3DS path and linked it into 3dsTrident.""" + + PROFILE = { + "emulator": "trident", "type": "libretro", + "source": "https://github.com/DanAlexMorton/3dsTrident", + "upstream": "https://github.com/wheremyfoodat/Panda3DS", + "source_commit": "b" * 40, + } + + def tearDown(self): + sitesources._tree_paths.cache_clear() + + def _render(self, tree: frozenset[str] | None) -> str: + original = sitesources._tree_paths + sitesources._tree_paths = lambda repo, pin: tree # type: ignore[assignment] + try: + return sitesources._source_ref_markdown( + self.PROFILE, "src/emulator.cpp:234; src/core/loader/ncch.cpp:151-164") + finally: + sitesources._tree_paths = original + + def test_a_path_the_tree_lacks_is_not_linked(self): + text = self._render(frozenset({"src/emulator.cpp"})) + self.assertIn("[`src/emulator.cpp:234`](https://github.com/DanAlexMorton/3dsTrident/", text) + self.assertIn("`src/core/loader/ncch.cpp:151-164`", text) + self.assertNotIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader", text) + + def test_without_a_cached_tree_the_declarations_decide(self): + text = self._render(None) + self.assertIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader/ncch.cpp", text) + + def test_every_rendered_link_is_in_its_cached_tree(self): + """Offline: each link the pages render points at a path the cached + tree of its repository holds.""" + if not sitesources.TREE_CACHE.is_dir(): + raise unittest.SkipTest("no cached tree listings") + from common import load_emulator_profiles # noqa: PLC0415 + + link = re.compile(r"\(https://github\.com/([^/]+/[^/]+)/blob/([0-9a-f]{40})/([^#)]+)") + checked, wrong = 0, [] + profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"), skip_aliases=False) + for key, profile in sorted(profiles.items()): + for entry in profile.get("files") or []: + if not isinstance(entry, dict) or not entry.get("source_ref"): + continue + for slug, pin, path in link.findall( + sitesources._source_ref_markdown(profile, entry["source_ref"])): + import upstream # noqa: PLC0415 + + repo = upstream.parse_repo(f"https://github.com/{slug}") + held = sitesources.tree_holds(repo, pin, urllib.parse.unquote(path)) + if held is None: + continue + checked += 1 + if not held: + wrong.append((key, slug, path)) + if not checked: + raise unittest.SkipTest("no rendered link has a cached tree") + self.assertEqual(wrong, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_render.py b/tests/test_site_render.py index efbefc53..dabf9d9a 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -224,7 +224,7 @@ class AnAliasTreeIsListedOnItsParentsPage(unittest.TestCase): members = {"freeintv-overlays": {"a.png": {"sha1": "02" * 20}}} items = generate_site.file_location_items({"files": {}}, self.PROFILES, members) self.assertEqual(items, [{"sha1": "02" * 20, - "url": "emulators/freeintv/#sha1-020202020202"}]) + "url": "/retrobios/emulators/freeintv/#sha1-020202020202"}]) def test_the_parent_page_lists_the_tree(self): import generate_site diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index e4deae8a..0d2df849 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -54,7 +54,7 @@ Every module in `tests/`, by the area it guards. | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_offline`, `test_install_selection` | | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture