feat: publish manifests, id indexes and licence

The CSV and SQLite carry zipped_file and the Recalbox flags, contracts
leave registry keys in a registry field and validate, every envelope
carries commit, lookup URLs are site-absolute, systems.json and
cores.json answer an id, the Data page shows the manifest fields and a
folder check, and platform overviews link emulators by name.

scripts/generate_site.py

@@ -106,6 +106,9 @@ STARLIGHT_CONTENT = "src/content/docs"
 SITE_NAME = "RetroBIOS"
 REPO_URL = "https://github.com/Abdess/retrobios"
 RELEASE_URL = f"{REPO_URL}/releases/latest"
+# What LICENSE grants over the published data: its MIT terms name the
+# metadata. The BIOS files themselves are under NOTICE, never this.
+DATA_LICENSE = "MIT (LICENSE: scripts, tooling, schemas, documentation and metadata)"
 SITE_URL = "https://abdess.github.io/retrobios/"
 GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads"]
 # Generated directories of files served as they are, not pages.
@@ -806,6 +809,36 @@ def _sha1_by_path(db: dict | None) -> dict[str, str]:
     return {str(e.get("path", "")): sha1 for sha1, e in ((db or {}).get("files") or {}).items()}
 
 
+@functools.lru_cache(maxsize=1)
+def _contract_keys() -> frozenset[str]:
+    """The top-level keys platform.schema.json allows in a platform's list."""
+    schema = json.loads((_REPO_ROOT / "schemas" / "platform.schema.json").read_text(encoding="utf-8"))
+    return frozenset(schema.get("properties") or {})
+
+
+def _split_contract(config: dict) -> tuple[dict, dict]:
+    """The list, valid against platform.schema.json, and the registry entry.
+
+    load_platform_config merges the registry into the list: `install`,
+    `logo`, `scraper` and `_key` made all 18 contracts fail the published
+    schema. They stay published, beside the contract.
+    """
+    allowed = _contract_keys()
+    contract = {key: value for key, value in config.items() if key in allowed}
+    registry = {key: value for key, value in config.items()
+                if key not in allowed and key != "_key"}
+    return contract, registry
+
+
+# Per-entry fields a folder check needs beyond the hashes: an md5 is a ZIP
+# member's when zipped_file is set, and a CSV without it judged 106 good
+# Batocera files wrong.
+ENTRY_EXPORT_FIELDS = (
+    "zipped_file", "hash_match_mandatory", "mandatory_declared", "curated",
+    "shared_group", "mirror_of", "zip_digest",
+)
+
+
 def _platform_export_rows(
     coverages: dict, db: dict | None = None,
 ) -> tuple[list[dict], list[dict]]:
@@ -816,6 +849,7 @@ def _platform_export_rows(
     for key, coverage in sorted(coverages.items()):
         config = coverage["config"]
         outcome = _verify_outcomes(coverage, sha1_by_path)
+        contract, registry = _split_contract(config)
         items.append({
             "id": key,
             "name": coverage["platform"],
@@ -826,7 +860,8 @@ def _platform_export_rows(
                     "core_present", "core_missing", "core_unsourceable",
                 )
             },
-            "contract": config,
+            "contract": contract,
+            "registry": registry,
         })
         for system_id, system in sorted(config.get("systems", {}).items()):
             for entry in system.get("files", []) or []:
@@ -846,11 +881,20 @@ def _platform_export_rows(
                     "sha256": _json_text(entry.get("sha256")),
                     "md5": _json_text(entry.get("md5")),
                     "crc32": _json_text(entry.get("crc32")),
+                    **{field: _export_flag(entry.get(field)) for field in ENTRY_EXPORT_FIELDS},
                     **outcome(system_id, entry.get("name", "")),
                 })
     return items, file_rows
 
 
+def _export_flag(value: object) -> object:
+    """A per-entry field as CSV and SQLite hold it: booleans stay booleans,
+    an absent field is empty."""
+    if value is None:
+        return ""
+    return value if isinstance(value, bool) else str(value)
+
+
 def _emulator_export_items(profiles: dict) -> list[dict]:
     return [
         {"id": key, "profile": profile}
@@ -1089,7 +1133,9 @@ def _write_sqlite_export(
                 destination TEXT NOT NULL, required INTEGER NOT NULL,
                 region TEXT, variant_group TEXT, size INTEGER,
                 sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT,
-                status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT
+                status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT,
+                zipped_file TEXT, hash_match_mandatory INTEGER, mandatory_declared INTEGER,
+                curated INTEGER, shared_group TEXT, mirror_of TEXT, zip_digest TEXT
             );
             CREATE TABLE emulators (
                 id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT,
@@ -1134,6 +1180,7 @@ def _write_sqlite_export(
             **({"commit": commit} if commit else {}),
             "source": REPO_URL,
             "scope": "metadata only, no BIOS or firmware payload bytes",
+            "license": DATA_LICENSE,
         }
         connection.executemany(
             "INSERT INTO metadata(key, value) VALUES (?, ?)",
@@ -1168,7 +1215,8 @@ def _write_sqlite_export(
             "INSERT INTO platform_files VALUES "
             "(:platform_id, :system, :name, :destination, :required, :region, "
             ":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, "
-            ":collection_path, :resolved_sha1)",
+            ":collection_path, :resolved_sha1, :zipped_file, :hash_match_mandatory, "
+            ":mandatory_declared, :curated, :shared_group, :mirror_of, :zip_digest)",
             platform_files,
         )
         for item in emulator_items:
@@ -1359,13 +1407,21 @@ def generate_data_exports(
         json.dumps(db, ensure_ascii=False, indent=2) + "\n",
     )
 
-    schema_names = (
-        "database.schema.json", "emulator.schema.json", "platform.schema.json",
-        "site-api-envelope.schema.json", "stats.schema.json",
-    )
-    for schema_name in schema_names:
-        shutil.copy2(Path("schemas") / schema_name, schemas_dest / schema_name)
-        _record(schemas_dest / schema_name)
+    # Every schema the published files are written against, the manifests'
+    # too: an integrator had the platform's list and not what the pack writes.
+    for schema_path in sorted((_REPO_ROOT / "schemas").glob("*.schema.json")):
+        shutil.copy2(schema_path, schemas_dest / schema_path.name)
+        _record(schemas_dest / schema_path.name)
+
+    manifest_assets = _publish_manifests(api)
+    for filename, (kind, items) in {
+        "systems.json": ("system-index", system_index_items(db, coverages, profiles)),
+        "cores.json": ("core-index", core_index_items(profiles)),
+    }.items():
+        write_if_changed(str(api / filename), json.dumps({
+            "schema_version": 1, "generated_at": generated_at, **stamp,
+            "kind": kind, "count": len(items), "items": items,
+        }, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
 
     file_rows = []
     for sha1, entry in sorted(db.get("files", {}).items()):
@@ -1400,7 +1456,7 @@ def generate_data_exports(
                 "platform_id", "platform", "system", "name", "destination",
                 "required", "region", "variant_group", "size", "sha1",
                 "sha256", "md5", "crc32", "status", "category",
-                "collection_path", "resolved_sha1",
+                "collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS,
             ],
             platform_files,
         ),
@@ -1452,6 +1508,11 @@ def generate_data_exports(
         (api / "gaps.json", "Verification and coverage gaps", "application/json", "schemas/site-api-envelope.schema.json"),
         (api / "file-locations.json", "Where each file's card is, by SHA1", "application/json", "schemas/site-api-envelope.schema.json"),
         (api / "stats.json", "Project statistics", "application/json", "schemas/stats.schema.json"),
+        (api / "systems.json", "System ids, their names and sections", "application/json",
+         "schemas/site-api-envelope.schema.json"),
+        (api / "cores.json", "Core names and the profile each one is", "application/json",
+         "schemas/site-api-envelope.schema.json"),
+        *manifest_assets,
         (downloads / "files.csv", "File hashes CSV", "text/csv", None),
         (downloads / "retrobios.dat", "Collection DAT (Logiqx XML, for clrmamepro and RomVault)",
          "application/xml", None),
@@ -1488,6 +1549,7 @@ def generate_data_exports(
         "generated_at": generated_at,
         **stamp,
         "kind": "catalog",
+        "license": DATA_LICENSE,
         "count": len(published),
         "items": published,
     }
@@ -1495,10 +1557,176 @@ def generate_data_exports(
         str(api / "catalog.json"),
         json.dumps(catalog, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
     )
+    # api/v1/ itself is a folder GitHub Pages answers with the 404 page: the
+    # entry point is a document of its own, naming where to start.
+    entry_points = [
+        {"title": "Catalog", "url": "catalog.json",
+         "description": "Every published file with its size, SHA256 and schema."},
+        {"title": "System ids", "url": "systems.json",
+         "description": "Each system id with its name and the section holding its files."},
+        {"title": "Core names", "url": "cores.json",
+         "description": "Each core name and alias with the profile it designates."},
+        {"title": "Schemas", "url": "schemas/site-api-envelope.schema.json",
+         "description": "The envelope every JSON document of api/v1 shares."},
+    ]
+    write_if_changed(str(api / "index.json"), json.dumps({
+        "schema_version": 1, "generated_at": generated_at, **stamp,
+        "kind": "index", "license": DATA_LICENSE,
+        "count": len(entry_points), "items": entry_points,
+    }, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
     return catalog_items
 
 
+def _publish_manifests(api: Path) -> list[tuple]:
+    """The install manifests and the release record, beside the API.
+
+    The platform's list says what a setup checks. The manifest says what its
+    pack installs: Batocera's list holds 353 files, its pack 14,508. The
+    only copy was install/<platform>.json on the repository's main branch,
+    which no page named. Copied byte for byte: the SHA256 the catalog gives
+    is the repository file's.
+    """
+    folder = api / "manifests"
+    folder.mkdir(parents=True, exist_ok=True)
+    assets: list[tuple] = []
+    for source in sorted((_REPO_ROOT / "install").glob("*.json")):
+        target = folder / source.name
+        shutil.copy2(source, target)
+        _record(target)
+        assets.append((target, f"Install manifest: {source.stem}", "application/json",
+                       "schemas/install-manifest.schema.json"))
+    release = _REPO_ROOT / "release.json"
+    if release.is_file():
+        shutil.copy2(release, api / "release.json")
+        _record(api / "release.json")
+        assets.append((api / "release.json", "Latest release: packs, sizes and assets",
+                       "application/json", "schemas/release.schema.json"))
+    return assets
+
+
+def _site_path(markdown_target: str) -> str:
+    """A page link as generators write it (systems/sony.md#psx) as a
+    site-absolute path (/retrobios/systems/sony/#psx), which resolves the same
+    from any document that holds it."""
+    page, _, anchor = markdown_target.partition("#")
+    base = urllib.parse.urlsplit(SITE_URL).path
+    page = page.removesuffix(".md")
+    page = "" if page == "index" else page.removesuffix("/index") + "/"
+    return f"{base}{page}" + (f"#{anchor}" if anchor else "")
+
+
+def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]:
+    """Each system id the platforms and profiles write, reachable by that id.
+
+    api/v1/systems/ is keyed by manufacturer and its consoles by display
+    name: a frontend keying by system id (sony-playstation) had no way in.
+    The url is the section system_section gives, the one every page links.
+    """
+    from siterender import system_ids, system_section
+
+    sha1_by_path = _sha1_by_path(db)
+    files = db.get("files") or {}
+    by_md5 = (db.get("indexes") or {}).get("by_md5") or {}
+    held: dict[str, set[str]] = {}
+    for coverage in coverages.values():
+        for detail in coverage.get("details") or []:
+            path = str(detail.get("path") or "")
+            sha1 = sha1_by_path.get(path) if detail.get("status") != "missing" else None
+            if sha1:
+                held.setdefault(str(detail.get("system", "")), set()).add(sha1)
+    for profile in profiles.values():
+        if profile.get("type") in ("alias", "test"):
+            continue
+        own = [str(s) for s in profile.get("systems") or []]
+        for entry in profile.get("files") or []:
+            if not isinstance(entry, dict):
+                continue
+            ids = ([str(entry["system"])] if entry.get("system")
+                   else [str(s) for s in entry.get("systems") or []] or
+                   (own if len(own) == 1 else []))
+            found = {v for v in _hash_values(entry.get("sha1")) if v in files}
+            for md5 in _hash_values(entry.get("md5")):
+                match = by_md5.get(md5)
+                found.update([match] if isinstance(match, str) else match or [])
+            for sys_id in ids:
+                held.setdefault(sys_id, set()).update(found)
+    manufacturers = {_slug(mfr): mfr for mfr in _group_by_manufacturer(db)}
+    items = []
+    for sys_id in sorted(system_ids(profiles, [c["config"] for c in coverages.values()])):
+        section = system_section(sys_id)
+        try:
+            name = system_name(sys_id)
+        except KeyError:
+            name = sys_id
+        items.append({
+            "id": sys_id,
+            "name": name,
+            "manufacturer": manufacturers.get(section[0], "") if section else "",
+            "url": _site_path(console_href(*section)) if section else None,
+            "sha1": sorted(held.get(sys_id, set()) & files.keys()),
+        })
+    return items
+
+
+def core_index_items(profiles: dict) -> list[dict]:
+    """Every name a core goes by, with the profile it designates.
+
+    A frontend knows mednafen_psx_libretro, and api/v1/emulators/ had only
+    beetle_psx.json: the reverse index resolve_platform_cores reads, aliases
+    included, said once. A name two profiles claim (pcsx2) lists both, the
+    one a platform list designates first.
+    """
+    from common import preferred_profile, upstream_profile_index
+
+    published = {name for name, p in profiles.items() if p.get("type") not in ("alias", "test")}
+    index = upstream_profile_index(profiles)
+    for name, profile in profiles.items():
+        target = str(profile.get("alias_of") or "")
+        if profile.get("type") == "alias" and target in published:
+            index.setdefault(name, set()).add(target)
+    items = []
+    for name in sorted(index):
+        claimants = sorted(index[name] & published)
+        preferred = preferred_profile({name: set(claimants)}, name)
+        for profile in sorted(claimants, key=lambda p: (p != preferred, p)):
+            items.append({"name": name, "profile": profile, "preferred": profile == preferred})
+    return items
+
+
+def _is_manifest(item: dict) -> bool:
+    return item["url"].startswith("api/v1/manifests/")
+
+
+def _schema_field_rows(schema: dict, keys: tuple[str, ...]) -> list[str]:
+    """One table row per field, its description read from the schema, so the
+    page and the published contract say the same thing."""
+    props = schema.get("properties") or {}
+    return [
+        f"| `{key}` | {_escape_tags(str(props[key]['description']))} |"
+        for key in keys if (props.get(key) or {}).get("description")
+    ]
+
+
+# The manifest fields an integrator reads first, in the order they matter.
+MANIFEST_TOP_FIELDS = ("platform", "version", "base_destination", "files",
+                       "omitted_files", "pack_files", "core_names", "system_cores")
+MANIFEST_FILE_FIELDS = ("dest", "sha1", "size", "repo_path", "release_asset", "cores",
+                        "systems", "category", "alternatives")
+
+
+def _short_sha256(item: dict, catalog: str) -> str:
+    """A digest cell that fits its column: 12 characters linking the catalog
+    that holds it, the whole value in the title. 64 characters pushed the
+    table to 932 px in a 720 px column."""
+    value = str(item["sha256"])
+    return f'<a href="{catalog}"><code title="{value}">{value[:12]}</code></a>'
+
+
 def generate_data_page(exports: list[dict]) -> str:
+    manifest_schema = json.loads(
+        (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8"))
+    snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8")
+    manifests = [item for item in exports if _is_manifest(item)]
     lines = [
         "# Data and API",
         "",
@@ -1511,20 +1739,21 @@ def generate_data_page(exports: list[dict]) -> str:
             markup.button("Collection DAT", "downloads/retrobios.dat"),
         ]),
         "",
-        "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game "
-        "per `bios/<Manufacturer>/<Console>/` folder of the collection, named "
-        "`Manufacturer\\Console` so the tools look in that subfolder, each file a "
-        "rom named by its path below that folder. It audits a copy of the "
-        "collection's `bios/` tree, or rebuilds that layout from your own dumps. A "
-        "platform pack has its own layout and is checked with `install.py --check`.",
+        f"**Licence:** the metadata is offered under the MIT terms of [LICENSE]({REPO_URL}/blob/main/LICENSE), "
+        "which cover the scripts, tooling, schemas, documentation and metadata. "
+        f"They never cover a BIOS or firmware file ([NOTICE]({REPO_URL}/blob/main/NOTICE)). "
+        "`catalog.json` and the SQLite `metadata` table carry the same `license`.",
         "",
         "## Stable JSON endpoints",
         "",
+        "Start at [`api/v1/index.json`](api/v1/index.json): it names the catalog, "
+        "the two id indexes and the envelope schema.",
+        "",
         "| Dataset | Endpoint | Schema | Size |",
         "|---------|----------|--------|-----:|",
     ]
     for item in exports:
-        if item["media_type"] != "application/json":
+        if item["media_type"] != "application/json" or _is_manifest(item):
             continue
         schema = (
             f"[JSON Schema](api/v1/{item['schema']})"
@@ -1536,17 +1765,28 @@ def generate_data_page(exports: list[dict]) -> str:
         )
     lines.extend([
         "",
-        "Every JSON document carries `schema_version`, and `commit`, the revision "
+        "Every envelope carries `schema_version`, `kind` and `commit`, the revision "
         "of this repository the data was read at. Breaking changes use a new "
-        "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1.",
+        "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. "
+        "Each schema's `$id` is the address it is served at.",
         "",
         "One emulator profile without the whole list: "
         "`api/v1/emulators/<id>.json`, the envelope of `emulators.json` holding "
         "that one item, its `profile` valid against "
         "[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each "
         "emulator page links its own. `api/v1/platforms/<id>.json` holds one "
-        "platform of `platforms.json`, and `api/v1/systems/<slug>.json` the files "
-        "of one system page by console. Each platform and system page links its own.",
+        "platform of `platforms.json`, its `contract` valid against "
+        "[`platform.schema.json`](api/v1/schemas/platform.schema.json) and its "
+        "`registry` the install and scraper settings beside the list. "
+        "`api/v1/systems/<slug>.json` holds the files of one system page by console. "
+        "Each platform and system page links its own.",
+        "",
+        "By id: [`systems.json`](api/v1/systems.json) gives each system id a platform "
+        "or a profile writes (`sony-playstation`) with its name, its manufacturer, the "
+        "`url` of the section holding its files and their SHA1. "
+        "[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a "
+        "platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), "
+        "whose document is `api/v1/emulators/<profile>.json`.",
         "",
         "One hash without the whole list: `api/v1/lookup/<xx>.json`, where `<xx>` "
         "is the first two characters of the hash in lowercase, holds every SHA1, "
@@ -1555,6 +1795,65 @@ def generate_data_page(exports: list[dict]) -> str:
         "profile), the file `name` and the `url` of the page that shows it. "
         "[Find a file](files/index.md) reads it.",
         "",
+        "Two kinds of `url`:",
+        "",
+        "- In `catalog.json` and `index.json`, a `url` is relative to the document "
+        "holding it, as RFC 3986 resolves it.",
+        "- In the lookup shards, `file-locations.json` and `systems.json`, a `url` "
+        f"is site-absolute (`/retrobios/systems/sony/#...`). Prefix "
+        f"`{SITE_URL.removesuffix(urllib.parse.urlsplit(SITE_URL).path)}` for a full address.",
+        "",
+        "## Platform manifests",
+        "",
+        "A platform's list is what the setup checks. Its manifest is what the pack "
+        "installs: the list's files plus every file the setup's cores load. "
+        "`api/v1/manifests/<platform>.json` is the file `install.py` reads, byte "
+        "for byte, valid against "
+        "[`install-manifest.schema.json`](api/v1/schemas/install-manifest.schema.json). "
+        "[`api/v1/release.json`](api/v1/release.json) records the packs of the latest "
+        "release and the files each one holds.",
+        "",
+        "Manifests: " + ", ".join(
+            f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})"
+            for item in manifests) + "." if manifests else "",
+        "",
+        "| Field | Meaning |",
+        "|-------|---------|",
+        *_schema_field_rows(manifest_schema, MANIFEST_TOP_FIELDS),
+        "",
+        "Each entry of `files`:",
+        "",
+        "| Field | Meaning |",
+        "|-------|---------|",
+        *_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS),
+        "",
+        "## Check a folder for one platform",
+        "",
+        "The job most tools come for: is this BIOS folder what the platform checks? "
+        "Read `api/v1/platforms/<id>.json` and apply the platform's own rules:",
+        "",
+        "- `verification_mode: existence`: a file at its `destination` passes.",
+        "- An entry with no hash in the mode's field is checked on presence too.",
+        "- `zipped_file` set: the md5 is that of this member inside the ZIP, never "
+        "the ZIP's. Member names compare casefolded. A ZIP declared once per member "
+        "appears once per member: Batocera lists 692 entries for 353 destinations.",
+        "- An md5 shorter than 32 characters (Batocera) compares by its prefix. "
+        "Recalbox writes several accepted md5 in one field, comma separated.",
+        "- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members "
+        "read in name order, never the archive's.",
+        "- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. "
+        "The script below does not apply it.",
+        "",
+        "```python",
+        snippet.rstrip("\n"),
+        "```",
+        "",
+        "The script is [`scripts/check_platform_folder.py`]"
+        f"({REPO_URL}/blob/main/scripts/check_platform_folder.py). A test runs it "
+        "against the verifier on a sample folder, so the two give the same answer. "
+        "`install.py --check` answers another question: whether the folder holds "
+        "what the pack installs, the manifest above.",
+        "",
         "## Bulk downloads",
         "",
         "| Export | Format | Size | SHA256 |",
@@ -1565,9 +1864,19 @@ def generate_data_page(exports: list[dict]) -> str:
             continue
         lines.append(
             f"| [{item['title']}]({item['url']}) | `{item['media_type']}` | "
-            f"{_fmt_size(item['bytes'])} | `{item['sha256']}` |"
+            f"{_fmt_size(item['bytes'])} | {_short_sha256(item, 'api/v1/catalog.json')} |"
         )
     lines.extend([
+        "",
+        "Each SHA256 is cut to 12 characters. Point at it for the whole value, or "
+        "read it in [`catalog.json`](api/v1/catalog.json).",
+        "",
+        "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game "
+        "per `bios/<Manufacturer>/<Console>/` folder of the collection, named "
+        "`Manufacturer\\Console` so the tools look in that subfolder, each file a "
+        "rom named by its path below that folder. It audits a copy of the "
+        "collection's `bios/` tree, or rebuilds that layout from your own dumps. A "
+        "platform pack has its own layout and is checked with `install.py --check`.",
         "",
         "The SQLite file contains indexed tables for content hashes, provenance "
         "catalog matches, platform declarations, emulator profiles and current "
@@ -1601,6 +1910,25 @@ def generate_data_page(exports: list[dict]) -> str:
         "`provenance-wanted.csv` is the Wanted list of [Dump provenance]"
         "(provenance.md): each dump a catalog describes and the collection lacks.",
         "",
+        "`platform-files.csv` and the `platform_files` table also carry the entry "
+        "fields a check needs:",
+        "",
+        "- `zipped_file` set: the md5 is the md5 of that member inside the ZIP.",
+        "- `zip_digest` is `composite` where the md5 of a ZIP covers its members.",
+        "- `hash_match_mandatory` and `mandatory_declared` are Recalbox's flags.",
+        "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's "
+        "own file does not carry.",
+        "",
+        "## Hosting and limits",
+        "",
+        "- Static files on GitHub Pages, read with a plain GET. No key, no account.",
+        "- Any origin may read them (`Access-Control-Allow-Origin: *`), so a "
+        "browser tool can call them directly.",
+        "- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). "
+        "Send the `ETag` back in `If-None-Match` to skip an unchanged download.",
+        "- The base is the project's address, "
+        f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.",
+        "",
         "## Semantics and limits",
         "",
         "Treat four questions separately: content identity (hashes), presence in "
@@ -2365,19 +2693,30 @@ def _blocks_in_runs(blocks: list[list[str]], budget: int) -> list[list[str]]:
     return runs
 
 
+def _parent_front_matter(parent: tuple[str, str] | None) -> list[str]:
+    """The `parent:` front matter of a part page: the page it belongs to,
+    by label and site path, which the breadcrumb and sidebar read."""
+    if parent is None:
+        return []
+    label, link = parent
+    return ["parent:", f"  label: {json.dumps(label)}", f"  link: {json.dumps(link)}"]
+
+
 def _list_pages(
     stem: str, title: str, intro: str, head: list[str], blocks: list[list[str]],
     moved: dict[str, str], budget: int = LIST_PART_BUDGET,
+    parent: tuple[str, str] | None = None,
 ) -> str:
     """Pages of their own for a long list: ``head`` (a table header, say)
     opens each part, ``blocks`` are cut between parts. Returns where the list
-    went, as the page it left says it."""
+    went, as the page it left says it. ``parent`` (label, site path) is the
+    page the parts belong to, for the breadcrumb."""
     runs = _blocks_in_runs(blocks, budget)
     stems = [stem + (f"-{n}" if n > 1 else "") for n in range(1, len(runs) + 1)]
     for n, (here, run) in enumerate(zip(stems, runs), 1):
         part = f" (part {n} of {len(runs)})" if len(runs) > 1 else ""
         moved[here] = "\n".join([
-            "---", "level: reference", "---", "",
+            "---", "level: reference", *_parent_front_matter(parent), "---", "",
             f"# {title}{part}", "", intro, "",
             *([", ".join(f"[part {i}]({other}.md)" if other != here else f"part {i}"
                          for i, other in enumerate(stems, 1)), ""] if len(runs) > 1 else []),
@@ -2404,6 +2743,11 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str:
     writes. A system with no section of its own is named, never sent to the
     page a storage folder chose.
     """
+    return f"{_system_named_link(sys_id, prefix)} <small>{_escape_tags(sys_id)}</small>"
+
+
+def _system_named_link(sys_id: str, prefix: str = "") -> str:
+    """A system's name, linked to the section system_section gives."""
     from siterender import system_section
 
     name = _escape_tags(system_name(sys_id))
@@ -2411,7 +2755,17 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str:
     if section is not None:
         slug, console = section
         name = f"[{name}]({console_href(slug, console, prefix)})"
-    return f"{name} <small>{_escape_tags(sys_id)}</small>"
+    return name
+
+
+def _emulator_named_link(key: str, prefix: str = "") -> str:
+    """An emulator by the name its page carries, linked to that page. A
+    profile with no page (an alias, a test) is named only."""
+    profile = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False).get(key, {})
+    label = _escape_tags(str(profile.get("emulator") or key))
+    if profile.get("type") in ("alias", "test"):
+        return label
+    return f"[{label}]({prefix}emulators/{key}.md)"
 
 
 def _served_line(f: dict) -> str:
@@ -2493,7 +2847,7 @@ def _render_platform_systems(
         anchor = sys_id.replace(" ", "-")
         body = [f"{ok_count}/{total} files verified"]
         if sys_emus:
-            emu_links = ", ".join(_emulator_link(e, "../") for e in sorted(sys_emus))
+            emu_links = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus))
             body[-1] += "\\"
             body.append(f"Emulators: {emu_links}")
         body.append("")
@@ -2620,8 +2974,12 @@ def _render_platform_systems(
                      for n in range(1, len(runs) + 1)]
             for n, (stem, run) in enumerate(zip(stems, runs), 1):
                 part = f" (part {n} of {len(runs)})" if len(runs) > 1 else ""
+                # A console query landed here with no way to the console's
+                # own page: the list says whose it is first, linked.
                 moved[stem] = "\n".join([
+                    "---", *_parent_front_matter((display, f"/platforms/{key}/")), "---", "",
                     f"# {display}: {system_name(sys_id)}{part}", "",
+                    f"{display}'s list for {_system_named_link(sys_id, '../')}. "
                     f"Part of the [{display} page]({key}.md#{anchor}), whose other systems "
                     f"are listed there. Each file below is as {display} declares it, with "
                     "its status in the pack.",
@@ -3121,10 +3479,12 @@ def generate_platform_page(
                 f'{non_ok} issue{"s" if non_ok > 1 else ""}</span>'
             )
         sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id))
-        emu_str = ", ".join(sys_emus[:3])
-        if len(sys_emus) > 3:
-            emu_str += f" +{len(sys_emus) - 3}"
         anchor = sys_id.replace(" ", "-")
+        # Named and linked as the detail blocks below do: the column held
+        # profile ids, none of 1,463 linked.
+        emu_str = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus)[:3])
+        if len(sys_emus) > 3:
+            emu_str += f" [+{len(sys_emus) - 3}](#{anchor})"
         # The name a reader filters on, the id the list writes beside it.
         lines.append(
             f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) <small>{sys_id}</small> "
@@ -3257,7 +3617,9 @@ def file_location_items(
     A card lives on its manufacturer's page, and a file refiled between
     manufacturers (mt32_control.rom, Commodore to Roland) moved its link
     with it. files/?sha1= reads this list and goes to the card wherever it
-    is now: the address cited depends on the content only.
+    is now: the address cited depends on the content only. The url is
+    site-absolute (/retrobios/systems/...): written from the site root, it
+    resolved against the document holding it to a page that does not exist.
     """
     items = []
     for sha1, entry in sorted(db.get("files", {}).items()):
@@ -3265,7 +3627,7 @@ def file_location_items(
         if len(parts) < 3 or parts[0] != "bios":
             continue
         page = card_page(entry.get("path", ""), sha1)
-        items.append({"sha1": sha1, "url": f"systems/{page}/#sha1-{sha1[:12]}"})
+        items.append({"sha1": sha1, "url": _site_path(f"systems/{page}.md#sha1-{sha1[:12]}")})
     # A data directory's files are listed on the page of an emulator reading
     # the tree: 2,534 hashes the packs ship were found nowhere on the site.
     owners = _data_member_owners(profiles or {})
@@ -3281,7 +3643,8 @@ def file_location_items(
             if sha1 and sha1 not in known:
                 known.add(sha1)
                 page = page_of.get(member, owners[ref])
-                items.append({"sha1": sha1, "url": f"emulators/{page}/#sha1-{sha1[:12]}"})
+                items.append({"sha1": sha1,
+                              "url": _site_path(f"emulators/{page}.md#sha1-{sha1[:12]}")})
     return sorted(items, key=lambda item: item["sha1"])
 
 
@@ -3325,7 +3688,7 @@ def file_lookup_records(
                 })
     for key, coverage in sorted(coverages.items()):
         for system_id, system in sorted((coverage["config"].get("systems") or {}).items()):
-            url = f"platforms/{key}/#{system_id.replace(' ', '-')}"
+            url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}")
             for entry in system.get("files") or []:
                 for algorithm in HASH_ALGORITHMS:
                     for value in _hash_values(entry.get(algorithm)):
@@ -3338,7 +3701,7 @@ def file_lookup_records(
             continue
         entries = profile.get("files") or []
         for entry, anchor in zip(entries, emulator_file_anchors(entries)):
-            url = f"emulators/{emulator_card_page(key, anchor)}/#{anchor}"
+            url = _site_path(f"emulators/{emulator_card_page(key, anchor)}.md#{anchor}")
             members = [(str(entry.get("name", "")), entry)] + [
                 (str(member["name"]), member) for member in entry.get("contents") or []
                 if isinstance(member, dict) and member.get("name")
@@ -3745,7 +4108,7 @@ def generate_provenance_page(
             f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}",
             f"Dumps {label} catalogues and the collection lacks, by DAT. Part of "
             f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).",
-            [], blocks, moved,
+            [], blocks, moved, parent=("Dump provenance", "/provenance/"),
         )
         lines.extend([
             f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: "
@@ -6092,12 +6455,27 @@ def _declared_hashes(config: dict, detail: dict) -> dict[str, list[str]]:
     return {}
 
 
+def short_hash(value: str) -> str:
+    """A hash in a table cell: its first 12 characters, the whole value in
+    the title. Full MD5 and SHA1 values pushed the gaps tables past the
+    column on every screen."""
+    value = str(value)
+    if len(value) <= 12:
+        return _code(value)
+    return f'<code title="{html.escape(value)}">{html.escape(value[:12])}</code>'
+
+
 def _hash_cell(hashes: dict[str, list[str]]) -> str:
     return "<br>".join(
-        f"{algo} " + ", ".join(_code(v) for v in values) for algo, values in hashes.items()
+        f"{algo} " + ", ".join(short_hash(v) for v in values) for algo, values in hashes.items()
     )
 
 
+def _reason_cell(text: str) -> str:
+    """A reason in a gaps table, in the class that keeps its column readable."""
+    return f'<span class="rb-reason">{text}</span>' if text else "-"
+
+
 def _copy_cell(path: str, algos: list[str], db: dict | None) -> str:
     """The file a path holds, with its hashes in the algorithms declared."""
     if not path:
@@ -6208,7 +6586,7 @@ def _render_problem_files(
                 plats = ", ".join(sorted(set(f["platforms"])))
                 declared = f["declared"]
                 lines.append(
-                    f"| {_code(f['name'])} | {plats} | {f['reason']} "
+                    f"| {_code(f['name'])} | {plats} | {_reason_cell(f['reason'])} "
                     f"| {_hash_cell(declared) or '-'} "
                     f"| {_copy_cell(f['path'], list(declared) or ['md5'], db)} |"
                 )
@@ -6230,7 +6608,7 @@ def _render_problem_files(
                 plats = ", ".join(sorted(set(f["platforms"])))
                 held = _held_accepted(f["discrepancy"], db)
                 cell = f"[{_code(held)}]({_fetch_url(held)})" if held else "-"
-                lines.append(f"| `{fname}` | {plats} | {f['discrepancy']} | {cell} |")
+                lines.append(f"| `{fname}` | {plats} | {_reason_cell(f['discrepancy'])} | {cell} |")
             lines.append("")
 
     if settled:
@@ -6752,6 +7130,7 @@ def generate_gap_analysis(
                     "the collection does not hold. Part of "
                     "[What the packs do not hold](gaps.md#files-to-source).",
                     head, [[row] for row in rows], moved,
+                    parent=("What the packs do not hold", "/gaps/"),
                 )
                 lines.extend([f"Listed on {where}.", ""])
         if bundled_rows:
@@ -6953,7 +7332,8 @@ def _render_xref_by_platform(
         )
         if moved is not None:
             moved[stem] = "\n".join([
-                "---", "level: reference", "---", "",
+                "---", "level: reference",
+                *_parent_front_matter(("Cores by platform", "/cross-reference/")), "---", "",
                 f"# Cores on {display}",
                 "",
                 f"{_count(count, 'core')}{_core_scope_note(config, unique)} that "
@@ -7869,7 +8249,7 @@ def _generate(args: argparse.Namespace) -> None:
         write_single_item(
             public / "api" / "v1" / "systems" / f"{slug}.json", "systems",
             system_export_item(slug, mfr, consoles),
-            str(db.get("generated_at") or _timestamp()),
+            str(db.get("generated_at") or _timestamp()), _source_commit(),
         )
     clash = split_system_stems() & {_slug(m) for m in manufacturers}
     if clash:
@@ -7936,18 +8316,11 @@ def _generate(args: argparse.Namespace) -> None:
         # The envelope and item of emulators.json, one item long: the
         # profile validates against the published schema, which an "id"
         # beside its fields broke on all 530, and schema_version is there.
-        write_if_changed(
-            str(public / "api" / "v1" / "emulators" / f"{name}.json"),
-            json.dumps(
-                {
-                    "schema_version": 1,
-                    "generated_at": str(db.get("generated_at") or _timestamp()),
-                    "kind": "emulators",
-                    "count": 1,
-                    "items": [{"id": name, "profile": profile}],
-                },
-                indent=2, sort_keys=True, ensure_ascii=False, default=str,
-            ) + "\n",
+        # The commit too, as every other envelope: 625 files lacked it.
+        write_single_item(
+            public / "api" / "v1" / "emulators" / f"{name}.json", "emulators",
+            {"id": name, "profile": profile},
+            str(db.get("generated_at") or _timestamp()), _source_commit(),
         )
         pages = emulator_pages(
             name, profile, db, consumers, suppl_names, data_registry,

tests/test_site_exports.py

@@ -685,14 +685,14 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase):
         records = generate_site.file_lookup_records(db, coverages, profiles, locations, members)
         found = {(r["layer"], r.get("owner", ""), r["algorithm"], r["value"]): r for r in records}
         self.assertEqual(found[("collection", "", "md5", md5)]["url"],
-                         "systems/sony/#sha1-" + "ab" * 6)
+                         "/retrobios/systems/sony/#sha1-" + "ab" * 6)
         self.assertIn(("collection", "", "crc32", "8d8cb7e4"), found)
         self.assertEqual(found[("data-directory", "", "sha1", "cd" * 20)]["name"], "font.bin")
         self.assertEqual(found[("platform", "batocera", "md5", md5)]["url"],
-                         "platforms/batocera/#sony-playstation")
+                         "/retrobios/platforms/batocera/#sony-playstation")
         self.assertIn(("platform", "batocera", "md5", "0123456789abcdef0123456789abcdef"), found)
         self.assertEqual(found[("emulator", "duckstation", "md5", md5)]["url"],
-                         "emulators/duckstation/#file-scph5501-bin-490f666e")
+                         "/retrobios/emulators/duckstation/#file-scph5501-bin-490f666e")
         self.assertEqual(found[("emulator", "duckstation", "crc32", "11223344")]["name"], "m.rom")
         self.assertNotIn(("emulator", "an-alias", "md5", md5), found)
 
@@ -724,7 +724,7 @@ class AFileHasAnAddressByContent(unittest.TestCase):
         }}
         self.assertEqual(
             generate_site.file_location_items(db),
-            [{"sha1": "ab" * 20, "url": "systems/roland/#sha1-" + "ab" * 6}],
+            [{"sha1": "ab" * 20, "url": "/retrobios/systems/roland/#sha1-" + "ab" * 6}],
         )
         self.assertIn('id="rb-file-lookup-status"', generate_site.generate_files_page())
         script = (Path(__file__).resolve().parents[1] / "website" / "src" / "scripts"
@@ -865,7 +865,8 @@ class ADataDirectoryFileCanBeLookedUp(unittest.TestCase):
             members = data_directory_members(Path(tmp))
         items = file_location_items(
             {"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}}, members)
-        self.assertEqual(items, [{"sha1": "c" * 40, "url": "emulators/dolphin/#sha1-cccccccccccc"}])
+        self.assertEqual(items, [{"sha1": "c" * 40,
+                                  "url": "/retrobios/emulators/dolphin/#sha1-cccccccccccc"}])
 
 
 class ACardSaysWhyAFileIsLeftOut(unittest.TestCase):

tests/test_site_platform_hashes.py

@@ -69,8 +69,9 @@ class AnUntestedRowNamesBothHashes(unittest.TestCase):
         db = {"files": {"f" * 40: {"path": path, "sha256": held}}}
         table = "\n".join(gs._render_problem_files([problem], db=db))
         row = next(line for line in table.splitlines() if "dsifirmware" in line)
-        self.assertIn(f"sha256 `{declared}`", row)
-        self.assertIn(f"sha256 `{held}`", row)
+        # Cut to 12 characters in the cell, the whole value in its title.
+        self.assertIn(f'sha256 <code title="{declared}">{declared[:12]}</code>', row)
+        self.assertIn(f'sha256 <code title="{held}">{held[:12]}</code>', row)
         self.assertIn(f"`{path}`", row)
 
 

tests/test_site_r2_gate.py

@@ -261,7 +261,7 @@ class TheRules(unittest.TestCase):
             {"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}},
             {"dolphin-sys": members})
         last = f"{generate_site.DATA_TREE_ROWS_PER_PAGE:040x}"
-        self.assertIn({"sha1": last, "url": f"emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items)
+        self.assertIn({"sha1": last, "url": f"/retrobios/emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items)
 
     def test_no_float_rule_for_right_aligned_images(self):
         css = (REPO_ROOT / "website" / "src" / "styles" / "retrobios.css").read_text(encoding="utf-8")

tests/test_site_r3_reference.py

@@ -0,0 +1,469 @@
+"""Reference pages and data (round 3, package 8).
+
+The CSV and SQLite dropped zipped_file, so a check built from them judged
+106 good Batocera files wrong. The manifest a pack installs was nowhere on
+the site. Contracts failed their own schema, 716 documents lacked `commit`,
+no endpoint answered a system id or a buildbot core name, lookup URLs
+resolved to 404s, the data's licence went unstated, platform overviews
+named emulators as unlinked ids, gaps tables overflowed with full hashes
+and a source link went into a repository that does not hold the file.
+Tests that read the built site skip when it has not been generated.
+"""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import io
+import json
+import os
+import re
+import shutil
+import sqlite3
+import sys
+import tempfile
+import unittest
+import urllib.parse
+import zipfile
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import check_platform_folder  # noqa: E402
+import generate_site  # noqa: E402
+import sitesources  # noqa: E402
+
+PUBLIC = REPO_ROOT / "website" / "public"
+API = PUBLIC / "api" / "v1"
+SITE = "https://abdess.github.io/retrobios/"
+TMP = REPO_ROOT / "tmp"
+
+
+def _built(path: Path) -> Path:
+    if not path.exists():
+        raise unittest.SkipTest(f"{path.relative_to(REPO_ROOT)} not generated")
+    return path
+
+
+def _verify_status(entry: dict, path: Path, mode: str) -> str:
+    """What verify.py says of one entry, the platform's native logic."""
+    from verify import verify_entry_existence, verify_entry_md5, verify_entry_sha1
+
+    local = str(path) if path.is_file() else None
+    if mode == "existence":
+        return verify_entry_existence(entry, local)["status"]
+    if mode == "sha1":
+        return verify_entry_sha1(entry, local)["status"]
+    return verify_entry_md5(entry, local)["status"]
+
+
+def _as_snippet(status: str) -> str:
+    return {"ok": "ok", "untested": "wrong", "missing": "missing"}[status]
+
+
+class TheExportsKeepWhatACheckNeeds(unittest.TestCase):
+    """8.1: zipped_file and the Recalbox flags reach the CSV and SQLite."""
+
+    COVERAGE = {"batocera": {
+        "platform": "Batocera", "details": [],
+        "config": {"platform": "Batocera", "verification_mode": "md5",
+                   "install": {"detect": []}, "logo": "x.svg", "_key": "batocera",
+                   "systems": {"bbc": {"files": [
+                       {"name": "bbcb.zip", "destination": "bbcb.zip",
+                        "md5": "521a197d", "zipped_file": "basic1.rom"},
+                       {"name": "n.rom", "destination": "n.rom", "md5": "ab",
+                        "hash_match_mandatory": True, "mandatory_declared": False,
+                        "shared_group": "group-x"},
+                   ]}}},
+    }}
+
+    def test_rows_carry_the_member_and_the_flags(self):
+        _items, rows = generate_site._platform_export_rows(self.COVERAGE)
+        zipped, flagged = rows
+        self.assertEqual(zipped["zipped_file"], "basic1.rom")
+        self.assertIs(flagged["hash_match_mandatory"], True)
+        self.assertIs(flagged["mandatory_declared"], False)
+        self.assertEqual(flagged["shared_group"], "group-x")
+        self.assertEqual(flagged["zipped_file"], "")
+
+    def test_the_csv_header_and_the_table_name_them(self):
+        source = Path(generate_site.__file__).read_text(encoding="utf-8")
+        self.assertIn('"collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS,', source)
+        for field in generate_site.ENTRY_EXPORT_FIELDS:
+            self.assertIn(f":{field}", source)
+        for field in ("zipped_file", "hash_match_mandatory", "mandatory_declared", "curated"):
+            self.assertIn(field, generate_site.ENTRY_EXPORT_FIELDS)
+
+    def test_the_built_csv_has_the_column(self):
+        text = _built(PUBLIC / "downloads" / "platform-files.csv").read_text(encoding="utf-8")
+        rows = list(csv.DictReader(io.StringIO(text)))
+        batocera = [row for row in rows if row["platform_id"] == "batocera"]
+        self.assertGreater(sum(1 for row in batocera if row["zipped_file"]), 300)
+        with sqlite3.connect(_built(PUBLIC / "downloads" / "retrobios.sqlite")) as db:
+            count = db.execute("SELECT count(*) FROM platform_files "
+                               "WHERE platform_id = 'batocera' AND zipped_file != ''").fetchone()[0]
+            license_row = db.execute("SELECT value FROM metadata WHERE key = 'license'").fetchone()
+        self.assertGreater(count, 300)
+        self.assertEqual(license_row[0], generate_site.DATA_LICENSE)
+
+
+class AFolderCheckAgreesWithTheVerifier(unittest.TestCase):
+    """8.1 and 8.3: the published example gives verify.py's verdicts."""
+
+    def _folder(self, tmp: Path) -> tuple[dict, Path]:
+        member = b"basic rom bytes"
+        root = tmp / "bios"
+        root.mkdir()
+        with zipfile.ZipFile(root / "bbcb.zip", "w") as archive:
+            archive.writestr("BASIC1.ROM", member)
+        (root / "plain.bin").write_bytes(b"plain")
+        (root / "wrong.bin").write_bytes(b"not it")
+        (root / "nohash.bin").write_bytes(b"x")
+        md5 = lambda data: hashlib.md5(data).hexdigest()  # noqa: E731
+        contract = {"verification_mode": "md5", "systems": {"s": {"files": [
+            {"name": "bbcb.zip", "destination": "bbcb.zip", "md5": md5(member),
+             "zipped_file": "basic1.rom"},
+            {"name": "bbcb.zip", "destination": "bbcb.zip", "md5": "0" * 32,
+             "zipped_file": "basic1.rom"},
+            {"name": "plain.bin", "destination": "plain.bin", "md5": md5(b"plain")[:29]},
+            {"name": "wrong.bin", "destination": "wrong.bin", "md5": md5(b"plain")},
+            {"name": "nohash.bin", "destination": "nohash.bin"},
+            {"name": "gone.bin", "destination": "gone.bin", "md5": "1" * 32},
+        ]}}}
+        return contract, root
+
+    def test_each_rule_on_a_sample_folder(self):
+        with tempfile.TemporaryDirectory(dir=TMP) as tmp:
+            contract, root = self._folder(Path(tmp))
+            platforms = Path(tmp) / "platforms"
+            platforms.mkdir()
+            (platforms / "sample.json").write_text(
+                json.dumps({"items": [{"contract": contract}]}), encoding="utf-8")
+            found = check_platform_folder.check("sample", root, platforms.as_uri() + "/")
+            expected = [
+                (entry["destination"], _as_snippet(_verify_status(entry, root / entry["destination"], "md5")))
+                for entry in contract["systems"]["s"]["files"]
+            ]
+        self.assertEqual(found, expected)
+        self.assertEqual([v for _d, v in found], ["ok", "wrong", "ok", "wrong", "ok", "missing"])
+
+    def test_batocera_from_the_csv_matches_the_verifier(self):
+        """A check driven by platform-files.csv, with zipped_file, gives the
+        verifier's verdict on every Batocera entry of a folder filled from
+        the collection."""
+        text = _built(PUBLIC / "downloads" / "platform-files.csv").read_text(encoding="utf-8")
+        rows = [row for row in csv.DictReader(io.StringIO(text)) if row["platform_id"] == "batocera"]
+        with tempfile.TemporaryDirectory(dir=TMP) as tmp:
+            root = Path(tmp)
+            for row in rows:
+                source = REPO_ROOT / row["collection_path"] if row["collection_path"] else None
+                target = root / row["destination"]
+                if source and source.is_file() and not target.exists():
+                    target.parent.mkdir(parents=True, exist_ok=True)
+                    try:
+                        os.link(source, target)
+                    except OSError:
+                        shutil.copyfile(source, target)
+            held = [row for row in rows if (root / row["destination"]).is_file()]
+            if len(held) < 100:
+                raise unittest.SkipTest("the collection files are not in this checkout")
+            # The verifier reads the platform's own entries, the check the CSV.
+            contract = json.loads(_built(API / "platforms" / "batocera.json").read_text())[
+                "items"][0]["contract"]
+            declared = [entry for _sys, system in sorted(contract["systems"].items())
+                        for entry in system.get("files") or []]
+            self.assertEqual([e["name"] for e in declared], [row["name"] for row in rows])
+            mismatched = []
+            zipped_ok = 0
+            for row, platform_entry in zip(rows, declared):
+                entry = {key: value for key, value in row.items() if value}
+                path = root / row["destination"]
+                ours = "ok" if check_platform_folder.accepts(entry, path, "md5") else (
+                    "wrong" if path.is_file() else "missing")
+                theirs = _as_snippet(_verify_status(platform_entry, path, "md5"))
+                if ours != theirs:
+                    mismatched.append((row["destination"], row.get("zipped_file"), ours, theirs))
+                zipped_ok += ours == "ok" and bool(row["zipped_file"])
+        self.assertEqual(mismatched, [])
+        self.assertGreater(zipped_ok, 100)
+
+    def test_the_data_page_shows_the_script_itself(self):
+        script = (REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8")
+        page = generate_site.generate_data_page([])
+        self.assertIn("```python\n" + script.rstrip("\n") + "\n```", page)
+        self.assertLess(len(script.splitlines()), 70)
+        self.assertIn("## Check a folder for one platform", page)
+
+
+class TheContractsValidate(unittest.TestCase):
+    """8.4: registry keys leave the contract, every $id is its served URL."""
+
+    def test_every_contract_validates_against_the_platform_schema(self):
+        from jsonschema import Draft202012Validator  # noqa: PLC0415
+
+        from common import list_registered_platforms, load_platform_config  # noqa: PLC0415
+
+        schema = json.loads((REPO_ROOT / "schemas" / "platform.schema.json").read_text())
+        validator = Draft202012Validator(schema)
+        platforms = str(REPO_ROOT / "platforms")
+        for name in list_registered_platforms(platforms, include_archived=True):
+            config = load_platform_config(name, platforms)
+            contract, registry = generate_site._split_contract(config)
+            with self.subTest(platform=name):
+                errors = [error.message for error in validator.iter_errors(contract)]
+                self.assertEqual(errors, [])
+                self.assertIn("install", registry)
+                self.assertNotIn("_key", registry)
+                self.assertEqual(set(contract) | set(registry) | {"_key"}, set(config) | {"_key"})
+
+    def test_every_schema_id_is_the_address_it_is_served_at(self):
+        for path in sorted((REPO_ROOT / "schemas").glob("*.schema.json")):
+            schema = json.loads(path.read_text())
+            with self.subTest(schema=path.name):
+                self.assertEqual(schema.get("$id"), f"{SITE}api/v1/schemas/{path.name}")
+
+    def test_every_schema_is_published(self):
+        published = {p.name for p in _built(API / "schemas").glob("*.json")}
+        self.assertEqual(published, {p.name for p in (REPO_ROOT / "schemas").glob("*.schema.json")})
+
+    def test_the_built_api_validates(self):
+        from jsonschema import Draft202012Validator  # noqa: PLC0415
+
+        schemas = {p.name: json.loads(p.read_text()) for p in (REPO_ROOT / "schemas").glob("*.json")}
+        envelope = Draft202012Validator(schemas["site-api-envelope.schema.json"])
+        platform = Draft202012Validator(schemas["platform.schema.json"])
+        document = json.loads(_built(API / "platforms.json").read_text())
+        self.assertEqual(list(envelope.iter_errors(document)), [])
+        for item in document["items"]:
+            with self.subTest(platform=item["id"]):
+                self.assertEqual([e.message for e in platform.iter_errors(item["contract"])], [])
+        for name in ("systems.json", "cores.json", "index.json", "catalog.json"):
+            with self.subTest(document=name):
+                self.assertEqual(
+                    [e.message for e in envelope.iter_errors(json.loads((API / name).read_text()))], [])
+
+
+class EveryEnvelopeCarriesTheCommit(unittest.TestCase):
+    """8.5: emulators/ and systems/ lacked `commit` (716 documents)."""
+
+    def test_the_writers_pass_it(self):
+        source = Path(generate_site.__file__).read_text(encoding="utf-8")
+        self.assertIn('str(db.get("generated_at") or _timestamp()), _source_commit(),\n        )\n'
+                      "    clash", source)
+        block = source[source.index("public_profiles = {"):source.index("pages = emulator_pages(")]
+        self.assertIn("write_single_item(", block)
+        self.assertIn("_source_commit()", block)
+
+    def test_every_built_envelope_has_it(self):
+        _built(API / "catalog.json")
+        if not generate_site._source_commit():
+            raise unittest.SkipTest("not a git checkout")
+        lacking = [
+            path.relative_to(API).as_posix() for path in sorted(API.rglob("*.json"))
+            if path.parent.name not in ("schemas", "manifests")
+            and path.name not in ("database.json", "stats.json", "release.json")
+            and "commit" not in json.loads(path.read_text())
+        ]
+        self.assertEqual(lacking, [])
+
+
+class ManifestsArePublished(unittest.TestCase):
+    """8.2: the manifest a pack installs, byte for byte, in the catalog."""
+
+    def test_each_manifest_is_the_repository_file(self):
+        catalog = json.loads(_built(API / "catalog.json").read_text())
+        listed = {item["url"]: item for item in catalog["items"]}
+        for source in sorted((REPO_ROOT / "install").glob("*.json")):
+            with self.subTest(manifest=source.name):
+                item = listed[f"manifests/{source.name}"]
+                self.assertEqual(item["sha256"], hashlib.sha256(source.read_bytes()).hexdigest())
+                self.assertEqual(item["schema"], "schemas/install-manifest.schema.json")
+        self.assertIn("release.json", listed)
+        self.assertEqual(catalog["license"], generate_site.DATA_LICENSE)
+
+    def test_the_data_page_explains_the_fields_from_the_schema(self):
+        page = generate_site.generate_data_page([])
+        schema = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text())
+        for field in ("dest", "repo_path", "release_asset", "cores"):
+            description = schema["$defs"]["file"]["properties"][field]["description"]
+            self.assertIn(f"| `{field}` | {generate_site._escape_tags(description)} |", page)
+        self.assertIn("## Platform manifests", page)
+        self.assertIn("**Licence:**", page)
+        self.assertIn("## Hosting and limits", page)
+        # The endpoints first, the DAT below them.
+        self.assertLess(page.index("## Stable JSON endpoints"), page.index("The DAT is Logiqx XML"))
+
+
+class AnIdLeadsToItsData(unittest.TestCase):
+    """8.6: a system by its id, a core by the name the buildbot gives it."""
+
+    def test_a_buildbot_name_designates_its_profile(self):
+        profiles = {
+            "beetle_psx": {"type": "libretro", "cores": ["mednafen_psx", "mednafen_psx_hw"]},
+            "pcsx2": {"type": "standalone"},
+            "lrps2": {"type": "libretro", "cores": ["pcsx2"]},
+            "mednafen_psx_alias": {"type": "alias", "alias_of": "beetle_psx"},
+            "probe": {"type": "test", "cores": ["probe_core"]},
+        }
+        items = generate_site.core_index_items(profiles)
+        by_name: dict[str, list[tuple[str, bool]]] = {}
+        for item in items:
+            by_name.setdefault(item["name"], []).append((item["profile"], item["preferred"]))
+        self.assertEqual(by_name["mednafen_psx"], [("beetle_psx", True)])
+        self.assertEqual(by_name["pcsx2"], [("pcsx2", True), ("lrps2", False)])
+        self.assertEqual(by_name["mednafen_psx_alias"], [("beetle_psx", True)])
+        self.assertNotIn("probe_core", by_name)
+
+    def test_the_built_index_resolves_ids_to_sections(self):
+        items = {item["id"]: item for item in
+                 json.loads(_built(API / "systems.json").read_text())["items"]}
+        color = items["wonderswan-color"]
+        self.assertEqual(color["name"], "Bandai WonderSwan Color")
+        self.assertTrue(color["url"].startswith("/retrobios/systems/bandai/"), color["url"])
+        self.assertTrue(items["sony-playstation"]["sha1"])
+        cores = json.loads((API / "cores.json").read_text())["items"]
+        self.assertIn({"name": "mednafen_psx", "profile": "beetle_psx", "preferred": True}, cores)
+
+
+class LookupUrlsResolve(unittest.TestCase):
+    """8.7: a record's url, joined to the shard holding it, is a page."""
+
+    def test_site_absolute(self):
+        db = {"files": {"ab" * 20: {"path": "bios/Sony/PlayStation/scph5501.bin"}}}
+        [item] = generate_site.file_location_items(db)
+        self.assertTrue(item["url"].startswith("/retrobios/systems/"), item["url"])
+        shard = SITE + "api/v1/lookup/ab.json"
+        self.assertTrue(urllib.parse.urljoin(shard, item["url"]).startswith(SITE + "systems/"))
+
+    def test_built_records_land_on_built_pages(self):
+        shard = json.loads(_built(API / "lookup" / "05.json").read_text())
+        dist = REPO_ROOT / "website" / "dist"
+        for record in shard["items"][:200]:
+            with self.subTest(url=record["url"]):
+                self.assertTrue(record["url"].startswith("/retrobios/"))
+                if dist.is_dir():
+                    page = urllib.parse.unquote(record["url"].split("#")[0].removeprefix("/retrobios/"))
+                    self.assertTrue((dist / page / "index.html").is_file(), page)
+
+
+class PlatformPagesLinkTheirSystemsAndEmulators(unittest.TestCase):
+    """8.9: the overview named emulators as ids, none of 1,463 linked."""
+
+    def test_an_emulator_is_named_and_linked(self):
+        link = generate_site._emulator_named_link("beetle_psx", "../")
+        self.assertRegex(link, r"^\[[^\]]+\]\(\.\./emulators/beetle_psx\.md\)$")
+        self.assertNotEqual(link, "[beetle_psx](../emulators/beetle_psx.md)")
+
+    def test_the_overview_column_uses_it(self):
+        source = Path(generate_site.__file__).read_text(encoding="utf-8")
+        block = source[source.index("### Systems overview"):source.index("_render_platform_systems(by_system")]
+        self.assertIn("_emulator_named_link(e, \"../\")", block)
+        self.assertNotIn('", ".join(sys_emus[:3])', block)
+
+    def test_a_moved_list_opens_on_its_console_and_names_its_parent(self):
+        # Past PLATFORM_SECTION_BUDGET, so the list leaves for a page of its own.
+        files = [{"name": f"{'f' * 200}{i}.bin", "status": "ok"} for i in range(1200)]
+        moved: dict[str, str] = {}
+        generate_site._render_platform_systems(
+            {"sony-playstation": files}, {}, {}, ("retroarch", "RetroArch"), moved, {})
+        self.assertTrue(moved)
+        page = next(iter(moved.values()))
+        self.assertTrue(page.startswith('---\nparent:\n  label: "RetroArch"\n'
+                                        '  link: "/platforms/retroarch/"\n---\n'), page[:120])
+        self.assertRegex(page, r"RetroArch's list for \[Sony PlayStation\]\(\.\./systems/")
+
+    def test_part_pages_of_gaps_and_cross_reference_name_their_parent(self):
+        moved: dict[str, str] = {}
+        generate_site._list_pages("gaps-x", "T", "intro", [], [["| a |"]], moved,
+                                  parent=("What the packs do not hold", "/gaps/"))
+        self.assertIn('parent:\n  label: "What the packs do not hold"\n  link: "/gaps/"', moved["gaps-x"])
+        source = Path(generate_site.__file__).read_text(encoding="utf-8")
+        self.assertIn('_parent_front_matter(("Cores by platform", "/cross-reference/"))', source)
+        self.assertIn('parent=("Dump provenance", "/provenance/")', source)
+
+
+class TablesFitTheirColumn(unittest.TestCase):
+    """8.12 and 8.13: a hash in a cell is 12 characters, the rest in its title."""
+
+    def test_the_downloads_table(self):
+        item = {"title": "SQLite snapshot", "url": "downloads/retrobios.sqlite",
+                "media_type": "application/vnd.sqlite3", "bytes": 10, "sha256": "a" * 64}
+        page = generate_site.generate_data_page([item])
+        row = next(line for line in page.splitlines() if "SQLite snapshot" in line and "|" in line)
+        self.assertIn(f'<code title="{"a" * 64}">{"a" * 12}</code>', row)
+        self.assertNotIn("a" * 13, row.replace(f'title="{"a" * 64}"', ""))
+
+    def test_the_gaps_tables(self):
+        problem = {"platform": "Batocera", "name": "x.bin", "status": "untested", "required": True,
+                   "reason": "expected md5 differs", "declared": {"md5": ["b" * 32]}, "path": ""}
+        table = "\n".join(generate_site._render_problem_files([problem], db={"files": {}}))
+        row = next(line for line in table.splitlines() if "x.bin" in line)
+        self.assertIn(f'<code title="{"b" * 32}">{"b" * 12}</code>', row)
+        self.assertIn('<span class="rb-reason">expected md5 differs</span>', row)
+
+
+class SourceLinksGoWhereTheFileIs(unittest.TestCase):
+    """8.14: trident cited a Panda3DS path and linked it into 3dsTrident."""
+
+    PROFILE = {
+        "emulator": "trident", "type": "libretro",
+        "source": "https://github.com/DanAlexMorton/3dsTrident",
+        "upstream": "https://github.com/wheremyfoodat/Panda3DS",
+        "source_commit": "b" * 40,
+    }
+
+    def tearDown(self):
+        sitesources._tree_paths.cache_clear()
+
+    def _render(self, tree: frozenset[str] | None) -> str:
+        original = sitesources._tree_paths
+        sitesources._tree_paths = lambda repo, pin: tree  # type: ignore[assignment]
+        try:
+            return sitesources._source_ref_markdown(
+                self.PROFILE, "src/emulator.cpp:234; src/core/loader/ncch.cpp:151-164")
+        finally:
+            sitesources._tree_paths = original
+
+    def test_a_path_the_tree_lacks_is_not_linked(self):
+        text = self._render(frozenset({"src/emulator.cpp"}))
+        self.assertIn("[`src/emulator.cpp:234`](https://github.com/DanAlexMorton/3dsTrident/", text)
+        self.assertIn("`src/core/loader/ncch.cpp:151-164`", text)
+        self.assertNotIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader", text)
+
+    def test_without_a_cached_tree_the_declarations_decide(self):
+        text = self._render(None)
+        self.assertIn("3dsTrident/blob/" + "b" * 40 + "/src/core/loader/ncch.cpp", text)
+
+    def test_every_rendered_link_is_in_its_cached_tree(self):
+        """Offline: each link the pages render points at a path the cached
+        tree of its repository holds."""
+        if not sitesources.TREE_CACHE.is_dir():
+            raise unittest.SkipTest("no cached tree listings")
+        from common import load_emulator_profiles  # noqa: PLC0415
+
+        link = re.compile(r"\(https://github\.com/([^/]+/[^/]+)/blob/([0-9a-f]{40})/([^#)]+)")
+        checked, wrong = 0, []
+        profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"), skip_aliases=False)
+        for key, profile in sorted(profiles.items()):
+            for entry in profile.get("files") or []:
+                if not isinstance(entry, dict) or not entry.get("source_ref"):
+                    continue
+                for slug, pin, path in link.findall(
+                        sitesources._source_ref_markdown(profile, entry["source_ref"])):
+                    import upstream  # noqa: PLC0415
+
+                    repo = upstream.parse_repo(f"https://github.com/{slug}")
+                    held = sitesources.tree_holds(repo, pin, urllib.parse.unquote(path))
+                    if held is None:
+                        continue
+                    checked += 1
+                    if not held:
+                        wrong.append((key, slug, path))
+        if not checked:
+            raise unittest.SkipTest("no rendered link has a cached tree")
+        self.assertEqual(wrong, [])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_site_render.py

@@ -224,7 +224,7 @@ class AnAliasTreeIsListedOnItsParentsPage(unittest.TestCase):
         members = {"freeintv-overlays": {"a.png": {"sha1": "02" * 20}}}
         items = generate_site.file_location_items({"files": {}}, self.PROFILES, members)
         self.assertEqual(items, [{"sha1": "02" * 20,
-                                  "url": "emulators/freeintv/#sha1-020202020202"}])
+                                  "url": "/retrobios/emulators/freeintv/#sha1-020202020202"}])
 
     def test_the_parent_page_lists_the_tree(self):
         import generate_site

wiki/testing-guide.md

@@ -54,7 +54,7 @@ Every module in `tests/`, by the area it guards.
 | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_offline`, `test_install_selection` |
 | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
-| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
 
 ## Test architecture