feat: cite sources and files on reference pages

# Conflicts:
#	wiki/testing-guide.md

schemas/site-api-envelope.schema.json

@@ -19,6 +19,11 @@
       "type": "string",
       "format": "date-time"
     },
+    "commit": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{40}$",
+      "description": "The repository commit the data was read at."
+    },
     "kind": {
       "type": "string",
       "enum": [

scripts/exporter/candidates.py

@@ -41,6 +41,37 @@ def _entry_hashes(entry: dict) -> set[str]:
     return values
 
 
+def _declared_hashes(entry: dict) -> dict[str, set[str]]:
+    """Hash field -> the values a platform entry declares, alt_md5 as md5."""
+    found = {f: set(_hash_values(entry, f)) for f in HASH_FIELDS}
+    found["md5"] |= set(_hash_values(entry, "alt_md5"))
+    return {f: values for f, values in found.items() if values}
+
+
+def _same(declared: str, accepted: str) -> bool:
+    """Equal, or a cut md5 that starts the accepted one."""
+    return declared == accepted or (
+        _CUT_MD5 <= len(declared) < _FULL_MD5 and accepted.startswith(declared)
+    )
+
+
+def _accepts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
+    """True when the profile entry, or a member of its archive, accepts a
+    hash the platform entry declares."""
+    accepted = _entry_hashes(profile_entry)
+    return any(_same(d, a) for values in declared.values() for d in values for a in accepted)
+
+
+def _contradicts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
+    """True when both sides state the same kind of hash and no value agrees:
+    the profile entry describes another dump under the same name."""
+    for field_name, values in declared.items():
+        own = set(_hash_values(profile_entry, field_name))
+        if own and not any(_same(d, a) for d in values for a in own):
+            return True
+    return False
+
+
 @dataclass
 class ReadEvidence:
     """What the examined profiles open, indexed for one lookup per entry."""
@@ -373,20 +404,34 @@ class EntryCategories:
                 )
 
     def _matching(self, key: str, kind: str, entry: dict) -> list[dict]:
-        """The profile entries of `key` that name the platform entry."""
+        """The profile entries of `key` that read the platform entry.
+
+        A platform entry that declares a hash is answered by the profile
+        entries accepting that hash: one name covers several revisions
+        (DuckStation lists scph1001.bin v2.2 at bios.cpp:36 and v2.0 at
+        bios.cpp:42), and a match by name cited the line of another dump.
+        A same-named entry is kept, marked `"by": "name"`, only when no
+        entry accepts the hash and its own hashes do not contradict it.
+        """
         names = {
             str(n).casefold()
             for n in (entry.get("name"), *(entry.get("aliases") or []))
             if n
         }
-        found = []
+        declared = _declared_hashes(entry)
+        by_hash: list[dict] = []
+        by_name: list[dict] = []
         for f in self._profiles[key].get("files") or []:
             if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA):
                 continue
+            cited = {k: f[k] for k in ("name", "source_ref") if f.get(k)}
+            if declared and _accepts(f, declared):
+                by_hash.append(cited)
+                continue
             own = {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n}
-            if own & names:
-                found.append({k: f[k] for k in ("name", "source_ref") if f.get(k)})
-        return found
+            if own & names and not (declared and _contradicts(f, declared)):
+                by_name.append({**cited, "by": "name"})
+        return by_hash or by_name
 
     def explain(self, entry: dict, destination: str, system: str) -> dict:
         """The category of an entry and what establishes it."""

scripts/sitesources.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import re
 import urllib.parse
+from dataclasses import dataclass
 import urllib.request
 
 import upstream
@@ -345,3 +346,98 @@ _PATH_RE = re.compile(r"^[A-Za-z0-9_.@/+~-]+$")
 def _guarded_spans(line: str) -> list[tuple[int, int]]:
     """Code spans and links of a markdown line."""
     return [m.span() for m in re.finditer(r"`[^`]*`|\[[^\]]*\]\([^)]*\)", line)]
+
+
+# Where a platform's list was read: the forge, the repository, the revision
+# and the path inside it, out of the URL its scraper pinned.
+_LIST_URL_RES = (
+    re.compile(r"^https://raw\.githubusercontent\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/"
+               r"(?P<rev>[^/]+)/(?P<path>.+)$"),
+    re.compile(r"^https://github\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/(?:blob|tree)/"
+               r"(?P<rev>[^/]+)(?:/(?P<path>.+))?$"),
+    re.compile(r"^https://codeload\.github\.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)/"
+               r"tar\.gz/(?P<rev>[^/]+)$"),
+    re.compile(r"^https://(?P<host>gitlab\.com)/(?P<owner>[^/]+)/(?P<repo>[^/]+)/-/"
+               r"(?:raw|blob)/(?P<rev>[^/]+)/(?P<path>.+)$"),
+)
+_BRANCHES = frozenset({"master", "main", "db"})
+
+
+@dataclass(frozen=True)
+class ListSource:
+    """A platform's list as its scraper read it."""
+
+    host: str
+    owner: str
+    repo: str
+    rev: str
+    path: str
+
+    @property
+    def moving(self) -> bool:
+        """The revision is a branch: what was read is not recorded."""
+        return not self.rev or self.rev in _BRANCHES
+
+    @property
+    def label(self) -> str:
+        """`batocera-systems at tag batocera-43.1`, said in words."""
+        name = self.path.rsplit("/", 1)[-1] if self.path else f"{self.repo} repository"
+        if not self.rev:
+            return f"{name}, default branch"
+        if re.fullmatch(r"[0-9a-f]{40}", self.rev):
+            return f"{name} at commit {self.rev[:12]}"
+        if self.moving:
+            return f"{name} on branch {self.rev}"
+        return f"{name} at tag {self.rev}"
+
+    def browse(self, path: str = "", start: int | None = None,
+               end: int | None = None) -> str:
+        """The page a reader opens: the file, or the tree, at the revision."""
+        path = path or self.path
+        if self.host == "gitlab.com":
+            url = f"https://gitlab.com/{self.owner}/{self.repo}/-/blob/{self.rev}/{path}"
+            anchor = f"#L{start}" + (f"-{end}" if end and end != start else "")
+        else:
+            base = f"https://github.com/{self.owner}/{self.repo}"
+            if not self.rev:
+                return base
+            kind = "blob" if path else "tree"
+            url = f"{base}/{kind}/{self.rev}" + (f"/{path}" if path else "")
+            anchor = f"#L{start}" + (f"-L{end}" if end and end != start else "")
+        return url + (anchor if start else "")
+
+
+def list_source(url: str) -> ListSource | None:
+    """The forge, repository, revision and path of a platform's `source:`."""
+    url = str(url or "").strip()
+    for pattern in _LIST_URL_RES:
+        found = pattern.match(url)
+        if found:
+            parts = found.groupdict()
+            return ListSource(parts.get("host") or "github.com", parts["owner"],
+                              parts["repo"], parts["rev"], parts.get("path") or "")
+    bare = re.match(r"^https://github\.com/([^/]+)/([^/]+?)/?$", url)
+    if bare:
+        return ListSource("github.com", bare[1], bare[2], "", "")
+    return None
+
+
+def launcher_ref_markdown(source: ListSource | None, ref: str) -> str:
+    """A launcher record's source_ref, each part a pinned link.
+
+    Linked only where the caller knows the launcher code sits in the
+    repository and revision of the platform's list. Otherwise each part is
+    plain code: a link to another repository is a false citation.
+    """
+    rendered: list[str] = []
+    for part in split_source_ref(ref):
+        display = part.path.rsplit("/", 1)[-1]
+        if part.start is not None:
+            display += f":{part.start}"
+            if part.end is not None and part.end != part.start:
+                display += f"-{part.end}"
+        if source is None or not source.rev or not _PATH_RE.fullmatch(part.path):
+            rendered.append(f"`{display}`")
+        else:
+            rendered.append(f"[`{display}`]({source.browse(part.path, part.start, part.end)})")
+    return ", ".join(rendered)

tests/test_entry_categories.py

@@ -88,9 +88,11 @@ class Evidence(unittest.TestCase):
     def test_game_data_names_its_reader_and_reference(self):
         found = self.cats.explain({"name": "diabdat.mpq"}, "ports/diablo/diabdat.mpq", "diablo")
         self.assertEqual(found["category"], GAME_DATA)
+        # The platform entry declares no hash: the reader is matched by name.
         self.assertEqual(found["read_by"], [{
             "emulator": "devilutionx", "proof": "name", "runs_system": True,
-            "entries": [{"name": "diabdat.mpq", "source_ref": "Source/init.cpp:120"}]}])
+            "entries": [{"name": "diabdat.mpq", "source_ref": "Source/init.cpp:120",
+                         "by": "name"}]}])
         self.assertEqual(verify.category_evidence_text(found),
                          "read by devilutionx (diabdat.mpq at Source/init.cpp:120)")
 

tests/test_site_cores.py

@@ -33,12 +33,17 @@ class SiteCoreCounts(unittest.TestCase):
         for name in list_registered_platforms(platforms, include_archived=True):
             config = load_platform_config(name, platforms)
             coverages[name] = {"platform": config.get("platform", name), "config": config}
-        page = generate_site.generate_cross_reference(coverages, profiles)
+        moved: dict[str, str] = {}
+        page = generate_site.generate_cross_reference(coverages, profiles, moved)
         shown = {
             m.group(1): int(m.group(2))
-            for m in re.finditer(
-                r'<summary>([^<]+)</summary>\n\n.*\n\n\*\*(\d+) cores', page)
+            for m in re.finditer(r"^\| \[([^\]]+)\]\(platforms/[^)]+\) \| (\d+)", page, re.M)
         }
+        for name, cov in coverages.items():
+            with self.subTest(page=name):
+                rows = [line for line in moved[f"cross-reference-{name}"].splitlines()
+                        if line.startswith("| [")]
+                self.assertEqual(len(rows), shown[cov["platform"]])
         for name, cov in coverages.items():
             with self.subTest(platform=name):
                 expected = len(resolve_platform_cores(cov["config"], unique))

tests/test_site_exports.py

@@ -622,7 +622,9 @@ class APlatformPageNamesTheFileItTranscribes(unittest.TestCase):
             return generate_site.generate_platform_page("p", cov)
 
         tag = "https://raw.githubusercontent.com/o/r/batocera-43.1/x/batocera-systems"
-        self.assertIn(f"| Transcribed from | <{tag}> |", page(tag))
+        self.assertIn(
+            "| Transcribed from | [batocera-systems at tag batocera-43.1]"
+            "(https://github.com/o/r/blob/batocera-43.1/x/batocera-systems) |", page(tag))
         branch = "https://raw.githubusercontent.com/o/r/master/x.json"
         self.assertIn("revision read is not recorded", page(branch))
         self.assertIn("revision read is not recorded", page("https://github.com/RetroDECK/components"))

tests/test_site_platform_hashes.py

@@ -10,11 +10,13 @@ from __future__ import annotations
 
 import sys
 import unittest
+from unittest import mock
 from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(REPO_ROOT / "scripts"))
 
+import siterender  # noqa: E402
 from generate_site import _render_platform_systems  # noqa: E402
 
 A, B, C = "a" * 32, "b" * 32, "c" * 32
@@ -36,7 +38,9 @@ class EachRowItsDeclaration(unittest.TestCase):
                 {"name": "boot.rom", "status": "ok", "system": "saturn"},
             ],
         }
-        text = "\n".join(_render_platform_systems(by_system, config_files, {}))
+        names = {"psx": "Sony PlayStation", "saturn": "Sega Saturn"}
+        with mock.patch.dict(siterender.SYSTEM_NAMES, names):
+            text = "\n".join(_render_platform_systems(by_system, config_files, {}))
         psx, saturn = text.split("saturn", 1)[0], text.split("saturn", 1)[1]
         self.assertIn(A, psx)
         self.assertNotIn(B, psx)

tests/test_site_r2_reference.py

@@ -0,0 +1,357 @@
+"""Reference pages (round 2, package 8).
+
+A reference page cites a hash, a source line or a count, and each citation
+has to hold when an expert checks it: the reader of a pinned dump is the
+profile entry accepting its hash, a source line is a pinned link, an entry
+names the file that serves it, and the exports carry what the page shows.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import re
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_site  # noqa: E402
+import siterender  # noqa: E402
+from common import (  # noqa: E402
+    list_registered_platforms,
+    load_emulator_profiles,
+    load_platform_config,
+)
+from exporter.baseline import HASH_FIELDS, _hash_values  # noqa: E402
+from exporter.candidates import EntryCategories  # noqa: E402
+from sitesources import launcher_ref_markdown, list_source  # noqa: E402
+
+PLATFORMS = str(REPO_ROOT / "platforms")
+DIST = REPO_ROOT / "website" / "dist"
+
+
+def _accepted(profile_entry: dict) -> set[str]:
+    values = {v for f in HASH_FIELDS for v in _hash_values(profile_entry, f)}
+    for member in profile_entry.get("contents") or []:
+        if isinstance(member, dict):
+            values |= {v for f in HASH_FIELDS for v in _hash_values(member, f)}
+    return values
+
+
+class ReaderMatchedByHash(unittest.TestCase):
+    """Batocera's scph1001.bin (md5 dc2b9bf8, SCPH-1001 v2.0) cited
+    DuckStation's bios.cpp:36, the line of the v2.2 dump: the reader was
+    matched by name."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
+
+    def test_batocera_scph1001_cites_the_v20_line(self):
+        config = load_platform_config("batocera", PLATFORMS)
+        cats = EntryCategories(config, self.profiles)
+        entry = next(e for e in config["systems"]["sony-playstation"]["files"]
+                     if e["name"] == "scph1001.bin")
+        found = cats.explain(entry, entry.get("destination") or entry["name"],
+                             "sony-playstation")
+        duck = next(r for r in found["read_by"] if r["emulator"] == "duckstation")
+        self.assertEqual(duck["entries"], [
+            {"name": "scph1001_v20.bin", "source_ref": "src/core/bios.cpp:42"}])
+
+    def test_every_cited_entry_accepts_the_md5_or_says_by_name(self):
+        by_name = 0
+        for platform in list_registered_platforms(PLATFORMS, include_archived=True):
+            config = load_platform_config(platform, PLATFORMS)
+            cats = EntryCategories(config, self.profiles)
+            for sys_id, system in (config.get("systems") or {}).items():
+                for entry in system.get("files") or []:
+                    md5s = set(_hash_values(entry, "md5")) | set(_hash_values(entry, "alt_md5"))
+                    if not md5s:
+                        continue
+                    # The same dump, named by whichever hash both sides state.
+                    declared = md5s | {v for f in HASH_FIELDS for v in _hash_values(entry, f)}
+                    found = cats.explain(entry, entry.get("destination") or entry["name"], sys_id)
+                    for reader in found.get("read_by", []):
+                        if reader["emulator"] == "launcher":
+                            continue
+                        files = self.profiles[reader["emulator"]].get("files") or []
+                        for cited in reader["entries"]:
+                            if cited.get("by") == "name":
+                                by_name += 1
+                                continue
+                            accepted = set().union(*(
+                                _accepted(f) for f in files
+                                if isinstance(f, dict) and f.get("name") == cited["name"]))
+                            with self.subTest(platform=platform, entry=entry["name"],
+                                              emulator=reader["emulator"]):
+                                self.assertTrue(any(
+                                    a == m or (29 <= len(m) < 32 and a.startswith(m))
+                                    for m in declared for a in accepted))
+        self.assertGreater(by_name, 0)
+
+    def test_a_contradicting_homonym_is_not_cited(self):
+        profiles = {"core": {"type": "libretro", "systems": ["s"], "files": [
+            {"name": "bios.bin", "md5": "1" * 32, "source_ref": "a.c:1"},
+            {"name": "bios_v2.bin", "md5": "2" * 32, "source_ref": "a.c:2"},
+        ]}}
+        config = {"cores": ["core"], "systems": {"s": {"files": []}}}
+        cats = EntryCategories(config, profiles)
+        found = cats.explain({"name": "bios.bin", "md5": "2" * 32}, "bios.bin", "s")
+        self.assertEqual(found["read_by"][0]["entries"],
+                         [{"name": "bios_v2.bin", "source_ref": "a.c:2"}])
+        found = cats.explain({"name": "bios.bin", "md5": "3" * 32}, "bios.bin", "s")
+        self.assertEqual(found["read_by"][0]["entries"], [])
+
+
+class EvidenceLinksItsSource(unittest.TestCase):
+    """Platform pages printed refs as code with the role word in front:
+    `source frontend/libretro.c:3864-3867`."""
+
+    def test_pinned_link_and_by_name(self):
+        evidence = {"read_by": [{
+            "emulator": "pcsx_rearmed", "proof": "name", "runs_system": True,
+            "entries": [{"name": "scph1001.bin", "by": "name",
+                         "source_ref": "source frontend/libretro.c:3864-3867 (listed_bios[6])"}],
+        }]}
+        text = generate_site._category_evidence_md(evidence)
+        self.assertRegex(text, r"\[`frontend/libretro\.c:3864-3867`\]\(https://github\.com/"
+                               r"libretro/pcsx_rearmed/blob/[0-9a-f]{40}/frontend/libretro\.c"
+                               r"#L3864-L3867\)")
+        self.assertIn("(by name)", text)
+        self.assertNotIn("`source ", text)
+
+    def test_launcher_record_named_with_its_lines(self):
+        evidence = {"read_by": [{"emulator": "launcher", "proof": "launcher",
+                                 "entries": [], "runs_system": True}]}
+        config = load_platform_config("batocera", PLATFORMS)
+        launched = generate_site._launcher_refs("batocera", config)["scph1001.bin"]
+        text = generate_site._category_evidence_md(evidence, launched)
+        self.assertNotIn("(by launcher)", text)
+        self.assertIn("https://github.com/batocera-linux/batocera.linux/blob/batocera-43.1/"
+                      "package/batocera/core/batocera-configgen/", text)
+        self.assertIn("duckstationGenerator.py:115-136", text)
+
+    def test_a_launcher_in_another_repository_is_not_linked(self):
+        source = list_source("https://gitlab.com/recalbox/recalbox/-/raw/10.1.1/x/es_bios.xml")
+        self.assertEqual(launcher_ref_markdown(None, "a/b.py:3"), "`b.py:3`")
+        self.assertIn("gitlab.com/recalbox/recalbox/-/blob/10.1.1/a/b.py#L3",
+                      launcher_ref_markdown(source, "a/b.py:3"))
+
+
+class EachEntryNamesItsFile(unittest.TestCase):
+    def test_served_held_and_missing(self):
+        path, digest = next(
+            (p, d) for p, d in generate_site._collection_sha1_by_path().items()
+            if p.startswith("bios/") and p.count("/") >= 3)
+        served = generate_site._served_line({"status": "ok", "path": path})
+        self.assertRegex(served, rf"\]\(\.\./systems/[^)]+\.md#sha1-{digest[:12]}\)")
+        self.assertTrue(served.startswith("Served by: "))
+        self.assertIn(f"`{digest[:8]}`", served)
+        self.assertEqual(generate_site._served_line({"status": "missing"}), "Not held")
+        self.assertTrue(generate_site._served_line(
+            {"status": "untested", "path": path}).startswith("Held, not the declared hash"))
+        # Existence mode reports no path: nothing is claimed, held or not.
+        self.assertEqual(generate_site._served_line({"status": "ok"}), "")
+
+    def test_a_long_system_list_is_cut_between_files(self):
+        body = ["2/2 files verified", ""]
+        for n in range(4):
+            body += [f"- `f{n}.bin` - OK", "    " + "x" * 100]
+        runs = generate_site._file_runs(body, 250)
+        self.assertEqual(len(runs), 2)
+        for run in runs:
+            self.assertEqual(run[:2], body[:2])
+            self.assertTrue(run[2].startswith("- "))
+        self.assertEqual(sum(len(run) - 2 for run in runs), len(body) - 2)
+
+
+class AlsoNamedIsNotCited(unittest.TestCase):
+    def test_other_system_cores_are_named_only(self):
+        evidence = {"read_by": [
+            {"emulator": "duckstation", "proof": "name", "runs_system": True,
+             "entries": [{"name": "scph1001_v20.bin", "source_ref": "src/core/bios.cpp:42"}]},
+            {"emulator": "mame", "proof": "name", "runs_system": False,
+             "entries": [{"name": "x.zip", "source_ref": "src/mame/x.cpp:1", "by": "name"}]},
+        ]}
+        text = generate_site._category_evidence_md(evidence)
+        also = text.split(", also named by ", 1)[1]
+        self.assertNotIn("x.cpp", also)
+        self.assertIn("mame", also)
+
+
+class PlatformReferenceInWords(unittest.TestCase):
+    def _page(self, mode: str, hash_type: str, systems: dict | None = None,
+              registry: dict | None = None) -> str:
+        details = [{"name": "f.bin", "status": "ok", "system": s} for s in (systems or {})]
+        cov = {"platform": "P", "details": details, "total": 1, "verified": 1,
+               "present": 1, "missing": 0, "untested": 0, "mode": mode,
+               "config": {"hash_type": hash_type, "systems": systems or {},
+                          "source": "https://github.com/o/r/tree/" + "a" * 40}}
+        return generate_site.generate_platform_page("p", cov, registry or {})
+
+    def test_hash_type_only_when_it_differs(self):
+        self.assertNotIn("Hash type", self._page("md5", "md5"))
+        self.assertNotIn("Hashes in the list", self._page("md5", "md5"))
+        page = self._page("existence", "sha1")
+        self.assertIn("| Hashes in the list | SHA-1 |", page)
+        self.assertIn(">name only</span>", page)
+        self.assertIn(">MD5</span>", self._page("md5", "md5"))
+        self.assertIn("[r repository at commit aaaaaaaaaaaa](https://github.com/o/r/tree/"
+                      + "a" * 40 + ")", page)
+
+    def test_systems_by_name_with_the_id_in_small_text(self):
+        systems = {"sony-playstation": {"files": [{"name": "f.bin"}]}}
+        page = self._page("md5", "md5", systems)
+        name = siterender.system_name("sony-playstation")
+        self.assertIn(f"| [{name}](#sony-playstation) <small>sony-playstation</small> |", page)
+        heading = next(line for line in page.splitlines() if line.startswith("### ")
+                       and "sony-playstation" in line)
+        self.assertIn(name, heading)
+        self.assertTrue(heading.endswith("<small>sony-playstation</small> { #sony-playstation }"))
+
+    def test_logo_in_front_matter_not_floated(self):
+        page = self._page("md5", "md5", registry={"p": {"logo": "https://x.test/l.png"}})
+        self.assertTrue(page.startswith('---\nlogo: "https://x.test/l.png"\n---\n'))
+        self.assertNotIn('align="right"', page)
+
+
+class GapsPageNumbers(unittest.TestCase):
+    def test_badge_only_for_missing_required_firmware(self):
+        self.assertNotIn("rb-badge", generate_site._ratio_cell(99, 100))
+        self.assertIn('class="rb-bar"', generate_site._ratio_cell(99, 100))
+        self.assertIn("rb-badge-danger", generate_site._ratio_cell(99, 100, 1))
+        cov = {"details": [
+            {"name": "a", "system": "s", "status": "missing", "required": True, "category": "bios"},
+            {"name": "b", "system": "s", "status": "missing", "required": True,
+             "category": "game_data"},
+            {"name": "c", "system": "s", "status": "missing", "required": False},
+        ]}
+        self.assertEqual(generate_site._missing_required_firmware(cov), 1)
+
+    def test_headings_and_header(self):
+        source = Path(generate_site.__file__).read_text(encoding="utf-8")
+        body = source[source.index("def generate_gap_analysis("):]
+        body = body[:body.index("\ndef ", 10)]
+        self.assertIn('"# What the packs do not hold"', body)
+        self.assertIn("On its BIOS list (destinations)", body)
+        self.assertIn('"### Files to source"', body)
+        self.assertNotIn("### Files to source (", body)
+
+
+class UpstreamsGrouped(unittest.TestCase):
+    def test_mame_trees_are_one_project(self):
+        keys = {generate_site.upstream_project(u)[0] for u in (
+            "https://github.com/mamedev/mame",
+            "https://github.com/mamedev/mame/tree/mame0289",
+            "https://github.com/mamedev/mame/tree/mame0135u4",
+        )}
+        self.assertEqual(keys, {"github.com/mamedev/mame"})
+        self.assertEqual(generate_site.upstream_project(
+            "https://github.com/mamedev/mame/tree/mame0289")[1], "mame0289")
+
+    def test_nested_groups_notes_and_archives(self):
+        project = generate_site.upstream_project
+        self.assertEqual(
+            project("https://gitlab.com/recalbox/packages/recalbox-cores/libretro-exel-hence"),
+            ("gitlab.com/recalbox/packages/recalbox-cores/libretro-exel-hence", ""))
+        self.assertEqual(project("https://gitlab.com/jgemu/geolith/-/tree/v1"),
+                         ("gitlab.com/jgemu/geolith", "v1"))
+        self.assertEqual(
+            project("https://git.9front.org/plan9front/plan9front (sys/src/games/nes/)"),
+            ("git.9front.org/plan9front/plan9front", "sys/src/games/nes/"))
+        key, _ = project("https://archive.softwareheritage.org/browse/revision/"
+                         "dc20a615275f66bee20a4fd851ef0231daca4f14/")
+        self.assertEqual(generate_site._upstream_label(key),
+                         "Software Heritage revision dc20a615275f")
+
+    def test_overview_and_a_page_per_platform(self):
+        profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
+        coverages = {}
+        for name in ("batocera", "retroarch"):
+            config = load_platform_config(name, PLATFORMS)
+            coverages[name] = {"platform": config.get("platform", name), "config": config}
+        moved: dict[str, str] = {}
+        page = generate_site.generate_cross_reference(coverages, profiles, moved)
+        self.assertEqual(set(moved), {"cross-reference-batocera", "cross-reference-retroarch"})
+        self.assertNotIn("<details", page)
+        upstream = page[page.index("## By upstream emulator"):]
+        self.assertEqual(len(re.findall(r"\| \[mamedev/<wbr>mame\]\(", upstream)), 1)
+        header = next(line for line in upstream.splitlines() if line.startswith("| Cores"))
+        self.assertTrue(header.startswith("| Cores | Upstream |"))
+
+
+class ExportsComplete(unittest.TestCase):
+    def test_gap_rows_once_with_their_outcome(self):
+        path = next(iter(generate_site._collection_sha1_by_path()))
+        detail = {"name": "x.bin", "system": "s", "status": "untested", "path": path,
+                  "category": "bios"}
+        coverages = {"p": {"platform": "P", "details": [
+            {"name": "h.xml", "system": "s", "status": "missing"},
+            {"name": "h.xml", "system": "s", "status": "missing"}, detail]}}
+        db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+        rows = generate_site._gap_export_rows(coverages, {}, db)
+        self.assertEqual([r["name"] for r in rows], ["h.xml", "x.bin"])
+        held = rows[1]
+        self.assertEqual(held["collection_path"], path)
+        self.assertEqual(held["resolved_sha1"], generate_site._collection_sha1_by_path()[path])
+        self.assertEqual(held["category"], "bios")
+
+    def test_platform_rows_carry_status_and_file(self):
+        db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+        sha1, entry = next(iter(db["files"].items()))
+        coverages = {"p": {"platform": "P", "config": {"systems": {"s": {"files": [
+            {"name": "a.bin"}, {"name": "b.bin"}]}}}, "details": [
+            {"name": "a.bin", "system": "s", "status": "ok", "path": entry["path"],
+             "category": "bios"},
+            {"name": "b.bin", "system": "s", "status": "missing", "category": "game_data"}]}}
+        _items, rows = generate_site._platform_export_rows(coverages, db)
+        self.assertEqual((rows[0]["status"], rows[0]["resolved_sha1"]), ("ok", sha1))
+        self.assertEqual(rows[0]["collection_path"], entry["path"])
+        self.assertEqual((rows[1]["status"], rows[1]["category"], rows[1]["collection_path"]),
+                         ("missing", "game_data", ""))
+
+    def test_commit_and_wanted_list(self):
+        commit = generate_site._source_commit()
+        self.assertRegex(commit, r"^[0-9a-f]{40}$")
+        schema = json.loads((REPO_ROOT / "schemas" / "site-api-envelope.schema.json")
+                            .read_text(encoding="utf-8"))
+        self.assertIn("commit", schema["properties"])
+        db = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+        rows = generate_site._provenance_wanted_rows(db)
+        self.assertTrue(rows)
+        self.assertTrue(all(r["catalog"] and r["name"] for r in rows))
+        text = generate_site._csv_document(list(generate_site.WANTED_FIELDS), rows)
+        self.assertEqual(next(csv.reader(io.StringIO(text))), list(generate_site.WANTED_FIELDS))
+
+
+class DatesAndNames(unittest.TestCase):
+    def test_snapshot_date_in_words(self):
+        self.assertEqual(generate_site._date_in_words("2026-08-07T10:00:00Z"),
+                         '<time datetime="2026-08-07">7 August 2026</time>')
+
+    def test_data_page_title(self):
+        self.assertTrue(generate_site.generate_data_page([]).startswith("# Data and API\n"))
+
+
+@unittest.skipUnless((DIST / "platforms" / "batocera" / "index.html").is_file(),
+                     "site not built")
+class BuiltBatoceraPage(unittest.TestCase):
+    def test_served_by_and_pinned_links(self):
+        main = (DIST / "platforms" / "batocera" / "index.html").read_text(encoding="utf-8")
+        self.assertTrue("batocera-systems at tag batocera-43.1" in main)
+        # PlayStation's list is long enough to sit on a page of its own.
+        pages = [main] + [p.read_text(encoding="utf-8") for p in
+                          (DIST / "platforms").glob("batocera-sony-playstation*/index.html")]
+        html = "".join(pages)
+        for text in ("Served by:", "scph1001_v20.bin", "https://github.com/stenzek/duckstation/blob/"):
+            with self.subTest(text=text):
+                self.assertTrue(text in html)
+        self.assertFalse("(by launcher)" in html)
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_site_render.py

@@ -312,7 +312,9 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
 
         by_system = {f"system-{i:02d}": [{"name": f"f{i}.bin", "status": "ok"}]
                      for i in range(11)}
-        lines = generate_site._render_platform_systems(by_system, {}, {})
+        names = {sys_id: f"System {sys_id[-2:]}" for sys_id in by_system}
+        with mock.patch.dict(siterender.SYSTEM_NAMES, names):
+            lines = generate_site._render_platform_systems(by_system, {}, {})
         headings = [line for line in lines if line.startswith("### ")]
         self.assertEqual(len(headings), 11)
         self.assertTrue(headings[0].endswith("{ #system-00 }"))

tests/test_site_validation.py

@@ -372,8 +372,12 @@ class PlatformPagesPrintTheDeclaredCrc32(unittest.TestCase):
         from tests import sitepages  # noqa: PLC0415
 
         sitepages.require()
-        page = sitepages.page("platforms/retroarch.md")
-        self.assertIn("CRC32: `91e93107`", page.read_text(encoding="utf-8"))
+        # The page and the systems moved to pages of their own.
+        text = "".join(
+            path.read_text(encoding="utf-8") for path in sitepages.pages()
+            if sitepages.name(path) == "platforms/retroarch.md"
+            or sitepages.name(path).startswith("platforms/retroarch-"))
+        self.assertIn("CRC32: `91e93107`", text)
 
 
 class PlainHouseStyle(unittest.TestCase):

wiki/testing-guide.md

@@ -55,7 +55,7 @@ Every module in `tests/`, by the area it guards.
 | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
 | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
-| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
 
 ## Test architecture