feat: keep the address of a file that left

files/?sha1= of a file the collection no longer holds answered that no
file the site knows has the hash. generate_db.py now records each sha1 a
rebuild drops in retired.json, with the day and the file in its place,
backfilled from the history since 0d586ef6b (4,139 files). The lookup
answers with it and leads to the successor's card.

.github/workflows/deploy-site.yml

@@ -53,6 +53,7 @@ on:
       - "scripts/sitemeasurements.py"
       - "scripts/siteexports.py"
       - "scripts/siterender.py"
+      - "scripts/retired_files.py"
       - "scripts/sitesources.py"
       - "scripts/sitestarlight.py"
       - "scripts/sitewrite.py"
@@ -79,6 +80,7 @@ on:
       - "database.json"
       - "release.json"
       - "derivable.json"
+      - "retired.json"
       - "measurements/**"
       # The Proof lines of the emulator pages read what an issue's runs showed.
       - "tests/campaign/observations.yml"

.github/workflows/validate.yml

@@ -29,6 +29,7 @@ on:
       - "database.json"
       - "release.json"
       - "derivable.json"
+      - "retired.json"
       - "measurements/**"
       - "website/**"
   # Spelled out twice because the workflow parser reads no YAML anchor. A test
@@ -54,6 +55,7 @@ on:
       - "database.json"
       - "release.json"
       - "derivable.json"
+      - "retired.json"
       - "measurements/**"
       - "website/**"
 

automation/jobs.yml

@@ -87,7 +87,7 @@ jobs:
         body: >-
           Regenerated by the offline pipeline, database.json and the
           manifests computed twice with the same result.
-        paths: [database.json, derivable.json, install/*.json, install/targets/*.json,
+        paths: [database.json, derivable.json, retired.json, install/*.json, install/targets/*.json,
                 install/games/*.json, install/games/*/*.json, README.md, CONTRIBUTING.md,
                 wiki/installer.md, website/src/data/*.json]
     scratch: [dist, tmp/autorun]

schemas/site-api-envelope.schema.json

@@ -390,11 +390,22 @@
                   "enum": [
                     "collection",
                     "archive-member",
+                    "retired",
                     "data-directory",
                     "platform",
                     "emulator"
                   ]
                 },
+                "left": {
+                  "type": "string",
+                  "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
+                  "description": "On a retired record: the day the file, whose SHA1 is `sha1`, left the collection (retired.json)."
+                },
+                "successor": {
+                  "type": "string",
+                  "pattern": "^[0-9a-f]{40}$",
+                  "description": "On a retired record: SHA1 of the collection file now holding its place, whose card `url` leads to."
+                },
                 "archive": {
                   "type": "string",
                   "description": "On an archive-member record: the name of the collection archive holding the member, whose SHA1 is `sha1`. Its value is the member's CRC32, read from the archive's central directory."

scripts/generate_db.py

@@ -34,6 +34,7 @@ from common import (
     write_text_atomic,
     yaml_load,
 )
+import retired_files
 
 # Platform and profile aliases are read from YAML: without pyyaml the
 # database lost every one of them and the run still exited 0.
@@ -484,6 +485,19 @@ def main() -> int:
         "indexes": indexes,
     }
 
+    # A sha1 the rebuild drops keeps its published address: the record
+    # beside the database says when it left and what holds its place.
+    output = Path(args.output)
+    if output.is_file():
+        previous = json.loads(output.read_text(encoding="utf-8"))
+        left = retired_files.update(
+            retired_files.retired_path(output), previous, database,
+            database["generated_at"][:10],
+        )
+        if left:
+            print(f"  Left the collection: {left} files, recorded in "
+                  f"{retired_files.retired_path(output)}")
+
     new_content = json.dumps(database, indent=2)
     written = write_if_changed(args.output, new_content)
 

scripts/generate_site.py

@@ -89,6 +89,7 @@ from generate_readme import (
 from provenance_report import build_report, distinct_missing
 import sitemarkup as markup
 import siteentries
+import retired_files
 import sitestarlight
 from siteexports import generate_exports_page, load_export_reports
 from siteguides import (
@@ -1618,7 +1619,8 @@ def generate_data_exports(
         "file-locations.json": ("file-locations", locations),
     }
     write_lookup_shards(
-        api, file_lookup_records(db, coverages, profiles, locations, members, consumers),
+        api, file_lookup_records(db, coverages, profiles, locations, members, consumers,
+                                 retired_files.load(_REPO_ROOT / "retired.json")),
         generated_at, commit,
     )
     for filename, (kind, items) in envelopes.items():
@@ -2201,8 +2203,9 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         "One hash without the whole list: `api/v1/lookup/<xx>.json`, where `<xx>` "
         "is the first two characters of the hash in lowercase, holds every SHA1, "
         "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` "
-        "(the collection, a member of a collection archive, a data directory, a "
-        "platform's list or an emulator profile), the file `name`, the `url` of the page that shows it and, "
+        "(the collection, a member of a collection archive, a file that left the "
+        "collection, a data directory, a platform's list or an emulator profile), "
+        "the file `name`, the `url` of the page that shows it and, "
         "when one is known, the `system_name` the site gives its system. "
         "[Find or check a file](files/index.md) reads it.",
         "",
@@ -2226,7 +2229,7 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         "  const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);",
         "  if (!response.ok) return [];",
         "  const shard = await response.json();",
-        "  // layer: collection, archive-member, data-directory, platform or emulator.",
+        "  // layer: collection, archive-member, retired, data-directory, platform, emulator.",
         "  return shard.items.filter((item) => item.value === sha1);",
         "}",
         "```",
@@ -4734,7 +4737,9 @@ def generate_files_page() -> str:
         "A file's card lives on its manufacturer's page, and a file refiled "
         "between manufacturers moves with it. `files/?sha1=<sha1>` is the address "
         "that does not move: it opens the card wherever it is now. The link icon "
-        "beside each card's name is that address.",
+        "beside each card's name is that address. A file that left the "
+        "collection keeps it: the page says when, and opens the card of the "
+        "file in its place.",
         "",
         "## The index behind it",
         "",
@@ -4924,6 +4929,39 @@ def _data_member_digests(
     return found
 
 
+def retired_records(retired: dict[str, dict], files: dict, locations: list[dict]) -> list[dict]:
+    """A record for each hash of a file that left the collection.
+
+    Its address (files/?sha1=, #sha1-<12>) was published and answered that
+    no file the site knows has the hash, inviting a contribution of a file
+    the collection held. The record says when it left, and leads to the
+    card of the file holding its place, followed to the one held today.
+    """
+    cards = {item["sha1"]: item["url"] for item in locations}
+    records = []
+    for sha1, record in sorted(retired.items()):
+        if sha1 in files:
+            continue
+        successor, seen = record.get("successor"), {sha1}
+        while successor and successor not in files and successor not in seen:
+            seen.add(successor)
+            successor = (retired.get(successor) or {}).get("successor")
+        url = cards.get(successor) if successor in files else None
+        extra = {"left": str(record["left"])}
+        if url:
+            extra["successor"] = successor
+        else:
+            # The page its card was on: the reader lands among its neighbours.
+            url = _site_path(f"systems/{card_page(str(record.get('path', '')), sha1)}.md")
+        for algorithm in HASH_ALGORITHMS:
+            value = sha1 if algorithm == "sha1" else record.get(algorithm)
+            for one in _hash_values(value):
+                records.append({"value": one, "algorithm": algorithm, "layer": "retired",
+                                "name": str(record.get("name", "")), "sha1": sha1,
+                                "url": url, **extra})
+    return records
+
+
 def _archive_member_records(entry: dict, sha1: str, url: str,
                             root: Path = _REPO_ROOT) -> list[dict]:
     """The CRC32 of each member of a collection archive, read from its
@@ -4956,6 +4994,7 @@ def file_lookup_records(
     locations: list[dict],
     members: dict | None = None,
     consumers: FileConsumers | None = None,
+    retired: dict | None = None,
 ) -> list[dict]:
     """Every hash the site knows, one record per value, for files/?hash=.
 
@@ -5033,6 +5072,7 @@ def file_lookup_records(
                             "owner": key, "name": name, "url": url,
                             **({"system_name": entry_label} if entry_label else {}),
                         })
+    records += retired_records(retired or {}, files, locations)
     records += _held_under_every_hash(records, files, db.get("indexes") or {})
     for record in records:
         if record["layer"] in ("platform", "emulator"):

scripts/retired_files.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Files that left the collection, and the file that took their place.
+
+A file's content address (files/?sha1=, #sha1-<12> on its card) is
+published, and a published address never stops answering. When a rebuild
+of database.json drops a sha1, the file it named is recorded here: its
+name, its last path, its hashes, the day it left and the sha1 the
+collection now holds at that path, or under that name in that folder.
+The site's lookup reads the record and answers with it.
+
+The record is append-only, written by generate_db.py beside the database.
+A sha1 that comes back leaves it. `--backfill` rebuilds it from the
+history of database.json since content addresses were published.
+
+Usage:
+    python scripts/retired_files.py --backfill
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+RETIRED_FILE = REPO_ROOT / "retired.json"
+# The commit that published content addresses: 0d586ef6b, 2026-10-19.
+FIRST_ADDRESSED = "0d586ef6b"
+KEPT_FIELDS = ("name", "path", "size", "md5", "crc32", "sha256")
+
+
+def retired_path(database: str | Path) -> Path:
+    """The record kept beside a database file."""
+    return Path(database).with_name("retired.json")
+
+
+def _successor(entry: dict, by_path: dict[str, str],
+               by_name: dict[str, list[tuple[str, str]]]) -> str | None:
+    """The sha1 the new database holds at the entry's path, else under its
+    name in the same folder, else the one file of that name: a2diskiing.zip
+    was rebuilt from its recipe and filed under Apple II the same day."""
+    path = str(entry.get("path", ""))
+    if path in by_path:
+        return by_path[path]
+    named = by_name.get(str(entry.get("name", "")), [])
+    folder = path.rsplit("/", 1)[0]
+    beside = [sha1 for sha1, other in named if other.rsplit("/", 1)[0] == folder]
+    for found in (beside, [sha1 for sha1, _other in named]):
+        if len(found) == 1:
+            return found[0]
+    return None
+
+
+def retired_between(old: dict, new: dict, day: str) -> dict[str, dict]:
+    """sha1 -> record, for each file of *old* that *new* no longer holds."""
+    old_files = old.get("files") or {}
+    new_files = new.get("files") or {}
+    by_path = {str(e.get("path", "")): sha1 for sha1, e in new_files.items()}
+    by_name: dict[str, list[tuple[str, str]]] = {}
+    for sha1, e in new_files.items():
+        by_name.setdefault(str(e.get("name", "")), []).append((sha1, str(e.get("path", ""))))
+    records = {}
+    for sha1 in sorted(set(old_files) - set(new_files)):
+        entry = old_files[sha1]
+        record = {key: entry[key] for key in KEPT_FIELDS if entry.get(key) not in (None, "")}
+        record["left"] = day
+        successor = _successor(entry, by_path, by_name)
+        if successor:
+            record["successor"] = successor
+        records[sha1] = record
+    return records
+
+
+def merge(record: dict[str, dict], left: dict[str, dict], new: dict) -> dict[str, dict]:
+    """The record with *left* added, first departure kept, and every sha1
+    the new database holds again taken out."""
+    held = set(new.get("files") or {})
+    merged = {sha1: entry for sha1, entry in record.items() if sha1 not in held}
+    for sha1, entry in left.items():
+        merged.setdefault(sha1, entry)
+    return dict(sorted(merged.items()))
+
+
+def load(path: Path) -> dict[str, dict]:
+    if not path.is_file():
+        return {}
+    return json.loads(path.read_text(encoding="utf-8")).get("files") or {}
+
+
+def dump(records: dict[str, dict]) -> str:
+    return json.dumps({"schema_version": 1, "files": records}, indent=1, sort_keys=True) + "\n"
+
+
+def update(path: Path, old: dict, new: dict, day: str) -> int:
+    """Records what *new* dropped from *old*. Returns how many left now."""
+    from common import write_if_changed  # noqa: PLC0415
+
+    left = retired_between(old, new, day)
+    record = load(path)
+    merged = merge(record, left, new)
+    if merged != record:
+        write_if_changed(str(path), dump(merged))
+    return len(set(left) - set(record))
+
+
+def _git(*args: str) -> str:
+    return subprocess.run(["git", *args], cwd=REPO_ROOT, capture_output=True, text=True,
+                          check=True).stdout
+
+
+def backfill() -> dict[str, dict]:
+    """Every sha1 that left database.json in the commits since content
+    addresses were published, with the day of the commit that dropped it."""
+    revisions = [line.split() for line in _git(
+        "log", "--reverse", "--format=%H %cs", f"{FIRST_ADDRESSED}^..HEAD", "--", "database.json",
+    ).splitlines() if line.strip()]
+    records: dict[str, dict] = {}
+    previous: dict | None = None
+    for commit, day in revisions:
+        current = json.loads(_git("show", f"{commit}:database.json"))
+        if previous is not None:
+            records = merge(records, retired_between(previous, current, day), current)
+        previous = current
+    return records
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
+    parser.add_argument("--backfill", action="store_true",
+                        help="rebuild the record from the history of database.json")
+    parser.add_argument("--output", default=str(RETIRED_FILE),
+                        help="the record to write (default: retired.json)")
+    args = parser.parse_args()
+    if not args.backfill:
+        parser.print_help()
+        return 2
+    sys.path.insert(0, str(Path(__file__).parent))
+    from common import write_if_changed  # noqa: PLC0415
+
+    records = backfill()
+    current = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+    records = merge(records, {}, current)
+    write_if_changed(args.output, dump(records))
+    print(f"{len(records)} files left the collection since {FIRST_ADDRESSED}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_retired_files.py

@@ -0,0 +1,103 @@
+"""A file that leaves the collection keeps its published address.
+
+a2diskiing.zip (bd02f6a0) was rebuilt from its recipe on 2026-10-22 and
+filed under Apple II as d9bb1897. Its card's address, files/?sha1=bd02f6a0,
+answered that no file the site knows has the hash and asked for a
+contribution of a file the collection had held.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import retired_files  # noqa: E402
+
+OLD = "bd" * 20
+NEW = "d9" * 20
+
+
+def _db(**files: dict) -> dict:
+    return {"files": files}
+
+
+class WhatLeftIsRecorded(unittest.TestCase):
+    def setUp(self):
+        self.old = _db(**{
+            OLD: {"name": "a2diskiing.zip", "path": "bios/Arcade/Arcade/a2diskiing.zip",
+                  "md5": "43" * 16, "crc32": "292ef826", "size": 589},
+            "11" * 20: {"name": "kept.bin", "path": "bios/A/B/kept.bin"},
+        })
+        self.new = _db(**{
+            NEW: {"name": "a2diskiing.zip", "path": "bios/Apple/Apple II/a2diskiing.zip"},
+            "11" * 20: {"name": "kept.bin", "path": "bios/A/B/kept.bin"},
+        })
+
+    def test_the_departure_and_the_file_in_its_place(self):
+        left = retired_files.retired_between(self.old, self.new, "2026-10-22")
+        self.assertEqual(list(left), [OLD])
+        self.assertEqual(left[OLD]["left"], "2026-10-22")
+        self.assertEqual(left[OLD]["successor"], NEW)
+        self.assertEqual(left[OLD]["md5"], "43" * 16)
+
+    def test_the_first_departure_stays_and_a_return_leaves_the_record(self):
+        first = retired_files.merge({}, retired_files.retired_between(self.old, self.new, "2026-10-22"),
+                                    self.new)
+        again = retired_files.merge(first, {OLD: {"left": "2026-11-01"}}, self.new)
+        self.assertEqual(again[OLD]["left"], "2026-10-22")
+        self.assertEqual(retired_files.merge(again, {}, self.old), {})
+
+    def test_generate_db_writes_it_beside_the_database(self):
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+            database = Path(tmp) / "database.json"
+            path = retired_files.retired_path(database)
+            self.assertEqual(path, Path(tmp) / "retired.json")
+            self.assertEqual(retired_files.update(path, self.old, self.new, "2026-10-22"), 1)
+            self.assertEqual(retired_files.update(path, self.old, self.new, "2026-10-23"), 0)
+            record = json.loads(path.read_text())["files"]
+            self.assertEqual(record[OLD]["left"], "2026-10-22")
+        source = (REPO_ROOT / "scripts" / "generate_db.py").read_text(encoding="utf-8")
+        self.assertIn("retired_files.update(", source)
+
+
+class TheLookupAnswersForIt(unittest.TestCase):
+    def test_records_lead_to_the_card_in_its_place(self):
+        import generate_site  # noqa: PLC0415
+
+        retired = {OLD: {"name": "a2diskiing.zip", "path": "bios/Arcade/Arcade/a2diskiing.zip",
+                         "crc32": "292ef826", "left": "2026-10-22", "successor": "ee" * 20},
+                   "ee" * 20: {"name": "a2diskiing.zip", "path": "bios/Apple/x/a2diskiing.zip",
+                               "left": "2026-10-23", "successor": NEW}}
+        files = {NEW: {"name": "a2diskiing.zip"}}
+        locations = [{"sha1": NEW, "url": "/retrobios/systems/apple/#sha1-d9d9d9d9d9d9"}]
+        records = generate_site.retired_records(retired, files, locations)
+        found = {(r["sha1"], r["algorithm"]): r for r in records}
+        # Followed to the file held today, through the copy that left after.
+        self.assertEqual(found[(OLD, "sha1")]["url"], locations[0]["url"])
+        self.assertEqual(found[(OLD, "sha1")]["successor"], NEW)
+        self.assertEqual(found[(OLD, "crc32")]["value"], "292ef826")
+        self.assertEqual(found[(OLD, "sha1")]["layer"], "retired")
+        self.assertEqual(found[(OLD, "sha1")]["left"], "2026-10-22")
+
+    def test_the_committed_record_holds_the_reported_file(self):
+        record = retired_files.load(REPO_ROOT / "retired.json")
+        if not record:
+            self.skipTest("retired.json is not written")
+        self.assertIn("bd02f6a0ce8accba7ea5415fd22c2e8f23ec4b10", record)
+        database = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+        self.assertFalse(set(record) & set(database["files"]))
+
+    def test_the_page_script_answers_with_it(self):
+        script = (REPO_ROOT / "website" / "src" / "scripts" / "site.ts").read_text(encoding="utf-8")
+        self.assertIn("item.layer === 'retired'", script)
+        self.assertIn("left the collection on", script)
+
+
+if __name__ == "__main__":
+    unittest.main()

website/src/scripts/site.ts

@@ -23,6 +23,9 @@ type Item = {
 	system_name?: string;
 	// On an archive-member record: the collection archive holding the member.
 	archive?: string;
+	// On a retired record: the day the file left, and the file in its place.
+	left?: string;
+	successor?: string;
 };
 
 const siteRoot = () => new URL(import.meta.env.BASE_URL.replace(/\/?$/, '/'), location.origin);
@@ -525,7 +528,7 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched:
 	const searched = new Set(matched.map((item) => item.value));
 	// A member's record names its archive, which is not the file searched for.
 	const carried = [...new Set(matched.flatMap((item) =>
-		(item.sha1 && item.layer !== 'archive-member' && !searched.has(item.sha1) ? [item.sha1] : [])))];
+		(item.sha1 && !['archive-member', 'retired'].includes(item.layer) && !searched.has(item.sha1) ? [item.sha1] : [])))];
 	const found = carried.length ? [...matched, ...(await lookupValues(carried).catch(() => [] as Item[]))] : matched;
 	const held = distinct(found.filter((item) => item.layer === 'collection'), (item) => item.sha1 ?? item.url);
 	const files = await Promise.all(held.map((item) => heldFile(item).catch(() => undefined)));
@@ -613,7 +616,9 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched:
 			),
 		);
 	});
-	if (!held.length && !members.length) blocks.push(paragraph('No file in the collection has this hash.'));
+	const retired = held.length ? [] : distinct(byLayer('retired'), (item) => item.sha1 ?? item.value);
+	if (!held.length && !members.length && !retired.length) blocks.push(paragraph('No file in the collection has this hash.'));
+	blocks.push(...retired.map(retiredParagraph));
 	if (members.length) {
 		// A ROM the collection holds inside a set: the set's card, by its name.
 		const list = document.createElement('ul');
@@ -657,6 +662,15 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched:
 	result.replaceChildren(...blocks);
 };
 
+// A file the collection held: its address keeps answering, with the day
+// it left and the card of the file in its place.
+const retiredParagraph = (item: Item) =>
+	paragraph(
+		code(item.name || item.value), ` left the collection on ${item.left}. `,
+		item.successor ? link(item.url, 'Open the card of the file in its place') : link(item.url, 'Open the page it was on'),
+		'.',
+	);
+
 const WANTED = 'provenance/#wanted-catalogued-dumps-this-project-does-not-have';
 
 // Nothing the site knows has the value: where a dump of one's own can go.
@@ -766,8 +780,12 @@ const resolveFileAddress = () => {
 			const matches = items.filter((item) =>
 				prefix ? item.algorithm === 'sha1' && item.value.startsWith(value) : item.value === value);
 			const cards = matches.filter((item) => item.layer === 'collection' || item.layer === 'data-directory');
+			const retired = matches.filter((item) => item.layer === 'retired' && item.algorithm === 'sha1');
 			if (new Set(cards.map((item) => item.url)).size === 1) {
 				location.replace(new URL(cards[0].url, siteRoot()));
+			} else if (!cards.length && retired.length) {
+				status.textContent = 'This file is no longer in the collection.';
+				result.replaceChildren(...distinct(retired, (item) => item.value).map(retiredParagraph));
 			} else if (matches.length) {
 				return renderMatches(status, result, prefix ? cards : matches);
 			} else {

wiki/architecture.md

@@ -52,6 +52,7 @@ website/                 Astro Starlight project; pages, data and navigation
 _mame_clones.json        MAME parent/clone set mappings
 database.json            file index built from bios/ (SHA1 primary key)
 derivable.json           files a held dump already contains, from provenance_report.py
+retired.json             files that left the collection, written by generate_db.py
 release.json             files, sizes and assets of each pack of the last release
 allowed_signers          public key that signs SHA256SUMS.txt
 dist/                    generated packs (gitignored)

wiki/testing-guide.md

@@ -65,7 +65,7 @@ Every module in `tests/`, by the area it guards.
 | Pack probe | the cores a platform runs for a system, staging grown to a fixed point, runs mapped to pack and manifest entries, the line and its schema, the probe lock per run, that no page shows its lines. One class builds a real pack narrowed to PlayStation and lays it out both ways, from the archive and with `install.py --dest`, and skips without the collection | `test_pack_probe` |
 | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
-| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` |
+| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` |
 | Games on the card | small DATs, listxml and setup sources, a fake fetcher, fixture install lists, and the committed data against its schemas | `test_arcade_parents`, `test_game_catalogs`, `test_game_headers`, `test_game_regions`, `test_games_index`, `test_romfolders_card`, `test_romfolders_es` |
 
 ## Test architecture
@@ -463,7 +463,7 @@ refactor: extract shared utils to common.py
 
 Files are staged by name, never with `git add -A` or `git add .`, which sweep in
 deletions and work that is not yours. Generated artifacts (`database.json`,
-`derivable.json`, `install/*.json`, `README.md`, `CONTRIBUTING.md`, `exports/`,
+`retired.json`, `derivable.json`, `install/*.json`, `README.md`, `CONTRIBUTING.md`, `exports/`,
 the generated tables and run samples of `wiki/installer.md`) are committed
 with the data they derive from, never alone: a manifest committed before the
 files it points at gives the installer a 404. The run samples count what the