feat: keep the address of a file that left

files/?sha1= of a file the collection no longer holds answered that no
file the site knows has the hash. generate_db.py now records each sha1 a
rebuild drops in retired.json, with the day and the file in its place,
backfilled from the history since 0d586ef6b (4,139 files). The lookup
answers with it and leads to the successor's card.

scripts/generate_db.py

@@ -34,6 +34,7 @@ from common import (
     write_text_atomic,
     yaml_load,
 )
+import retired_files
 
 # Platform and profile aliases are read from YAML: without pyyaml the
 # database lost every one of them and the run still exited 0.
@@ -484,6 +485,19 @@ def main() -> int:
         "indexes": indexes,
     }
 
+    # A sha1 the rebuild drops keeps its published address: the record
+    # beside the database says when it left and what holds its place.
+    output = Path(args.output)
+    if output.is_file():
+        previous = json.loads(output.read_text(encoding="utf-8"))
+        left = retired_files.update(
+            retired_files.retired_path(output), previous, database,
+            database["generated_at"][:10],
+        )
+        if left:
+            print(f"  Left the collection: {left} files, recorded in "
+                  f"{retired_files.retired_path(output)}")
+
     new_content = json.dumps(database, indent=2)
     written = write_if_changed(args.output, new_content)
 

scripts/generate_site.py

@@ -89,6 +89,7 @@ from generate_readme import (
 from provenance_report import build_report, distinct_missing
 import sitemarkup as markup
 import siteentries
+import retired_files
 import sitestarlight
 from siteexports import generate_exports_page, load_export_reports
 from siteguides import (
@@ -1618,7 +1619,8 @@ def generate_data_exports(
         "file-locations.json": ("file-locations", locations),
     }
     write_lookup_shards(
-        api, file_lookup_records(db, coverages, profiles, locations, members, consumers),
+        api, file_lookup_records(db, coverages, profiles, locations, members, consumers,
+                                 retired_files.load(_REPO_ROOT / "retired.json")),
         generated_at, commit,
     )
     for filename, (kind, items) in envelopes.items():
@@ -2201,8 +2203,9 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         "One hash without the whole list: `api/v1/lookup/<xx>.json`, where `<xx>` "
         "is the first two characters of the hash in lowercase, holds every SHA1, "
         "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` "
-        "(the collection, a member of a collection archive, a data directory, a "
-        "platform's list or an emulator profile), the file `name`, the `url` of the page that shows it and, "
+        "(the collection, a member of a collection archive, a file that left the "
+        "collection, a data directory, a platform's list or an emulator profile), "
+        "the file `name`, the `url` of the page that shows it and, "
         "when one is known, the `system_name` the site gives its system. "
         "[Find or check a file](files/index.md) reads it.",
         "",
@@ -2226,7 +2229,7 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         "  const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);",
         "  if (!response.ok) return [];",
         "  const shard = await response.json();",
-        "  // layer: collection, archive-member, data-directory, platform or emulator.",
+        "  // layer: collection, archive-member, retired, data-directory, platform, emulator.",
         "  return shard.items.filter((item) => item.value === sha1);",
         "}",
         "```",
@@ -4734,7 +4737,9 @@ def generate_files_page() -> str:
         "A file's card lives on its manufacturer's page, and a file refiled "
         "between manufacturers moves with it. `files/?sha1=<sha1>` is the address "
         "that does not move: it opens the card wherever it is now. The link icon "
-        "beside each card's name is that address.",
+        "beside each card's name is that address. A file that left the "
+        "collection keeps it: the page says when, and opens the card of the "
+        "file in its place.",
         "",
         "## The index behind it",
         "",
@@ -4924,6 +4929,39 @@ def _data_member_digests(
     return found
 
 
+def retired_records(retired: dict[str, dict], files: dict, locations: list[dict]) -> list[dict]:
+    """A record for each hash of a file that left the collection.
+
+    Its address (files/?sha1=, #sha1-<12>) was published and answered that
+    no file the site knows has the hash, inviting a contribution of a file
+    the collection held. The record says when it left, and leads to the
+    card of the file holding its place, followed to the one held today.
+    """
+    cards = {item["sha1"]: item["url"] for item in locations}
+    records = []
+    for sha1, record in sorted(retired.items()):
+        if sha1 in files:
+            continue
+        successor, seen = record.get("successor"), {sha1}
+        while successor and successor not in files and successor not in seen:
+            seen.add(successor)
+            successor = (retired.get(successor) or {}).get("successor")
+        url = cards.get(successor) if successor in files else None
+        extra = {"left": str(record["left"])}
+        if url:
+            extra["successor"] = successor
+        else:
+            # The page its card was on: the reader lands among its neighbours.
+            url = _site_path(f"systems/{card_page(str(record.get('path', '')), sha1)}.md")
+        for algorithm in HASH_ALGORITHMS:
+            value = sha1 if algorithm == "sha1" else record.get(algorithm)
+            for one in _hash_values(value):
+                records.append({"value": one, "algorithm": algorithm, "layer": "retired",
+                                "name": str(record.get("name", "")), "sha1": sha1,
+                                "url": url, **extra})
+    return records
+
+
 def _archive_member_records(entry: dict, sha1: str, url: str,
                             root: Path = _REPO_ROOT) -> list[dict]:
     """The CRC32 of each member of a collection archive, read from its
@@ -4956,6 +4994,7 @@ def file_lookup_records(
     locations: list[dict],
     members: dict | None = None,
     consumers: FileConsumers | None = None,
+    retired: dict | None = None,
 ) -> list[dict]:
     """Every hash the site knows, one record per value, for files/?hash=.
 
@@ -5033,6 +5072,7 @@ def file_lookup_records(
                             "owner": key, "name": name, "url": url,
                             **({"system_name": entry_label} if entry_label else {}),
                         })
+    records += retired_records(retired or {}, files, locations)
     records += _held_under_every_hash(records, files, db.get("indexes") or {})
     for record in records:
         if record["layer"] in ("platform", "emulator"):

scripts/retired_files.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Files that left the collection, and the file that took their place.
+
+A file's content address (files/?sha1=, #sha1-<12> on its card) is
+published, and a published address never stops answering. When a rebuild
+of database.json drops a sha1, the file it named is recorded here: its
+name, its last path, its hashes, the day it left and the sha1 the
+collection now holds at that path, or under that name in that folder.
+The site's lookup reads the record and answers with it.
+
+The record is append-only, written by generate_db.py beside the database.
+A sha1 that comes back leaves it. `--backfill` rebuilds it from the
+history of database.json since content addresses were published.
+
+Usage:
+    python scripts/retired_files.py --backfill
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+RETIRED_FILE = REPO_ROOT / "retired.json"
+# The commit that published content addresses: 0d586ef6b, 2026-10-19.
+FIRST_ADDRESSED = "0d586ef6b"
+KEPT_FIELDS = ("name", "path", "size", "md5", "crc32", "sha256")
+
+
+def retired_path(database: str | Path) -> Path:
+    """The record kept beside a database file."""
+    return Path(database).with_name("retired.json")
+
+
+def _successor(entry: dict, by_path: dict[str, str],
+               by_name: dict[str, list[tuple[str, str]]]) -> str | None:
+    """The sha1 the new database holds at the entry's path, else under its
+    name in the same folder, else the one file of that name: a2diskiing.zip
+    was rebuilt from its recipe and filed under Apple II the same day."""
+    path = str(entry.get("path", ""))
+    if path in by_path:
+        return by_path[path]
+    named = by_name.get(str(entry.get("name", "")), [])
+    folder = path.rsplit("/", 1)[0]
+    beside = [sha1 for sha1, other in named if other.rsplit("/", 1)[0] == folder]
+    for found in (beside, [sha1 for sha1, _other in named]):
+        if len(found) == 1:
+            return found[0]
+    return None
+
+
+def retired_between(old: dict, new: dict, day: str) -> dict[str, dict]:
+    """sha1 -> record, for each file of *old* that *new* no longer holds."""
+    old_files = old.get("files") or {}
+    new_files = new.get("files") or {}
+    by_path = {str(e.get("path", "")): sha1 for sha1, e in new_files.items()}
+    by_name: dict[str, list[tuple[str, str]]] = {}
+    for sha1, e in new_files.items():
+        by_name.setdefault(str(e.get("name", "")), []).append((sha1, str(e.get("path", ""))))
+    records = {}
+    for sha1 in sorted(set(old_files) - set(new_files)):
+        entry = old_files[sha1]
+        record = {key: entry[key] for key in KEPT_FIELDS if entry.get(key) not in (None, "")}
+        record["left"] = day
+        successor = _successor(entry, by_path, by_name)
+        if successor:
+            record["successor"] = successor
+        records[sha1] = record
+    return records
+
+
+def merge(record: dict[str, dict], left: dict[str, dict], new: dict) -> dict[str, dict]:
+    """The record with *left* added, first departure kept, and every sha1
+    the new database holds again taken out."""
+    held = set(new.get("files") or {})
+    merged = {sha1: entry for sha1, entry in record.items() if sha1 not in held}
+    for sha1, entry in left.items():
+        merged.setdefault(sha1, entry)
+    return dict(sorted(merged.items()))
+
+
+def load(path: Path) -> dict[str, dict]:
+    if not path.is_file():
+        return {}
+    return json.loads(path.read_text(encoding="utf-8")).get("files") or {}
+
+
+def dump(records: dict[str, dict]) -> str:
+    return json.dumps({"schema_version": 1, "files": records}, indent=1, sort_keys=True) + "\n"
+
+
+def update(path: Path, old: dict, new: dict, day: str) -> int:
+    """Records what *new* dropped from *old*. Returns how many left now."""
+    from common import write_if_changed  # noqa: PLC0415
+
+    left = retired_between(old, new, day)
+    record = load(path)
+    merged = merge(record, left, new)
+    if merged != record:
+        write_if_changed(str(path), dump(merged))
+    return len(set(left) - set(record))
+
+
+def _git(*args: str) -> str:
+    return subprocess.run(["git", *args], cwd=REPO_ROOT, capture_output=True, text=True,
+                          check=True).stdout
+
+
+def backfill() -> dict[str, dict]:
+    """Every sha1 that left database.json in the commits since content
+    addresses were published, with the day of the commit that dropped it."""
+    revisions = [line.split() for line in _git(
+        "log", "--reverse", "--format=%H %cs", f"{FIRST_ADDRESSED}^..HEAD", "--", "database.json",
+    ).splitlines() if line.strip()]
+    records: dict[str, dict] = {}
+    previous: dict | None = None
+    for commit, day in revisions:
+        current = json.loads(_git("show", f"{commit}:database.json"))
+        if previous is not None:
+            records = merge(records, retired_between(previous, current, day), current)
+        previous = current
+    return records
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
+    parser.add_argument("--backfill", action="store_true",
+                        help="rebuild the record from the history of database.json")
+    parser.add_argument("--output", default=str(RETIRED_FILE),
+                        help="the record to write (default: retired.json)")
+    args = parser.parse_args()
+    if not args.backfill:
+        parser.print_help()
+        return 2
+    sys.path.insert(0, str(Path(__file__).parent))
+    from common import write_if_changed  # noqa: PLC0415
+
+    records = backfill()
+    current = json.loads((REPO_ROOT / "database.json").read_text(encoding="utf-8"))
+    records = merge(records, {}, current)
+    write_if_changed(args.output, dump(records))
+    print(f"{len(records)} files left the collection since {FIRST_ADDRESSED}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())