fix: stop asking for dumps the collection can make

The Wanted page listed 79 dumps the collection already produces,
such as a Kickstart's byte-swapped words or the halves of a C128
character ROM. The pipeline now records which missing dumps can be
derived, in derivable.json, and the page moves them to a section of
their own that names the held file and the transform.

scripts/generate_site.py

@@ -1690,7 +1690,22 @@ def _prov_title(data: dict) -> str:
     return ": ".join(p for p in parts if p).replace('"', """)
 
 
-def generate_provenance_page(db: dict, report: dict) -> str:
+def load_derivable(path: str = "derivable.json") -> dict[str, dict]:
+    """Missing sha1 -> how the collection produces it, as the pipeline found.
+
+    Empty when the pipeline has not recorded it: the page then lists every
+    missing dump, as it did before the record existed.
+    """
+    try:
+        with open(path, encoding="utf-8") as handle:
+            return json.load(handle)
+    except FileNotFoundError:
+        return {}
+
+
+def generate_provenance_page(
+    db: dict, report: dict, derivable: dict[str, dict] | None = None
+) -> str:
     """Page explaining the verified dump badges and listing catalog gaps."""
     matched_files = count_catalog_matched(db)
     total_files = compute_composition(db)["systems"]["files"]
@@ -1774,6 +1789,20 @@ def generate_provenance_page(db: dict, report: dict) -> str:
             ]
         )
 
+    # A dump the collection already produces (the odd bytes of a Kickstart,
+    # the halves of a character ROM) is no acquisition target: the page asked
+    # the public for 79 of them.
+    derivable = derivable or {}
+    derived_entries: dict[str, dict] = {}
+    for data in report.values():
+        kept = []
+        for entry in data["missing"]:
+            sha1 = (entry.get("sha1") or "").lower()
+            if sha1 in derivable:
+                derived_entries.setdefault(sha1, entry)
+            else:
+                kept.append(entry)
+        data["missing"] = kept
     total_missing = distinct_missing(report)
     lines.extend(
         [
@@ -1811,6 +1840,26 @@ def generate_provenance_page(db: dict, report: dict) -> str:
                 lines.append(f"    | `{name}` | {desc} | `{sha1}` |")
             lines.append("")
 
+    if derived_entries:
+        lines.extend(
+            [
+                "## Derivable: dumps the collection already produces",
+                "",
+                f"**{len(derived_entries):,}** catalogued dumps are not held as "
+                "files but come out of one that is: a byte or word split, a "
+                "half, a swap, or two chips joined. They are not wanted.",
+                "",
+                "| Name | Comes from | How | SHA1 |",
+                "|------|------------|-----|------|",
+            ]
+        )
+        for sha1, entry in sorted(derived_entries.items(), key=lambda kv: kv[1]["name"]):
+            how = derivable[sha1]
+            sources = " + ".join(f"`{path}`" for path in how.get("from", []))
+            name = entry["name"].replace("|", "-")
+            lines.append(f"| `{name}` | {sources} | {how.get('transform', '')} | `{sha1}` |")
+        lines.append("")
+
     lines.append(f'<div class="rb-timestamp">Generated on {_timestamp()}.</div>')
     return "\n".join(lines) + "\n"
 
@@ -3927,7 +3976,8 @@ def main() -> None:
     print("Generating dump provenance page...")
     provenance_report = build_report(db, load_provenance_snapshots())
     write_if_changed(
-        str(docs / "provenance.md"), generate_provenance_page(db, provenance_report)
+        str(docs / "provenance.md"),
+        generate_provenance_page(db, provenance_report, load_derivable()),
     )
 
     # Wiki pages: copy manually maintained sources + generate dynamic ones

scripts/pipeline.py

@@ -3,7 +3,7 @@
 
 Steps:
   1. generate_db.py --force     (rebuild database.json from bios/)
-  1b. provenance_report.py      (dump-catalog coverage from provenance/)
+  1b. provenance_report.py      (dump-catalog coverage, derivable.json)
   1c. romset_recipes.py         (archive identification, reconstruction targets)
   2. refresh_data_dirs.py       (update Dolphin Sys, PPSSPP, etc.)
   2b2. check_release_assets.py  (large-files release vs gitignored files, online)
@@ -400,9 +400,12 @@ def main() -> None:
         print("\nDatabase generation failed, aborting.")
         sys.exit(1)
 
-    # Step 1b: Dump-catalog coverage (offline, reads provenance/ snapshots)
+    # Step 1b: Dump-catalog coverage (offline, reads provenance/ snapshots),
+    # and which missing dumps the collection already produces: recorded in
+    # derivable.json, which the site reads instead of rereading bios/.
     ok, _ = run(
-        [sys.executable, "scripts/provenance_report.py"],
+        [sys.executable, "scripts/provenance_report.py", "--derive",
+         "--write-derivable", "derivable.json"],
         "1b provenance report",
     )
     results["provenance"] = ok

scripts/provenance_report.py

@@ -22,7 +22,12 @@ import sys
 import zipfile
 
 sys.path.insert(0, os.path.dirname(__file__))
-from common import DEFAULT_PROVENANCE_DIR, load_database, load_provenance_snapshots
+from common import (
+    DEFAULT_PROVENANCE_DIR,
+    load_database,
+    load_provenance_snapshots,
+    write_if_changed,
+)
 
 
 def archive_members(db: dict) -> dict[tuple[str, int], list[tuple[str, str]]]:
@@ -176,7 +181,15 @@ def main() -> int:
         "byte swaps, concatenations); reads the collection, about 90 s",
     )
     parser.add_argument("--json", action="store_true", help="Full report as JSON")
+    parser.add_argument(
+        "--write-derivable",
+        metavar="PATH",
+        help="With --derive, record the derivable entries for the site "
+        "(the pipeline writes derivable.json)",
+    )
     args = parser.parse_args()
+    if args.write_derivable and not args.derive:
+        parser.error("--write-derivable records what --derive finds; pass both")
 
     db = load_database(args.db)
     snapshots = load_provenance_snapshots(args.provenance_dir)
@@ -195,6 +208,14 @@ def main() -> int:
                 e["sha1"]: derived[e["sha1"]] for e in data["missing"]
                 if e.get("sha1") in derived
             }
+        if args.write_derivable:
+            # The site cannot spend 80 s reading the collection on every
+            # build: the pipeline records what this pass found, and the
+            # Wanted page stops asking for dumps the collection can produce.
+            write_if_changed(
+                args.write_derivable,
+                json.dumps(dict(sorted(derived.items())), indent=2, sort_keys=True) + "\n",
+            )
 
     if args.json:
         print(json.dumps(report, indent=2))