feat: mark catalog gaps a held dump contains

scripts/derivations.py

@@ -0,0 +1,157 @@
+"""Catalog entries whose bytes a held dump already contains.
+
+A catalog lists chips as they come off the board; a collection often holds
+the image an emulator reads. The Amiga 3.1 Kickstart pair 391773/391774 is
+the high and low 16-bit words of amiga-os-310-a1200.rom, byte-swapped; the
+TOSEC "Basic V7" is basiclo and basichi end to end. Listed as missing, they
+send a maintainer after files that are already here. Each transform below is
+one the source boards use; a match is by sha1, never by size or name.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from collections import defaultdict
+from collections.abc import Callable, Iterable
+
+# A held file larger than this is not read: no chip image is near it.
+MAX_SOURCE_SIZE = 64 * 1024 * 1024
+
+
+def _swap16(data: bytes) -> bytes:
+    swapped = bytearray(data)
+    swapped[0::2], swapped[1::2] = data[1::2], data[0::2]
+    return bytes(swapped)
+
+
+def _words(data: bytes, start: int) -> bytes:
+    return b"".join(data[i:i + 2] for i in range(start * 2, len(data), 4))
+
+
+# (name, size of the output for an input of size n or None, transform)
+TRANSFORMS: tuple[tuple[str, Callable[[int], int | None], Callable[[bytes], bytes]], ...] = (
+    ("first half", lambda n: n // 2 if n % 2 == 0 else None, lambda d: d[:len(d) // 2]),
+    ("second half", lambda n: n // 2 if n % 2 == 0 else None, lambda d: d[len(d) // 2:]),
+    *(
+        (f"quarter {q + 1}", lambda n: n // 4 if n % 4 == 0 else None,
+         lambda d, q=q: d[q * len(d) // 4:(q + 1) * len(d) // 4])
+        for q in range(4)
+    ),
+    ("even bytes", lambda n: n // 2 if n % 2 == 0 else None, lambda d: d[0::2]),
+    ("odd bytes", lambda n: n // 2 if n % 2 == 0 else None, lambda d: d[1::2]),
+    ("even words", lambda n: n // 2 if n % 4 == 0 else None, lambda d: _words(d, 0)),
+    ("odd words", lambda n: n // 2 if n % 4 == 0 else None, lambda d: _words(d, 1)),
+    ("even words, byte-swapped", lambda n: n // 2 if n % 4 == 0 else None,
+     lambda d: _swap16(_words(d, 0))),
+    ("odd words, byte-swapped", lambda n: n // 2 if n % 4 == 0 else None,
+     lambda d: _swap16(_words(d, 1))),
+    ("byte-swapped", lambda n: n if n % 2 == 0 else None, _swap16),
+    ("mirrored twice", lambda n: n * 2, lambda d: d + d),
+)
+
+
+def _gf2_times(matrix: list[int], vector: int) -> int:
+    total, row = 0, 0
+    while vector:
+        if vector & 1:
+            total ^= matrix[row]
+        vector >>= 1
+        row += 1
+    return total
+
+
+def _gf2_square(matrix: list[int]) -> list[int]:
+    return [_gf2_times(matrix, matrix[n]) for n in range(32)]
+
+
+def crc32_combine(crc1: int, crc2: int, len2: int) -> int:
+    """crc32 of A + B from crc32(A), crc32(B) and len(B), as zlib computes it."""
+    if len2 <= 0:
+        return crc1
+    odd = [0xEDB88320] + [1 << (n - 1) for n in range(1, 32)]
+    even = _gf2_square(odd)
+    odd = _gf2_square(even)
+    while True:
+        even = _gf2_square(odd)
+        if len2 & 1:
+            crc1 = _gf2_times(even, crc1)
+        len2 >>= 1
+        if not len2:
+            break
+        odd = _gf2_square(even)
+        if len2 & 1:
+            crc1 = _gf2_times(odd, crc1)
+        len2 >>= 1
+        if not len2:
+            break
+    return crc1 ^ crc2
+
+
+def _read(path: str) -> bytes | None:
+    try:
+        return open(path, "rb").read()
+    except OSError:
+        return None
+
+
+def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]:
+    """Missing sha1 to the held file and transform that produce its bytes.
+
+    Only held files whose size can produce a wanted size are read, and a
+    concatenation is only tried between files of one directory: the chip
+    pairs a catalog splits come from one machine.
+    """
+    wanted: dict[tuple[int, str], str] = {}
+    by_size_crc: dict[tuple[int, int], list[str]] = defaultdict(list)
+    for entry in entries:
+        sha1, size = (entry.get("sha1") or "").lower(), entry.get("size")
+        if sha1 and isinstance(size, int) and size > 0:
+            wanted[(size, sha1)] = sha1
+            crc = entry.get("crc32")
+            if crc:
+                by_size_crc[(size, int(str(crc), 16))].append(sha1)
+    sizes = {size for size, _sha1 in wanted}
+    found: dict[str, dict] = {}
+
+    files = [f for f in db.get("files", {}).values()
+             if 0 < f.get("size", 0) <= MAX_SOURCE_SIZE and os.path.exists(f["path"])]
+    for held in files:
+        n = held["size"]
+        useful = [t for t in TRANSFORMS if t[1](n) in sizes]
+        if not useful:
+            continue
+        data = _read(held["path"])
+        if data is None:
+            continue
+        for name, out_size, transform in useful:
+            out = transform(data)
+            sha1 = wanted.get((out_size(n), hashlib.sha1(out).hexdigest()))
+            if sha1 and sha1 not in found:
+                found[sha1] = {"from": [held["path"]], "transform": name}
+
+    by_dir: dict[str, list[dict]] = defaultdict(list)
+    for held in files:
+        if held.get("crc32"):
+            by_dir[os.path.dirname(held["path"])].append(held)
+    for group in by_dir.values():
+        for first in group:
+            for second in group:
+                if first is second:
+                    continue
+                total = first["size"] + second["size"]
+                if total not in sizes:
+                    continue
+                crc = crc32_combine(int(first["crc32"], 16), int(second["crc32"], 16),
+                                    second["size"])
+                candidates = [s for s in by_size_crc.get((total, crc), []) if s not in found]
+                if not candidates:
+                    continue
+                a, b = _read(first["path"]), _read(second["path"])
+                if a is None or b is None:
+                    continue
+                sha1 = hashlib.sha1(a + b).hexdigest()
+                if sha1 in candidates:
+                    found[sha1] = {"from": [first["path"], second["path"]],
+                                   "transform": "concatenated"}
+    return found

scripts/provenance_report.py

@@ -135,6 +135,12 @@ def main() -> int:
         help="Directory with dump-catalog snapshots",
     )
     parser.add_argument("--missing", action="store_true", help="List missing entries")
+    parser.add_argument(
+        "--derive",
+        action="store_true",
+        help="Find missing entries a held dump contains (halves, word splits, "
+        "byte swaps, concatenations); reads the collection, about 90 s",
+    )
     parser.add_argument("--json", action="store_true", help="Full report as JSON")
     args = parser.parse_args()
 
@@ -145,6 +151,16 @@ def main() -> int:
         return 0
 
     report = build_report(db, snapshots, archive_members(db))
+    derived: dict[str, dict] = {}
+    if args.derive:
+        from derivations import derivable  # noqa: PLC0415
+
+        derived = derivable((e for d in report.values() for e in d["missing"]), db)
+        for data in report.values():
+            data["derivable"] = {
+                e["sha1"]: derived[e["sha1"]] for e in data["missing"]
+                if e.get("sha1") in derived
+            }
 
     if args.json:
         print(json.dumps(report, indent=2))
@@ -166,11 +182,19 @@ def main() -> int:
         if args.missing:
             for entry in data["missing"]:
                 label = entry.get("description") or entry["name"]
-                print(f"    MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}")
+                source = derived.get(entry.get("sha1", ""))
+                if source:
+                    print(f"    DERIVABLE {entry['name']} ({label}): {source['transform']} "
+                          f"of {' + '.join(source['from'])}")
+                else:
+                    print(f"    MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}")
 
     total_missing = sum(len(d["missing"]) for d in report.values())
     if total_missing:
-        print(f"  {total_missing} catalog entries missing from collection")
+        line = f"  {total_missing} catalog entries missing from collection"
+        if args.derive:
+            line += f", {len(derived)} of them derivable from held dumps"
+        print(line)
     return 0
 
 

tests/test_derivations.py

@@ -0,0 +1,68 @@
+"""Catalog entries the collection already holds in another layout.
+
+391773/391774, the two Kickstart 3.1 chips of the A1200, are the high and low
+16-bit words of the image the collection keeps, byte-swapped; TOSEC's Basic V7
+is two held ROMs end to end. The gap report listed them as acquisition targets.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+import zlib
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import derivations  # noqa: E402
+
+
+def _held(path: Path, data: bytes) -> dict:
+    path.write_bytes(data)
+    return {"path": str(path), "size": len(data), "crc32": format(zlib.crc32(data), "08x"),
+            "sha1": hashlib.sha1(data).hexdigest()}
+
+
+def _wanted(data: bytes) -> dict:
+    return {"sha1": hashlib.sha1(data).hexdigest(), "size": len(data),
+            "crc32": format(zlib.crc32(data), "08x")}
+
+
+class HeldDumpsAnswerCatalogEntries(unittest.TestCase):
+    def setUp(self):
+        self.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
+        self.addCleanup(shutil.rmtree, self.tmp, True)
+
+    def test_combine_agrees_with_zlib(self):
+        for a, b in ((b"abc", b"defgh"), (os.urandom(999), os.urandom(4097))):
+            self.assertEqual(derivations.crc32_combine(zlib.crc32(a), zlib.crc32(b), len(b)),
+                             zlib.crc32(a + b))
+
+    def test_a_word_split_byte_swapped_chip_is_found(self):
+        image = os.urandom(64)
+        db = {"files": {"x": _held(self.tmp / "kick.rom", image)}}
+        high = derivations._swap16(derivations._words(image, 0))
+        found = derivations.derivable([_wanted(high)], db)
+        self.assertEqual(found[hashlib.sha1(high).hexdigest()]["transform"],
+                         "even words, byte-swapped")
+
+    def test_two_held_halves_make_the_catalogued_whole(self):
+        low, high = os.urandom(32), os.urandom(32)
+        db = {"files": {"a": _held(self.tmp / "basiclo.bin", low),
+                        "b": _held(self.tmp / "basichi.bin", high)}}
+        found = derivations.derivable([_wanted(low + high)], db)
+        self.assertEqual(found[hashlib.sha1(low + high).hexdigest()]["from"],
+                         [str(self.tmp / "basiclo.bin"), str(self.tmp / "basichi.bin")])
+
+    def test_unrelated_bytes_are_not_claimed(self):
+        db = {"files": {"a": _held(self.tmp / "a.bin", os.urandom(64))}}
+        self.assertEqual(derivations.derivable([_wanted(os.urandom(32))], db), {})
+
+
+if __name__ == "__main__":
+    unittest.main()

wiki/tools.md

@@ -544,6 +544,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6
 | `download.sh` | Same, as a shell one-liner (`curl` + `unzip`) |
 | `split_pack.py` | Cut a pack over the release asset limit into parts that are each a ZIP of whole files, read back before the pack is removed |
 | `provenance_report.py` | Dump-catalog coverage and acquisition targets (see above) |
+| `find_pinned_revision.py` | Walk the upstream commits that touched a file and return the revision whose bytes match a platform's pinned md5 (`--write` saves them) |
 | `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database |
 | `generate_site.py` | Generate all MkDocs site pages (this documentation) |
 | `validate_site.py` | Validate rendered metadata, headings, image alternatives, JSON-LD, local resources, links and fragments |
@@ -695,9 +696,18 @@ python -m scripts.scraper.dat_pack_importer --source no-intro --pack no-intro.zi
 python -m scripts.scraper.dat_pack_importer --source tosec --pack TOSEC-v2025.zip
 python scripts/provenance_report.py            # coverage summary (pipeline step 1b)
 python scripts/provenance_report.py --missing  # list acquisition targets
+python scripts/provenance_report.py --missing --derive  # mark targets a held dump already contains
 python scripts/provenance_report.py --json     # full report
 ```
 
+A catalog lists chips as they come off the board, the collection often holds
+the image an emulator reads. `--derive` reads the collection (about 90 s) and
+marks a missing entry DERIVABLE when a held file produces its sha1 by halving,
+quartering, splitting into even and odd bytes or 16-bit words (byte-swapped
+or not), swapping bytes, mirroring, or joining two files of one directory.
+The Amiga Kickstart chip pairs and TOSEC's split C64 and VES ROMs are found
+this way: 67 entries on the current snapshots.
+
 | Catalog | Acquisition | Notes |
 |---------|------------|-------|
 | Redump | direct fetch from redump.info `/static/bios` | 4 BIOS DATs. redump.org serves stale DATs |