From 0fc71793a9a147c574d8c509766f95b99e53120a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sat, 17 Oct 2026 13:40:05 +0200 Subject: [PATCH] feat: mark catalog gaps a held dump contains --- scripts/derivations.py | 157 +++++++++++++++++++++++++++++++++++ scripts/provenance_report.py | 28 ++++++- tests/test_derivations.py | 68 +++++++++++++++ wiki/tools.md | 10 +++ 4 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 scripts/derivations.py create mode 100644 tests/test_derivations.py diff --git a/scripts/derivations.py b/scripts/derivations.py new file mode 100644 index 00000000..467c1f3d --- /dev/null +++ b/scripts/derivations.py @@ -0,0 +1,157 @@ +"""Catalog entries whose bytes a held dump already contains. + +A catalog lists chips as they come off the board; a collection often holds +the image an emulator reads. The Amiga 3.1 Kickstart pair 391773/391774 is +the high and low 16-bit words of amiga-os-310-a1200.rom, byte-swapped; the +TOSEC "Basic V7" is basiclo and basichi end to end. Listed as missing, they +send a maintainer after files that are already here. Each transform below is +one the source boards use; a match is by sha1, never by size or name. +""" + +from __future__ import annotations + +import hashlib +import os +from collections import defaultdict +from collections.abc import Callable, Iterable + +# A held file larger than this is not read: no chip image is near it. +MAX_SOURCE_SIZE = 64 * 1024 * 1024 + + +def _swap16(data: bytes) -> bytes: + swapped = bytearray(data) + swapped[0::2], swapped[1::2] = data[1::2], data[0::2] + return bytes(swapped) + + +def _words(data: bytes, start: int) -> bytes: + return b"".join(data[i:i + 2] for i in range(start * 2, len(data), 4)) + + +# (name, size of the output for an input of size n or None, transform) +TRANSFORMS: tuple[tuple[str, Callable[[int], int | None], Callable[[bytes], bytes]], ...] = ( + ("first half", lambda n: n // 2 if n % 2 == 0 else None, lambda d: d[:len(d) // 2]), + ("second half", lambda n: n // 2 if n % 2 == 0 else None, lambda d: d[len(d) // 2:]), + *( + (f"quarter {q + 1}", lambda n: n // 4 if n % 4 == 0 else None, + lambda d, q=q: d[q * len(d) // 4:(q + 1) * len(d) // 4]) + for q in range(4) + ), + ("even bytes", lambda n: n // 2 if n % 2 == 0 else None, lambda d: d[0::2]), + ("odd bytes", lambda n: n // 2 if n % 2 == 0 else None, lambda d: d[1::2]), + ("even words", lambda n: n // 2 if n % 4 == 0 else None, lambda d: _words(d, 0)), + ("odd words", lambda n: n // 2 if n % 4 == 0 else None, lambda d: _words(d, 1)), + ("even words, byte-swapped", lambda n: n // 2 if n % 4 == 0 else None, + lambda d: _swap16(_words(d, 0))), + ("odd words, byte-swapped", lambda n: n // 2 if n % 4 == 0 else None, + lambda d: _swap16(_words(d, 1))), + ("byte-swapped", lambda n: n if n % 2 == 0 else None, _swap16), + ("mirrored twice", lambda n: n * 2, lambda d: d + d), +) + + +def _gf2_times(matrix: list[int], vector: int) -> int: + total, row = 0, 0 + while vector: + if vector & 1: + total ^= matrix[row] + vector >>= 1 + row += 1 + return total + + +def _gf2_square(matrix: list[int]) -> list[int]: + return [_gf2_times(matrix, matrix[n]) for n in range(32)] + + +def crc32_combine(crc1: int, crc2: int, len2: int) -> int: + """crc32 of A + B from crc32(A), crc32(B) and len(B), as zlib computes it.""" + if len2 <= 0: + return crc1 + odd = [0xEDB88320] + [1 << (n - 1) for n in range(1, 32)] + even = _gf2_square(odd) + odd = _gf2_square(even) + while True: + even = _gf2_square(odd) + if len2 & 1: + crc1 = _gf2_times(even, crc1) + len2 >>= 1 + if not len2: + break + odd = _gf2_square(even) + if len2 & 1: + crc1 = _gf2_times(odd, crc1) + len2 >>= 1 + if not len2: + break + return crc1 ^ crc2 + + +def _read(path: str) -> bytes | None: + try: + return open(path, "rb").read() + except OSError: + return None + + +def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]: + """Missing sha1 to the held file and transform that produce its bytes. + + Only held files whose size can produce a wanted size are read, and a + concatenation is only tried between files of one directory: the chip + pairs a catalog splits come from one machine. + """ + wanted: dict[tuple[int, str], str] = {} + by_size_crc: dict[tuple[int, int], list[str]] = defaultdict(list) + for entry in entries: + sha1, size = (entry.get("sha1") or "").lower(), entry.get("size") + if sha1 and isinstance(size, int) and size > 0: + wanted[(size, sha1)] = sha1 + crc = entry.get("crc32") + if crc: + by_size_crc[(size, int(str(crc), 16))].append(sha1) + sizes = {size for size, _sha1 in wanted} + found: dict[str, dict] = {} + + files = [f for f in db.get("files", {}).values() + if 0 < f.get("size", 0) <= MAX_SOURCE_SIZE and os.path.exists(f["path"])] + for held in files: + n = held["size"] + useful = [t for t in TRANSFORMS if t[1](n) in sizes] + if not useful: + continue + data = _read(held["path"]) + if data is None: + continue + for name, out_size, transform in useful: + out = transform(data) + sha1 = wanted.get((out_size(n), hashlib.sha1(out).hexdigest())) + if sha1 and sha1 not in found: + found[sha1] = {"from": [held["path"]], "transform": name} + + by_dir: dict[str, list[dict]] = defaultdict(list) + for held in files: + if held.get("crc32"): + by_dir[os.path.dirname(held["path"])].append(held) + for group in by_dir.values(): + for first in group: + for second in group: + if first is second: + continue + total = first["size"] + second["size"] + if total not in sizes: + continue + crc = crc32_combine(int(first["crc32"], 16), int(second["crc32"], 16), + second["size"]) + candidates = [s for s in by_size_crc.get((total, crc), []) if s not in found] + if not candidates: + continue + a, b = _read(first["path"]), _read(second["path"]) + if a is None or b is None: + continue + sha1 = hashlib.sha1(a + b).hexdigest() + if sha1 in candidates: + found[sha1] = {"from": [first["path"], second["path"]], + "transform": "concatenated"} + return found diff --git a/scripts/provenance_report.py b/scripts/provenance_report.py index 6100a231..51e18e9b 100644 --- a/scripts/provenance_report.py +++ b/scripts/provenance_report.py @@ -135,6 +135,12 @@ def main() -> int: help="Directory with dump-catalog snapshots", ) parser.add_argument("--missing", action="store_true", help="List missing entries") + parser.add_argument( + "--derive", + action="store_true", + help="Find missing entries a held dump contains (halves, word splits, " + "byte swaps, concatenations); reads the collection, about 90 s", + ) parser.add_argument("--json", action="store_true", help="Full report as JSON") args = parser.parse_args() @@ -145,6 +151,16 @@ def main() -> int: return 0 report = build_report(db, snapshots, archive_members(db)) + derived: dict[str, dict] = {} + if args.derive: + from derivations import derivable # noqa: PLC0415 + + derived = derivable((e for d in report.values() for e in d["missing"]), db) + for data in report.values(): + data["derivable"] = { + e["sha1"]: derived[e["sha1"]] for e in data["missing"] + if e.get("sha1") in derived + } if args.json: print(json.dumps(report, indent=2)) @@ -166,11 +182,19 @@ def main() -> int: if args.missing: for entry in data["missing"]: label = entry.get("description") or entry["name"] - print(f" MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}") + source = derived.get(entry.get("sha1", "")) + if source: + print(f" DERIVABLE {entry['name']} ({label}): {source['transform']} " + f"of {' + '.join(source['from'])}") + else: + print(f" MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}") total_missing = sum(len(d["missing"]) for d in report.values()) if total_missing: - print(f" {total_missing} catalog entries missing from collection") + line = f" {total_missing} catalog entries missing from collection" + if args.derive: + line += f", {len(derived)} of them derivable from held dumps" + print(line) return 0 diff --git a/tests/test_derivations.py b/tests/test_derivations.py new file mode 100644 index 00000000..2b1c8993 --- /dev/null +++ b/tests/test_derivations.py @@ -0,0 +1,68 @@ +"""Catalog entries the collection already holds in another layout. + +391773/391774, the two Kickstart 3.1 chips of the A1200, are the high and low +16-bit words of the image the collection keeps, byte-swapped; TOSEC's Basic V7 +is two held ROMs end to end. The gap report listed them as acquisition targets. +""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import sys +import tempfile +import unittest +import zlib +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import derivations # noqa: E402 + + +def _held(path: Path, data: bytes) -> dict: + path.write_bytes(data) + return {"path": str(path), "size": len(data), "crc32": format(zlib.crc32(data), "08x"), + "sha1": hashlib.sha1(data).hexdigest()} + + +def _wanted(data: bytes) -> dict: + return {"sha1": hashlib.sha1(data).hexdigest(), "size": len(data), + "crc32": format(zlib.crc32(data), "08x")} + + +class HeldDumpsAnswerCatalogEntries(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, self.tmp, True) + + def test_combine_agrees_with_zlib(self): + for a, b in ((b"abc", b"defgh"), (os.urandom(999), os.urandom(4097))): + self.assertEqual(derivations.crc32_combine(zlib.crc32(a), zlib.crc32(b), len(b)), + zlib.crc32(a + b)) + + def test_a_word_split_byte_swapped_chip_is_found(self): + image = os.urandom(64) + db = {"files": {"x": _held(self.tmp / "kick.rom", image)}} + high = derivations._swap16(derivations._words(image, 0)) + found = derivations.derivable([_wanted(high)], db) + self.assertEqual(found[hashlib.sha1(high).hexdigest()]["transform"], + "even words, byte-swapped") + + def test_two_held_halves_make_the_catalogued_whole(self): + low, high = os.urandom(32), os.urandom(32) + db = {"files": {"a": _held(self.tmp / "basiclo.bin", low), + "b": _held(self.tmp / "basichi.bin", high)}} + found = derivations.derivable([_wanted(low + high)], db) + self.assertEqual(found[hashlib.sha1(low + high).hexdigest()]["from"], + [str(self.tmp / "basiclo.bin"), str(self.tmp / "basichi.bin")]) + + def test_unrelated_bytes_are_not_claimed(self): + db = {"files": {"a": _held(self.tmp / "a.bin", os.urandom(64))}} + self.assertEqual(derivations.derivable([_wanted(os.urandom(32))], db), {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/tools.md b/wiki/tools.md index 02295c30..c05238ac 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -544,6 +544,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6 | `download.sh` | Same, as a shell one-liner (`curl` + `unzip`) | | `split_pack.py` | Cut a pack over the release asset limit into parts that are each a ZIP of whole files, read back before the pack is removed | | `provenance_report.py` | Dump-catalog coverage and acquisition targets (see above) | +| `find_pinned_revision.py` | Walk the upstream commits that touched a file and return the revision whose bytes match a platform's pinned md5 (`--write` saves them) | | `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database | | `generate_site.py` | Generate all MkDocs site pages (this documentation) | | `validate_site.py` | Validate rendered metadata, headings, image alternatives, JSON-LD, local resources, links and fragments | @@ -695,9 +696,18 @@ python -m scripts.scraper.dat_pack_importer --source no-intro --pack no-intro.zi python -m scripts.scraper.dat_pack_importer --source tosec --pack TOSEC-v2025.zip python scripts/provenance_report.py # coverage summary (pipeline step 1b) python scripts/provenance_report.py --missing # list acquisition targets +python scripts/provenance_report.py --missing --derive # mark targets a held dump already contains python scripts/provenance_report.py --json # full report ``` +A catalog lists chips as they come off the board, the collection often holds +the image an emulator reads. `--derive` reads the collection (about 90 s) and +marks a missing entry DERIVABLE when a held file produces its sha1 by halving, +quartering, splitting into even and odd bytes or 16-bit words (byte-swapped +or not), swapping bytes, mirroring, or joining two files of one directory. +The Amiga Kickstart chip pairs and TOSEC's split C64 and VES ROMs are found +this way: 67 entries on the current snapshots. + | Catalog | Acquisition | Notes | |---------|------------|-------| | Redump | direct fetch from redump.info `/static/bios` | 4 BIOS DATs. redump.org serves stale DATs |