chore: merge archive recipes

This commit is contained in:
Abdessamad Derraz committed 2026-11-07 00:05:51 +01:00
commit 21dfb961f9
17 files changed
+262465 -94902

No files matched your search

+349
View File
@@ -0,0 +1,349 @@
"""Every archive has a recipe, every member is held, and the build reproduces it.
The collection must regenerate each archive it serves from the ROMs it holds
plus recipes/archives.json, without a copy of that archive from elsewhere.
"""
from __future__ import annotations
import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache
import hashlib
import json
import subprocess
import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
from unittest import mock
ROOT = Path(__file__).resolve().parent.parent
TMP_ROOT = ROOT / "tmp" / "tests"
TMP_ROOT.mkdir(parents=True, exist_ok=True)
sys.path.insert(0, str(ROOT / "scripts"))
import archive_build # noqa: E402
from torrentzip import build_torrentzip # noqa: E402
ROM_A = b"first rom payload" * 50
ROM_B = b"second rom payload" * 40
ROM_C = b"third rom, held only inside its own archive" * 30
def _sha1(data: bytes) -> str:
return hashlib.sha1(data).hexdigest()
class TheLedgerCoversTheCollection(unittest.TestCase):
"""Read from the committed files: database.json and recipes/archives.json."""
@classmethod
def setUpClass(cls):
with (ROOT / "database.json").open(encoding="utf-8") as handle:
cls.db = json.load(handle)
cls.ledger = archive_build.load_ledger()
cls.archives = archive_build.database_archives(cls.db)
def test_every_database_archive_has_a_recipe(self):
absent = sorted(set(self.archives) - set(self.ledger))
self.assertEqual(
absent, [],
f"{len(absent)} archives without a recipe; run "
"`python scripts/romset_recipes.py --capture`",
)
def test_no_recipe_outlives_its_archive(self):
self.assertEqual(sorted(set(self.ledger) - set(self.archives)), [])
def test_each_recipe_names_the_container_the_database_holds(self):
wrong = [
path for path, record in self.ledger.items()
if path in self.archives and any(
record[key] != self.archives[path][key] for key in ("sha1", "md5", "size")
)
]
self.assertEqual(wrong, [], "recaptured archives keep a stale recipe")
def test_each_recipe_is_complete(self):
for path, record in self.ledger.items():
self.assertIn(record["rebuild"], ("exact", "canonical"), path)
self.assertEqual(record["sources"][-1], "collection", path)
self.assertTrue(record["members"], path)
for member in record["members"]:
self.assertEqual(set(member), {"name", "size", "crc32", "sha1"}, path)
if record.get("bytes_pinned"):
self.assertTrue(record["pins"], path)
def test_every_member_is_held_in_the_collection(self):
"""The list printed here is the acquisition list: a member neither held
as a loose file nor inside any archive the database lists."""
missing = archive_build.missing_members(self.ledger, self.db)
listing = "\n".join(
f" {m['archive']}: {m['member']} ({m['sha1']})" for m in missing
)
self.assertEqual(missing, [], f"members held nowhere:\n{listing}")
def test_the_file_is_written_the_one_way(self):
text = archive_build.ARCHIVES_FILE.read_text(encoding="utf-8")
self.assertEqual(text, archive_build.ledger_text(self.ledger))
def test_recipes_describe_the_archives_on_disk(self):
"""Names in order, sizes and crc32 against each central directory."""
checked = 0
for path, record in sorted(self.ledger.items()):
archive = ROOT / path
if not archive.is_file():
continue # a release asset not restored in this checkout
with zipfile.ZipFile(archive) as handle:
held = [
(info.filename, info.file_size, f"{info.CRC:08x}")
for info in handle.infolist() if not info.is_dir()
]
recorded = [(m["name"], m["size"], m["crc32"]) for m in record["members"]]
self.assertEqual(held, recorded, path)
checked += 1
if not checked:
self.skipTest("bios/ is not present")
class Fixture(unittest.TestCase):
"""A small collection: a loose ROM, a TorrentZip set, a plain ZIP."""
def setUp(self):
self.temp = tempfile.TemporaryDirectory(dir=TMP_ROOT)
self.root = Path(self.temp.name)
bios = self.root / "bios"
bios.mkdir()
(bios / "a.rom").write_bytes(ROM_A)
self.exact_path = bios / "set.zip"
self.exact_path.write_bytes(
build_torrentzip([("a.rom", ROM_A), ("b.rom", ROM_B), ("c.rom", ROM_C)])
)
self.other_path = bios / "other.zip"
with zipfile.ZipFile(self.other_path, "w", zipfile.ZIP_DEFLATED) as archive:
archive.writestr("dir/", b"")
archive.writestr("second.rom", ROM_B)
archive.writestr("first.rom", ROM_A)
self.db = {"files": {}}
for path in (bios / "a.rom", self.exact_path, self.other_path):
data = path.read_bytes()
self.db["files"][_sha1(data)] = {
"path": path.relative_to(self.root).as_posix(),
"name": path.name,
"sha1": _sha1(data),
"md5": hashlib.md5(data).hexdigest(),
"size": len(data),
}
def tearDown(self):
self.temp.cleanup()
def capture(self, **kwargs) -> dict[str, dict]:
records, failed = archive_build.capture(
self.db, kwargs.get("previous", {}), kwargs.get("index", {}),
kwargs.get("pins", {}), workers=1, log=lambda _line: None,
)
self.assertEqual(failed, {})
return records
class Capture(Fixture):
def setUp(self):
super().setUp()
self._cwd = Path.cwd()
import os
os.chdir(self.root)
def tearDown(self):
import os
os.chdir(self._cwd)
super().tearDown()
def test_a_torrentzip_archive_is_exact_and_another_canonical(self):
records = self.capture()
exact, other = records["bios/set.zip"], records["bios/other.zip"]
self.assertEqual(exact["rebuild"], "exact")
self.assertEqual(other["rebuild"], "canonical")
self.assertEqual(
[m["name"] for m in other["members"]], ["second.rom", "first.rom"],
"members keep the archive's own order",
)
self.assertEqual(other["directories"], ["dir/"])
self.assertEqual(other["members"][0]["sha1"], _sha1(ROM_B))
self.assertEqual(exact["sources"], ["collection"])
def test_dat_agreement_and_pins_are_recorded(self):
key = archive_build.member_key(
[{"name": n, "crc32": c} for n, c in (
("a.rom", self._crc(ROM_A)), ("b.rom", self._crc(ROM_B)),
("c.rom", self._crc(ROM_C)),
)]
)
md5 = hashlib.md5(self.exact_path.read_bytes()).hexdigest()
records = self.capture(
index={key: {"mame:MAME mame0289"}}, pins={md5: {"platform:demo"}}
)
record = records["bios/set.zip"]
self.assertEqual(record["sources"], ["mame:MAME mame0289", "collection"])
self.assertEqual((record["bytes_pinned"], record["pins"]), (True, ["platform:demo"]))
self.assertNotIn("bytes_pinned", records["bios/other.zip"])
def test_an_unchanged_archive_is_not_read_again(self):
previous = self.capture()
previous["bios/set.zip"]["members"][0]["sha1"] = "kept"
records = self.capture(previous=previous)
self.assertEqual(records["bios/set.zip"]["members"][0]["sha1"], "kept")
def test_zlib_ng_refuses_to_capture(self):
with mock.patch.object(archive_build, "zlib_ng_version", lambda: "2.2.4"):
with self.assertRaises(archive_build.ZlibNgError):
self.capture()
def test_composite_and_zipped_pins_name_no_container(self):
platforms = self.root / "platforms"
platforms.mkdir()
(platforms / "_registry.yml").write_text(
"platforms:\n"
" plain:\n config: plain.yml\n"
" composite:\n config: composite.yml\n zip_digest: composite\n",
encoding="utf-8",
)
entry = " - name: set.zip\n destination: set.zip\n"
(platforms / "plain.yml").write_text(
"platform: Plain\nsystems:\n arcade:\n files:\n"
f"{entry} md5: {'a' * 32}\n"
f"{entry} md5: {'b' * 32}\n zipped_file: a.rom\n",
encoding="utf-8",
)
(platforms / "composite.yml").write_text(
"platform: Composite\nsystems:\n arcade:\n files:\n"
f"{entry} md5: {'c' * 32}\n",
encoding="utf-8",
)
emulators = self.root / "emulators"
emulators.mkdir()
(emulators / "core.yml").write_text(
"emulator: Core\ntype: libretro\nfiles:\n"
f" - name: set.zip\n sha1: {'d' * 40}\n",
encoding="utf-8",
)
pins = archive_build.container_pins(str(platforms), str(emulators))
self.assertEqual(pins, {"a" * 32: {"platform:plain"}, "d" * 40: {"profile:core"}})
@staticmethod
def _crc(data: bytes) -> str:
import zlib
return f"{zlib.crc32(data) & 0xFFFFFFFF:08x}"
class Build(Fixture):
def setUp(self):
super().setUp()
import os
self._cwd = Path.cwd()
os.chdir(self.root)
self.ledger = self.capture()
os.chdir(self._cwd)
self.atoms = archive_build.AtomIndex.from_collection(self.ledger, self.db, self.root)
self.out = self.root / "out"
def build(self, path: str):
return archive_build.build_archive(self.ledger[path], self.atoms, self.out, self.root)
def test_an_exact_recipe_rebuilds_the_bytes(self):
result = self.build("bios/set.zip")
self.assertEqual(result.status, "exact")
built = self.out / "set.zip"
self.assertEqual(built.read_bytes(), self.exact_path.read_bytes())
self.assertEqual(result.own_members, 1, "c.rom is held only in set.zip")
def test_members_come_from_a_loose_file_then_another_archive(self):
own = self.root / "bios" / "set.zip"
self.assertEqual(self.atoms.find(_sha1(ROM_A), own).member, None)
self.assertEqual(self.atoms.find(_sha1(ROM_B), own).path.name, "other.zip")
self.assertEqual(self.atoms.find(_sha1(ROM_C), own).path, own)
def test_a_canonical_recipe_is_checked_by_its_members(self):
result = self.build("bios/other.zip")
self.assertEqual(result.status, "canonical")
with zipfile.ZipFile(self.out / "other.zip") as archive:
self.assertEqual(sorted(archive.namelist()), ["first.rom", "second.rom"])
self.assertTrue(archive.comment.startswith(b"TORRENTZIPPED-"))
def test_a_member_held_nowhere_makes_it_impossible(self):
record = dict(self.ledger["bios/set.zip"])
record["members"] = record["members"] + [
{"name": "gone.rom", "size": 1, "crc32": "00000000", "sha1": "f" * 40}
]
result = archive_build.build_archive(record, self.atoms, self.out, self.root)
self.assertEqual(result.status, "impossible")
self.assertEqual([m["name"] for m in result.missing], ["gone.rom"])
self.assertFalse(self.out.exists())
def test_a_mismatch_is_never_promoted(self):
record = dict(self.ledger["bios/set.zip"], sha1="0" * 40)
result = archive_build.build_archive(record, self.atoms, self.out, self.root)
self.assertEqual(result.status, "failed")
self.assertIn("is not the recorded", result.detail)
self.assertEqual(list(self.out.rglob("*")), [], "no file, no scratch left")
def test_a_canonical_mismatch_is_never_promoted(self):
record = dict(self.ledger["bios/other.zip"])
record["members"] = [dict(m) for m in record["members"]]
record["members"][0]["crc32"] = "00000000"
result = archive_build.build_archive(record, self.atoms, self.out, self.root)
self.assertEqual(result.status, "failed")
self.assertEqual(list(self.out.rglob("*")), [])
def test_zlib_ng_never_claims_exact(self):
with mock.patch.object(archive_build, "zlib_ng_version", lambda: "2.2.4"):
result = self.build("bios/set.zip")
self.assertEqual(result.status, "canonical")
self.assertIn("zlib-ng 2.2.4", result.detail)
def test_an_archive_is_named_by_path_or_by_name(self):
self.assertEqual(
[r["path"] for r in archive_build.select(self.ledger, "set.zip", self.root)],
["bios/set.zip"],
)
self.assertEqual(
len(archive_build.select(self.ledger, str(self.exact_path), self.root)), 1
)
self.assertEqual(archive_build.select(self.ledger, "none.zip", self.root), [])
def test_missing_members_reads_the_committed_data(self):
self.assertEqual(archive_build.missing_members(self.ledger, self.db), [])
del self.db["files"][_sha1(self.exact_path.read_bytes())]
missing = archive_build.missing_members(self.ledger, self.db)
self.assertEqual(
[(m["archive"], m["member"]) for m in missing], [("bios/set.zip", "c.rom")]
)
def test_the_command_reports_each_archive(self):
archives = self.root / "archives.json"
archives.write_text(archive_build.ledger_text(self.ledger), encoding="utf-8")
db = self.root / "database.json"
db.write_text(json.dumps(self.db), encoding="utf-8")
run = subprocess.run(
[
sys.executable, str(ROOT / "scripts" / "romset_recipes.py"),
"--build-all", "--output-dir", str(self.out), "--discard",
"--bios-dir", str(self.root / "bios"), "--db", str(db),
"--archives", str(archives),
],
capture_output=True, text=True, check=False, timeout=120,
)
self.assertEqual(run.returncode, 0, run.stdout + run.stderr)
self.assertIn("EXACT: bios/set.zip", run.stdout)
self.assertIn("CANONICAL: bios/other.zip", run.stdout)
self.assertIn("2 archives: 1 rebuilt exact, 1 canonical, 0 impossible", run.stdout)
self.assertEqual(list(self.out.rglob("*.zip")), [], "--discard keeps nothing")
if __name__ == "__main__":
unittest.main()
+7
View File
@@ -66,6 +66,13 @@ KEPT = {
"bios/Mattel/Aquarius/.variants/aquarius_ar.zip.73026a45": (
"a platform pack copy; its TMCTV.png is held only here"
),
# Each carries slot ROMs beside its own set. Its own members, TorrentZipped,
# are byte for byte the archive under the newer MAME name in Arcade/MAME,
# so trimming it makes a duplicate: whether the old name stays is the
# owner's storage decision (2026-11-06).
"bios/Arcade/Arcade/fccpu20.zip": "MAME 0.196 to 0.258 fccpu20, now sys68k_cpu20.zip",
"bios/Arcade/Arcade/fcisio1.zip": "MAME 0.196 to 0.258 fcisio1, now sys68k_isio1.zip",
"bios/Arcade/Arcade/interpod.zip": "MAME 0.177 to 0.215 interpod, now cbm_interpod.zip",
"bios/Sega/Dreamcast/dc.zip": (
"RetroBat's dc/dc.zip holds dc_boot.bin and dc_flash.bin, the names the "
"Dreamcast BIOS goes by outside MAME, not MAME's dc set"
+100
View File
@@ -246,6 +246,25 @@ class ReportsOverAFixture(unittest.TestCase):
self.assertEqual(len(report["unreproducible"]), 1)
self.assertEqual(report["unreproducible"][0]["recipes_tried"], ["v1"])
def test_a_composite_md5_is_no_container_target(self):
"""Recalbox hashes a ZIP by its members: its md5 names no archive bytes."""
platforms = self.root / "platforms"
platforms.mkdir()
(platforms / "_registry.yml").write_text(
"platforms:\n demo:\n config: demo.yml\n zip_digest: composite\n",
encoding="utf-8",
)
(platforms / "demo.yml").write_text(
"platform: Demo\nsystems:\n arcade:\n files:\n"
" - name: set.zip\n destination: set.zip\n"
f" md5: {'f' * 32}\n",
encoding="utf-8",
)
self.db["indexes"]["by_md5"] = {}
report = missing_report(self.recipes, self.pool, self.db, str(platforms))
self.assertEqual(report, {"constructible": [], "unreproducible": [], "no_recipe": []})
DAT = """<?xml version="1.0"?>
<datafile>
@@ -410,6 +429,87 @@ class ListXmlRecipes(unittest.TestCase):
[m["name"] for m in entries[0]["members"]], ["own.rom", "shared.rom"]
)
def test_a_set_kept_under_another_name_is_matched_by_members(self):
"""aga.zip holds exactly MAME's isa_aga: the entry carries the
collection's name and keeps the upstream one in ``set``."""
from scripts.scraper.romset_dat_importer import member_list
child = [
{"name": "own.rom", "crc32": _crc(ROM_B)},
{"name": "shared.rom", "crc32": _crc(ROM_A)},
]
entries = listxml_entries(
self._opener(self._document()), "MAME 0.289", {"other.zip"},
by_members={member_list(child): {"mine.zip"}},
)
self.assertEqual(
[(entry["name"], entry["set"]) for entry in entries],
[("mine.zip", "childset")],
)
self.assertEqual(
[m["name"] for m in entries[0]["members"]], ["own.rom", "shared.rom"]
)
def test_matching_by_members_keeps_the_named_sets(self):
from scripts.scraper.romset_dat_importer import member_list
entries = listxml_entries(
self._opener(self._document()), "MAME 0.289", {"parentset.zip"},
by_members={member_list([{"name": "nothing", "crc32": "0"}]): {"x.zip"}},
)
self.assertEqual([entry["name"] for entry in entries], ["parentset.zip"])
def test_one_entry_per_archive_its_own_set_first(self):
"""Thirty keyboard sets share one ROM: the set named like the archive wins."""
from scripts.scraper.romset_dat_importer import one_set_per_archive
members = [{"name": "kbd.rom", "crc32": "12345678"}]
entries = one_set_per_archive([
{"name": "a500kbd.zip", "set": "a2000kbd", "members": members},
{"name": "a500kbd.zip", "set": "a500kbd", "members": members},
{"name": "a500kbd.zip", "set": "a600kbd", "members": members},
])
self.assertEqual([entry["set"] for entry in entries], ["a500kbd"])
def test_a_logiqx_set_is_matched_by_members_too(self):
from scripts.scraper.romset_dat_importer import member_list
members = [
{"name": "01", "crc32": _crc(ROM_A)},
{"name": "23", "crc32": _crc(ROM_B)},
]
entries = recipe_entries(
parse_logiqx(DAT.format(crc_a=_crc(ROM_A), crc_b=_crc(ROM_B))),
keep=set(), by_members={member_list(members): {"salora.zip"}},
)
self.assertEqual(
[(entry["name"], entry["set"]) for entry in entries],
[("salora.zip", "manager")],
)
def test_the_collection_names_and_member_lists(self):
"""A .variants/ copy stands for the name its tag is cut from."""
from scripts.scraper.romset_dat_importer import collection_archives, member_list
with tempfile.TemporaryDirectory(dir=TMP_ROOT) as temp:
root = Path(temp)
plain, variant = root / "set.zip", root / "other.zip.1a2b3c4d"
for path in (plain, variant):
with zipfile.ZipFile(path, "w") as archive:
archive.writestr("a.rom", ROM_A)
loose = root / "loose.bin"
loose.write_bytes(ROM_B)
db = root / "db.json"
db.write_text(json.dumps({"files": {
str(i): {"path": str(path)} for i, path in enumerate((plain, variant, loose))
}}), encoding="utf-8")
names, by_members = collection_archives(str(db))
self.assertEqual(names, {"set.zip", "other.zip"})
self.assertEqual(
by_members,
{member_list([{"name": "a.rom", "crc32": _crc(ROM_A)}]): {"set.zip", "other.zip"}},
)
def test_a_set_holding_a_missing_catalog_dump_is_kept(self):
"""fm7740sx carries three No-Intro dumps the collection lacks. No
profile names it, so the missing report named no place to find them."""
+1 -1
View File
@@ -752,7 +752,7 @@ class TheRomsetLineMatchesByCrc(unittest.TestCase):
if main is None or not (REPO_ROOT / main["path"]).is_file():
self.skipTest("the main neogeo.zip is not in this checkout")
line = gs._romset_line(main)
self.assertIn("Holds every ROM of MAME 0.190 to 0.289", line)
self.assertIn("Holds every ROM of MAME 0.177 to 0.289", line)
self.assertNotIn("No match", line)
+66
View File
@@ -155,6 +155,72 @@ def _as_file(test: unittest.TestCase, data: bytes) -> str:
return path
class StreamingWriter(unittest.TestCase):
"""Members stream through in chunks, one in memory at a time."""
def _members(self) -> list[tuple[str, bytes]]:
import random
noise = random.Random(7).randbytes(70_000)
return [("b.rom", noise + bytes(50_000) + noise[:9_000]), ("A.rom", b"x" * 1_000)]
def test_chunked_input_gives_the_one_shot_bytes(self):
"""Deflate output does not depend on how its input is cut."""
import io
import torrentzip
members = self._members()
whole = torrentzip.build_torrentzip(members)
saved = torrentzip.CHUNK
try:
for chunk in (1, 333, 4096):
torrentzip.CHUNK = chunk
self.assertEqual(torrentzip.build_torrentzip(members), whole, chunk)
finally:
torrentzip.CHUNK = saved
sink = io.BytesIO()
written = torrentzip.write_torrentzip(
sink, [(name, lambda d=data: io.BytesIO(d)) for name, data in members]
)
self.assertEqual(sink.getvalue(), whole)
self.assertEqual(
[(w.name, w.size, w.sha1) for w in written],
[(name, len(data), hashlib.sha1(data).hexdigest())
for name, data in sorted(members, key=lambda m: m[0].lower())],
)
def test_a_member_past_32_bits_takes_a_zip64_field(self):
"""cspect-next-8gb.zip holds an 8 GiB image: its size needs ZIP64."""
import torrentzip
members = self._members()
saved = torrentzip.ZIP32_LIMIT
torrentzip.ZIP32_LIMIT = 60_000
try:
data = torrentzip.build_torrentzip(members)
finally:
torrentzip.ZIP32_LIMIT = saved
with zipfile.ZipFile(_as_file(self, data)) as zf:
self.assertIsNone(zf.testzip())
self.assertEqual(
{info.filename: zf.read(info) for info in zf.infolist()}, dict(members)
)
self.assertNotEqual(data, build_torrentzip(members))
def test_zlib_ng_is_recognised(self):
"""python.org 3.14 on Windows links zlib-ng, which gives other bytes."""
from types import SimpleNamespace
from torrentzip import zlib_ng_version
stock = SimpleNamespace(ZLIB_RUNTIME_VERSION="1.3.1")
windows = SimpleNamespace(ZLIB_RUNTIME_VERSION="1.3.1.zlib-ng")
binding = SimpleNamespace(ZLIB_RUNTIME_VERSION="1.3.1", ZLIBNG_VERSION="2.2.4")
self.assertIsNone(zlib_ng_version(stock))
self.assertEqual(zlib_ng_version(windows), "1.3.1.zlib-ng")
self.assertEqual(zlib_ng_version(binding), "2.2.4")
class CheckModeExitCode(unittest.TestCase):
"""--check printed `conform: False` and exited 0, so no script could use it."""