diff --git a/scripts/scraper/romset_dat_importer.py b/scripts/scraper/romset_dat_importer.py index 78ef91c0..80e663cd 100644 --- a/scripts/scraper/romset_dat_importer.py +++ b/scripts/scraper/romset_dat_importer.py @@ -11,6 +11,7 @@ in its repository, so both are fetched without a browser: python -m scripts.scraper.romset_dat_importer --source mame --fetch python -m scripts.scraper.romset_dat_importer --source mame --fetch mame0289 python -m scripts.scraper.romset_dat_importer --source fbneo --fetch + python -m scripts.scraper.romset_dat_importer --source fbneo --fetch v1.0.0.02 python -m scripts.scraper.romset_dat_importer --source mame --pack local.dat Only sets the collection has a reason to know are kept, so the committed @@ -33,6 +34,7 @@ import os import re import shutil import sys +import urllib.parse import urllib.request import zipfile from collections.abc import Callable, Iterator @@ -374,7 +376,7 @@ MAME_LISTXML_URL = ( "https://github.com/mamedev/mame/releases/download/{tag}/{tag}lx.zip" ) MAME_LATEST_API = "https://api.github.com/repos/mamedev/mame/releases/latest" -FBNEO_DATS_API = "https://api.github.com/repos/libretro/FBNeo/contents/dats" +FBNEO_API = "https://api.github.com/repos/libretro/FBNeo" def _api_json(url: str) -> object: @@ -400,7 +402,8 @@ def latest_mame_tag() -> str: def fetch_tag(source: str, requested: str) -> str: - """The tag a ``--fetch`` names: bare, the newest MAME release.""" + """The revision a ``--fetch`` names: bare, the newest MAME release or + the tip of FBNeo's default branch.""" if source == "mame" and requested == "latest": tag = latest_mame_tag() print(f" Derniere release MAME : {tag}") @@ -408,6 +411,10 @@ def fetch_tag(source: str, requested: str) -> str: return requested +class BlobMismatchError(ValueError): + """A downloaded DAT is not the blob the repository listing names.""" + + def git_blob_sha(data: bytes) -> str: """The sha git gives a blob, so a cached file can be checked against a repository listing without any other state.""" @@ -452,7 +459,15 @@ def fetch_pack( {}, ) - listing = _api_json(FBNEO_DATS_API) + # The revision is resolved to a commit first and everything is read at + # that commit: a branch URL is served through a CDN cache, so the bytes + # could predate the listing that named their blob. + ref = "HEAD" if tag == "latest" else urllib.parse.quote(tag, safe="") + commit = _api_json(f"{FBNEO_API}/commits/{ref}") + sha = commit.get("sha") if isinstance(commit, dict) else None + if not sha: + raise BlobMismatchError(f"libretro/FBNeo has no revision {tag}") + listing = _api_json(f"{FBNEO_API}/contents/dats?ref={sha}") target = cache_dir / "fbneo-dats" target.mkdir(parents=True, exist_ok=True) blobs: dict[str, str] = {} @@ -465,7 +480,17 @@ def fetch_pack( if destination.is_file() and git_blob_sha(destination.read_bytes()) != item["sha"]: destination.unlink() _download(item["download_url"], destination) + if git_blob_sha(destination.read_bytes()) != item["sha"]: + destination.unlink() + raise BlobMismatchError( + f"{item['name']} downloaded from {item['download_url']} is not " + f"blob {item['sha']} of libretro/FBNeo {sha}" + ) blobs[item["name"]] = item["sha"] + # A DAT the revision does not carry is not read from an older fetch. + for stale in target.glob("*"): + if stale.is_file() and stale.name not in blobs: + stale.unlink() return target, blobs @@ -509,8 +534,9 @@ def main() -> int: "--fetch", nargs="?", const="latest", - help="download upstream instead of using --pack " - "(a MAME tag such as mame0289, or bare for the newest release)", + help="download upstream instead of using --pack: a MAME tag such as " + "mame0289 or an FBNeo tag, branch or commit; bare, the newest MAME " + "release or the tip of FBNeo's default branch", ) parser.add_argument( "--cache-dir", default=".cache/dats", @@ -547,11 +573,24 @@ def main() -> int: args = parser.parse_args() blobs: dict[str, str] = {} + # FBNeo's DAT headers keep one version across commits, so a DAT read at + # a named revision is labelled with it: under the bare header name it + # merged into the tip's recipes. Its blobs describe that revision, not + # the upstream state the freshness check compares with, and are not + # recorded. + revision = "" if args.fetch: - pack, blobs = fetch_pack( - args.source, fetch_tag(args.source, args.fetch), Path(args.cache_dir) - ) + try: + pack, blobs = fetch_pack( + args.source, fetch_tag(args.source, args.fetch), Path(args.cache_dir) + ) + except BlobMismatchError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 print(f" Recupere {pack}") + if args.source == "fbneo" and args.fetch != "latest": + revision = args.fetch + blobs = {} elif args.pack: pack = Path(args.pack) else: @@ -596,6 +635,8 @@ def main() -> int: if not dat.roms: skipped += 1 continue + if revision: + dat.name = f"{dat.name or member} @ {revision}" found = recipe_entries(dat, keep, targets) if not found: continue diff --git a/tests/test_romset_recipes.py b/tests/test_romset_recipes.py index 3e5d30ed..e4d57feb 100644 --- a/tests/test_romset_recipes.py +++ b/tests/test_romset_recipes.py @@ -694,6 +694,88 @@ class AnOversizedDatIsNamed(unittest.TestCase): +FBNEO_DAT = ( + b'\n
FinalBurn Neo - Demo' + b"1.0.0.03
" + b'Demo BIOS' + b'' + b"\n" +) + + +class FbneoFetchReadsOneRevision(unittest.TestCase): + """The FBNeo import listed blobs at the tip but downloaded branch URLs a + CDN caches, and --fetch read the tip whatever was named.""" + + def setUp(self): + from scripts.scraper import romset_dat_importer as importer + + self.importer = importer + self._tmp = tempfile.TemporaryDirectory(dir=TMP_ROOT) + self.dir = Path(self._tmp.name) + self.calls: list[str] = [] + self.served = FBNEO_DAT + + def tearDown(self): + self._tmp.cleanup() + + def _api(self, url: str) -> object: + self.calls.append(url) + if "/commits/" in url: + return {"sha": "c0ffee"} + return [{ + "type": "file", "name": "Demo.dat", + "sha": self.importer.git_blob_sha(FBNEO_DAT), + "download_url": "https://raw.githubusercontent.com/libretro/FBNeo/c0ffee/dats/Demo.dat", + }] + + def _download(self, url: str, destination: Path) -> Path: + destination.write_bytes(self.served) + return destination + + def _patched(self): + from unittest import mock + + return mock.patch.multiple( + self.importer, _api_json=self._api, _download=self._download + ) + + def test_bytes_that_are_not_the_listed_blob_are_refused(self): + self.served = FBNEO_DAT.replace(b"1.0.0.03", b"1.0.0.02") + with self._patched(), self.assertRaises(self.importer.BlobMismatchError): + self.importer.fetch_pack("fbneo", "latest", self.dir) + self.assertFalse((self.dir / "fbneo-dats" / "Demo.dat").exists()) + + def test_the_named_revision_is_the_one_listed(self): + with self._patched(): + target, blobs = self.importer.fetch_pack("fbneo", "v1.0.0.02", self.dir) + self.assertEqual( + self.calls, + [f"{self.importer.FBNEO_API}/commits/v1.0.0.02", + f"{self.importer.FBNEO_API}/contents/dats?ref=c0ffee"], + ) + self.assertEqual(list(blobs), ["Demo.dat"]) + + def test_a_revision_import_is_labelled_and_leaves_the_tip_blobs(self): + import contextlib + import io + from unittest import mock + + output = self.dir / "fbneo.json" + output.write_text(json.dumps({"upstream": {"blobs": {"Demo.dat": "tip"}}})) + argv = ["romset_dat_importer", "--source", "fbneo", "--fetch", "v1.0.0.02", + "--all", "--cache-dir", str(self.dir), "--output", str(output)] + with self._patched(), mock.patch.object(sys, "argv", argv), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(self.importer.main(), 0) + snapshot = json.loads(output.read_text()) + self.assertEqual(snapshot["upstream"]["blobs"], {"Demo.dat": "tip"}) + self.assertEqual( + [entry["dat"] for entry in snapshot["entries"]], + ["FinalBurn Neo - Demo @ v1.0.0.02"], + ) + + class LooseRomsAreAtoms(unittest.TestCase): """AtomPool read only ZIP members: nmk004.zip stayed unidentified while its one ROM sat loose in bios/Arcade/FBNeo/nmk004.bin.""" diff --git a/wiki/tools.md b/wiki/tools.md index fdb2d81b..840fdbc8 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -651,7 +651,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6 | `generate_site.py` | Generate all MkDocs site pages (this documentation) | | `validate_site.py` | Validate rendered metadata, headings, image alternatives, JSON-LD, local resources, links and fragments | | `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held | -| `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha | +| `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch ` reads FBNeo at that tag, branch or commit | | `deterministic_zip.py` | Rebuild MAME BIOS ZIPs deterministically (same ROMs = same hash) | | `torrentzip.py` | Build TorrentZip archives for MAME/FBNeo ROM sets (archive bytes depend only on contents) | | `crypto_verify.py` | 3DS RSA signature and AES crypto verification | @@ -756,8 +756,17 @@ FBNeo keeps its DATs in the repository. python -m scripts.scraper.romset_dat_importer --source mame --fetch mame0289 python -m scripts.scraper.romset_dat_importer --source mame --fetch mame0250 python -m scripts.scraper.romset_dat_importer --source fbneo --fetch +python -m scripts.scraper.romset_dat_importer --source fbneo --fetch v1.0.0.02 ``` +FBNeo DATs are read at one commit: the revision named (tag, branch or commit, +the default branch when bare) is resolved first, and every DAT downloaded is +held against the blob sha the listing gives, so a cached branch URL cannot +serve older bytes. Their headers keep one version across commits, so DATs read +at a named revision are labelled ` @ ` and accumulate +beside the tip's. Only an import of the tip records the blobs the freshness +check compares with. + Snapshots land in `recipes/`, not `provenance/`: the latter holds dump catalogues and a recipe is not one. Versions accumulate rather than replace each other, because a platform pins the archive of whichever version its list was built