fix: read fbneo dats at one checked commit

The FBNeo import recorded the listing's blob sha for bytes fetched from a
branch URL a CDN caches, and --fetch <revision> still read the tip. The
revision is now resolved to a commit, listed and downloaded there, each
DAT is held against its blob sha, and a DAT read at a named revision is
labelled with it without replacing the tip's recorded blobs.

scripts/scraper/romset_dat_importer.py

@@ -11,6 +11,7 @@ in its repository, so both are fetched without a browser:
     python -m scripts.scraper.romset_dat_importer --source mame --fetch
     python -m scripts.scraper.romset_dat_importer --source mame --fetch mame0289
     python -m scripts.scraper.romset_dat_importer --source fbneo --fetch
+    python -m scripts.scraper.romset_dat_importer --source fbneo --fetch v1.0.0.02
     python -m scripts.scraper.romset_dat_importer --source mame --pack local.dat
 
 Only sets the collection has a reason to know are kept, so the committed
@@ -33,6 +34,7 @@ import os
 import re
 import shutil
 import sys
+import urllib.parse
 import urllib.request
 import zipfile
 from collections.abc import Callable, Iterator
@@ -374,7 +376,7 @@ MAME_LISTXML_URL = (
     "https://github.com/mamedev/mame/releases/download/{tag}/{tag}lx.zip"
 )
 MAME_LATEST_API = "https://api.github.com/repos/mamedev/mame/releases/latest"
-FBNEO_DATS_API = "https://api.github.com/repos/libretro/FBNeo/contents/dats"
+FBNEO_API = "https://api.github.com/repos/libretro/FBNeo"
 
 
 def _api_json(url: str) -> object:
@@ -400,7 +402,8 @@ def latest_mame_tag() -> str:
 
 
 def fetch_tag(source: str, requested: str) -> str:
-    """The tag a ``--fetch`` names: bare, the newest MAME release."""
+    """The revision a ``--fetch`` names: bare, the newest MAME release or
+    the tip of FBNeo's default branch."""
     if source == "mame" and requested == "latest":
         tag = latest_mame_tag()
         print(f"  Derniere release MAME : {tag}")
@@ -408,6 +411,10 @@ def fetch_tag(source: str, requested: str) -> str:
     return requested
 
 
+class BlobMismatchError(ValueError):
+    """A downloaded DAT is not the blob the repository listing names."""
+
+
 def git_blob_sha(data: bytes) -> str:
     """The sha git gives a blob, so a cached file can be checked against a
     repository listing without any other state."""
@@ -452,7 +459,15 @@ def fetch_pack(
             {},
         )
 
-    listing = _api_json(FBNEO_DATS_API)
+    # The revision is resolved to a commit first and everything is read at
+    # that commit: a branch URL is served through a CDN cache, so the bytes
+    # could predate the listing that named their blob.
+    ref = "HEAD" if tag == "latest" else urllib.parse.quote(tag, safe="")
+    commit = _api_json(f"{FBNEO_API}/commits/{ref}")
+    sha = commit.get("sha") if isinstance(commit, dict) else None
+    if not sha:
+        raise BlobMismatchError(f"libretro/FBNeo has no revision {tag}")
+    listing = _api_json(f"{FBNEO_API}/contents/dats?ref={sha}")
     target = cache_dir / "fbneo-dats"
     target.mkdir(parents=True, exist_ok=True)
     blobs: dict[str, str] = {}
@@ -465,7 +480,17 @@ def fetch_pack(
         if destination.is_file() and git_blob_sha(destination.read_bytes()) != item["sha"]:
             destination.unlink()
         _download(item["download_url"], destination)
+        if git_blob_sha(destination.read_bytes()) != item["sha"]:
+            destination.unlink()
+            raise BlobMismatchError(
+                f"{item['name']} downloaded from {item['download_url']} is not "
+                f"blob {item['sha']} of libretro/FBNeo {sha}"
+            )
         blobs[item["name"]] = item["sha"]
+    # A DAT the revision does not carry is not read from an older fetch.
+    for stale in target.glob("*"):
+        if stale.is_file() and stale.name not in blobs:
+            stale.unlink()
     return target, blobs
 
 
@@ -509,8 +534,9 @@ def main() -> int:
         "--fetch",
         nargs="?",
         const="latest",
-        help="download upstream instead of using --pack "
-        "(a MAME tag such as mame0289, or bare for the newest release)",
+        help="download upstream instead of using --pack: a MAME tag such as "
+        "mame0289 or an FBNeo tag, branch or commit; bare, the newest MAME "
+        "release or the tip of FBNeo's default branch",
     )
     parser.add_argument(
         "--cache-dir", default=".cache/dats",
@@ -547,11 +573,24 @@ def main() -> int:
     args = parser.parse_args()
 
     blobs: dict[str, str] = {}
+    # FBNeo's DAT headers keep one version across commits, so a DAT read at
+    # a named revision is labelled with it: under the bare header name it
+    # merged into the tip's recipes. Its blobs describe that revision, not
+    # the upstream state the freshness check compares with, and are not
+    # recorded.
+    revision = ""
     if args.fetch:
-        pack, blobs = fetch_pack(
-            args.source, fetch_tag(args.source, args.fetch), Path(args.cache_dir)
-        )
+        try:
+            pack, blobs = fetch_pack(
+                args.source, fetch_tag(args.source, args.fetch), Path(args.cache_dir)
+            )
+        except BlobMismatchError as exc:
+            print(f"ERROR: {exc}", file=sys.stderr)
+            return 1
         print(f"  Recupere {pack}")
+        if args.source == "fbneo" and args.fetch != "latest":
+            revision = args.fetch
+            blobs = {}
     elif args.pack:
         pack = Path(args.pack)
     else:
@@ -596,6 +635,8 @@ def main() -> int:
         if not dat.roms:
             skipped += 1
             continue
+        if revision:
+            dat.name = f"{dat.name or member} @ {revision}"
         found = recipe_entries(dat, keep, targets)
         if not found:
             continue

tests/test_romset_recipes.py

@@ -694,6 +694,88 @@ class AnOversizedDatIsNamed(unittest.TestCase):
 
 
 
+FBNEO_DAT = (
+    b'<?xml version="1.0"?>\n<datafile><header><name>FinalBurn Neo - Demo</name>'
+    b"<version>1.0.0.03</version></header>"
+    b'<game name="demobios"><description>Demo BIOS</description>'
+    b'<rom name="a.bin" size="17" crc="' + _crc(ROM_A).encode() + b'"/></game>'
+    b"</datafile>\n"
+)
+
+
+class FbneoFetchReadsOneRevision(unittest.TestCase):
+    """The FBNeo import listed blobs at the tip but downloaded branch URLs a
+    CDN caches, and --fetch <revision> read the tip whatever was named."""
+
+    def setUp(self):
+        from scripts.scraper import romset_dat_importer as importer
+
+        self.importer = importer
+        self._tmp = tempfile.TemporaryDirectory(dir=TMP_ROOT)
+        self.dir = Path(self._tmp.name)
+        self.calls: list[str] = []
+        self.served = FBNEO_DAT
+
+    def tearDown(self):
+        self._tmp.cleanup()
+
+    def _api(self, url: str) -> object:
+        self.calls.append(url)
+        if "/commits/" in url:
+            return {"sha": "c0ffee"}
+        return [{
+            "type": "file", "name": "Demo.dat",
+            "sha": self.importer.git_blob_sha(FBNEO_DAT),
+            "download_url": "https://raw.githubusercontent.com/libretro/FBNeo/c0ffee/dats/Demo.dat",
+        }]
+
+    def _download(self, url: str, destination: Path) -> Path:
+        destination.write_bytes(self.served)
+        return destination
+
+    def _patched(self):
+        from unittest import mock
+
+        return mock.patch.multiple(
+            self.importer, _api_json=self._api, _download=self._download
+        )
+
+    def test_bytes_that_are_not_the_listed_blob_are_refused(self):
+        self.served = FBNEO_DAT.replace(b"1.0.0.03", b"1.0.0.02")
+        with self._patched(), self.assertRaises(self.importer.BlobMismatchError):
+            self.importer.fetch_pack("fbneo", "latest", self.dir)
+        self.assertFalse((self.dir / "fbneo-dats" / "Demo.dat").exists())
+
+    def test_the_named_revision_is_the_one_listed(self):
+        with self._patched():
+            target, blobs = self.importer.fetch_pack("fbneo", "v1.0.0.02", self.dir)
+        self.assertEqual(
+            self.calls,
+            [f"{self.importer.FBNEO_API}/commits/v1.0.0.02",
+             f"{self.importer.FBNEO_API}/contents/dats?ref=c0ffee"],
+        )
+        self.assertEqual(list(blobs), ["Demo.dat"])
+
+    def test_a_revision_import_is_labelled_and_leaves_the_tip_blobs(self):
+        import contextlib
+        import io
+        from unittest import mock
+
+        output = self.dir / "fbneo.json"
+        output.write_text(json.dumps({"upstream": {"blobs": {"Demo.dat": "tip"}}}))
+        argv = ["romset_dat_importer", "--source", "fbneo", "--fetch", "v1.0.0.02",
+                "--all", "--cache-dir", str(self.dir), "--output", str(output)]
+        with self._patched(), mock.patch.object(sys, "argv", argv), \
+                contextlib.redirect_stdout(io.StringIO()):
+            self.assertEqual(self.importer.main(), 0)
+        snapshot = json.loads(output.read_text())
+        self.assertEqual(snapshot["upstream"]["blobs"], {"Demo.dat": "tip"})
+        self.assertEqual(
+            [entry["dat"] for entry in snapshot["entries"]],
+            ["FinalBurn Neo - Demo @ v1.0.0.02"],
+        )
+
+
 class LooseRomsAreAtoms(unittest.TestCase):
     """AtomPool read only ZIP members: nmk004.zip stayed unidentified while
     its one ROM sat loose in bios/Arcade/FBNeo/nmk004.bin."""

wiki/tools.md

@@ -651,7 +651,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6
 | `generate_site.py` | Generate all MkDocs site pages (this documentation) |
 | `validate_site.py` | Validate rendered metadata, headings, image alternatives, JSON-LD, local resources, links and fragments |
 | `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held |
-| `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha |
+| `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch <revision>` reads FBNeo at that tag, branch or commit |
 | `deterministic_zip.py` | Rebuild MAME BIOS ZIPs deterministically (same ROMs = same hash) |
 | `torrentzip.py` | Build TorrentZip archives for MAME/FBNeo ROM sets (archive bytes depend only on contents) |
 | `crypto_verify.py` | 3DS RSA signature and AES crypto verification |
@@ -756,8 +756,17 @@ FBNeo keeps its DATs in the repository.
 python -m scripts.scraper.romset_dat_importer --source mame --fetch mame0289
 python -m scripts.scraper.romset_dat_importer --source mame --fetch mame0250
 python -m scripts.scraper.romset_dat_importer --source fbneo --fetch
+python -m scripts.scraper.romset_dat_importer --source fbneo --fetch v1.0.0.02
 ```
 
+FBNeo DATs are read at one commit: the revision named (tag, branch or commit,
+the default branch when bare) is resolved first, and every DAT downloaded is
+held against the blob sha the listing gives, so a cached branch URL cannot
+serve older bytes. Their headers keep one version across commits, so DATs read
+at a named revision are labelled `<DAT name> @ <revision>` and accumulate
+beside the tip's. Only an import of the tip records the blobs the freshness
+check compares with.
+
 Snapshots land in `recipes/`, not `provenance/`: the latter holds dump catalogues
 and a recipe is not one. Versions accumulate rather than replace each other,
 because a platform pins the archive of whichever version its list was built