chore: merge audit fixes (scrapers)

scripts/check_freshness.py

@@ -40,6 +40,7 @@ import shutil
 import subprocess
 import sys
 import tempfile
+from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import asdict, dataclass, field
 from datetime import datetime, timezone
@@ -56,6 +57,7 @@ from common import (  # noqa: E402
     list_registered_platforms,
     load_emulator_profiles,
     upstream_profile_index,
+    write_text_atomic,
     yaml_load,
 )
 from exporter import discover_exporters  # noqa: E402
@@ -65,6 +67,8 @@ from scripts.scraper.targets import discover_target_scrapers  # noqa: E402
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 CORE_INFO_REPO = "https://github.com/libretro/libretro-core-info"
+# The git blob of every .info as last reviewed: `--record-coreinfo` writes it.
+CORE_INFO_RECORD = REPO_ROOT / "platforms" / "_core_info.json"
 NO_INTRO_MIRROR = "https://github.com/hugo19941994/auto-datfile-generator"
 NO_INTRO_RELEASE = "Daily_Rebuild"
 NO_INTRO_ASSET = "no-intro.zip"
@@ -107,6 +111,9 @@ class PlatformDiff:
     files_changed: list[str] = field(default_factory=list)
     cores_added: list[str] = field(default_factory=list)
     cores_removed: list[str] = field(default_factory=list)
+    # Any other field the scrape writes: launchers, dual_mode_cores, a
+    # system's core, native_id or data_directories.
+    fields_changed: list[str] = field(default_factory=list)
 
     @property
     def content_changed(self) -> bool:
@@ -119,6 +126,7 @@ class PlatformDiff:
                 self.files_changed,
                 self.cores_added,
                 self.cores_removed,
+                self.fields_changed,
             )
         )
 
@@ -138,6 +146,7 @@ class PlatformDiff:
             ("~files", self.files_changed),
             ("+cores", self.cores_added),
             ("-cores", self.cores_removed),
+            ("~fields", self.fields_changed),
         ):
             if items:
                 shown = ", ".join(items[:SHOWN_ITEMS]) + (", ..." if len(items) > SHOWN_ITEMS else "")
@@ -177,6 +186,33 @@ def _keyed_files(files: list[dict]) -> dict[str, dict]:
     return keyed
 
 
+# Compared on their own above. `source` is the revision read: it moves with
+# every upstream commit, and what that commit changes is diffed field by field.
+_DIFFED_APART = frozenset({"version", "systems", "cores", "standalone_cores", "source"})
+
+
+def _without_refs(value: object) -> object:
+    """*value* without its `source_ref` lines.
+
+    A launcher cites lines of the revision read, so they shift with any
+    upstream commit above them (es_features.cfg:11638 read as :11649 one
+    RetroBat commit later). The cited content is what is compared.
+    """
+    if isinstance(value, dict):
+        return {k: _without_refs(v) for k, v in value.items() if k != "source_ref"}
+    if isinstance(value, list):
+        return [_without_refs(v) for v in value]
+    return value
+
+
+def _changed_fields(old: dict, new: dict, skip: frozenset[str], prefix: str = "") -> list[str]:
+    return [
+        f"{prefix}{key}"
+        for key in sorted(set(old) | set(new), key=str)
+        if key not in skip and _without_refs(old.get(key)) != _without_refs(new.get(key))
+    ]
+
+
 def diff_platform(old: dict, new: dict) -> PlatformDiff:
     """Compare two platform files the way a reviewer reads the diff.
 
@@ -205,6 +241,11 @@ def diff_platform(old: dict, new: dict) -> PlatformDiff:
             for k in sorted(set(old_files) & set(new_files))
             if old_files[k] != new_files[k]
         )
+        diff.fields_changed.extend(_changed_fields(
+            old_systems[system] or {}, new_systems[system] or {},
+            frozenset({"files"}), f"{system}/",
+        ))
+    diff.fields_changed = _changed_fields(old, new, _DIFFED_APART) + diff.fields_changed
     old_cores, new_cores = _core_names(old), _core_names(new)
     diff.cores_added = sorted(new_cores - old_cores)
     diff.cores_removed = sorted(old_cores - new_cores)
@@ -432,6 +473,27 @@ def _transcribed_at(platforms_dir: Path, platform: str) -> float:
     return newest
 
 
+# How the truth model settles two cores' hashes: generate_truth reports it,
+# and it says nothing about whether a cached original is current.
+_MODEL_NOTES = ("WARNING: hash conflict",)
+
+
+@contextlib.contextmanager
+def _without_model_notes() -> Iterator[None]:
+    """Keep the model build's own notes out of the freshness report.
+
+    Anything else written to stderr passes through.
+    """
+    sink = io.StringIO()
+    try:
+        with contextlib.redirect_stderr(sink):
+            yield
+    finally:
+        for line in sink.getvalue().splitlines(keepends=True):
+            if not line.startswith(_MODEL_NOTES):
+                sys.stderr.write(line)
+
+
 def check_native(
     platforms_dir: Path, cache_dir: Path, truth_dir: Path,
     published: Path = Path("exports"),
@@ -444,24 +506,25 @@ def check_native(
         exporter_class = exporters.get(name)
         if not exporter_class:
             continue
-        try:
-            truth, scraped = export_native.load_inputs(name, truth_dir, str(platforms_dir))
-        except export_native.StaleTruth as exc:
-            findings.append(Finding("native", name, STALE, detail=str(exc)))
-            continue
-        systems, _ = build_native_model(truth or {}, scraped)
-        wanted = export_native.wanted_sources(exporter_class(), systems, scraped)
-        reason = native_cache_state(
-            wanted, cache_dir / name, recorded, index, _transcribed_at(platforms_dir, name)
-        )
-        if reason:
-            reason += ", export_native.py --refresh-cache fetches it"
-        else:
-            reason = published_export_state(
-                name, exporter_class, truth_dir, platforms_dir, cache_dir, published
+        with _without_model_notes():
+            try:
+                truth, scraped = export_native.load_inputs(name, truth_dir, str(platforms_dir))
+            except export_native.StaleTruth as exc:
+                findings.append(Finding("native", name, STALE, detail=str(exc)))
+                continue
+            systems, _ = build_native_model(truth or {}, scraped)
+            wanted = export_native.wanted_sources(exporter_class(), systems, scraped)
+            reason = native_cache_state(
+                wanted, cache_dir / name, recorded, index, _transcribed_at(platforms_dir, name)
             )
             if reason:
-                reason += f", export_native.py --platform {name} writes it again"
+                reason += ", export_native.py --refresh-cache fetches it"
+            else:
+                reason = published_export_state(
+                    name, exporter_class, truth_dir, platforms_dir, cache_dir, published
+                )
+                if reason:
+                    reason += f", export_native.py --platform {name} writes it again"
         findings.append(
             Finding(
                 "native",
@@ -660,8 +723,9 @@ def _age_days(stamp: str, now: datetime) -> int | None:
 # --- core-info --------------------------------------------------------------
 
 
-def core_info_names(cache_dir: str, offline: bool) -> list[str] | None:
-    """Every core the libretro core-info repository describes."""
+def core_info_blobs(cache_dir: str, offline: bool) -> dict[str, str] | None:
+    """Every core the libretro core-info repository describes, with the git
+    blob of its .info."""
     repo = upstream.parse_repo(CORE_INFO_REPO)
     if repo is None:
         return None
@@ -673,12 +737,54 @@ def core_info_names(cache_dir: str, offline: bool) -> list[str] | None:
     )
     if not isinstance(payload, list):
         return None
-    names = []
+    blobs = {}
     for entry in payload:
         name = entry.get("name", "") if isinstance(entry, dict) else ""
         if name.endswith("_libretro.info"):
-            names.append(name[: -len("_libretro.info")])
-    return sorted(names)
+            blobs[name[: -len("_libretro.info")]] = str(entry.get("sha") or "")
+    return dict(sorted(blobs.items()))
+
+
+def core_info_names(cache_dir: str, offline: bool) -> list[str] | None:
+    """Every core the libretro core-info repository describes."""
+    blobs = core_info_blobs(cache_dir, offline)
+    return None if blobs is None else list(blobs)
+
+
+def load_core_info_record(path: Path = CORE_INFO_RECORD) -> dict[str, str] | None:
+    if not path.is_file():
+        return None
+    return dict((json.loads(path.read_text(encoding="utf-8")) or {}).get("blobs") or {})
+
+
+def record_core_info(
+    blobs: dict[str, str], names: list[str], path: Path = CORE_INFO_RECORD
+) -> list[str]:
+    """Record the current blob of *names* (every .info when empty) as reviewed.
+
+    Naming the cores keeps a change nobody read out of the record.
+    """
+    unknown = sorted(set(names) - set(blobs))
+    if unknown:
+        raise ValueError(f"no such .info in core-info: {', '.join(unknown)}")
+    recorded = load_core_info_record(path) or {}
+    recorded.update({n: blobs[n] for n in (names or blobs)})
+    payload = {"repository": CORE_INFO_REPO, "blobs": dict(sorted(recorded.items()))}
+    write_text_atomic(str(path), json.dumps(payload, indent=2) + "\n")
+    return sorted(names or blobs)
+
+
+def coreinfo_drift(
+    blobs: dict[str, str], recorded: dict[str, str], profiles: dict[str, dict]
+) -> list[str]:
+    """Profiled cores whose .info is not the one recorded as reviewed.
+
+    The .info is no authority, but a new firmwareN_path in it is the
+    cheapest sign that a profile needs reading again.
+    """
+    index = {key.casefold() for key in profile_name_index(profiles)}
+    return [n for n, sha in blobs.items()
+            if n.casefold() in index and recorded.get(n) != sha]
 
 
 def coreinfo_gaps(
@@ -708,10 +814,14 @@ def coreinfo_gaps(
     return unprofiled, standalone
 
 
-def check_coreinfo(profiles: dict[str, dict], cache_dir: str, offline: bool) -> list[Finding]:
-    names = core_info_names(cache_dir, offline)
-    if names is None:
+def check_coreinfo(
+    profiles: dict[str, dict], cache_dir: str, offline: bool,
+    record_path: Path = CORE_INFO_RECORD,
+) -> list[Finding]:
+    blobs = core_info_blobs(cache_dir, offline)
+    if blobs is None:
         return [Finding("coreinfo", "libretro-core-info", UNKNOWN, detail="listing unavailable")]
+    names = list(blobs)
     unprofiled, standalone = coreinfo_gaps(names, profiles)
     findings = [
         Finding(
@@ -736,6 +846,22 @@ def check_coreinfo(profiles: dict[str, dict], cache_dir: str, offline: bool) ->
         )
         for name, key in standalone
     )
+    recorded = load_core_info_record(record_path)
+    if recorded is None:
+        findings.append(Finding(
+            "coreinfo", "reviewed .info", UNKNOWN,
+            detail=f"no {record_path.name}: run check_freshness.py --record-coreinfo",
+        ))
+        return findings
+    findings.extend(
+        Finding(
+            "coreinfo", name, STALE,
+            local=(recorded.get(name) or "not recorded")[:12], remote=blobs[name][:12],
+            detail=(".info changed since reviewed: read its firmware, then "
+                    f"check_freshness.py --record-coreinfo {name}"),
+        )
+        for name in coreinfo_drift(blobs, recorded, profiles)
+    )
     return findings
 
 
@@ -1278,8 +1404,21 @@ def main() -> int:
         "--emulators-dir", default="emulators",
         help="emulator profile directory (default: %(default)s)",
     )
+    parser.add_argument(
+        "--record-coreinfo", nargs="*", metavar="CORE",
+        help="record the current .info of CORE (every core when none is named) "
+             f"as reviewed in {CORE_INFO_RECORD.relative_to(REPO_ROOT)}, then exit",
+    )
     args = parser.parse_args()
 
+    if args.record_coreinfo is not None:
+        blobs = core_info_blobs(args.cache_dir, args.offline)
+        if not blobs:
+            raise RuntimeError("core-info listing unavailable: nothing recorded")
+        done = record_core_info(blobs, args.record_coreinfo)
+        print(f"recorded {len(done)} .info file(s) as reviewed")
+        return 0
+
     areas = tuple(a.strip() for a in args.only.split(",")) if args.only else AREAS
     unknown = sorted(set(areas) - set(AREAS))
     if unknown:

scripts/generate_pack.py

@@ -3225,46 +3225,48 @@ def _main(held: contextlib.ExitStack) -> None:
         return
 
     data_registry = load_data_dir_registry(args.platforms_dir)
-    if data_registry and not args.offline and not args.no_refresh_data:
-        from refresh_data_dirs import load_registry, refresh_all
+    # The output directory is claimed before the data directory refresh: a
+    # refresh waits for every run that holds a cache, so a second build
+    # during a pipeline sat silent for hours instead of being refused.
+    with _pack_output_lock(args.output_dir):
+        if data_registry and not args.offline and not args.no_refresh_data:
+            from refresh_data_dirs import load_registry, refresh_all
 
-        registry = load_registry(os.path.join(args.platforms_dir, "_data_dirs.yml"))
-        results = refresh_all(registry, force=args.refresh_data)
-        updated = sum(1 for v in results.values() if v)
-        if updated:
-            print(f"Refreshed {updated} data director{'ies' if updated > 1 else 'y'}")
-        failed = sorted(k for k, v in results.items() if v is None)
-        if failed:
-            print(f"WARNING: data directory refresh failed: {', '.join(failed)}")
-    if data_registry:
-        hold_data_dirs(held, data_registry)
+            registry = load_registry(os.path.join(args.platforms_dir, "_data_dirs.yml"))
+            results = refresh_all(registry, force=args.refresh_data)
+            updated = sum(1 for v in results.values() if v)
+            if updated:
+                print(f"Refreshed {updated} data director{'ies' if updated > 1 else 'y'}")
+            failed = sorted(k for k, v in results.items() if v is None)
+            if failed:
+                print(f"WARNING: data directory refresh failed: {', '.join(failed)}")
+        if data_registry:
+            hold_data_dirs(held, data_registry)
 
-    emu_profiles = load_emulator_profiles(args.emulators_dir)
+        emu_profiles = load_emulator_profiles(args.emulators_dir)
 
-    target_cores_cache: dict[str, set[str] | None] = {}
-    if args.target:
-        try:
-            target_cores_cache, platforms = build_target_cores_cache(
-                platforms,
-                args.target,
-                args.platforms_dir,
-                is_all=args.all,
-            )
-        except (FileNotFoundError, ValueError) as e:
-            print(f"ERROR: {e}", file=sys.stderr)
-            sys.exit(1)
+        target_cores_cache: dict[str, set[str] | None] = {}
+        if args.target:
+            try:
+                target_cores_cache, platforms = build_target_cores_cache(
+                    platforms,
+                    args.target,
+                    args.platforms_dir,
+                    is_all=args.all,
+                )
+            except (FileNotFoundError, ValueError) as e:
+                print(f"ERROR: {e}", file=sys.stderr)
+                sys.exit(1)
 
-    groups = group_identical_platforms(
-        platforms, args.platforms_dir, target_cores_cache if args.target else None
-    )
+        groups = group_identical_platforms(
+            platforms, args.platforms_dir, target_cores_cache if args.target else None
+        )
 
-    if args.manifest:
-        with _pack_output_lock(args.output_dir):
+        if args.manifest:
             _run_manifest_mode(
                 args, groups, db, zip_contents, emu_profiles, target_cores_cache
             )
-    else:
-        with _pack_output_lock(args.output_dir):
+        else:
             _run_platform_packs(
                 args,
                 groups,
@@ -3275,9 +3277,9 @@ def _main(held: contextlib.ExitStack) -> None:
                 target_cores_cache,
                 system_filter,
             )
-        if args.verify_packs:
-            with _pack_output_lock(args.output_dir, exclusive=False):
-                _run_verify_packs(args)
+    if not args.manifest and args.verify_packs:
+        with _pack_output_lock(args.output_dir, exclusive=False):
+            _run_verify_packs(args)
 
 
 # Manifest generation (JSON inventory for install.py)

scripts/largefiles.py

@@ -164,18 +164,15 @@ def _fetch_asset(
 
     if os.path.isdir(cached):
         return None
+    if os.path.exists(cached) and not (expected_sha1 or expected_md5):
+        return cached
     if os.path.exists(cached):
-        try:
-            hashes = compute_hashes(cached) if (expected_sha1 or expected_md5) else {}
-        except FileNotFoundError:
-            hashes = None
-        if hashes is None:
+        snapshot = _read_once(cached)
+        if snapshot is None:
             pass
-        elif not (expected_sha1 or expected_md5) or _matches(
-            hashes, expected_sha1, expected_md5
-        ):
+        elif _matches(snapshot[0], expected_sha1, expected_md5):
             return cached
-        elif not _replaced_on_release(name, cached, offline):
+        elif not _replaced_on_release(name, snapshot[0]["sha256"], snapshot[1], offline):
             # A copy of this asset that answers another hash: the caller
             # wants a different revision under the same name, and the
             # release still serves this one. Keeping it is what lets the
@@ -249,8 +246,32 @@ def _asset_urls(name: str) -> list[tuple[str, str]]:
     ]
 
 
-def _replaced_on_release(name: str, cached: str, offline: bool) -> bool:
-    """Whether the release now serves other bytes than the cached copy.
+def _read_once(path: str) -> tuple[dict[str, str], int] | None:
+    """Digests and size of the file at *path*, all read from one file, or
+    None when it is gone.
+
+    Every judgement on a cache copy reads this one snapshot. Hashed, then
+    hashed again by path for the release check, a copy another run promoted
+    in between answered the second question: the copy the caller wanted was
+    in the cache and the caller was told there was none.
+    """
+    # A promotion is an os.replace, so it changes the inode: the same inode
+    # on both sides of the read means the digests describe that one file.
+    for _attempt in range(3):
+        try:
+            before = os.stat(path)
+            hashes = compute_hashes(path)
+            after = os.stat(path)
+        except FileNotFoundError:
+            return None
+        if (before.st_ino, before.st_mtime_ns) == (after.st_ino, after.st_mtime_ns):
+            return hashes, after.st_size
+    return None
+
+
+def _replaced_on_release(name: str, sha256: str, size: int, offline: bool) -> bool:
+    """Whether the release now serves other bytes than the cached copy,
+    whose *sha256* and *size* the caller read.
 
     The release lists a sha256 per asset: a dump rebuilt at the same size
     (the eight 251658304-byte DSi NANDs) differs only there, and judged by
@@ -266,9 +287,9 @@ def _replaced_on_release(name: str, cached: str, offline: bool) -> bool:
         None,
     )
     if digest:
-        return compute_hashes(cached, frozenset({"sha256"}))["sha256"] != digest
+        return sha256 != digest
     served = _served_size(name)
-    return served is not None and served != os.path.getsize(cached)
+    return served is not None and served != size
 
 
 _digests_cache: dict[str, str] | None = None

scripts/refresh_data_dirs.py

@@ -23,6 +23,7 @@ import tempfile
 import urllib.error
 import urllib.request
 import zipfile
+from collections.abc import Iterator
 from pathlib import Path
 from artifacts import file_lock as _file_lock
 from common import github_token, platform_arg, require_platform, yaml_load
@@ -61,7 +62,34 @@ def cache_lock(
     swapped the tree shipped part of it, or nothing.
     """
     cache_dir = Path(cache_dir)
-    return _file_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared=shared)
+    return _announced_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared, cache_dir)
+
+
+@contextlib.contextmanager
+def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[None]:
+    """file_lock, saying so when it has to wait.
+
+    A pipeline holds every cache for its whole run: a refresh started beside
+    it waited hours with nothing on screen.
+    """
+    try:
+        import fcntl
+    except ImportError:
+        with _file_lock(lock_path, shared=shared):
+            yield
+        return
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    mode = fcntl.LOCK_SH if shared else fcntl.LOCK_EX
+    with open(lock_path, "a") as handle:
+        try:
+            fcntl.flock(handle, mode | fcntl.LOCK_NB)
+        except BlockingIOError:
+            log.warning("waiting for %s: another run holds it", cache_dir)
+            fcntl.flock(handle, mode)
+        try:
+            yield
+        finally:
+            fcntl.flock(handle, fcntl.LOCK_UN)
 
 
 def hold_data_dirs(stack: contextlib.ExitStack, registry: dict) -> None:
@@ -383,6 +411,14 @@ def _get_remote_etag(source_url: str) -> str | None:
         return None
 
 
+def _short_tag(tag: str | None) -> str:
+    """A commit sha or ETag cut for a log line, quotes dropped first: a
+    quoted ETag cut at 12 characters kept its opening quote only."""
+    if not tag:
+        return "none"
+    return tag.strip().removeprefix("W/").strip('"')[:12] or "none"
+
+
 def refresh_entry(
     key: str,
     entry: dict,
@@ -431,16 +467,17 @@ def _refresh_entry(
         needs_refresh = remote_tag != cached_tag
 
     if not needs_refresh:
-        log.info("[%s] up to date (tag: %s)", key, (cached_tag or "?")[:12])
+        log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag))
         return False
 
     if dry_run:
-        log.info(
-            "[%s] would refresh (type: %s, cached: %s)",
-            key,
-            source_type,
-            cached_tag or "none",
-        )
+        if remote_tag is not None:
+            reason = f"cached: {_short_tag(cached_tag)}, remote: {_short_tag(remote_tag)}"
+        elif force:
+            reason = "forced"
+        else:
+            reason = "not cached"
+        log.info("[%s] would refresh (type: %s, %s)", key, source_type, reason)
         return True
 
     if remote_tag is None:
@@ -516,6 +553,16 @@ def refresh_all(
     return results
 
 
+def summarize(outcomes: dict[str, bool | None], dry_run: bool) -> str:
+    """One closing line: how many entries changed, were current, failed."""
+    refreshed = sum(1 for o in outcomes.values() if o is True)
+    current = sum(1 for o in outcomes.values() if o is False)
+    failed = sum(1 for o in outcomes.values() if o is None)
+    verb = "would refresh" if dry_run else "refreshed"
+    line = f"{len(outcomes)} entries: {refreshed} {verb}, {current} up to date"
+    return f"{line}, {failed} failed" if failed else line
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(
         allow_abbrev=False,
@@ -567,6 +614,7 @@ def main() -> None:
             registry, force=args.force, dry_run=args.dry_run, platform=args.platform
         )
 
+    log.info(summarize(outcomes, args.dry_run))
     failed = sorted(key for key, outcome in outcomes.items() if outcome is None)
     if failed:
         log.error("refresh failed: %s", ", ".join(failed))

scripts/refresh_stale.py

@@ -25,7 +25,8 @@ Mapping (stale -> command):
     catalogs/fbneo recipes  python -m scripts.scraper.romset_dat_importer
                             --source fbneo --fetch
 
-Surfaced only (no refresh): coreinfo gaps, redump/no-intro/tosec packs,
+Surfaced only (no refresh): data rows that are not a _data_dirs.yml key
+(the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs,
 CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs
 a human read (new .info file, manual DAT download, PyPI version bump).
 
@@ -46,12 +47,14 @@ import sys
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
+from functools import lru_cache
 from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 LOG_DIR = REPO_ROOT / "tmp" / "refresh_stale"
 CHECK_FRESHNESS = REPO_ROOT / "scripts" / "check_freshness.py"
 PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
+DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
 
 AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
 MANUAL_AREAS = ("coreinfo", "ci", "profiles")
@@ -91,6 +94,15 @@ def _load_platform_registry() -> dict[str, dict]:
         return (yaml.safe_load(fh) or {}).get("platforms") or {}
 
 
+@lru_cache(maxsize=1)
+def _data_dir_keys() -> frozenset[str]:
+    """The entries refresh_data_dirs.py --key accepts."""
+    import yaml
+
+    with DATA_DIRS_REGISTRY.open(encoding="utf-8") as fh:
+        return frozenset((yaml.safe_load(fh) or {}).get("data_directories") or {})
+
+
 def _run_check_freshness(areas: tuple[str, ...], extra: list[str]) -> list[dict]:
     """Run check_freshness.py --json and return its findings."""
     cmd = [sys.executable, str(CHECK_FRESHNESS), "--json"]
@@ -246,8 +258,9 @@ def _command_for(
         ]
 
     if area == "data":
-        if subject == "_data_dirs.yml":
-            # The registry file itself cannot be refreshed, but its entries can.
+        if subject not in _data_dir_keys():
+            # "_data_dirs.yml" and "buildbot system" name the registry and
+            # what it does not track yet: a human adds the entry.
             return None
         return [
             sys.executable,

scripts/scraper/base_scraper.py

@@ -163,6 +163,11 @@ def fetch_bytes(
 class BaseScraper(ABC):
     """Abstract base class for platform BIOS requirement scrapers."""
 
+    # False for a platform whose file list is inherited and whose scraper
+    # reads only its core list (RetroPie). Otherwise an empty read is an
+    # upstream that changed shape, never a platform with no file.
+    lists_files: bool = True
+
     def __init__(self, url: str = "") -> None:
         self.url = url
         self._raw_data: str | None = None
@@ -367,6 +372,12 @@ def scraper_cli(
         reqs = scraper.fetch_requirements()
     except (OSError, ValueError, RuntimeError) as e:
         _exit_on_scrape_error(e)
+    if scraper.lists_files and not reqs:
+        # Written, an empty read became a platform with no system that
+        # verify and the pack count 0/0 and call consistent.
+        _exit_on_scrape_error(RuntimeError(
+            f"{scraper_class.__module__}: upstream returned no BIOS entry"
+        ))
 
     if args.dry_run:
         by_system: dict[str, list] = {}
@@ -426,6 +437,12 @@ def scraper_cli(
                 if req.zipped_file:
                     entry["zipped_file"] = req.zipped_file
                 config["systems"][sys_id]["files"].append(entry)
+        if scraper.lists_files and not any(
+            (system or {}).get("files") for system in (config.get("systems") or {}).values()
+        ):
+            _exit_on_scrape_error(RuntimeError(
+                f"{scraper_class.__module__}: the generated platform lists no file"
+            ))
         # Merge into existing YAML: preserve fields the scraper doesn't generate
         # (data_directories, case_insensitive_fs, manually added metadata).
         # The scraper replaces systems + files, everything else is preserved.

scripts/scraper/fbneo_hash_scraper.py

@@ -42,6 +42,8 @@ def _is_cache_fresh() -> bool:
         return False
     try:
         data = json.loads(CACHE_PATH.read_text(encoding="utf-8"))
+        if not data.get("bios_sets"):
+            return False
         fetched_at = datetime.fromisoformat(data["fetched_at"])
         return datetime.now(timezone.utc) - fetched_at < timedelta(hours=STALE_HOURS)
     except (json.JSONDecodeError, KeyError, ValueError):
@@ -178,6 +180,10 @@ def fetch_and_cache(force: bool = False) -> dict[str, Any]:
 
         log.info("parsing source tree")
         bios_sets = parse_fbneo_source_tree(str(CLONE_DIR))
+        if not bios_sets or not commit:
+            # An empty read was cached for a day and merged nothing, and
+            # the pipeline step reported OK.
+            raise RuntimeError(f"no BIOS set read at FBNeo {version} ({commit[:12]})")
 
         cache: dict[str, Any] = {
             "source": "finalburnneo/FBNeo",

scripts/scraper/mame_hash_scraper.py

@@ -62,7 +62,7 @@ def _load_cache() -> dict[str, Any] | None:
 
 
 def _is_stale(cache: dict[str, Any] | None) -> bool:
-    if cache is None:
+    if cache is None or not cache.get("bios_sets"):
         return True
     fetched_at = cache.get("fetched_at")
     if not fetched_at:
@@ -292,6 +292,11 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
         bios_sets = parse_mame_source_tree(str(_CLONE_DIR))
         version = _parse_version_tag(tag)
         commit = _get_commit()
+        if not bios_sets or not commit:
+            # An empty read was cached for a day and every profile entry
+            # came out "out of scope", so nothing was refreshed and the
+            # pipeline step reported OK.
+            raise RuntimeError(f"no BIOS root set read at {tag}")
 
         data: dict[str, Any] = {
             "source": "mamedev/mame",

scripts/scraper/retropie_scraper.py

@@ -59,6 +59,8 @@ _PACKAGE_DIRS = _SCRIPT_SECTIONS
 class Scraper(BaseScraper):
     """Scraper for the RetroPie package list."""
 
+    lists_files = False
+
     def __init__(self, url: str | None = None) -> None:
         # The package list is read from master: the commit read is recorded.
         if url is None:
@@ -122,6 +124,8 @@ class Scraper(BaseScraper):
                 else:
                     found.add(match.group(1))
 
+        if not found:
+            raise ValueError(f"{self.url}: no rp_module_id under {SCRIPTMODULES}/")
         self._modules = sorted(found)
         return self._modules
 

scripts/scraper/romset_dat_importer.py

@@ -15,7 +15,9 @@ in its repository, so both are fetched without a browser:
 
 Only sets the collection has a reason to know are kept, so the committed
 snapshot stays small: the archive names that emulator profiles or platform
-BIOS lists reference. ``--all`` keeps everything.
+BIOS lists reference, and the sets holding a dump-catalog entry the
+collection lacks (provenance_report names the set as where to find it).
+``--all`` keeps everything.
 
 Members without a CRC32 are undumped. A real romset does not carry them, so
 they are dropped from the recipe rather than disqualifying the whole set.
@@ -42,7 +44,7 @@ from ..common import (
     load_emulator_profiles,
     load_platform_config,
 )
-from ..dumpcatalog import build_snapshot, write_snapshot
+from ..dumpcatalog import build_snapshot, load_provenance_snapshots, write_snapshot
 from .logiqx_parser import LogiqxDat, parse_logiqx
 
 MAX_MEMBER_SIZE = 200 * 1024 * 1024
@@ -74,8 +76,45 @@ def relevant_archives(emulators_dir: str, platforms_dir: str) -> set[str]:
     return names
 
 
+Targets = dict[tuple[str, int], set[str]]
+
+
+def catalog_targets(provenance_dir: str, db_path: str) -> Targets:
+    """(crc32, size) -> sha1 of each dump-catalog entry the collection lacks.
+
+    fm7740sx carries DICROM.rom, EXTSUB.rom and KANJI2.rom, which No-Intro
+    lists and the collection misses: filtered on archive names alone, the
+    snapshot never held the set and the missing report named no source.
+    """
+    with open(db_path, encoding="utf-8") as handle:
+        held = set(json.load(handle).get("files", {}))
+    targets: Targets = {}
+    for snapshot in load_provenance_snapshots(provenance_dir).values():
+        for entry in snapshot.get("entries", []):
+            crc, size = str(entry.get("crc32") or "").lower(), entry.get("size")
+            sha1 = str(entry.get("sha1") or "").lower()
+            if not crc or size is None or (sha1 and sha1 in held):
+                continue
+            targets.setdefault((crc.zfill(8), int(size)), set()).add(sha1)
+    return targets
+
+
+def holds_target(members: list[dict], targets: Targets | None) -> bool:
+    """Whether a set carries a missing catalog dump. A sha1 on both sides
+    must agree: crc32 and size alone collide across 40 000 ROMs."""
+    if not targets:
+        return False
+    for member in members:
+        key = (str(member.get("crc32") or "").lower().zfill(8), int(member.get("size") or 0))
+        wanted = targets.get(key)
+        sha1 = str(member.get("sha1") or "").lower()
+        if wanted is not None and (not sha1 or "" in wanted or sha1 in wanted):
+            return True
+    return False
+
+
 def recipe_entries(
-    dat: LogiqxDat, keep: set[str] | None = None
+    dat: LogiqxDat, keep: set[str] | None = None, targets: Targets | None = None
 ) -> list[dict]:
     """Group a parsed DAT into one recipe per set."""
     by_set: dict[str, list] = {}
@@ -87,7 +126,9 @@ def recipe_entries(
     entries: list[dict] = []
     for set_name, roms in sorted(by_set.items()):
         archive = f"{set_name}.zip"
-        if keep is not None and archive not in keep:
+        if keep is not None and archive not in keep and not holds_target(
+            [{"crc32": r.crc32, "size": r.size, "sha1": r.sha1} for r in roms], targets
+        ):
             continue
         entries.append(
             {
@@ -149,9 +190,12 @@ def _machine_roms(blob: bytes) -> list[dict]:
 
 
 def _scan_listxml(
-    stream: io.BufferedIOBase, selector: Callable[[str], bool]
+    stream: io.BufferedIOBase,
+    selector: Callable[[str], bool],
+    targets: Targets | None = None,
 ) -> dict[str, tuple[str | None, list[dict]]]:
-    """Stream a MAME -listxml document, keeping the machines *selector* wants.
+    """Stream a MAME -listxml document, keeping the machines *selector* wants
+    and those holding one of *targets*.
 
     The document is 311 MB for MAME 0.289, so it is never held whole: the
     reader keeps a sliding window just large enough to close the element it
@@ -170,19 +214,22 @@ def _scan_listxml(
             if end == -1:
                 break
             name = match.group(1).decode()
-            if selector(name):
-                parent = _ROMOF.search(match.group(2).decode())
-                found[name] = (
-                    parent.group(1) if parent else None,
-                    _machine_roms(buffer[match.end() : end]),
-                )
+            wanted = selector(name)
+            if wanted or targets:
+                roms = _machine_roms(buffer[match.end() : end])
+                if wanted or holds_target(roms, targets):
+                    parent = _ROMOF.search(match.group(2).decode())
+                    found[name] = (parent.group(1) if parent else None, roms)
             position = end + len(b"</machine>")
         buffer = buffer[position:] if position else buffer[-(2 * LISTXML_CHUNK) :]
     return found
 
 
 def listxml_entries(
-    open_stream: Callable[[], io.BufferedIOBase], label: str, keep: set[str] | None
+    open_stream: Callable[[], io.BufferedIOBase],
+    label: str,
+    keep: set[str] | None,
+    targets: Targets | None = None,
 ) -> list[dict]:
     """Recipes from a MAME -listxml document, parents resolved.
 
@@ -193,7 +240,7 @@ def listxml_entries(
     """
     with open_stream() as stream:
         wanted = _scan_listxml(
-            stream, lambda name: keep is None or f"{name}.zip" in keep
+            stream, lambda name: keep is None or f"{name}.zip" in keep, targets
         )
     requested = set(wanted)
     parents = {
@@ -484,6 +531,15 @@ def main() -> int:
     parser.add_argument(
         "--all", action="store_true", help="keep every set, not only referenced ones"
     )
+    parser.add_argument(
+        "--provenance-dir", default="provenance",
+        help="dump catalogs whose missing entries keep the sets holding them "
+             "(default: %(default)s)",
+    )
+    parser.add_argument(
+        "--db", default="database.json",
+        help="collection database the catalogs are compared with (default: %(default)s)",
+    )
     parser.add_argument(
         "--dry-run", action="store_true",
         help="print what would change and write nothing",
@@ -511,6 +567,7 @@ def main() -> int:
         if args.all
         else relevant_archives(args.emulators_dir, args.platforms_dir)
     )
+    targets = None if args.all else catalog_targets(args.provenance_dir, args.db)
 
     dats: dict[str, str] = {}
     entries: list[dict] = []
@@ -518,7 +575,7 @@ def main() -> int:
 
     if _is_listxml(pack):
         label = f"MAME {pack.stem.replace('lx', '')}"
-        entries = listxml_entries(_listxml_opener(pack), label, keep)
+        entries = listxml_entries(_listxml_opener(pack), label, keep, targets)
         dats[label] = pack.stem
         print(f"  {label}: {len(entries)} sets retenus")
         members = sum(len(entry["members"]) for entry in entries)
@@ -539,7 +596,7 @@ def main() -> int:
         if not dat.roms:
             skipped += 1
             continue
-        found = recipe_entries(dat, keep)
+        found = recipe_entries(dat, keep, targets)
         if not found:
             continue
         dats[dat.name or member] = dat.version

scripts/scraper/targets/__init__.py

@@ -43,6 +43,7 @@ class BaseTargetScraper(ABC):
             r"^https://raw\.githubusercontent\.com/[^/]+/[^/]+/([^/]+)/",
             r"^https://github\.com/[^/]+/[^/]+/(?:blob|tree)/([^/]+)/",
             r"^https://gitlab\.com/.+?/-/(?:raw|blob|tree)/([^/]+)/",
+            r"^https://codeload\.github\.com/[^/]+/[^/]+/(?:tar\.gz|zip)/([^/]+)$",
         ):
             match = re.match(pattern, source)
             if match:

scripts/scraper/targets/emudeck_targets_scraper.py

@@ -1,8 +1,11 @@
 """Scraper for EmuDeck emulator targets.
 
 Sources:
-  SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh
-  Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1
+  SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh, read at the
+           revision platforms/emudeck.yml was scraped at
+  Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1, default branch:
+           the platform file is read from dragoonDorise/EmuDeck alone and
+           pins no emudeck-we revision
 """
 
 from __future__ import annotations
@@ -89,8 +92,18 @@ def _script_to_core(filename: str) -> str | None:
 class Scraper(BaseTargetScraper):
     """Fetches emulator lists for EmuDeck SteamOS and Windows targets."""
 
-    def __init__(self, url: str = "https://github.com/dragoonDorise/EmuDeck") -> None:
+    def __init__(
+        self, url: str = "https://github.com/dragoonDorise/EmuDeck", revision: str | None = None
+    ) -> None:
         super().__init__(url=url)
+        self._revision = revision
+
+    @property
+    def revision(self) -> str:
+        """The checkBIOS.sh commit platforms/emudeck.yml was read at."""
+        if self._revision is None:
+            self._revision = self.pinned_revision(PLATFORM_NAME)
+        return self._revision
 
     def _fetch_cores_for_target(
         self, api_url: str, label: str, arch: str = "x86_64"
@@ -141,7 +154,9 @@ class Scraper(BaseTargetScraper):
         raise RuntimeError(f"no {arch} core list in {target_path}")
 
     def fetch_targets(self) -> dict:
-        steamos_cores = self._fetch_cores_for_target(STEAMOS_API, "SteamOS")
+        steamos_cores = self._fetch_cores_for_target(
+            f"{STEAMOS_API}?ref={self.revision}", "SteamOS"
+        )
         windows_cores = self._fetch_cores_for_target(WINDOWS_API, "Windows")
 
         targets: dict[str, dict] = {}
@@ -158,7 +173,7 @@ class Scraper(BaseTargetScraper):
 
         return {
             "platform": "emudeck",
-            "source": self.url,
+            "source": f"{self.url}/tree/{self.revision}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
             "targets": targets,
         }

scripts/scraper/targets/retropie_targets_scraper.py

@@ -1,6 +1,7 @@
 """Scraper for RetroPie package availability per platform.
 
-Source: https://github.com/RetroPie/RetroPie-Setup/tree/master/scriptmodules
+Source: https://github.com/RetroPie/RetroPie-Setup/tree/<rev>/scriptmodules,
+read at the revision platforms/retropie.yml was scraped at.
 Parses rp_module_id and rp_module_flags from each scriptmodule of the
 package sections the platform list reads (emulators, libretrocores, ports)
 to determine which platforms each package supports.
@@ -26,7 +27,7 @@ GITHUB_API_URL = (
     "https://api.github.com/repos/RetroPie/RetroPie-Setup/contents/scriptmodules"
 )
 RAW_BASE_URL = (
-    "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/master/scriptmodules/"
+    "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/{rev}/scriptmodules/"
 )
 # The sections the platform scraper reads: a standalone package missing here
 # is filtered out of every targeted pack.
@@ -114,14 +115,23 @@ def _parse_module(content: str) -> tuple[str | None, str]:
 class Scraper(BaseTargetScraper):
     """Fetches RetroPie package availability by parsing scriptmodules."""
 
-    def __init__(self, url: str = GITHUB_API_URL) -> None:
+    def __init__(self, url: str = GITHUB_API_URL, revision: str | None = None) -> None:
         super().__init__(url=url)
+        self._revision = revision
+
+    @property
+    def revision(self) -> str:
+        """The commit the package list was read at: the flags of another
+        revision filter packages the platform file does not describe."""
+        if self._revision is None:
+            self._revision = self.pinned_revision(PLATFORM_NAME)
+        return self._revision
 
     def _list_scriptmodules(self) -> list[str]:
         """Return section/filename for every .sh of the package sections."""
         names: list[str] = []
         for section in PACKAGE_DIRS:
-            url = f"{self.url}/{section}"
+            url = f"{self.url}/{section}?ref={self.revision}"
             entries = json.loads(_fetch(url, accept="application/vnd.github+json"))
             listed = [
                 f"{section}/{e['name']}" for e in entries if e.get("name", "").endswith(".sh")
@@ -132,7 +142,7 @@ class Scraper(BaseTargetScraper):
         return names
 
     def _fetch_module(self, filename: str) -> str:
-        return _fetch(f"{RAW_BASE_URL}{filename}")
+        return _fetch(f"{RAW_BASE_URL.format(rev=self.revision)}{filename}")
 
     def fetch_targets(self) -> dict:
         print("  listing RetroPie scriptmodules...", file=sys.stderr)
@@ -170,7 +180,7 @@ class Scraper(BaseTargetScraper):
 
         return {
             "platform": "retropie",
-            "source": self.url,
+            "source": f"{self.url}?ref={self.revision}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
             "targets": targets,
         }