chore: merge audit fixes (scrapers)

scripts/scraper/base_scraper.py

@@ -163,6 +163,11 @@ def fetch_bytes(
 class BaseScraper(ABC):
     """Abstract base class for platform BIOS requirement scrapers."""
 
+    # False for a platform whose file list is inherited and whose scraper
+    # reads only its core list (RetroPie). Otherwise an empty read is an
+    # upstream that changed shape, never a platform with no file.
+    lists_files: bool = True
+
     def __init__(self, url: str = "") -> None:
         self.url = url
         self._raw_data: str | None = None
@@ -367,6 +372,12 @@ def scraper_cli(
         reqs = scraper.fetch_requirements()
     except (OSError, ValueError, RuntimeError) as e:
         _exit_on_scrape_error(e)
+    if scraper.lists_files and not reqs:
+        # Written, an empty read became a platform with no system that
+        # verify and the pack count 0/0 and call consistent.
+        _exit_on_scrape_error(RuntimeError(
+            f"{scraper_class.__module__}: upstream returned no BIOS entry"
+        ))
 
     if args.dry_run:
         by_system: dict[str, list] = {}
@@ -426,6 +437,12 @@ def scraper_cli(
                 if req.zipped_file:
                     entry["zipped_file"] = req.zipped_file
                 config["systems"][sys_id]["files"].append(entry)
+        if scraper.lists_files and not any(
+            (system or {}).get("files") for system in (config.get("systems") or {}).values()
+        ):
+            _exit_on_scrape_error(RuntimeError(
+                f"{scraper_class.__module__}: the generated platform lists no file"
+            ))
         # Merge into existing YAML: preserve fields the scraper doesn't generate
         # (data_directories, case_insensitive_fs, manually added metadata).
         # The scraper replaces systems + files, everything else is preserved.

scripts/scraper/fbneo_hash_scraper.py

@@ -42,6 +42,8 @@ def _is_cache_fresh() -> bool:
         return False
     try:
         data = json.loads(CACHE_PATH.read_text(encoding="utf-8"))
+        if not data.get("bios_sets"):
+            return False
         fetched_at = datetime.fromisoformat(data["fetched_at"])
         return datetime.now(timezone.utc) - fetched_at < timedelta(hours=STALE_HOURS)
     except (json.JSONDecodeError, KeyError, ValueError):
@@ -178,6 +180,10 @@ def fetch_and_cache(force: bool = False) -> dict[str, Any]:
 
         log.info("parsing source tree")
         bios_sets = parse_fbneo_source_tree(str(CLONE_DIR))
+        if not bios_sets or not commit:
+            # An empty read was cached for a day and merged nothing, and
+            # the pipeline step reported OK.
+            raise RuntimeError(f"no BIOS set read at FBNeo {version} ({commit[:12]})")
 
         cache: dict[str, Any] = {
             "source": "finalburnneo/FBNeo",

scripts/scraper/mame_hash_scraper.py

@@ -62,7 +62,7 @@ def _load_cache() -> dict[str, Any] | None:
 
 
 def _is_stale(cache: dict[str, Any] | None) -> bool:
-    if cache is None:
+    if cache is None or not cache.get("bios_sets"):
         return True
     fetched_at = cache.get("fetched_at")
     if not fetched_at:
@@ -292,6 +292,11 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
         bios_sets = parse_mame_source_tree(str(_CLONE_DIR))
         version = _parse_version_tag(tag)
         commit = _get_commit()
+        if not bios_sets or not commit:
+            # An empty read was cached for a day and every profile entry
+            # came out "out of scope", so nothing was refreshed and the
+            # pipeline step reported OK.
+            raise RuntimeError(f"no BIOS root set read at {tag}")
 
         data: dict[str, Any] = {
             "source": "mamedev/mame",

scripts/scraper/retropie_scraper.py

@@ -59,6 +59,8 @@ _PACKAGE_DIRS = _SCRIPT_SECTIONS
 class Scraper(BaseScraper):
     """Scraper for the RetroPie package list."""
 
+    lists_files = False
+
     def __init__(self, url: str | None = None) -> None:
         # The package list is read from master: the commit read is recorded.
         if url is None:
@@ -122,6 +124,8 @@ class Scraper(BaseScraper):
                 else:
                     found.add(match.group(1))
 
+        if not found:
+            raise ValueError(f"{self.url}: no rp_module_id under {SCRIPTMODULES}/")
         self._modules = sorted(found)
         return self._modules
 

scripts/scraper/romset_dat_importer.py

@@ -15,7 +15,9 @@ in its repository, so both are fetched without a browser:
 
 Only sets the collection has a reason to know are kept, so the committed
 snapshot stays small: the archive names that emulator profiles or platform
-BIOS lists reference. ``--all`` keeps everything.
+BIOS lists reference, and the sets holding a dump-catalog entry the
+collection lacks (provenance_report names the set as where to find it).
+``--all`` keeps everything.
 
 Members without a CRC32 are undumped. A real romset does not carry them, so
 they are dropped from the recipe rather than disqualifying the whole set.
@@ -42,7 +44,7 @@ from ..common import (
     load_emulator_profiles,
     load_platform_config,
 )
-from ..dumpcatalog import build_snapshot, write_snapshot
+from ..dumpcatalog import build_snapshot, load_provenance_snapshots, write_snapshot
 from .logiqx_parser import LogiqxDat, parse_logiqx
 
 MAX_MEMBER_SIZE = 200 * 1024 * 1024
@@ -74,8 +76,45 @@ def relevant_archives(emulators_dir: str, platforms_dir: str) -> set[str]:
     return names
 
 
+Targets = dict[tuple[str, int], set[str]]
+
+
+def catalog_targets(provenance_dir: str, db_path: str) -> Targets:
+    """(crc32, size) -> sha1 of each dump-catalog entry the collection lacks.
+
+    fm7740sx carries DICROM.rom, EXTSUB.rom and KANJI2.rom, which No-Intro
+    lists and the collection misses: filtered on archive names alone, the
+    snapshot never held the set and the missing report named no source.
+    """
+    with open(db_path, encoding="utf-8") as handle:
+        held = set(json.load(handle).get("files", {}))
+    targets: Targets = {}
+    for snapshot in load_provenance_snapshots(provenance_dir).values():
+        for entry in snapshot.get("entries", []):
+            crc, size = str(entry.get("crc32") or "").lower(), entry.get("size")
+            sha1 = str(entry.get("sha1") or "").lower()
+            if not crc or size is None or (sha1 and sha1 in held):
+                continue
+            targets.setdefault((crc.zfill(8), int(size)), set()).add(sha1)
+    return targets
+
+
+def holds_target(members: list[dict], targets: Targets | None) -> bool:
+    """Whether a set carries a missing catalog dump. A sha1 on both sides
+    must agree: crc32 and size alone collide across 40 000 ROMs."""
+    if not targets:
+        return False
+    for member in members:
+        key = (str(member.get("crc32") or "").lower().zfill(8), int(member.get("size") or 0))
+        wanted = targets.get(key)
+        sha1 = str(member.get("sha1") or "").lower()
+        if wanted is not None and (not sha1 or "" in wanted or sha1 in wanted):
+            return True
+    return False
+
+
 def recipe_entries(
-    dat: LogiqxDat, keep: set[str] | None = None
+    dat: LogiqxDat, keep: set[str] | None = None, targets: Targets | None = None
 ) -> list[dict]:
     """Group a parsed DAT into one recipe per set."""
     by_set: dict[str, list] = {}
@@ -87,7 +126,9 @@ def recipe_entries(
     entries: list[dict] = []
     for set_name, roms in sorted(by_set.items()):
         archive = f"{set_name}.zip"
-        if keep is not None and archive not in keep:
+        if keep is not None and archive not in keep and not holds_target(
+            [{"crc32": r.crc32, "size": r.size, "sha1": r.sha1} for r in roms], targets
+        ):
             continue
         entries.append(
             {
@@ -149,9 +190,12 @@ def _machine_roms(blob: bytes) -> list[dict]:
 
 
 def _scan_listxml(
-    stream: io.BufferedIOBase, selector: Callable[[str], bool]
+    stream: io.BufferedIOBase,
+    selector: Callable[[str], bool],
+    targets: Targets | None = None,
 ) -> dict[str, tuple[str | None, list[dict]]]:
-    """Stream a MAME -listxml document, keeping the machines *selector* wants.
+    """Stream a MAME -listxml document, keeping the machines *selector* wants
+    and those holding one of *targets*.
 
     The document is 311 MB for MAME 0.289, so it is never held whole: the
     reader keeps a sliding window just large enough to close the element it
@@ -170,19 +214,22 @@ def _scan_listxml(
             if end == -1:
                 break
             name = match.group(1).decode()
-            if selector(name):
-                parent = _ROMOF.search(match.group(2).decode())
-                found[name] = (
-                    parent.group(1) if parent else None,
-                    _machine_roms(buffer[match.end() : end]),
-                )
+            wanted = selector(name)
+            if wanted or targets:
+                roms = _machine_roms(buffer[match.end() : end])
+                if wanted or holds_target(roms, targets):
+                    parent = _ROMOF.search(match.group(2).decode())
+                    found[name] = (parent.group(1) if parent else None, roms)
             position = end + len(b"</machine>")
         buffer = buffer[position:] if position else buffer[-(2 * LISTXML_CHUNK) :]
     return found
 
 
 def listxml_entries(
-    open_stream: Callable[[], io.BufferedIOBase], label: str, keep: set[str] | None
+    open_stream: Callable[[], io.BufferedIOBase],
+    label: str,
+    keep: set[str] | None,
+    targets: Targets | None = None,
 ) -> list[dict]:
     """Recipes from a MAME -listxml document, parents resolved.
 
@@ -193,7 +240,7 @@ def listxml_entries(
     """
     with open_stream() as stream:
         wanted = _scan_listxml(
-            stream, lambda name: keep is None or f"{name}.zip" in keep
+            stream, lambda name: keep is None or f"{name}.zip" in keep, targets
         )
     requested = set(wanted)
     parents = {
@@ -484,6 +531,15 @@ def main() -> int:
     parser.add_argument(
         "--all", action="store_true", help="keep every set, not only referenced ones"
     )
+    parser.add_argument(
+        "--provenance-dir", default="provenance",
+        help="dump catalogs whose missing entries keep the sets holding them "
+             "(default: %(default)s)",
+    )
+    parser.add_argument(
+        "--db", default="database.json",
+        help="collection database the catalogs are compared with (default: %(default)s)",
+    )
     parser.add_argument(
         "--dry-run", action="store_true",
         help="print what would change and write nothing",
@@ -511,6 +567,7 @@ def main() -> int:
         if args.all
         else relevant_archives(args.emulators_dir, args.platforms_dir)
     )
+    targets = None if args.all else catalog_targets(args.provenance_dir, args.db)
 
     dats: dict[str, str] = {}
     entries: list[dict] = []
@@ -518,7 +575,7 @@ def main() -> int:
 
     if _is_listxml(pack):
         label = f"MAME {pack.stem.replace('lx', '')}"
-        entries = listxml_entries(_listxml_opener(pack), label, keep)
+        entries = listxml_entries(_listxml_opener(pack), label, keep, targets)
         dats[label] = pack.stem
         print(f"  {label}: {len(entries)} sets retenus")
         members = sum(len(entry["members"]) for entry in entries)
@@ -539,7 +596,7 @@ def main() -> int:
         if not dat.roms:
             skipped += 1
             continue
-        found = recipe_entries(dat, keep)
+        found = recipe_entries(dat, keep, targets)
         if not found:
             continue
         dats[dat.name or member] = dat.version

scripts/scraper/targets/__init__.py

@@ -43,6 +43,7 @@ class BaseTargetScraper(ABC):
             r"^https://raw\.githubusercontent\.com/[^/]+/[^/]+/([^/]+)/",
             r"^https://github\.com/[^/]+/[^/]+/(?:blob|tree)/([^/]+)/",
             r"^https://gitlab\.com/.+?/-/(?:raw|blob|tree)/([^/]+)/",
+            r"^https://codeload\.github\.com/[^/]+/[^/]+/(?:tar\.gz|zip)/([^/]+)$",
         ):
             match = re.match(pattern, source)
             if match:

scripts/scraper/targets/emudeck_targets_scraper.py

@@ -1,8 +1,11 @@
 """Scraper for EmuDeck emulator targets.
 
 Sources:
-  SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh
-  Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1
+  SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh, read at the
+           revision platforms/emudeck.yml was scraped at
+  Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1, default branch:
+           the platform file is read from dragoonDorise/EmuDeck alone and
+           pins no emudeck-we revision
 """
 
 from __future__ import annotations
@@ -89,8 +92,18 @@ def _script_to_core(filename: str) -> str | None:
 class Scraper(BaseTargetScraper):
     """Fetches emulator lists for EmuDeck SteamOS and Windows targets."""
 
-    def __init__(self, url: str = "https://github.com/dragoonDorise/EmuDeck") -> None:
+    def __init__(
+        self, url: str = "https://github.com/dragoonDorise/EmuDeck", revision: str | None = None
+    ) -> None:
         super().__init__(url=url)
+        self._revision = revision
+
+    @property
+    def revision(self) -> str:
+        """The checkBIOS.sh commit platforms/emudeck.yml was read at."""
+        if self._revision is None:
+            self._revision = self.pinned_revision(PLATFORM_NAME)
+        return self._revision
 
     def _fetch_cores_for_target(
         self, api_url: str, label: str, arch: str = "x86_64"
@@ -141,7 +154,9 @@ class Scraper(BaseTargetScraper):
         raise RuntimeError(f"no {arch} core list in {target_path}")
 
     def fetch_targets(self) -> dict:
-        steamos_cores = self._fetch_cores_for_target(STEAMOS_API, "SteamOS")
+        steamos_cores = self._fetch_cores_for_target(
+            f"{STEAMOS_API}?ref={self.revision}", "SteamOS"
+        )
         windows_cores = self._fetch_cores_for_target(WINDOWS_API, "Windows")
 
         targets: dict[str, dict] = {}
@@ -158,7 +173,7 @@ class Scraper(BaseTargetScraper):
 
         return {
             "platform": "emudeck",
-            "source": self.url,
+            "source": f"{self.url}/tree/{self.revision}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
             "targets": targets,
         }

scripts/scraper/targets/retropie_targets_scraper.py

@@ -1,6 +1,7 @@
 """Scraper for RetroPie package availability per platform.
 
-Source: https://github.com/RetroPie/RetroPie-Setup/tree/master/scriptmodules
+Source: https://github.com/RetroPie/RetroPie-Setup/tree/<rev>/scriptmodules,
+read at the revision platforms/retropie.yml was scraped at.
 Parses rp_module_id and rp_module_flags from each scriptmodule of the
 package sections the platform list reads (emulators, libretrocores, ports)
 to determine which platforms each package supports.
@@ -26,7 +27,7 @@ GITHUB_API_URL = (
     "https://api.github.com/repos/RetroPie/RetroPie-Setup/contents/scriptmodules"
 )
 RAW_BASE_URL = (
-    "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/master/scriptmodules/"
+    "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/{rev}/scriptmodules/"
 )
 # The sections the platform scraper reads: a standalone package missing here
 # is filtered out of every targeted pack.
@@ -114,14 +115,23 @@ def _parse_module(content: str) -> tuple[str | None, str]:
 class Scraper(BaseTargetScraper):
     """Fetches RetroPie package availability by parsing scriptmodules."""
 
-    def __init__(self, url: str = GITHUB_API_URL) -> None:
+    def __init__(self, url: str = GITHUB_API_URL, revision: str | None = None) -> None:
         super().__init__(url=url)
+        self._revision = revision
+
+    @property
+    def revision(self) -> str:
+        """The commit the package list was read at: the flags of another
+        revision filter packages the platform file does not describe."""
+        if self._revision is None:
+            self._revision = self.pinned_revision(PLATFORM_NAME)
+        return self._revision
 
     def _list_scriptmodules(self) -> list[str]:
         """Return section/filename for every .sh of the package sections."""
         names: list[str] = []
         for section in PACKAGE_DIRS:
-            url = f"{self.url}/{section}"
+            url = f"{self.url}/{section}?ref={self.revision}"
             entries = json.loads(_fetch(url, accept="application/vnd.github+json"))
             listed = [
                 f"{section}/{e['name']}" for e in entries if e.get("name", "").endswith(".sh")
@@ -132,7 +142,7 @@ class Scraper(BaseTargetScraper):
         return names
 
     def _fetch_module(self, filename: str) -> str:
-        return _fetch(f"{RAW_BASE_URL}{filename}")
+        return _fetch(f"{RAW_BASE_URL.format(rev=self.revision)}{filename}")
 
     def fetch_targets(self) -> dict:
         print("  listing RetroPie scriptmodules...", file=sys.stderr)
@@ -170,7 +180,7 @@ class Scraper(BaseTargetScraper):
 
         return {
             "platform": "retropie",
-            "source": self.url,
+            "source": f"{self.url}?ref={self.revision}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
             "targets": targets,
         }