fix: scrape targets at the pinned platform release

scripts/scraper/recalbox_scraper.py

@@ -199,8 +199,9 @@ def board_cores(text: str, board: str, enabled: set[str]) -> set[str]:
 class Scraper(BaseScraper):
     """Scraper for Recalbox es_bios.xml."""
 
-    def __init__(self):
-        self.tag = _fetch_gitlab_stable_tag()
+    def __init__(self, tag: str | None = None):
+        # A target scraper passes the tag platforms/recalbox.yml pins.
+        self.tag = tag or _fetch_gitlab_stable_tag()
         super().__init__(url=SOURCE_URL.format(tag=self.tag))
 
     def _system_inis(self, systems: list[str]) -> list[str]:

scripts/scraper/targets/__init__.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import importlib
 import pkgutil
+import re
 from abc import ABC, abstractmethod
 from pathlib import Path
 
@@ -22,6 +23,35 @@ class BaseTargetScraper(ABC):
         """Fetch targets and their core lists. Returns dict matching target YAML format."""
         ...
 
+    @staticmethod
+    def pinned_revision(platform: str, platforms_dir: str | None = None) -> str:
+        """The upstream revision platforms/<platform>.yml was scraped at.
+
+        A target list describes the same release as the platform's BIOS
+        list: read on its own "latest stable tag", a Recalbox 10.2 board list
+        filtered 10.1.1 systems. Moving to a new release is a rescrape of the
+        platform first.
+        """
+        import yaml
+
+        directory = Path(platforms_dir) if platforms_dir else (
+            Path(__file__).resolve().parents[3] / "platforms"
+        )
+        with open(directory / f"{platform}.yml", encoding="utf-8") as handle:
+            source = str((yaml.safe_load(handle) or {}).get("source", ""))
+        for pattern in (
+            r"^https://raw\.githubusercontent\.com/[^/]+/[^/]+/([^/]+)/",
+            r"^https://github\.com/[^/]+/[^/]+/(?:blob|tree)/([^/]+)/",
+            r"^https://gitlab\.com/.+?/-/(?:raw|blob|tree)/([^/]+)/",
+        ):
+            match = re.match(pattern, source)
+            if match:
+                return match.group(1)
+        raise RuntimeError(
+            f"platforms/{platform}.yml names no pinned revision in its source "
+            f"({source!r}): rescrape the platform first"
+        )
+
     def write_output(self, data: dict, output_path: str) -> None:
         """Write target data to YAML file."""
         try:

scripts/scraper/targets/batocera_targets_scraper.py

@@ -32,10 +32,8 @@ PLATFORM_NAME = "batocera"
 GITHUB_API = "https://api.github.com/repos/batocera-linux/batocera.linux/contents"
 try:
     from ..base_scraper import github_headers
-    from ..batocera_scraper import fetch_stable_tag
 except ImportError:
     from scraper.base_scraper import github_headers
-    from scraper.batocera_scraper import fetch_stable_tag
 
 RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/{tag}"
 
@@ -355,7 +353,7 @@ class Scraper(BaseTargetScraper):
 
     def fetch_targets(self) -> dict:
         """Build per-board emulator availability map."""
-        self.tag = fetch_stable_tag()
+        self.tag = self.pinned_revision(PLATFORM_NAME)
         print("  fetching board list...", file=sys.stderr)
         boards = self._list_boards()
 
@@ -422,7 +420,7 @@ class Scraper(BaseTargetScraper):
 
         return {
             "platform": "batocera",
-            "source": self.url,
+            "source": f"{self.url}/tree/{self.tag}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
             "targets": targets,
         }

scripts/scraper/targets/recalbox_targets_scraper.py

@@ -62,8 +62,8 @@ class Scraper(BaseTargetScraper):
         super().__init__(url=url)
 
     def fetch_targets(self) -> dict:
-        platform = PlatformScraper()
-        tag = platform.tag
+        tag = self.pinned_revision(PLATFORM_NAME)
+        platform = PlatformScraper(tag)
         listing = json.loads(fetch_bytes(CONFIGS_TREE_URL.format(tag=tag)))
         boards = {
             match.group(1): item["name"]

scripts/scraper/targets/rocknix_targets_scraper.py

@@ -30,10 +30,8 @@ DEVICES_DIR = "projects/ROCKNIX/devices"
 
 try:
     from ..base_scraper import fetch_bytes, github_headers
-    from ..rocknix_scraper import GITHUB_REPO
 except ImportError:
     from scraper.base_scraper import fetch_bytes, github_headers
-    from scraper.rocknix_scraper import GITHUB_REPO
 
 # `NAME="a b \
 #   c"` or `NAME+=" a b"`, continuation lines joined first.
@@ -148,11 +146,7 @@ class Scraper(BaseTargetScraper):
         return sorted(devices)
 
     def fetch_targets(self) -> dict:
-        from scraper.base_scraper import fetch_github_latest_version  # noqa: PLC0415
-
-        self.tag = fetch_github_latest_version(GITHUB_REPO)
-        if not self.tag:
-            raise RuntimeError("cannot read ROCKNIX's latest release tag")
+        self.tag = self.pinned_revision(PLATFORM_NAME)
         package_mk = self._raw(EMULATORS_MK)
         if "LIBRETRO_CORES" not in package_mk:
             raise RuntimeError(f"no LIBRETRO_CORES in {EMULATORS_MK} at {self.tag}")