From 27d119cb7f8b4a33ed26c5ddf5b369afc27d469f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:01:36 +0200 Subject: [PATCH] fix: scrape targets at the pinned platform release --- platforms/targets/batocera.yml | 4 +-- scripts/scraper/recalbox_scraper.py | 5 ++-- scripts/scraper/targets/__init__.py | 30 +++++++++++++++++++ .../targets/batocera_targets_scraper.py | 6 ++-- .../targets/recalbox_targets_scraper.py | 4 +-- .../targets/rocknix_targets_scraper.py | 8 +---- tests/test_audit_regressions.py | 30 +++++++++++++++++++ 7 files changed, 70 insertions(+), 17 deletions(-) diff --git a/platforms/targets/batocera.yml b/platforms/targets/batocera.yml index 9bcabd15..22a12e46 100644 --- a/platforms/targets/batocera.yml +++ b/platforms/targets/batocera.yml @@ -1,6 +1,6 @@ platform: batocera -source: https://github.com/batocera-linux/batocera.linux -scraped_at: '2026-10-17T10:20:31Z' +source: https://github.com/batocera-linux/batocera.linux/tree/batocera-43.1 +scraped_at: '2026-10-17T22:01:01Z' targets: a3gen2: architecture: aarch64 diff --git a/scripts/scraper/recalbox_scraper.py b/scripts/scraper/recalbox_scraper.py index e7a3fa26..0b8f03b5 100644 --- a/scripts/scraper/recalbox_scraper.py +++ b/scripts/scraper/recalbox_scraper.py @@ -199,8 +199,9 @@ def board_cores(text: str, board: str, enabled: set[str]) -> set[str]: class Scraper(BaseScraper): """Scraper for Recalbox es_bios.xml.""" - def __init__(self): - self.tag = _fetch_gitlab_stable_tag() + def __init__(self, tag: str | None = None): + # A target scraper passes the tag platforms/recalbox.yml pins. + self.tag = tag or _fetch_gitlab_stable_tag() super().__init__(url=SOURCE_URL.format(tag=self.tag)) def _system_inis(self, systems: list[str]) -> list[str]: diff --git a/scripts/scraper/targets/__init__.py b/scripts/scraper/targets/__init__.py index b193eda4..aafebfac 100644 --- a/scripts/scraper/targets/__init__.py +++ b/scripts/scraper/targets/__init__.py @@ -7,6 +7,7 @@ from __future__ import annotations import importlib import pkgutil +import re from abc import ABC, abstractmethod from pathlib import Path @@ -22,6 +23,35 @@ class BaseTargetScraper(ABC): """Fetch targets and their core lists. Returns dict matching target YAML format.""" ... + @staticmethod + def pinned_revision(platform: str, platforms_dir: str | None = None) -> str: + """The upstream revision platforms/.yml was scraped at. + + A target list describes the same release as the platform's BIOS + list: read on its own "latest stable tag", a Recalbox 10.2 board list + filtered 10.1.1 systems. Moving to a new release is a rescrape of the + platform first. + """ + import yaml + + directory = Path(platforms_dir) if platforms_dir else ( + Path(__file__).resolve().parents[3] / "platforms" + ) + with open(directory / f"{platform}.yml", encoding="utf-8") as handle: + source = str((yaml.safe_load(handle) or {}).get("source", "")) + for pattern in ( + r"^https://raw\.githubusercontent\.com/[^/]+/[^/]+/([^/]+)/", + r"^https://github\.com/[^/]+/[^/]+/(?:blob|tree)/([^/]+)/", + r"^https://gitlab\.com/.+?/-/(?:raw|blob|tree)/([^/]+)/", + ): + match = re.match(pattern, source) + if match: + return match.group(1) + raise RuntimeError( + f"platforms/{platform}.yml names no pinned revision in its source " + f"({source!r}): rescrape the platform first" + ) + def write_output(self, data: dict, output_path: str) -> None: """Write target data to YAML file.""" try: diff --git a/scripts/scraper/targets/batocera_targets_scraper.py b/scripts/scraper/targets/batocera_targets_scraper.py index 434babed..49adcbc5 100644 --- a/scripts/scraper/targets/batocera_targets_scraper.py +++ b/scripts/scraper/targets/batocera_targets_scraper.py @@ -32,10 +32,8 @@ PLATFORM_NAME = "batocera" GITHUB_API = "https://api.github.com/repos/batocera-linux/batocera.linux/contents" try: from ..base_scraper import github_headers - from ..batocera_scraper import fetch_stable_tag except ImportError: from scraper.base_scraper import github_headers - from scraper.batocera_scraper import fetch_stable_tag RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/{tag}" @@ -355,7 +353,7 @@ class Scraper(BaseTargetScraper): def fetch_targets(self) -> dict: """Build per-board emulator availability map.""" - self.tag = fetch_stable_tag() + self.tag = self.pinned_revision(PLATFORM_NAME) print(" fetching board list...", file=sys.stderr) boards = self._list_boards() @@ -422,7 +420,7 @@ class Scraper(BaseTargetScraper): return { "platform": "batocera", - "source": self.url, + "source": f"{self.url}/tree/{self.tag}", "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "targets": targets, } diff --git a/scripts/scraper/targets/recalbox_targets_scraper.py b/scripts/scraper/targets/recalbox_targets_scraper.py index 5382848a..e9063e41 100644 --- a/scripts/scraper/targets/recalbox_targets_scraper.py +++ b/scripts/scraper/targets/recalbox_targets_scraper.py @@ -62,8 +62,8 @@ class Scraper(BaseTargetScraper): super().__init__(url=url) def fetch_targets(self) -> dict: - platform = PlatformScraper() - tag = platform.tag + tag = self.pinned_revision(PLATFORM_NAME) + platform = PlatformScraper(tag) listing = json.loads(fetch_bytes(CONFIGS_TREE_URL.format(tag=tag))) boards = { match.group(1): item["name"] diff --git a/scripts/scraper/targets/rocknix_targets_scraper.py b/scripts/scraper/targets/rocknix_targets_scraper.py index 161170b2..2a1c019d 100644 --- a/scripts/scraper/targets/rocknix_targets_scraper.py +++ b/scripts/scraper/targets/rocknix_targets_scraper.py @@ -30,10 +30,8 @@ DEVICES_DIR = "projects/ROCKNIX/devices" try: from ..base_scraper import fetch_bytes, github_headers - from ..rocknix_scraper import GITHUB_REPO except ImportError: from scraper.base_scraper import fetch_bytes, github_headers - from scraper.rocknix_scraper import GITHUB_REPO # `NAME="a b \ # c"` or `NAME+=" a b"`, continuation lines joined first. @@ -148,11 +146,7 @@ class Scraper(BaseTargetScraper): return sorted(devices) def fetch_targets(self) -> dict: - from scraper.base_scraper import fetch_github_latest_version # noqa: PLC0415 - - self.tag = fetch_github_latest_version(GITHUB_REPO) - if not self.tag: - raise RuntimeError("cannot read ROCKNIX's latest release tag") + self.tag = self.pinned_revision(PLATFORM_NAME) package_mk = self._raw(EMULATORS_MK) if "LIBRETRO_CORES" not in package_mk: raise RuntimeError(f"no LIBRETRO_CORES in {EMULATORS_MK} at {self.tag}") diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 6bcec033..7d61ac49 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -3717,5 +3717,35 @@ class CompanionsComeFromADirectoryIndex(unittest.TestCase): self.assertIs(packextras._files_by_directory(files_db), index) + +class TargetListsReadThePinnedRelease(unittest.TestCase): + """Each target scraper asked upstream for its latest tag: a Recalbox + 10.2 board list could filter 10.1.1 systems, and Batocera's target file + did not even record the tag it read.""" + + def test_the_revision_comes_from_the_platform_yaml(self): + from scripts.scraper.targets import BaseTargetScraper + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp: + for name, source, tag in ( + ("a", "https://raw.githubusercontent.com/o/r/batocera-43.1/x/y", "batocera-43.1"), + ("b", "https://gitlab.com/recalbox/recalbox/-/raw/10.1.1/a/b.xml", "10.1.1"), + ("c", "https://github.com/o/r/blob/v2/f.cs", "v2"), + ): + Path(tmp, f"{name}.yml").write_text(f"source: {source}\n") + self.assertEqual(BaseTargetScraper.pinned_revision(name, tmp), tag) + Path(tmp, "d.yml").write_text("source: https://example.test/x\n") + with self.assertRaises(RuntimeError): + BaseTargetScraper.pinned_revision("d", tmp) + + def test_no_target_scraper_asks_for_the_latest_tag(self): + for name in ("batocera", "rocknix", "recalbox"): + source = (ROOT / "scripts" / "scraper" / "targets" / f"{name}_targets_scraper.py").read_text() + with self.subTest(scraper=name): + self.assertIn("self.pinned_revision(PLATFORM_NAME)", source) + self.assertNotIn("fetch_stable_tag()", source) + self.assertNotIn("fetch_github_latest_version(", source) + + if __name__ == "__main__": unittest.main()