From 905316061e825a34c48c17ab38c7208f57b037a0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:24:04 +0200 Subject: [PATCH] fix: read retropie and emudeck targets at the pin The RetroPie target scraper read master and the EmuDeck one the default branch, while both platform files are pinned on a commit: a module flag changed in between filtered packages the pinned list does not describe. Both now read the revision pinned_revision returns, which also accepts the codeload URL the RetroPie scraper records. EmuDeck Windows stays on the emudeck-we default branch: the platform file pins no revision of it. --- scripts/scraper/targets/__init__.py | 1 + .../targets/emudeck_targets_scraper.py | 25 ++++++++++--- .../targets/retropie_targets_scraper.py | 22 ++++++++---- tests/test_target_scrapers.py | 36 ++++++++++++++++++- 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/scripts/scraper/targets/__init__.py b/scripts/scraper/targets/__init__.py index 21339b343..08381331f 100644 --- a/scripts/scraper/targets/__init__.py +++ b/scripts/scraper/targets/__init__.py @@ -43,6 +43,7 @@ class BaseTargetScraper(ABC): r"^https://raw\.githubusercontent\.com/[^/]+/[^/]+/([^/]+)/", r"^https://github\.com/[^/]+/[^/]+/(?:blob|tree)/([^/]+)/", r"^https://gitlab\.com/.+?/-/(?:raw|blob|tree)/([^/]+)/", + r"^https://codeload\.github\.com/[^/]+/[^/]+/(?:tar\.gz|zip)/([^/]+)$", ): match = re.match(pattern, source) if match: diff --git a/scripts/scraper/targets/emudeck_targets_scraper.py b/scripts/scraper/targets/emudeck_targets_scraper.py index 393bbb07b..2b356ca62 100644 --- a/scripts/scraper/targets/emudeck_targets_scraper.py +++ b/scripts/scraper/targets/emudeck_targets_scraper.py @@ -1,8 +1,11 @@ """Scraper for EmuDeck emulator targets. Sources: - SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh - Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1 + SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh, read at the + revision platforms/emudeck.yml was scraped at + Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1, default branch: + the platform file is read from dragoonDorise/EmuDeck alone and + pins no emudeck-we revision """ from __future__ import annotations @@ -90,8 +93,18 @@ def _script_to_core(filename: str) -> str | None: class Scraper(BaseTargetScraper): """Fetches emulator lists for EmuDeck SteamOS and Windows targets.""" - def __init__(self, url: str = "https://github.com/dragoonDorise/EmuDeck") -> None: + def __init__( + self, url: str = "https://github.com/dragoonDorise/EmuDeck", revision: str | None = None + ) -> None: super().__init__(url=url) + self._revision = revision + + @property + def revision(self) -> str: + """The checkBIOS.sh commit platforms/emudeck.yml was read at.""" + if self._revision is None: + self._revision = self.pinned_revision(PLATFORM_NAME) + return self._revision def _fetch_cores_for_target( self, api_url: str, label: str, arch: str = "x86_64" @@ -142,7 +155,9 @@ class Scraper(BaseTargetScraper): raise RuntimeError(f"no {arch} core list in {target_path}") def fetch_targets(self) -> dict: - steamos_cores = self._fetch_cores_for_target(STEAMOS_API, "SteamOS") + steamos_cores = self._fetch_cores_for_target( + f"{STEAMOS_API}?ref={self.revision}", "SteamOS" + ) windows_cores = self._fetch_cores_for_target(WINDOWS_API, "Windows") targets: dict[str, dict] = {} @@ -159,7 +174,7 @@ class Scraper(BaseTargetScraper): return { "platform": "emudeck", - "source": self.url, + "source": f"{self.url}/tree/{self.revision}", "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "targets": targets, } diff --git a/scripts/scraper/targets/retropie_targets_scraper.py b/scripts/scraper/targets/retropie_targets_scraper.py index 64d4e3a80..7f7b1d303 100644 --- a/scripts/scraper/targets/retropie_targets_scraper.py +++ b/scripts/scraper/targets/retropie_targets_scraper.py @@ -1,6 +1,7 @@ """Scraper for RetroPie package availability per platform. -Source: https://github.com/RetroPie/RetroPie-Setup/tree/master/scriptmodules +Source: https://github.com/RetroPie/RetroPie-Setup/tree//scriptmodules, +read at the revision platforms/retropie.yml was scraped at. Parses rp_module_id and rp_module_flags from each scriptmodule of the package sections the platform list reads (emulators, libretrocores, ports) to determine which platforms each package supports. @@ -27,7 +28,7 @@ GITHUB_API_URL = ( "https://api.github.com/repos/RetroPie/RetroPie-Setup/contents/scriptmodules" ) RAW_BASE_URL = ( - "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/master/scriptmodules/" + "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/{rev}/scriptmodules/" ) # The sections the platform scraper reads: a standalone package missing here # is filtered out of every targeted pack. @@ -113,14 +114,23 @@ def _parse_module(content: str) -> tuple[str | None, str]: class Scraper(BaseTargetScraper): """Fetches RetroPie package availability by parsing scriptmodules.""" - def __init__(self, url: str = GITHUB_API_URL) -> None: + def __init__(self, url: str = GITHUB_API_URL, revision: str | None = None) -> None: super().__init__(url=url) + self._revision = revision + + @property + def revision(self) -> str: + """The commit the package list was read at: the flags of another + revision filter packages the platform file does not describe.""" + if self._revision is None: + self._revision = self.pinned_revision(PLATFORM_NAME) + return self._revision def _list_scriptmodules(self) -> list[str]: """Return section/filename for every .sh of the package sections.""" names: list[str] = [] for section in PACKAGE_DIRS: - url = f"{self.url}/{section}" + url = f"{self.url}/{section}?ref={self.revision}" entries = json.loads(_fetch(url, accept="application/vnd.github+json")) listed = [ f"{section}/{e['name']}" for e in entries if e.get("name", "").endswith(".sh") @@ -131,7 +141,7 @@ class Scraper(BaseTargetScraper): return names def _fetch_module(self, filename: str) -> str: - return _fetch(f"{RAW_BASE_URL}{filename}") + return _fetch(f"{RAW_BASE_URL.format(rev=self.revision)}{filename}") def fetch_targets(self) -> dict: print(" listing RetroPie scriptmodules...", file=sys.stderr) @@ -169,7 +179,7 @@ class Scraper(BaseTargetScraper): return { "platform": "retropie", - "source": self.url, + "source": f"{self.url}?ref={self.revision}", "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "targets": targets, } diff --git a/tests/test_target_scrapers.py b/tests/test_target_scrapers.py index c97334534..0bfdd60c6 100644 --- a/tests/test_target_scrapers.py +++ b/tests/test_target_scrapers.py @@ -307,7 +307,7 @@ class RetroPieModuleFlags(unittest.TestCase): } def fake_fetch(url, **_kwargs): - section = url.rstrip("/").rsplit("/", 1)[-1] + section = url.split("?", 1)[0].rstrip("/").rsplit("/", 1)[-1] if section in listings: return json.dumps(listings[section]) return modules[url.split("scriptmodules/", 1)[1]] @@ -321,6 +321,40 @@ class RetroPieModuleFlags(unittest.TestCase): +class TargetsReadThePlatformRevision(unittest.TestCase): + """A target list read on master filtered packages by flags the pinned + package list does not have: every request names the platform's pin.""" + + def _urls(self, module, scraper) -> list[str]: + seen: list[str] = [] + + def record(url, **_kwargs): + seen.append(url) + if "contents" in url: + return json.dumps([{"name": "lr-mgba.sh"}]) + return 'rp_module_id="lr-mgba"' + + with mock.patch.object(module, "_fetch", record): + scraper.fetch_targets() + return seen + + def test_retropie_reads_its_codeload_pin(self): + pin = retropie_targets_scraper.Scraper.pinned_revision("retropie") + self.assertRegex(pin, r"^[0-9a-f]{40}$") + urls = self._urls(retropie_targets_scraper, retropie_targets_scraper.Scraper()) + self.assertTrue(urls) + self.assertTrue(all(pin in url for url in urls), urls) + + def test_emudeck_steamos_reads_its_pin(self): + pin = emudeck_targets_scraper.Scraper.pinned_revision("emudeck") + scraper = emudeck_targets_scraper.Scraper() + with mock.patch.object(scraper, "_load_retroarch_cores", return_value=[]): + urls = self._urls(emudeck_targets_scraper, scraper) + steamos = [u for u in urls if "dragoonDorise" in u] + self.assertTrue(steamos) + self.assertTrue(all(u.endswith(f"?ref={pin}") for u in steamos), steamos) + + class HashScrapersRefuseAnUnknownVersion(unittest.TestCase): """mame_hash_scraper wrote core_version "unknown" after a failed API call and cached it for a day, while the pipeline step still read OK."""