From 16f6ef8cbfb461f53e617fa90edcc236fe4ac5fd Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:06:18 +0200 Subject: [PATCH] feat: record the commit a branch scrape read Six platforms are read from a branch: System.dat, batocera-systems.json, checkBIOS.sh, the MiSTer BiosDB, the RetroPie scriptmodules and the RetroDECK component manifests. Their YAML named the branch, so the native export patched whatever the branch had become since the scrape. Each scraper now resolves the branch head once, reads every file at that commit and writes it into source. The export follows a recorded tarball and a recorded tree as it already followed a raw file URL. --- platforms/emudeck.yml | 2 +- platforms/misterfpga.yml | 2 +- platforms/retroarch.yml | 2 +- platforms/retrobat.yml | 2 +- platforms/retrodeck.yml | 2 +- platforms/retropie.yml | 2 +- scripts/export_native.py | 29 ++++++++++++- scripts/scraper/base_scraper.py | 20 +++++++++ scripts/scraper/emudeck_scraper.py | 14 +++++-- scripts/scraper/libretro_scraper.py | 15 +++++-- scripts/scraper/misterfpga_scraper.py | 12 ++++-- scripts/scraper/retrobat_scraper.py | 12 ++++-- scripts/scraper/retrodeck_scraper.py | 37 +++++++++++++---- scripts/scraper/retropie_scraper.py | 15 +++---- tests/test_e2e.py | 2 +- tests/test_retrodeck_scraper_errors.py | 16 ++++++++ tests/test_scraper_contract.py | 57 +++++++++++++++++++++++++- 17 files changed, 203 insertions(+), 38 deletions(-) diff --git a/platforms/emudeck.yml b/platforms/emudeck.yml index 7f9a9be4..0e0e1970 100644 --- a/platforms/emudeck.yml +++ b/platforms/emudeck.yml @@ -1,7 +1,7 @@ platform: EmuDeck version: 2.7EA homepage: "https://www.emudeck.com" -source: "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/main/functions/checkBIOS.sh" +source: "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/dddb30229adbbd54d5ff7283cfd70e11c4322c26/functions/checkBIOS.sh" base_destination: bios hash_type: md5 verification_mode: md5 diff --git a/platforms/misterfpga.yml b/platforms/misterfpga.yml index 9eec4e12..0b3cc3ab 100644 --- a/platforms/misterfpga.yml +++ b/platforms/misterfpga.yml @@ -1,7 +1,7 @@ platform: MiSTer FPGA version: '2026-09-29' homepage: "https://misterfpga.org" -source: "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/db/bios_db.json.zip" +source: "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/7cfa7a3e84af078d0efba74fa46c50357c4f567c/bios_db.json.zip" base_destination: games hash_type: md5 verification_mode: md5 diff --git a/platforms/retroarch.yml b/platforms/retroarch.yml index c6289703..50e32746 100644 --- a/platforms/retroarch.yml +++ b/platforms/retroarch.yml @@ -2,7 +2,7 @@ platform: RetroArch version: v1.22.2 dat_version: v1.19.0 homepage: "https://www.retroarch.com" -source: "https://github.com/libretro/libretro-database/blob/master/dat/System.dat" +source: "https://github.com/libretro/libretro-database/blob/12c21cf79f1e20eff6ac68b80875816c717d0a7a/dat/System.dat" base_destination: system hash_type: sha1 verification_mode: existence diff --git a/platforms/retrobat.yml b/platforms/retrobat.yml index ccb9bbfc..19d5e1cf 100644 --- a/platforms/retrobat.yml +++ b/platforms/retrobat.yml @@ -1,7 +1,7 @@ platform: RetroBat version: 8.3.1 homepage: "https://www.retrobat.org" -source: "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/master/batocera-systems/Resources/batocera-systems.json" +source: "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/48cdc61623c5943bb17647b80a60d32d395c3725/batocera-systems/Resources/batocera-systems.json" base_destination: bios hash_type: md5 verification_mode: md5 diff --git a/platforms/retrodeck.yml b/platforms/retrodeck.yml index a28af67a..5e9da15d 100644 --- a/platforms/retrodeck.yml +++ b/platforms/retrodeck.yml @@ -1,7 +1,7 @@ platform: RetroDECK version: 0.10.10b homepage: "https://retrodeck.net" -source: "https://github.com/RetroDECK/components" +source: "https://github.com/RetroDECK/components/tree/fcd0d72995d7aa2e741e2753ac6914025b2ff1ac" base_destination: '' hash_type: md5 verification_mode: md5 diff --git a/platforms/retropie.yml b/platforms/retropie.yml index 50a888a4..6f670b3a 100644 --- a/platforms/retropie.yml +++ b/platforms/retropie.yml @@ -2,7 +2,7 @@ inherits: retroarch platform: RetroPie version: 4.8.12 homepage: "https://retropie.org.uk" -source: "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master" +source: "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/46128cd252cdecae852a4d0bafa2aaf9d03ab563" base_destination: BIOS cores: - "81" diff --git a/scripts/export_native.py b/scripts/export_native.py index 1fb1ce32..1315589b 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -163,12 +163,20 @@ def pinned_base(wanted: dict[str, str], scraped: dict | None) -> str: correct a file our data never described. """ source = _raw_url(str((scraped or {}).get("source", ""))) + tree = _GITHUB_TREE.match(source) + if tree: + # RetroDECK reads one manifest per component: the YAML records the + # tree at the commit read, and every file is taken at that commit. + return "https://raw.githubusercontent.com/{}/{}/{}/".format(*tree.groups()) for relative in wanted: if source.endswith("/" + relative): return source[: -len(relative)] return "" +_GITHUB_TREE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/tree/([0-9a-f]{40})/?$") + + def _raw_url(url: str) -> str: """A GitHub blob page names a revision but serves HTML, raw serves bytes.""" marker = "/blob/" @@ -196,7 +204,26 @@ def wanted_sources( relative: _at_revision(wanted[relative], base) or base + relative for relative in wanted } - return wanted + source = str((scraped or {}).get("source", "")) + return { + relative: _tarball_at(url, source) or url for relative, url in wanted.items() + } + + +_CODELOAD = re.compile(r"^https://codeload\.github\.com/([^/]+)/([^/]+)/tar\.gz/") + + +def _tarball_at(url: str, source: str) -> str: + """The recorded tarball when the original is a tarball of the same repository. + + RetroPie's declarations are spread over every scriptmodule, so its scraper + reads the repository archive. The archive the YAML records is the one at + the commit it was read from, the exporter's own URL names the branch. + """ + wanted, recorded = _CODELOAD.match(url), _CODELOAD.match(source) + if wanted and recorded and wanted.group(1, 2) == recorded.group(1, 2): + return source + return "" _RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/") diff --git a/scripts/scraper/base_scraper.py b/scripts/scraper/base_scraper.py index ac9b2362..e4703ff9 100644 --- a/scripts/scraper/base_scraper.py +++ b/scripts/scraper/base_scraper.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import os +import re import sys import time import urllib.error @@ -296,6 +297,25 @@ def fetch_github_latest_version(repo: str) -> str: return tag +def fetch_branch_head(repo: str, branch: str) -> str: + """The commit a branch of a GitHub repo points at right now. + + A platform read from a moving branch records this sha in its source URL, + so the exporter later patches the revision the data was read from and + not whatever the branch has become. Raises like the release lookup. + """ + url = f"https://api.github.com/repos/{repo}/commits/{branch}" + try: + req = urllib.request.Request(url, headers=github_headers()) + with urllib.request.urlopen(req, timeout=15) as resp: + sha = json.loads(resp.read()).get("sha", "") + except (urllib.error.URLError, json.JSONDecodeError) as e: + raise RuntimeError(f"cannot read the head of {repo}@{branch}: {e}") from e + if not re.fullmatch(r"[0-9a-f]{40}", sha or ""): + raise RuntimeError(f"{repo}@{branch} answered no commit sha") + return sha + + class _PlatformDumper(yaml.SafeDumper): """Quote scalars that a YAML reader could misparse. diff --git a/scripts/scraper/emudeck_scraper.py b/scripts/scraper/emudeck_scraper.py index 0fd6ad23..1b51b909 100644 --- a/scripts/scraper/emudeck_scraper.py +++ b/scripts/scraper/emudeck_scraper.py @@ -21,6 +21,7 @@ try: from .base_scraper import ( BaseScraper, BiosRequirement, + fetch_branch_head, fetch_github_latest_version, github_headers, requirement_entry, @@ -29,6 +30,7 @@ except ImportError: from base_scraper import ( BaseScraper, BiosRequirement, + fetch_branch_head, fetch_github_latest_version, github_headers, requirement_entry, @@ -36,9 +38,10 @@ except ImportError: PLATFORM_NAME = "emudeck" +EMUDECK_REPO = "dragoonDorise/EmuDeck" CHECKBIOS_URL = ( "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/" - "main/functions/checkBIOS.sh" + "{rev}/functions/checkBIOS.sh" ) CSV_BASE_URL = ( @@ -220,8 +223,13 @@ class Scraper(BaseScraper): """Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets.""" def __init__( - self, checkbios_url: str = CHECKBIOS_URL, csv_base_url: str = CSV_BASE_URL + self, checkbios_url: str | None = None, csv_base_url: str = CSV_BASE_URL ) -> None: + # checkBIOS.sh is read from main: the commit read is recorded. + if checkbios_url is None: + checkbios_url = CHECKBIOS_URL.format( + rev=fetch_branch_head(EMUDECK_REPO, "main") + ) super().__init__(url=checkbios_url) self.checkbios_url = checkbios_url self.csv_base_url = csv_base_url @@ -436,7 +444,7 @@ class Scraper(BaseScraper): "platform": "EmuDeck", "version": version or "", "homepage": "https://www.emudeck.com", - "source": CHECKBIOS_URL, + "source": self.checkbios_url, "base_destination": "bios", "hash_type": "md5", "verification_mode": "md5", diff --git a/scripts/scraper/libretro_scraper.py b/scripts/scraper/libretro_scraper.py index 227b5506..846c2234 100644 --- a/scripts/scraper/libretro_scraper.py +++ b/scripts/scraper/libretro_scraper.py @@ -14,6 +14,7 @@ import urllib.request from .base_scraper import ( BaseScraper, BiosRequirement, + fetch_branch_head, fetch_github_latest_version, github_headers, requirement_entry, @@ -22,8 +23,9 @@ from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format PLATFORM_NAME = "libretro" +DATABASE_REPO = "libretro/libretro-database" SOURCE_URL = ( - "https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat" + "https://raw.githubusercontent.com/libretro/libretro-database/{rev}/dat/System.dat" ) # Libretro cores that expect BIOS files in a subdirectory of system/. @@ -125,8 +127,10 @@ def _core_info_files() -> list[str]: class Scraper(BaseScraper): """Scraper for libretro System.dat.""" - def __init__(self, url: str = SOURCE_URL) -> None: - super().__init__(url=url) + def __init__(self, url: str | None = None) -> None: + # System.dat lives on master: the commit read is the one recorded. + self.revision = fetch_branch_head(DATABASE_REPO, "master") if url is None else "" + super().__init__(url=url or SOURCE_URL.format(rev=self.revision)) def fetch_requirements(self) -> list[BiosRequirement]: """Parse System.dat and return BIOS requirements.""" @@ -504,7 +508,10 @@ class Scraper(BaseScraper): "version": metadata["retroarch_version"] or "", "dat_version": metadata["dat_version"] or "", "homepage": "https://www.retroarch.com", - "source": "https://github.com/libretro/libretro-database/blob/master/dat/System.dat", + "source": ( + f"https://github.com/{DATABASE_REPO}/blob/" + f"{self.revision or 'master'}/dat/System.dat" + ), "base_destination": "system", "hash_type": "sha1", "verification_mode": "existence", diff --git a/scripts/scraper/misterfpga_scraper.py b/scripts/scraper/misterfpga_scraper.py index 3553c339..78585f6e 100644 --- a/scripts/scraper/misterfpga_scraper.py +++ b/scripts/scraper/misterfpga_scraper.py @@ -33,6 +33,7 @@ try: BaseScraper, BiosRequirement, _read_limited, + fetch_branch_head, requirement_entry, ) except ImportError: @@ -40,13 +41,15 @@ except ImportError: BaseScraper, BiosRequirement, _read_limited, + fetch_branch_head, requirement_entry, ) PLATFORM_NAME = "misterfpga" +BIOSDB_REPO = "ajgowans/BiosDB_MiSTer" SOURCE_URL = ( - "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/db/bios_db.json.zip" + "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/{rev}/bios_db.json.zip" ) DB_MEMBER = "bios_db.json" @@ -126,7 +129,10 @@ def system_of(path: str) -> str: class Scraper(BaseScraper): """Scraper for the MiSTer BiosDB database.""" - def __init__(self, url: str = SOURCE_URL) -> None: + def __init__(self, url: str | None = None) -> None: + # The database is published on the db branch: the commit read is recorded. + if url is None: + url = SOURCE_URL.format(rev=fetch_branch_head(BIOSDB_REPO, "db")) super().__init__(url=url) def _fetch_raw(self) -> str: @@ -220,7 +226,7 @@ class Scraper(BaseScraper): "platform": "MiSTer FPGA", "version": released, "homepage": "https://misterfpga.org", - "source": SOURCE_URL, + "source": self.url, "base_destination": "games", "hash_type": "md5", "verification_mode": "md5", diff --git a/scripts/scraper/retrobat_scraper.py b/scripts/scraper/retrobat_scraper.py index c7b0736b..7b2d6ce9 100644 --- a/scripts/scraper/retrobat_scraper.py +++ b/scripts/scraper/retrobat_scraper.py @@ -16,6 +16,7 @@ try: from .base_scraper import ( BaseScraper, BiosRequirement, + fetch_branch_head, fetch_github_latest_version, requirement_entry, ) @@ -24,6 +25,7 @@ except ImportError: from base_scraper import ( BaseScraper, BiosRequirement, + fetch_branch_head, fetch_github_latest_version, requirement_entry, ) @@ -31,9 +33,10 @@ except ImportError: PLATFORM_NAME = "retrobat" +LAUNCHER_REPO = "RetroBat-Official/emulatorlauncher" SOURCE_URL = ( "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/" - "master/batocera-systems/Resources/batocera-systems.json" + "{rev}/batocera-systems/Resources/batocera-systems.json" ) GITHUB_REPO = "RetroBat-Official/retrobat" @@ -103,7 +106,10 @@ SYSTEM_SLUG_MAP = { class Scraper(BaseScraper): """Scraper for RetroBat batocera-systems.json.""" - def __init__(self, url: str = SOURCE_URL) -> None: + def __init__(self, url: str | None = None) -> None: + # batocera-systems.json is read from master: the commit read is recorded. + if url is None: + url = SOURCE_URL.format(rev=fetch_branch_head(LAUNCHER_REPO, "master")) super().__init__(url=url) self._parsed: dict | None = None @@ -234,7 +240,7 @@ class Scraper(BaseScraper): "platform": "RetroBat", "version": version, "homepage": "https://www.retrobat.org", - "source": SOURCE_URL, + "source": self.url, "base_destination": "bios", "hash_type": "md5", "verification_mode": "md5", diff --git a/scripts/scraper/retrodeck_scraper.py b/scripts/scraper/retrodeck_scraper.py index 97e28f40..d5cf4114 100644 --- a/scripts/scraper/retrodeck_scraper.py +++ b/scripts/scraper/retrodeck_scraper.py @@ -36,18 +36,26 @@ import urllib.request from pathlib import Path try: - from .base_scraper import BaseScraper, BiosRequirement, requirement_entry + from .base_scraper import ( + BaseScraper, + BiosRequirement, + fetch_branch_head, + requirement_entry, + ) except ImportError: sys.path.insert(0, str(Path(__file__).parent.parent)) - from scraper.base_scraper import BaseScraper, BiosRequirement, requirement_entry + from scraper.base_scraper import ( + BaseScraper, + BiosRequirement, + fetch_branch_head, + requirement_entry, + ) PLATFORM_NAME = "retrodeck" COMPONENTS_REPO = "RetroDECK/components" COMPONENTS_BRANCH = "main" -COMPONENTS_API_URL = ( - f"https://api.github.com/repos/{COMPONENTS_REPO}/git/trees/{COMPONENTS_BRANCH}" -) -RAW_BASE = f"https://raw.githubusercontent.com/{COMPONENTS_REPO}/{COMPONENTS_BRANCH}" +COMPONENTS_API_URL = "https://api.github.com/repos/" + COMPONENTS_REPO + "/git/trees/{rev}" +RAW_BASE = "https://raw.githubusercontent.com/" + COMPONENTS_REPO + "/{rev}" SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"} NON_EMULATOR_COMPONENTS = { "framework", @@ -225,6 +233,11 @@ class Scraper(BaseScraper): super().__init__() self.manifests_dir = manifests_dir self._manifests: list[tuple[str, dict]] | None = None + # The manifests are read from main: one commit for the tree and for + # every file, and that commit is the one recorded. + self.revision = ( + "" if manifests_dir else fetch_branch_head(COMPONENTS_REPO, COMPONENTS_BRANCH) + ) def _get_manifests(self) -> list[tuple[str, dict]]: """Fetch manifests once, cache for reuse.""" @@ -244,7 +257,9 @@ class Scraper(BaseScraper): headers["Authorization"] = f"token {token}" try: - req = urllib.request.Request(COMPONENTS_API_URL, headers=headers) + req = urllib.request.Request( + COMPONENTS_API_URL.format(rev=self.revision), headers=headers + ) with urllib.request.urlopen(req, timeout=30) as resp: tree = json.loads(resp.read().decode()) except (urllib.error.HTTPError, urllib.error.URLError) as e: @@ -263,7 +278,7 @@ class Scraper(BaseScraper): manifests: list[tuple[str, dict]] = [] for comp in sorted(component_dirs): - url = f"{RAW_BASE}/{comp}/component_manifest.json" + url = f"{RAW_BASE.format(rev=self.revision)}/{comp}/component_manifest.json" print(f" {comp} ...", file=sys.stderr, end="", flush=True) # Only a 404 says the component has no manifest. A 429, a 503 or # a dropped connection is no answer: skipping it wrote @@ -450,7 +465,11 @@ class Scraper(BaseScraper): "platform": "RetroDECK", "version": version, "homepage": "https://retrodeck.net", - "source": "https://github.com/RetroDECK/components", + "source": ( + f"https://github.com/{COMPONENTS_REPO}/tree/{self.revision}" + if self.revision + else f"https://github.com/{COMPONENTS_REPO}" + ), "base_destination": "", "hash_type": "md5", "verification_mode": "md5", diff --git a/scripts/scraper/retropie_scraper.py b/scripts/scraper/retropie_scraper.py index 6c2eefe2..07a3d77c 100644 --- a/scripts/scraper/retropie_scraper.py +++ b/scripts/scraper/retropie_scraper.py @@ -26,15 +26,13 @@ import urllib.error import urllib.request try: - from .base_scraper import BaseScraper, BiosRequirement + from .base_scraper import BaseScraper, BiosRequirement, fetch_branch_head except ImportError: - from base_scraper import BaseScraper, BiosRequirement + from base_scraper import BaseScraper, BiosRequirement, fetch_branch_head PLATFORM_NAME = "retropie" -SOURCE_URL = ( - "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master" -) +SOURCE_URL = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/{rev}" GITHUB_REPO = "RetroPie/RetroPie-Setup" MAX_ARCHIVE = 64 * 1024 * 1024 @@ -51,7 +49,10 @@ _PACKAGE_DIRS = ("emulators", "libretrocores", "ports") class Scraper(BaseScraper): """Scraper for the RetroPie package list.""" - def __init__(self, url: str = SOURCE_URL) -> None: + def __init__(self, url: str | None = None) -> None: + # The package list is read from master: the commit read is recorded. + if url is None: + url = SOURCE_URL.format(rev=fetch_branch_head(GITHUB_REPO, "master")) super().__init__(url=url) self._modules: list[str] | None = None self._version = "" @@ -135,7 +136,7 @@ class Scraper(BaseScraper): "platform": "RetroPie", "version": self._version, "homepage": "https://retropie.org.uk", - "source": SOURCE_URL, + "source": self.url, "base_destination": "BIOS", "cores": cores, } diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 0003f3c2..3a7472a2 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -5497,7 +5497,7 @@ struct BurnDriver BurnDrvneogeo = { from scraper.misterfpga_scraper import Scraper - scraper = Scraper() + scraper = Scraper(url="https://example.test/bios_db.json.zip") scraper._raw_data = json.dumps(self.MISTER_DB) return scraper diff --git a/tests/test_retrodeck_scraper_errors.py b/tests/test_retrodeck_scraper_errors.py index 08db2667..2ab272d9 100644 --- a/tests/test_retrodeck_scraper_errors.py +++ b/tests/test_retrodeck_scraper_errors.py @@ -34,6 +34,8 @@ class _Response(io.BytesIO): def _opener(failure): def urlopen(req, timeout=0): url = req.full_url + if "/commits/" in url: + return _Response(json.dumps({"sha": "a" * 40}).encode()) if "api.github.com" in url: return _Response(json.dumps(TREE).encode()) if "pcsx2" in url and failure is not None: @@ -60,6 +62,20 @@ class ManifestErrors(unittest.TestCase): error = urllib.error.HTTPError("u", 404, "not found", {}, None) self.assertEqual([name for name, _ in self.fetch(error)], ["ppsspp"]) + def test_the_tree_and_files_are_read_at_one_commit(self): + seen = [] + opener = _opener(None) + + def recording(req, timeout=0): + seen.append(req.full_url) + return opener(req, timeout) + + with mock.patch("urllib.request.urlopen", recording): + Scraper()._fetch_remote_manifests() + reads = [url for url in seen if "/commits/" not in url] + self.assertTrue(reads) + self.assertTrue(all("a" * 40 in url for url in reads), reads) + def test_a_truncated_tree_stops_the_scrape(self): TREE["truncated"] = True self.addCleanup(TREE.pop, "truncated") diff --git a/tests/test_scraper_contract.py b/tests/test_scraper_contract.py index e4fc31c0..bdbbac99 100644 --- a/tests/test_scraper_contract.py +++ b/tests/test_scraper_contract.py @@ -11,6 +11,8 @@ exercised is a promise to contributors backed by nothing. from __future__ import annotations import importlib +import io +import json import subprocess import sys import tempfile @@ -246,7 +248,9 @@ class RetroPiePackageList(unittest.TestCase): def _scraper(self, modules: dict[str, str]): from scraper.retropie_scraper import Scraper - scraper = Scraper() + # An explicit archive URL: the tests read a local archive, never the + # branch head. + scraper = Scraper(url="https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/" + "0" * 40) scraper.module_ids(self._archive(modules)) return scraper @@ -502,6 +506,11 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase): "scraper.romm_scraper", "scraper.batocera_scraper", "scraper.recalbox_scraper", + "scraper.libretro_scraper", + "scraper.retrobat_scraper", + "scraper.emudeck_scraper", + "scraper.misterfpga_scraper", + "scraper.retropie_scraper", ): module = importlib.import_module(name) with self.subTest(scraper=name), mock.patch( @@ -509,6 +518,52 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase): ), self.assertRaises(RuntimeError): module.Scraper() + def test_a_branch_read_records_its_commit(self): + """System.dat, batocera-systems.json, checkBIOS.sh, the BiosDB and the + RetroPie scriptmodules are read from a branch. The YAML named the + branch, so the exporter patched whatever it had become since.""" + sha = "0123456789abcdef0123456789abcdef01234567" + + class Answer(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *_exc): + return False + + def answer(request, timeout=None): + return Answer(json.dumps({"sha": sha}).encode()) + + for name in ( + "scraper.libretro_scraper", + "scraper.retrobat_scraper", + "scraper.emudeck_scraper", + "scraper.misterfpga_scraper", + "scraper.retropie_scraper", + ): + module = importlib.import_module(name) + with self.subTest(scraper=name), mock.patch("urllib.request.urlopen", answer): + self.assertRegex(module.Scraper().url, f"/{sha}(/|$)") + + def test_the_export_reads_the_recorded_tarball(self): + from export_native import _tarball_at + + branch = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master" + pinned = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/" + "a" * 40 + self.assertEqual(_tarball_at(branch, pinned), pinned) + other = "https://codeload.github.com/Other/Repo/tar.gz/" + "b" * 40 + self.assertEqual(_tarball_at(branch, other), "") + + def test_the_export_reads_every_component_at_the_recorded_tree(self): + from export_native import pinned_base + + sha = "c" * 40 + base = pinned_base( + {"pcsx2/component_manifest.json": "unused"}, + {"source": f"https://github.com/RetroDECK/components/tree/{sha}"}, + ) + self.assertEqual(base, f"https://raw.githubusercontent.com/RetroDECK/components/{sha}/") + def test_a_release_lookup_raises(self): with mock.patch("urllib.request.urlopen", self._refuse), self.assertRaises( RuntimeError