feat: record the commit a branch scrape read

Six platforms are read from a branch: System.dat, batocera-systems.json, checkBIOS.sh, the MiSTer BiosDB, the RetroPie scriptmodules and the RetroDECK component manifests. Their YAML named the branch, so the native export patched whatever the branch had become since the scrape. Each scraper now resolves the branch head once, reads every file at that commit and writes it into source. The export follows a recorded tarball and a recorded tree as it already followed a raw file URL.

scripts/export_native.py

@@ -163,12 +163,20 @@ def pinned_base(wanted: dict[str, str], scraped: dict | None) -> str:
     correct a file our data never described.
     """
     source = _raw_url(str((scraped or {}).get("source", "")))
+    tree = _GITHUB_TREE.match(source)
+    if tree:
+        # RetroDECK reads one manifest per component: the YAML records the
+        # tree at the commit read, and every file is taken at that commit.
+        return "https://raw.githubusercontent.com/{}/{}/{}/".format(*tree.groups())
     for relative in wanted:
         if source.endswith("/" + relative):
             return source[: -len(relative)]
     return ""
 
 
+_GITHUB_TREE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/tree/([0-9a-f]{40})/?$")
+
+
 def _raw_url(url: str) -> str:
     """A GitHub blob page names a revision but serves HTML, raw serves bytes."""
     marker = "/blob/"
@@ -196,7 +204,26 @@ def wanted_sources(
             relative: _at_revision(wanted[relative], base) or base + relative
             for relative in wanted
         }
-    return wanted
+    source = str((scraped or {}).get("source", ""))
+    return {
+        relative: _tarball_at(url, source) or url for relative, url in wanted.items()
+    }
+
+
+_CODELOAD = re.compile(r"^https://codeload\.github\.com/([^/]+)/([^/]+)/tar\.gz/")
+
+
+def _tarball_at(url: str, source: str) -> str:
+    """The recorded tarball when the original is a tarball of the same repository.
+
+    RetroPie's declarations are spread over every scriptmodule, so its scraper
+    reads the repository archive. The archive the YAML records is the one at
+    the commit it was read from, the exporter's own URL names the branch.
+    """
+    wanted, recorded = _CODELOAD.match(url), _CODELOAD.match(source)
+    if wanted and recorded and wanted.group(1, 2) == recorded.group(1, 2):
+        return source
+    return ""
 
 
 _RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/")

scripts/scraper/base_scraper.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import sys
 import time
 import urllib.error
@@ -296,6 +297,25 @@ def fetch_github_latest_version(repo: str) -> str:
     return tag
 
 
+def fetch_branch_head(repo: str, branch: str) -> str:
+    """The commit a branch of a GitHub repo points at right now.
+
+    A platform read from a moving branch records this sha in its source URL,
+    so the exporter later patches the revision the data was read from and
+    not whatever the branch has become. Raises like the release lookup.
+    """
+    url = f"https://api.github.com/repos/{repo}/commits/{branch}"
+    try:
+        req = urllib.request.Request(url, headers=github_headers())
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            sha = json.loads(resp.read()).get("sha", "")
+    except (urllib.error.URLError, json.JSONDecodeError) as e:
+        raise RuntimeError(f"cannot read the head of {repo}@{branch}: {e}") from e
+    if not re.fullmatch(r"[0-9a-f]{40}", sha or ""):
+        raise RuntimeError(f"{repo}@{branch} answered no commit sha")
+    return sha
+
+
 class _PlatformDumper(yaml.SafeDumper):
     """Quote scalars that a YAML reader could misparse.
 

scripts/scraper/emudeck_scraper.py

@@ -21,6 +21,7 @@ try:
     from .base_scraper import (
         BaseScraper,
         BiosRequirement,
+        fetch_branch_head,
         fetch_github_latest_version,
         github_headers,
         requirement_entry,
@@ -29,6 +30,7 @@ except ImportError:
     from base_scraper import (
         BaseScraper,
         BiosRequirement,
+        fetch_branch_head,
         fetch_github_latest_version,
         github_headers,
         requirement_entry,
@@ -36,9 +38,10 @@ except ImportError:
 
 PLATFORM_NAME = "emudeck"
 
+EMUDECK_REPO = "dragoonDorise/EmuDeck"
 CHECKBIOS_URL = (
     "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/"
-    "main/functions/checkBIOS.sh"
+    "{rev}/functions/checkBIOS.sh"
 )
 
 CSV_BASE_URL = (
@@ -220,8 +223,13 @@ class Scraper(BaseScraper):
     """Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets."""
 
     def __init__(
-        self, checkbios_url: str = CHECKBIOS_URL, csv_base_url: str = CSV_BASE_URL
+        self, checkbios_url: str | None = None, csv_base_url: str = CSV_BASE_URL
     ) -> None:
+        # checkBIOS.sh is read from main: the commit read is recorded.
+        if checkbios_url is None:
+            checkbios_url = CHECKBIOS_URL.format(
+                rev=fetch_branch_head(EMUDECK_REPO, "main")
+            )
         super().__init__(url=checkbios_url)
         self.checkbios_url = checkbios_url
         self.csv_base_url = csv_base_url
@@ -436,7 +444,7 @@ class Scraper(BaseScraper):
             "platform": "EmuDeck",
             "version": version or "",
             "homepage": "https://www.emudeck.com",
-            "source": CHECKBIOS_URL,
+            "source": self.checkbios_url,
             "base_destination": "bios",
             "hash_type": "md5",
             "verification_mode": "md5",

scripts/scraper/libretro_scraper.py

@@ -14,6 +14,7 @@ import urllib.request
 from .base_scraper import (
     BaseScraper,
     BiosRequirement,
+    fetch_branch_head,
     fetch_github_latest_version,
     github_headers,
     requirement_entry,
@@ -22,8 +23,9 @@ from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format
 
 PLATFORM_NAME = "libretro"
 
+DATABASE_REPO = "libretro/libretro-database"
 SOURCE_URL = (
-    "https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"
+    "https://raw.githubusercontent.com/libretro/libretro-database/{rev}/dat/System.dat"
 )
 
 # Libretro cores that expect BIOS files in a subdirectory of system/.
@@ -125,8 +127,10 @@ def _core_info_files() -> list[str]:
 class Scraper(BaseScraper):
     """Scraper for libretro System.dat."""
 
-    def __init__(self, url: str = SOURCE_URL) -> None:
-        super().__init__(url=url)
+    def __init__(self, url: str | None = None) -> None:
+        # System.dat lives on master: the commit read is the one recorded.
+        self.revision = fetch_branch_head(DATABASE_REPO, "master") if url is None else ""
+        super().__init__(url=url or SOURCE_URL.format(rev=self.revision))
 
     def fetch_requirements(self) -> list[BiosRequirement]:
         """Parse System.dat and return BIOS requirements."""
@@ -504,7 +508,10 @@ class Scraper(BaseScraper):
             "version": metadata["retroarch_version"] or "",
             "dat_version": metadata["dat_version"] or "",
             "homepage": "https://www.retroarch.com",
-            "source": "https://github.com/libretro/libretro-database/blob/master/dat/System.dat",
+            "source": (
+                f"https://github.com/{DATABASE_REPO}/blob/"
+                f"{self.revision or 'master'}/dat/System.dat"
+            ),
             "base_destination": "system",
             "hash_type": "sha1",
             "verification_mode": "existence",

scripts/scraper/misterfpga_scraper.py

@@ -33,6 +33,7 @@ try:
         BaseScraper,
         BiosRequirement,
         _read_limited,
+        fetch_branch_head,
         requirement_entry,
     )
 except ImportError:
@@ -40,13 +41,15 @@ except ImportError:
         BaseScraper,
         BiosRequirement,
         _read_limited,
+        fetch_branch_head,
         requirement_entry,
     )
 
 PLATFORM_NAME = "misterfpga"
 
+BIOSDB_REPO = "ajgowans/BiosDB_MiSTer"
 SOURCE_URL = (
-    "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/db/bios_db.json.zip"
+    "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/{rev}/bios_db.json.zip"
 )
 
 DB_MEMBER = "bios_db.json"
@@ -126,7 +129,10 @@ def system_of(path: str) -> str:
 class Scraper(BaseScraper):
     """Scraper for the MiSTer BiosDB database."""
 
-    def __init__(self, url: str = SOURCE_URL) -> None:
+    def __init__(self, url: str | None = None) -> None:
+        # The database is published on the db branch: the commit read is recorded.
+        if url is None:
+            url = SOURCE_URL.format(rev=fetch_branch_head(BIOSDB_REPO, "db"))
         super().__init__(url=url)
 
     def _fetch_raw(self) -> str:
@@ -220,7 +226,7 @@ class Scraper(BaseScraper):
             "platform": "MiSTer FPGA",
             "version": released,
             "homepage": "https://misterfpga.org",
-            "source": SOURCE_URL,
+            "source": self.url,
             "base_destination": "games",
             "hash_type": "md5",
             "verification_mode": "md5",

scripts/scraper/retrobat_scraper.py

@@ -16,6 +16,7 @@ try:
     from .base_scraper import (
         BaseScraper,
         BiosRequirement,
+        fetch_branch_head,
         fetch_github_latest_version,
         requirement_entry,
     )
@@ -24,6 +25,7 @@ except ImportError:
     from base_scraper import (
         BaseScraper,
         BiosRequirement,
+        fetch_branch_head,
         fetch_github_latest_version,
         requirement_entry,
     )
@@ -31,9 +33,10 @@ except ImportError:
 
 PLATFORM_NAME = "retrobat"
 
+LAUNCHER_REPO = "RetroBat-Official/emulatorlauncher"
 SOURCE_URL = (
     "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/"
-    "master/batocera-systems/Resources/batocera-systems.json"
+    "{rev}/batocera-systems/Resources/batocera-systems.json"
 )
 
 GITHUB_REPO = "RetroBat-Official/retrobat"
@@ -103,7 +106,10 @@ SYSTEM_SLUG_MAP = {
 class Scraper(BaseScraper):
     """Scraper for RetroBat batocera-systems.json."""
 
-    def __init__(self, url: str = SOURCE_URL) -> None:
+    def __init__(self, url: str | None = None) -> None:
+        # batocera-systems.json is read from master: the commit read is recorded.
+        if url is None:
+            url = SOURCE_URL.format(rev=fetch_branch_head(LAUNCHER_REPO, "master"))
         super().__init__(url=url)
         self._parsed: dict | None = None
 
@@ -234,7 +240,7 @@ class Scraper(BaseScraper):
             "platform": "RetroBat",
             "version": version,
             "homepage": "https://www.retrobat.org",
-            "source": SOURCE_URL,
+            "source": self.url,
             "base_destination": "bios",
             "hash_type": "md5",
             "verification_mode": "md5",

scripts/scraper/retrodeck_scraper.py

@@ -36,18 +36,26 @@ import urllib.request
 from pathlib import Path
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
+    from .base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_branch_head,
+        requirement_entry,
+    )
 except ImportError:
     sys.path.insert(0, str(Path(__file__).parent.parent))
-    from scraper.base_scraper import BaseScraper, BiosRequirement, requirement_entry
+    from scraper.base_scraper import (
+        BaseScraper,
+        BiosRequirement,
+        fetch_branch_head,
+        requirement_entry,
+    )
 
 PLATFORM_NAME = "retrodeck"
 COMPONENTS_REPO = "RetroDECK/components"
 COMPONENTS_BRANCH = "main"
-COMPONENTS_API_URL = (
-    f"https://api.github.com/repos/{COMPONENTS_REPO}/git/trees/{COMPONENTS_BRANCH}"
-)
-RAW_BASE = f"https://raw.githubusercontent.com/{COMPONENTS_REPO}/{COMPONENTS_BRANCH}"
+COMPONENTS_API_URL = "https://api.github.com/repos/" + COMPONENTS_REPO + "/git/trees/{rev}"
+RAW_BASE = "https://raw.githubusercontent.com/" + COMPONENTS_REPO + "/{rev}"
 SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"}
 NON_EMULATOR_COMPONENTS = {
     "framework",
@@ -225,6 +233,11 @@ class Scraper(BaseScraper):
         super().__init__()
         self.manifests_dir = manifests_dir
         self._manifests: list[tuple[str, dict]] | None = None
+        # The manifests are read from main: one commit for the tree and for
+        # every file, and that commit is the one recorded.
+        self.revision = (
+            "" if manifests_dir else fetch_branch_head(COMPONENTS_REPO, COMPONENTS_BRANCH)
+        )
 
     def _get_manifests(self) -> list[tuple[str, dict]]:
         """Fetch manifests once, cache for reuse."""
@@ -244,7 +257,9 @@ class Scraper(BaseScraper):
             headers["Authorization"] = f"token {token}"
 
         try:
-            req = urllib.request.Request(COMPONENTS_API_URL, headers=headers)
+            req = urllib.request.Request(
+                COMPONENTS_API_URL.format(rev=self.revision), headers=headers
+            )
             with urllib.request.urlopen(req, timeout=30) as resp:
                 tree = json.loads(resp.read().decode())
         except (urllib.error.HTTPError, urllib.error.URLError) as e:
@@ -263,7 +278,7 @@ class Scraper(BaseScraper):
 
         manifests: list[tuple[str, dict]] = []
         for comp in sorted(component_dirs):
-            url = f"{RAW_BASE}/{comp}/component_manifest.json"
+            url = f"{RAW_BASE.format(rev=self.revision)}/{comp}/component_manifest.json"
             print(f"  {comp} ...", file=sys.stderr, end="", flush=True)
             # Only a 404 says the component has no manifest. A 429, a 503 or
             # a dropped connection is no answer: skipping it wrote
@@ -450,7 +465,11 @@ class Scraper(BaseScraper):
             "platform": "RetroDECK",
             "version": version,
             "homepage": "https://retrodeck.net",
-            "source": "https://github.com/RetroDECK/components",
+            "source": (
+                f"https://github.com/{COMPONENTS_REPO}/tree/{self.revision}"
+                if self.revision
+                else f"https://github.com/{COMPONENTS_REPO}"
+            ),
             "base_destination": "",
             "hash_type": "md5",
             "verification_mode": "md5",

scripts/scraper/retropie_scraper.py

@@ -26,15 +26,13 @@ import urllib.error
 import urllib.request
 
 try:
-    from .base_scraper import BaseScraper, BiosRequirement
+    from .base_scraper import BaseScraper, BiosRequirement, fetch_branch_head
 except ImportError:
-    from base_scraper import BaseScraper, BiosRequirement
+    from base_scraper import BaseScraper, BiosRequirement, fetch_branch_head
 
 PLATFORM_NAME = "retropie"
 
-SOURCE_URL = (
-    "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master"
-)
+SOURCE_URL = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/{rev}"
 GITHUB_REPO = "RetroPie/RetroPie-Setup"
 MAX_ARCHIVE = 64 * 1024 * 1024
 
@@ -51,7 +49,10 @@ _PACKAGE_DIRS = ("emulators", "libretrocores", "ports")
 class Scraper(BaseScraper):
     """Scraper for the RetroPie package list."""
 
-    def __init__(self, url: str = SOURCE_URL) -> None:
+    def __init__(self, url: str | None = None) -> None:
+        # The package list is read from master: the commit read is recorded.
+        if url is None:
+            url = SOURCE_URL.format(rev=fetch_branch_head(GITHUB_REPO, "master"))
         super().__init__(url=url)
         self._modules: list[str] | None = None
         self._version = ""
@@ -135,7 +136,7 @@ class Scraper(BaseScraper):
             "platform": "RetroPie",
             "version": self._version,
             "homepage": "https://retropie.org.uk",
-            "source": SOURCE_URL,
+            "source": self.url,
             "base_destination": "BIOS",
             "cores": cores,
         }