feat: record the commit a branch scrape read

Six platforms are read from a branch: System.dat, batocera-systems.json, checkBIOS.sh, the MiSTer BiosDB, the RetroPie scriptmodules and the RetroDECK component manifests. Their YAML named the branch, so the native export patched whatever the branch had become since the scrape. Each scraper now resolves the branch head once, reads every file at that commit and writes it into source. The export follows a recorded tarball and a recorded tree as it already followed a raw file URL.
This commit is contained in:
Abdessamad Derraz committed 2026-10-19 22:50:45 +02:00
1 parent eafb73bf55
commit 16f6ef8cbf
17 files changed
+203 -38

No files matched your search

+1 -1
View File
@@ -1,7 +1,7 @@
platform: EmuDeck
version: 2.7EA
homepage: "https://www.emudeck.com"
source: "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/main/functions/checkBIOS.sh"
source: "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/dddb30229adbbd54d5ff7283cfd70e11c4322c26/functions/checkBIOS.sh"
base_destination: bios
hash_type: md5
verification_mode: md5
+1 -1
View File
@@ -1,7 +1,7 @@
platform: MiSTer FPGA
version: '2026-09-29'
homepage: "https://misterfpga.org"
source: "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/db/bios_db.json.zip"
source: "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/7cfa7a3e84af078d0efba74fa46c50357c4f567c/bios_db.json.zip"
base_destination: games
hash_type: md5
verification_mode: md5
+1 -1
View File
@@ -2,7 +2,7 @@ platform: RetroArch
version: v1.22.2
dat_version: v1.19.0
homepage: "https://www.retroarch.com"
source: "https://github.com/libretro/libretro-database/blob/master/dat/System.dat"
source: "https://github.com/libretro/libretro-database/blob/12c21cf79f1e20eff6ac68b80875816c717d0a7a/dat/System.dat"
base_destination: system
hash_type: sha1
verification_mode: existence
+1 -1
View File
@@ -1,7 +1,7 @@
platform: RetroBat
version: 8.3.1
homepage: "https://www.retrobat.org"
source: "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/master/batocera-systems/Resources/batocera-systems.json"
source: "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/48cdc61623c5943bb17647b80a60d32d395c3725/batocera-systems/Resources/batocera-systems.json"
base_destination: bios
hash_type: md5
verification_mode: md5
+1 -1
View File
@@ -1,7 +1,7 @@
platform: RetroDECK
version: 0.10.10b
homepage: "https://retrodeck.net"
source: "https://github.com/RetroDECK/components"
source: "https://github.com/RetroDECK/components/tree/fcd0d72995d7aa2e741e2753ac6914025b2ff1ac"
base_destination: ''
hash_type: md5
verification_mode: md5
+1 -1
View File
@@ -2,7 +2,7 @@ inherits: retroarch
platform: RetroPie
version: 4.8.12
homepage: "https://retropie.org.uk"
source: "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master"
source: "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/46128cd252cdecae852a4d0bafa2aaf9d03ab563"
base_destination: BIOS
cores:
- "81"
+28 -1
View File
@@ -163,12 +163,20 @@ def pinned_base(wanted: dict[str, str], scraped: dict | None) -> str:
correct a file our data never described.
"""
source = _raw_url(str((scraped or {}).get("source", "")))
tree = _GITHUB_TREE.match(source)
if tree:
# RetroDECK reads one manifest per component: the YAML records the
# tree at the commit read, and every file is taken at that commit.
return "https://raw.githubusercontent.com/{}/{}/{}/".format(*tree.groups())
for relative in wanted:
if source.endswith("/" + relative):
return source[: -len(relative)]
return ""
_GITHUB_TREE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/tree/([0-9a-f]{40})/?$")
def _raw_url(url: str) -> str:
"""A GitHub blob page names a revision but serves HTML, raw serves bytes."""
marker = "/blob/"
@@ -196,7 +204,26 @@ def wanted_sources(
relative: _at_revision(wanted[relative], base) or base + relative
for relative in wanted
}
return wanted
source = str((scraped or {}).get("source", ""))
return {
relative: _tarball_at(url, source) or url for relative, url in wanted.items()
}
_CODELOAD = re.compile(r"^https://codeload\.github\.com/([^/]+)/([^/]+)/tar\.gz/")
def _tarball_at(url: str, source: str) -> str:
"""The recorded tarball when the original is a tarball of the same repository.
RetroPie's declarations are spread over every scriptmodule, so its scraper
reads the repository archive. The archive the YAML records is the one at
the commit it was read from, the exporter's own URL names the branch.
"""
wanted, recorded = _CODELOAD.match(url), _CODELOAD.match(source)
if wanted and recorded and wanted.group(1, 2) == recorded.group(1, 2):
return source
return ""
_RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/")
+20
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
import os
import re
import sys
import time
import urllib.error
@@ -296,6 +297,25 @@ def fetch_github_latest_version(repo: str) -> str:
return tag
def fetch_branch_head(repo: str, branch: str) -> str:
"""The commit a branch of a GitHub repo points at right now.
A platform read from a moving branch records this sha in its source URL,
so the exporter later patches the revision the data was read from and
not whatever the branch has become. Raises like the release lookup.
"""
url = f"https://api.github.com/repos/{repo}/commits/{branch}"
try:
req = urllib.request.Request(url, headers=github_headers())
with urllib.request.urlopen(req, timeout=15) as resp:
sha = json.loads(resp.read()).get("sha", "")
except (urllib.error.URLError, json.JSONDecodeError) as e:
raise RuntimeError(f"cannot read the head of {repo}@{branch}: {e}") from e
if not re.fullmatch(r"[0-9a-f]{40}", sha or ""):
raise RuntimeError(f"{repo}@{branch} answered no commit sha")
return sha
class _PlatformDumper(yaml.SafeDumper):
"""Quote scalars that a YAML reader could misparse.
+11 -3
View File
@@ -21,6 +21,7 @@ try:
from .base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
fetch_github_latest_version,
github_headers,
requirement_entry,
@@ -29,6 +30,7 @@ except ImportError:
from base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
fetch_github_latest_version,
github_headers,
requirement_entry,
@@ -36,9 +38,10 @@ except ImportError:
PLATFORM_NAME = "emudeck"
EMUDECK_REPO = "dragoonDorise/EmuDeck"
CHECKBIOS_URL = (
"https://raw.githubusercontent.com/dragoonDorise/EmuDeck/"
"main/functions/checkBIOS.sh"
"{rev}/functions/checkBIOS.sh"
)
CSV_BASE_URL = (
@@ -220,8 +223,13 @@ class Scraper(BaseScraper):
"""Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets."""
def __init__(
self, checkbios_url: str = CHECKBIOS_URL, csv_base_url: str = CSV_BASE_URL
self, checkbios_url: str | None = None, csv_base_url: str = CSV_BASE_URL
) -> None:
# checkBIOS.sh is read from main: the commit read is recorded.
if checkbios_url is None:
checkbios_url = CHECKBIOS_URL.format(
rev=fetch_branch_head(EMUDECK_REPO, "main")
)
super().__init__(url=checkbios_url)
self.checkbios_url = checkbios_url
self.csv_base_url = csv_base_url
@@ -436,7 +444,7 @@ class Scraper(BaseScraper):
"platform": "EmuDeck",
"version": version or "",
"homepage": "https://www.emudeck.com",
"source": CHECKBIOS_URL,
"source": self.checkbios_url,
"base_destination": "bios",
"hash_type": "md5",
"verification_mode": "md5",
+11 -4
View File
@@ -14,6 +14,7 @@ import urllib.request
from .base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
fetch_github_latest_version,
github_headers,
requirement_entry,
@@ -22,8 +23,9 @@ from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format
PLATFORM_NAME = "libretro"
DATABASE_REPO = "libretro/libretro-database"
SOURCE_URL = (
"https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"
"https://raw.githubusercontent.com/libretro/libretro-database/{rev}/dat/System.dat"
)
# Libretro cores that expect BIOS files in a subdirectory of system/.
@@ -125,8 +127,10 @@ def _core_info_files() -> list[str]:
class Scraper(BaseScraper):
"""Scraper for libretro System.dat."""
def __init__(self, url: str = SOURCE_URL) -> None:
super().__init__(url=url)
def __init__(self, url: str | None = None) -> None:
# System.dat lives on master: the commit read is the one recorded.
self.revision = fetch_branch_head(DATABASE_REPO, "master") if url is None else ""
super().__init__(url=url or SOURCE_URL.format(rev=self.revision))
def fetch_requirements(self) -> list[BiosRequirement]:
"""Parse System.dat and return BIOS requirements."""
@@ -504,7 +508,10 @@ class Scraper(BaseScraper):
"version": metadata["retroarch_version"] or "",
"dat_version": metadata["dat_version"] or "",
"homepage": "https://www.retroarch.com",
"source": "https://github.com/libretro/libretro-database/blob/master/dat/System.dat",
"source": (
f"https://github.com/{DATABASE_REPO}/blob/"
f"{self.revision or 'master'}/dat/System.dat"
),
"base_destination": "system",
"hash_type": "sha1",
"verification_mode": "existence",
+9 -3
View File
@@ -33,6 +33,7 @@ try:
BaseScraper,
BiosRequirement,
_read_limited,
fetch_branch_head,
requirement_entry,
)
except ImportError:
@@ -40,13 +41,15 @@ except ImportError:
BaseScraper,
BiosRequirement,
_read_limited,
fetch_branch_head,
requirement_entry,
)
PLATFORM_NAME = "misterfpga"
BIOSDB_REPO = "ajgowans/BiosDB_MiSTer"
SOURCE_URL = (
"https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/db/bios_db.json.zip"
"https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/{rev}/bios_db.json.zip"
)
DB_MEMBER = "bios_db.json"
@@ -126,7 +129,10 @@ def system_of(path: str) -> str:
class Scraper(BaseScraper):
"""Scraper for the MiSTer BiosDB database."""
def __init__(self, url: str = SOURCE_URL) -> None:
def __init__(self, url: str | None = None) -> None:
# The database is published on the db branch: the commit read is recorded.
if url is None:
url = SOURCE_URL.format(rev=fetch_branch_head(BIOSDB_REPO, "db"))
super().__init__(url=url)
def _fetch_raw(self) -> str:
@@ -220,7 +226,7 @@ class Scraper(BaseScraper):
"platform": "MiSTer FPGA",
"version": released,
"homepage": "https://misterfpga.org",
"source": SOURCE_URL,
"source": self.url,
"base_destination": "games",
"hash_type": "md5",
"verification_mode": "md5",
+9 -3
View File
@@ -16,6 +16,7 @@ try:
from .base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
fetch_github_latest_version,
requirement_entry,
)
@@ -24,6 +25,7 @@ except ImportError:
from base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
fetch_github_latest_version,
requirement_entry,
)
@@ -31,9 +33,10 @@ except ImportError:
PLATFORM_NAME = "retrobat"
LAUNCHER_REPO = "RetroBat-Official/emulatorlauncher"
SOURCE_URL = (
"https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/"
"master/batocera-systems/Resources/batocera-systems.json"
"{rev}/batocera-systems/Resources/batocera-systems.json"
)
GITHUB_REPO = "RetroBat-Official/retrobat"
@@ -103,7 +106,10 @@ SYSTEM_SLUG_MAP = {
class Scraper(BaseScraper):
"""Scraper for RetroBat batocera-systems.json."""
def __init__(self, url: str = SOURCE_URL) -> None:
def __init__(self, url: str | None = None) -> None:
# batocera-systems.json is read from master: the commit read is recorded.
if url is None:
url = SOURCE_URL.format(rev=fetch_branch_head(LAUNCHER_REPO, "master"))
super().__init__(url=url)
self._parsed: dict | None = None
@@ -234,7 +240,7 @@ class Scraper(BaseScraper):
"platform": "RetroBat",
"version": version,
"homepage": "https://www.retrobat.org",
"source": SOURCE_URL,
"source": self.url,
"base_destination": "bios",
"hash_type": "md5",
"verification_mode": "md5",
+28 -9
View File
@@ -36,18 +36,26 @@ import urllib.request
from pathlib import Path
try:
from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
from .base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
requirement_entry,
)
except ImportError:
sys.path.insert(0, str(Path(__file__).parent.parent))
from scraper.base_scraper import BaseScraper, BiosRequirement, requirement_entry
from scraper.base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
requirement_entry,
)
PLATFORM_NAME = "retrodeck"
COMPONENTS_REPO = "RetroDECK/components"
COMPONENTS_BRANCH = "main"
COMPONENTS_API_URL = (
f"https://api.github.com/repos/{COMPONENTS_REPO}/git/trees/{COMPONENTS_BRANCH}"
)
RAW_BASE = f"https://raw.githubusercontent.com/{COMPONENTS_REPO}/{COMPONENTS_BRANCH}"
COMPONENTS_API_URL = "https://api.github.com/repos/" + COMPONENTS_REPO + "/git/trees/{rev}"
RAW_BASE = "https://raw.githubusercontent.com/" + COMPONENTS_REPO + "/{rev}"
SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"}
NON_EMULATOR_COMPONENTS = {
"framework",
@@ -225,6 +233,11 @@ class Scraper(BaseScraper):
super().__init__()
self.manifests_dir = manifests_dir
self._manifests: list[tuple[str, dict]] | None = None
# The manifests are read from main: one commit for the tree and for
# every file, and that commit is the one recorded.
self.revision = (
"" if manifests_dir else fetch_branch_head(COMPONENTS_REPO, COMPONENTS_BRANCH)
)
def _get_manifests(self) -> list[tuple[str, dict]]:
"""Fetch manifests once, cache for reuse."""
@@ -244,7 +257,9 @@ class Scraper(BaseScraper):
headers["Authorization"] = f"token {token}"
try:
req = urllib.request.Request(COMPONENTS_API_URL, headers=headers)
req = urllib.request.Request(
COMPONENTS_API_URL.format(rev=self.revision), headers=headers
)
with urllib.request.urlopen(req, timeout=30) as resp:
tree = json.loads(resp.read().decode())
except (urllib.error.HTTPError, urllib.error.URLError) as e:
@@ -263,7 +278,7 @@ class Scraper(BaseScraper):
manifests: list[tuple[str, dict]] = []
for comp in sorted(component_dirs):
url = f"{RAW_BASE}/{comp}/component_manifest.json"
url = f"{RAW_BASE.format(rev=self.revision)}/{comp}/component_manifest.json"
print(f" {comp} ...", file=sys.stderr, end="", flush=True)
# Only a 404 says the component has no manifest. A 429, a 503 or
# a dropped connection is no answer: skipping it wrote
@@ -450,7 +465,11 @@ class Scraper(BaseScraper):
"platform": "RetroDECK",
"version": version,
"homepage": "https://retrodeck.net",
"source": "https://github.com/RetroDECK/components",
"source": (
f"https://github.com/{COMPONENTS_REPO}/tree/{self.revision}"
if self.revision
else f"https://github.com/{COMPONENTS_REPO}"
),
"base_destination": "",
"hash_type": "md5",
"verification_mode": "md5",
+8 -7
View File
@@ -26,15 +26,13 @@ import urllib.error
import urllib.request
try:
from .base_scraper import BaseScraper, BiosRequirement
from .base_scraper import BaseScraper, BiosRequirement, fetch_branch_head
except ImportError:
from base_scraper import BaseScraper, BiosRequirement
from base_scraper import BaseScraper, BiosRequirement, fetch_branch_head
PLATFORM_NAME = "retropie"
SOURCE_URL = (
"https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master"
)
SOURCE_URL = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/{rev}"
GITHUB_REPO = "RetroPie/RetroPie-Setup"
MAX_ARCHIVE = 64 * 1024 * 1024
@@ -51,7 +49,10 @@ _PACKAGE_DIRS = ("emulators", "libretrocores", "ports")
class Scraper(BaseScraper):
"""Scraper for the RetroPie package list."""
def __init__(self, url: str = SOURCE_URL) -> None:
def __init__(self, url: str | None = None) -> None:
# The package list is read from master: the commit read is recorded.
if url is None:
url = SOURCE_URL.format(rev=fetch_branch_head(GITHUB_REPO, "master"))
super().__init__(url=url)
self._modules: list[str] | None = None
self._version = ""
@@ -135,7 +136,7 @@ class Scraper(BaseScraper):
"platform": "RetroPie",
"version": self._version,
"homepage": "https://retropie.org.uk",
"source": SOURCE_URL,
"source": self.url,
"base_destination": "BIOS",
"cores": cores,
}
+1 -1
View File
@@ -5497,7 +5497,7 @@ struct BurnDriver BurnDrvneogeo = {
from scraper.misterfpga_scraper import Scraper
scraper = Scraper()
scraper = Scraper(url="https://example.test/bios_db.json.zip")
scraper._raw_data = json.dumps(self.MISTER_DB)
return scraper
+16
View File
@@ -34,6 +34,8 @@ class _Response(io.BytesIO):
def _opener(failure):
def urlopen(req, timeout=0):
url = req.full_url
if "/commits/" in url:
return _Response(json.dumps({"sha": "a" * 40}).encode())
if "api.github.com" in url:
return _Response(json.dumps(TREE).encode())
if "pcsx2" in url and failure is not None:
@@ -60,6 +62,20 @@ class ManifestErrors(unittest.TestCase):
error = urllib.error.HTTPError("u", 404, "not found", {}, None)
self.assertEqual([name for name, _ in self.fetch(error)], ["ppsspp"])
def test_the_tree_and_files_are_read_at_one_commit(self):
seen = []
opener = _opener(None)
def recording(req, timeout=0):
seen.append(req.full_url)
return opener(req, timeout)
with mock.patch("urllib.request.urlopen", recording):
Scraper()._fetch_remote_manifests()
reads = [url for url in seen if "/commits/" not in url]
self.assertTrue(reads)
self.assertTrue(all("a" * 40 in url for url in reads), reads)
def test_a_truncated_tree_stops_the_scrape(self):
TREE["truncated"] = True
self.addCleanup(TREE.pop, "truncated")
+56 -1
View File
@@ -11,6 +11,8 @@ exercised is a promise to contributors backed by nothing.
from __future__ import annotations
import importlib
import io
import json
import subprocess
import sys
import tempfile
@@ -246,7 +248,9 @@ class RetroPiePackageList(unittest.TestCase):
def _scraper(self, modules: dict[str, str]):
from scraper.retropie_scraper import Scraper
scraper = Scraper()
# An explicit archive URL: the tests read a local archive, never the
# branch head.
scraper = Scraper(url="https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/" + "0" * 40)
scraper.module_ids(self._archive(modules))
return scraper
@@ -502,6 +506,11 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase):
"scraper.romm_scraper",
"scraper.batocera_scraper",
"scraper.recalbox_scraper",
"scraper.libretro_scraper",
"scraper.retrobat_scraper",
"scraper.emudeck_scraper",
"scraper.misterfpga_scraper",
"scraper.retropie_scraper",
):
module = importlib.import_module(name)
with self.subTest(scraper=name), mock.patch(
@@ -509,6 +518,52 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase):
), self.assertRaises(RuntimeError):
module.Scraper()
def test_a_branch_read_records_its_commit(self):
"""System.dat, batocera-systems.json, checkBIOS.sh, the BiosDB and the
RetroPie scriptmodules are read from a branch. The YAML named the
branch, so the exporter patched whatever it had become since."""
sha = "0123456789abcdef0123456789abcdef01234567"
class Answer(io.BytesIO):
def __enter__(self):
return self
def __exit__(self, *_exc):
return False
def answer(request, timeout=None):
return Answer(json.dumps({"sha": sha}).encode())
for name in (
"scraper.libretro_scraper",
"scraper.retrobat_scraper",
"scraper.emudeck_scraper",
"scraper.misterfpga_scraper",
"scraper.retropie_scraper",
):
module = importlib.import_module(name)
with self.subTest(scraper=name), mock.patch("urllib.request.urlopen", answer):
self.assertRegex(module.Scraper().url, f"/{sha}(/|$)")
def test_the_export_reads_the_recorded_tarball(self):
from export_native import _tarball_at
branch = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master"
pinned = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/" + "a" * 40
self.assertEqual(_tarball_at(branch, pinned), pinned)
other = "https://codeload.github.com/Other/Repo/tar.gz/" + "b" * 40
self.assertEqual(_tarball_at(branch, other), "")
def test_the_export_reads_every_component_at_the_recorded_tree(self):
from export_native import pinned_base
sha = "c" * 40
base = pinned_base(
{"pcsx2/component_manifest.json": "unused"},
{"source": f"https://github.com/RetroDECK/components/tree/{sha}"},
)
self.assertEqual(base, f"https://raw.githubusercontent.com/RetroDECK/components/{sha}/")
def test_a_release_lookup_raises(self):
with mock.patch("urllib.request.urlopen", self._refuse), self.assertRaises(
RuntimeError