feat: record the commit a branch scrape read

Six platforms are read from a branch: System.dat, batocera-systems.json, checkBIOS.sh, the MiSTer BiosDB, the RetroPie scriptmodules and the RetroDECK component manifests. Their YAML named the branch, so the native export patched whatever the branch had become since the scrape. Each scraper now resolves the branch head once, reads every file at that commit and writes it into source. The export follows a recorded tarball and a recorded tree as it already followed a raw file URL.
This commit is contained in:
Abdessamad Derraz committed 2026-10-19 22:50:45 +02:00
1 parent eafb73bf55
commit 16f6ef8cbf
17 files changed
+203 -38

No files matched your search

+28 -1
View File
@@ -163,12 +163,20 @@ def pinned_base(wanted: dict[str, str], scraped: dict | None) -> str:
correct a file our data never described.
"""
source = _raw_url(str((scraped or {}).get("source", "")))
tree = _GITHUB_TREE.match(source)
if tree:
# RetroDECK reads one manifest per component: the YAML records the
# tree at the commit read, and every file is taken at that commit.
return "https://raw.githubusercontent.com/{}/{}/{}/".format(*tree.groups())
for relative in wanted:
if source.endswith("/" + relative):
return source[: -len(relative)]
return ""
_GITHUB_TREE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/tree/([0-9a-f]{40})/?$")
def _raw_url(url: str) -> str:
"""A GitHub blob page names a revision but serves HTML, raw serves bytes."""
marker = "/blob/"
@@ -196,7 +204,26 @@ def wanted_sources(
relative: _at_revision(wanted[relative], base) or base + relative
for relative in wanted
}
return wanted
source = str((scraped or {}).get("source", ""))
return {
relative: _tarball_at(url, source) or url for relative, url in wanted.items()
}
_CODELOAD = re.compile(r"^https://codeload\.github\.com/([^/]+)/([^/]+)/tar\.gz/")
def _tarball_at(url: str, source: str) -> str:
"""The recorded tarball when the original is a tarball of the same repository.
RetroPie's declarations are spread over every scriptmodule, so its scraper
reads the repository archive. The archive the YAML records is the one at
the commit it was read from, the exporter's own URL names the branch.
"""
wanted, recorded = _CODELOAD.match(url), _CODELOAD.match(source)
if wanted and recorded and wanted.group(1, 2) == recorded.group(1, 2):
return source
return ""
_RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/")