mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-20 18:59:07 -05:00
feat: record the commit a branch scrape read
Six platforms are read from a branch: System.dat, batocera-systems.json, checkBIOS.sh, the MiSTer BiosDB, the RetroPie scriptmodules and the RetroDECK component manifests. Their YAML named the branch, so the native export patched whatever the branch had become since the scrape. Each scraper now resolves the branch head once, reads every file at that commit and writes it into source. The export follows a recorded tarball and a recorded tree as it already followed a raw file URL.
This commit is contained in:
1 parent
eafb73bf55
commit
16f6ef8cbf
17 files changed
+203
-38
No files matched your search
@@ -163,12 +163,20 @@ def pinned_base(wanted: dict[str, str], scraped: dict | None) -> str:
|
||||
correct a file our data never described.
|
||||
"""
|
||||
source = _raw_url(str((scraped or {}).get("source", "")))
|
||||
tree = _GITHUB_TREE.match(source)
|
||||
if tree:
|
||||
# RetroDECK reads one manifest per component: the YAML records the
|
||||
# tree at the commit read, and every file is taken at that commit.
|
||||
return "https://raw.githubusercontent.com/{}/{}/{}/".format(*tree.groups())
|
||||
for relative in wanted:
|
||||
if source.endswith("/" + relative):
|
||||
return source[: -len(relative)]
|
||||
return ""
|
||||
|
||||
|
||||
_GITHUB_TREE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/tree/([0-9a-f]{40})/?$")
|
||||
|
||||
|
||||
def _raw_url(url: str) -> str:
|
||||
"""A GitHub blob page names a revision but serves HTML, raw serves bytes."""
|
||||
marker = "/blob/"
|
||||
@@ -196,7 +204,26 @@ def wanted_sources(
|
||||
relative: _at_revision(wanted[relative], base) or base + relative
|
||||
for relative in wanted
|
||||
}
|
||||
return wanted
|
||||
source = str((scraped or {}).get("source", ""))
|
||||
return {
|
||||
relative: _tarball_at(url, source) or url for relative, url in wanted.items()
|
||||
}
|
||||
|
||||
|
||||
_CODELOAD = re.compile(r"^https://codeload\.github\.com/([^/]+)/([^/]+)/tar\.gz/")
|
||||
|
||||
|
||||
def _tarball_at(url: str, source: str) -> str:
|
||||
"""The recorded tarball when the original is a tarball of the same repository.
|
||||
|
||||
RetroPie's declarations are spread over every scriptmodule, so its scraper
|
||||
reads the repository archive. The archive the YAML records is the one at
|
||||
the commit it was read from, the exporter's own URL names the branch.
|
||||
"""
|
||||
wanted, recorded = _CODELOAD.match(url), _CODELOAD.match(source)
|
||||
if wanted and recorded and wanted.group(1, 2) == recorded.group(1, 2):
|
||||
return source
|
||||
return ""
|
||||
|
||||
|
||||
_RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/")
|
||||
|
||||
Reference in new issue
Block a user