mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-21 11:19:11 -05:00
feat: record the commit a branch scrape read
Six platforms are read from a branch: System.dat, batocera-systems.json, checkBIOS.sh, the MiSTer BiosDB, the RetroPie scriptmodules and the RetroDECK component manifests. Their YAML named the branch, so the native export patched whatever the branch had become since the scrape. Each scraper now resolves the branch head once, reads every file at that commit and writes it into source. The export follows a recorded tarball and a recorded tree as it already followed a raw file URL.
This commit is contained in:
1 parent
eafb73bf55
commit
16f6ef8cbf
17 files changed
+203
-38
No files matched your search
@@ -1,7 +1,7 @@
|
||||
platform: EmuDeck
|
||||
version: 2.7EA
|
||||
homepage: "https://www.emudeck.com"
|
||||
source: "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/main/functions/checkBIOS.sh"
|
||||
source: "https://raw.githubusercontent.com/dragoonDorise/EmuDeck/dddb30229adbbd54d5ff7283cfd70e11c4322c26/functions/checkBIOS.sh"
|
||||
base_destination: bios
|
||||
hash_type: md5
|
||||
verification_mode: md5
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
platform: MiSTer FPGA
|
||||
version: '2026-09-29'
|
||||
homepage: "https://misterfpga.org"
|
||||
source: "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/db/bios_db.json.zip"
|
||||
source: "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/7cfa7a3e84af078d0efba74fa46c50357c4f567c/bios_db.json.zip"
|
||||
base_destination: games
|
||||
hash_type: md5
|
||||
verification_mode: md5
|
||||
|
||||
@@ -2,7 +2,7 @@ platform: RetroArch
|
||||
version: v1.22.2
|
||||
dat_version: v1.19.0
|
||||
homepage: "https://www.retroarch.com"
|
||||
source: "https://github.com/libretro/libretro-database/blob/master/dat/System.dat"
|
||||
source: "https://github.com/libretro/libretro-database/blob/12c21cf79f1e20eff6ac68b80875816c717d0a7a/dat/System.dat"
|
||||
base_destination: system
|
||||
hash_type: sha1
|
||||
verification_mode: existence
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
platform: RetroBat
|
||||
version: 8.3.1
|
||||
homepage: "https://www.retrobat.org"
|
||||
source: "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/master/batocera-systems/Resources/batocera-systems.json"
|
||||
source: "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/48cdc61623c5943bb17647b80a60d32d395c3725/batocera-systems/Resources/batocera-systems.json"
|
||||
base_destination: bios
|
||||
hash_type: md5
|
||||
verification_mode: md5
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
platform: RetroDECK
|
||||
version: 0.10.10b
|
||||
homepage: "https://retrodeck.net"
|
||||
source: "https://github.com/RetroDECK/components"
|
||||
source: "https://github.com/RetroDECK/components/tree/fcd0d72995d7aa2e741e2753ac6914025b2ff1ac"
|
||||
base_destination: ''
|
||||
hash_type: md5
|
||||
verification_mode: md5
|
||||
|
||||
@@ -2,7 +2,7 @@ inherits: retroarch
|
||||
platform: RetroPie
|
||||
version: 4.8.12
|
||||
homepage: "https://retropie.org.uk"
|
||||
source: "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master"
|
||||
source: "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/46128cd252cdecae852a4d0bafa2aaf9d03ab563"
|
||||
base_destination: BIOS
|
||||
cores:
|
||||
- "81"
|
||||
|
||||
@@ -163,12 +163,20 @@ def pinned_base(wanted: dict[str, str], scraped: dict | None) -> str:
|
||||
correct a file our data never described.
|
||||
"""
|
||||
source = _raw_url(str((scraped or {}).get("source", "")))
|
||||
tree = _GITHUB_TREE.match(source)
|
||||
if tree:
|
||||
# RetroDECK reads one manifest per component: the YAML records the
|
||||
# tree at the commit read, and every file is taken at that commit.
|
||||
return "https://raw.githubusercontent.com/{}/{}/{}/".format(*tree.groups())
|
||||
for relative in wanted:
|
||||
if source.endswith("/" + relative):
|
||||
return source[: -len(relative)]
|
||||
return ""
|
||||
|
||||
|
||||
_GITHUB_TREE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/tree/([0-9a-f]{40})/?$")
|
||||
|
||||
|
||||
def _raw_url(url: str) -> str:
|
||||
"""A GitHub blob page names a revision but serves HTML, raw serves bytes."""
|
||||
marker = "/blob/"
|
||||
@@ -196,7 +204,26 @@ def wanted_sources(
|
||||
relative: _at_revision(wanted[relative], base) or base + relative
|
||||
for relative in wanted
|
||||
}
|
||||
return wanted
|
||||
source = str((scraped or {}).get("source", ""))
|
||||
return {
|
||||
relative: _tarball_at(url, source) or url for relative, url in wanted.items()
|
||||
}
|
||||
|
||||
|
||||
_CODELOAD = re.compile(r"^https://codeload\.github\.com/([^/]+)/([^/]+)/tar\.gz/")
|
||||
|
||||
|
||||
def _tarball_at(url: str, source: str) -> str:
|
||||
"""The recorded tarball when the original is a tarball of the same repository.
|
||||
|
||||
RetroPie's declarations are spread over every scriptmodule, so its scraper
|
||||
reads the repository archive. The archive the YAML records is the one at
|
||||
the commit it was read from, the exporter's own URL names the branch.
|
||||
"""
|
||||
wanted, recorded = _CODELOAD.match(url), _CODELOAD.match(source)
|
||||
if wanted and recorded and wanted.group(1, 2) == recorded.group(1, 2):
|
||||
return source
|
||||
return ""
|
||||
|
||||
|
||||
_RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/")
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -296,6 +297,25 @@ def fetch_github_latest_version(repo: str) -> str:
|
||||
return tag
|
||||
|
||||
|
||||
def fetch_branch_head(repo: str, branch: str) -> str:
|
||||
"""The commit a branch of a GitHub repo points at right now.
|
||||
|
||||
A platform read from a moving branch records this sha in its source URL,
|
||||
so the exporter later patches the revision the data was read from and
|
||||
not whatever the branch has become. Raises like the release lookup.
|
||||
"""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{branch}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=github_headers())
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
sha = json.loads(resp.read()).get("sha", "")
|
||||
except (urllib.error.URLError, json.JSONDecodeError) as e:
|
||||
raise RuntimeError(f"cannot read the head of {repo}@{branch}: {e}") from e
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", sha or ""):
|
||||
raise RuntimeError(f"{repo}@{branch} answered no commit sha")
|
||||
return sha
|
||||
|
||||
|
||||
class _PlatformDumper(yaml.SafeDumper):
|
||||
"""Quote scalars that a YAML reader could misparse.
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ try:
|
||||
from .base_scraper import (
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
fetch_branch_head,
|
||||
fetch_github_latest_version,
|
||||
github_headers,
|
||||
requirement_entry,
|
||||
@@ -29,6 +30,7 @@ except ImportError:
|
||||
from base_scraper import (
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
fetch_branch_head,
|
||||
fetch_github_latest_version,
|
||||
github_headers,
|
||||
requirement_entry,
|
||||
@@ -36,9 +38,10 @@ except ImportError:
|
||||
|
||||
PLATFORM_NAME = "emudeck"
|
||||
|
||||
EMUDECK_REPO = "dragoonDorise/EmuDeck"
|
||||
CHECKBIOS_URL = (
|
||||
"https://raw.githubusercontent.com/dragoonDorise/EmuDeck/"
|
||||
"main/functions/checkBIOS.sh"
|
||||
"{rev}/functions/checkBIOS.sh"
|
||||
)
|
||||
|
||||
CSV_BASE_URL = (
|
||||
@@ -220,8 +223,13 @@ class Scraper(BaseScraper):
|
||||
"""Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets."""
|
||||
|
||||
def __init__(
|
||||
self, checkbios_url: str = CHECKBIOS_URL, csv_base_url: str = CSV_BASE_URL
|
||||
self, checkbios_url: str | None = None, csv_base_url: str = CSV_BASE_URL
|
||||
) -> None:
|
||||
# checkBIOS.sh is read from main: the commit read is recorded.
|
||||
if checkbios_url is None:
|
||||
checkbios_url = CHECKBIOS_URL.format(
|
||||
rev=fetch_branch_head(EMUDECK_REPO, "main")
|
||||
)
|
||||
super().__init__(url=checkbios_url)
|
||||
self.checkbios_url = checkbios_url
|
||||
self.csv_base_url = csv_base_url
|
||||
@@ -436,7 +444,7 @@ class Scraper(BaseScraper):
|
||||
"platform": "EmuDeck",
|
||||
"version": version or "",
|
||||
"homepage": "https://www.emudeck.com",
|
||||
"source": CHECKBIOS_URL,
|
||||
"source": self.checkbios_url,
|
||||
"base_destination": "bios",
|
||||
"hash_type": "md5",
|
||||
"verification_mode": "md5",
|
||||
|
||||
@@ -14,6 +14,7 @@ import urllib.request
|
||||
from .base_scraper import (
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
fetch_branch_head,
|
||||
fetch_github_latest_version,
|
||||
github_headers,
|
||||
requirement_entry,
|
||||
@@ -22,8 +23,9 @@ from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format
|
||||
|
||||
PLATFORM_NAME = "libretro"
|
||||
|
||||
DATABASE_REPO = "libretro/libretro-database"
|
||||
SOURCE_URL = (
|
||||
"https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"
|
||||
"https://raw.githubusercontent.com/libretro/libretro-database/{rev}/dat/System.dat"
|
||||
)
|
||||
|
||||
# Libretro cores that expect BIOS files in a subdirectory of system/.
|
||||
@@ -125,8 +127,10 @@ def _core_info_files() -> list[str]:
|
||||
class Scraper(BaseScraper):
|
||||
"""Scraper for libretro System.dat."""
|
||||
|
||||
def __init__(self, url: str = SOURCE_URL) -> None:
|
||||
super().__init__(url=url)
|
||||
def __init__(self, url: str | None = None) -> None:
|
||||
# System.dat lives on master: the commit read is the one recorded.
|
||||
self.revision = fetch_branch_head(DATABASE_REPO, "master") if url is None else ""
|
||||
super().__init__(url=url or SOURCE_URL.format(rev=self.revision))
|
||||
|
||||
def fetch_requirements(self) -> list[BiosRequirement]:
|
||||
"""Parse System.dat and return BIOS requirements."""
|
||||
@@ -504,7 +508,10 @@ class Scraper(BaseScraper):
|
||||
"version": metadata["retroarch_version"] or "",
|
||||
"dat_version": metadata["dat_version"] or "",
|
||||
"homepage": "https://www.retroarch.com",
|
||||
"source": "https://github.com/libretro/libretro-database/blob/master/dat/System.dat",
|
||||
"source": (
|
||||
f"https://github.com/{DATABASE_REPO}/blob/"
|
||||
f"{self.revision or 'master'}/dat/System.dat"
|
||||
),
|
||||
"base_destination": "system",
|
||||
"hash_type": "sha1",
|
||||
"verification_mode": "existence",
|
||||
|
||||
@@ -33,6 +33,7 @@ try:
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
_read_limited,
|
||||
fetch_branch_head,
|
||||
requirement_entry,
|
||||
)
|
||||
except ImportError:
|
||||
@@ -40,13 +41,15 @@ except ImportError:
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
_read_limited,
|
||||
fetch_branch_head,
|
||||
requirement_entry,
|
||||
)
|
||||
|
||||
PLATFORM_NAME = "misterfpga"
|
||||
|
||||
BIOSDB_REPO = "ajgowans/BiosDB_MiSTer"
|
||||
SOURCE_URL = (
|
||||
"https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/db/bios_db.json.zip"
|
||||
"https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/{rev}/bios_db.json.zip"
|
||||
)
|
||||
|
||||
DB_MEMBER = "bios_db.json"
|
||||
@@ -126,7 +129,10 @@ def system_of(path: str) -> str:
|
||||
class Scraper(BaseScraper):
|
||||
"""Scraper for the MiSTer BiosDB database."""
|
||||
|
||||
def __init__(self, url: str = SOURCE_URL) -> None:
|
||||
def __init__(self, url: str | None = None) -> None:
|
||||
# The database is published on the db branch: the commit read is recorded.
|
||||
if url is None:
|
||||
url = SOURCE_URL.format(rev=fetch_branch_head(BIOSDB_REPO, "db"))
|
||||
super().__init__(url=url)
|
||||
|
||||
def _fetch_raw(self) -> str:
|
||||
@@ -220,7 +226,7 @@ class Scraper(BaseScraper):
|
||||
"platform": "MiSTer FPGA",
|
||||
"version": released,
|
||||
"homepage": "https://misterfpga.org",
|
||||
"source": SOURCE_URL,
|
||||
"source": self.url,
|
||||
"base_destination": "games",
|
||||
"hash_type": "md5",
|
||||
"verification_mode": "md5",
|
||||
|
||||
@@ -16,6 +16,7 @@ try:
|
||||
from .base_scraper import (
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
fetch_branch_head,
|
||||
fetch_github_latest_version,
|
||||
requirement_entry,
|
||||
)
|
||||
@@ -24,6 +25,7 @@ except ImportError:
|
||||
from base_scraper import (
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
fetch_branch_head,
|
||||
fetch_github_latest_version,
|
||||
requirement_entry,
|
||||
)
|
||||
@@ -31,9 +33,10 @@ except ImportError:
|
||||
|
||||
PLATFORM_NAME = "retrobat"
|
||||
|
||||
LAUNCHER_REPO = "RetroBat-Official/emulatorlauncher"
|
||||
SOURCE_URL = (
|
||||
"https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/"
|
||||
"master/batocera-systems/Resources/batocera-systems.json"
|
||||
"{rev}/batocera-systems/Resources/batocera-systems.json"
|
||||
)
|
||||
|
||||
GITHUB_REPO = "RetroBat-Official/retrobat"
|
||||
@@ -103,7 +106,10 @@ SYSTEM_SLUG_MAP = {
|
||||
class Scraper(BaseScraper):
|
||||
"""Scraper for RetroBat batocera-systems.json."""
|
||||
|
||||
def __init__(self, url: str = SOURCE_URL) -> None:
|
||||
def __init__(self, url: str | None = None) -> None:
|
||||
# batocera-systems.json is read from master: the commit read is recorded.
|
||||
if url is None:
|
||||
url = SOURCE_URL.format(rev=fetch_branch_head(LAUNCHER_REPO, "master"))
|
||||
super().__init__(url=url)
|
||||
self._parsed: dict | None = None
|
||||
|
||||
@@ -234,7 +240,7 @@ class Scraper(BaseScraper):
|
||||
"platform": "RetroBat",
|
||||
"version": version,
|
||||
"homepage": "https://www.retrobat.org",
|
||||
"source": SOURCE_URL,
|
||||
"source": self.url,
|
||||
"base_destination": "bios",
|
||||
"hash_type": "md5",
|
||||
"verification_mode": "md5",
|
||||
|
||||
@@ -36,18 +36,26 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
|
||||
from .base_scraper import (
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
fetch_branch_head,
|
||||
requirement_entry,
|
||||
)
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
from scraper.base_scraper import BaseScraper, BiosRequirement, requirement_entry
|
||||
from scraper.base_scraper import (
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
fetch_branch_head,
|
||||
requirement_entry,
|
||||
)
|
||||
|
||||
PLATFORM_NAME = "retrodeck"
|
||||
COMPONENTS_REPO = "RetroDECK/components"
|
||||
COMPONENTS_BRANCH = "main"
|
||||
COMPONENTS_API_URL = (
|
||||
f"https://api.github.com/repos/{COMPONENTS_REPO}/git/trees/{COMPONENTS_BRANCH}"
|
||||
)
|
||||
RAW_BASE = f"https://raw.githubusercontent.com/{COMPONENTS_REPO}/{COMPONENTS_BRANCH}"
|
||||
COMPONENTS_API_URL = "https://api.github.com/repos/" + COMPONENTS_REPO + "/git/trees/{rev}"
|
||||
RAW_BASE = "https://raw.githubusercontent.com/" + COMPONENTS_REPO + "/{rev}"
|
||||
SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"}
|
||||
NON_EMULATOR_COMPONENTS = {
|
||||
"framework",
|
||||
@@ -225,6 +233,11 @@ class Scraper(BaseScraper):
|
||||
super().__init__()
|
||||
self.manifests_dir = manifests_dir
|
||||
self._manifests: list[tuple[str, dict]] | None = None
|
||||
# The manifests are read from main: one commit for the tree and for
|
||||
# every file, and that commit is the one recorded.
|
||||
self.revision = (
|
||||
"" if manifests_dir else fetch_branch_head(COMPONENTS_REPO, COMPONENTS_BRANCH)
|
||||
)
|
||||
|
||||
def _get_manifests(self) -> list[tuple[str, dict]]:
|
||||
"""Fetch manifests once, cache for reuse."""
|
||||
@@ -244,7 +257,9 @@ class Scraper(BaseScraper):
|
||||
headers["Authorization"] = f"token {token}"
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(COMPONENTS_API_URL, headers=headers)
|
||||
req = urllib.request.Request(
|
||||
COMPONENTS_API_URL.format(rev=self.revision), headers=headers
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
tree = json.loads(resp.read().decode())
|
||||
except (urllib.error.HTTPError, urllib.error.URLError) as e:
|
||||
@@ -263,7 +278,7 @@ class Scraper(BaseScraper):
|
||||
|
||||
manifests: list[tuple[str, dict]] = []
|
||||
for comp in sorted(component_dirs):
|
||||
url = f"{RAW_BASE}/{comp}/component_manifest.json"
|
||||
url = f"{RAW_BASE.format(rev=self.revision)}/{comp}/component_manifest.json"
|
||||
print(f" {comp} ...", file=sys.stderr, end="", flush=True)
|
||||
# Only a 404 says the component has no manifest. A 429, a 503 or
|
||||
# a dropped connection is no answer: skipping it wrote
|
||||
@@ -450,7 +465,11 @@ class Scraper(BaseScraper):
|
||||
"platform": "RetroDECK",
|
||||
"version": version,
|
||||
"homepage": "https://retrodeck.net",
|
||||
"source": "https://github.com/RetroDECK/components",
|
||||
"source": (
|
||||
f"https://github.com/{COMPONENTS_REPO}/tree/{self.revision}"
|
||||
if self.revision
|
||||
else f"https://github.com/{COMPONENTS_REPO}"
|
||||
),
|
||||
"base_destination": "",
|
||||
"hash_type": "md5",
|
||||
"verification_mode": "md5",
|
||||
|
||||
@@ -26,15 +26,13 @@ import urllib.error
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
from .base_scraper import BaseScraper, BiosRequirement
|
||||
from .base_scraper import BaseScraper, BiosRequirement, fetch_branch_head
|
||||
except ImportError:
|
||||
from base_scraper import BaseScraper, BiosRequirement
|
||||
from base_scraper import BaseScraper, BiosRequirement, fetch_branch_head
|
||||
|
||||
PLATFORM_NAME = "retropie"
|
||||
|
||||
SOURCE_URL = (
|
||||
"https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master"
|
||||
)
|
||||
SOURCE_URL = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/{rev}"
|
||||
GITHUB_REPO = "RetroPie/RetroPie-Setup"
|
||||
MAX_ARCHIVE = 64 * 1024 * 1024
|
||||
|
||||
@@ -51,7 +49,10 @@ _PACKAGE_DIRS = ("emulators", "libretrocores", "ports")
|
||||
class Scraper(BaseScraper):
|
||||
"""Scraper for the RetroPie package list."""
|
||||
|
||||
def __init__(self, url: str = SOURCE_URL) -> None:
|
||||
def __init__(self, url: str | None = None) -> None:
|
||||
# The package list is read from master: the commit read is recorded.
|
||||
if url is None:
|
||||
url = SOURCE_URL.format(rev=fetch_branch_head(GITHUB_REPO, "master"))
|
||||
super().__init__(url=url)
|
||||
self._modules: list[str] | None = None
|
||||
self._version = ""
|
||||
@@ -135,7 +136,7 @@ class Scraper(BaseScraper):
|
||||
"platform": "RetroPie",
|
||||
"version": self._version,
|
||||
"homepage": "https://retropie.org.uk",
|
||||
"source": SOURCE_URL,
|
||||
"source": self.url,
|
||||
"base_destination": "BIOS",
|
||||
"cores": cores,
|
||||
}
|
||||
|
||||
+1
-1
@@ -5497,7 +5497,7 @@ struct BurnDriver BurnDrvneogeo = {
|
||||
|
||||
from scraper.misterfpga_scraper import Scraper
|
||||
|
||||
scraper = Scraper()
|
||||
scraper = Scraper(url="https://example.test/bios_db.json.zip")
|
||||
scraper._raw_data = json.dumps(self.MISTER_DB)
|
||||
return scraper
|
||||
|
||||
|
||||
@@ -34,6 +34,8 @@ class _Response(io.BytesIO):
|
||||
def _opener(failure):
|
||||
def urlopen(req, timeout=0):
|
||||
url = req.full_url
|
||||
if "/commits/" in url:
|
||||
return _Response(json.dumps({"sha": "a" * 40}).encode())
|
||||
if "api.github.com" in url:
|
||||
return _Response(json.dumps(TREE).encode())
|
||||
if "pcsx2" in url and failure is not None:
|
||||
@@ -60,6 +62,20 @@ class ManifestErrors(unittest.TestCase):
|
||||
error = urllib.error.HTTPError("u", 404, "not found", {}, None)
|
||||
self.assertEqual([name for name, _ in self.fetch(error)], ["ppsspp"])
|
||||
|
||||
def test_the_tree_and_files_are_read_at_one_commit(self):
|
||||
seen = []
|
||||
opener = _opener(None)
|
||||
|
||||
def recording(req, timeout=0):
|
||||
seen.append(req.full_url)
|
||||
return opener(req, timeout)
|
||||
|
||||
with mock.patch("urllib.request.urlopen", recording):
|
||||
Scraper()._fetch_remote_manifests()
|
||||
reads = [url for url in seen if "/commits/" not in url]
|
||||
self.assertTrue(reads)
|
||||
self.assertTrue(all("a" * 40 in url for url in reads), reads)
|
||||
|
||||
def test_a_truncated_tree_stops_the_scrape(self):
|
||||
TREE["truncated"] = True
|
||||
self.addCleanup(TREE.pop, "truncated")
|
||||
|
||||
@@ -11,6 +11,8 @@ exercised is a promise to contributors backed by nothing.
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -246,7 +248,9 @@ class RetroPiePackageList(unittest.TestCase):
|
||||
def _scraper(self, modules: dict[str, str]):
|
||||
from scraper.retropie_scraper import Scraper
|
||||
|
||||
scraper = Scraper()
|
||||
# An explicit archive URL: the tests read a local archive, never the
|
||||
# branch head.
|
||||
scraper = Scraper(url="https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/" + "0" * 40)
|
||||
scraper.module_ids(self._archive(modules))
|
||||
return scraper
|
||||
|
||||
@@ -502,6 +506,11 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase):
|
||||
"scraper.romm_scraper",
|
||||
"scraper.batocera_scraper",
|
||||
"scraper.recalbox_scraper",
|
||||
"scraper.libretro_scraper",
|
||||
"scraper.retrobat_scraper",
|
||||
"scraper.emudeck_scraper",
|
||||
"scraper.misterfpga_scraper",
|
||||
"scraper.retropie_scraper",
|
||||
):
|
||||
module = importlib.import_module(name)
|
||||
with self.subTest(scraper=name), mock.patch(
|
||||
@@ -509,6 +518,52 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase):
|
||||
), self.assertRaises(RuntimeError):
|
||||
module.Scraper()
|
||||
|
||||
def test_a_branch_read_records_its_commit(self):
|
||||
"""System.dat, batocera-systems.json, checkBIOS.sh, the BiosDB and the
|
||||
RetroPie scriptmodules are read from a branch. The YAML named the
|
||||
branch, so the exporter patched whatever it had become since."""
|
||||
sha = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
class Answer(io.BytesIO):
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
return False
|
||||
|
||||
def answer(request, timeout=None):
|
||||
return Answer(json.dumps({"sha": sha}).encode())
|
||||
|
||||
for name in (
|
||||
"scraper.libretro_scraper",
|
||||
"scraper.retrobat_scraper",
|
||||
"scraper.emudeck_scraper",
|
||||
"scraper.misterfpga_scraper",
|
||||
"scraper.retropie_scraper",
|
||||
):
|
||||
module = importlib.import_module(name)
|
||||
with self.subTest(scraper=name), mock.patch("urllib.request.urlopen", answer):
|
||||
self.assertRegex(module.Scraper().url, f"/{sha}(/|$)")
|
||||
|
||||
def test_the_export_reads_the_recorded_tarball(self):
|
||||
from export_native import _tarball_at
|
||||
|
||||
branch = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/refs/heads/master"
|
||||
pinned = "https://codeload.github.com/RetroPie/RetroPie-Setup/tar.gz/" + "a" * 40
|
||||
self.assertEqual(_tarball_at(branch, pinned), pinned)
|
||||
other = "https://codeload.github.com/Other/Repo/tar.gz/" + "b" * 40
|
||||
self.assertEqual(_tarball_at(branch, other), "")
|
||||
|
||||
def test_the_export_reads_every_component_at_the_recorded_tree(self):
|
||||
from export_native import pinned_base
|
||||
|
||||
sha = "c" * 40
|
||||
base = pinned_base(
|
||||
{"pcsx2/component_manifest.json": "unused"},
|
||||
{"source": f"https://github.com/RetroDECK/components/tree/{sha}"},
|
||||
)
|
||||
self.assertEqual(base, f"https://raw.githubusercontent.com/RetroDECK/components/{sha}/")
|
||||
|
||||
def test_a_release_lookup_raises(self):
|
||||
with mock.patch("urllib.request.urlopen", self._refuse), self.assertRaises(
|
||||
RuntimeError
|
||||
|
||||
Reference in new issue
Block a user