mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-17 17:03:27 -05:00
Abdessamad Derraz
6280f4c370
es_bios.xml names only the cores that need a BIOS, so a hand-kept registry list stood in, with mame and mamemess that Recalbox does not ship and without vice, ymir, vircon32 and others it does. The scraper now reads each system.ini of recalbox-romfs2 at the scraped tag, and the registry list is gone.
318 lines
11 KiB
Python
318 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Scraper for Recalbox BIOS requirements.
|
|
|
|
Source: https://gitlab.com/recalbox/recalbox/-/raw/master/board/recalbox/fsoverlay/recalbox/share_init/system/.emulationstation/es_bios.xml
|
|
Format: XML (es_bios.xml)
|
|
Hash: MD5 (multiple valid hashes per entry, comma-separated)
|
|
|
|
Recalbox verification logic:
|
|
- Checks MD5 of file on disk against list of valid hashes
|
|
- Multiple MD5s accepted per BIOS (different ROM revisions)
|
|
- Alternate file paths (pipe-separated)
|
|
- hashMatchMandatory flag: if false, wrong hash = warning (YELLOW) not error (RED)
|
|
- ZIP files get composite MD5 calculation
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from common import parse_untrusted_xml
|
|
|
|
from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
|
|
|
|
PLATFORM_NAME = "recalbox"
|
|
|
|
def _fetch_gitlab_stable_tag() -> str:
|
|
"""Fetch the latest stable x.y.z tag from the Recalbox GitLab."""
|
|
import json
|
|
import re
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
url = "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tags?per_page=50"
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
tags = json.loads(resp.read())
|
|
except (urllib.error.URLError, json.JSONDecodeError) as e:
|
|
raise RuntimeError(f"cannot list Recalbox tags: {e}") from e
|
|
stable = [t["name"] for t in tags if re.fullmatch(r"[0-9]+\.[0-9]+(\.[0-9]+)?", t["name"])]
|
|
if not stable:
|
|
raise RuntimeError("no stable Recalbox tag among the latest 50")
|
|
return stable[0]
|
|
|
|
|
|
SOURCE_URL = (
|
|
"https://gitlab.com/recalbox/recalbox/-/raw/{tag}/"
|
|
"board/recalbox/fsoverlay/recalbox/share_init/system/"
|
|
".emulationstation/es_bios.xml"
|
|
)
|
|
|
|
SYSTEM_SLUG_MAP = {
|
|
"3do": "3do",
|
|
"amiga600": "commodore-amiga",
|
|
"amiga1200": "commodore-amiga",
|
|
"amigacd32": "commodore-amiga",
|
|
"amigacdtv": "commodore-amiga",
|
|
"amstradcpc": "amstrad-cpc",
|
|
"atari800": "atari-400-800",
|
|
"atari5200": "atari-5200",
|
|
"atari7800": "atari-7800",
|
|
"atarilynx": "atari-lynx",
|
|
"atarist": "atari-st",
|
|
"c64": "commodore-c64",
|
|
"channelf": "fairchild-channel-f",
|
|
"colecovision": "coleco-colecovision",
|
|
"dreamcast": "sega-dreamcast",
|
|
"fds": "nintendo-fds",
|
|
"gamecube": "nintendo-gamecube",
|
|
"gamegear": "sega-game-gear",
|
|
"gb": "nintendo-gb",
|
|
"gba": "nintendo-gba",
|
|
"gbc": "nintendo-gbc",
|
|
"intellivision": "mattel-intellivision",
|
|
"jaguar": "atari-jaguar",
|
|
"mastersystem": "sega-master-system",
|
|
"megadrive": "sega-mega-drive",
|
|
"msx": "microsoft-msx",
|
|
"msx1": "microsoft-msx",
|
|
"msx2": "microsoft-msx",
|
|
"n64": "nintendo-64",
|
|
"naomi": "sega-dreamcast-arcade",
|
|
"naomigd": "sega-dreamcast-arcade",
|
|
"atomiswave": "sega-dreamcast-arcade",
|
|
"nds": "nintendo-ds",
|
|
"neogeo": "snk-neogeo",
|
|
"neogeocd": "snk-neogeo-cd",
|
|
"o2em": "magnavox-odyssey2",
|
|
"pcengine": "nec-pc-engine",
|
|
"pcenginecd": "nec-pc-engine",
|
|
"pcfx": "nec-pc-fx",
|
|
"ps2": "sony-playstation-2",
|
|
"psx": "sony-playstation",
|
|
"saturn": "sega-saturn",
|
|
"scummvm": "scummvm",
|
|
"segacd": "sega-mega-cd",
|
|
"snes": "nintendo-snes",
|
|
"supergrafx": "nec-pc-engine",
|
|
"x68000": "sharp-x68000",
|
|
"zxspectrum": "sinclair-zx-spectrum",
|
|
}
|
|
|
|
|
|
SYSTEMS_TREE_URL = (
|
|
"https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tree"
|
|
"?path=package/recalbox-romfs2/systems&ref={tag}&per_page=100&page={page}"
|
|
)
|
|
SYSTEM_INI_URL = (
|
|
"https://gitlab.com/recalbox/recalbox/-/raw/{tag}"
|
|
"/package/recalbox-romfs2/systems/{system}/system.ini"
|
|
)
|
|
|
|
|
|
def system_ini_cores(text: str) -> tuple[set[str], set[str]]:
|
|
"""The cores one Recalbox system.ini offers, and those run standalone.
|
|
|
|
Each ``[core.N]`` section (or its board override ``[core.N.<board>]``)
|
|
names an ``emulator`` and a ``core``: ``libretro`` with the core's name,
|
|
or a standalone emulator. es_bios.xml only names the cores that need a
|
|
BIOS, so read alone it left most of Recalbox's cores outside the pack.
|
|
"""
|
|
import re
|
|
|
|
# Recalbox's .ini files are not strict INI (a value can run on a bare
|
|
# line), so only sections and key = value lines are read.
|
|
sections: dict[str, dict[str, str]] = {}
|
|
current: dict[str, str] | None = None
|
|
for raw in text.splitlines():
|
|
line = raw.strip()
|
|
if not line or line.startswith((";", "#")):
|
|
continue
|
|
header = re.fullmatch(r"\[([^\]]+)\]", line)
|
|
if header:
|
|
current = sections.setdefault(header.group(1).strip(), {})
|
|
continue
|
|
if current is not None and "=" in line:
|
|
key, value = line.split("=", 1)
|
|
current[key.strip()] = value.strip().strip('"')
|
|
cores: set[str] = set()
|
|
standalone: set[str] = set()
|
|
for section, values in sections.items():
|
|
if not re.fullmatch(r"core\.\d+(\.[\w-]+)?", section):
|
|
continue
|
|
emulator = values.get("emulator", "")
|
|
core = values.get("core", "")
|
|
if emulator == "libretro":
|
|
if core:
|
|
cores.add(core)
|
|
elif emulator:
|
|
cores.add(emulator)
|
|
standalone.add(emulator)
|
|
if core and core != emulator:
|
|
cores.add(core)
|
|
standalone.add(core)
|
|
return cores, standalone
|
|
|
|
|
|
class Scraper(BaseScraper):
|
|
"""Scraper for Recalbox es_bios.xml."""
|
|
|
|
def __init__(self):
|
|
self.tag = _fetch_gitlab_stable_tag()
|
|
super().__init__(url=SOURCE_URL.format(tag=self.tag))
|
|
|
|
def _fetch_cores(self) -> tuple[list[str], list[str]]:
|
|
"""Every core Recalbox's systems offer, and those it runs standalone.
|
|
|
|
Read from each system.ini of recalbox-romfs2 at the scraped tag: the
|
|
systems ES lists and the cores a user can pick for each.
|
|
"""
|
|
import json
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
def get(url: str) -> bytes:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return resp.read()
|
|
except urllib.error.URLError as e:
|
|
raise ConnectionError(f"Failed to fetch {url}: {e}") from e
|
|
|
|
systems: list[str] = []
|
|
page = 1
|
|
while True:
|
|
listing = json.loads(get(SYSTEMS_TREE_URL.format(tag=self.tag, page=page)))
|
|
systems += [
|
|
item["name"] for item in listing
|
|
if item.get("type") == "tree" and not item["name"].startswith(".")
|
|
]
|
|
if len(listing) < 100:
|
|
break
|
|
page += 1
|
|
if not systems:
|
|
raise RuntimeError(f"no Recalbox system found at {self.tag}")
|
|
cores: set[str] = set()
|
|
standalone: set[str] = set()
|
|
for system in systems:
|
|
text = get(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8")
|
|
found, alone = system_ini_cores(text)
|
|
cores |= found
|
|
standalone |= alone
|
|
if not cores:
|
|
raise RuntimeError(f"no core in Recalbox's system.ini files at {self.tag}")
|
|
return sorted(cores), sorted(standalone)
|
|
|
|
def fetch_requirements(self) -> list[BiosRequirement]:
|
|
"""Parse es_bios.xml and return BIOS requirements."""
|
|
raw = self._fetch_raw()
|
|
|
|
if not self.validate_format(raw):
|
|
raise ValueError("es_bios.xml format validation failed")
|
|
|
|
root = parse_untrusted_xml(raw, "es_bios.xml")
|
|
requirements = []
|
|
seen = set()
|
|
|
|
for system_elem in root.findall(".//system"):
|
|
platform = system_elem.get("platform", "")
|
|
system_slug = SYSTEM_SLUG_MAP.get(platform, platform)
|
|
fullname = system_elem.get("fullname", "")
|
|
|
|
for bios_elem in system_elem.findall("bios"):
|
|
paths_str = bios_elem.get("path", "")
|
|
md5_str = bios_elem.get("md5", "")
|
|
mandatory = bios_elem.get("mandatory", "true") != "false"
|
|
|
|
paths = [p.strip() for p in paths_str.split("|") if p.strip()]
|
|
if not paths:
|
|
continue
|
|
|
|
primary_path = paths[0]
|
|
name = (
|
|
primary_path.split("/")[-1] if "/" in primary_path else primary_path
|
|
)
|
|
|
|
md5_list = [m.strip() for m in md5_str.split(",") if m.strip()]
|
|
all_md5 = ",".join(md5_list) if md5_list else None
|
|
|
|
dedup_key = (platform, primary_path)
|
|
if dedup_key in seen:
|
|
continue
|
|
seen.add(dedup_key)
|
|
|
|
native: dict[str, object] = {}
|
|
if fullname:
|
|
native["native_name"] = fullname
|
|
core = bios_elem.get("core", "").strip()
|
|
if core:
|
|
native["core"] = core
|
|
note = bios_elem.get("note", "").strip()
|
|
if note:
|
|
native["note"] = note
|
|
# Recalbox reads a missing attribute as true for both flags,
|
|
# so only the explicit value carries information.
|
|
hash_match = bios_elem.get("hashMatchMandatory")
|
|
if hash_match is not None:
|
|
native["hash_match_mandatory"] = hash_match != "false"
|
|
if bios_elem.get("mandatory") is not None:
|
|
native["mandatory_declared"] = mandatory
|
|
if len(paths) > 1:
|
|
native["alt_paths"] = paths[1:]
|
|
|
|
requirements.append(
|
|
BiosRequirement(
|
|
name=name,
|
|
system=system_slug,
|
|
md5=all_md5,
|
|
destination=primary_path,
|
|
required=mandatory,
|
|
native_id=platform,
|
|
native=native,
|
|
)
|
|
)
|
|
|
|
return requirements
|
|
|
|
def validate_format(self, raw_data: str) -> bool:
|
|
"""Validate es_bios.xml format."""
|
|
return "<biosList" in raw_data and "<system" in raw_data and "<bios" in raw_data
|
|
|
|
def generate_platform_yaml(self) -> dict:
|
|
"""Generate a platform YAML config dict from scraped data."""
|
|
requirements = self.fetch_requirements()
|
|
|
|
systems: dict[str, dict] = {}
|
|
for req in requirements:
|
|
if req.system not in systems:
|
|
sys_entry: dict = {"files": []}
|
|
if req.native_id:
|
|
sys_entry["native_id"] = req.native_id
|
|
systems[req.system] = sys_entry
|
|
|
|
systems[req.system]["files"].append(requirement_entry(req))
|
|
|
|
cores, standalone = self._fetch_cores()
|
|
return {
|
|
"platform": "Recalbox",
|
|
"version": self.tag,
|
|
"homepage": "https://www.recalbox.com",
|
|
"source": self.url,
|
|
"base_destination": "bios",
|
|
"hash_type": "md5",
|
|
"verification_mode": "md5",
|
|
"cores": cores,
|
|
"standalone_cores": standalone,
|
|
"systems": systems,
|
|
}
|
|
|
|
|
|
def main():
|
|
"""CLI entry point."""
|
|
from .base_scraper import scraper_cli
|
|
|
|
scraper_cli(Scraper, "Scrape Recalbox es_bios.xml")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|