mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-18 17:33:30 -05:00
Abdessamad Derraz
e540dac996
RetroDECK filed coco as tandy-coco and Dragon as dragon-32-64, MiSTer filed COCO3 as tandy-coco and Recalbox kept its raw ids, so none of them met xroar's coco, coco3 and dragon64 and its ROMs reached no system. The scraper maps now agree and a test holds every shared native id to one system.
320 lines
11 KiB
Python
320 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Scraper for Recalbox BIOS requirements.
|
|
|
|
Source: https://gitlab.com/recalbox/recalbox/-/raw/master/board/recalbox/fsoverlay/recalbox/share_init/system/.emulationstation/es_bios.xml
|
|
Format: XML (es_bios.xml)
|
|
Hash: MD5 (multiple valid hashes per entry, comma-separated)
|
|
|
|
Recalbox verification logic:
|
|
- Checks MD5 of file on disk against list of valid hashes
|
|
- Multiple MD5s accepted per BIOS (different ROM revisions)
|
|
- Alternate file paths (pipe-separated)
|
|
- hashMatchMandatory flag: if false, wrong hash = warning (YELLOW) not error (RED)
|
|
- ZIP files get composite MD5 calculation
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from common import parse_untrusted_xml
|
|
|
|
from .base_scraper import BaseScraper, BiosRequirement, requirement_entry
|
|
|
|
PLATFORM_NAME = "recalbox"
|
|
|
|
def _fetch_gitlab_stable_tag() -> str:
|
|
"""Fetch the latest stable x.y.z tag from the Recalbox GitLab."""
|
|
import json
|
|
import re
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
url = "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tags?per_page=50"
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
tags = json.loads(resp.read())
|
|
except (urllib.error.URLError, json.JSONDecodeError) as e:
|
|
raise RuntimeError(f"cannot list Recalbox tags: {e}") from e
|
|
stable = [t["name"] for t in tags if re.fullmatch(r"[0-9]+\.[0-9]+(\.[0-9]+)?", t["name"])]
|
|
if not stable:
|
|
raise RuntimeError("no stable Recalbox tag among the latest 50")
|
|
return stable[0]
|
|
|
|
|
|
SOURCE_URL = (
|
|
"https://gitlab.com/recalbox/recalbox/-/raw/{tag}/"
|
|
"board/recalbox/fsoverlay/recalbox/share_init/system/"
|
|
".emulationstation/es_bios.xml"
|
|
)
|
|
|
|
SYSTEM_SLUG_MAP = {
|
|
"3do": "3do",
|
|
"dragon": "dragon64",
|
|
"trs80coco": "coco",
|
|
"amiga600": "commodore-amiga",
|
|
"amiga1200": "commodore-amiga",
|
|
"amigacd32": "commodore-amiga",
|
|
"amigacdtv": "commodore-amiga",
|
|
"amstradcpc": "amstrad-cpc",
|
|
"atari800": "atari-400-800",
|
|
"atari5200": "atari-5200",
|
|
"atari7800": "atari-7800",
|
|
"atarilynx": "atari-lynx",
|
|
"atarist": "atari-st",
|
|
"c64": "commodore-c64",
|
|
"channelf": "fairchild-channel-f",
|
|
"colecovision": "coleco-colecovision",
|
|
"dreamcast": "sega-dreamcast",
|
|
"fds": "nintendo-fds",
|
|
"gamecube": "nintendo-gamecube",
|
|
"gamegear": "sega-game-gear",
|
|
"gb": "nintendo-gb",
|
|
"gba": "nintendo-gba",
|
|
"gbc": "nintendo-gbc",
|
|
"intellivision": "mattel-intellivision",
|
|
"jaguar": "atari-jaguar",
|
|
"mastersystem": "sega-master-system",
|
|
"megadrive": "sega-mega-drive",
|
|
"msx": "microsoft-msx",
|
|
"msx1": "microsoft-msx",
|
|
"msx2": "microsoft-msx",
|
|
"n64": "nintendo-64",
|
|
"naomi": "sega-dreamcast-arcade",
|
|
"naomigd": "sega-dreamcast-arcade",
|
|
"atomiswave": "sega-dreamcast-arcade",
|
|
"nds": "nintendo-ds",
|
|
"neogeo": "snk-neogeo",
|
|
"neogeocd": "snk-neogeo-cd",
|
|
"o2em": "magnavox-odyssey2",
|
|
"pcengine": "nec-pc-engine",
|
|
"pcenginecd": "nec-pc-engine",
|
|
"pcfx": "nec-pc-fx",
|
|
"ps2": "sony-playstation-2",
|
|
"psx": "sony-playstation",
|
|
"saturn": "sega-saturn",
|
|
"scummvm": "scummvm",
|
|
"segacd": "sega-mega-cd",
|
|
"snes": "nintendo-snes",
|
|
"supergrafx": "nec-pc-engine",
|
|
"x68000": "sharp-x68000",
|
|
"zxspectrum": "sinclair-zx-spectrum",
|
|
}
|
|
|
|
|
|
SYSTEMS_TREE_URL = (
|
|
"https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tree"
|
|
"?path=package/recalbox-romfs2/systems&ref={tag}&per_page=100&page={page}"
|
|
)
|
|
SYSTEM_INI_URL = (
|
|
"https://gitlab.com/recalbox/recalbox/-/raw/{tag}"
|
|
"/package/recalbox-romfs2/systems/{system}/system.ini"
|
|
)
|
|
|
|
|
|
def system_ini_cores(text: str) -> tuple[set[str], set[str]]:
|
|
"""The cores one Recalbox system.ini offers, and those run standalone.
|
|
|
|
Each ``[core.N]`` section (or its board override ``[core.N.<board>]``)
|
|
names an ``emulator`` and a ``core``: ``libretro`` with the core's name,
|
|
or a standalone emulator. es_bios.xml only names the cores that need a
|
|
BIOS, so read alone it left most of Recalbox's cores outside the pack.
|
|
"""
|
|
import re
|
|
|
|
# Recalbox's .ini files are not strict INI (a value can run on a bare
|
|
# line), so only sections and key = value lines are read.
|
|
sections: dict[str, dict[str, str]] = {}
|
|
current: dict[str, str] | None = None
|
|
for raw in text.splitlines():
|
|
line = raw.strip()
|
|
if not line or line.startswith((";", "#")):
|
|
continue
|
|
header = re.fullmatch(r"\[([^\]]+)\]", line)
|
|
if header:
|
|
current = sections.setdefault(header.group(1).strip(), {})
|
|
continue
|
|
if current is not None and "=" in line:
|
|
key, value = line.split("=", 1)
|
|
current[key.strip()] = value.strip().strip('"')
|
|
cores: set[str] = set()
|
|
standalone: set[str] = set()
|
|
for section, values in sections.items():
|
|
if not re.fullmatch(r"core\.\d+(\.[\w-]+)?", section):
|
|
continue
|
|
emulator = values.get("emulator", "")
|
|
core = values.get("core", "")
|
|
if emulator == "libretro":
|
|
if core:
|
|
cores.add(core)
|
|
elif emulator:
|
|
cores.add(emulator)
|
|
standalone.add(emulator)
|
|
if core and core != emulator:
|
|
cores.add(core)
|
|
standalone.add(core)
|
|
return cores, standalone
|
|
|
|
|
|
class Scraper(BaseScraper):
|
|
"""Scraper for Recalbox es_bios.xml."""
|
|
|
|
def __init__(self):
|
|
self.tag = _fetch_gitlab_stable_tag()
|
|
super().__init__(url=SOURCE_URL.format(tag=self.tag))
|
|
|
|
def _fetch_cores(self) -> tuple[list[str], list[str]]:
|
|
"""Every core Recalbox's systems offer, and those it runs standalone.
|
|
|
|
Read from each system.ini of recalbox-romfs2 at the scraped tag: the
|
|
systems ES lists and the cores a user can pick for each.
|
|
"""
|
|
import json
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
def get(url: str) -> bytes:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return resp.read()
|
|
except urllib.error.URLError as e:
|
|
raise ConnectionError(f"Failed to fetch {url}: {e}") from e
|
|
|
|
systems: list[str] = []
|
|
page = 1
|
|
while True:
|
|
listing = json.loads(get(SYSTEMS_TREE_URL.format(tag=self.tag, page=page)))
|
|
systems += [
|
|
item["name"] for item in listing
|
|
if item.get("type") == "tree" and not item["name"].startswith(".")
|
|
]
|
|
if len(listing) < 100:
|
|
break
|
|
page += 1
|
|
if not systems:
|
|
raise RuntimeError(f"no Recalbox system found at {self.tag}")
|
|
cores: set[str] = set()
|
|
standalone: set[str] = set()
|
|
for system in systems:
|
|
text = get(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8")
|
|
found, alone = system_ini_cores(text)
|
|
cores |= found
|
|
standalone |= alone
|
|
if not cores:
|
|
raise RuntimeError(f"no core in Recalbox's system.ini files at {self.tag}")
|
|
return sorted(cores), sorted(standalone)
|
|
|
|
def fetch_requirements(self) -> list[BiosRequirement]:
|
|
"""Parse es_bios.xml and return BIOS requirements."""
|
|
raw = self._fetch_raw()
|
|
|
|
if not self.validate_format(raw):
|
|
raise ValueError("es_bios.xml format validation failed")
|
|
|
|
root = parse_untrusted_xml(raw, "es_bios.xml")
|
|
requirements = []
|
|
seen = set()
|
|
|
|
for system_elem in root.findall(".//system"):
|
|
platform = system_elem.get("platform", "")
|
|
system_slug = SYSTEM_SLUG_MAP.get(platform, platform)
|
|
fullname = system_elem.get("fullname", "")
|
|
|
|
for bios_elem in system_elem.findall("bios"):
|
|
paths_str = bios_elem.get("path", "")
|
|
md5_str = bios_elem.get("md5", "")
|
|
mandatory = bios_elem.get("mandatory", "true") != "false"
|
|
|
|
paths = [p.strip() for p in paths_str.split("|") if p.strip()]
|
|
if not paths:
|
|
continue
|
|
|
|
primary_path = paths[0]
|
|
name = (
|
|
primary_path.split("/")[-1] if "/" in primary_path else primary_path
|
|
)
|
|
|
|
md5_list = [m.strip() for m in md5_str.split(",") if m.strip()]
|
|
all_md5 = ",".join(md5_list) if md5_list else None
|
|
|
|
dedup_key = (platform, primary_path)
|
|
if dedup_key in seen:
|
|
continue
|
|
seen.add(dedup_key)
|
|
|
|
native: dict[str, object] = {}
|
|
if fullname:
|
|
native["native_name"] = fullname
|
|
core = bios_elem.get("core", "").strip()
|
|
if core:
|
|
native["core"] = core
|
|
note = bios_elem.get("note", "").strip()
|
|
if note:
|
|
native["note"] = note
|
|
# Recalbox reads a missing attribute as true for both flags,
|
|
# so only the explicit value carries information.
|
|
hash_match = bios_elem.get("hashMatchMandatory")
|
|
if hash_match is not None:
|
|
native["hash_match_mandatory"] = hash_match != "false"
|
|
if bios_elem.get("mandatory") is not None:
|
|
native["mandatory_declared"] = mandatory
|
|
if len(paths) > 1:
|
|
native["alt_paths"] = paths[1:]
|
|
|
|
requirements.append(
|
|
BiosRequirement(
|
|
name=name,
|
|
system=system_slug,
|
|
md5=all_md5,
|
|
destination=primary_path,
|
|
required=mandatory,
|
|
native_id=platform,
|
|
native=native,
|
|
)
|
|
)
|
|
|
|
return requirements
|
|
|
|
def validate_format(self, raw_data: str) -> bool:
|
|
"""Validate es_bios.xml format."""
|
|
return "<biosList" in raw_data and "<system" in raw_data and "<bios" in raw_data
|
|
|
|
def generate_platform_yaml(self) -> dict:
|
|
"""Generate a platform YAML config dict from scraped data."""
|
|
requirements = self.fetch_requirements()
|
|
|
|
systems: dict[str, dict] = {}
|
|
for req in requirements:
|
|
if req.system not in systems:
|
|
sys_entry: dict = {"files": []}
|
|
if req.native_id:
|
|
sys_entry["native_id"] = req.native_id
|
|
systems[req.system] = sys_entry
|
|
|
|
systems[req.system]["files"].append(requirement_entry(req))
|
|
|
|
cores, standalone = self._fetch_cores()
|
|
return {
|
|
"platform": "Recalbox",
|
|
"version": self.tag,
|
|
"homepage": "https://www.recalbox.com",
|
|
"source": self.url,
|
|
"base_destination": "bios",
|
|
"hash_type": "md5",
|
|
"verification_mode": "md5",
|
|
"cores": cores,
|
|
"standalone_cores": standalone,
|
|
"systems": systems,
|
|
}
|
|
|
|
|
|
def main():
|
|
"""CLI entry point."""
|
|
from .base_scraper import scraper_cli
|
|
|
|
scraper_cli(Scraper, "Scrape Recalbox es_bios.xml")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|