From 6280f4c370d170f88019f55d33e74080b0de3378 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sat, 17 Oct 2026 03:31:11 +0200 Subject: [PATCH] feat: read recalbox's cores from its system files es_bios.xml names only the cores that need a BIOS, so a hand-kept registry list stood in, with mame and mamemess that Recalbox does not ship and without vice, ymir, vircon32 and others it does. The scraper now reads each system.ini of recalbox-romfs2 at the scraped tag, and the registry list is gone. --- platforms/_registry.yml | 195 ----------------- platforms/recalbox.yml | 312 ++++++++++++++++++++-------- scripts/scraper/recalbox_scraper.py | 107 ++++++++-- tests/test_scraper_contract.py | 50 +++-- 4 files changed, 357 insertions(+), 307 deletions(-) diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 2da6ff0a..12584541 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -213,201 +213,6 @@ platforms: schedule: monthly target_scraper: null target_source: null - cores: - - '2048' - - '81' - - a5200 - - advancemame - - amiberry - - applewin - - arduous - - atari800 - - b2 - - beebem - - bk - - bluemsx - - boom3 - - bsnes - - bsneshd - - cannonball - - cap32 - - cdi2015 - - corsixth - - craft - - crocods - - daphne - - desmume - - dice - - dinothawr - - dirksimple - - dolphin - - dolphin-gui - - dosbox - - dosbox_pure - - duckstation - - easyrpg - - ecwolf - - emuscv - - fake08 - - fba2x - - fbneo - - fceumm - - flycast - - flycast-next - - fmsx - - freechaf - - freeintv - - frotz - - fuse - - gambatte - - gearcoleco - - geargrafx - - gearsystem - - genesisplusgx - - genesisplusgx_ex - - genesisplusgxwide - - geolith - - glide64mk2 - - gliden64 - - gliden64_20 - - gong - - gpsp - - gsplus - - gw - - handy - - hatari - - hatarib - - holani - - imageviewer - - julius - - kronos - - lowresnx - - lutro - - mame - - mame0258 - - mame0278 - - mame2000 - - mame2003 - - mame2003_plus - - mame2010 - - mame2015 - - mame2016 - - mamemess - - mednafen_lynx - - mednafen_ngp - - mednafen_pce_fast - - mednafen_pcfx - - mednafen_psx - - mednafen_psx_hw - - mednafen_saturn - - mednafen_supafaust - - mednafen_supergrafx - - mednafen_vb - - mednafen_wswan - - melonds - - mesen - - mesen_s - - meteor - - mgba - - minivmac - - mojozork - - moonlight - - mrboom - - mu - - mupen64plus - - mupen64plus_next - - n64_gles2 - - neocd - - nestopia - - np2kai - - nxengine - - o2em - - openbor - - openlara - - opera - - oricutron - - parallel_n64 - - pcsx2 - - pcsx_rearmed - - pico8 - - picodrive - - pisnes - - pokemini - - potator - - ppsspp - - prboom - - prosystem - - ps2 - - puae - - px68k - - quasi88 - - quicknes - - race - - rb5000 - - reicast - - reminiscence - - retro8 - - retrodream - - rice - - rice_gles2 - - same_cdi - - sameboy - - sameduck - - scummvm - - sdlpop - - simcoupe - - snes9x - - snes9x2002 - - snes9x2005 - - snes9x2010 - - solarus - - stella - - stella2014 - - stonesoup - - supermodel - - swanstation - - tamalibretro - - tgbdual - - theodore - - thepowdertoy - - ti99sim - - tic80 - - tyrquake - - uae4all - - uae4arm - - uzem - - vecx - - vice_x128 - - vice_x64 - - vice_x64sc - - vice_xcbm2 - - vice_xcbm5x0 - - vice_xpet - - vice_xplus4 - - vice_xscpu64 - - vice_xvic - - virtualjaguar - - vitaquake2 - - vitaquake3 - - vitavoyager - - vpinball - - vvvvvv - - wasm4 - - x1 - - x128 - - x64 - - x64sx - - xcbm2 - - xcbm5x0 - - xemu - - xpet - - xplus4 - - xrick - - xroar - - xscpu64 - - xvic - - yabasanshiro - - yabause install: detect: - os: linux diff --git a/platforms/recalbox.yml b/platforms/recalbox.yml index 490d517b..297390a6 100644 --- a/platforms/recalbox.yml +++ b/platforms/recalbox.yml @@ -6,112 +6,262 @@ base_destination: bios hash_type: md5 verification_mode: md5 cores: +- "2048" +- "81" +- a5200 +- advancemame - amiberry +- applewin +- arduous +- atari800 +- b2 +- beebem +- bk +- bluemsx +- boom3 +- bsnes +- bsneshd +- cannonball +- cap32 +- cdi2015 +- corsixth +- craft +- crocods +- daphne +- desmume +- dice +- dinothawr +- dirksimple - dolphin +- dolphin-gui +- dosbox +- dosbox_pure - duckstation +- easyrpg +- ecwolf +- eightyone +- emuscv +- exel-hence +- fake08 +- fba2x +- fbneo +- fceumm +- flycast +- flycast-next +- fmsx +- freechaf +- freeintv +- frotz +- fuse +- gambatte +- gearcoleco +- geargrafx +- gearsystem +- genesisplusgx +- genesisplusgx_ex +- genesisplusgxwide +- geolith +- glide64mk2 +- gliden64 +- gliden64_20 +- gong +- gpsp - gsplus +- gw +- handy - hatari -- libretro-hatari -- libretro/ mame2003-plus -- libretro/a5200 -- libretro/amiberry -- libretro/atari800 -- libretro/bk -- libretro/bluemsx -- libretro/bsnes -- libretro/bsnes-hd -- libretro/cdi2015 -- libretro/desmume -- libretro/dolphin -- libretro/emuscv -- libretro/fba2x -- libretro/fbneo -- libretro/fceumm -- libretro/flycast -- libretro/fmsx -- libretro/freechaf -- libretro/freeintv -- libretro/fuse -- libretro/gambatte -- libretro/gearcoleco -- libretro/geargrafx -- libretro/gearsystem -- libretro/genesisplusgx -- libretro/genesisplusgxwide -- libretro/geolith -- libretro/gpsp -- libretro/handy -- libretro/hatari -- libretro/kronos -- libretro/mame2000 -- libretro/mame2003 -- libretro/mame2003-plus -- libretro/mame2010 -- libretro/mame2015 -- libretro/mednafen_lynx -- libretro/mednafen_pce_fast -- libretro/mednafen_psx -- libretro/mednafen_psx_hw -- libretro/mednafen_saturn -- libretro/mednafen_stv -- libretro/mednafen_supergrafx -- libretro/melonds -- libretro/mesen -- libretro/mesen_s -- libretro/meteor -- libretro/mgba -- libretro/minivmac -- libretro/mu -- libretro/mupen64plus_next -- libretro/neocd -- libretro/nestopia -- libretro/np2kai -- libretro/o2em -- libretro/opera -- libretro/parallel_n64 -- libretro/pcfx -- libretro/pcsx2 -- libretro/pcsx_rearmed -- libretro/picodrive -- libretro/pokemini -- libretro/prosystem -- libretro/px68k -- libretro/quasi88 -- libretro/same_cdi -- libretro/sameboy -- libretro/scummvm -- libretro/snes9x -- libretro/swanstation -- libretro/uae -- libretro/uae4all -- libretro/uae4arm -- libretro/vice_x128 -- libretro/vice_x64 -- libretro/vice_x64sc -- libretro/xmil -- libretro/xscpu64 -- libretro/yabasanshiro -- libretro/yabause +- hatarib +- holani +- hypseus_singe +- imageviewer +- julius +- kronos +- lowresnx +- lutro +- m2000 +- mame0258 +- mame0278 +- mame2000 +- mame2003 +- mame2003_plus +- mame2010 +- mame2015 +- mame2016 +- mednafen_lynx +- mednafen_ngp +- mednafen_pce_fast +- mednafen_pcfx +- mednafen_psx +- mednafen_psx_hw +- mednafen_saturn +- mednafen_stv +- mednafen_supafaust +- mednafen_supergrafx +- mednafen_vb +- mednafen_wswan +- melonds +- mesen +- mesen_s +- meteor +- mgba +- minivmac +- mojozork +- moonlight +- mrboom +- mu +- mupen64plus +- mupen64plus_next +- n64_gles2 +- neocd +- nestopia +- np2kai +- nxengine +- o2em +- openbor +- openlara +- opera - oricutron +- parallel_n64 - pcsx2 +- pcsx_rearmed +- pd777 +- pico8 +- picodrive +- pisnes +- pokemini +- potator +- ppsspp +- prboom +- prosystem +- ps2 +- puae +- puae2021 +- px68k +- quasi88 +- quicknes +- race +- rb5000 - reicast +- reminiscence +- retro8 +- retrodream +- rice +- rice_gles2 +- same_cdi +- sameboy +- sameduck - scummvm +- sdlpop +- simcoupe +- snes9x +- snes9x2002 +- snes9x2005 +- snes9x2010 +- solarus +- stella +- stella2014 +- stonesoup +- sugarbox2 +- supermodel +- swanstation +- tamalibretro +- tgbdual +- theodore +- thepowdertoy - ti99sim +- tic80 +- tyrquake +- uae4all +- uae4arm +- uzem +- vecx +- vice +- vice_x128 +- vice_x64 +- vice_x64sc +- vice_xcbm2 +- vice_xcbm5x0 +- vice_xpet +- vice_xplus4 +- vice_xscpu64 +- vice_xvic +- vircon32 +- virtualjaguar +- vitaquake2 +- vitaquake3 +- vitavoyager +- vpinball +- vvvvvv +- wasm4 +- x1 +- x128 +- x64 +- x64sx +- xcbm2 +- xcbm5x0 - xemu +- xpet +- xplus4 +- xrick - xroar +- xscpu64 +- xvic +- yabasanshiro +- yabause +- ymir standalone_cores: +- advancemame - amiberry +- beebem +- corsixth +- daphne - dolphin +- dolphin-gui +- dosbox - duckstation +- fba2x +- frotz +- glide64mk2 +- gliden64 +- gliden64_20 - gsplus - hatari +- julius +- moonlight +- mupen64plus +- n64_gles2 +- openbor +- openlara - oricutron - pcsx2 +- pcsx_rearmed +- pico8 +- pisnes +- ppsspp +- rb5000 - reicast +- rice +- rice_gles2 - scummvm +- sdlpop +- simcoupe +- solarus +- supermodel - ti99sim +- vice +- vpinball +- vvvvvv +- x128 +- x64 +- x64sx +- xcbm2 +- xcbm5x0 - xemu +- xpet +- xplus4 - xroar +- xscpu64 +- xvic systems: commodore-amiga: files: diff --git a/scripts/scraper/recalbox_scraper.py b/scripts/scraper/recalbox_scraper.py index 22bfc1fb..62173df7 100644 --- a/scripts/scraper/recalbox_scraper.py +++ b/scripts/scraper/recalbox_scraper.py @@ -99,11 +99,58 @@ SYSTEM_SLUG_MAP = { } -def split_cores(names) -> tuple[list[str], list[str]]: - """(every core name, the standalone ones) from es_bios.xml ``core`` values.""" - cores = {name for name in names if name} - standalone = {name for name in cores if not name.startswith("libretro")} - return sorted(cores), sorted(standalone) +SYSTEMS_TREE_URL = ( + "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tree" + "?path=package/recalbox-romfs2/systems&ref={tag}&per_page=100&page={page}" +) +SYSTEM_INI_URL = ( + "https://gitlab.com/recalbox/recalbox/-/raw/{tag}" + "/package/recalbox-romfs2/systems/{system}/system.ini" +) + + +def system_ini_cores(text: str) -> tuple[set[str], set[str]]: + """The cores one Recalbox system.ini offers, and those run standalone. + + Each ``[core.N]`` section (or its board override ``[core.N.]``) + names an ``emulator`` and a ``core``: ``libretro`` with the core's name, + or a standalone emulator. es_bios.xml only names the cores that need a + BIOS, so read alone it left most of Recalbox's cores outside the pack. + """ + import re + + # Recalbox's .ini files are not strict INI (a value can run on a bare + # line), so only sections and key = value lines are read. + sections: dict[str, dict[str, str]] = {} + current: dict[str, str] | None = None + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith((";", "#")): + continue + header = re.fullmatch(r"\[([^\]]+)\]", line) + if header: + current = sections.setdefault(header.group(1).strip(), {}) + continue + if current is not None and "=" in line: + key, value = line.split("=", 1) + current[key.strip()] = value.strip().strip('"') + cores: set[str] = set() + standalone: set[str] = set() + for section, values in sections.items(): + if not re.fullmatch(r"core\.\d+(\.[\w-]+)?", section): + continue + emulator = values.get("emulator", "") + core = values.get("core", "") + if emulator == "libretro": + if core: + cores.add(core) + elif emulator: + cores.add(emulator) + standalone.add(emulator) + if core and core != emulator: + cores.add(core) + standalone.add(core) + return cores, standalone class Scraper(BaseScraper): @@ -114,20 +161,46 @@ class Scraper(BaseScraper): super().__init__(url=SOURCE_URL.format(tag=self.tag)) def _fetch_cores(self) -> tuple[list[str], list[str]]: - """Core names from es_bios.xml, and those Recalbox runs standalone. + """Every core Recalbox's systems offer, and those it runs standalone. - The ``core`` attribute writes a libretro core as ``libretro/`` - and a standalone emulator by its bare name (``dolphin``, ``pcsx2``, - ``xemu``), so the prefix is the build mode. One entry spells - ``libretro-hatari``; a name that starts with ``libretro`` is a core. + Read from each system.ini of recalbox-romfs2 at the scraped tag: the + systems ES lists and the cores a user can pick for each. """ - raw = self._fetch_raw() - root = parse_untrusted_xml(raw, "es_bios.xml") - return split_cores( - part.strip() - for bios_elem in root.findall(".//system/bios") - for part in bios_elem.get("core", "").split(",") - ) + import json + import urllib.error + import urllib.request + + def get(url: str) -> bytes: + req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"}) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read() + except urllib.error.URLError as e: + raise ConnectionError(f"Failed to fetch {url}: {e}") from e + + systems: list[str] = [] + page = 1 + while True: + listing = json.loads(get(SYSTEMS_TREE_URL.format(tag=self.tag, page=page))) + systems += [ + item["name"] for item in listing + if item.get("type") == "tree" and not item["name"].startswith(".") + ] + if len(listing) < 100: + break + page += 1 + if not systems: + raise RuntimeError(f"no Recalbox system found at {self.tag}") + cores: set[str] = set() + standalone: set[str] = set() + for system in systems: + text = get(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8") + found, alone = system_ini_cores(text) + cores |= found + standalone |= alone + if not cores: + raise RuntimeError(f"no core in Recalbox's system.ini files at {self.tag}") + return sorted(cores), sorted(standalone) def fetch_requirements(self) -> list[BiosRequirement]: """Parse es_bios.xml and return BIOS requirements.""" diff --git a/tests/test_scraper_contract.py b/tests/test_scraper_contract.py index 16b38f8a..920634ed 100644 --- a/tests/test_scraper_contract.py +++ b/tests/test_scraper_contract.py @@ -219,23 +219,45 @@ class RetroPiePackageList(unittest.TestCase): class RecalboxBuildMode(unittest.TestCase): - """es_bios.xml names a libretro core as libretro/ and a standalone - emulator by its bare name: the prefix is the build mode, and a platform - that runs an emulator standalone must say so or the pack lays its files - out for the libretro build.""" + """Recalbox's cores come from each system.ini of recalbox-romfs2: a + `[core.N]` section names `emulator = "libretro"` with the core, or a + standalone emulator. es_bios.xml only names the cores that need a BIOS, + so read alone it left most of Recalbox's cores, mame0258 and mame0278 + among them, outside the pack.""" - def test_bare_names_are_the_standalone_emulators(self): - from scraper.recalbox_scraper import split_cores + SYSTEM_INI = """\ +[system] +name = "mame" +fullname = "MAME" + extra line the parser must survive. - cores, standalone = split_cores( - ["libretro/uae", "amiberry", "", "dolphin", "libretro/dolphin", "libretro-hatari", "hatari"] - ) - self.assertEqual( - cores, - ["amiberry", "dolphin", "hatari", "libretro-hatari", "libretro/dolphin", "libretro/uae"], - ) - self.assertEqual(standalone, ["amiberry", "dolphin", "hatari"]) +[core.0] +package = BR2_PACKAGE_LIBRETRO_MAME0278 +emulator = "libretro" +core = "mame0278" +[core.1] +emulator = "advancemame" +core = "advancemame" + +[core.2.rpi4] +emulator = "libretro" +core = "mame2003_plus" +""" + + def test_libretro_cores_and_standalone_emulators(self): + from scraper.recalbox_scraper import system_ini_cores + + cores, standalone = system_ini_cores(self.SYSTEM_INI) + self.assertEqual(cores, {"mame0278", "advancemame", "mame2003_plus"}) + self.assertEqual(standalone, {"advancemame"}) + + def test_the_registry_leaves_the_list_to_the_scraper(self): + import yaml + + root = Path(__file__).resolve().parents[1] + registry = yaml.safe_load((root / "platforms" / "_registry.yml").read_text()) + self.assertNotIn("cores", registry["platforms"]["recalbox"]) class UnreadableReleaseStopsTheScrape(unittest.TestCase):