diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 2da6ff0a..12584541 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -213,201 +213,6 @@ platforms: schedule: monthly target_scraper: null target_source: null - cores: - - '2048' - - '81' - - a5200 - - advancemame - - amiberry - - applewin - - arduous - - atari800 - - b2 - - beebem - - bk - - bluemsx - - boom3 - - bsnes - - bsneshd - - cannonball - - cap32 - - cdi2015 - - corsixth - - craft - - crocods - - daphne - - desmume - - dice - - dinothawr - - dirksimple - - dolphin - - dolphin-gui - - dosbox - - dosbox_pure - - duckstation - - easyrpg - - ecwolf - - emuscv - - fake08 - - fba2x - - fbneo - - fceumm - - flycast - - flycast-next - - fmsx - - freechaf - - freeintv - - frotz - - fuse - - gambatte - - gearcoleco - - geargrafx - - gearsystem - - genesisplusgx - - genesisplusgx_ex - - genesisplusgxwide - - geolith - - glide64mk2 - - gliden64 - - gliden64_20 - - gong - - gpsp - - gsplus - - gw - - handy - - hatari - - hatarib - - holani - - imageviewer - - julius - - kronos - - lowresnx - - lutro - - mame - - mame0258 - - mame0278 - - mame2000 - - mame2003 - - mame2003_plus - - mame2010 - - mame2015 - - mame2016 - - mamemess - - mednafen_lynx - - mednafen_ngp - - mednafen_pce_fast - - mednafen_pcfx - - mednafen_psx - - mednafen_psx_hw - - mednafen_saturn - - mednafen_supafaust - - mednafen_supergrafx - - mednafen_vb - - mednafen_wswan - - melonds - - mesen - - mesen_s - - meteor - - mgba - - minivmac - - mojozork - - moonlight - - mrboom - - mu - - mupen64plus - - mupen64plus_next - - n64_gles2 - - neocd - - nestopia - - np2kai - - nxengine - - o2em - - openbor - - openlara - - opera - - oricutron - - parallel_n64 - - pcsx2 - - pcsx_rearmed - - pico8 - - picodrive - - pisnes - - pokemini - - potator - - ppsspp - - prboom - - prosystem - - ps2 - - puae - - px68k - - quasi88 - - quicknes - - race - - rb5000 - - reicast - - reminiscence - - retro8 - - retrodream - - rice - - rice_gles2 - - same_cdi - - sameboy - - sameduck - - scummvm - - sdlpop - - simcoupe - - snes9x - - snes9x2002 - - snes9x2005 - - snes9x2010 - - solarus - - stella - - stella2014 - - stonesoup - - supermodel - - swanstation - - tamalibretro - - tgbdual - - theodore - - thepowdertoy - - ti99sim - - tic80 - - tyrquake - - uae4all - - uae4arm - - uzem - - vecx - - vice_x128 - - vice_x64 - - vice_x64sc - - vice_xcbm2 - - vice_xcbm5x0 - - vice_xpet - - vice_xplus4 - - vice_xscpu64 - - vice_xvic - - virtualjaguar - - vitaquake2 - - vitaquake3 - - vitavoyager - - vpinball - - vvvvvv - - wasm4 - - x1 - - x128 - - x64 - - x64sx - - xcbm2 - - xcbm5x0 - - xemu - - xpet - - xplus4 - - xrick - - xroar - - xscpu64 - - xvic - - yabasanshiro - - yabause install: detect: - os: linux diff --git a/platforms/recalbox.yml b/platforms/recalbox.yml index 490d517b..297390a6 100644 --- a/platforms/recalbox.yml +++ b/platforms/recalbox.yml @@ -6,112 +6,262 @@ base_destination: bios hash_type: md5 verification_mode: md5 cores: +- "2048" +- "81" +- a5200 +- advancemame - amiberry +- applewin +- arduous +- atari800 +- b2 +- beebem +- bk +- bluemsx +- boom3 +- bsnes +- bsneshd +- cannonball +- cap32 +- cdi2015 +- corsixth +- craft +- crocods +- daphne +- desmume +- dice +- dinothawr +- dirksimple - dolphin +- dolphin-gui +- dosbox +- dosbox_pure - duckstation +- easyrpg +- ecwolf +- eightyone +- emuscv +- exel-hence +- fake08 +- fba2x +- fbneo +- fceumm +- flycast +- flycast-next +- fmsx +- freechaf +- freeintv +- frotz +- fuse +- gambatte +- gearcoleco +- geargrafx +- gearsystem +- genesisplusgx +- genesisplusgx_ex +- genesisplusgxwide +- geolith +- glide64mk2 +- gliden64 +- gliden64_20 +- gong +- gpsp - gsplus +- gw +- handy - hatari -- libretro-hatari -- libretro/ mame2003-plus -- libretro/a5200 -- libretro/amiberry -- libretro/atari800 -- libretro/bk -- libretro/bluemsx -- libretro/bsnes -- libretro/bsnes-hd -- libretro/cdi2015 -- libretro/desmume -- libretro/dolphin -- libretro/emuscv -- libretro/fba2x -- libretro/fbneo -- libretro/fceumm -- libretro/flycast -- libretro/fmsx -- libretro/freechaf -- libretro/freeintv -- libretro/fuse -- libretro/gambatte -- libretro/gearcoleco -- libretro/geargrafx -- libretro/gearsystem -- libretro/genesisplusgx -- libretro/genesisplusgxwide -- libretro/geolith -- libretro/gpsp -- libretro/handy -- libretro/hatari -- libretro/kronos -- libretro/mame2000 -- libretro/mame2003 -- libretro/mame2003-plus -- libretro/mame2010 -- libretro/mame2015 -- libretro/mednafen_lynx -- libretro/mednafen_pce_fast -- libretro/mednafen_psx -- libretro/mednafen_psx_hw -- libretro/mednafen_saturn -- libretro/mednafen_stv -- libretro/mednafen_supergrafx -- libretro/melonds -- libretro/mesen -- libretro/mesen_s -- libretro/meteor -- libretro/mgba -- libretro/minivmac -- libretro/mu -- libretro/mupen64plus_next -- libretro/neocd -- libretro/nestopia -- libretro/np2kai -- libretro/o2em -- libretro/opera -- libretro/parallel_n64 -- libretro/pcfx -- libretro/pcsx2 -- libretro/pcsx_rearmed -- libretro/picodrive -- libretro/pokemini -- libretro/prosystem -- libretro/px68k -- libretro/quasi88 -- libretro/same_cdi -- libretro/sameboy -- libretro/scummvm -- libretro/snes9x -- libretro/swanstation -- libretro/uae -- libretro/uae4all -- libretro/uae4arm -- libretro/vice_x128 -- libretro/vice_x64 -- libretro/vice_x64sc -- libretro/xmil -- libretro/xscpu64 -- libretro/yabasanshiro -- libretro/yabause +- hatarib +- holani +- hypseus_singe +- imageviewer +- julius +- kronos +- lowresnx +- lutro +- m2000 +- mame0258 +- mame0278 +- mame2000 +- mame2003 +- mame2003_plus +- mame2010 +- mame2015 +- mame2016 +- mednafen_lynx +- mednafen_ngp +- mednafen_pce_fast +- mednafen_pcfx +- mednafen_psx +- mednafen_psx_hw +- mednafen_saturn +- mednafen_stv +- mednafen_supafaust +- mednafen_supergrafx +- mednafen_vb +- mednafen_wswan +- melonds +- mesen +- mesen_s +- meteor +- mgba +- minivmac +- mojozork +- moonlight +- mrboom +- mu +- mupen64plus +- mupen64plus_next +- n64_gles2 +- neocd +- nestopia +- np2kai +- nxengine +- o2em +- openbor +- openlara +- opera - oricutron +- parallel_n64 - pcsx2 +- pcsx_rearmed +- pd777 +- pico8 +- picodrive +- pisnes +- pokemini +- potator +- ppsspp +- prboom +- prosystem +- ps2 +- puae +- puae2021 +- px68k +- quasi88 +- quicknes +- race +- rb5000 - reicast +- reminiscence +- retro8 +- retrodream +- rice +- rice_gles2 +- same_cdi +- sameboy +- sameduck - scummvm +- sdlpop +- simcoupe +- snes9x +- snes9x2002 +- snes9x2005 +- snes9x2010 +- solarus +- stella +- stella2014 +- stonesoup +- sugarbox2 +- supermodel +- swanstation +- tamalibretro +- tgbdual +- theodore +- thepowdertoy - ti99sim +- tic80 +- tyrquake +- uae4all +- uae4arm +- uzem +- vecx +- vice +- vice_x128 +- vice_x64 +- vice_x64sc +- vice_xcbm2 +- vice_xcbm5x0 +- vice_xpet +- vice_xplus4 +- vice_xscpu64 +- vice_xvic +- vircon32 +- virtualjaguar +- vitaquake2 +- vitaquake3 +- vitavoyager +- vpinball +- vvvvvv +- wasm4 +- x1 +- x128 +- x64 +- x64sx +- xcbm2 +- xcbm5x0 - xemu +- xpet +- xplus4 +- xrick - xroar +- xscpu64 +- xvic +- yabasanshiro +- yabause +- ymir standalone_cores: +- advancemame - amiberry +- beebem +- corsixth +- daphne - dolphin +- dolphin-gui +- dosbox - duckstation +- fba2x +- frotz +- glide64mk2 +- gliden64 +- gliden64_20 - gsplus - hatari +- julius +- moonlight +- mupen64plus +- n64_gles2 +- openbor +- openlara - oricutron - pcsx2 +- pcsx_rearmed +- pico8 +- pisnes +- ppsspp +- rb5000 - reicast +- rice +- rice_gles2 - scummvm +- sdlpop +- simcoupe +- solarus +- supermodel - ti99sim +- vice +- vpinball +- vvvvvv +- x128 +- x64 +- x64sx +- xcbm2 +- xcbm5x0 - xemu +- xpet +- xplus4 - xroar +- xscpu64 +- xvic systems: commodore-amiga: files: diff --git a/scripts/scraper/recalbox_scraper.py b/scripts/scraper/recalbox_scraper.py index 22bfc1fb..62173df7 100644 --- a/scripts/scraper/recalbox_scraper.py +++ b/scripts/scraper/recalbox_scraper.py @@ -99,11 +99,58 @@ SYSTEM_SLUG_MAP = { } -def split_cores(names) -> tuple[list[str], list[str]]: - """(every core name, the standalone ones) from es_bios.xml ``core`` values.""" - cores = {name for name in names if name} - standalone = {name for name in cores if not name.startswith("libretro")} - return sorted(cores), sorted(standalone) +SYSTEMS_TREE_URL = ( + "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tree" + "?path=package/recalbox-romfs2/systems&ref={tag}&per_page=100&page={page}" +) +SYSTEM_INI_URL = ( + "https://gitlab.com/recalbox/recalbox/-/raw/{tag}" + "/package/recalbox-romfs2/systems/{system}/system.ini" +) + + +def system_ini_cores(text: str) -> tuple[set[str], set[str]]: + """The cores one Recalbox system.ini offers, and those run standalone. + + Each ``[core.N]`` section (or its board override ``[core.N.]``) + names an ``emulator`` and a ``core``: ``libretro`` with the core's name, + or a standalone emulator. es_bios.xml only names the cores that need a + BIOS, so read alone it left most of Recalbox's cores outside the pack. + """ + import re + + # Recalbox's .ini files are not strict INI (a value can run on a bare + # line), so only sections and key = value lines are read. + sections: dict[str, dict[str, str]] = {} + current: dict[str, str] | None = None + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith((";", "#")): + continue + header = re.fullmatch(r"\[([^\]]+)\]", line) + if header: + current = sections.setdefault(header.group(1).strip(), {}) + continue + if current is not None and "=" in line: + key, value = line.split("=", 1) + current[key.strip()] = value.strip().strip('"') + cores: set[str] = set() + standalone: set[str] = set() + for section, values in sections.items(): + if not re.fullmatch(r"core\.\d+(\.[\w-]+)?", section): + continue + emulator = values.get("emulator", "") + core = values.get("core", "") + if emulator == "libretro": + if core: + cores.add(core) + elif emulator: + cores.add(emulator) + standalone.add(emulator) + if core and core != emulator: + cores.add(core) + standalone.add(core) + return cores, standalone class Scraper(BaseScraper): @@ -114,20 +161,46 @@ class Scraper(BaseScraper): super().__init__(url=SOURCE_URL.format(tag=self.tag)) def _fetch_cores(self) -> tuple[list[str], list[str]]: - """Core names from es_bios.xml, and those Recalbox runs standalone. + """Every core Recalbox's systems offer, and those it runs standalone. - The ``core`` attribute writes a libretro core as ``libretro/`` - and a standalone emulator by its bare name (``dolphin``, ``pcsx2``, - ``xemu``), so the prefix is the build mode. One entry spells - ``libretro-hatari``; a name that starts with ``libretro`` is a core. + Read from each system.ini of recalbox-romfs2 at the scraped tag: the + systems ES lists and the cores a user can pick for each. """ - raw = self._fetch_raw() - root = parse_untrusted_xml(raw, "es_bios.xml") - return split_cores( - part.strip() - for bios_elem in root.findall(".//system/bios") - for part in bios_elem.get("core", "").split(",") - ) + import json + import urllib.error + import urllib.request + + def get(url: str) -> bytes: + req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"}) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read() + except urllib.error.URLError as e: + raise ConnectionError(f"Failed to fetch {url}: {e}") from e + + systems: list[str] = [] + page = 1 + while True: + listing = json.loads(get(SYSTEMS_TREE_URL.format(tag=self.tag, page=page))) + systems += [ + item["name"] for item in listing + if item.get("type") == "tree" and not item["name"].startswith(".") + ] + if len(listing) < 100: + break + page += 1 + if not systems: + raise RuntimeError(f"no Recalbox system found at {self.tag}") + cores: set[str] = set() + standalone: set[str] = set() + for system in systems: + text = get(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8") + found, alone = system_ini_cores(text) + cores |= found + standalone |= alone + if not cores: + raise RuntimeError(f"no core in Recalbox's system.ini files at {self.tag}") + return sorted(cores), sorted(standalone) def fetch_requirements(self) -> list[BiosRequirement]: """Parse es_bios.xml and return BIOS requirements.""" diff --git a/tests/test_scraper_contract.py b/tests/test_scraper_contract.py index 16b38f8a..920634ed 100644 --- a/tests/test_scraper_contract.py +++ b/tests/test_scraper_contract.py @@ -219,23 +219,45 @@ class RetroPiePackageList(unittest.TestCase): class RecalboxBuildMode(unittest.TestCase): - """es_bios.xml names a libretro core as libretro/ and a standalone - emulator by its bare name: the prefix is the build mode, and a platform - that runs an emulator standalone must say so or the pack lays its files - out for the libretro build.""" + """Recalbox's cores come from each system.ini of recalbox-romfs2: a + `[core.N]` section names `emulator = "libretro"` with the core, or a + standalone emulator. es_bios.xml only names the cores that need a BIOS, + so read alone it left most of Recalbox's cores, mame0258 and mame0278 + among them, outside the pack.""" - def test_bare_names_are_the_standalone_emulators(self): - from scraper.recalbox_scraper import split_cores + SYSTEM_INI = """\ +[system] +name = "mame" +fullname = "MAME" + extra line the parser must survive. - cores, standalone = split_cores( - ["libretro/uae", "amiberry", "", "dolphin", "libretro/dolphin", "libretro-hatari", "hatari"] - ) - self.assertEqual( - cores, - ["amiberry", "dolphin", "hatari", "libretro-hatari", "libretro/dolphin", "libretro/uae"], - ) - self.assertEqual(standalone, ["amiberry", "dolphin", "hatari"]) +[core.0] +package = BR2_PACKAGE_LIBRETRO_MAME0278 +emulator = "libretro" +core = "mame0278" +[core.1] +emulator = "advancemame" +core = "advancemame" + +[core.2.rpi4] +emulator = "libretro" +core = "mame2003_plus" +""" + + def test_libretro_cores_and_standalone_emulators(self): + from scraper.recalbox_scraper import system_ini_cores + + cores, standalone = system_ini_cores(self.SYSTEM_INI) + self.assertEqual(cores, {"mame0278", "advancemame", "mame2003_plus"}) + self.assertEqual(standalone, {"advancemame"}) + + def test_the_registry_leaves_the_list_to_the_scraper(self): + import yaml + + root = Path(__file__).resolve().parents[1] + registry = yaml.safe_load((root / "platforms" / "_registry.yml").read_text()) + self.assertNotIn("cores", registry["platforms"]["recalbox"]) class UnreadableReleaseStopsTheScrape(unittest.TestCase):