feat: read recalbox's cores from its system files

es_bios.xml names only the cores that need a BIOS, so a hand-kept
registry list stood in, with mame and mamemess that Recalbox does not
ship and without vice, ymir, vircon32 and others it does. The scraper
now reads each system.ini of recalbox-romfs2 at the scraped tag, and
the registry list is gone.

platforms/_registry.yml

@@ -213,201 +213,6 @@ platforms:
     schedule: monthly
     target_scraper: null
     target_source: null
-    cores:
-    - '2048'
-    - '81'
-    - a5200
-    - advancemame
-    - amiberry
-    - applewin
-    - arduous
-    - atari800
-    - b2
-    - beebem
-    - bk
-    - bluemsx
-    - boom3
-    - bsnes
-    - bsneshd
-    - cannonball
-    - cap32
-    - cdi2015
-    - corsixth
-    - craft
-    - crocods
-    - daphne
-    - desmume
-    - dice
-    - dinothawr
-    - dirksimple
-    - dolphin
-    - dolphin-gui
-    - dosbox
-    - dosbox_pure
-    - duckstation
-    - easyrpg
-    - ecwolf
-    - emuscv
-    - fake08
-    - fba2x
-    - fbneo
-    - fceumm
-    - flycast
-    - flycast-next
-    - fmsx
-    - freechaf
-    - freeintv
-    - frotz
-    - fuse
-    - gambatte
-    - gearcoleco
-    - geargrafx
-    - gearsystem
-    - genesisplusgx
-    - genesisplusgx_ex
-    - genesisplusgxwide
-    - geolith
-    - glide64mk2
-    - gliden64
-    - gliden64_20
-    - gong
-    - gpsp
-    - gsplus
-    - gw
-    - handy
-    - hatari
-    - hatarib
-    - holani
-    - imageviewer
-    - julius
-    - kronos
-    - lowresnx
-    - lutro
-    - mame
-    - mame0258
-    - mame0278
-    - mame2000
-    - mame2003
-    - mame2003_plus
-    - mame2010
-    - mame2015
-    - mame2016
-    - mamemess
-    - mednafen_lynx
-    - mednafen_ngp
-    - mednafen_pce_fast
-    - mednafen_pcfx
-    - mednafen_psx
-    - mednafen_psx_hw
-    - mednafen_saturn
-    - mednafen_supafaust
-    - mednafen_supergrafx
-    - mednafen_vb
-    - mednafen_wswan
-    - melonds
-    - mesen
-    - mesen_s
-    - meteor
-    - mgba
-    - minivmac
-    - mojozork
-    - moonlight
-    - mrboom
-    - mu
-    - mupen64plus
-    - mupen64plus_next
-    - n64_gles2
-    - neocd
-    - nestopia
-    - np2kai
-    - nxengine
-    - o2em
-    - openbor
-    - openlara
-    - opera
-    - oricutron
-    - parallel_n64
-    - pcsx2
-    - pcsx_rearmed
-    - pico8
-    - picodrive
-    - pisnes
-    - pokemini
-    - potator
-    - ppsspp
-    - prboom
-    - prosystem
-    - ps2
-    - puae
-    - px68k
-    - quasi88
-    - quicknes
-    - race
-    - rb5000
-    - reicast
-    - reminiscence
-    - retro8
-    - retrodream
-    - rice
-    - rice_gles2
-    - same_cdi
-    - sameboy
-    - sameduck
-    - scummvm
-    - sdlpop
-    - simcoupe
-    - snes9x
-    - snes9x2002
-    - snes9x2005
-    - snes9x2010
-    - solarus
-    - stella
-    - stella2014
-    - stonesoup
-    - supermodel
-    - swanstation
-    - tamalibretro
-    - tgbdual
-    - theodore
-    - thepowdertoy
-    - ti99sim
-    - tic80
-    - tyrquake
-    - uae4all
-    - uae4arm
-    - uzem
-    - vecx
-    - vice_x128
-    - vice_x64
-    - vice_x64sc
-    - vice_xcbm2
-    - vice_xcbm5x0
-    - vice_xpet
-    - vice_xplus4
-    - vice_xscpu64
-    - vice_xvic
-    - virtualjaguar
-    - vitaquake2
-    - vitaquake3
-    - vitavoyager
-    - vpinball
-    - vvvvvv
-    - wasm4
-    - x1
-    - x128
-    - x64
-    - x64sx
-    - xcbm2
-    - xcbm5x0
-    - xemu
-    - xpet
-    - xplus4
-    - xrick
-    - xroar
-    - xscpu64
-    - xvic
-    - yabasanshiro
-    - yabause
     install:
       detect:
       - os: linux

platforms/recalbox.yml

@@ -6,112 +6,262 @@ base_destination: bios
 hash_type: md5
 verification_mode: md5
 cores:
+- "2048"
+- "81"
+- a5200
+- advancemame
 - amiberry
+- applewin
+- arduous
+- atari800
+- b2
+- beebem
+- bk
+- bluemsx
+- boom3
+- bsnes
+- bsneshd
+- cannonball
+- cap32
+- cdi2015
+- corsixth
+- craft
+- crocods
+- daphne
+- desmume
+- dice
+- dinothawr
+- dirksimple
 - dolphin
+- dolphin-gui
+- dosbox
+- dosbox_pure
 - duckstation
+- easyrpg
+- ecwolf
+- eightyone
+- emuscv
+- exel-hence
+- fake08
+- fba2x
+- fbneo
+- fceumm
+- flycast
+- flycast-next
+- fmsx
+- freechaf
+- freeintv
+- frotz
+- fuse
+- gambatte
+- gearcoleco
+- geargrafx
+- gearsystem
+- genesisplusgx
+- genesisplusgx_ex
+- genesisplusgxwide
+- geolith
+- glide64mk2
+- gliden64
+- gliden64_20
+- gong
+- gpsp
 - gsplus
+- gw
+- handy
 - hatari
-- libretro-hatari
-- libretro/ mame2003-plus
-- libretro/a5200
-- libretro/amiberry
-- libretro/atari800
-- libretro/bk
-- libretro/bluemsx
-- libretro/bsnes
-- libretro/bsnes-hd
-- libretro/cdi2015
-- libretro/desmume
-- libretro/dolphin
-- libretro/emuscv
-- libretro/fba2x
-- libretro/fbneo
-- libretro/fceumm
-- libretro/flycast
-- libretro/fmsx
-- libretro/freechaf
-- libretro/freeintv
-- libretro/fuse
-- libretro/gambatte
-- libretro/gearcoleco
-- libretro/geargrafx
-- libretro/gearsystem
-- libretro/genesisplusgx
-- libretro/genesisplusgxwide
-- libretro/geolith
-- libretro/gpsp
-- libretro/handy
-- libretro/hatari
-- libretro/kronos
-- libretro/mame2000
-- libretro/mame2003
-- libretro/mame2003-plus
-- libretro/mame2010
-- libretro/mame2015
-- libretro/mednafen_lynx
-- libretro/mednafen_pce_fast
-- libretro/mednafen_psx
-- libretro/mednafen_psx_hw
-- libretro/mednafen_saturn
-- libretro/mednafen_stv
-- libretro/mednafen_supergrafx
-- libretro/melonds
-- libretro/mesen
-- libretro/mesen_s
-- libretro/meteor
-- libretro/mgba
-- libretro/minivmac
-- libretro/mu
-- libretro/mupen64plus_next
-- libretro/neocd
-- libretro/nestopia
-- libretro/np2kai
-- libretro/o2em
-- libretro/opera
-- libretro/parallel_n64
-- libretro/pcfx
-- libretro/pcsx2
-- libretro/pcsx_rearmed
-- libretro/picodrive
-- libretro/pokemini
-- libretro/prosystem
-- libretro/px68k
-- libretro/quasi88
-- libretro/same_cdi
-- libretro/sameboy
-- libretro/scummvm
-- libretro/snes9x
-- libretro/swanstation
-- libretro/uae
-- libretro/uae4all
-- libretro/uae4arm
-- libretro/vice_x128
-- libretro/vice_x64
-- libretro/vice_x64sc
-- libretro/xmil
-- libretro/xscpu64
-- libretro/yabasanshiro
-- libretro/yabause
+- hatarib
+- holani
+- hypseus_singe
+- imageviewer
+- julius
+- kronos
+- lowresnx
+- lutro
+- m2000
+- mame0258
+- mame0278
+- mame2000
+- mame2003
+- mame2003_plus
+- mame2010
+- mame2015
+- mame2016
+- mednafen_lynx
+- mednafen_ngp
+- mednafen_pce_fast
+- mednafen_pcfx
+- mednafen_psx
+- mednafen_psx_hw
+- mednafen_saturn
+- mednafen_stv
+- mednafen_supafaust
+- mednafen_supergrafx
+- mednafen_vb
+- mednafen_wswan
+- melonds
+- mesen
+- mesen_s
+- meteor
+- mgba
+- minivmac
+- mojozork
+- moonlight
+- mrboom
+- mu
+- mupen64plus
+- mupen64plus_next
+- n64_gles2
+- neocd
+- nestopia
+- np2kai
+- nxengine
+- o2em
+- openbor
+- openlara
+- opera
 - oricutron
+- parallel_n64
 - pcsx2
+- pcsx_rearmed
+- pd777
+- pico8
+- picodrive
+- pisnes
+- pokemini
+- potator
+- ppsspp
+- prboom
+- prosystem
+- ps2
+- puae
+- puae2021
+- px68k
+- quasi88
+- quicknes
+- race
+- rb5000
 - reicast
+- reminiscence
+- retro8
+- retrodream
+- rice
+- rice_gles2
+- same_cdi
+- sameboy
+- sameduck
 - scummvm
+- sdlpop
+- simcoupe
+- snes9x
+- snes9x2002
+- snes9x2005
+- snes9x2010
+- solarus
+- stella
+- stella2014
+- stonesoup
+- sugarbox2
+- supermodel
+- swanstation
+- tamalibretro
+- tgbdual
+- theodore
+- thepowdertoy
 - ti99sim
+- tic80
+- tyrquake
+- uae4all
+- uae4arm
+- uzem
+- vecx
+- vice
+- vice_x128
+- vice_x64
+- vice_x64sc
+- vice_xcbm2
+- vice_xcbm5x0
+- vice_xpet
+- vice_xplus4
+- vice_xscpu64
+- vice_xvic
+- vircon32
+- virtualjaguar
+- vitaquake2
+- vitaquake3
+- vitavoyager
+- vpinball
+- vvvvvv
+- wasm4
+- x1
+- x128
+- x64
+- x64sx
+- xcbm2
+- xcbm5x0
 - xemu
+- xpet
+- xplus4
+- xrick
 - xroar
+- xscpu64
+- xvic
+- yabasanshiro
+- yabause
+- ymir
 standalone_cores:
+- advancemame
 - amiberry
+- beebem
+- corsixth
+- daphne
 - dolphin
+- dolphin-gui
+- dosbox
 - duckstation
+- fba2x
+- frotz
+- glide64mk2
+- gliden64
+- gliden64_20
 - gsplus
 - hatari
+- julius
+- moonlight
+- mupen64plus
+- n64_gles2
+- openbor
+- openlara
 - oricutron
 - pcsx2
+- pcsx_rearmed
+- pico8
+- pisnes
+- ppsspp
+- rb5000
 - reicast
+- rice
+- rice_gles2
 - scummvm
+- sdlpop
+- simcoupe
+- solarus
+- supermodel
 - ti99sim
+- vice
+- vpinball
+- vvvvvv
+- x128
+- x64
+- x64sx
+- xcbm2
+- xcbm5x0
 - xemu
+- xpet
+- xplus4
 - xroar
+- xscpu64
+- xvic
 systems:
   commodore-amiga:
     files:

scripts/scraper/recalbox_scraper.py

@@ -99,11 +99,58 @@ SYSTEM_SLUG_MAP = {
 }
 
 
-def split_cores(names) -> tuple[list[str], list[str]]:
-    """(every core name, the standalone ones) from es_bios.xml ``core`` values."""
-    cores = {name for name in names if name}
-    standalone = {name for name in cores if not name.startswith("libretro")}
-    return sorted(cores), sorted(standalone)
+SYSTEMS_TREE_URL = (
+    "https://gitlab.com/api/v4/projects/recalbox%2Frecalbox/repository/tree"
+    "?path=package/recalbox-romfs2/systems&ref={tag}&per_page=100&page={page}"
+)
+SYSTEM_INI_URL = (
+    "https://gitlab.com/recalbox/recalbox/-/raw/{tag}"
+    "/package/recalbox-romfs2/systems/{system}/system.ini"
+)
+
+
+def system_ini_cores(text: str) -> tuple[set[str], set[str]]:
+    """The cores one Recalbox system.ini offers, and those run standalone.
+
+    Each ``[core.N]`` section (or its board override ``[core.N.<board>]``)
+    names an ``emulator`` and a ``core``: ``libretro`` with the core's name,
+    or a standalone emulator. es_bios.xml only names the cores that need a
+    BIOS, so read alone it left most of Recalbox's cores outside the pack.
+    """
+    import re
+
+    # Recalbox's .ini files are not strict INI (a value can run on a bare
+    # line), so only sections and key = value lines are read.
+    sections: dict[str, dict[str, str]] = {}
+    current: dict[str, str] | None = None
+    for raw in text.splitlines():
+        line = raw.strip()
+        if not line or line.startswith((";", "#")):
+            continue
+        header = re.fullmatch(r"\[([^\]]+)\]", line)
+        if header:
+            current = sections.setdefault(header.group(1).strip(), {})
+            continue
+        if current is not None and "=" in line:
+            key, value = line.split("=", 1)
+            current[key.strip()] = value.strip().strip('"')
+    cores: set[str] = set()
+    standalone: set[str] = set()
+    for section, values in sections.items():
+        if not re.fullmatch(r"core\.\d+(\.[\w-]+)?", section):
+            continue
+        emulator = values.get("emulator", "")
+        core = values.get("core", "")
+        if emulator == "libretro":
+            if core:
+                cores.add(core)
+        elif emulator:
+            cores.add(emulator)
+            standalone.add(emulator)
+            if core and core != emulator:
+                cores.add(core)
+                standalone.add(core)
+    return cores, standalone
 
 
 class Scraper(BaseScraper):
@@ -114,20 +161,46 @@ class Scraper(BaseScraper):
         super().__init__(url=SOURCE_URL.format(tag=self.tag))
 
     def _fetch_cores(self) -> tuple[list[str], list[str]]:
-        """Core names from es_bios.xml, and those Recalbox runs standalone.
+        """Every core Recalbox's systems offer, and those it runs standalone.
 
-        The ``core`` attribute writes a libretro core as ``libretro/<name>``
-        and a standalone emulator by its bare name (``dolphin``, ``pcsx2``,
-        ``xemu``), so the prefix is the build mode. One entry spells
-        ``libretro-hatari``; a name that starts with ``libretro`` is a core.
+        Read from each system.ini of recalbox-romfs2 at the scraped tag: the
+        systems ES lists and the cores a user can pick for each.
         """
-        raw = self._fetch_raw()
-        root = parse_untrusted_xml(raw, "es_bios.xml")
-        return split_cores(
-            part.strip()
-            for bios_elem in root.findall(".//system/bios")
-            for part in bios_elem.get("core", "").split(",")
-        )
+        import json
+        import urllib.error
+        import urllib.request
+
+        def get(url: str) -> bytes:
+            req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
+            try:
+                with urllib.request.urlopen(req, timeout=30) as resp:
+                    return resp.read()
+            except urllib.error.URLError as e:
+                raise ConnectionError(f"Failed to fetch {url}: {e}") from e
+
+        systems: list[str] = []
+        page = 1
+        while True:
+            listing = json.loads(get(SYSTEMS_TREE_URL.format(tag=self.tag, page=page)))
+            systems += [
+                item["name"] for item in listing
+                if item.get("type") == "tree" and not item["name"].startswith(".")
+            ]
+            if len(listing) < 100:
+                break
+            page += 1
+        if not systems:
+            raise RuntimeError(f"no Recalbox system found at {self.tag}")
+        cores: set[str] = set()
+        standalone: set[str] = set()
+        for system in systems:
+            text = get(SYSTEM_INI_URL.format(tag=self.tag, system=system)).decode("utf-8")
+            found, alone = system_ini_cores(text)
+            cores |= found
+            standalone |= alone
+        if not cores:
+            raise RuntimeError(f"no core in Recalbox's system.ini files at {self.tag}")
+        return sorted(cores), sorted(standalone)
 
     def fetch_requirements(self) -> list[BiosRequirement]:
         """Parse es_bios.xml and return BIOS requirements."""

tests/test_scraper_contract.py

@@ -219,23 +219,45 @@ class RetroPiePackageList(unittest.TestCase):
 
 
 class RecalboxBuildMode(unittest.TestCase):
-    """es_bios.xml names a libretro core as libretro/<name> and a standalone
-    emulator by its bare name: the prefix is the build mode, and a platform
-    that runs an emulator standalone must say so or the pack lays its files
-    out for the libretro build."""
+    """Recalbox's cores come from each system.ini of recalbox-romfs2: a
+    `[core.N]` section names `emulator = "libretro"` with the core, or a
+    standalone emulator. es_bios.xml only names the cores that need a BIOS,
+    so read alone it left most of Recalbox's cores, mame0258 and mame0278
+    among them, outside the pack."""
 
-    def test_bare_names_are_the_standalone_emulators(self):
-        from scraper.recalbox_scraper import split_cores
+    SYSTEM_INI = """\
+[system]
+name = "mame"
+fullname = "MAME"
+  extra line the parser must survive.
 
-        cores, standalone = split_cores(
-            ["libretro/uae", "amiberry", "", "dolphin", "libretro/dolphin", "libretro-hatari", "hatari"]
-        )
-        self.assertEqual(
-            cores,
-            ["amiberry", "dolphin", "hatari", "libretro-hatari", "libretro/dolphin", "libretro/uae"],
-        )
-        self.assertEqual(standalone, ["amiberry", "dolphin", "hatari"])
+[core.0]
+package = BR2_PACKAGE_LIBRETRO_MAME0278
+emulator = "libretro"
+core = "mame0278"
 
+[core.1]
+emulator = "advancemame"
+core = "advancemame"
+
+[core.2.rpi4]
+emulator = "libretro"
+core = "mame2003_plus"
+"""
+
+    def test_libretro_cores_and_standalone_emulators(self):
+        from scraper.recalbox_scraper import system_ini_cores
+
+        cores, standalone = system_ini_cores(self.SYSTEM_INI)
+        self.assertEqual(cores, {"mame0278", "advancemame", "mame2003_plus"})
+        self.assertEqual(standalone, {"advancemame"})
+
+    def test_the_registry_leaves_the_list_to_the_scraper(self):
+        import yaml
+
+        root = Path(__file__).resolve().parents[1]
+        registry = yaml.safe_load((root / "platforms" / "_registry.yml").read_text())
+        self.assertNotIn("cores", registry["platforms"]["recalbox"])
 
 
 class UnreadableReleaseStopsTheScrape(unittest.TestCase):