feat: pack every core batocera offers in es

Batocera's cores were read from configgen-defaults.yml, one per system,
and a hand-kept registry list overrode even that; DOSBox Staging,
melonDS DS and nine other cores a user can pick in ES were outside the
pack, while mamemess, which Batocera does not ship, was in. The
scraper now reads es_systems.yml, the registry list is gone, and the
target scraper names standalone emulators (amiberry, fsuae, vice)
instead of only the machines listed under them.

platforms/_registry.yml

@@ -190,213 +190,6 @@ platforms:
     source_format: python_dict
     hash_type: md5
     schedule: weekly
-    cores:
-    - '81'
-    - a5200
-    - abuse
-    - amiberry
-    - applewin
-    - arduous
-    - atari800
-    - azahar
-    - beetle-saturn
-    - bennugd
-    - bigpemu
-    - bk
-    - blastem
-    - bluemsx
-    - boom3
-    - bsnes
-    - bstone
-    - cannonball
-    - cap32
-    - catacombgl
-    - cdogs
-    - cemu
-    - cgenius
-    - citron
-    - clk
-    - corsixth
-    - demul
-    - desmume
-    - devilutionx
-    - dhewm3
-    - dice
-    - dolphin
-    - dosbox
-    - dosbox_pure
-    - duckstation
-    - dxx-rebirth
-    - easyrpg
-    - ecwolf
-    - eduke32
-    - eka2l1
-    - emuscv
-    - ep128emu-core
-    - etlegacy
-    - fake08
-    - fallout1-ce
-    - fallout2-ce
-    - fbneo
-    - fceumm
-    - flatpak
-    - flycast
-    - fmsx
-    - freechaf
-    - freeintv
-    - freej2me
-    - fsuae
-    - fury
-    - fuse
-    - gambatte
-    - gearcoleco
-    - gearsystem
-    - genesisplusgx
-    - glide64mk2
-    - gong
-    - gpsp
-    - gsplus
-    - gw
-    - gzdoom
-    - handy
-    - hatari
-    - hcl
-    - holani
-    - hurrican
-    - hypseus-singe
-    - ikemen
-    - ioquake3
-    - iortcw
-    - jazz2-native
-    - kronos
-    - lindbergh-loader
-    - lowresnx
-    - lutro
-    - mame
-    - mame078plus
-    - mamemess
-    - mednafen_lynx
-    - mednafen_ngp
-    - mednafen_psx
-    - mednafen_supergrafx
-    - mednafen_wswan
-    - melonds
-    - mesen
-    - mesen-s
-    - mgba
-    - minivmac
-    - model2emu
-    - moonlight
-    - mrboom
-    - mupen64plus-next
-    - neocd
-    - nestopia
-    - np2kai
-    - nxengine
-    - o2em
-    - odcommander
-    - openbor6412
-    - openjazz
-    - openjk
-    - openjkdf2
-    - openmohaa
-    - openmsx
-    - opera
-    - parallel_n64
-    - pce_fast
-    - pcfx
-    - pcsx2
-    - pcsx_rearmed
-    - pd777
-    - picodrive
-    - play
-    - pokemini
-    - potator
-    - ppsspp
-    - prboom
-    - prosystem
-    - puae
-    - puae2021
-    - px68k
-    - pygame
-    - pyxel
-    - quasi88
-    - raze
-    - redream
-    - reminiscence
-    - rpcs3
-    - ruffle
-    - ryujinx
-    - samcoupe
-    - same_cdi
-    - sameduck
-    - scummvm
-    - sdlpop
-    - sh
-    - shadps4
-    - smsplus
-    - snes9x
-    - solarus
-    - sonic-mania
-    - sonic2013
-    - sonic3-air
-    - squirreljme
-    - steam
-    - stella
-    - stella2014
-    - superbroswar
-    - supermodel
-    - swanstation
-    - taradino
-    - tgbdual
-    - theforceengine
-    - theodore
-    - thextech
-    - tic80
-    - tr1x
-    - tr2x
-    - tsugaru
-    - tyrian
-    - tyrquake
-    - uae4arm
-    - uqm
-    - uzem
-    - vb
-    - vba-m
-    - vecx
-    - vemulator
-    - vice
-    - vice_x128
-    - vice_x64
-    - vice_x64sc
-    - vice_xpet
-    - vice_xplus4
-    - vice_xscpu64
-    - vice_xvic
-    - vircon32
-    - virtualjaguar
-    - vita3k
-    - vitaquake2
-    - vox_official
-    - vpinball
-    - wasm4
-    - wine-tkg
-    - x1
-    - x128
-    - x16emu
-    - xash3d_fwgs
-    - xemu
-    - xenia
-    - xenia-canary
-    - xpet
-    - xplus4
-    - xrick
-    - xroar
-    - xvic
-    - yabasanshiro
-    - ymir
-    - yquake2
-    - zc210
     target_scraper: batocera_targets
     target_source: https://github.com/batocera-linux/batocera.linux
     install:

platforms/batocera.yml

@@ -9,13 +9,20 @@ cores:
 - "81"
 - a5200
 - abuse
+- amiberry
+- applewin
 - arduous
 - atari800
 - azahar
+- beetle-saturn
 - bennugd
+- bigpemu
 - bk
+- blastem
 - bluemsx
+- boom3
 - bsnes
+- bsnes_hd
 - bstone
 - cannonball
 - cap32
@@ -27,44 +34,67 @@ cores:
 - clk
 - corsixth
 - demul
+- desmume
 - devilutionx
 - dhewm3
 - dice
 - dolphin
+- dosbox
+- dosbox-x
 - dosbox_pure
+- dosbox_staging
+- dosboxx
+- duckstation
+- duckstation-legacy
 - dxx-rebirth
 - easyrpg
 - ecwolf
 - eduke32
 - eka2l1
 - emuscv
+- ep128emu-core
 - etlegacy
 - fake08
 - fallout1-ce
 - fallout2-ce
+- fba2x
+- fbalpha
 - fbneo
 - fceumm
-- flatpak
 - flycast
+- flycastvl
+- fmsx
 - freechaf
 - freeintv
+- freej2me
+- fsuae
 - fury
 - fuse
 - gambatte
+- gearcoleco
 - gearsystem
 - genesisplusgx
-- glide64mk2
+- genesisplusgx-expanded
+- genesisplusgx-wide
 - gong
+- gpsp
 - gsplus
 - gw
 - gzdoom
+- handy
 - hatari
+- hatarib
 - hcl
+- holani
 - hurrican
 - hypseus-singe
 - ikemen
+- imame4all
 - ioquake3
+- iortcw
 - jazz2-native
+- kronos
+- lightspark
 - lindbergh-loader
 - lowresnx
 - lutro
@@ -72,25 +102,39 @@ cores:
 - mame078plus
 - mednafen_lynx
 - mednafen_ngp
+- mednafen_psx
 - mednafen_supergrafx
 - mednafen_wswan
 - melonds
+- melondsds
+- mesen
+- mesen-s
 - mgba
 - minivmac
 - model2emu
 - moonlight
 - mrboom
+- mugen
+- mupen64plus
+- mupen64plus-next
 - neocd
+- nestopia
 - np2kai
 - nxengine
 - o2em
-- odcommander
+- openbor
+- openbor4432
 - openbor6412
+- openbor7142
+- openbor7530
 - openjazz
 - openjk
 - openjkdf2
 - openmohaa
+- openmsx
 - opera
+- parallel_n64
+- pce
 - pce_fast
 - pcfx
 - pcsx2
@@ -98,35 +142,45 @@ cores:
 - pd777
 - picodrive
 - play
+- pocketsnes
 - pokemini
 - potator
 - ppsspp
 - prboom
 - prosystem
 - puae
+- puae2021
 - px68k
 - pygame
 - pyxel
 - quasi88
 - raze
+- redream
 - reminiscence
 - rpcs3
 - ruffle
+- ryujinx
 - samcoupe
+- same_cdi
 - sameduck
 - scummvm
 - sdlpop
-- sh
 - shadps4
+- smsplus
 - snes9x
+- snes9x_next
 - solarus
 - sonic-mania
 - sonic2013
 - sonic3-air
-- steam
+- soniccd
+- squirreljme
 - stella
+- stella2014
 - superbroswar
 - supermodel
+- supermodel-legacy
+- swanstation
 - taradino
 - tgbdual
 - theforceengine
@@ -138,29 +192,49 @@ cores:
 - tsugaru
 - tyrian
 - tyrquake
+- uae4arm
 - uqm
 - uzem
 - vb
+- vba-m
 - vecx
+- vemulator
+- vice
+- vice_x128
 - vice_x64
+- vice_x64sc
+- vice_xpet
+- vice_xplus4
+- vice_xscpu64
+- vice_xvic
 - vircon32
 - virtualjaguar
 - vita3k
+- vitaquake2
+- vkquake
+- vkquake2
+- vkquake3
 - vox_official
 - vpinball
 - wasm4
-- wine-tkg
+- wine
 - x1
 - x128
 - x16emu
+- x64
+- x64dtv
 - xash3d_fwgs
 - xemu
+- xenia
 - xenia-canary
 - xpet
 - xplus4
 - xrick
+- xroar
+- xscpu64
 - xvic
 - yabasanshiro
+- ymir
 - yquake2
 - zc210
 systems:
@@ -5618,7 +5692,10 @@ systems:
     name: Elektronika BK
 standalone_cores:
 - abuse
+- amiberry
+- applewin
 - azahar
+- bigpemu
 - bstone
 - cannonball
 - catacombgl
@@ -5632,6 +5709,12 @@ standalone_cores:
 - devilutionx
 - dhewm3
 - dolphin
+- dosbox
+- dosbox-x
+- dosbox_staging
+- dosboxx
+- duckstation
+- duckstation-legacy
 - dxx-rebirth
 - easyrpg
 - ecwolf
@@ -5640,11 +5723,13 @@ standalone_cores:
 - etlegacy
 - fallout1-ce
 - fallout2-ce
-- flatpak
+- fba2x
+- flycast
+- fsuae
 - fury
-- glide64mk2
 - gsplus
 - gzdoom
+- hatari
 - hcl
 - hurrican
 - hypseus-singe
@@ -5654,41 +5739,50 @@ standalone_cores:
 - jazz2-native
 - kodi
 - lexaloffle
+- lightspark
 - lindbergh-loader
 - mame
+- melonds
 - model2emu
 - moonlight
 - mugen
 - mupen64plus
-- odcommander
 - openbor
+- openbor4432
 - openbor6412
+- openbor7142
+- openbor7530
 - openjazz
 - openjk
 - openjkdf2
 - openmohaa
+- openmsx
 - pcsx2
 - play
 - ppsspp
 - pygame
 - pyxel
 - raze
+- redream
 - rpcs3
 - ruffle
+- ryujinx
 - samcoupe
 - scummvm
 - sdlpop
-- sh
 - shadps4
 - solarus
 - sonic-mania
 - sonic2013
 - sonic3-air
-- steam
+- soniccd
+- stella
 - supermodel
+- supermodel-legacy
 - taradino
 - theforceengine
 - thextech
+- tic80
 - tr1x
 - tr2x
 - tsugaru
@@ -5696,16 +5790,24 @@ standalone_cores:
 - uqm
 - vice
 - vita3k
+- vkquake
+- vkquake2
+- vkquake3
 - vox_official
 - vpinball
 - wine
-- wine-tkg
 - x128
 - x16emu
+- x64
+- x64dtv
 - xash3d_fwgs
 - xemu
+- xenia
 - xenia-canary
 - xpet
 - xplus4
+- xroar
+- xscpu64
 - xvic
+- ymir
 - yquake2

scripts/scraper/batocera_scraper.py

@@ -73,6 +73,59 @@ CONFIGGEN_DEFAULTS_URL = (
     "configgen-defaults.yml"
 )
 
+ES_SYSTEMS_URL = (
+    _RAW_BASE + "/{tag}"
+    "/package/batocera/emulationstation/batocera-es-system/es_systems.yml"
+)
+
+# Names es_systems.yml files under an emulator that are not emulators:
+# machine selectors, video plugins, launchers. Read from the overrides the
+# target scraper applies, so one list says what is not a core.
+OVERRIDES_PATH = Path(__file__).resolve().parents[2] / "platforms" / "targets" / "_overrides.yml"
+
+
+def _not_cores() -> set[str]:
+    data = yaml_load(OVERRIDES_PATH.read_text(encoding="utf-8")) or {}
+    default = (data.get("batocera", {}).get("targets", {}) or {}).get("_default", {}) or {}
+    return {str(name) for name in default.get("remove_cores", [])}
+
+
+def es_systems_emulators(text: str) -> tuple[set[str], set[str]]:
+    """Every core es_systems.yml offers, and those that run standalone.
+
+    ES lists, per system, each emulator a user may pick and the cores under
+    it; configgen-defaults.yml only names the one picked by default, so
+    reading it alone left DOSBox Staging, melonDS DS and nine other built
+    cores outside the pack.
+    """
+    data = yaml_load(text)
+    if not isinstance(data, dict):
+        raise ValueError("es_systems.yml is not a mapping")
+    cores: set[str] = set()
+    standalone: set[str] = set()
+    for system in data.values():
+        emulators = system.get("emulators") if isinstance(system, dict) else None
+        if not isinstance(emulators, dict):
+            continue
+        for emulator, entries in emulators.items():
+            if not isinstance(entries, dict):
+                continue
+            # A standalone emulator is its own core; what it lists under it
+            # are machines or builds, and the non-cores among them are named
+            # in the overrides. Keys whose value is not a mapping
+            # (archs_include) describe the entry, not a core.
+            if emulator != "libretro":
+                cores.add(str(emulator))
+                standalone.add(str(emulator))
+            for core, spec in entries.items():
+                if not isinstance(spec, dict):
+                    continue
+                cores.add(str(core))
+                if emulator != "libretro":
+                    standalone.add(str(core))
+    return cores, standalone
+
+
 SYSTEM_SLUG_MAP = {
     "atari800": "atari-400-800",
     "atari5200": "atari-5200",
@@ -207,8 +260,11 @@ class Scraper(BaseScraper):
                 f"Failed to fetch {url}: {e}"
             ) from e
         data = yaml_load(raw)
-        cores: set[str] = set()
-        standalone: set[str] = set()
+        cores, standalone = es_systems_emulators(
+            self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag))
+        )
+        if not cores:
+            raise RuntimeError("es_systems.yml names no emulator")
         for system, cfg in data.items():
             if system == "default" or not isinstance(cfg, dict):
                 continue
@@ -220,7 +276,16 @@ class Scraper(BaseScraper):
                 standalone.add(emulator)
                 if core and core != emulator:
                     standalone.add(core)
-        return sorted(cores), sorted(standalone)
+        dropped = _not_cores()
+        return sorted(cores - dropped), sorted(standalone - dropped)
+
+    def _fetch_text(self, url: str) -> str:
+        req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                return resp.read().decode("utf-8")
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"Failed to fetch {url}: {e}") from e
 
     def _extract_systems_dict(self, raw: str) -> dict:
         """Extract and parse the 'systems' dict from the Python source via ast.literal_eval."""

scripts/scraper/targets/batocera_targets_scraper.py

@@ -228,7 +228,7 @@ def _parse_es_systems(text: str) -> dict[str, list[str]]:
         emulators = system_data.get("emulators")
         if not isinstance(emulators, dict):
             continue
-        for _group_name, group_data in emulators.items():
+        for group_name, group_data in emulators.items():
             if not isinstance(group_data, dict):
                 continue
             for core_name, core_data in group_data.items():
@@ -237,9 +237,13 @@ def _parse_es_systems(text: str) -> dict[str, list[str]]:
                 require = core_data.get("requireAnyOf", [])
                 if not isinstance(require, list):
                     continue
+                # A standalone emulator is its own core: amiberry, fsuae and
+                # vice list machines under their name, and the package that
+                # builds a machine builds the emulator.
+                names = [core_name] if group_name == "libretro" else [group_name, core_name]
                 for pkg_flag in require:
                     if isinstance(pkg_flag, str):
-                        package_to_emulators.setdefault(pkg_flag, []).append(core_name)
+                        package_to_emulators.setdefault(pkg_flag, []).extend(names)
 
     return package_to_emulators
 

tests/test_batocera_cores.py

@@ -0,0 +1,70 @@
+"""Batocera runs every emulator its ES lists, not only each system's default.
+
+The platform scraper read configgen-defaults.yml alone, one core per system,
+so DOSBox Staging, melonDS DS and nine other cores Batocera builds and lets a
+user pick stayed outside the pack. The target scraper read es_systems.yml but
+dropped a standalone emulator's own name, keeping only the machines listed
+under it: amiberry, fsuae and vice were never in a target list.
+"""
+
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT))
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+ES_SYSTEMS = """\
+dos:
+  emulators:
+    libretro:
+      dosbox_pure: { requireAnyOf: [BR2_PACKAGE_LIBRETRO_DOSBOX_PURE] }
+      dosbox_staging: { requireAnyOf: [BR2_PACKAGE_LIBRETRO_DOSBOX_STAGING] }
+    dosbox:
+      dosbox: { requireAnyOf: [BR2_PACKAGE_DOSBOX] }
+amiga500:
+  emulators:
+    amiberry:
+      A500: { requireAnyOf: [BR2_PACKAGE_AMIBERRY] }
+      archs_include: [x86_64]
+"""
+
+
+class ThePlatformListsEveryOfferedCore(unittest.TestCase):
+    def test_libretro_cores_and_standalone_emulators(self):
+        from scripts.scraper.batocera_scraper import es_systems_emulators
+
+        cores, standalone = es_systems_emulators(ES_SYSTEMS)
+        self.assertEqual(
+            cores, {"dosbox_pure", "dosbox_staging", "dosbox", "amiberry", "A500"}
+        )
+        self.assertEqual(standalone, {"dosbox", "amiberry", "A500"})
+
+    def test_the_overrides_drop_the_machine_selectors(self):
+        from scripts.scraper.batocera_scraper import _not_cores
+
+        self.assertIn("A500", _not_cores())
+
+    def test_the_registry_leaves_the_list_to_the_scraper(self):
+        import yaml
+
+        registry = yaml.safe_load((REPO_ROOT / "platforms" / "_registry.yml").read_text())
+        self.assertNotIn("cores", registry["platforms"]["batocera"])
+        platform = yaml.safe_load((REPO_ROOT / "platforms" / "batocera.yml").read_text())
+        self.assertIn("dosbox_staging", platform["cores"])
+
+
+class TheTargetsNameTheStandaloneEmulator(unittest.TestCase):
+    def test_the_group_name_is_a_core(self):
+        from scripts.scraper.targets.batocera_targets_scraper import _parse_es_systems
+
+        mapping = _parse_es_systems(ES_SYSTEMS)
+        self.assertEqual(mapping["BR2_PACKAGE_AMIBERRY"], ["amiberry", "A500"])
+        self.assertEqual(mapping["BR2_PACKAGE_LIBRETRO_DOSBOX_STAGING"], ["dosbox_staging"])
+
+
+if __name__ == "__main__":
+    unittest.main()