chore: merge the elec core names

scripts/scraper/amberelec_scraper.py

@@ -37,11 +37,12 @@ from .batocera_scraper import (
     SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
 )
 from .batocera_scraper import (
+    _not_cores,
     extract_systems_dict,
     platform_systems,
     requirements_from_systems,
 )
-from .es_emulators import es_cores, xml_emulators
+from .es_emulators import attach_system_cores, es_cores, system_cores, xml_systems
 
 PLATFORM_NAME = "amberelec"
 
@@ -65,9 +66,14 @@ SYSTEM_SLUG_MAP = {
 # names. `mame` lists MAME machines and runs them on the libretro MAME core
 # (lines 233-261 set self.core = 'mame', line 355 builds the RetroArch command).
 RUNNERS: dict[str, str | None] = {"retroarch": None, "retrorun": None, "mame": "mame"}
-# runemu.py:113, PPSSPPSDL runs /usr/bin/ppsspp.sh, installed by the
-# ppssppsa package (packages/games/emulators/ppssppsa).
-RENAMED = {"PPSSPPSDL": "ppsspp"}
+# runemu.py:110-128 starts every other emulator through the script its
+# standalone_emulators table names, keyed by the emulator name es_systems.cfg
+# writes. Three are the standalone build of an emulator the list also offers
+# as a libretro core, the package's PKG_SITE naming the same project:
+# hatarisa hatari/hatari (packages/games/emulators/hatarisa), ppssppsa
+# hrydgard/ppsspp (PPSSPPSDL runs its ppsspp.sh, line 116) and scummvmsa
+# scummvm/scummvm.
+BUILDS_OF = {"HATARISA": "hatari", "PPSSPPSDL": "ppsspp", "SCUMMVMSA": "scummvm"}
 
 _PRERELEASE_COMMIT = re.compile(r"AmberELEC/AmberELEC@([0-9a-f]{40})")
 
@@ -128,11 +134,12 @@ class Scraper(BaseScraper):
         systems = platform_systems(
             self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
         )
-        cores, standalone, dual = es_cores(
-            xml_emulators(self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.commit))),
-            RUNNERS,
-            RENAMED,
-        )
+        # es_systems.cfg gives each <system> its <emulators> in menu order
+        # (atarist, lines 267-288: retroarch's hatari, then HATARISA).
+        es_systems = xml_systems(self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.commit)))
+        not_cores = _not_cores(PLATFORM_NAME)
+        cores, standalone, dual = es_cores(es_systems, RUNNERS, BUILDS_OF, not_cores)
+        attach_system_cores(systems, system_cores(es_systems, RUNNERS, not_cores))
         return {
             "platform": "AmberELEC",
             "version": self.tag,

scripts/scraper/batocera_scraper.py

@@ -142,6 +142,44 @@ def es_systems_libretro(text: str) -> set[str]:
     return found
 
 
+def es_systems_by_system(
+    es_systems_text: str, defaults_text: str, dropped: set[str]
+) -> dict[str, list[str]]:
+    """system -> the cores es_systems.yml offers it, in the file's order.
+
+    Read as configgen_cores reads the whole list: a core under `libretro:`
+    is a libretro core, any other emulator is its own core and so is each
+    build it lists. The core configgen-defaults.yml starts by default comes
+    last when es_systems.yml does not offer it.
+    """
+    data = yaml_load_upstream(es_systems_text)
+    defaults = yaml_load_upstream(defaults_text)
+    if not isinstance(data, dict) or not isinstance(defaults, dict):
+        raise ValueError("es_systems.yml or configgen-defaults.yml is not a mapping")
+    result: dict[str, list[str]] = {}
+    for system, spec in data.items():
+        emulators = spec.get("emulators") if isinstance(spec, dict) else None
+        names: list[str] = []
+        if isinstance(emulators, dict):
+            for emulator, entries in emulators.items():
+                if not isinstance(entries, dict):
+                    continue
+                if emulator != "libretro":
+                    names.append(str(emulator))
+                names.extend(str(c) for c, s in entries.items() if isinstance(s, dict))
+        default = defaults.get(system)
+        if isinstance(default, dict):
+            emulator, core = default.get("emulator"), default.get("core")
+            if emulator and emulator != "libretro":
+                names.append(str(emulator))
+            if core:
+                names.append(str(core))
+        names = [n for n in dict.fromkeys(names) if n not in dropped]
+        if names:
+            result[str(system)] = names
+    return result
+
+
 SYSTEM_SLUG_MAP = {
     "advision": "entex-advision",
     "amiga500": "commodore-amiga",

scripts/scraper/emuelec_scraper.py

@@ -27,19 +27,29 @@ Verification (batocera-systems at the release tag):
 from __future__ import annotations
 
 import re
+import shlex
 import urllib.error
 import urllib.request
 
+from safeparse import yaml_load_upstream
+
 from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
 from .batocera_scraper import (
     SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
 )
 from .batocera_scraper import (
+    _not_cores,
     extract_systems_dict,
     platform_systems,
     requirements_from_systems,
 )
-from .es_emulators import es_cores, json_emulators
+from .es_emulators import (
+    SystemEmulators,
+    attach_system_cores,
+    es_cores,
+    json_systems,
+    system_cores,
+)
 
 PLATFORM_NAME = "emuelec"
 
@@ -47,12 +57,14 @@ REPO = "EmuELEC/EmuELEC"
 _RAW = "https://raw.githubusercontent.com/EmuELEC/EmuELEC/{ref}/"
 SOURCE_TEMPLATE = _RAW + "packages/sx05re/emuelec/bin/batocera/batocera-systems"
 ES_SYSTEMS_TEMPLATE = _RAW + "packages/sx05re/emuelec-emulationstation/config/es_systems.json"
+PORTS_TEMPLATE = _RAW + "packages/sx05re/emuelec-ports/sources/ports.yaml"
 
 # Batocera's names, plus the keys EmuELEC writes that Batocera does not:
 # console keys under the slug the other scrapers give them, and each port
 # under the system of the profile of the engine its name gives
 # (devilutionx, prboom, opentyrian, cannonball, nxengine, tyrquake,
-# reminiscence, xrick, vvvvvv, eduke32, sonic2013, sonic-mania, dinothawr).
+# reminiscence, xrick, vvvvvv, eduke32, sonic2013, sonic-mania, dinothawr,
+# rigelengine).
 SYSTEM_SLUG_MAP = {
     **BATOCERA_SLUGS,
     "atari7800": "atari-7800",
@@ -75,14 +87,77 @@ SYSTEM_SLUG_MAP = {
     "sonic1": "sonic-1-2013",
     "sonic2": "sonic-2-2013",
     "sonicmania": "sonic-mania",
+    "RigelEngine": "duke-nukem-2",
 }
 
 # es_systems.json names libretro cores under `libretro`, and `retrorun`
-# loads the libretro core it lists.
+# loads the libretro core it lists. emuelecRunEmu.sh:101-116 turns either
+# into ${CORE}_libretro. Any other emulator runs the core it names
+# (EMU="${CORE}", line 107) through the case at lines 164-353.
 RUNNERS: dict[str, str | None] = {"libretro": None, "retrorun": None}
-# packages/sx05re/emulators/PPSSPPSDL builds PPSSPP's SDL front end.
-RENAMED = {"PPSSPPSDL": "ppsspp"}
 
+# A standalone emulator and the libretro core of the same emulator, from the
+# packages each one builds (packages/sx05re/emulators/<pkg>/package.mk
+# PKG_SITE): hatarisa hatari/hatari, stellasa stella-emu/stella, fbneoSA
+# finalburnneo/FBNeo, flycastsa flyinghead/flycast, scummvmsa
+# scummvm/scummvm, PPSSPPSDL hrydgard/ppsspp.
+BUILDS_OF = {
+    "HATARISA": "hatari",
+    "STELLASA": "stella",
+    "FBNEOSA": "fbneo",
+    "flycastsa": "flycast",
+    "SCUMMVMSA": "scummvm",
+    "PPSSPPSDL": "ppsspp",
+}
+
+# The ES system `ports` (es_systems.json:2570-2590) runs one generated
+# script per entry of ports.yaml through emuelecRunApp.sh:22, and its single
+# emulator `ports` is that dispatch, not a program. The scripts are rendered
+# from port.sh.j2 by port_builder.py (emuelec-ports/package.mk:62-64): each
+# runs its `commands`, so the engine is the libretro core a command hands
+# emuelecRunEmu.sh with -C (lines 367-369 load <core>_libretro.so), or the
+# program `port_exe` names.
+PORTS_SYSTEM = "ports"
+
+
+def port_engines(text: str) -> list[tuple[str, str, str | None, bool]]:
+    """(port, engine, check_bios key, libretro?) for each entry of ports.yaml.
+
+    The engine is written as the OS writes it: the libretro core a command
+    hands emuelecRunEmu.sh with -C, else `port_exe`. `check_bios` is the
+    batocera-systems key the script hands ee_check_bios on a failed start
+    (port.sh.j2:23-25), the key the port's files are listed under.
+    """
+    data = yaml_load_upstream(text)
+    if not isinstance(data, dict) or not data:
+        raise ValueError("ports.yaml names no port")
+    engines = []
+    for port, spec in data.items():
+        core = None
+        for command in map(str, spec.get("commands") or []):
+            if "emuelecRunEmu.sh" in command:
+                core = next(
+                    (w[2:] for w in shlex.split(command) if w.startswith("-C")), core
+                )
+        engine = core or spec.get("port_exe")
+        if not engine:
+            raise ValueError(f"ports.yaml: {port} names neither a core nor port_exe")
+        engines.append((str(port), str(engine), spec.get("check_bios"), core is not None))
+    return engines
+
+
+def with_ports(
+    systems: SystemEmulators, engines: list[tuple[str, str, str | None, bool]]
+) -> SystemEmulators:
+    """The ES systems, each port engine filed under `ports` and its own key."""
+    def entry(engine: str, libretro: bool) -> tuple[str, list[str]]:
+        return ("libretro", [engine]) if libretro else (engine, [engine])
+
+    result = {**systems, PORTS_SYSTEM: [entry(e, lr) for _, e, _, lr in engines]}
+    for _, engine, key, libretro in engines:
+        if key:
+            result[key] = [*result.get(key, []), entry(engine, libretro)]
+    return result
 
 class Scraper(BaseScraper):
     """Scraper for EmuELEC's batocera-systems."""
@@ -115,11 +190,16 @@ class Scraper(BaseScraper):
         systems = platform_systems(
             self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
         )
-        cores, standalone, dual = es_cores(
-            json_emulators(self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))),
-            RUNNERS,
-            RENAMED,
+        # es_systems.json gives each system its emulators in menu order (psx,
+        # lines 3610-3660: libretro pcsx_rearmed_32b, pcsx_rearmed,
+        # swanstation, then Duckstation).
+        es_systems = with_ports(
+            json_systems(self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))),
+            port_engines(self._fetch_text(PORTS_TEMPLATE.format(ref=self.tag))),
         )
+        not_cores = _not_cores(PLATFORM_NAME)
+        cores, standalone, dual = es_cores(es_systems, RUNNERS, BUILDS_OF, not_cores)
+        attach_system_cores(systems, system_cores(es_systems, RUNNERS, not_cores))
         return {
             "platform": "EmuELEC",
             "version": self.tag.removeprefix("v"),

scripts/scraper/es_emulators.py

@@ -1,88 +1,161 @@
-"""The emulators an EmulationStation system list offers, by profile names.
+"""The emulators an EmulationStation system list offers, by system.
 
 AmberELEC writes its list as es_systems.cfg XML, EmuELEC as an es_systems.json
-it turns into that XML at build time. Both name, per system, the emulators a
-user may pick and the cores under each. Some emulators are libretro front
-ends (RetroArch, retrorun) whose cores are libretro core names. Every other
-emulator is a standalone program named by the OS (`HATARISA`, `PPSSPPSDL`).
+it turns into that XML at build time. Both name, per system and in the order
+the menu shows them, the emulators a user may pick and the cores under each.
+Some emulators are libretro front ends (RetroArch, retrorun) whose cores are
+libretro core names. Every other emulator is a standalone program, named as
+the OS's launch code dispatches on it (`HATARISA`, `PPSSPPSDL`): profiles
+answer to that spelling in their `cores:`.
 """
 
 from __future__ import annotations
 
 import json
-import re
+from collections.abc import Iterable, Mapping
 
 from safeparse import parse_untrusted_xml
 
+# system -> [(emulator, [core, ...]), ...], both in the list's order.
+SystemEmulators = dict[str, list[tuple[str, list[str]]]]
+
+
+def xml_systems(text: str) -> SystemEmulators:
+    """Each system's emulators and cores, from an es_systems.cfg."""
+    root = parse_untrusted_xml(text, "es_systems.cfg")
+    systems: SystemEmulators = {}
+    for system in root.iter("system"):
+        name = (system.findtext("name") or "").strip()
+        if not name:
+            continue
+        systems[name] = [
+            (emulator.get("name"),
+             [core.text.strip() for core in emulator.iter("core") if core.text])
+            for emulator in system.iter("emulator")
+            if emulator.get("name")
+        ]
+    if not any(systems.values()):
+        raise ValueError("es_systems.cfg names no emulator")
+    return systems
+
+
+def json_systems(text: str) -> SystemEmulators:
+    """Each system's emulators and cores, from an es_systems.json."""
+    systems: SystemEmulators = {}
+    for system in json.loads(text).get("systems", []):
+        systems[system["name"]] = [
+            (emulator["name"], [core["name"] for core in emulator.get("cores") or []])
+            for emulator in system.get("emulators") or []
+        ]
+    if not any(systems.values()):
+        raise ValueError("es_systems.json names no emulator")
+    return systems
+
+
+def _pairs(systems: SystemEmulators) -> dict[str, set[str]]:
+    pairs: dict[str, set[str]] = {}
+    for emulators in systems.values():
+        for emulator, cores in emulators:
+            pairs.setdefault(emulator, set()).update(cores)
+    return pairs
+
 
 def xml_emulators(text: str) -> dict[str, set[str]]:
     """Emulator -> cores, from an es_systems.cfg."""
-    root = parse_untrusted_xml(text, "es_systems.cfg")
-    pairs: dict[str, set[str]] = {}
-    for emulator in root.iter("emulator"):
-        name = emulator.get("name")
-        if name:
-            pairs.setdefault(name, set()).update(
-                core.text.strip() for core in emulator.iter("core") if core.text
-            )
-    if not pairs:
-        raise ValueError("es_systems.cfg names no emulator")
-    return pairs
+    return _pairs(xml_systems(text))
 
 
 def json_emulators(text: str) -> dict[str, set[str]]:
     """Emulator -> cores, from an es_systems.json."""
-    pairs: dict[str, set[str]] = {}
-    for system in json.loads(text).get("systems", []):
-        for emulator in system.get("emulators") or []:
-            pairs.setdefault(emulator["name"], set()).update(
-                core["name"] for core in emulator.get("cores") or []
-            )
-    if not pairs:
-        raise ValueError("es_systems.json names no emulator")
-    return pairs
+    return _pairs(json_systems(text))
 
 
-def standalone_names(emulator: str) -> list[str]:
-    """The names a profile may know a standalone emulator by.
+def _names(
+    emulator: str,
+    cores: Iterable[str],
+    runners: Mapping[str, str | None],
+) -> list[str]:
+    """The core names one emulator entry stands for.
 
-    The OS's own spelling, its lowercase, and the lowercase without the
-    `sa` suffix the ELEC family appends to a standalone build of an emulator
-    that also exists as a core (`HATARISA`, `scummvmsa`, `yabasanshiroSA`).
+    A runner's entry stands for the libretro cores it lists, or for the one
+    core it always loads. A standalone emulator stands for itself.
     """
-    lower = emulator.lower()
-    names = [emulator, lower]
-    stripped = re.sub(r"-?sa$", "", lower)
-    if stripped:
-        names.append(stripped)
-    return list(dict.fromkeys(names))
+    if emulator in runners:
+        fixed = runners[emulator]
+        return [fixed] if fixed else list(cores)
+    return [emulator]
+
+
+def system_cores(
+    systems: SystemEmulators,
+    runners: Mapping[str, str | None],
+    not_cores: Iterable[str] = (),
+) -> dict[str, list[str]]:
+    """system -> the cores it runs, in the list's order, as the OS writes them."""
+    dropped = set(not_cores)
+    result: dict[str, list[str]] = {}
+    for system, emulators in systems.items():
+        names = [
+            name
+            for emulator, cores in emulators
+            for name in _names(emulator, cores, runners)
+            if name not in dropped
+        ]
+        if names:
+            result[system] = list(dict.fromkeys(names))
+    return result
 
 
 def es_cores(
-    pairs: dict[str, set[str]],
-    runners: dict[str, str | None],
-    renamed: dict[str, str] | None = None,
+    systems: SystemEmulators,
+    runners: Mapping[str, str | None],
+    builds_of: Mapping[str, str] | None = None,
+    not_cores: Iterable[str] = (),
 ) -> tuple[list[str], list[str], list[str]]:
     """(cores, standalone_cores, dual_mode_cores) of an ES system list.
 
     `runners` maps each emulator that runs libretro cores to None when the
     cores it lists are libretro core names, or to the one core it always
     loads (AmberELEC's `mame` lists MAME machines and loads the MAME core).
-    `renamed` gives the emulator a standalone name stands for when the
-    suffix rule cannot (`PPSSPPSDL` is PPSSPP), each one read in the OS's
-    launch code.
+    `builds_of` maps a standalone emulator to the libretro core of the same
+    emulator, each pair read in the OS's packages: the standalone one is
+    dual-mode when the list also offers that core, as is a standalone name
+    the list also offers as a core. `not_cores` names what
+    the list files as an emulator without emulating anything (a media
+    player, a runtime).
     """
-    renamed = renamed or {}
+    builds_of = builds_of or {}
+    dropped = set(not_cores)
     libretro: set[str] = set()
-    for runner, fixed in runners.items():
-        if runner in pairs:
-            libretro.update([fixed] if fixed else pairs[runner])
     standalone: set[str] = set()
-    for emulator in pairs:
-        if emulator in runners:
-            continue
-        standalone.update(standalone_names(emulator))
-        if emulator in renamed:
-            standalone.add(renamed[emulator])
-    dual = {name for name in standalone if name in libretro}
+    for emulators in systems.values():
+        for emulator, cores in emulators:
+            names = {n for n in _names(emulator, cores, runners) if n not in dropped}
+            (libretro if emulator in runners else standalone).update(names)
+    dual = {
+        name for name in standalone
+        if name in libretro or builds_of.get(name) in libretro
+    }
     return sorted(libretro | standalone), sorted(standalone), sorted(dual)
+
+
+def attach_system_cores(
+    platform_systems: dict[str, dict], by_native: Mapping[str, list[str]]
+) -> None:
+    """Give each platform system the cores its native systems run.
+
+    A slug can gather several native systems (Diablo and Hellfire under
+    diablo): its list is their union, in the order first met. A system the
+    ES list does not name gets no list, as nothing says what runs it.
+    """
+    for entry in platform_systems.values():
+        natives = [entry.get("native_id")] + [
+            f.get("native_system") for f in entry.get("files") or []
+        ]
+        cores = [
+            core
+            for native in dict.fromkeys(n for n in natives if n)
+            for core in by_native.get(native, [])
+        ]
+        if cores:
+            entry["cores"] = list(dict.fromkeys(cores))

scripts/scraper/knulli_scraper.py

@@ -37,10 +37,12 @@ from .batocera_scraper import (
 from .batocera_scraper import (
     _not_cores,
     configgen_cores,
+    es_systems_by_system,
     extract_systems_dict,
     platform_systems,
     requirements_from_systems,
 )
+from .es_emulators import attach_system_cores
 
 PLATFORM_NAME = "knulli"
 
@@ -93,12 +95,16 @@ class Scraper(BaseScraper):
             self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
         )
         # es_systems.yml files machine selectors and plugins under an
-        # emulator as Batocera's does: Batocera's list of them applies.
-        cores, standalone, dual = configgen_cores(
-            self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag)),
-            self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag)),
-            _not_cores("batocera"),
-        )
+        # emulator as Batocera's does: Batocera's list of them applies, with
+        # KNULLI's own.
+        defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag))
+        es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))
+        dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME)
+        cores, standalone, dual = configgen_cores(defaults, es_systems, dropped)
+        # Each system's own list (psx, es_systems.yml:640-653: libretro
+        # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and
+        # duckstation-legacy).
+        attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped))
         result = {
             "platform": "KNULLI",
             "version": self.tag,