feat: add knulli, amberelec and emuelec platforms

Each OS publishes Batocera's systems dict (knulli-systems, and
batocera-systems at AmberELEC's newest prerelease commit and EmuELEC v4.8),
read with the shared parser. Cores come from each OS's own ES list.
AmberELEC and EmuELEC declare neogeo/ and ports/ beside bios/ from
/storage/roms, so their packs extract there and core_destination puts
core files under bios/.

scripts/common.py

@@ -2842,10 +2842,14 @@ def declared_root(config: dict) -> str:
     Only a platform whose base_destination is explicitly empty does that
     (RetroDECK writes bios/ in front of each file), and core files go under
     the same directory. A config that does not state a base says nothing,
-    and its common folder is a system's, not the BIOS root.
+    and its common folder is a system's, not the BIOS root. A platform whose
+    list also names folders beside the BIOS one states it as
+    core_destination: EmuELEC's ports/ entries are a quarter of its list.
     """
     if config.get("base_destination", None) != "":
         return ""
+    if config.get("core_destination"):
+        return str(config["core_destination"])
     roots = [
         d.split("/", 1)[0]
         for system in config.get("systems", {}).values()

scripts/exporter/amberelec_exporter.py

@@ -0,0 +1,46 @@
+"""Exporter for AmberELEC's batocera-systems.
+
+Batocera's script shape, so only the systems dict is rewritten. Two things
+differ. Paths are relative to /storage/roms, where the list also names
+neogeo/ beside bios/, and our destinations start there too. No entry carries
+an emulator or core gate: coreExists() lets an ungated item through for
+every build, and an addition is written the same way.
+"""
+
+from __future__ import annotations
+
+from .base_exporter import BaseExporter
+from .baseline import NativeFile
+from .batocera_exporter import Exporter as BatoceraExporter
+
+SOURCE_URL = (
+    "https://raw.githubusercontent.com/AmberELEC/AmberELEC/dev"
+    "/packages/amberelec/sources/scripts/batocera-systems"
+)
+
+
+class Exporter(BatoceraExporter):
+    """Write AmberELEC's batocera-systems, corrected."""
+
+    PATH_PREFIX = ""
+
+    @classmethod
+    def writable(cls, fe: NativeFile, require: str = "") -> bool:
+        # The list gates nothing on a runner, so an addition needs none.
+        return BaseExporter.writable.__func__(cls, fe, require)
+
+    def _addition_keys(self, fe: NativeFile) -> list[tuple[str, str | None]] | None:
+        return []
+
+    @staticmethod
+    def platform_name() -> str:
+        return "amberelec"
+
+    @staticmethod
+    def file_modes() -> dict[str, int]:
+        # 100755 at a21975bf (git/trees).
+        return {"batocera-systems": 0o755}
+
+    @staticmethod
+    def native_sources() -> dict[str, str]:
+        return {"batocera-systems": SOURCE_URL}

scripts/exporter/batocera_exporter.py

@@ -35,6 +35,27 @@ def _commented(text: str, position: int) -> bool:
     return "#" in _STRING.sub("", text[line_start:position])
 
 
+def _closing_line(text: str, start: int) -> int | None:
+    """Index of the line holding the brace that closes `systems = {`.
+
+    Read from the parsed module, never executed. None when the script does
+    not parse or names no such assignment at line `start`.
+    """
+    try:
+        tree = ast.parse(text)
+    except SyntaxError:
+        return None
+    for node in tree.body:
+        if (
+            isinstance(node, ast.Assign)
+            and node.lineno - 1 == start
+            and isinstance(node.value, ast.Dict)
+            and node.end_lineno is not None
+        ):
+            return node.end_lineno - 1
+    return None
+
+
 def _note_key(text: str) -> tuple[str, str]:
     """(file, zippedFile) of the item a line holds, ("", "") for none."""
     path = re.search(r'"file"\s*:\s*"([^"]+)"', text)
@@ -55,6 +76,11 @@ class Exporter(BaseExporter):
     # (duckstation-sa), which its checker does not know.
     STANDALONE_RUNNERS = True
 
+    # What the script's paths add in front of a destination. Batocera's
+    # paths are relative to /userdata and our destinations to its bios/;
+    # AmberELEC's and EmuELEC's destinations already start at /storage/roms.
+    PATH_PREFIX = "bios/"
+
     def __init__(self) -> None:
         super().__init__()
         # The emulator and core each system entry of the original is gated
@@ -200,8 +226,7 @@ class Exporter(BaseExporter):
         alt = fe.native("alt_md5", "")
         if alt:
             item.append(f'"altmd5": "{alt}"')
-        declared = fe.native("native_path", "")
-        path = str(declared) if declared else f"bios/{fe.destination}"
+        path = self.declared_path(fe)
         item.append(f'"file": "{path}"')
         zipped = fe.native("zipped_file", "")
         if zipped:
@@ -209,6 +234,12 @@ class Exporter(BaseExporter):
         item.extend(self._runner_keys(originals.get(path)) if runner is None else runner)
         return "{ " + ", ".join(item) + " }"
 
+    @classmethod
+    def declared_path(cls, fe: NativeFile) -> str:
+        """The `file` the script names for an entry."""
+        declared = fe.native("native_path", "")
+        return str(declared) if declared else f"{cls.PATH_PREFIX}{fe.destination}"
+
     @staticmethod
     def _own_entry(system: NativeSystem, before: dict | None) -> dict:
         """The maintainer's entry for this system, out of the parsed pair."""
@@ -396,6 +427,9 @@ class Exporter(BaseExporter):
             (i for i in range(start + 1, len(lines)) if lines[i].startswith("}")),
             None,
         )
+        if end is None:
+            # EmuELEC closes the dict with an indented brace.
+            end = _closing_line(original, start)
         if end is None:
             raise ValueError("batocera-systems: the systems dict is not closed")
         return lines[: start + 1], lines[start + 1 : end], lines[end:]
@@ -520,14 +554,12 @@ class Exporter(BaseExporter):
         content = produced[self.native_filename()]
         issues: list[str] = []
 
-        block = content.split("\nsystems = {", 1)
-        if len(block) != 2:
-            return ["no systems dict in the output"]
-        end = block[1].find("\n}")
-        if end < 0:
-            return ["the systems dict is not closed"]
         try:
-            exported = ast.literal_eval("{" + block[1][:end] + "\n}")
+            _head, body, _tail = self._split_original(content)
+        except ValueError as exc:
+            return [str(exc)]
+        try:
+            exported = ast.literal_eval("{" + "\n".join(body) + "\n}")
         except (SyntaxError, ValueError) as exc:
             return [f"the systems dict does not parse: {exc}"]
 
@@ -541,8 +573,7 @@ class Exporter(BaseExporter):
                 continue
             declared = {bios.get("file", "") for bios in entry.get("biosFiles", [])}
             for fe in files:
-                path = str(fe.native("native_path", "")) or f"bios/{fe.destination}"
-                if path not in declared:
+                if self.declared_path(fe) not in declared:
                     issues.append(f"absent: {system.native_id}/{fe.name}")
 
         for native_id, entry in exported.items():

scripts/exporter/emuelec_exporter.py

@@ -0,0 +1,32 @@
+"""Exporter for EmuELEC's batocera-systems.
+
+AmberELEC's case: paths relative to /storage/roms with ports/ and neogeo/
+beside bios/, and no runner gate, since EmuELEC's checkBios never reads one.
+The dict closes on an indented brace.
+"""
+
+from __future__ import annotations
+
+from .amberelec_exporter import Exporter as AmberELECExporter
+
+SOURCE_URL = (
+    "https://raw.githubusercontent.com/EmuELEC/EmuELEC/master"
+    "/packages/sx05re/emuelec/bin/batocera/batocera-systems"
+)
+
+
+class Exporter(AmberELECExporter):
+    """Write EmuELEC's batocera-systems, corrected."""
+
+    @staticmethod
+    def platform_name() -> str:
+        return "emuelec"
+
+    @staticmethod
+    def file_modes() -> dict[str, int]:
+        # 100755 at v4.8 (git/trees).
+        return {"batocera-systems": 0o755}
+
+    @staticmethod
+    def native_sources() -> dict[str, str]:
+        return {"batocera-systems": SOURCE_URL}

scripts/exporter/knulli_exporter.py

@@ -0,0 +1,31 @@
+"""Exporter for KNULLI's knulli-systems.
+
+A fork of Batocera's script, checked the same way: es_systems names a
+libretro core under `libretro` and a standalone by itself, and entries carry
+Batocera's emulator and core gates.
+"""
+
+from __future__ import annotations
+
+from .batocera_exporter import Exporter as BatoceraExporter
+
+SOURCE_URL = (
+    "https://raw.githubusercontent.com/knulli-cfw/knulli-linux/knulli-main"
+    "/package/system/knulli-scripts/scripts/knulli-systems"
+)
+
+
+class Exporter(BatoceraExporter):
+    """Write KNULLI's knulli-systems, corrected."""
+
+    @staticmethod
+    def platform_name() -> str:
+        return "knulli"
+
+    @staticmethod
+    def native_filename() -> str:
+        return "knulli-systems"
+
+    @staticmethod
+    def native_sources() -> dict[str, str]:
+        return {"knulli-systems": SOURCE_URL}

scripts/scraper/amberelec_scraper.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Scraper for AmberELEC's batocera-systems.
+
+Source: packages/amberelec/sources/scripts/batocera-systems
+Format: Python source, Batocera's `systems` dict, parsed by batocera_scraper
+Hash: MD5
+
+AmberELEC has published no release since 20230203. Its builds come from the
+dev branch as prereleases in AmberELEC/AmberELEC-prerelease, each naming the
+commit it was built from. The scraper reads the list at the commit of the
+newest one. That list accepts every md5 the 20230203 one does for the files
+it keeps (syscard3.pce 38179df8, BS-X.bin 96cf17bf) and adds
+neogeo/aes.zip.
+
+Verification (batocera-systems at that commit):
+- checkBios (lines 583-643): md5sum() of the file at /storage/roms/<file>
+  against the declared md5, an empty md5 meaning presence only. For a
+  zippedFile entry, checkInsideZip() (lines 530-561) matches the member name
+  with casefold() and hashes it.
+- Run with no argument by EmulationStation's BIOS screen
+  (AmberELEC/emulationstation e3846751, es-app/src/ApiSystem.cpp:603-613),
+  which also passes `--filter <system>`. ee_check_bios
+  (packages/amberelec/profile.d/99-distribution.conf:320) passes
+  `--strictfilter`, which the script's argument switch (lines 665-687) does
+  not handle: that call prints nothing and exits 0.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.error
+import urllib.request
+
+from .base_scraper import BaseScraper, BiosRequirement, github_headers
+from .batocera_scraper import (
+    SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
+)
+from .batocera_scraper import (
+    extract_systems_dict,
+    platform_systems,
+    requirements_from_systems,
+)
+from .es_emulators import es_cores, xml_emulators
+
+PLATFORM_NAME = "amberelec"
+
+REPO = "AmberELEC/AmberELEC"
+PRERELEASE_REPO = "AmberELEC/AmberELEC-prerelease"
+_RAW = "https://raw.githubusercontent.com/AmberELEC/AmberELEC/{ref}/"
+SOURCE_TEMPLATE = _RAW + "packages/amberelec/sources/scripts/batocera-systems"
+ES_SYSTEMS_TEMPLATE = _RAW + "packages/ui/emulationstation/config/es_systems.cfg"
+
+# Batocera's names, plus the keys AmberELEC writes that Batocera does not,
+# filed under the slug the other scrapers already give them.
+SYSTEM_SLUG_MAP = {
+    **BATOCERA_SLUGS,
+    "atari7800": "atari-7800",
+    "atarilynx": "atari-lynx",
+    "n64": "nintendo-64",
+    "o2em": "magnavox-odyssey2",
+}
+
+# runemu.py: `retroarch` and `retrorun` load the libretro core es_systems.cfg
+# names. `mame` lists MAME machines and runs them on the libretro MAME core
+# (lines 233-261 set self.core = 'mame', line 355 builds the RetroArch command).
+RUNNERS: dict[str, str | None] = {"retroarch": None, "retrorun": None, "mame": "mame"}
+# runemu.py:113, PPSSPPSDL runs /usr/bin/ppsspp.sh, installed by the
+# ppssppsa package (packages/games/emulators/ppssppsa).
+RENAMED = {"PPSSPPSDL": "ppsspp"}
+
+_PRERELEASE_COMMIT = re.compile(r"AmberELEC/AmberELEC@([0-9a-f]{40})")
+
+
+def prerelease_commit(body: str) -> str:
+    """The commit a prerelease's notes say it was built from."""
+    found = _PRERELEASE_COMMIT.search(body or "")
+    if found is None:
+        raise RuntimeError("the newest AmberELEC prerelease names no commit")
+    return found.group(1)
+
+
+def fetch_newest_prerelease() -> tuple[str, str]:
+    """(tag, commit) of the newest AmberELEC prerelease."""
+    url = f"https://api.github.com/repos/{PRERELEASE_REPO}/releases?per_page=1"
+    try:
+        req = urllib.request.Request(url, headers=github_headers())
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            releases = json.loads(resp.read())
+    except (urllib.error.URLError, json.JSONDecodeError) as e:
+        raise RuntimeError(f"cannot list the releases of {PRERELEASE_REPO}: {e}") from e
+    if not releases or not releases[0].get("tag_name"):
+        raise RuntimeError(f"{PRERELEASE_REPO} lists no release")
+    return releases[0]["tag_name"], prerelease_commit(releases[0].get("body", ""))
+
+
+class Scraper(BaseScraper):
+    """Scraper for AmberELEC's batocera-systems."""
+
+    def __init__(self, url: str | None = None) -> None:
+        if url:
+            self.tag, self.commit = "", ""
+        else:
+            self.tag, self.commit = fetch_newest_prerelease()
+        super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.commit))
+
+    def _fetch_text(self, url: str) -> str:
+        req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                return resp.read().decode("utf-8")
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"Failed to fetch {url}: {e}") from e
+
+    def validate_format(self, raw_data: str) -> bool:
+        has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
+        return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data
+
+    def fetch_requirements(self) -> list[BiosRequirement]:
+        raw = self._fetch_raw()
+        if not self.validate_format(raw):
+            raise ValueError("batocera-systems format validation failed")
+        # Paths are relative to /storage/roms (`prefix`, lines 667-675): the
+        # list names neogeo/neogeo.zip beside bios/.
+        return requirements_from_systems(extract_systems_dict(raw), SYSTEM_SLUG_MAP, strip="")
+
+    def generate_platform_yaml(self) -> dict:
+        systems = platform_systems(
+            self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
+        )
+        cores, standalone, dual = es_cores(
+            xml_emulators(self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.commit))),
+            RUNNERS,
+            RENAMED,
+        )
+        return {
+            "platform": "AmberELEC",
+            "version": self.tag,
+            "homepage": "https://amberelec.org",
+            "source": self.url,
+            "base_destination": "",
+            # RetroArch's system_directory is ~/roms/bios
+            # (packages/games/tools/retroarch/sources/retroarch.cfg:611).
+            "core_destination": "bios",
+            "hash_type": "md5",
+            "verification_mode": "md5",
+            "cores": cores,
+            "standalone_cores": standalone,
+            "dual_mode_cores": dual,
+            "systems": systems,
+        }
+
+
+def main() -> None:
+    from .base_scraper import scraper_cli
+
+    scraper_cli(Scraper, "Scrape AmberELEC BIOS requirements")
+
+
+if __name__ == "__main__":
+    main()

scripts/scraper/emuelec_scraper.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Scraper for EmuELEC's batocera-systems.
+
+Source: packages/sx05re/emuelec/bin/batocera/batocera-systems
+Format: Python source, Batocera's `systems` dict, parsed by batocera_scraper
+Hash: MD5
+
+`naomi` and `Dinothawr` are written twice in the dict. Python keeps the later
+value of a repeated key when the script runs, and so does the parser: naomi
+is the entry at line 461 (naomi_boot.bin and naomi.zip, line 444 naming
+naomi.zip alone), Dinothawr the one at line 776 (dinothawr.game, line 494
+naming level_3-3.tmx).
+
+Verification (batocera-systems at the release tag):
+- checkBios (lines 817-833): md5sum() of the file at /storage/roms/<file>
+  against the declared md5, an empty md5 meaning presence only. It never
+  reads zippedFile: the FM Towns entries that name a member hash the whole
+  archive against the member's md5. There is no emulator or core gate.
+- Run with no argument by EmulationStation's BIOS screen
+  (EmuELEC/emuelec-emulationstation 09356d0d, es-app/src/ApiSystem.cpp:
+  645-655, `--filter <system>` for one system), and with `--strictfilter
+  <platform>` by ee_check_bios after a failed launch
+  (packages/sx05re/emuelec/profile.d/99-emuelec.conf:261, called from
+  emuelecRunEmu.sh:605).
+"""
+
+from __future__ import annotations
+
+import re
+import urllib.error
+import urllib.request
+
+from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+from .batocera_scraper import (
+    SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
+)
+from .batocera_scraper import (
+    extract_systems_dict,
+    platform_systems,
+    requirements_from_systems,
+)
+from .es_emulators import es_cores, json_emulators
+
+PLATFORM_NAME = "emuelec"
+
+REPO = "EmuELEC/EmuELEC"
+_RAW = "https://raw.githubusercontent.com/EmuELEC/EmuELEC/{ref}/"
+SOURCE_TEMPLATE = _RAW + "packages/sx05re/emuelec/bin/batocera/batocera-systems"
+ES_SYSTEMS_TEMPLATE = _RAW + "packages/sx05re/emuelec-emulationstation/config/es_systems.json"
+
+# Batocera's names, plus the keys EmuELEC writes that Batocera does not:
+# console keys under the slug the other scrapers give them, and each port
+# under the system of the profile of the engine its name gives
+# (devilutionx, prboom, opentyrian, cannonball, nxengine, tyrquake,
+# reminiscence, xrick, vvvvvv, eduke32, sonic2013, sonic-mania, dinothawr).
+SYSTEM_SLUG_MAP = {
+    **BATOCERA_SLUGS,
+    "atari7800": "atari-7800",
+    "atarilynx": "atari-lynx",
+    "n64": "nintendo-64",
+    "o2em": "magnavox-odyssey2",
+    "Diablo": "diablo",
+    "Hellfire": "diablo",
+    "Dinothawr": "dinothawr",
+    "Quake": "quake",
+    "VVVVVV": "vvvvvv",
+    "DOOM1": "doom",
+    "DOOM2": "doom",
+    "OpenTyrian": "tyrian",
+    "Cannonball": "sega-outrun",
+    "CaveStory": "cave-story",
+    "REminiscence": "flashback",
+    "RickDangerous": "xrick",
+    "eduke32": "duke3d",
+    "sonic1": "sonic-1-2013",
+    "sonic2": "sonic-2-2013",
+    "sonicmania": "sonic-mania",
+}
+
+# es_systems.json names libretro cores under `libretro`, and `retrorun`
+# loads the libretro core it lists.
+RUNNERS: dict[str, str | None] = {"libretro": None, "retrorun": None}
+# packages/sx05re/emulators/PPSSPPSDL builds PPSSPP's SDL front end.
+RENAMED = {"PPSSPPSDL": "ppsspp"}
+
+
+class Scraper(BaseScraper):
+    """Scraper for EmuELEC's batocera-systems."""
+
+    def __init__(self, url: str | None = None) -> None:
+        self.tag = "" if url else fetch_github_latest_version(REPO)
+        super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.tag))
+
+    def _fetch_text(self, url: str) -> str:
+        req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                return resp.read().decode("utf-8")
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"Failed to fetch {url}: {e}") from e
+
+    def validate_format(self, raw_data: str) -> bool:
+        has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
+        return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data
+
+    def fetch_requirements(self) -> list[BiosRequirement]:
+        raw = self._fetch_raw()
+        if not self.validate_format(raw):
+            raise ValueError("batocera-systems format validation failed")
+        # Paths are relative to /storage/roms (`prefix`, lines 859-865): the
+        # list names neogeo/ and ports/ beside bios/.
+        return requirements_from_systems(extract_systems_dict(raw), SYSTEM_SLUG_MAP, strip="")
+
+    def generate_platform_yaml(self) -> dict:
+        systems = platform_systems(
+            self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
+        )
+        cores, standalone, dual = es_cores(
+            json_emulators(self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))),
+            RUNNERS,
+            RENAMED,
+        )
+        return {
+            "platform": "EmuELEC",
+            "version": self.tag.removeprefix("v"),
+            "homepage": "https://emuelec.org",
+            "source": self.url,
+            "base_destination": "",
+            # RetroArch's system_directory is /storage/roms/bios
+            # (packages/sx05re/libretro_base/retroarch/package.mk:117).
+            "core_destination": "bios",
+            "hash_type": "md5",
+            "verification_mode": "md5",
+            "cores": cores,
+            "standalone_cores": standalone,
+            "dual_mode_cores": dual,
+            "systems": systems,
+        }
+
+
+def main() -> None:
+    from .base_scraper import scraper_cli
+
+    scraper_cli(Scraper, "Scrape EmuELEC BIOS requirements")
+
+
+if __name__ == "__main__":
+    main()

scripts/scraper/es_emulators.py

@@ -0,0 +1,88 @@
+"""The emulators an EmulationStation system list offers, by profile names.
+
+AmberELEC writes its list as es_systems.cfg XML, EmuELEC as an es_systems.json
+it turns into that XML at build time. Both name, per system, the emulators a
+user may pick and the cores under each. Some emulators are libretro front
+ends (RetroArch, retrorun) whose cores are libretro core names. Every other
+emulator is a standalone program named by the OS (`HATARISA`, `PPSSPPSDL`).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+from safeparse import parse_untrusted_xml
+
+
+def xml_emulators(text: str) -> dict[str, set[str]]:
+    """Emulator -> cores, from an es_systems.cfg."""
+    root = parse_untrusted_xml(text, "es_systems.cfg")
+    pairs: dict[str, set[str]] = {}
+    for emulator in root.iter("emulator"):
+        name = emulator.get("name")
+        if name:
+            pairs.setdefault(name, set()).update(
+                core.text.strip() for core in emulator.iter("core") if core.text
+            )
+    if not pairs:
+        raise ValueError("es_systems.cfg names no emulator")
+    return pairs
+
+
+def json_emulators(text: str) -> dict[str, set[str]]:
+    """Emulator -> cores, from an es_systems.json."""
+    pairs: dict[str, set[str]] = {}
+    for system in json.loads(text).get("systems", []):
+        for emulator in system.get("emulators") or []:
+            pairs.setdefault(emulator["name"], set()).update(
+                core["name"] for core in emulator.get("cores") or []
+            )
+    if not pairs:
+        raise ValueError("es_systems.json names no emulator")
+    return pairs
+
+
+def standalone_names(emulator: str) -> list[str]:
+    """The names a profile may know a standalone emulator by.
+
+    The OS's own spelling, its lowercase, and the lowercase without the
+    `sa` suffix the ELEC family appends to a standalone build of an emulator
+    that also exists as a core (`HATARISA`, `scummvmsa`, `yabasanshiroSA`).
+    """
+    lower = emulator.lower()
+    names = [emulator, lower]
+    stripped = re.sub(r"-?sa$", "", lower)
+    if stripped:
+        names.append(stripped)
+    return list(dict.fromkeys(names))
+
+
+def es_cores(
+    pairs: dict[str, set[str]],
+    runners: dict[str, str | None],
+    renamed: dict[str, str] | None = None,
+) -> tuple[list[str], list[str], list[str]]:
+    """(cores, standalone_cores, dual_mode_cores) of an ES system list.
+
+    `runners` maps each emulator that runs libretro cores to None when the
+    cores it lists are libretro core names, or to the one core it always
+    loads (AmberELEC's `mame` lists MAME machines and loads the MAME core).
+    `renamed` gives the emulator a standalone name stands for when the
+    suffix rule cannot (`PPSSPPSDL` is PPSSPP), each one read in the OS's
+    launch code.
+    """
+    renamed = renamed or {}
+    libretro: set[str] = set()
+    for runner, fixed in runners.items():
+        if runner in pairs:
+            libretro.update([fixed] if fixed else pairs[runner])
+    standalone: set[str] = set()
+    for emulator in pairs:
+        if emulator in runners:
+            continue
+        standalone.update(standalone_names(emulator))
+        if emulator in renamed:
+            standalone.add(renamed[emulator])
+    dual = {name for name in standalone if name in libretro}
+    return sorted(libretro | standalone), sorted(standalone), sorted(dual)

scripts/scraper/knulli_scraper.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Scraper for KNULLI's knulli-systems.
+
+Source: package/system/knulli-scripts/scripts/knulli-systems
+Format: Python source, Batocera's `systems` dict, parsed by batocera_scraper
+Hash: MD5
+
+`msx1` and `msx2+` are written twice in the dict. Python keeps the later
+value of a repeated key when the script runs, and so does the parser.
+
+Verification (knulli-systems at the release tag):
+- checkBios (lines 867-927): md5sum() of the file at /userdata/<file>
+  against the declared md5, an empty md5 meaning presence only. For a
+  zippedFile entry, checkInsideZip() (lines 814-845) matches the member name
+  with casefold() and hashes it. A system or file carrying `emulator` and
+  `core` is checked only when es_systems.cfg offers that pair (coreExists,
+  lines 847-865).
+- Run by EmulationStation's BIOS screen (knulli-cfw/batocera-emulationstation
+  10834d00, es-app/src/ApiSystem.cpp:764-774) and by knulli-support:69.
+
+es_systems.yml and configgen-defaults.yml have Batocera's shape, so the cores
+are read with Batocera's code. The configgen generators do not: the hatari,
+melonds and gsplus generators and libretroOptions.py are older forms than the
+ones batocera_launchers reads, so no launcher layer is written.
+"""
+
+from __future__ import annotations
+
+import re
+import urllib.error
+import urllib.request
+
+from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
+from .batocera_scraper import (
+    SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
+)
+from .batocera_scraper import (
+    _not_cores,
+    configgen_cores,
+    extract_systems_dict,
+    platform_systems,
+    requirements_from_systems,
+)
+
+PLATFORM_NAME = "knulli"
+
+REPO = "knulli-cfw/knulli-linux"
+_RAW = "https://raw.githubusercontent.com/knulli-cfw/knulli-linux/{ref}/"
+SOURCE_TEMPLATE = _RAW + "package/system/knulli-scripts/scripts/knulli-systems"
+CONFIGGEN_DEFAULTS_TEMPLATE = (
+    _RAW + "package/system/knulli-configgen/configs/configgen-defaults.yml"
+)
+ES_SYSTEMS_TEMPLATE = _RAW + "package/emulationstation/knulli-es-system/es_systems.yml"
+
+# Batocera's names, plus the keys KNULLI writes that Batocera does not,
+# filed under the slug the other scrapers already give them. msu-md holds the
+# Mega-CD BIOS files Batocera files under megadrive-msu.
+SYSTEM_SLUG_MAP = {
+    **BATOCERA_SLUGS,
+    "o2em": "magnavox-odyssey2",
+    "msu-md": "sega-mega-cd",
+    "xbox": "microsoft-xbox",
+}
+
+
+class Scraper(BaseScraper):
+    """Scraper for KNULLI's knulli-systems."""
+
+    def __init__(self, url: str | None = None) -> None:
+        self.tag = "" if url else fetch_github_latest_version(REPO)
+        super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.tag))
+
+    def _fetch_text(self, url: str) -> str:
+        req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                return resp.read().decode("utf-8")
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"Failed to fetch {url}: {e}") from e
+
+    def validate_format(self, raw_data: str) -> bool:
+        has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
+        return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data
+
+    def fetch_requirements(self) -> list[BiosRequirement]:
+        raw = self._fetch_raw()
+        if not self.validate_format(raw):
+            raise ValueError("knulli-systems format validation failed")
+        return requirements_from_systems(extract_systems_dict(raw), SYSTEM_SLUG_MAP)
+
+    def generate_platform_yaml(self) -> dict:
+        systems = platform_systems(
+            self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
+        )
+        # es_systems.yml files machine selectors and plugins under an
+        # emulator as Batocera's does: Batocera's list of them applies.
+        cores, standalone, dual = configgen_cores(
+            self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag)),
+            self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag)),
+            _not_cores("batocera"),
+        )
+        result = {
+            "platform": "KNULLI",
+            "version": self.tag,
+            "homepage": "https://knulli.org",
+            "source": self.url,
+            "base_destination": "bios",
+            "hash_type": "md5",
+            "verification_mode": "md5",
+            "cores": cores,
+            "systems": systems,
+        }
+        if standalone:
+            result["standalone_cores"] = standalone
+        if dual:
+            result["dual_mode_cores"] = dual
+        return result
+
+
+def main() -> None:
+    from .base_scraper import scraper_cli
+
+    scraper_cli(Scraper, "Scrape KNULLI BIOS requirements")
+
+
+if __name__ == "__main__":
+    main()