fix: name each exported addition's core

Batocera, ROCKNIX and Recalbox find a file through the core that wants
it. Additions now carry that core under the platform's own name: the
emulator/core pair checkBios reads (ROCKNIX's from its emulators
package), or Recalbox's core attribute. One with no nameable core, or
filed in a system gated on another core, is counted as not stated.

scripts/export_native.py

@@ -16,6 +16,7 @@ Usage:
 from __future__ import annotations
 
 import argparse
+import re
 import json
 import sys
 import urllib.error
@@ -146,10 +147,29 @@ def wanted_sources(
 
     base = pinned_base(wanted, scraped)
     if base:
-        wanted = {relative: base + relative for relative in wanted}
+        wanted = {
+            relative: _at_revision(wanted[relative], base) or base + relative
+            for relative in wanted
+        }
     return wanted
 
 
+_RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/")
+
+
+def _at_revision(url: str, base: str) -> str:
+    """A file of the pinned repository, read at the pinned revision; empty
+    when the URL is not a raw GitHub URL of that repository.
+
+    ROCKNIX's checker sits in packages/rocknix/, the emulators it names in
+    packages/virtual/: no shared directory, one revision.
+    """
+    pinned, other = _RAW_GITHUB.match(base), _RAW_GITHUB.match(url)
+    if pinned and other and pinned.group(1, 2) == other.group(1, 2):
+        return pinned.group(0) + url[other.end():]
+    return ""
+
+
 def collect_originals(
     exporter: object,
     systems: dict,
@@ -347,7 +367,7 @@ def export_platform(
         for entry in system.files:
             if not entry.name:
                 continue
-            if exporter.writable(entry):
+            if exporter.lands(entry):
                 landed += entry.platform is None
             elif entry.platform is None:
                 refused += 1

scripts/exporter/base_exporter.py

@@ -127,6 +127,11 @@ class BaseExporter(ABC):
         """
         return False
 
+    def lands(self, fe: NativeFile) -> bool:
+        """Whether this render writes the entry: writable(), plus whatever
+        the render learned from the platform's own file."""
+        return self.writable(fe)
+
     @classmethod
     def writable(cls, fe: NativeFile, require: str = "") -> bool:
         """Whether this format can carry the entry.

scripts/exporter/baseline.py

@@ -65,6 +65,9 @@ class NativeFile:
     filled: list[str] = field(default_factory=list)
     # An addition filed under our own slug: the platform names no such system.
     foreign_system: bool = False
+    # For an addition, each core that wants it as (runs standalone, name the
+    # platform gives it); empty when no core the platform runs is named.
+    runners: list[tuple[bool, str]] = field(default_factory=list)
 
     @property
     def origin(self) -> str:
@@ -198,13 +201,14 @@ class NativeFile:
         return default
 
     def cores(self) -> list[str]:
-        """Cores that want this file, the platform's naming preferred."""
+        """Cores that want this file, as Recalbox writes them: the declared
+        value, else each runner, a standalone bare and a core under
+        libretro/. A profile key is not a name the platform knows:
+        `libretro/duckstation` named no core Recalbox runs."""
         declared = self.native("core", "")
         if declared:
             return [c.strip() for c in str(declared).split(",") if c.strip()]
-        if self.truth:
-            return [f"libretro/{c}" for c in self.truth.get("_cores", [])]
-        return []
+        return [name if standalone else f"libretro/{name}" for standalone, name in self.runners]
 
 
 @dataclass
@@ -483,9 +487,46 @@ def build_native_model(
         tail = [fe for fe in system.files if fe.platform is None]
         system.files = head + search_order(tail)
 
+    _attach_runners(systems, scraped)
     return systems, report
 
 
+def _attach_runners(systems: dict[str, NativeSystem], scraped: dict | None) -> None:
+    """Name the cores behind each addition the way the platform names them.
+
+    The platform lists its cores by its own names (Batocera's
+    `mednafen_psx`, Recalbox's `mednafen_psx`), our truth by profile key
+    (`beetle_psx`); a profile's `cores:` holds the names it answers to.
+    The build a platform runs an emulator as decides the runner.
+    """
+    if not scraped:
+        return
+    from common import load_emulator_profiles, runs_standalone
+
+    declared = scraped.get("cores")
+    libretro = {str(c) for c in declared} if isinstance(declared, list) else set()
+    standalone = {str(c) for c in scraped.get("standalone_cores") or []}
+    profiles = load_emulator_profiles(str(Path(__file__).resolve().parents[2] / "emulators"))
+    for system in systems.values():
+        for fe in system.files:
+            if fe.platform is not None or not fe.truth:
+                continue
+            runners: list[tuple[bool, str]] = []
+            for key in sorted(fe.truth.get("_cores") or ()):
+                profile = profiles.get(key, {})
+                names = [key, *(str(c) for c in profile.get("cores") or [])]
+                if runs_standalone(key, profile, standalone):
+                    name = next((n for n in names if n in standalone), None)
+                    runner = (True, name) if name else None
+                else:
+                    name = next((n for n in names if n in libretro - standalone), None) \
+                        or next((n for n in names if n in libretro), None)
+                    runner = (False, name) if name else None
+                if runner and runner not in runners:
+                    runners.append(runner)
+            fe.runners = runners
+
+
 def search_order(files: list[NativeFile]) -> list[NativeFile]:
     """Order files the way the code looks for them, best first.
 

scripts/exporter/batocera_exporter.py

@@ -25,6 +25,17 @@ _INDENT = " " * 4
 class Exporter(BaseExporter):
     """Write Batocera's batocera-systems, corrected."""
 
+    # checkBios reads emulator names from es_systems, and Batocera's
+    # standalone_cores are those names. ROCKNIX's list holds package names
+    # (duckstation-sa), which its checker does not know.
+    STANDALONE_RUNNERS = True
+
+    def __init__(self) -> None:
+        super().__init__()
+        # The emulator and core each system entry of the original is gated
+        # on, read by render.
+        self._gates: dict[str, tuple[str, str] | None] = {}
+
     @staticmethod
     def platform_name() -> str:
         return "batocera"
@@ -55,8 +66,60 @@ class Exporter(BaseExporter):
         """
         if fe.platform is None and len(set(fe.hashes("md5"))) > 1:
             return False
+        if fe.platform is None and not cls._usable_runners(fe):
+            # Written without its core's emulator and core, checkBios asks
+            # every build for it (the defect _runner_keys repairs for the
+            # platform's own items).
+            return False
         return super().writable(fe, require)
 
+    @classmethod
+    def _usable_runners(cls, fe: NativeFile) -> list[tuple[bool, str]]:
+        return [r for r in fe.runners if cls.STANDALONE_RUNNERS or not r[0]]
+
+    def _runner_pair(self, runner: tuple[bool, str]) -> tuple[str, str | None] | None:
+        """The emulator and core checkBios finds a runner under, None when
+        the checker has no name for it. Batocera's es_systems names a
+        libretro core under `libretro` and a standalone by itself."""
+        standalone, name = runner
+        return (name, None) if standalone else ("libretro", name)
+
+    def _addition_keys(self, fe: NativeFile) -> list[tuple[str, str | None]] | None:
+        """The emulator/core pairs an addition is written with, [] when its
+        system's own gate already names its core, None when the gate names
+        another: checkBios walks a gated system only when that core is
+        installed, so a file of another core there is asked of the wrong
+        users and never of its own."""
+        pairs = [
+            pair for r in self._usable_runners(fe)
+            if (pair := self._runner_pair(r)) is not None
+        ]
+        if not pairs:
+            return None
+        gate = self._gates.get(fe.native_system)
+        if gate is None:
+            return pairs
+        return [] if gate in pairs else None
+
+    def exportable(
+        self,
+        systems: dict[str, NativeSystem],
+        require: str = "",
+    ) -> list[tuple[NativeSystem, list[NativeFile]]]:
+        """What the format can carry, less the additions its gates refuse:
+        render and validate read the same list."""
+        result = []
+        for system, files in super().exportable(systems, require):
+            kept = [fe for fe in files if self.lands(fe)]
+            if kept:
+                result.append((system, kept))
+        return result
+
+    def lands(self, fe: NativeFile) -> bool:
+        if not self.writable(fe):
+            return False
+        return fe.platform is not None or self._addition_keys(fe) is not None
+
     def states(self, fe: NativeFile, field_name: str) -> bool:
         if field_name == "md5" and "md5" in fe.filled:
             return bool(self.single_value(fe, "md5"))
@@ -96,22 +159,38 @@ class Exporter(BaseExporter):
         }
         parts: list[str] = []
         for fe in files:
-            # The platform states an unhashed file as an empty md5 rather
-            # than leaving it out, and so do we.
-            item = [f'"md5": "{self.single_value(fe, "md5")}"']
-            alt = fe.native("alt_md5", "")
-            if alt:
-                item.append(f'"altmd5": "{alt}"')
-            declared = fe.native("native_path", "")
-            path = str(declared) if declared else f"bios/{fe.destination}"
-            item.append(f'"file": "{path}"')
-            zipped = fe.native("zipped_file", "")
-            if zipped:
-                item.append(f'"zippedFile": "{zipped}"')
-            item.extend(self._runner_keys(originals.get(path)))
-            parts.append("{ " + ", ".join(item) + " }")
+            for runner in self._item_runners(fe):
+                parts.append(self._bios_item(fe, runner, originals))
         return parts
 
+    def _item_runners(self, fe: NativeFile) -> list[list[str] | None]:
+        """One item per runner an addition names; None keeps the original's."""
+        if fe.platform is not None:
+            return [None]
+        pairs = self._addition_keys(fe) or []
+        if not pairs:
+            return [[]]
+        return [
+            [f'"emulator": "{emulator}"'] + ([f'"core": "{core}"'] if core else [])
+            for emulator, core in pairs
+        ]
+
+    def _bios_item(self, fe: NativeFile, runner: list[str] | None, originals: dict) -> str:
+        # The platform states an unhashed file as an empty md5 rather
+        # than leaving it out, and so do we.
+        item = [f'"md5": "{self.single_value(fe, "md5")}"']
+        alt = fe.native("alt_md5", "")
+        if alt:
+            item.append(f'"altmd5": "{alt}"')
+        declared = fe.native("native_path", "")
+        path = str(declared) if declared else f"bios/{fe.destination}"
+        item.append(f'"file": "{path}"')
+        zipped = fe.native("zipped_file", "")
+        if zipped:
+            item.append(f'"zippedFile": "{zipped}"')
+        item.extend(self._runner_keys(originals.get(path)) if runner is None else runner)
+        return "{ " + ", ".join(item) + " }"
+
     @staticmethod
     def _own_entry(system: NativeSystem, before: dict | None) -> dict:
         """The maintainer's entry for this system, out of the parsed pair."""
@@ -257,11 +336,6 @@ class Exporter(BaseExporter):
         originals: dict[str, str],
         scraped: dict | None = None,
     ) -> dict[str, str]:
-        exportable = dict(
-            (system.native_id, (system, files))
-            for system, files in self.exportable(systems, require="md5")
-        )
-
         original = originals.get(self.native_filename(), "")
         if not original:
             raise ValueError(
@@ -272,6 +346,18 @@ class Exporter(BaseExporter):
 
         head, body, tail = self._split_original(original)
         spans = self._entry_spans(body)
+        for key, (start, end) in spans.items():
+            parsed = self._parse_entry(body[start:end + 1]) or {}
+            own = parsed.get(key) if isinstance(parsed.get(key), dict) else {}
+            emulator, core = own.get("emulator"), own.get("core")
+            self._gates[key] = (
+                (emulator, core if isinstance(core, str) else None)
+                if isinstance(emulator, str) else None
+            )
+        exportable = {
+            system.native_id: (system, files)
+            for system, files in self.exportable(systems, require="md5")
+        }
 
         rebuilt: list[str] = []
         written: set[str] = set()

scripts/exporter/rocknix_exporter.py

@@ -6,6 +6,9 @@ runs, so only the mapping is rewritten.
 
 from __future__ import annotations
 
+import re
+
+from .baseline import NativeSystem, Report
 from .batocera_exporter import Exporter as BatoceraExporter
 
 SOURCE_URL = (
@@ -14,9 +17,47 @@ SOURCE_URL = (
 )
 
 
+EMULATORS_URL = (
+    "https://raw.githubusercontent.com/ROCKNIX/distribution/next/projects/ROCKNIX"
+    "/packages/virtual/emulators/package.mk"
+)
+EMULATORS_MK = "emulators/package.mk"
+# `add_emu_core <system> <emulator> <core> <default>`: the pairs es_systems.cfg
+# lists, and so the names checkBios's emuDict holds.
+_ADD_EMU_CORE = re.compile(r"^\s*add_emu_core\s+(\S+)\s+(\S+)\s+(\S+)", re.MULTILINE)
+
+
 class Exporter(BatoceraExporter):
     """Write ROCKNIX's rocknix-systems, corrected."""
 
+    def __init__(self) -> None:
+        super().__init__()
+        self._emulator_of: dict[str, str] = {}
+
+    def _runner_pair(self, runner: tuple[bool, str]) -> tuple[str, str | None] | None:
+        """ROCKNIX registers libretro cores under `retroarch` and each
+        standalone under its own emulator (`azahar azahar-sa`), in the
+        emulators package; the `libretro` Batocera writes names nothing
+        there."""
+        _standalone, name = runner
+        emulator = self._emulator_of.get(name)
+        return (emulator, name) if emulator else None
+
+    def render(
+        self,
+        systems: dict[str, NativeSystem],
+        report: Report,
+        originals: dict[str, str],
+        scraped: dict | None = None,
+    ) -> dict[str, str]:
+        self._emulator_of = {
+            core: emulator
+            for _system, emulator, core in _ADD_EMU_CORE.findall(
+                originals.get(EMULATORS_MK, "")
+            )
+        }
+        return super().render(systems, report, originals, scraped)
+
     @staticmethod
     def platform_name() -> str:
         return "rocknix"
@@ -27,4 +68,4 @@ class Exporter(BatoceraExporter):
 
     @staticmethod
     def native_sources() -> dict[str, str]:
-        return {"rocknix-systems": SOURCE_URL}
+        return {"rocknix-systems": SOURCE_URL, EMULATORS_MK: EMULATORS_URL}

tests/test_export_counts.py

@@ -331,7 +331,8 @@ class OneMd5FormatsAddWhole(unittest.TestCase):
         from exporter.batocera_exporter import Exporter as Batocera  # noqa: PLC0415
 
         several = NativeFile("boot.bin", "boot.bin", "dc", truth={"md5": [A, B, C]})
-        single = NativeFile("boot2.bin", "boot2.bin", "dc", truth={"md5": A})
+        single = NativeFile("boot2.bin", "boot2.bin", "dc", truth={"md5": A},
+                          runners=[(False, "flycast")])
         self.assertFalse(Batocera.writable(several))
         self.assertTrue(Batocera.writable(single))
 

tests/test_exporters.py

@@ -43,6 +43,8 @@ from export_native import (  # noqa: E402
 )
 from exporter.baseline import (  # noqa: E402
     NativeFile,
+    NativeSystem,
+    Report,
     build_native_model,
 )
 
@@ -85,6 +87,11 @@ TRUTH = {
 }
 
 SCRAPED = {
+    # The cores the platform runs, by its own names: an addition is written
+    # with the runner that wants it.
+    "cores": ["atari800", "duckstation", "fuse", "melonds", "o2em", "picodrive",
+              "snes9x", "beetle_psx"],
+    "standalone_cores": ["duckstation"],
     "systems": {
         "sony-playstation": {
             "native_id": "psx",
@@ -949,6 +956,7 @@ class BatoceraExport(unittest.TestCase):
         self.assertTrue(foreign.foreign_system)
         own = next(fe for fe in systems["psx"].files if fe.name == "scph7001.bin")
         self.assertFalse(own.foreign_system)
+        self.assertEqual(own.runners, [(True, "duckstation")])
         for platform in ("batocera", "rocknix", "retrobat", "recalbox"):
             with self.subTest(platform=platform):
                 exporter = discover_exporters()[platform]
@@ -963,7 +971,8 @@ class BatoceraExport(unittest.TestCase):
 
     def test_a_system_the_platform_names_without_files_is_its_own(self):
         """A system the platform declares empty still has the platform's id."""
-        scraped = {"systems": {"nintendo-snes": {"native_id": "snes", "files": []}}}
+        scraped = {"cores": ["snes9x"],
+                   "systems": {"nintendo-snes": {"native_id": "snes", "files": []}}}
         truth = {"systems": {"nintendo-snes": {"files": [{
             "name": "st010.zip", "path": "st010.zip", "md5": "2" * 32,
             "required": False, "_cores": ["snes9x"],
@@ -1389,3 +1398,109 @@ class ARecordDescribesOneDump(unittest.TestCase):
             source = (Path(__file__).resolve().parent.parent / "scripts" / "exporter" / module).read_text()
             with self.subTest(module=module):
                 self.assertNotRegex(source, r'= fe\.hash\("(crc32|md5|sha1)"\)')
+
+
+class AnAdditionCarriesItsRunner(unittest.TestCase):
+    """Batocera's apple2 entry is gated on libretro/mame; Clock Signal's
+    apple2.rom written inside it was asked of mame users and never of clk
+    users. Recalbox got core="libretro/duckstation", a core it never runs."""
+
+    ORIGINAL = (
+        "#!/usr/bin/env python\n"
+        "systems = {\n"
+        '    "apple2": { "name": "Apple II", "emulator": "libretro", "core": "mame", '
+        '"biosFiles": [ { "md5": "", "file": "bios/apple2e.zip" } ] },\n'
+        '    "macintosh": { "name": "Macintosh", "biosFiles": [ '
+        '{ "md5": "", "file": "bios/MacII.ROM", "emulator": "libretro", "core": "minivmac" } ] },\n'
+        "}\n"
+        "def checkBios(systems, prefix, filterROMs):\n"
+        "    return {}\n"
+    )
+
+    def _systems(self):
+        def add(native, name, runners):
+            return NativeFile(name, name, native, truth={"md5": "1" * 32}, runners=runners)
+
+        apple2 = NativeSystem("apple2", "Apple II", from_platform=True, files=[
+            NativeFile("apple2e.zip", "apple2e.zip", "apple2", platform={"md5": ""}),
+            add("apple2", "apple2.rom", [(True, "clk")]),
+            add("apple2", "apple2e.rom", [(False, "mame")]),
+        ])
+        mac = NativeSystem("macintosh", "Macintosh", from_platform=True, files=[
+            NativeFile("MacII.ROM", "MacII.ROM", "macintosh", platform={"md5": ""}),
+            add("macintosh", "macplus.rom", [(True, "clk")]),
+            add("macintosh", "nothing.rom", []),
+        ])
+        return {"apple2": apple2, "macintosh": mac}
+
+    def test_batocera_writes_the_runner_or_counts_the_file_out(self):
+        exporter = discover_exporters()["batocera"]()
+        systems = self._systems()
+        out = exporter.render(systems, Report(), {"batocera-systems": self.ORIGINAL})
+        text = out["batocera-systems"]
+        self.assertNotIn("apple2.rom", text)
+        self.assertIn('"file": "bios/apple2e.rom" }', text)
+        self.assertIn('"file": "bios/macplus.rom", "emulator": "clk" }', text)
+        self.assertNotIn("nothing.rom", text)
+        landed = {fe.name for s in systems.values() for fe in s.files
+                  if fe.platform is None and exporter.lands(fe)}
+        self.assertEqual(landed, {"apple2e.rom", "macplus.rom"})
+
+    def test_recalbox_names_cores_as_it_runs_them(self):
+        fe = NativeFile("scph5000.bin", "scph5000.bin", "psx", truth={"md5": "1" * 32},
+                        runners=[(True, "duckstation"), (False, "mednafen_psx")])
+        self.assertEqual(fe.cores(), ["duckstation", "libretro/mednafen_psx"])
+        self.assertEqual(NativeFile("x", "x", "psx", truth={"md5": "1" * 32}).cores(), [])
+
+    def test_runners_use_the_platform_name_of_the_profile(self):
+        scraped = {"cores": ["mednafen_psx", "duckstation"],
+                   "standalone_cores": ["duckstation"],
+                   "systems": {"sony-playstation": {"native_id": "psx", "files": []}}}
+        truth = {"systems": {"sony-playstation": {"files": [
+            {"name": "a.bin", "md5": "1" * 32, "_cores": ["beetle_psx"]},
+            {"name": "b.bin", "md5": "2" * 32, "_cores": ["duckstation"]},
+            {"name": "c.bin", "md5": "3" * 32, "_cores": ["no-such-core"]},
+        ]}}}
+        systems, _ = build_native_model(truth, scraped)
+        runners = {fe.name: fe.runners for fe in systems["psx"].files}
+        self.assertEqual(runners["a.bin"], [(False, "mednafen_psx")])
+        self.assertEqual(runners["b.bin"], [(True, "duckstation")])
+        self.assertEqual(runners["c.bin"], [])
+
+
+class RocknixNamesRunnersItsOwnWay(unittest.TestCase):
+    """ROCKNIX's es_systems comes from add_emu_core in its emulators package:
+    libretro cores under `retroarch`, standalones under their emulator. The
+    `libretro` pair Batocera writes names nothing there."""
+
+    ORIGINAL = (
+        "#!/usr/bin/env python\n"
+        "systems = {\n"
+        '    "psx": { "name": "PS1", "biosFiles": [ { "md5": "", "file": "bios/scph5501.bin" } ] },\n'
+        "}\n"
+        "def checkBios(systems, prefix, filterROMs):\n"
+        "    return {}\n"
+    )
+    PACKAGE = (
+        "  add_emu_core psx retroarch beetle_psx false\n"
+        "  add_emu_core psx duckstation duckstation-sa false\n"
+    )
+
+    def test_the_package_names_the_runner(self):
+        exporter = discover_exporters()["rocknix"]()
+        psx = NativeSystem("psx", "PS1", from_platform=True, files=[
+            NativeFile("scph5501.bin", "scph5501.bin", "psx", platform={"md5": ""}),
+            NativeFile("a.bin", "a.bin", "psx", truth={"md5": "1" * 32},
+                       runners=[(False, "beetle_psx")]),
+            NativeFile("b.bin", "b.bin", "psx", truth={"md5": "2" * 32},
+                       runners=[(True, "duckstation-sa")]),
+            NativeFile("c.bin", "c.bin", "psx", truth={"md5": "3" * 32},
+                       runners=[(False, "unknown_core")]),
+        ])
+        text = exporter.render(
+            {"psx": psx}, Report(),
+            {"rocknix-systems": self.ORIGINAL, "emulators/package.mk": self.PACKAGE},
+        )["rocknix-systems"]
+        self.assertIn('"file": "bios/a.bin", "emulator": "retroarch", "core": "beetle_psx" }', text)
+        self.assertIn('"file": "bios/b.bin", "emulator": "duckstation", "core": "duckstation-sa" }', text)
+        self.assertNotIn("c.bin", text)