diff --git a/scripts/export_native.py b/scripts/export_native.py index 584de668..03766171 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -16,6 +16,7 @@ Usage: from __future__ import annotations import argparse +import re import json import sys import urllib.error @@ -146,10 +147,29 @@ def wanted_sources( base = pinned_base(wanted, scraped) if base: - wanted = {relative: base + relative for relative in wanted} + wanted = { + relative: _at_revision(wanted[relative], base) or base + relative + for relative in wanted + } return wanted +_RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/") + + +def _at_revision(url: str, base: str) -> str: + """A file of the pinned repository, read at the pinned revision; empty + when the URL is not a raw GitHub URL of that repository. + + ROCKNIX's checker sits in packages/rocknix/, the emulators it names in + packages/virtual/: no shared directory, one revision. + """ + pinned, other = _RAW_GITHUB.match(base), _RAW_GITHUB.match(url) + if pinned and other and pinned.group(1, 2) == other.group(1, 2): + return pinned.group(0) + url[other.end():] + return "" + + def collect_originals( exporter: object, systems: dict, @@ -347,7 +367,7 @@ def export_platform( for entry in system.files: if not entry.name: continue - if exporter.writable(entry): + if exporter.lands(entry): landed += entry.platform is None elif entry.platform is None: refused += 1 diff --git a/scripts/exporter/base_exporter.py b/scripts/exporter/base_exporter.py index c0fd4fc4..d3f37166 100644 --- a/scripts/exporter/base_exporter.py +++ b/scripts/exporter/base_exporter.py @@ -127,6 +127,11 @@ class BaseExporter(ABC): """ return False + def lands(self, fe: NativeFile) -> bool: + """Whether this render writes the entry: writable(), plus whatever + the render learned from the platform's own file.""" + return self.writable(fe) + @classmethod def writable(cls, fe: NativeFile, require: str = "") -> bool: """Whether this format can carry the entry. diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index 775adafd..0f5ac5c5 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -65,6 +65,9 @@ class NativeFile: filled: list[str] = field(default_factory=list) # An addition filed under our own slug: the platform names no such system. foreign_system: bool = False + # For an addition, each core that wants it as (runs standalone, name the + # platform gives it); empty when no core the platform runs is named. + runners: list[tuple[bool, str]] = field(default_factory=list) @property def origin(self) -> str: @@ -198,13 +201,14 @@ class NativeFile: return default def cores(self) -> list[str]: - """Cores that want this file, the platform's naming preferred.""" + """Cores that want this file, as Recalbox writes them: the declared + value, else each runner, a standalone bare and a core under + libretro/. A profile key is not a name the platform knows: + `libretro/duckstation` named no core Recalbox runs.""" declared = self.native("core", "") if declared: return [c.strip() for c in str(declared).split(",") if c.strip()] - if self.truth: - return [f"libretro/{c}" for c in self.truth.get("_cores", [])] - return [] + return [name if standalone else f"libretro/{name}" for standalone, name in self.runners] @dataclass @@ -483,9 +487,46 @@ def build_native_model( tail = [fe for fe in system.files if fe.platform is None] system.files = head + search_order(tail) + _attach_runners(systems, scraped) return systems, report +def _attach_runners(systems: dict[str, NativeSystem], scraped: dict | None) -> None: + """Name the cores behind each addition the way the platform names them. + + The platform lists its cores by its own names (Batocera's + `mednafen_psx`, Recalbox's `mednafen_psx`), our truth by profile key + (`beetle_psx`); a profile's `cores:` holds the names it answers to. + The build a platform runs an emulator as decides the runner. + """ + if not scraped: + return + from common import load_emulator_profiles, runs_standalone + + declared = scraped.get("cores") + libretro = {str(c) for c in declared} if isinstance(declared, list) else set() + standalone = {str(c) for c in scraped.get("standalone_cores") or []} + profiles = load_emulator_profiles(str(Path(__file__).resolve().parents[2] / "emulators")) + for system in systems.values(): + for fe in system.files: + if fe.platform is not None or not fe.truth: + continue + runners: list[tuple[bool, str]] = [] + for key in sorted(fe.truth.get("_cores") or ()): + profile = profiles.get(key, {}) + names = [key, *(str(c) for c in profile.get("cores") or [])] + if runs_standalone(key, profile, standalone): + name = next((n for n in names if n in standalone), None) + runner = (True, name) if name else None + else: + name = next((n for n in names if n in libretro - standalone), None) \ + or next((n for n in names if n in libretro), None) + runner = (False, name) if name else None + if runner and runner not in runners: + runners.append(runner) + fe.runners = runners + + def search_order(files: list[NativeFile]) -> list[NativeFile]: """Order files the way the code looks for them, best first. diff --git a/scripts/exporter/batocera_exporter.py b/scripts/exporter/batocera_exporter.py index 38fda274..16b22292 100644 --- a/scripts/exporter/batocera_exporter.py +++ b/scripts/exporter/batocera_exporter.py @@ -25,6 +25,17 @@ _INDENT = " " * 4 class Exporter(BaseExporter): """Write Batocera's batocera-systems, corrected.""" + # checkBios reads emulator names from es_systems, and Batocera's + # standalone_cores are those names. ROCKNIX's list holds package names + # (duckstation-sa), which its checker does not know. + STANDALONE_RUNNERS = True + + def __init__(self) -> None: + super().__init__() + # The emulator and core each system entry of the original is gated + # on, read by render. + self._gates: dict[str, tuple[str, str] | None] = {} + @staticmethod def platform_name() -> str: return "batocera" @@ -55,8 +66,60 @@ class Exporter(BaseExporter): """ if fe.platform is None and len(set(fe.hashes("md5"))) > 1: return False + if fe.platform is None and not cls._usable_runners(fe): + # Written without its core's emulator and core, checkBios asks + # every build for it (the defect _runner_keys repairs for the + # platform's own items). + return False return super().writable(fe, require) + @classmethod + def _usable_runners(cls, fe: NativeFile) -> list[tuple[bool, str]]: + return [r for r in fe.runners if cls.STANDALONE_RUNNERS or not r[0]] + + def _runner_pair(self, runner: tuple[bool, str]) -> tuple[str, str | None] | None: + """The emulator and core checkBios finds a runner under, None when + the checker has no name for it. Batocera's es_systems names a + libretro core under `libretro` and a standalone by itself.""" + standalone, name = runner + return (name, None) if standalone else ("libretro", name) + + def _addition_keys(self, fe: NativeFile) -> list[tuple[str, str | None]] | None: + """The emulator/core pairs an addition is written with, [] when its + system's own gate already names its core, None when the gate names + another: checkBios walks a gated system only when that core is + installed, so a file of another core there is asked of the wrong + users and never of its own.""" + pairs = [ + pair for r in self._usable_runners(fe) + if (pair := self._runner_pair(r)) is not None + ] + if not pairs: + return None + gate = self._gates.get(fe.native_system) + if gate is None: + return pairs + return [] if gate in pairs else None + + def exportable( + self, + systems: dict[str, NativeSystem], + require: str = "", + ) -> list[tuple[NativeSystem, list[NativeFile]]]: + """What the format can carry, less the additions its gates refuse: + render and validate read the same list.""" + result = [] + for system, files in super().exportable(systems, require): + kept = [fe for fe in files if self.lands(fe)] + if kept: + result.append((system, kept)) + return result + + def lands(self, fe: NativeFile) -> bool: + if not self.writable(fe): + return False + return fe.platform is not None or self._addition_keys(fe) is not None + def states(self, fe: NativeFile, field_name: str) -> bool: if field_name == "md5" and "md5" in fe.filled: return bool(self.single_value(fe, "md5")) @@ -96,22 +159,38 @@ class Exporter(BaseExporter): } parts: list[str] = [] for fe in files: - # The platform states an unhashed file as an empty md5 rather - # than leaving it out, and so do we. - item = [f'"md5": "{self.single_value(fe, "md5")}"'] - alt = fe.native("alt_md5", "") - if alt: - item.append(f'"altmd5": "{alt}"') - declared = fe.native("native_path", "") - path = str(declared) if declared else f"bios/{fe.destination}" - item.append(f'"file": "{path}"') - zipped = fe.native("zipped_file", "") - if zipped: - item.append(f'"zippedFile": "{zipped}"') - item.extend(self._runner_keys(originals.get(path))) - parts.append("{ " + ", ".join(item) + " }") + for runner in self._item_runners(fe): + parts.append(self._bios_item(fe, runner, originals)) return parts + def _item_runners(self, fe: NativeFile) -> list[list[str] | None]: + """One item per runner an addition names; None keeps the original's.""" + if fe.platform is not None: + return [None] + pairs = self._addition_keys(fe) or [] + if not pairs: + return [[]] + return [ + [f'"emulator": "{emulator}"'] + ([f'"core": "{core}"'] if core else []) + for emulator, core in pairs + ] + + def _bios_item(self, fe: NativeFile, runner: list[str] | None, originals: dict) -> str: + # The platform states an unhashed file as an empty md5 rather + # than leaving it out, and so do we. + item = [f'"md5": "{self.single_value(fe, "md5")}"'] + alt = fe.native("alt_md5", "") + if alt: + item.append(f'"altmd5": "{alt}"') + declared = fe.native("native_path", "") + path = str(declared) if declared else f"bios/{fe.destination}" + item.append(f'"file": "{path}"') + zipped = fe.native("zipped_file", "") + if zipped: + item.append(f'"zippedFile": "{zipped}"') + item.extend(self._runner_keys(originals.get(path)) if runner is None else runner) + return "{ " + ", ".join(item) + " }" + @staticmethod def _own_entry(system: NativeSystem, before: dict | None) -> dict: """The maintainer's entry for this system, out of the parsed pair.""" @@ -257,11 +336,6 @@ class Exporter(BaseExporter): originals: dict[str, str], scraped: dict | None = None, ) -> dict[str, str]: - exportable = dict( - (system.native_id, (system, files)) - for system, files in self.exportable(systems, require="md5") - ) - original = originals.get(self.native_filename(), "") if not original: raise ValueError( @@ -272,6 +346,18 @@ class Exporter(BaseExporter): head, body, tail = self._split_original(original) spans = self._entry_spans(body) + for key, (start, end) in spans.items(): + parsed = self._parse_entry(body[start:end + 1]) or {} + own = parsed.get(key) if isinstance(parsed.get(key), dict) else {} + emulator, core = own.get("emulator"), own.get("core") + self._gates[key] = ( + (emulator, core if isinstance(core, str) else None) + if isinstance(emulator, str) else None + ) + exportable = { + system.native_id: (system, files) + for system, files in self.exportable(systems, require="md5") + } rebuilt: list[str] = [] written: set[str] = set() diff --git a/scripts/exporter/rocknix_exporter.py b/scripts/exporter/rocknix_exporter.py index 7aed30c0..95119bf3 100644 --- a/scripts/exporter/rocknix_exporter.py +++ b/scripts/exporter/rocknix_exporter.py @@ -6,6 +6,9 @@ runs, so only the mapping is rewritten. from __future__ import annotations +import re + +from .baseline import NativeSystem, Report from .batocera_exporter import Exporter as BatoceraExporter SOURCE_URL = ( @@ -14,9 +17,47 @@ SOURCE_URL = ( ) +EMULATORS_URL = ( + "https://raw.githubusercontent.com/ROCKNIX/distribution/next/projects/ROCKNIX" + "/packages/virtual/emulators/package.mk" +) +EMULATORS_MK = "emulators/package.mk" +# `add_emu_core `: the pairs es_systems.cfg +# lists, and so the names checkBios's emuDict holds. +_ADD_EMU_CORE = re.compile(r"^\s*add_emu_core\s+(\S+)\s+(\S+)\s+(\S+)", re.MULTILINE) + + class Exporter(BatoceraExporter): """Write ROCKNIX's rocknix-systems, corrected.""" + def __init__(self) -> None: + super().__init__() + self._emulator_of: dict[str, str] = {} + + def _runner_pair(self, runner: tuple[bool, str]) -> tuple[str, str | None] | None: + """ROCKNIX registers libretro cores under `retroarch` and each + standalone under its own emulator (`azahar azahar-sa`), in the + emulators package; the `libretro` Batocera writes names nothing + there.""" + _standalone, name = runner + emulator = self._emulator_of.get(name) + return (emulator, name) if emulator else None + + def render( + self, + systems: dict[str, NativeSystem], + report: Report, + originals: dict[str, str], + scraped: dict | None = None, + ) -> dict[str, str]: + self._emulator_of = { + core: emulator + for _system, emulator, core in _ADD_EMU_CORE.findall( + originals.get(EMULATORS_MK, "") + ) + } + return super().render(systems, report, originals, scraped) + @staticmethod def platform_name() -> str: return "rocknix" @@ -27,4 +68,4 @@ class Exporter(BatoceraExporter): @staticmethod def native_sources() -> dict[str, str]: - return {"rocknix-systems": SOURCE_URL} + return {"rocknix-systems": SOURCE_URL, EMULATORS_MK: EMULATORS_URL} diff --git a/tests/test_export_counts.py b/tests/test_export_counts.py index 3d399afc..23cec8f7 100644 --- a/tests/test_export_counts.py +++ b/tests/test_export_counts.py @@ -331,7 +331,8 @@ class OneMd5FormatsAddWhole(unittest.TestCase): from exporter.batocera_exporter import Exporter as Batocera # noqa: PLC0415 several = NativeFile("boot.bin", "boot.bin", "dc", truth={"md5": [A, B, C]}) - single = NativeFile("boot2.bin", "boot2.bin", "dc", truth={"md5": A}) + single = NativeFile("boot2.bin", "boot2.bin", "dc", truth={"md5": A}, + runners=[(False, "flycast")]) self.assertFalse(Batocera.writable(several)) self.assertTrue(Batocera.writable(single)) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b0c25428..6a478ce6 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -43,6 +43,8 @@ from export_native import ( # noqa: E402 ) from exporter.baseline import ( # noqa: E402 NativeFile, + NativeSystem, + Report, build_native_model, ) @@ -85,6 +87,11 @@ TRUTH = { } SCRAPED = { + # The cores the platform runs, by its own names: an addition is written + # with the runner that wants it. + "cores": ["atari800", "duckstation", "fuse", "melonds", "o2em", "picodrive", + "snes9x", "beetle_psx"], + "standalone_cores": ["duckstation"], "systems": { "sony-playstation": { "native_id": "psx", @@ -949,6 +956,7 @@ class BatoceraExport(unittest.TestCase): self.assertTrue(foreign.foreign_system) own = next(fe for fe in systems["psx"].files if fe.name == "scph7001.bin") self.assertFalse(own.foreign_system) + self.assertEqual(own.runners, [(True, "duckstation")]) for platform in ("batocera", "rocknix", "retrobat", "recalbox"): with self.subTest(platform=platform): exporter = discover_exporters()[platform] @@ -963,7 +971,8 @@ class BatoceraExport(unittest.TestCase): def test_a_system_the_platform_names_without_files_is_its_own(self): """A system the platform declares empty still has the platform's id.""" - scraped = {"systems": {"nintendo-snes": {"native_id": "snes", "files": []}}} + scraped = {"cores": ["snes9x"], + "systems": {"nintendo-snes": {"native_id": "snes", "files": []}}} truth = {"systems": {"nintendo-snes": {"files": [{ "name": "st010.zip", "path": "st010.zip", "md5": "2" * 32, "required": False, "_cores": ["snes9x"], @@ -1389,3 +1398,109 @@ class ARecordDescribesOneDump(unittest.TestCase): source = (Path(__file__).resolve().parent.parent / "scripts" / "exporter" / module).read_text() with self.subTest(module=module): self.assertNotRegex(source, r'= fe\.hash\("(crc32|md5|sha1)"\)') + + +class AnAdditionCarriesItsRunner(unittest.TestCase): + """Batocera's apple2 entry is gated on libretro/mame; Clock Signal's + apple2.rom written inside it was asked of mame users and never of clk + users. Recalbox got core="libretro/duckstation", a core it never runs.""" + + ORIGINAL = ( + "#!/usr/bin/env python\n" + "systems = {\n" + ' "apple2": { "name": "Apple II", "emulator": "libretro", "core": "mame", ' + '"biosFiles": [ { "md5": "", "file": "bios/apple2e.zip" } ] },\n' + ' "macintosh": { "name": "Macintosh", "biosFiles": [ ' + '{ "md5": "", "file": "bios/MacII.ROM", "emulator": "libretro", "core": "minivmac" } ] },\n' + "}\n" + "def checkBios(systems, prefix, filterROMs):\n" + " return {}\n" + ) + + def _systems(self): + def add(native, name, runners): + return NativeFile(name, name, native, truth={"md5": "1" * 32}, runners=runners) + + apple2 = NativeSystem("apple2", "Apple II", from_platform=True, files=[ + NativeFile("apple2e.zip", "apple2e.zip", "apple2", platform={"md5": ""}), + add("apple2", "apple2.rom", [(True, "clk")]), + add("apple2", "apple2e.rom", [(False, "mame")]), + ]) + mac = NativeSystem("macintosh", "Macintosh", from_platform=True, files=[ + NativeFile("MacII.ROM", "MacII.ROM", "macintosh", platform={"md5": ""}), + add("macintosh", "macplus.rom", [(True, "clk")]), + add("macintosh", "nothing.rom", []), + ]) + return {"apple2": apple2, "macintosh": mac} + + def test_batocera_writes_the_runner_or_counts_the_file_out(self): + exporter = discover_exporters()["batocera"]() + systems = self._systems() + out = exporter.render(systems, Report(), {"batocera-systems": self.ORIGINAL}) + text = out["batocera-systems"] + self.assertNotIn("apple2.rom", text) + self.assertIn('"file": "bios/apple2e.rom" }', text) + self.assertIn('"file": "bios/macplus.rom", "emulator": "clk" }', text) + self.assertNotIn("nothing.rom", text) + landed = {fe.name for s in systems.values() for fe in s.files + if fe.platform is None and exporter.lands(fe)} + self.assertEqual(landed, {"apple2e.rom", "macplus.rom"}) + + def test_recalbox_names_cores_as_it_runs_them(self): + fe = NativeFile("scph5000.bin", "scph5000.bin", "psx", truth={"md5": "1" * 32}, + runners=[(True, "duckstation"), (False, "mednafen_psx")]) + self.assertEqual(fe.cores(), ["duckstation", "libretro/mednafen_psx"]) + self.assertEqual(NativeFile("x", "x", "psx", truth={"md5": "1" * 32}).cores(), []) + + def test_runners_use_the_platform_name_of_the_profile(self): + scraped = {"cores": ["mednafen_psx", "duckstation"], + "standalone_cores": ["duckstation"], + "systems": {"sony-playstation": {"native_id": "psx", "files": []}}} + truth = {"systems": {"sony-playstation": {"files": [ + {"name": "a.bin", "md5": "1" * 32, "_cores": ["beetle_psx"]}, + {"name": "b.bin", "md5": "2" * 32, "_cores": ["duckstation"]}, + {"name": "c.bin", "md5": "3" * 32, "_cores": ["no-such-core"]}, + ]}}} + systems, _ = build_native_model(truth, scraped) + runners = {fe.name: fe.runners for fe in systems["psx"].files} + self.assertEqual(runners["a.bin"], [(False, "mednafen_psx")]) + self.assertEqual(runners["b.bin"], [(True, "duckstation")]) + self.assertEqual(runners["c.bin"], []) + + +class RocknixNamesRunnersItsOwnWay(unittest.TestCase): + """ROCKNIX's es_systems comes from add_emu_core in its emulators package: + libretro cores under `retroarch`, standalones under their emulator. The + `libretro` pair Batocera writes names nothing there.""" + + ORIGINAL = ( + "#!/usr/bin/env python\n" + "systems = {\n" + ' "psx": { "name": "PS1", "biosFiles": [ { "md5": "", "file": "bios/scph5501.bin" } ] },\n' + "}\n" + "def checkBios(systems, prefix, filterROMs):\n" + " return {}\n" + ) + PACKAGE = ( + " add_emu_core psx retroarch beetle_psx false\n" + " add_emu_core psx duckstation duckstation-sa false\n" + ) + + def test_the_package_names_the_runner(self): + exporter = discover_exporters()["rocknix"]() + psx = NativeSystem("psx", "PS1", from_platform=True, files=[ + NativeFile("scph5501.bin", "scph5501.bin", "psx", platform={"md5": ""}), + NativeFile("a.bin", "a.bin", "psx", truth={"md5": "1" * 32}, + runners=[(False, "beetle_psx")]), + NativeFile("b.bin", "b.bin", "psx", truth={"md5": "2" * 32}, + runners=[(True, "duckstation-sa")]), + NativeFile("c.bin", "c.bin", "psx", truth={"md5": "3" * 32}, + runners=[(False, "unknown_core")]), + ]) + text = exporter.render( + {"psx": psx}, Report(), + {"rocknix-systems": self.ORIGINAL, "emulators/package.mk": self.PACKAGE}, + )["rocknix-systems"] + self.assertIn('"file": "bios/a.bin", "emulator": "retroarch", "core": "beetle_psx" }', text) + self.assertIn('"file": "bios/b.bin", "emulator": "duckstation", "core": "duckstation-sa" }', text) + self.assertNotIn("c.bin", text)