diff --git a/platforms/retroarch.yml b/platforms/retroarch.yml index ed29e31c..27ce473c 100644 --- a/platforms/retroarch.yml +++ b/platforms/retroarch.yml @@ -384,6 +384,7 @@ systems: - name: qsound.zip destination: qsound.zip required: true + curated: true native_id: Arcade core: fbneo manufacturer: Various @@ -1907,24 +1908,31 @@ systems: - name: gc-ntsc-12.bin destination: dolphin-emu/Sys/GC/USA/IPL.bin required: false + curated: true - name: gc-pal-12.bin destination: dolphin-emu/Sys/GC/EUR/IPL.bin required: false + curated: true - name: gc-ntsc-12.bin destination: dolphin-emu/Sys/GC/JAP/IPL.bin required: false + curated: true - name: dsp_coef.bin destination: dolphin-emu/Sys/GC/dsp_coef.bin required: true + curated: true - name: dsp_rom.bin destination: dolphin-emu/Sys/GC/dsp_rom.bin required: true + curated: true - name: font_western.bin destination: dolphin-emu/Sys/GC/font_western.bin required: false + curated: true - name: font_japanese.bin destination: dolphin-emu/Sys/GC/font_japanese.bin required: false + curated: true native_id: Nintendo - GameCube core: dolphin manufacturer: Nintendo @@ -1997,15 +2005,19 @@ systems: - name: dsi_bios7.bin destination: dsi_bios7.bin required: true + curated: true - name: dsi_bios9.bin destination: dsi_bios9.bin required: true + curated: true - name: dsi_firmware.bin destination: dsi_firmware.bin required: true + curated: true - name: dsi_nand.bin destination: dsi_nand.bin required: true + curated: true native_id: Nintendo - Nintendo DS core: desmume manufacturer: Nintendo @@ -2554,6 +2566,7 @@ systems: - name: segasp.zip destination: dc/segasp.zip required: true + curated: true native_id: Sega - Dreamcast-based Arcade sega-game-gear: files: @@ -4731,6 +4744,7 @@ systems: - name: data.zip destination: xrick/data.zip required: true + curated: true core: xrick manufacturer: Other docs: "https://docs.libretro.com/library/xrick/" diff --git a/platforms/romm.yml b/platforms/romm.yml index e2ff594a..bc3c7eba 100644 --- a/platforms/romm.yml +++ b/platforms/romm.yml @@ -787,6 +787,7 @@ systems: crc32: 51a12d90 size: 32768 native_system: turbografx-cd + mirror_of: tg16 - name: gexpress.pce destination: turbografx-cd/gexpress.pce required: true @@ -795,6 +796,7 @@ systems: crc32: 51a12d90 size: 32768 native_system: turbografx-cd + mirror_of: tg16 - name: syscard1.pce destination: turbografx-cd/syscard1.pce required: true @@ -803,6 +805,7 @@ systems: crc32: 3f9f95a4 size: 262144 native_system: turbografx-cd + mirror_of: tg16 - name: syscard2.pce destination: turbografx-cd/syscard2.pce required: true @@ -811,6 +814,7 @@ systems: crc32: 283b74e0 size: 262144 native_system: turbografx-cd + mirror_of: tg16 - name: syscard2u.pce destination: turbografx-cd/syscard2u.pce required: true @@ -819,6 +823,7 @@ systems: crc32: ff2a5ec3 size: 262144 native_system: turbografx-cd + mirror_of: tg16 - name: syscard3.pce destination: turbografx-cd/syscard3.pce required: true @@ -827,6 +832,7 @@ systems: crc32: 6d9a73ef size: 262144 native_system: turbografx-cd + mirror_of: tg16 - name: syscard3u.pce destination: turbografx-cd/syscard3u.pce required: true @@ -835,6 +841,7 @@ systems: crc32: 2b5b75fe size: 262144 native_system: turbografx-cd + mirror_of: tg16 nec-pc-98: files: - name: 2608_bd.wav diff --git a/schemas/platform.schema.json b/schemas/platform.schema.json index f7f3bde7..1523af8a 100644 --- a/schemas/platform.schema.json +++ b/schemas/platform.schema.json @@ -280,6 +280,14 @@ "type": "string", "description": "System id the platform itself uses for this file" }, + "mirror_of": { + "type": "string", + "description": "Native system whose entry this one repeats; the platform's own file does not carry the copy" + }, + "curated": { + "type": "boolean", + "description": "Added by our scraper from the cores' source, not declared in the platform's own file" + }, "native_name": { "type": "string", "description": "Display name the platform gives its own system" diff --git a/scripts/common.py b/scripts/common.py index 2541dbc3..cfacf831 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -159,7 +159,11 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") - key = (gf.get("name"), gf.get("destination", gf.get("name"))) dest_lower = gf.get("destination", gf.get("name", "")).lower() if key not in existing and dest_lower not in existing_lower: - system.setdefault("files", []).append(gf) + # Marked as ours: the platform's own file does not + # carry it, and an export must not count it kept. + system.setdefault("files", []).append( + {**gf, "shared_group": group_name} + ) existing.add(key) # Merge metadata from _registry.yml. The registry is our curated source; diff --git a/scripts/export_native.py b/scripts/export_native.py index 770043d5..df106445 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -205,7 +205,34 @@ def load_inputs( scraped = load_platform_config(platform, platforms_dir) except (FileNotFoundError, OSError): scraped = None - return truth, scraped + return truth, _native_layer(scraped) + + +def _native_layer(config: dict | None) -> dict | None: + """The platform config without what we add to it. + + A shared group's files join the config for the pack, but the platform's + file does not carry them: counted kept, they were written into + libretro's System.dat (N88EXT0-3.ROM) as if libretro declared them. What + a core needs from a group reaches the export through the truth, as an + addition the format's gates apply to. A scraper's mirror of an entry + under another system (RomM's tg16 firmware under turbografx-cd), and a + file our scraper adds from the cores' source (Dolphin's DSP ROMs in + RetroArch's config), are not in the platform's file either. + """ + if config is None: + return None + systems = { + sys_id: { + **system, + "files": [ + f for f in system.get("files", []) + if not ({"shared_group", "mirror_of", "curated"} & f.keys()) + ], + } + for sys_id, system in config.get("systems", {}).items() + } + return {**config, "systems": systems} def refresh_cache( diff --git a/scripts/scraper/base_scraper.py b/scripts/scraper/base_scraper.py index c5c84a95..290d983a 100644 --- a/scripts/scraper/base_scraper.py +++ b/scripts/scraper/base_scraper.py @@ -35,6 +35,9 @@ class BiosRequirement: # is not always derivable from ours: libretro writes iplromco.dat bare # but ep128emu/roms/cpc464.rom with its directory. native_path: str | None = None + # The native system whose entry this one repeats under another system. + # The platform's own file does not carry the copy: an export skips it. + mirror_of: str | None = None # Fields the platform declares that have no equivalent in our model. # Kept verbatim so the native file can be written back unchanged. native: dict[str, object] = field(default_factory=dict) @@ -95,6 +98,8 @@ def requirement_entry(req: BiosRequirement) -> dict: entry["native_system"] = req.native_id if req.native_path and req.native_path != req.name: entry["native_path"] = req.native_path + if req.mirror_of: + entry["mirror_of"] = req.mirror_of for key in sorted(req.native): value = req.native[key] if value not in (None, "", [], {}): diff --git a/scripts/scraper/libretro_scraper.py b/scripts/scraper/libretro_scraper.py index e3609677..ca2e661b 100644 --- a/scripts/scraper/libretro_scraper.py +++ b/scripts/scraper/libretro_scraper.py @@ -279,9 +279,14 @@ class Scraper(BaseScraper): "docs": "https://docs.libretro.com/library/xrick/", }, } + # Every entry below is ours, not System.dat's: marked curated so an + # export of System.dat does not take it for libretro's. for sys_id, sys_data in EXTRA_SYSTEMS.items(): if sys_id not in systems: - systems[sys_id] = sys_data + systems[sys_id] = { + **sys_data, + "files": [{**f, "curated": True} for f in sys_data["files"]], + } # Arcade BIOS present in the repo but absent from System.dat. # FBNeo expects them in system/ or system/fbneo/. @@ -310,7 +315,7 @@ class Scraper(BaseScraper): existing = {f["name"] for f in systems["arcade"].get("files", [])} for ef in EXTRA_ARCADE_FILES: if ef["name"] not in existing: - systems["arcade"]["files"].append(ef) + systems["arcade"]["files"].append({**ef, "curated": True}) # segasp.zip for Sega System SP (Flycast) if "sega-dreamcast-arcade" in systems: @@ -323,6 +328,7 @@ class Scraper(BaseScraper): "name": "segasp.zip", "destination": "dc/segasp.zip", "required": True, + "curated": True, } ) @@ -442,7 +448,7 @@ class Scraper(BaseScraper): for ef in extra_files: key = (ef["name"], ef.get("destination", ef["name"])) if key not in existing: - systems[sys_id]["files"].append(ef) + systems[sys_id]["files"].append({**ef, "curated": True}) existing.add(key) # ep128emu shared group for Enterprise diff --git a/scripts/scraper/romm_scraper.py b/scripts/scraper/romm_scraper.py index c386d328..020d4f17 100644 --- a/scripts/scraper/romm_scraper.py +++ b/scripts/scraper/romm_scraper.py @@ -175,6 +175,7 @@ class Scraper(BaseScraper): destination=f"{slug}/{filename}", required=True, native_id=slug, + mirror_of=None if slug == igdb_slug else igdb_slug, ) ) diff --git a/tests/test_export_native_layer.py b/tests/test_export_native_layer.py new file mode 100644 index 00000000..dcdb2abf --- /dev/null +++ b/tests/test_export_native_layer.py @@ -0,0 +1,68 @@ +"""An export starts from the platform's own file, not from what we add to it. + +load_platform_config folds _shared.yml groups into a platform for the pack, +the RomM scraper repeats tg16's firmware under turbografx-cd, and the +libretro scraper adds files it traced in the cores' source. None of them is +in the platform's file. The export counted them kept and wrote them into +libretro's System.dat (N88EXT0-3.ROM) as if libretro declared them. +""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import common # noqa: E402 +from export_native import load_inputs # noqa: E402 + + +class TheNativeLayerIsThePlatformsOwn(unittest.TestCase): + def setUp(self): + common._platform_config_cache.clear() + common._shared_yml_cache.clear() + self.addCleanup(common._platform_config_cache.clear) + self.addCleanup(common._shared_yml_cache.clear) + + def test_shared_and_mirrored_entries_are_left_out(self): + with tempfile.TemporaryDirectory() as tmp: + platforms = Path(tmp) + (platforms / "_shared.yml").write_text( + "shared_groups:\n quasi88:\n" + " - name: N88EXT0.ROM\n destination: quasi88/N88EXT0.ROM\n" + ) + (platforms / "demo.yml").write_text( + "platform: Demo\nsystems:\n" + " nec-pc-88:\n includes: [quasi88]\n files:\n" + " - name: n88.rom\n destination: quasi88/n88.rom\n" + " tg16:\n files:\n" + " - name: syscard3.pce\n destination: tg16/syscard3.pce\n" + " - name: syscard3.pce\n destination: turbografx-cd/syscard3.pce\n" + " mirror_of: tg16\n" + " nintendo-gamecube:\n files:\n" + " - name: dsp_rom.bin\n destination: dolphin-emu/Sys/GC/dsp_rom.bin\n" + " curated: true\n" + ) + pack_view = common.load_platform_config("demo", str(platforms)) + _truth, native = load_inputs("demo", platforms, str(platforms)) + + self.assertEqual( + [f["name"] for f in pack_view["systems"]["nec-pc-88"]["files"]], + ["n88.rom", "N88EXT0.ROM"], + ) + self.assertEqual( + [f["name"] for f in native["systems"]["nec-pc-88"]["files"]], ["n88.rom"] + ) + self.assertEqual( + [f["destination"] for f in native["systems"]["tg16"]["files"]], + ["tg16/syscard3.pce"], + ) + self.assertEqual(native["systems"]["nintendo-gamecube"]["files"], []) + + +if __name__ == "__main__": + unittest.main()