fix: export only what the platform declares

platforms/retroarch.yml

@@ -384,6 +384,7 @@ systems:
     - name: qsound.zip
       destination: qsound.zip
       required: true
+      curated: true
     native_id: Arcade
     core: fbneo
     manufacturer: Various
@@ -1907,24 +1908,31 @@ systems:
     - name: gc-ntsc-12.bin
       destination: dolphin-emu/Sys/GC/USA/IPL.bin
       required: false
+      curated: true
     - name: gc-pal-12.bin
       destination: dolphin-emu/Sys/GC/EUR/IPL.bin
       required: false
+      curated: true
     - name: gc-ntsc-12.bin
       destination: dolphin-emu/Sys/GC/JAP/IPL.bin
       required: false
+      curated: true
     - name: dsp_coef.bin
       destination: dolphin-emu/Sys/GC/dsp_coef.bin
       required: true
+      curated: true
     - name: dsp_rom.bin
       destination: dolphin-emu/Sys/GC/dsp_rom.bin
       required: true
+      curated: true
     - name: font_western.bin
       destination: dolphin-emu/Sys/GC/font_western.bin
       required: false
+      curated: true
     - name: font_japanese.bin
       destination: dolphin-emu/Sys/GC/font_japanese.bin
       required: false
+      curated: true
     native_id: Nintendo - GameCube
     core: dolphin
     manufacturer: Nintendo
@@ -1997,15 +2005,19 @@ systems:
     - name: dsi_bios7.bin
       destination: dsi_bios7.bin
       required: true
+      curated: true
     - name: dsi_bios9.bin
       destination: dsi_bios9.bin
       required: true
+      curated: true
     - name: dsi_firmware.bin
       destination: dsi_firmware.bin
       required: true
+      curated: true
     - name: dsi_nand.bin
       destination: dsi_nand.bin
       required: true
+      curated: true
     native_id: Nintendo - Nintendo DS
     core: desmume
     manufacturer: Nintendo
@@ -2554,6 +2566,7 @@ systems:
     - name: segasp.zip
       destination: dc/segasp.zip
       required: true
+      curated: true
     native_id: Sega - Dreamcast-based Arcade
   sega-game-gear:
     files:
@@ -4731,6 +4744,7 @@ systems:
     - name: data.zip
       destination: xrick/data.zip
       required: true
+      curated: true
     core: xrick
     manufacturer: Other
     docs: "https://docs.libretro.com/library/xrick/"

platforms/romm.yml

@@ -787,6 +787,7 @@ systems:
       crc32: 51a12d90
       size: 32768
       native_system: turbografx-cd
+      mirror_of: tg16
     - name: gexpress.pce
       destination: turbografx-cd/gexpress.pce
       required: true
@@ -795,6 +796,7 @@ systems:
       crc32: 51a12d90
       size: 32768
       native_system: turbografx-cd
+      mirror_of: tg16
     - name: syscard1.pce
       destination: turbografx-cd/syscard1.pce
       required: true
@@ -803,6 +805,7 @@ systems:
       crc32: 3f9f95a4
       size: 262144
       native_system: turbografx-cd
+      mirror_of: tg16
     - name: syscard2.pce
       destination: turbografx-cd/syscard2.pce
       required: true
@@ -811,6 +814,7 @@ systems:
       crc32: 283b74e0
       size: 262144
       native_system: turbografx-cd
+      mirror_of: tg16
     - name: syscard2u.pce
       destination: turbografx-cd/syscard2u.pce
       required: true
@@ -819,6 +823,7 @@ systems:
       crc32: ff2a5ec3
       size: 262144
       native_system: turbografx-cd
+      mirror_of: tg16
     - name: syscard3.pce
       destination: turbografx-cd/syscard3.pce
       required: true
@@ -827,6 +832,7 @@ systems:
       crc32: 6d9a73ef
       size: 262144
       native_system: turbografx-cd
+      mirror_of: tg16
     - name: syscard3u.pce
       destination: turbografx-cd/syscard3u.pce
       required: true
@@ -835,6 +841,7 @@ systems:
       crc32: 2b5b75fe
       size: 262144
       native_system: turbografx-cd
+      mirror_of: tg16
   nec-pc-98:
     files:
     - name: 2608_bd.wav

schemas/platform.schema.json

@@ -280,6 +280,14 @@
           "type": "string",
           "description": "System id the platform itself uses for this file"
         },
+        "mirror_of": {
+          "type": "string",
+          "description": "Native system whose entry this one repeats; the platform's own file does not carry the copy"
+        },
+        "curated": {
+          "type": "boolean",
+          "description": "Added by our scraper from the cores' source, not declared in the platform's own file"
+        },
         "native_name": {
           "type": "string",
           "description": "Display name the platform gives its own system"

scripts/common.py

@@ -159,7 +159,11 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
                         key = (gf.get("name"), gf.get("destination", gf.get("name")))
                         dest_lower = gf.get("destination", gf.get("name", "")).lower()
                         if key not in existing and dest_lower not in existing_lower:
-                            system.setdefault("files", []).append(gf)
+                            # Marked as ours: the platform's own file does not
+                            # carry it, and an export must not count it kept.
+                            system.setdefault("files", []).append(
+                                {**gf, "shared_group": group_name}
+                            )
                             existing.add(key)
 
     # Merge metadata from _registry.yml. The registry is our curated source;

scripts/export_native.py

@@ -205,7 +205,34 @@ def load_inputs(
         scraped = load_platform_config(platform, platforms_dir)
     except (FileNotFoundError, OSError):
         scraped = None
-    return truth, scraped
+    return truth, _native_layer(scraped)
+
+
+def _native_layer(config: dict | None) -> dict | None:
+    """The platform config without what we add to it.
+
+    A shared group's files join the config for the pack, but the platform's
+    file does not carry them: counted kept, they were written into
+    libretro's System.dat (N88EXT0-3.ROM) as if libretro declared them. What
+    a core needs from a group reaches the export through the truth, as an
+    addition the format's gates apply to. A scraper's mirror of an entry
+    under another system (RomM's tg16 firmware under turbografx-cd), and a
+    file our scraper adds from the cores' source (Dolphin's DSP ROMs in
+    RetroArch's config), are not in the platform's file either.
+    """
+    if config is None:
+        return None
+    systems = {
+        sys_id: {
+            **system,
+            "files": [
+                f for f in system.get("files", [])
+                if not ({"shared_group", "mirror_of", "curated"} & f.keys())
+            ],
+        }
+        for sys_id, system in config.get("systems", {}).items()
+    }
+    return {**config, "systems": systems}
 
 
 def refresh_cache(

scripts/scraper/base_scraper.py

@@ -35,6 +35,9 @@ class BiosRequirement:
     # is not always derivable from ours: libretro writes iplromco.dat bare
     # but ep128emu/roms/cpc464.rom with its directory.
     native_path: str | None = None
+    # The native system whose entry this one repeats under another system.
+    # The platform's own file does not carry the copy: an export skips it.
+    mirror_of: str | None = None
     # Fields the platform declares that have no equivalent in our model.
     # Kept verbatim so the native file can be written back unchanged.
     native: dict[str, object] = field(default_factory=dict)
@@ -95,6 +98,8 @@ def requirement_entry(req: BiosRequirement) -> dict:
         entry["native_system"] = req.native_id
     if req.native_path and req.native_path != req.name:
         entry["native_path"] = req.native_path
+    if req.mirror_of:
+        entry["mirror_of"] = req.mirror_of
     for key in sorted(req.native):
         value = req.native[key]
         if value not in (None, "", [], {}):

scripts/scraper/libretro_scraper.py

@@ -279,9 +279,14 @@ class Scraper(BaseScraper):
                 "docs": "https://docs.libretro.com/library/xrick/",
             },
         }
+        # Every entry below is ours, not System.dat's: marked curated so an
+        # export of System.dat does not take it for libretro's.
         for sys_id, sys_data in EXTRA_SYSTEMS.items():
             if sys_id not in systems:
-                systems[sys_id] = sys_data
+                systems[sys_id] = {
+                    **sys_data,
+                    "files": [{**f, "curated": True} for f in sys_data["files"]],
+                }
 
         # Arcade BIOS present in the repo but absent from System.dat.
         # FBNeo expects them in system/ or system/fbneo/.
@@ -310,7 +315,7 @@ class Scraper(BaseScraper):
             existing = {f["name"] for f in systems["arcade"].get("files", [])}
             for ef in EXTRA_ARCADE_FILES:
                 if ef["name"] not in existing:
-                    systems["arcade"]["files"].append(ef)
+                    systems["arcade"]["files"].append({**ef, "curated": True})
 
         # segasp.zip for Sega System SP (Flycast)
         if "sega-dreamcast-arcade" in systems:
@@ -323,6 +328,7 @@ class Scraper(BaseScraper):
                         "name": "segasp.zip",
                         "destination": "dc/segasp.zip",
                         "required": True,
+                        "curated": True,
                     }
                 )
 
@@ -442,7 +448,7 @@ class Scraper(BaseScraper):
                 for ef in extra_files:
                     key = (ef["name"], ef.get("destination", ef["name"]))
                     if key not in existing:
-                        systems[sys_id]["files"].append(ef)
+                        systems[sys_id]["files"].append({**ef, "curated": True})
                         existing.add(key)
 
         # ep128emu shared group for Enterprise

scripts/scraper/romm_scraper.py

@@ -175,6 +175,7 @@ class Scraper(BaseScraper):
                         destination=f"{slug}/{filename}",
                         required=True,
                         native_id=slug,
+                        mirror_of=None if slug == igdb_slug else igdb_slug,
                     )
                 )
 

tests/test_export_native_layer.py

@@ -0,0 +1,68 @@
+"""An export starts from the platform's own file, not from what we add to it.
+
+load_platform_config folds _shared.yml groups into a platform for the pack,
+the RomM scraper repeats tg16's firmware under turbografx-cd, and the
+libretro scraper adds files it traced in the cores' source. None of them is
+in the platform's file. The export counted them kept and wrote them into
+libretro's System.dat (N88EXT0-3.ROM) as if libretro declared them.
+"""
+
+from __future__ import annotations
+
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import common  # noqa: E402
+from export_native import load_inputs  # noqa: E402
+
+
+class TheNativeLayerIsThePlatformsOwn(unittest.TestCase):
+    def setUp(self):
+        common._platform_config_cache.clear()
+        common._shared_yml_cache.clear()
+        self.addCleanup(common._platform_config_cache.clear)
+        self.addCleanup(common._shared_yml_cache.clear)
+
+    def test_shared_and_mirrored_entries_are_left_out(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            platforms = Path(tmp)
+            (platforms / "_shared.yml").write_text(
+                "shared_groups:\n  quasi88:\n"
+                "    - name: N88EXT0.ROM\n      destination: quasi88/N88EXT0.ROM\n"
+            )
+            (platforms / "demo.yml").write_text(
+                "platform: Demo\nsystems:\n"
+                "  nec-pc-88:\n    includes: [quasi88]\n    files:\n"
+                "      - name: n88.rom\n        destination: quasi88/n88.rom\n"
+                "  tg16:\n    files:\n"
+                "      - name: syscard3.pce\n        destination: tg16/syscard3.pce\n"
+                "      - name: syscard3.pce\n        destination: turbografx-cd/syscard3.pce\n"
+                "        mirror_of: tg16\n"
+                "  nintendo-gamecube:\n    files:\n"
+                "      - name: dsp_rom.bin\n        destination: dolphin-emu/Sys/GC/dsp_rom.bin\n"
+                "        curated: true\n"
+            )
+            pack_view = common.load_platform_config("demo", str(platforms))
+            _truth, native = load_inputs("demo", platforms, str(platforms))
+
+        self.assertEqual(
+            [f["name"] for f in pack_view["systems"]["nec-pc-88"]["files"]],
+            ["n88.rom", "N88EXT0.ROM"],
+        )
+        self.assertEqual(
+            [f["name"] for f in native["systems"]["nec-pc-88"]["files"]], ["n88.rom"]
+        )
+        self.assertEqual(
+            [f["destination"] for f in native["systems"]["tg16"]["files"]],
+            ["tg16/syscard3.pce"],
+        )
+        self.assertEqual(native["systems"]["nintendo-gamecube"]["files"], [])
+
+
+if __name__ == "__main__":
+    unittest.main()