fix: diff the truth against the platform's own list

scripts/common.py

@@ -201,6 +201,33 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
     return config
 
 
+def native_layer(config: dict | None) -> dict | None:
+    """The platform config without what we add to it.
+
+    A shared group's files join the config for the pack, but the platform's
+    file does not carry them: counted kept, they were written into
+    libretro's System.dat (N88EXT0-3.ROM) as if libretro declared them. What
+    a core needs from a group reaches the export through the truth, as an
+    addition the format's gates apply to. A scraper's mirror of an entry
+    under another system (RomM's tg16 firmware under turbografx-cd), and a
+    file our scraper adds from the cores' source (Dolphin's DSP ROMs in
+    RetroArch's config), are not in the platform's file either.
+    """
+    if config is None:
+        return None
+    systems = {
+        sys_id: {
+            **system,
+            "files": [
+                f for f in system.get("files", [])
+                if not ({"shared_group", "mirror_of", "curated"} & f.keys())
+            ],
+        }
+        for sys_id, system in config.get("systems", {}).items()
+    }
+    return {**config, "systems": systems}
+
+
 def load_data_dir_registry(platforms_dir: str = "platforms") -> dict:
     """Load the data directory registry from _data_dirs.yml."""
     registry_path = os.path.join(platforms_dir, "_data_dirs.yml")

scripts/diff_truth.py

@@ -16,7 +16,13 @@ import os
 import sys
 
 sys.path.insert(0, os.path.dirname(__file__))
-from common import list_registered_platforms, load_platform_config, require_yaml, yaml_load
+from common import (
+    list_registered_platforms,
+    load_platform_config,
+    native_layer,
+    require_yaml,
+    yaml_load,
+)
 from truth import diff_platform_truth
 
 yaml = require_yaml()
@@ -188,7 +194,9 @@ def main() -> None:
             continue
 
         try:
-            scraped = load_platform_config(platform, args.platforms_dir)
+            # What the platform's own file declares: our shared groups,
+            # mirrors and curated additions are not theirs to be credited.
+            scraped = native_layer(load_platform_config(platform, args.platforms_dir))
         except FileNotFoundError:
             print(f"skip {platform}: no scraped config", file=sys.stderr)
             continue

scripts/export_native.py

@@ -26,7 +26,7 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
 from artifacts import file_lock, write_bytes_atomic, write_text_atomic
-from common import list_registered_platforms, load_platform_config, yaml_load
+from common import list_registered_platforms, load_platform_config, native_layer, yaml_load
 from exporter import discover_exporters
 from exporter.baseline import HASH_FIELDS, _hash_values, _single_size, build_native_model
 
@@ -206,34 +206,7 @@ def load_inputs(
         scraped = load_platform_config(platform, platforms_dir)
     except (FileNotFoundError, OSError):
         scraped = None
-    return truth, _native_layer(scraped)
-
-
-def _native_layer(config: dict | None) -> dict | None:
-    """The platform config without what we add to it.
-
-    A shared group's files join the config for the pack, but the platform's
-    file does not carry them: counted kept, they were written into
-    libretro's System.dat (N88EXT0-3.ROM) as if libretro declared them. What
-    a core needs from a group reaches the export through the truth, as an
-    addition the format's gates apply to. A scraper's mirror of an entry
-    under another system (RomM's tg16 firmware under turbografx-cd), and a
-    file our scraper adds from the cores' source (Dolphin's DSP ROMs in
-    RetroArch's config), are not in the platform's file either.
-    """
-    if config is None:
-        return None
-    systems = {
-        sys_id: {
-            **system,
-            "files": [
-                f for f in system.get("files", [])
-                if not ({"shared_group", "mirror_of", "curated"} & f.keys())
-            ],
-        }
-        for sys_id, system in config.get("systems", {}).items()
-    }
-    return {**config, "systems": systems}
+    return truth, native_layer(scraped)
 
 
 def refresh_cache(

scripts/generate_site.py

@@ -38,6 +38,7 @@ from common import (
     load_database,
     load_emulator_profiles,
     load_provenance_snapshots,
+    native_layer,
     require_yaml,
     resolution_is_hash_exact,
     resolve_local_file,
@@ -2742,7 +2743,7 @@ def generate_gap_analysis(
         truth_data = generate_platform_truth(
             pname, cov["config"], (registry or {}).get(pname, {}), profiles, db
         )
-        s = diff_platform_truth(truth_data, cov["config"])["summary"]
+        s = diff_platform_truth(truth_data, native_layer(cov["config"]))["summary"]
         if s["systems_compared"] == 0:
             div_rows.append(
                 f"| [{cov['platform']}](platforms/{pname}.md) | - | - | - | - |"

tests/test_export_native_layer.py

@@ -64,5 +64,65 @@ class TheNativeLayerIsThePlatformsOwn(unittest.TestCase):
         self.assertEqual(native["systems"]["nintendo-gamecube"]["files"], [])
 
 
+class TheTruthDiffReadsTheSameLayer(unittest.TestCase):
+    """diff_truth credited libretro with the files our scraper adds.
+
+    Compared against the pack view, RetroArch's curated DSP ROMs, fonts and
+    DSi files counted as declared by System.dat: 1008 missing instead of
+    1016, while the export of the same input added them as missing.
+    """
+
+    def test_a_curated_file_is_missing_from_the_platform(self):
+        import json  # noqa: PLC0415
+        import subprocess  # noqa: PLC0415
+
+        with tempfile.TemporaryDirectory(dir=str(REPO_ROOT / "tmp")) as tmp:
+            root = Path(tmp)
+            (root / "demo.yml").write_text(
+                "platform: Demo\nsystems:\n"
+                "  nintendo-gamecube:\n    files:\n"
+                "      - name: dsp_rom.bin\n        destination: dolphin-emu/Sys/GC/dsp_rom.bin\n"
+                "        curated: true\n"
+                "      - name: IPL.bin\n        destination: dolphin-emu/Sys/GC/USA/IPL.bin\n"
+            )
+            truth = root / "truth"
+            truth.mkdir()
+            (truth / "demo.yml").write_text(
+                "systems:\n  nintendo-gamecube:\n    files:\n"
+                "      - name: dsp_rom.bin\n        _cores: [dolphin]\n"
+                "      - name: IPL.bin\n        _cores: [dolphin]\n"
+            )
+            proc = subprocess.run(
+                [sys.executable, "scripts/diff_truth.py", "--platform", "demo", "--json",
+                 "--platforms-dir", str(root), "--truth-dir", str(truth)],
+                capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=300, check=False,
+            )
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        (report,) = json.loads(proc.stdout)
+        missing = [
+            entry["name"]
+            for system in report["divergences"].values()
+            for entry in system.get("missing", [])
+        ]
+        self.assertEqual(missing, ["dsp_rom.bin"])
+
+    def test_every_truth_diff_caller_reads_the_native_layer(self):
+        """The site's platform-vs-source table answers the same question."""
+        import re  # noqa: PLC0415
+
+        for path in sorted((REPO_ROOT / "scripts").glob("*.py")):
+            text = path.read_text(encoding="utf-8")
+            for match in re.finditer(
+                r"(?<!def )diff_platform_truth\(\s*[^,()]+,\s*(native_layer\(|\w+)", text
+            ):
+                scraped = match.group(1)
+                with self.subTest(path=path.name):
+                    self.assertTrue(
+                        scraped == "native_layer("
+                        or re.search(rf"\b{scraped} = native_layer\(", text),
+                        f"{path.name}: diff_platform_truth reads {scraped}, the pack view",
+                    )
+
+
 if __name__ == "__main__":
     unittest.main()