diff --git a/scripts/common.py b/scripts/common.py index 8c9d91f3..b158f6bb 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -201,6 +201,33 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") - return config +def native_layer(config: dict | None) -> dict | None: + """The platform config without what we add to it. + + A shared group's files join the config for the pack, but the platform's + file does not carry them: counted kept, they were written into + libretro's System.dat (N88EXT0-3.ROM) as if libretro declared them. What + a core needs from a group reaches the export through the truth, as an + addition the format's gates apply to. A scraper's mirror of an entry + under another system (RomM's tg16 firmware under turbografx-cd), and a + file our scraper adds from the cores' source (Dolphin's DSP ROMs in + RetroArch's config), are not in the platform's file either. + """ + if config is None: + return None + systems = { + sys_id: { + **system, + "files": [ + f for f in system.get("files", []) + if not ({"shared_group", "mirror_of", "curated"} & f.keys()) + ], + } + for sys_id, system in config.get("systems", {}).items() + } + return {**config, "systems": systems} + + def load_data_dir_registry(platforms_dir: str = "platforms") -> dict: """Load the data directory registry from _data_dirs.yml.""" registry_path = os.path.join(platforms_dir, "_data_dirs.yml") diff --git a/scripts/diff_truth.py b/scripts/diff_truth.py index d1f42733..a9b2b387 100644 --- a/scripts/diff_truth.py +++ b/scripts/diff_truth.py @@ -16,7 +16,13 @@ import os import sys sys.path.insert(0, os.path.dirname(__file__)) -from common import list_registered_platforms, load_platform_config, require_yaml, yaml_load +from common import ( + list_registered_platforms, + load_platform_config, + native_layer, + require_yaml, + yaml_load, +) from truth import diff_platform_truth yaml = require_yaml() @@ -188,7 +194,9 @@ def main() -> None: continue try: - scraped = load_platform_config(platform, args.platforms_dir) + # What the platform's own file declares: our shared groups, + # mirrors and curated additions are not theirs to be credited. + scraped = native_layer(load_platform_config(platform, args.platforms_dir)) except FileNotFoundError: print(f"skip {platform}: no scraped config", file=sys.stderr) continue diff --git a/scripts/export_native.py b/scripts/export_native.py index 8997647c..584de668 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -26,7 +26,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from artifacts import file_lock, write_bytes_atomic, write_text_atomic -from common import list_registered_platforms, load_platform_config, yaml_load +from common import list_registered_platforms, load_platform_config, native_layer, yaml_load from exporter import discover_exporters from exporter.baseline import HASH_FIELDS, _hash_values, _single_size, build_native_model @@ -206,34 +206,7 @@ def load_inputs( scraped = load_platform_config(platform, platforms_dir) except (FileNotFoundError, OSError): scraped = None - return truth, _native_layer(scraped) - - -def _native_layer(config: dict | None) -> dict | None: - """The platform config without what we add to it. - - A shared group's files join the config for the pack, but the platform's - file does not carry them: counted kept, they were written into - libretro's System.dat (N88EXT0-3.ROM) as if libretro declared them. What - a core needs from a group reaches the export through the truth, as an - addition the format's gates apply to. A scraper's mirror of an entry - under another system (RomM's tg16 firmware under turbografx-cd), and a - file our scraper adds from the cores' source (Dolphin's DSP ROMs in - RetroArch's config), are not in the platform's file either. - """ - if config is None: - return None - systems = { - sys_id: { - **system, - "files": [ - f for f in system.get("files", []) - if not ({"shared_group", "mirror_of", "curated"} & f.keys()) - ], - } - for sys_id, system in config.get("systems", {}).items() - } - return {**config, "systems": systems} + return truth, native_layer(scraped) def refresh_cache( diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 031221d7..c6065b4b 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -38,6 +38,7 @@ from common import ( load_database, load_emulator_profiles, load_provenance_snapshots, + native_layer, require_yaml, resolution_is_hash_exact, resolve_local_file, @@ -2742,7 +2743,7 @@ def generate_gap_analysis( truth_data = generate_platform_truth( pname, cov["config"], (registry or {}).get(pname, {}), profiles, db ) - s = diff_platform_truth(truth_data, cov["config"])["summary"] + s = diff_platform_truth(truth_data, native_layer(cov["config"]))["summary"] if s["systems_compared"] == 0: div_rows.append( f"| [{cov['platform']}](platforms/{pname}.md) | - | - | - | - |" diff --git a/tests/test_export_native_layer.py b/tests/test_export_native_layer.py index dcdb2abf..1cbcdb91 100644 --- a/tests/test_export_native_layer.py +++ b/tests/test_export_native_layer.py @@ -64,5 +64,65 @@ class TheNativeLayerIsThePlatformsOwn(unittest.TestCase): self.assertEqual(native["systems"]["nintendo-gamecube"]["files"], []) +class TheTruthDiffReadsTheSameLayer(unittest.TestCase): + """diff_truth credited libretro with the files our scraper adds. + + Compared against the pack view, RetroArch's curated DSP ROMs, fonts and + DSi files counted as declared by System.dat: 1008 missing instead of + 1016, while the export of the same input added them as missing. + """ + + def test_a_curated_file_is_missing_from_the_platform(self): + import json # noqa: PLC0415 + import subprocess # noqa: PLC0415 + + with tempfile.TemporaryDirectory(dir=str(REPO_ROOT / "tmp")) as tmp: + root = Path(tmp) + (root / "demo.yml").write_text( + "platform: Demo\nsystems:\n" + " nintendo-gamecube:\n files:\n" + " - name: dsp_rom.bin\n destination: dolphin-emu/Sys/GC/dsp_rom.bin\n" + " curated: true\n" + " - name: IPL.bin\n destination: dolphin-emu/Sys/GC/USA/IPL.bin\n" + ) + truth = root / "truth" + truth.mkdir() + (truth / "demo.yml").write_text( + "systems:\n nintendo-gamecube:\n files:\n" + " - name: dsp_rom.bin\n _cores: [dolphin]\n" + " - name: IPL.bin\n _cores: [dolphin]\n" + ) + proc = subprocess.run( + [sys.executable, "scripts/diff_truth.py", "--platform", "demo", "--json", + "--platforms-dir", str(root), "--truth-dir", str(truth)], + capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=300, check=False, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + (report,) = json.loads(proc.stdout) + missing = [ + entry["name"] + for system in report["divergences"].values() + for entry in system.get("missing", []) + ] + self.assertEqual(missing, ["dsp_rom.bin"]) + + def test_every_truth_diff_caller_reads_the_native_layer(self): + """The site's platform-vs-source table answers the same question.""" + import re # noqa: PLC0415 + + for path in sorted((REPO_ROOT / "scripts").glob("*.py")): + text = path.read_text(encoding="utf-8") + for match in re.finditer( + r"(?