From a5d7a6572de6182b97b9bf7413087b5528a429af Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:39:21 +0200 Subject: [PATCH] fix: ask for no file the emulator installs The pack leaves out what an emulator installs itself (validation.deliverable); the truth did not. RetroPie's gngeopi help asked users to copy gngeo_data.zip, which make install puts in the package's data directory, and amiberry's whdboot tree, scummvm's extra data and dosbox-x's FREECG98.BMP went into platform lists as files to bring. The truth applies deliverable per platform and per build: an entry nobody has to bring is kept and marked bundled, never added, still corrected where the platform declares it. A rebuilt archive is installed only when every member is. The runner written beside an addition is the build that needs the file (_brought_by), and the RetroPie sentence names only what its package's core must be given. --- scripts/exporter/baseline.py | 19 +++++++-- scripts/exporter/retropie_exporter.py | 8 +++- scripts/truth.py | 44 +++++++++++++++++-- tests/test_export_counts.py | 14 ++++++ tests/test_exporters.py | 42 ++++++++++++++++++ tests/test_truth_layout.py | 61 +++++++++++++++++++++++++++ 6 files changed, 180 insertions(+), 8 deletions(-) diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index 3c7b1b79..35a90d2b 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -564,6 +564,9 @@ def build_native_model( if is_label(truth_entry): report.labels += 1 continue + # The emulator installs it: no user is asked to bring it. + if truth_entry.get("bundled"): + continue # A file another core's entry already claimed is still the # platform's file, not a new one: scph101.bin, declared by two @@ -616,7 +619,9 @@ def _attach_runners(systems: dict[str, NativeSystem], scraped: dict | None) -> N The platform lists its cores by its own names (Batocera's `mednafen_psx`, Recalbox's `mednafen_psx`), our truth by profile key (`beetle_psx`), and a profile's `cores:` holds the names it answers to. - The build a platform runs an emulator as decides the runner. + The build a platform runs an emulator as decides the runner, and the + truth says which build needs the file (`_brought_by`): ROCKNIX's + armsx2-sa installs pcsx2/resources itself, the libretro build does not. """ if not scraped: return @@ -631,10 +636,18 @@ def _attach_runners(systems: dict[str, NativeSystem], scraped: dict | None) -> N if fe.platform is not None or not fe.truth: continue runners: list[tuple[bool, str]] = [] - for key in sorted(fe.truth.get("_cores") or ()): + brought = fe.truth.get("_brought_by") + layers = ( + sorted((key, build == "standalone") for key, _, build in + (b.partition(":") for b in brought)) + if brought is not None + else [(key, runs_standalone(key, profiles.get(key, {}), standalone)) + for key in sorted(fe.truth.get("_cores") or ())] + ) + for key, as_standalone in layers: profile = profiles.get(key, {}) names = [key, *(str(c) for c in profile.get("cores") or [])] - if runs_standalone(key, profile, standalone): + if as_standalone: name = next((n for n in names if n in standalone), None) runner = (True, name) if name else None else: diff --git a/scripts/exporter/retropie_exporter.py b/scripts/exporter/retropie_exporter.py index edf2309c..a3806137 100644 --- a/scripts/exporter/retropie_exporter.py +++ b/scripts/exporter/retropie_exporter.py @@ -228,7 +228,13 @@ class Exporter(BaseExporter): runs in HLE without a BIOS, yet four were proposed as required for its package because other PSX cores require them. """ - required_by = (fe.truth or {}).get("_required_by") + truth = fe.truth or {} + # A file the package installs is not one to copy: gngeopi's + # gngeo_data.zip lands in its data directory with make install. + brought = truth.get("_brought_by") + if brought is not None and not any(b.startswith(f"{core}:") for b in brought): + return False + required_by = truth.get("_required_by") if required_by is None: return fe.required return core in required_by diff --git a/scripts/truth.py b/scripts/truth.py index 2bc1e7c7..dc569291 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -16,7 +16,13 @@ from common import ( standalone_set as platform_standalone_set, ) from hashing import parse_md5_list -from validation import filter_files_by_mode, opens_by_name, read_from_system_dir +from validation import ( + filter_files_by_mode, + installed_with_emulator, + opens_by_name, + read_from_system_dir, + unread_on, +) @@ -209,6 +215,14 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non ) +def _brought(file_entry: dict, emu_name: str) -> set[str]: + """The core and build that need the file brought, empty when the + emulator installs it. An entry read without layer (a test, a caller of + its own) is brought by its core's libretro build.""" + brought = file_entry.get("_brought", f"{emu_name}:libretro") + return {brought} if brought else set() + + def _shares_hash(a: dict, b: dict) -> bool: return any( _declared(a, field) & _declared(b, field) @@ -238,6 +252,7 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None: existing["_cores"] = existing.get("_cores", set()) | {emu_name} if file_entry.get("_opens_by_name", True): existing["_named_by"] = existing.get("_named_by", set()) | {emu_name} + existing["_brought_by"] = existing.get("_brought_by", set()) | _brought(file_entry, emu_name) sr = file_entry.get("source_ref") if sr is not None: sr_key = _serialize_source_ref(sr) @@ -335,6 +350,7 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict: entry["_cores"] = {emu_name} entry["_required_by"] = {emu_name} if file_entry.get("required") else set() entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set() + entry["_brought_by"] = _brought(file_entry, emu_name) sr = file_entry.get("source_ref") entry["_source_refs"] = {_serialize_source_ref(sr)} if sr is not None else set() if db: @@ -389,7 +405,8 @@ def _archives_not_members(files: list[dict]) -> list[dict]: key = (archive, fe.get("system", "")) entry = archives.get(key) if entry is None: - entry = {"name": archive, "category": "bios_zip", "contents": []} + entry = {"name": archive, "category": "bios_zip", "contents": [], + "_brought": None} if fe.get("system"): entry["system"] = fe["system"] if fe.get("source_ref") is not None: @@ -398,6 +415,12 @@ def _archives_not_members(files: list[dict]) -> list[dict]: out.append(entry) if fe.get("required"): entry["required"] = True + # The archive is brought when one member is: gngeo_data.zip, every + # member of it installed by make install, is not a file to copy. + if "_brought" not in fe: + entry.pop("_brought", None) + elif "_brought" in entry and entry["_brought"] is None: + entry["_brought"] = fe["_brought"] entry["contents"].append({ field: fe[field] for field in ("name", "size", "crc32", "sha1", "md5") @@ -562,10 +585,19 @@ def generate_platform_truth( continue cores_profiled.add(emu_name) + # The pack's rule (validation.deliverable), per build: a file the + # emulator installs is kept and marked, one the launcher never opens + # is left out. filtered = _archives_not_members([ - {**fe, "_opens_by_name": opens_by_name(fe, profile, standalone)} + { + **fe, + "_opens_by_name": opens_by_name(fe, profile, standalone), + "_brought": None if installed_with_emulator(fe, platform_name) + else f"{emu_name}:{'standalone' if standalone else 'libretro'}", + } for view, standalone in platform_layouts(emu_name, profile, standalone_set) for fe in _system_dir_files(view, standalone) + if not unread_on(fe, platform_name) ]) on_platform = { @@ -656,6 +688,9 @@ def generate_platform_truth( fe["_cores"] = sorted(fe.get("_cores", set())) fe["_required_by"] = sorted(fe.get("_required_by", set())) fe["_named_by"] = sorted(fe.get("_named_by", set())) + fe["_brought_by"] = sorted(fe.get("_brought_by", set())) + if not fe["_brought_by"]: + fe["bundled"] = True fe["_source_refs"] = sorted(fe.get("_source_refs", set())) # Add per-system coverage cov = system_cores.get(sys_id, {}) @@ -756,7 +791,8 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict: for t_index, t_entry in enumerate(truth_files): s_position = pairs.get(t_index) if s_position is None: - if not is_label(t_entry) and not platform_declares(t_entry, scraped_files): + if (not is_label(t_entry) and not t_entry.get("bundled") + and not platform_declares(t_entry, scraped_files)): missing.append( { "name": t_entry["name"], diff --git a/tests/test_export_counts.py b/tests/test_export_counts.py index 25346058..a333788f 100644 --- a/tests/test_export_counts.py +++ b/tests/test_export_counts.py @@ -596,6 +596,20 @@ class RetroPieProposals(unittest.TestCase): self.assertTrue(RetroPie._required_for(fe, "beetle_psx")) self.assertFalse(RetroPie._required_for(fe, "pcsx_rearmed")) + def test_a_file_the_package_installs_is_not_proposed(self): + """gngeopi's help asked to copy gngeo_data.zip, which make install + puts in the package's data directory (gngeopi.yml).""" + installed = NativeFile("gngeo_data.zip", "gngeo_data.zip", "neogeo", + truth={"required": True, "_required_by": ["gngeopi"], + "_cores": ["gngeopi"], "_brought_by": [], + "bundled": True}) + brought = NativeFile("neogeo.zip", "neogeo.zip", "neogeo", + truth={"required": True, "_required_by": ["gngeopi"], + "_cores": ["gngeopi"], + "_brought_by": ["gngeopi:standalone"]}) + self.assertFalse(RetroPie._required_for(installed, "gngeopi")) + self.assertTrue(RetroPie._required_for(brought, "gngeopi")) + def test_a_list_of_alternatives_is_not_extended(self): alternatives = "Copy the required BIOS file a.rom or b.rom to $biosdir" enumeration = "Copy the required BIOS files a.rom and b.rom to $biosdir" diff --git a/tests/test_exporters.py b/tests/test_exporters.py index ff4bdc11..2c3734d1 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -373,6 +373,48 @@ class Reconciliation(unittest.TestCase): "_cores": ["duckstation"], "_named_by": []}]} self.assertNotIn("missing", _diff_system(truth_sys, {"files": []})) + def test_a_bundled_truth_entry_is_never_added(self): + truth = {"systems": {"neogeo": {"files": [ + {"name": "gngeo_data.zip", "md5": "a" * 32, "bundled": True, + "_cores": ["gngeopi"], "_named_by": ["gngeopi"], "_brought_by": []}, + {"name": "neogeo.zip", "md5": "b" * 32, + "_cores": ["gngeopi"], "_named_by": ["gngeopi"], + "_brought_by": ["gngeopi:standalone"]}, + ]}}} + systems, report = build_native_model(truth, {"systems": {}}) + self.assertEqual([fe.name for fe in systems["neogeo"].files], ["neogeo.zip"]) + self.assertEqual(report.files_added, 1) + + def test_a_bundled_entry_the_platform_declares_is_kept(self): + truth = {"systems": {"neogeo": {"files": [ + {"name": "gngeo_data.zip", "md5": "a" * 32, "bundled": True, + "_cores": ["gngeopi"], "_named_by": ["gngeopi"], "_brought_by": []}]}}} + scraped = {"systems": {"neogeo": {"native_id": "neogeo", "files": [ + {"name": "gngeo_data.zip", "destination": "gngeo_data.zip", + "native_system": "neogeo", "md5": "c" * 32}]}}} + systems, _ = build_native_model(truth, scraped) + (kept,) = systems["neogeo"].files + self.assertIsNotNone(kept.truth) + + def test_the_runner_is_the_build_that_needs_the_file(self): + """ROCKNIX runs ARMSX2 standalone only, and the libretro build's + pcsx2/resources entries went out as armsx2-sa files.""" + truth = {"systems": {"sony-playstation-2": {"files": [ + {"name": "GameIndex.yaml", "md5": "a" * 32, + "_cores": ["armsx2"], "_named_by": ["armsx2"], + "_brought_by": ["armsx2:libretro"]}]}}} + scraped = {"cores": ["armsx2"], "standalone_cores": ["armsx2"], "systems": {}} + systems, _ = build_native_model(truth, scraped) + (added,) = systems["sony-playstation-2"].files + self.assertEqual(added.runners, [(False, "armsx2")]) + + def test_the_diff_names_no_missing_bundled_file(self): + from truth import _diff_system + truth_sys = {"files": [{"name": "gngeo_data.zip", "md5": "a" * 32, "bundled": True, + "_cores": ["gngeopi"], "_named_by": ["gngeopi"], + "_brought_by": []}]} + self.assertNotIn("missing", _diff_system(truth_sys, {"files": []})) + def test_the_diff_pairs_by_the_same_rule(self): """diff_truth reads the same pairing, or it names a mismatch the export does not correct.""" diff --git a/tests/test_truth_layout.py b/tests/test_truth_layout.py index 24625459..fff48cea 100644 --- a/tests/test_truth_layout.py +++ b/tests/test_truth_layout.py @@ -339,5 +339,66 @@ class TruthRecordsWhoOpensByName(unittest.TestCase): self.assertEqual(standalone["systems"]["sega-saturn"]["files"][0]["_named_by"], []) +class TruthKnowsWhatTheEmulatorInstalls(unittest.TestCase): + """The pack leaves out a file the emulator installs itself (validation. + deliverable). The truth did not, so RetroPie's gngeopi help asked users + to copy gngeo_data.zip, which make install puts beside the package, and + RetroDECK's manifests listed xemu's xbox_hdd.qcow2.""" + + @staticmethod + def truth(files: list[dict], platform: str = "p", standalone: list | None = None, + profile_type: str = "libretro") -> dict: + profile = {"type": profile_type, "systems": ["neogeo"], "files": files} + config = {"cores": ["emu"], "standalone_cores": standalone or [], + "systems": {"neogeo": {"files": []}}} + return generate_platform_truth(platform, config, {}, {"emu": profile}) + + @staticmethod + def entry(truth: dict, name: str) -> dict: + return next(f for f in truth["systems"]["neogeo"]["files"] if f["name"] == name) + + def test_a_bundled_entry_is_kept_and_marked(self): + truth = self.truth([{"name": "data.zip", "md5": "a" * 32, "bundled": True}, + {"name": "bios.zip", "md5": "b" * 32}]) + self.assertTrue(self.entry(truth, "data.zip")["bundled"]) + self.assertNotIn("bundled", self.entry(truth, "bios.zip")) + self.assertEqual(self.entry(truth, "bios.zip")["_brought_by"], ["emu:libretro"]) + + def test_installed_on_scopes_to_the_platform(self): + files = [{"name": "data.zip", "md5": "a" * 32, "installed_on": ["retropie"]}] + self.assertTrue(self.entry(self.truth(files, "retropie"), "data.zip")["bundled"]) + self.assertNotIn("bundled", self.entry(self.truth(files, "batocera"), "data.zip")) + + def test_not_bundled_on_brings_it_back(self): + files = [{"name": "data.zip", "md5": "a" * 32, "bundled": True, + "not_bundled_on": ["batocera"]}] + self.assertNotIn("bundled", self.entry(self.truth(files, "batocera"), "data.zip")) + + def test_bundled_for_one_build_only(self): + """PPSSPP's standalone release ships its assets, the core does not.""" + files = [{"name": "assets.zip", "md5": "a" * 32, "bundled": True, "mode": "standalone"}, + {"name": "assets.zip", "md5": "a" * 32, "mode": "libretro"}] + standalone = self.truth(files, standalone=["emu"], profile_type="standalone + libretro") + self.assertTrue(self.entry(standalone, "assets.zip")["bundled"]) + libretro = self.truth(files, profile_type="standalone + libretro") + self.assertNotIn("bundled", self.entry(libretro, "assets.zip")) + + def test_an_archive_is_installed_when_every_member_is(self): + """gngeopi declares gngeo_data.zip and its members, all bundled. The + archive rebuilt from its members carried no mark, counted as brought, + and RetroPie's help named it as a file to copy.""" + files = [{"name": "data.zip", "md5": "a" * 32, "bundled": True}, + {"name": "cmc42.xor", "archive": "data.zip", "crc32": "11111111", + "bundled": True}] + self.assertTrue(self.entry(self.truth(files), "data.zip")["bundled"]) + files[1] = {**files[1], "bundled": False} + self.assertNotIn("bundled", self.entry(self.truth(files), "data.zip")) + + def test_unread_on_leaves_the_entry_out(self): + files = [{"name": "cdbios.bin", "md5": "a" * 32, "unread_on": ["retrobat"]}] + names = [f["name"] for f in self.truth(files, "retrobat")["systems"]["neogeo"].get("files", [])] + self.assertEqual(names, []) + + if __name__ == "__main__": unittest.main()