fix: ask for no file the emulator installs

The pack leaves out what an emulator installs itself
(validation.deliverable); the truth did not. RetroPie's gngeopi help
asked users to copy gngeo_data.zip, which make install puts in the
package's data directory, and amiberry's whdboot tree, scummvm's extra
data and dosbox-x's FREECG98.BMP went into platform lists as files to
bring.

The truth applies deliverable per platform and per build: an entry
nobody has to bring is kept and marked bundled, never added, still
corrected where the platform declares it. A rebuilt archive is
installed only when every member is. The runner written beside an
addition is the build that needs the file (_brought_by), and the
RetroPie sentence names only what its package's core must be given.

scripts/exporter/baseline.py

@@ -564,6 +564,9 @@ def build_native_model(
             if is_label(truth_entry):
                 report.labels += 1
                 continue
+            # The emulator installs it: no user is asked to bring it.
+            if truth_entry.get("bundled"):
+                continue
 
             # A file another core's entry already claimed is still the
             # platform's file, not a new one: scph101.bin, declared by two
@@ -616,7 +619,9 @@ def _attach_runners(systems: dict[str, NativeSystem], scraped: dict | None) -> N
     The platform lists its cores by its own names (Batocera's
     `mednafen_psx`, Recalbox's `mednafen_psx`), our truth by profile key
     (`beetle_psx`), and a profile's `cores:` holds the names it answers to.
-    The build a platform runs an emulator as decides the runner.
+    The build a platform runs an emulator as decides the runner, and the
+    truth says which build needs the file (`_brought_by`): ROCKNIX's
+    armsx2-sa installs pcsx2/resources itself, the libretro build does not.
     """
     if not scraped:
         return
@@ -631,10 +636,18 @@ def _attach_runners(systems: dict[str, NativeSystem], scraped: dict | None) -> N
             if fe.platform is not None or not fe.truth:
                 continue
             runners: list[tuple[bool, str]] = []
-            for key in sorted(fe.truth.get("_cores") or ()):
+            brought = fe.truth.get("_brought_by")
+            layers = (
+                sorted((key, build == "standalone") for key, _, build in
+                       (b.partition(":") for b in brought))
+                if brought is not None
+                else [(key, runs_standalone(key, profiles.get(key, {}), standalone))
+                      for key in sorted(fe.truth.get("_cores") or ())]
+            )
+            for key, as_standalone in layers:
                 profile = profiles.get(key, {})
                 names = [key, *(str(c) for c in profile.get("cores") or [])]
-                if runs_standalone(key, profile, standalone):
+                if as_standalone:
                     name = next((n for n in names if n in standalone), None)
                     runner = (True, name) if name else None
                 else:

scripts/exporter/retropie_exporter.py

@@ -228,7 +228,13 @@ class Exporter(BaseExporter):
         runs in HLE without a BIOS, yet four were proposed as required for
         its package because other PSX cores require them.
         """
-        required_by = (fe.truth or {}).get("_required_by")
+        truth = fe.truth or {}
+        # A file the package installs is not one to copy: gngeopi's
+        # gngeo_data.zip lands in its data directory with make install.
+        brought = truth.get("_brought_by")
+        if brought is not None and not any(b.startswith(f"{core}:") for b in brought):
+            return False
+        required_by = truth.get("_required_by")
         if required_by is None:
             return fe.required
         return core in required_by

scripts/truth.py

@@ -16,7 +16,13 @@ from common import (
     standalone_set as platform_standalone_set,
 )
 from hashing import parse_md5_list
-from validation import filter_files_by_mode, opens_by_name, read_from_system_dir
+from validation import (
+    filter_files_by_mode,
+    installed_with_emulator,
+    opens_by_name,
+    read_from_system_dir,
+    unread_on,
+)
 
 
 
@@ -209,6 +215,14 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non
     )
 
 
+def _brought(file_entry: dict, emu_name: str) -> set[str]:
+    """The core and build that need the file brought, empty when the
+    emulator installs it. An entry read without layer (a test, a caller of
+    its own) is brought by its core's libretro build."""
+    brought = file_entry.get("_brought", f"{emu_name}:libretro")
+    return {brought} if brought else set()
+
+
 def _shares_hash(a: dict, b: dict) -> bool:
     return any(
         _declared(a, field) & _declared(b, field)
@@ -238,6 +252,7 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None:
     existing["_cores"] = existing.get("_cores", set()) | {emu_name}
     if file_entry.get("_opens_by_name", True):
         existing["_named_by"] = existing.get("_named_by", set()) | {emu_name}
+    existing["_brought_by"] = existing.get("_brought_by", set()) | _brought(file_entry, emu_name)
     sr = file_entry.get("source_ref")
     if sr is not None:
         sr_key = _serialize_source_ref(sr)
@@ -335,6 +350,7 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
     entry["_cores"] = {emu_name}
     entry["_required_by"] = {emu_name} if file_entry.get("required") else set()
     entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set()
+    entry["_brought_by"] = _brought(file_entry, emu_name)
     sr = file_entry.get("source_ref")
     entry["_source_refs"] = {_serialize_source_ref(sr)} if sr is not None else set()
     if db:
@@ -389,7 +405,8 @@ def _archives_not_members(files: list[dict]) -> list[dict]:
         key = (archive, fe.get("system", ""))
         entry = archives.get(key)
         if entry is None:
-            entry = {"name": archive, "category": "bios_zip", "contents": []}
+            entry = {"name": archive, "category": "bios_zip", "contents": [],
+                     "_brought": None}
             if fe.get("system"):
                 entry["system"] = fe["system"]
             if fe.get("source_ref") is not None:
@@ -398,6 +415,12 @@ def _archives_not_members(files: list[dict]) -> list[dict]:
             out.append(entry)
         if fe.get("required"):
             entry["required"] = True
+        # The archive is brought when one member is: gngeo_data.zip, every
+        # member of it installed by make install, is not a file to copy.
+        if "_brought" not in fe:
+            entry.pop("_brought", None)
+        elif "_brought" in entry and entry["_brought"] is None:
+            entry["_brought"] = fe["_brought"]
         entry["contents"].append({
             field: fe[field]
             for field in ("name", "size", "crc32", "sha1", "md5")
@@ -562,10 +585,19 @@ def generate_platform_truth(
             continue
         cores_profiled.add(emu_name)
 
+        # The pack's rule (validation.deliverable), per build: a file the
+        # emulator installs is kept and marked, one the launcher never opens
+        # is left out.
         filtered = _archives_not_members([
-            {**fe, "_opens_by_name": opens_by_name(fe, profile, standalone)}
+            {
+                **fe,
+                "_opens_by_name": opens_by_name(fe, profile, standalone),
+                "_brought": None if installed_with_emulator(fe, platform_name)
+                else f"{emu_name}:{'standalone' if standalone else 'libretro'}",
+            }
             for view, standalone in platform_layouts(emu_name, profile, standalone_set)
             for fe in _system_dir_files(view, standalone)
+            if not unread_on(fe, platform_name)
         ])
 
         on_platform = {
@@ -656,6 +688,9 @@ def generate_platform_truth(
             fe["_cores"] = sorted(fe.get("_cores", set()))
             fe["_required_by"] = sorted(fe.get("_required_by", set()))
             fe["_named_by"] = sorted(fe.get("_named_by", set()))
+            fe["_brought_by"] = sorted(fe.get("_brought_by", set()))
+            if not fe["_brought_by"]:
+                fe["bundled"] = True
             fe["_source_refs"] = sorted(fe.get("_source_refs", set()))
         # Add per-system coverage
         cov = system_cores.get(sys_id, {})
@@ -756,7 +791,8 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
     for t_index, t_entry in enumerate(truth_files):
         s_position = pairs.get(t_index)
         if s_position is None:
-            if not is_label(t_entry) and not platform_declares(t_entry, scraped_files):
+            if (not is_label(t_entry) and not t_entry.get("bundled")
+                    and not platform_declares(t_entry, scraped_files)):
                 missing.append(
                     {
                         "name": t_entry["name"],

tests/test_export_counts.py

@@ -596,6 +596,20 @@ class RetroPieProposals(unittest.TestCase):
         self.assertTrue(RetroPie._required_for(fe, "beetle_psx"))
         self.assertFalse(RetroPie._required_for(fe, "pcsx_rearmed"))
 
+    def test_a_file_the_package_installs_is_not_proposed(self):
+        """gngeopi's help asked to copy gngeo_data.zip, which make install
+        puts in the package's data directory (gngeopi.yml)."""
+        installed = NativeFile("gngeo_data.zip", "gngeo_data.zip", "neogeo",
+                               truth={"required": True, "_required_by": ["gngeopi"],
+                                      "_cores": ["gngeopi"], "_brought_by": [],
+                                      "bundled": True})
+        brought = NativeFile("neogeo.zip", "neogeo.zip", "neogeo",
+                             truth={"required": True, "_required_by": ["gngeopi"],
+                                    "_cores": ["gngeopi"],
+                                    "_brought_by": ["gngeopi:standalone"]})
+        self.assertFalse(RetroPie._required_for(installed, "gngeopi"))
+        self.assertTrue(RetroPie._required_for(brought, "gngeopi"))
+
     def test_a_list_of_alternatives_is_not_extended(self):
         alternatives = "Copy the required BIOS file a.rom or b.rom to $biosdir"
         enumeration = "Copy the required BIOS files a.rom and b.rom to $biosdir"

tests/test_exporters.py

@@ -373,6 +373,48 @@ class Reconciliation(unittest.TestCase):
                                 "_cores": ["duckstation"], "_named_by": []}]}
         self.assertNotIn("missing", _diff_system(truth_sys, {"files": []}))
 
+    def test_a_bundled_truth_entry_is_never_added(self):
+        truth = {"systems": {"neogeo": {"files": [
+            {"name": "gngeo_data.zip", "md5": "a" * 32, "bundled": True,
+             "_cores": ["gngeopi"], "_named_by": ["gngeopi"], "_brought_by": []},
+            {"name": "neogeo.zip", "md5": "b" * 32,
+             "_cores": ["gngeopi"], "_named_by": ["gngeopi"],
+             "_brought_by": ["gngeopi:standalone"]},
+        ]}}}
+        systems, report = build_native_model(truth, {"systems": {}})
+        self.assertEqual([fe.name for fe in systems["neogeo"].files], ["neogeo.zip"])
+        self.assertEqual(report.files_added, 1)
+
+    def test_a_bundled_entry_the_platform_declares_is_kept(self):
+        truth = {"systems": {"neogeo": {"files": [
+            {"name": "gngeo_data.zip", "md5": "a" * 32, "bundled": True,
+             "_cores": ["gngeopi"], "_named_by": ["gngeopi"], "_brought_by": []}]}}}
+        scraped = {"systems": {"neogeo": {"native_id": "neogeo", "files": [
+            {"name": "gngeo_data.zip", "destination": "gngeo_data.zip",
+             "native_system": "neogeo", "md5": "c" * 32}]}}}
+        systems, _ = build_native_model(truth, scraped)
+        (kept,) = systems["neogeo"].files
+        self.assertIsNotNone(kept.truth)
+
+    def test_the_runner_is_the_build_that_needs_the_file(self):
+        """ROCKNIX runs ARMSX2 standalone only, and the libretro build's
+        pcsx2/resources entries went out as armsx2-sa files."""
+        truth = {"systems": {"sony-playstation-2": {"files": [
+            {"name": "GameIndex.yaml", "md5": "a" * 32,
+             "_cores": ["armsx2"], "_named_by": ["armsx2"],
+             "_brought_by": ["armsx2:libretro"]}]}}}
+        scraped = {"cores": ["armsx2"], "standalone_cores": ["armsx2"], "systems": {}}
+        systems, _ = build_native_model(truth, scraped)
+        (added,) = systems["sony-playstation-2"].files
+        self.assertEqual(added.runners, [(False, "armsx2")])
+
+    def test_the_diff_names_no_missing_bundled_file(self):
+        from truth import _diff_system
+        truth_sys = {"files": [{"name": "gngeo_data.zip", "md5": "a" * 32, "bundled": True,
+                                "_cores": ["gngeopi"], "_named_by": ["gngeopi"],
+                                "_brought_by": []}]}
+        self.assertNotIn("missing", _diff_system(truth_sys, {"files": []}))
+
     def test_the_diff_pairs_by_the_same_rule(self):
         """diff_truth reads the same pairing, or it names a mismatch the
         export does not correct."""

tests/test_truth_layout.py

@@ -339,5 +339,66 @@ class TruthRecordsWhoOpensByName(unittest.TestCase):
         self.assertEqual(standalone["systems"]["sega-saturn"]["files"][0]["_named_by"], [])
 
 
+class TruthKnowsWhatTheEmulatorInstalls(unittest.TestCase):
+    """The pack leaves out a file the emulator installs itself (validation.
+    deliverable). The truth did not, so RetroPie's gngeopi help asked users
+    to copy gngeo_data.zip, which make install puts beside the package, and
+    RetroDECK's manifests listed xemu's xbox_hdd.qcow2."""
+
+    @staticmethod
+    def truth(files: list[dict], platform: str = "p", standalone: list | None = None,
+              profile_type: str = "libretro") -> dict:
+        profile = {"type": profile_type, "systems": ["neogeo"], "files": files}
+        config = {"cores": ["emu"], "standalone_cores": standalone or [],
+                  "systems": {"neogeo": {"files": []}}}
+        return generate_platform_truth(platform, config, {}, {"emu": profile})
+
+    @staticmethod
+    def entry(truth: dict, name: str) -> dict:
+        return next(f for f in truth["systems"]["neogeo"]["files"] if f["name"] == name)
+
+    def test_a_bundled_entry_is_kept_and_marked(self):
+        truth = self.truth([{"name": "data.zip", "md5": "a" * 32, "bundled": True},
+                            {"name": "bios.zip", "md5": "b" * 32}])
+        self.assertTrue(self.entry(truth, "data.zip")["bundled"])
+        self.assertNotIn("bundled", self.entry(truth, "bios.zip"))
+        self.assertEqual(self.entry(truth, "bios.zip")["_brought_by"], ["emu:libretro"])
+
+    def test_installed_on_scopes_to_the_platform(self):
+        files = [{"name": "data.zip", "md5": "a" * 32, "installed_on": ["retropie"]}]
+        self.assertTrue(self.entry(self.truth(files, "retropie"), "data.zip")["bundled"])
+        self.assertNotIn("bundled", self.entry(self.truth(files, "batocera"), "data.zip"))
+
+    def test_not_bundled_on_brings_it_back(self):
+        files = [{"name": "data.zip", "md5": "a" * 32, "bundled": True,
+                  "not_bundled_on": ["batocera"]}]
+        self.assertNotIn("bundled", self.entry(self.truth(files, "batocera"), "data.zip"))
+
+    def test_bundled_for_one_build_only(self):
+        """PPSSPP's standalone release ships its assets, the core does not."""
+        files = [{"name": "assets.zip", "md5": "a" * 32, "bundled": True, "mode": "standalone"},
+                 {"name": "assets.zip", "md5": "a" * 32, "mode": "libretro"}]
+        standalone = self.truth(files, standalone=["emu"], profile_type="standalone + libretro")
+        self.assertTrue(self.entry(standalone, "assets.zip")["bundled"])
+        libretro = self.truth(files, profile_type="standalone + libretro")
+        self.assertNotIn("bundled", self.entry(libretro, "assets.zip"))
+
+    def test_an_archive_is_installed_when_every_member_is(self):
+        """gngeopi declares gngeo_data.zip and its members, all bundled. The
+        archive rebuilt from its members carried no mark, counted as brought,
+        and RetroPie's help named it as a file to copy."""
+        files = [{"name": "data.zip", "md5": "a" * 32, "bundled": True},
+                 {"name": "cmc42.xor", "archive": "data.zip", "crc32": "11111111",
+                  "bundled": True}]
+        self.assertTrue(self.entry(self.truth(files), "data.zip")["bundled"])
+        files[1] = {**files[1], "bundled": False}
+        self.assertNotIn("bundled", self.entry(self.truth(files), "data.zip"))
+
+    def test_unread_on_leaves_the_entry_out(self):
+        files = [{"name": "cdbios.bin", "md5": "a" * 32, "unread_on": ["retrobat"]}]
+        names = [f["name"] for f in self.truth(files, "retrobat")["systems"]["neogeo"].get("files", [])]
+        self.assertEqual(names, [])
+
+
 if __name__ == "__main__":
     unittest.main()