chore: merge pack and verify fixes

tests/test_audit_regressions.py

@@ -2059,6 +2059,44 @@ class CheckoutCompletenessRegressions(unittest.TestCase):
             finally:
                 os.chdir(cwd)
 
+    def test_restore_refuses_a_copy_replaced_after_the_index(self):
+        """The cache is hashed first and copied later: another run that
+        replaced the cached name in between put its bytes at a path the
+        database pins to the old ones."""
+        from scripts import restore_large_files
+
+        with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
+            root = Path(directory)
+            cache = root / "cache"
+            cache.mkdir()
+            payload = b"firmware bytes"
+            (cache / "asset.bin").write_bytes(payload)
+            sha1 = hashlib.sha1(payload).hexdigest()
+            (root / ".gitignore").write_text("bios/Sony/big.pup\n", encoding="utf-8")
+            (root / "database.json").write_text(
+                json.dumps({"files": {sha1: {"path": "bios/Sony/big.pup"}}}),
+                encoding="utf-8",
+            )
+            real_index = restore_large_files.index_cache
+
+            def index_then_replace(cache_dir: str) -> dict[str, str]:
+                index = real_index(cache_dir)
+                (cache / "asset.bin").write_bytes(b"re-uploaded bytes")
+                return index
+
+            cwd = os.getcwd()
+            os.chdir(root)
+            try:
+                with mock.patch.object(restore_large_files, "index_cache", index_then_replace):
+                    restored, unsatisfied = restore_large_files.restore(
+                        str(cache), "database.json", ".gitignore"
+                    )
+                self.assertEqual((restored, unsatisfied), (0, ["bios/Sony/big.pup"]))
+                self.assertFalse((root / "bios/Sony/big.pup").exists())
+                self.assertEqual(os.listdir(root / "bios/Sony"), [])
+            finally:
+                os.chdir(cwd)
+
     def test_restore_leaves_tracked_paths_alone(self):
         from scripts.restore_large_files import restore
 

tests/test_case_insensitive_platforms.py

@@ -7,7 +7,12 @@ went red. The flag lived in retrobat.yml and vanished at the next scrape.
 
 from __future__ import annotations
 
+import contextlib
+import hashlib
+import io
+import os
 import sys
+import tempfile
 import unittest
 from pathlib import Path
 
@@ -29,5 +34,82 @@ class CaseInsensitiveWhereTheFilesystemIs(unittest.TestCase):
                     self.assertTrue(config.get("case_insensitive_fs"))
 
 
+class ThePackReadmeNamesTheSpellingsItLeavesOut(unittest.TestCase):
+    """The EmuDeck pack is folded for Windows and extracted on a Steam Deck,
+    where the core asking for bios.col finds only BIOS.col. Only the
+    installer adds that spelling (case_variants), and the pack's README said
+    nothing about it."""
+
+    def setUp(self):
+        import common
+        import generate_db
+        import generate_pack
+
+        self.gp = generate_pack
+        tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
+        self.addCleanup(tmp.cleanup)
+        previous = os.getcwd()
+        os.chdir(tmp.name)
+        self.addCleanup(os.chdir, previous)
+        Path("bios/Coleco").mkdir(parents=True)
+        payload = b"coleco bios"
+        Path("bios/Coleco/BIOS.col").write_bytes(payload)
+        sha1 = hashlib.sha1(payload).hexdigest()
+        files = {sha1: {"path": "bios/Coleco/BIOS.col", "name": "BIOS.col",
+                        "size": len(payload), "sha1": sha1,
+                        "md5": hashlib.md5(payload).hexdigest(),
+                        "sha256": hashlib.sha256(payload).hexdigest(),
+                        "crc32": "00000001"}}
+        self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+        Path("platforms").mkdir()
+        Path("emulators").mkdir()
+        self.config = {
+            "platform": "P", "verification_mode": "existence",
+            "case_insensitive_fs": True, "cores": ["mesence"],
+            "systems": {"coleco": {"files": [{"name": "BIOS.col"}]}},
+        }
+        Path("emulators/mesence.yml").write_text(yaml.dump({
+            "emulator": "mesence", "type": "libretro", "systems": ["coleco"],
+            "files": [{"name": "bios.col", "required": True}],
+        }))
+        common._platform_config_cache.clear()
+        common._emulator_profiles_cache.clear()
+        self.addCleanup(common._platform_config_cache.clear)
+        self.addCleanup(common._emulator_profiles_cache.clear)
+
+    def _readme(self, systems: list[str]) -> str:
+        import common
+
+        Path("platforms/p.yml").write_text(yaml.dump(self.config))
+        Path("platforms/_registry.yml").write_text(yaml.dump({"platforms": {"p": {
+            "status": "active",
+            "install": {"detect": [{"os": os_name} for os_name in systems]},
+        }}}))
+        common._platform_config_cache.clear()
+        texts: dict[str, str] = {}
+        real = self.gp._write_generated_member
+
+        def capture(zf, arcname, text):
+            texts[arcname] = text
+            return real(zf, arcname, text)
+
+        self.gp._write_generated_member = capture
+        self.addCleanup(setattr, self.gp, "_write_generated_member", real)
+        with contextlib.redirect_stdout(io.StringIO()):
+            self.gp.generate_pack(
+                "p", "platforms", self.db, "bios", "", emulators_dir="emulators",
+                offline=True, record=self.gp.PackRecord(),
+            )
+        return texts["README.txt"]
+
+    def test_a_platform_also_on_linux_names_them(self):
+        readme = self._readme(["windows", "linux"])
+        self.assertIn("NAMES THAT DIFFER ONLY BY CASE", readme)
+        self.assertIn("    bios.col  (the pack has BIOS.col)", readme)
+
+    def test_a_platform_only_on_windows_does_not(self):
+        self.assertNotIn("DIFFER ONLY BY CASE", self._readme(["windows"]))
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_foreign_tree_names.py

@@ -185,5 +185,53 @@ class RetailDashboardTree(unittest.TestCase):
                     self.assertEqual(status, "path_exact", path)
 
 
+class TheGapReportAsksAsTheEmulator(unittest.TestCase):
+    """dosbox-x's et4000.bin, no hash: pcem's et4000.BIN was its only
+    casing. The resolver, asked as dosbox-x, refuses it and the pack and the
+    installer leave the file out, while the gap report, verify's core files
+    and the manifest's held flag counted it available."""
+
+    _hold = ForeignTreeNames._hold
+
+    def setUp(self):
+        ForeignTreeNames.setUp(self)
+        self._hold("bios/Other/pcem/roms/et4000.BIN", b"E" * 64)
+        self.entry = {"name": "et4000.bin", "required": True}
+
+    def _source(self, owner: str) -> str | None:
+        from cross_reference import entry_source
+
+        by_name = self.db["indexes"]["by_name"]
+        index = {
+            "by_name": by_name, "by_name_lower": {k.lower(): k for k in by_name},
+            "by_path_suffix": {}, "by_md5": {}, "by_sha256": {}, "by_crc32": {},
+            "db_files": self.db["files"], "data_names": None,
+        }
+        return entry_source({**self.entry, "source_profile": owner}, index)
+
+    def test_another_tree_casing_is_not_held(self):
+        self.assertIsNone(self._source("dosbox-x"))
+        entry = {**self.entry, "source_profile": "dosbox-x"}
+        self.assertEqual(resolve_local_file(entry, self.db), (None, "not_found"))
+
+    def test_the_tree_owner_holds_it(self):
+        self.assertEqual(self._source("pcem"), "bios")
+
+    def test_core_files_follow_the_resolver(self):
+        from verify import find_undeclared_files
+
+        profiles = {"dosbox-x": {
+            "emulator": "DOSBox-X", "type": "libretro", "systems": ["dos"],
+            "files": [self.entry],
+        }}
+        config = {"platform": "P", "verification_mode": "md5", "cores": ["dosbox-x"],
+                  "systems": {"dos": {"files": []}}}
+        (found,) = [
+            u for u in find_undeclared_files(config, "emulators", self.db, profiles)
+            if u["name"] == "et4000.bin"
+        ]
+        self.assertFalse(found["in_repo"])
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_install_selection.py

@@ -1148,6 +1148,109 @@ class AStandaloneLayoutBelowTheBiosFolder(unittest.TestCase):
         self.assertEqual({f["dest"] for f in files}, {f["dest"] for f in built["files"]})
 
 
+class AnExtraBesideItsDeclarationBelowTheBiosFolder(unittest.TestCase):
+    """RetroDECK declares bios/scph5501.bin and writes core files below bios/.
+    Beetle PSX's scph5501.bin was grouped bare, a key whose region lookup
+    another core leaves untagged, while the declaration's key resolved to
+    DuckStation's American tag: `--region japan` dropped the declaration and
+    the pack kept the file through the core's claim, while the installer,
+    which records the path once, dropped it."""
+
+    def setUp(self):
+        import common
+        import generate_db
+        import generate_pack
+
+        self.gp = generate_pack
+        tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
+        self.addCleanup(tmp.cleanup)
+        previous = os.getcwd()
+        os.chdir(tmp.name)
+        self.addCleanup(os.chdir, previous)
+        Path("bios/Sys").mkdir(parents=True)
+        files = {}
+        for name in ("us.bin", "jp.bin"):
+            payload = name.encode()
+            Path(f"bios/Sys/{name}").write_bytes(payload)
+            sha1 = hashlib.sha1(payload).hexdigest()
+            files[sha1] = {"path": f"bios/Sys/{name}", "name": name,
+                           "size": len(payload), "sha1": sha1,
+                           "md5": hashlib.md5(payload).hexdigest(),
+                           "sha256": hashlib.sha256(payload).hexdigest(),
+                           "crc32": "00000001"}
+        self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+        Path("platforms").mkdir()
+        Path("platforms/p.yml").write_text(json.dumps({
+            "platform": "P", "version": "2.1", "verification_mode": "existence",
+            "base_destination": "", "cores": ["beetle", "dsa", "other"],
+            "standalone_cores": ["dsa"], "systems": {"sys": {"files": [
+                {"name": name, "destination": f"bios/{name}", "required": False}
+                for name in ("us.bin", "jp.bin")
+            ]}},
+        }))
+        Path("platforms/_registry.yml").write_text(
+            json.dumps({"platforms": {"p": {"status": "active"}}})
+        )
+        Path("emulators").mkdir()
+        Path("emulators/dsa.yml").write_text(json.dumps({
+            "emulator": "dsa", "type": "standalone + libretro", "systems": ["sys"],
+            "files": [
+                {"name": name, "standalone_path": f"bios/{name}", "system": "sys",
+                 "region": [region_name], "required": False}
+                for name, region_name in (("us.bin", "north-america"), ("jp.bin", "japan"))
+            ]}))
+        # Read before dsa, as beetle_psx is before duckstation, and untagged
+        # by name through a second reader, as pcsx1 leaves scph5501.bin.
+        Path("emulators/beetle.yml").write_text(json.dumps({
+            "emulator": "beetle", "type": "libretro", "systems": ["sys"], "files": [
+                {"name": "us.bin", "system": "sys", "region": ["north-america"],
+                 "required": False},
+            ]}))
+        Path("emulators/other.yml").write_text(json.dumps({
+            "emulator": "other", "type": "libretro", "systems": ["sys"], "files": [
+                {"name": "us.bin", "path": "alt/us.bin", "system": "sys", "required": False},
+            ]}))
+        common._platform_config_cache.clear()
+        common._emulator_profiles_cache.clear()
+        self.addCleanup(common._platform_config_cache.clear)
+        self.addCleanup(common._emulator_profiles_cache.clear)
+        self.profiles = common.load_emulator_profiles("emulators")
+
+    def _manifest(self, regions):
+        return self.gp.generate_manifest(
+            "p", "platforms", self.db, "bios", "platforms/_registry.yml",
+            emulators_dir="emulators", emu_profiles=self.profiles,
+            regions=regions, offline=True,
+        )
+
+    def _pack(self, regions):
+        record = self.gp.PackRecord()
+        self.gp.generate_pack(
+            "p", "platforms", self.db, "bios", "", emulators_dir="emulators",
+            emu_profiles=self.profiles, regions=regions, offline=True, record=record,
+        )
+        return {m for m in record.members if m.startswith("bios/")}
+
+    def test_the_slot_choices_are_computed_once(self):
+        """pack_overrides recomputed the choices the manifest had just
+        computed with the same arguments: 8% of a manifest run."""
+        import slots
+
+        real = slots.slot_choices
+        with mock.patch.object(slots, "slot_choices", side_effect=real) as counted:
+            self._manifest(None)
+        self.assertEqual(counted.call_count, 1)
+
+    def test_the_pack_and_the_installer_drop_the_same_files(self):
+        full = self._manifest(None)
+        files, _omitted = install.narrow(
+            full["files"], full["omitted_files"], [], [], ["japan"]
+        )
+        installed = {f["dest"] for f in files if f["dest"].startswith("bios/")}
+        self.assertIn("bios/jp.bin", installed)
+        self.assertEqual(self._pack(["japan"]), installed)
+
+
 class ACoreWithNothingHeldIsAnOmission(unittest.TestCase):
     """DOSBox serves SC-55/WAVEROM3.BIN, DOSBox Pure wants it and nothing is
     held for it. A pack built for a target without DOSBox lists it as left

tests/test_large_file_cache.py

@@ -158,9 +158,9 @@ class LargeFileCacheTest(unittest.TestCase):
             raise AssertionError("must not download when the cache is valid")
 
         largefiles.urllib.request.urlopen = fail
-        self.assertEqual(
-            common.fetch_large_file("asset.bin", dest_dir=self.dir), str(cached)
-        )
+        self.assertTrue(os.path.samefile(
+            common.fetch_large_file("asset.bin", dest_dir=self.dir), cached
+        ))
 
     def test_offline_cache_miss_never_opens_the_network(self):
         def fail(req, timeout=None):
@@ -182,15 +182,15 @@ class LargeFileCacheTest(unittest.TestCase):
             raise AssertionError("offline cache hit must not open the network")
 
         largefiles.urllib.request.urlopen = fail
-        self.assertEqual(
+        self.assertTrue(os.path.samefile(
             common.fetch_large_file(
                 "asset.bin",
                 dest_dir=self.dir,
                 expected_sha1=hashlib.sha1(PAYLOAD_A).hexdigest(),
                 offline=True,
             ),
-            str(cached),
-        )
+            cached,
+        ))
 
     def _serve(self, payload: bytes, gets: list):
         class _Head(io.BytesIO):
@@ -222,7 +222,7 @@ class LargeFileCacheTest(unittest.TestCase):
             "asset.bin", dest_dir=self.dir,
             expected_sha1=hashlib.sha1(rebuilt).hexdigest(),
         )
-        self.assertEqual(result, str(cached))
+        self.assertTrue(os.path.samefile(result, cached))
         self.assertEqual(cached.read_bytes(), rebuilt)
         self.assertEqual(len(gets), 1)
 
@@ -582,7 +582,7 @@ class ReleaseAssetNames(unittest.TestCase):
                 offline=True,
                 registered=self.REGISTERED,
             )
-            self.assertEqual(got, os.path.join(cache, names[self.RTCW]))
+            self.assertTrue(os.path.samefile(got, os.path.join(cache, names[self.RTCW])))
             # The cache entry of the other asset is a valid file, not stale.
             self.assertTrue(Path(cache, names[self.ET]).exists())
 
@@ -762,7 +762,7 @@ class ASameSizeReuploadIsSeen(unittest.TestCase):
                 "dsi_nand.bin", str(self.dir), hashlib.sha1(new).hexdigest(), "",
                 offline=False,
             )
-        self.assertEqual(found, str(self.cached))
+        self.assertTrue(os.path.samefile(found, self.cached))
 
     def test_the_release_check_compares_digests_at_equal_size(self):
         expected = {"bios/Nintendo/DSi/dsi_nand.bin": 8}
@@ -790,5 +790,51 @@ class ASameSizeReuploadIsSeen(unittest.TestCase):
         )
 
 
+class AVerifiedPathKeepsItsBytes(unittest.TestCase):
+    """fetch_large_file verified a cached copy and returned its name. Another
+    run that found the release re-uploaded then replaced that name with the
+    new revision, and the first run packed bytes it never verified, under a
+    manifest sha1 that describes the old ones."""
+
+    def setUp(self):
+        _test_registry(self)
+        self.dir = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
+        self.addCleanup(shutil.rmtree, self.dir, True)
+        (self.dir / "asset.bin").write_bytes(PAYLOAD_A)
+        self.served = {"asset.bin": hashlib.sha256(PAYLOAD_B).hexdigest()}
+
+    def _fetch(self, payload: bytes, offline: bool) -> str | None:
+        def urlopen(req, timeout=None):
+            return _SlowResponse(PAYLOAD_B)
+
+        with mock.patch.object(largefiles, "_served_digests", lambda: self.served), \
+                mock.patch.object(largefiles.urllib.request, "urlopen", urlopen):
+            return largefiles.fetch_large_file(
+                "asset.bin", dest_dir=str(self.dir),
+                expected_sha1=hashlib.sha1(payload).hexdigest(), offline=offline,
+            )
+
+    def test_a_revision_promoted_after_the_check_is_not_read(self):
+        first = self._fetch(PAYLOAD_A, offline=True)
+        second = self._fetch(PAYLOAD_B, offline=False)
+        self.assertEqual((self.dir / "asset.bin").read_bytes(), PAYLOAD_B)
+        self.assertEqual(Path(second).read_bytes(), PAYLOAD_B)
+        self.assertEqual(Path(first).read_bytes(), PAYLOAD_A)
+        self.assertEqual(Path(first).name, "asset.bin")
+
+    def test_an_orphaned_revision_is_dropped_after_the_grace(self):
+        first = self._fetch(PAYLOAD_A, offline=True)
+        self._fetch(PAYLOAD_B, offline=False)
+        self.assertTrue(Path(first).exists())
+        with mock.patch.object(largefiles, "ORPHAN_GRACE", -1):
+            largefiles._prune_verified(str(self.dir))
+        self.assertFalse(Path(first).exists())
+        self.assertEqual((self.dir / "asset.bin").read_bytes(), PAYLOAD_B)
+        self.assertEqual(
+            [p.name for p in (self.dir / largefiles.VERIFIED_STORE).iterdir()],
+            [hashlib.sha1(PAYLOAD_B).hexdigest()],
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_split_extras.py

@@ -7,6 +7,7 @@ and in no part (4507 of RetroArch's 5430).
 
 from __future__ import annotations
 
+import hashlib
 import sys
 import tempfile
 import unittest
@@ -19,7 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "scripts"))
 
 from common import build_zip_contents_index, compute_hashes, load_emulator_profiles  # noqa: E402
-from generate_pack import generate_split_packs  # noqa: E402
+from generate_pack import PackRecord, generate_pack, generate_split_packs  # noqa: E402
 
 
 class SplitPartsCoverTheExtras(unittest.TestCase):
@@ -70,5 +71,172 @@ class SplitPartsCoverTheExtras(unittest.TestCase):
             self.assertIn("core_c.bin", members)
 
 
+class SplitPartsHoldWhatTheFullPackHolds(unittest.TestCase):
+    """A part sees neither the other systems' declarations nor their extras.
+
+    RetroDECK's Other_Cores part wrote bios/SGB1.sfc/program.rom beside the
+    SNES part's file bios/SGB1.sfc, a file/directory clash the full pack
+    refuses. RetroBat's ColecoVision part wrote openmsx/... beside the MSX
+    part's openMSX/..., case twins the full pack folds. A path two systems
+    declare got each system's file, one per part.
+    """
+
+    def _build(self, platform: dict, profiles: dict[str, dict], blobs: dict[str, bytes]):
+        tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
+        self.addCleanup(tmp.cleanup)
+        root = Path(tmp.name)
+        (root / "platforms").mkdir()
+        (root / "emulators").mkdir()
+        files = {}
+        for rel, payload in blobs.items():
+            path = root / "bios" / rel
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(payload)
+            files[rel] = (str(path), compute_hashes(str(path)))
+        by_name: dict[str, list[str]] = {}
+        for rel, (_p, h) in files.items():
+            by_name.setdefault(Path(rel).name, []).append(h["sha1"])
+        db = {
+            "files": {
+                h["sha1"]: {"name": Path(rel).name, "md5": h["md5"], "sha1": h["sha1"],
+                            "sha256": h["sha256"], "path": p, "size": 1}
+                for rel, (p, h) in files.items()
+            },
+            "indexes": {
+                "by_md5": {h["md5"]: h["sha1"] for _p, h in files.values()},
+                "by_name": by_name, "by_crc32": {}, "by_path_suffix": {},
+            },
+        }
+        (root / "platforms" / "_registry.yml").write_text(
+            yaml.dump({"platforms": {"plat": {"status": "active"}}})
+        )
+        (root / "platforms" / "plat.yml").write_text(yaml.dump(platform))
+        for name, profile in profiles.items():
+            (root / "emulators" / f"{name}.yml").write_text(yaml.dump(profile))
+        args = dict(
+            emulators_dir=str(root / "emulators"),
+            zip_contents=build_zip_contents_index(db),
+            emu_profiles=load_emulator_profiles(str(root / "emulators")),
+            offline=True,
+        )
+        full = PackRecord()
+        generate_pack(
+            "plat", str(root / "platforms"), db, str(root / "bios"), "", record=full, **args
+        )
+        parts: dict[str, PackRecord] = {}
+        generate_split_packs(
+            "plat", str(root / "platforms"), db, str(root / "bios"), str(root / "dist"),
+            records=parts, **args,
+        )
+        self.assertFalse((root / "dist").exists())
+        return full, parts
+
+    def _assert_parts_add_up(self, full: PackRecord, parts: dict[str, PackRecord]) -> None:
+        notes = {"README.txt"}
+        union = {m for r in parts.values() for m in r.members} - notes
+        self.assertEqual(union, set(full.members) - notes)
+        for record in parts.values():
+            for member, source in record.sources.items():
+                self.assertEqual(
+                    compute_hashes(source)["sha1"],
+                    compute_hashes(full.sources[member])["sha1"], member,
+                )
+
+    def test_a_file_directory_conflict_stays_out_of_every_part(self):
+        full, parts = self._build(
+            {
+                "platform": "Plat", "verification_mode": "existence", "cores": ["sgb"],
+                "systems": {"snes": {"files": [{"name": "SGB1.sfc"}]}},
+            },
+            {"sgb": {
+                "emulator": "SGB", "type": "libretro", "systems": ["sgb-other"],
+                "files": [{"name": "program.rom", "path": "SGB1.sfc/program.rom"}],
+            }},
+            {"SGB1.sfc": b"cartridge", "sgb/program.rom": b"program"},
+        )
+        self.assertNotIn("SGB1.sfc/program.rom", full.members)
+        self._assert_parts_add_up(full, parts)
+
+    def test_a_case_twin_is_written_once(self):
+        full, parts = self._build(
+            {
+                "platform": "Plat", "verification_mode": "existence",
+                "case_insensitive_fs": True, "cores": ["msxcore"],
+                "systems": {
+                    "msx": {"files": [{"name": "disk.rom", "destination": "openMSX/disk.rom"}]},
+                    "coleco": {"files": []},
+                },
+            },
+            {"msxcore": {
+                "emulator": "MSX Core", "type": "libretro", "systems": ["msx", "coleco"],
+                "files": [{"name": "disk.rom", "path": "openmsx/disk.rom"}],
+            }},
+            {"MSX/disk.rom": b"disk"},
+        )
+        self.assertNotIn("openmsx/disk.rom", full.members)
+        self._assert_parts_add_up(full, parts)
+
+    def test_a_path_two_systems_declare_holds_one_file(self):
+        full, parts = self._build(
+            {
+                "platform": "Plat", "verification_mode": "existence",
+                "systems": {
+                    "coco": {"files": [{
+                        "name": "disk.rom", "sha1": hashlib.sha1(b"coco disk").hexdigest(),
+                    }]},
+                    "pc88": {"files": [{
+                        "name": "disk.rom", "sha1": hashlib.sha1(b"pc-88 disk").hexdigest(),
+                    }]},
+                },
+            },
+            {},
+            {"CoCo/disk.rom": b"coco disk", "PC88/disk.rom": b"pc-88 disk"},
+        )
+        self._assert_parts_add_up(full, parts)
+
+
+class PipelineComparesTheSplitParts(unittest.TestCase):
+    """Step 5 compared the --region, --target, --system, --required-only and
+    --no-game-data packs with verify, and nothing compared the --split parts
+    with the full pack: RetroDECK's SGB1.sfc file/directory clash and
+    RetroBat's 21 openmsx/openMSX twins passed every check."""
+
+    def setUp(self):
+        import pipeline
+
+        self.pipeline = pipeline
+
+    def _record(self, sources: dict[str, str]) -> PackRecord:
+        record = PackRecord()
+        record.members = [*sources, "README.txt"]
+        record.sources = dict(sources)
+        return record
+
+    def test_equal_parts_report_nothing(self):
+        full = self._record({"a.bin": "x", "b.bin": "y"})
+        parts = {"A": self._record({"a.bin": "x"}), "B": self._record({"b.bin": "y"})}
+        self.assertEqual(self.pipeline.split_differences(full, parts, str.__eq__), [])
+
+    def test_each_difference_is_named(self):
+        full = self._record({"SGB1.sfc": "x", "kept.bin": "k", "lost.bin": "l"})
+        parts = {
+            "Snes": self._record({"SGB1.sfc": "x", "kept.bin": "other"}),
+            "Other": self._record({"SGB1.sfc/program.rom": "p"}),
+        }
+        self.assertEqual(
+            self.pipeline.split_differences(full, parts, str.__eq__),
+            [
+                "in a part, not in the full pack: SGB1.sfc/program.rom",
+                "in the full pack, in no part: lost.bin",
+                "other bytes in Snes: kept.bin",
+            ],
+        )
+
+    def test_step_five_runs_the_comparison(self):
+        source = (REPO_ROOT / "scripts" / "pipeline.py").read_text()
+        step = source[source.index("# Step 5: Consistency check"):source.index("# Step 6:")]
+        self.assertIn("check_split_packs(", step)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_system_ids.py

@@ -73,6 +73,18 @@ class PlatformNamesReachTheProfileSystem(unittest.TestCase):
             with self.subTest(alias=alias):
                 self.assertEqual(_norm_system_id(alias), _norm_system_id(canonical))
 
+    def test_each_id_is_normalized_once(self):
+        """A manifest run asked 1.7 million times for about a thousand ids,
+        1.5 seconds of prefix loops: the answer is kept per id."""
+        _norm_system_id.cache_clear()
+        for _ in range(3):
+            _norm_system_id("Nintendo_WiiU")
+        info = _norm_system_id.cache_info()
+        self.assertEqual((info.misses, info.hits), (1, 2))
+        self.assertEqual(
+            _norm_system_id("Nintendo_WiiU"), _norm_system_id.__wrapped__("Nintendo_WiiU")
+        )
+
 
 class AFoldedClaimKeepsItsProof(unittest.TestCase):
     """One spelling per machine folds bsnes-hd's SGB1.sfc onto bsnes's: the

tests/test_twin_declaration_status.py

@@ -0,0 +1,98 @@
+"""A path declared twice, once unpinned and once with a pin the copy misses.
+
+The unpinned declaration ships the file and the pinned one refuses it. The
+builder wrote "excluded" over the shipped path, so its summary said "1 unsafe
+excluded" for a file inside the archive, while verify counted it untested
+and listed no exclusion. Both orders of the two systems are checked: the
+refusal read first left "excluded" behind the file the second one ships.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import io
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import common  # noqa: E402
+import generate_db  # noqa: E402
+import generate_pack as builder  # noqa: E402
+from verify import Status, verify_platform  # noqa: E402
+
+PAYLOAD = b"a console firmware"
+SHA1 = hashlib.sha1(PAYLOAD).hexdigest()
+
+
+class TwinDeclarationStatus(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
+        self._cwd = os.getcwd()
+        os.chdir(self._tmp.name)
+        for d in ("emulators", "platforms", "bios/Twin"):
+            Path(d).mkdir(parents=True)
+        Path("bios/Twin/fw.bin").write_bytes(PAYLOAD)
+        files = {SHA1: {
+            "path": "bios/Twin/fw.bin", "name": "fw.bin", "size": len(PAYLOAD),
+            "sha1": SHA1, "md5": hashlib.md5(PAYLOAD).hexdigest(),
+            "sha256": hashlib.sha256(PAYLOAD).hexdigest(), "crc32": "00000001",
+        }}
+        self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+        Path("platforms/_registry.yml").write_text(
+            yaml.dump({"platforms": {"twin": {"status": "active"}}})
+        )
+        self.addCleanup(common._platform_config_cache.clear)
+
+    def tearDown(self):
+        os.chdir(self._cwd)
+        self._tmp.cleanup()
+
+    def _config(self, unpinned: str, pinned: str) -> dict:
+        config = {
+            "platform": "Twin", "verification_mode": "md5",
+            "systems": {
+                unpinned: {"files": [{"name": "fw.bin", "destination": "fw.bin"}]},
+                pinned: {"files": [{
+                    "name": "fw.bin", "destination": "fw.bin", "md5": "0" * 32,
+                }]},
+            },
+        }
+        Path("platforms/twin.yml").write_text(yaml.dump(config))
+        common._platform_config_cache.clear()
+        return config
+
+    def _check(self, config: dict) -> None:
+        record = builder.PackRecord()
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            builder.generate_pack(
+                "twin", "platforms", self.db, "bios", "out",
+                emulators_dir="emulators", emu_profiles={}, record=record,
+                offline=True,
+            )
+        self.assertIn("fw.bin", record.members)
+        summary = out.getvalue()
+        self.assertNotIn("excluded", summary.lower())
+        self.assertIn("0/1 files OK, 1 untested", summary)
+        self.assertIn("UNTESTED: fw.bin", summary)
+        report = verify_platform(config, self.db, "emulators", {})
+        self.assertEqual(report["status_counts"], {Status.UNTESTED: 1})
+        self.assertEqual(report["excluded_files"], [])
+
+    def test_unpinned_declaration_read_first(self):
+        self._check(self._config("aaa", "bbb"))
+
+    def test_pinned_declaration_read_first(self):
+        self._check(self._config("zzz", "bbb"))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_verify_data_dir_notice.py

@@ -0,0 +1,74 @@
+"""verify --platform names the data directories that were never refreshed.
+
+A platform entry can resolve only through a data directory tree (AmberELEC's
+Machines/*/config.ini in the blueMSX tree). On a checkout where data/ was
+never refreshed, verify counted 200 such entries missing with no word on the
+cause, while an online pack build refreshes the trees and packs them.
+verify --emulator already named its uncached trees.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_db  # noqa: E402
+from verify import print_platform_result, verify_platform  # noqa: E402
+
+
+class UncachedDataDirectoryNotice(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
+        self._cwd = os.getcwd()
+        os.chdir(self._tmp.name)
+        Path("emulators").mkdir()
+        Path("data/present").mkdir(parents=True)
+        self.db = {"files": {}, "indexes": generate_db.build_indexes({}, {})}
+        self.config = {
+            "platform": "Tree", "verification_mode": "md5",
+            "systems": {"msx": {"files": [{
+                "name": "config.ini", "destination": "Machines/MSX/config.ini",
+            }]}},
+        }
+        self.registry = {
+            "absent": {"local_cache": "data/absent"},
+            "present": {"local_cache": "data/present"},
+        }
+
+    def tearDown(self):
+        os.chdir(self._cwd)
+        self._tmp.cleanup()
+
+    def _run(self) -> tuple[dict, str]:
+        result = verify_platform(
+            self.config, self.db, "emulators", {}, data_dir_registry=self.registry,
+        )
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            print_platform_result(result, ["Tree"])
+        return result, out.getvalue()
+
+    def test_an_uncached_tree_is_named(self):
+        result, printed = self._run()
+        self.assertEqual(result["data_dir_notices"], ["absent"])
+        self.assertIn("Data directories not cached (1)", printed)
+        self.assertIn("refresh_data_dirs.py", printed)
+        self.assertIn(": absent", printed)
+
+    def test_no_note_when_every_tree_is_cached(self):
+        Path("data/absent").mkdir()
+        result, printed = self._run()
+        self.assertEqual(result["data_dir_notices"], [])
+        self.assertNotIn("refresh_data_dirs", printed)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_verify_release_asset.py

@@ -29,7 +29,7 @@ import common  # noqa: E402
 import generate_db  # noqa: E402
 import generate_pack as builder  # noqa: E402
 import packresolve  # noqa: E402
-from verify import Status, verify_platform  # noqa: E402
+from verify import Status, verify_emulator, verify_platform  # noqa: E402
 
 PAYLOAD = b"a firmware image the repository does not commit"
 SHA1 = hashlib.sha1(PAYLOAD).hexdigest()
@@ -176,6 +176,65 @@ class CoreExtraReleaseAssetNotCached(ReleaseAssetNotCached):
     test_the_online_build_packs_it = None
 
 
+class EmulatorReleaseAssetNotCached(ReleaseAssetNotCached):
+    """verify --emulator against generate_emulator_pack: the emulator pack
+    resolves with offline=offline and fetched the asset online, while verify
+    forced offline resolution and called it missing."""
+
+    def setUp(self):
+        super().setUp()
+        self.profile = {
+            "emulator": "Big Core", "type": "libretro", "systems": ["sys"],
+            "files": [{"name": "big.bin", "required": True, "sha1": SHA1}],
+        }
+        Path("emulators/bigcore.yml").write_text(yaml.dump(self.profile))
+        common._emulator_profiles_cache.clear()
+
+    def _report(self) -> dict:
+        common._emulator_profiles_cache.clear()
+        return verify_emulator(["bigcore"], "emulators", self.db, platforms_dir="platforms")
+
+    def _entry(self) -> dict:
+        return next(d for d in self._report()["details"] if d["name"] == "big.bin")
+
+    def test_counted_as_fetched_when_the_pack_is_built(self):
+        entry = self._entry()
+        self.assertEqual(entry["status"], Status.OK)
+        self.assertEqual(entry["release_asset"], REGISTERED[0])
+
+    def test_its_path_is_a_pack_destination(self):
+        self.assertIn("big.bin", self._report()["pack_destinations"])
+
+    def test_offline_counts_it_missing(self):
+        packresolve.set_offline(True)
+        self.assertEqual(self._entry()["status"], Status.MISSING)
+        self.assertNotIn("big.bin", self._report()["pack_destinations"])
+
+    def test_an_asset_with_other_content_is_not_counted(self):
+        self.profile["files"][0]["sha1"] = "0" * 40
+        Path("emulators/bigcore.yml").write_text(yaml.dump(self.profile))
+        self.assertEqual(self._entry()["status"], Status.MISSING)
+
+    def test_an_archive_asset_is_counted_too(self):
+        self.profile["files"] = [{"name": "inner.rom", "archive": "big.bin"}]
+        Path("emulators/bigcore.yml").write_text(yaml.dump(self.profile))
+        self.assertIn("big.bin", self._report()["pack_destinations"])
+
+    def test_the_online_build_packs_it(self):
+        cache = Path("cache/big.bin")
+        cache.parent.mkdir()
+        cache.write_bytes(PAYLOAD)
+        record = builder.PackRecord()
+        with mock.patch.object(
+            packresolve, "fetch_large_file", return_value=str(cache)
+        ):
+            builder.generate_emulator_pack(
+                ["bigcore"], "emulators", self.db, "bios", "out",
+                offline=False, platforms_dir="platforms", record=record,
+            )
+        self.assertEqual(set(record.files), set(self._report()["pack_destinations"]))
+
+
 class OfflinePipelineVerifiesOffline(unittest.TestCase):
     """The offline pipeline builds packs without release assets: its verify
     step has to count them the same way, or step 5 compares two answers."""