chore: merge round 2 fixes (integrity)

tests/test_archive_choice.py

@@ -316,6 +316,20 @@ class AnArchiveRankedByWhatItHolds(unittest.TestCase):
             for name in ("epr-21330.ic27", "epr-21332.ic27")]}
         self.assertEqual(self._resolve(entry, db), "bios/MAME/hod2bios.zip")
 
+    def test_a_copy_holding_no_declared_member_is_not_the_archive(self):
+        """galgames.zip held only an assembler listing and none of the six
+        ROMs MAME declares: the name step served it as the BIOS set."""
+        from common import resolve_local_file
+
+        db = self._db([("bios/MAME/hod2bios.zip", ["listing.lst"])])
+        entry = {"name": "hod2bios.zip", "contents": [
+            {"name": name, "crc32": format(zlib.crc32(self.roms[name]), "08x")}
+            for name in ("epr-21330.ic27", "epr-21332.ic27")]}
+        path, status = resolve_local_file(entry, db)
+        self.assertEqual(status, "hash_mismatch")
+        self.assertEqual(Path(path).relative_to(self.root).as_posix(),
+                         "bios/MAME/hod2bios.zip")
+
     def test_without_a_layout_a_superset_primary_wins(self):
         db = self._db([("bios/Arcade/hod2bios.zip", ["epr-21330.ic27"]),
                        ("bios/MAME/hod2bios.zip", list(self.roms))])

tests/test_audit_regressions.py

@@ -1276,7 +1276,7 @@ class OneBaselineVerdict(unittest.TestCase):
             self.assertTrue(digest_verdict(plain, 0, lambda: {"md5": raw}, "md5"))
 
     def test_a_release_asset_keeps_its_answer(self):
-        from scripts.slots import apply_override
+        from scripts.packresolve import apply_override
 
         self.assertEqual(
             apply_override("bios/x.bin", "/cache/big.bin", "release_asset"),
@@ -1298,9 +1298,49 @@ class OneBaselineVerdict(unittest.TestCase):
             body = ast.get_source_segment(source, function) or ""
             with self.subTest(function=name):
                 self.assertIn("baseline_verdict(", body)
-                self.assertIn("slots.apply_override(", body)
+                self.assertIn("resolve_declaration(", body)
                 self.assertNotIn('"slot_arbitrated"', body)
 
+    def test_verify_and_the_region_pass_resolve_as_the_builder(self):
+        """verify called common.resolve_local_file and put the slot override
+        first, the region pass passed a destination hint the builder never
+        gives: a gitignored file absent from bios/ read as missing in verify
+        while the pack shipped its cached release asset."""
+        for module, name in (("verify.py", "_resolve_for_platform"),
+                             ("packextras.py", "_ships")):
+            source = (ROOT / "scripts" / module).read_text(encoding="utf-8")
+            function = next(
+                node for node in ast.parse(source).body
+                if isinstance(node, ast.FunctionDef) and node.name == name
+            )
+            body = ast.get_source_segment(source, function) or ""
+            with self.subTest(function=name):
+                self.assertIn("resolve_declaration(", body)
+                self.assertNotIn("resolve_local_file(", body)
+                self.assertNotIn('"slot_arbitrated"', body)
+
+    def test_verify_finds_the_cached_release_asset(self):
+        # verify imports the scripts directory's modules by their bare names.
+        import packresolve
+        from scripts.verify import _resolve_for_platform
+
+        with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
+            cached = Path(directory) / "BIG.PUP"
+            cached.write_bytes(b"large firmware")
+            sha1 = hashlib.sha1(cached.read_bytes()).hexdigest()
+            db = {"files": {sha1: {"path": f"{directory}/absent/BIG.PUP",
+                                   "name": "BIG.PUP", "sha1": sha1}},
+                  "indexes": {"by_name": {"BIG.PUP": [sha1]}}}
+            entry = {"name": "BIG.PUP", "sha1": sha1, "destination": "BIG.PUP"}
+            with mock.patch.object(
+                packresolve, "fetch_large_file", return_value=str(cached)
+            ):
+                found = _resolve_for_platform(
+                    entry, "sys", db, {}, None, {"BIG.PUP": "bios/other.bin"},
+                    "md5", {},
+                )
+        self.assertEqual(found, (str(cached), "release_asset"))
+
 
 class PackExclusionRegressions(unittest.TestCase):
     def test_slug_platform_core_requirement_uses_the_generated_destination(self):

tests/test_cross_reference_sources.py

@@ -110,5 +110,60 @@ class GapReportSources(unittest.TestCase):
         )
 
 
+class PlatformScopedEntries(unittest.TestCase):
+    """Under --platform the report reads the platform's scoping fields
+    through the functions verify and the builder call: blastem's cdbios.bin
+    is never opened by RetroBat (unread_on), touchHLE's fonts are not in
+    ROCKNIX's package (not_bundled_on), fMSX's fonts come with Recalbox's
+    (installed_on)."""
+
+    PROFILES = {"emu": {"emulator": "Emu", "systems": ["sys"], "files": [
+        {"name": "skipped.bin", "required": True, "unread_on": ["plat"]},
+        {"name": "font.ttf", "required": True, "bundled": True,
+         "not_bundled_on": ["plat"]},
+        {"name": "held.fnt", "installed_on": ["plat"]},
+    ]}}
+    DB = {"files": {"a" * 40: {"path": "bios/held.fnt", "name": "held.fnt"}},
+          "indexes": {"by_name": {"held.fnt": ["a" * 40]}, "by_md5": {},
+                      "by_crc32": {}, "by_sha256": {}, "by_path_suffix": {}}}
+
+    def _gaps(self, platform: str) -> dict[str, dict]:
+        report = cross_reference(self.PROFILES, {}, self.DB, platform=platform)
+        return {g["name"]: g for g in report["emu"]["gap_details"]}
+
+    def test_an_entry_the_platform_never_opens_is_not_a_gap(self):
+        self.assertNotIn("skipped.bin", self._gaps("plat"))
+        self.assertIn("skipped.bin", self._gaps("other"))
+        self.assertIn("skipped.bin", self._gaps(""))
+
+    def test_bundled_follows_the_platform_package(self):
+        gaps = self._gaps("plat")
+        self.assertFalse(gaps["font.ttf"]["bundled"])
+        self.assertTrue(gaps["held.fnt"]["bundled"])
+        gaps = self._gaps("other")
+        self.assertTrue(gaps["font.ttf"]["bundled"])
+        self.assertFalse(gaps["held.fnt"]["bundled"])
+
+    def test_a_held_bundled_file_is_not_announced_as_available(self):
+        import contextlib
+        import io
+
+        report = cross_reference(self.PROFILES, {}, self.DB, platform="plat")
+        with contextlib.redirect_stdout(io.StringIO()) as out:
+            print_report(report)
+        text = out.getvalue()
+        self.assertIn("held.fnt [BUNDLED]", text)
+        self.assertIn("0 available", text)
+        self.assertIn("1 missing (need to be sourced)", text)
+
+    def test_the_report_reads_the_shared_rules(self):
+        """One rule, one function: a local `bool(f.get("bundled"))` answered
+        differently from verify for every installed_on/not_bundled_on entry."""
+        source = (REPO_ROOT / "scripts" / "cross_reference.py").read_text()
+        self.assertNotIn('bool(f.get("bundled"))', source)
+        self.assertIn("installed_with_emulator(f, platform)", source)
+        self.assertIn("unread_on(f, platform)", source)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_file_consumers.py

@@ -83,6 +83,36 @@ class ReadersResolve(unittest.TestCase):
         self.assertNotIn("ioquake3", et)
 
 
+class PlatformReadersResolveAsThePack(unittest.TestCase):
+    """A hashless declaration resolves without its destination in the
+    builder, the manifest and verify. The site passed the destination and
+    credited RetroBat with another revision of its 63 mame/hash lists than
+    the one its manifest installs."""
+
+    def test_the_declared_file_is_the_one_the_pack_ships(self):
+        from packresolve import resolve_declaration
+
+        first, first_entry = _collected("bios/Arcade/MAME/hash/apfm1000.xml", b"one")
+        other, other_entry = _collected("bios/Other/mame/hash/apfm1000.xml", b"two")
+        db = {
+            "files": {first: first_entry, other: other_entry},
+            "indexes": {
+                "by_name": {"apfm1000.xml": [first, other]},
+                "by_path_suffix": {"mame/hash/apfm1000.xml": [other],
+                                   "Other/mame/hash/apfm1000.xml": [other],
+                                   "MAME/hash/apfm1000.xml": [first],
+                                   "Arcade/MAME/hash/apfm1000.xml": [first]},
+            },
+        }
+        entry = {"name": "apfm1000.xml", "destination": "mame/hash/apfm1000.xml"}
+        shipped, _status = resolve_declaration(entry, db, "bios", None, None, None,
+                                               offline=True)
+        consumers = FileConsumers(db, data_dir_registry={})
+        consumers.add_platforms({"plat": {"systems": {"s": {"files": [entry]}}}})
+        self.assertEqual(shipped, first_entry["path"])
+        self.assertEqual(consumers.platforms, {first: {"plat"}})
+
+
 class ReadersFollowTheDataTrees(unittest.TestCase):
     """Dolphin names codehandler.bin without a hash. The pack takes it from
     data/dolphin-sys, and the site credited Dolphin and Triforce with the

tests/test_foreign_tree.py

@@ -68,5 +68,76 @@ class ForeignTree(unittest.TestCase):
         self.assertNotEqual(status, "path_exact")
 
 
+class AReleaseNamedAfterTheTree(unittest.TestCase):
+    """cxbx-reloaded-2023 asks for EEPROM.bin. Its emulator's tree holds the
+    decrypted dump Cxbx reads, xemu's aliases name the encrypted retail one
+    eeprom.bin. The tree read as foreign to the suffixed profile, so the
+    case-folded fallback served the encrypted dump."""
+
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self._cwd = os.getcwd()
+        os.chdir(self._tmp.name)
+        files = {}
+        for rel, payload in (
+            ("bios/Other/cxbx-reloaded/EEPROM.bin", b"decrypted"),
+            ("bios/Microsoft/Xbox/eeprom.bin", b"encrypted"),
+        ):
+            path = Path(rel)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(payload)
+            sha1 = hashlib.sha1(payload).hexdigest()
+            files[sha1] = {"path": rel, "name": path.name, "size": len(payload),
+                           "sha1": sha1, "md5": hashlib.md5(payload).hexdigest(),
+                           "sha256": hashlib.sha256(payload).hexdigest(),
+                           "crc32": f"{len(files) + 1:08x}"}
+        self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+
+    def tearDown(self):
+        os.chdir(self._cwd)
+        self._tmp.cleanup()
+
+    def _resolve(self, owner: str, name: str):
+        entry = {"name": name, "source_profile": owner}
+        return resolve_local_file(entry, self.db, {})
+
+    def test_the_suffixed_release_reads_its_emulator_tree(self):
+        local, _status = self._resolve("cxbx-reloaded-2023", "EEPROM.bin")
+        self.assertEqual(local, "bios/Other/cxbx-reloaded/EEPROM.bin")
+
+    def test_only_whole_leading_words_name_the_tree(self):
+        """dolphin-mmjr2 is not a release of dolphin-mmjr: mmjr2 is one word."""
+        from common import _owner_tokens
+
+        self.assertNotIn("dolphinmmjr", _owner_tokens({"source_profile": "dolphin-mmjr2"}))
+        self.assertIn("cxbxreloaded", _owner_tokens({"source_profile": "cxbx-reloaded-2023"}))
+
+
+class AnotherMachinesRomIsNotServed(unittest.TestCase):
+    """np2pi reads a 16 KB PC-98 SCSI BIOS as scsi.rom. The only copy of that
+    size was Clock Signal's Apple II SCSI card ROM, and the name step served
+    it. Reads the repository."""
+
+    def test_np2pi_does_not_receive_the_apple_ii_rom(self):
+        import yaml
+
+        from common import load_database
+
+        if not (REPO_ROOT / "database.json").is_file():
+            self.skipTest("database.json is absent")
+        profile = yaml.safe_load((REPO_ROOT / "emulators" / "np2pi.yml").read_text())
+        entry = next(f for f in profile["files"] if f.get("name") == "scsi.rom")
+        previous = os.getcwd()
+        os.chdir(REPO_ROOT)
+        try:
+            db = load_database("database.json")
+            local, _status = resolve_local_file(
+                {**entry, "source_profile": "np2pi"}, db, {}, dest_hint=entry["path"]
+            )
+        finally:
+            os.chdir(previous)
+        self.assertFalse(local and "Clock Signal" in local, local)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_pack_core_extras.py

@@ -0,0 +1,91 @@
+"""The pack integrity check counts the core files the builder places.
+
+It rebuilt the core extras with an empty set of taken paths and tested
+presence alone. On a Recalbox pack it credited 68 core files at a path the
+platform's own file holds, 15 of them other bytes (fuse/trdos.rom, the PS1
+BIOS DuckStation reads), and printed a core count that contradicted the
+builder's in the same run.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import sys
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import packverify  # noqa: E402
+
+
+def _held(files: dict, by_name: dict, path: str, payload: bytes) -> str:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    Path(path).write_bytes(payload)
+    sha1 = hashlib.sha1(payload).hexdigest()
+    name = Path(path).name
+    files[sha1] = {"path": path, "name": name, "sha1": sha1,
+                   "md5": hashlib.md5(payload).hexdigest(), "size": len(payload)}
+    by_name.setdefault(name, []).append(sha1)
+    return sha1
+
+
+class CoreFilesAsTheBuilderPlacesThem(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
+        self.addCleanup(tmp.cleanup)
+        previous = os.getcwd()
+        os.chdir(tmp.name)
+        self.addCleanup(os.chdir, previous)
+        files: dict = {}
+        by_name: dict = {}
+        _held(files, by_name, "bios/plat/trdos.rom", b"platform dump")
+        core_trdos = _held(files, by_name, "bios/core/trdos.rom", b"core dump")
+        _held(files, by_name, "bios/core/other.rom", b"core file")
+        self.db = {"files": files, "indexes": {
+            "by_name": by_name, "by_md5": {e["md5"]: s for s, e in files.items()},
+            "by_path_suffix": {}, "by_crc32": {}, "by_sha256": {}}}
+        Path("platforms").mkdir()
+        config = {"platform": "P", "verification_mode": "existence", "cores": ["fuse"],
+                  "systems": {"zx": {"files": [
+                      {"name": "trdos.rom", "destination": "fuse/trdos.rom"}]}}}
+        Path("platforms/p.yml").write_text(yaml.safe_dump(config), encoding="utf-8")
+        Path("emulators").mkdir()
+        self.profiles = {"fuse": {
+            "emulator": "fuse", "type": "libretro", "systems": ["zx"], "files": [
+                {"name": "trdos.rom", "path": "fuse/trdos.rom", "sha1": core_trdos},
+                {"name": "other.rom", "path": "core/other.rom"},
+            ]}}
+
+    def _check(self, other: bytes) -> tuple:
+        with zipfile.ZipFile("pack.zip", "w") as zf:
+            zf.writestr("fuse/trdos.rom", b"platform dump")
+            zf.writestr("core/other.rom", other)
+        return packverify.verify_pack_against_platform(
+            "pack.zip", "p", "platforms", self.db, emulators_dir="emulators",
+            emu_profiles=self.profiles,
+        )
+
+    def test_a_path_the_platform_file_holds_is_not_a_core_file(self):
+        ok, _checked, _present, errors, _bc, _bp, core_checked, core_present, _ex = (
+            self._check(b"core file")
+        )
+        self.assertTrue(ok, errors)
+        self.assertEqual((core_checked, core_present), (1, 1))
+
+    def test_other_bytes_at_a_core_path_are_an_error(self):
+        ok, _checked, _present, errors, *_rest = self._check(b"something else")
+        self.assertFalse(ok)
+        self.assertTrue(
+            any(e.startswith("core file differs: core/other.rom") for e in errors), errors
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_pack_counts.py

@@ -551,5 +551,41 @@ class TheReleaseNotesReadTheRecord(unittest.TestCase):
         self.assertIn("scripts/release_notes.py", page)
 
 
+class CoreCopiesKeepTheirCategory(unittest.TestCase):
+    """The second pass copies a file the platform declares to the path a core
+    reads it from. Its extra named no gap report entry, so verify looked its
+    category up by an empty key and counted RetroArch's six hiscore.dat
+    copies, game data, under core files."""
+
+    def test_a_copy_counts_as_its_entry_says(self):
+        from packextras import _collect_emulator_extras, hidden_extras
+
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+            held = Path(tmp) / "x.dat"
+            held.write_bytes(b"data")
+            sha1 = hashlib.sha1(b"data").hexdigest()
+            config = {"platform": "P", "_key": "p", "cores": ["demo"], "systems": {
+                "s": {"files": [{"name": "x.dat", "destination": "x.dat"}]}}}
+            profiles = {"demo": {
+                "emulator": "Demo", "type": "libretro", "cores": ["demo"],
+                "systems": ["s"],
+                "files": [{"name": "x.dat", "path": "core/x.dat",
+                           "category": "game_data"}],
+            }}
+            db = {"files": {sha1: {"path": str(held), "name": "x.dat", "sha1": sha1}},
+                  "indexes": {"by_name": {"x.dat": [sha1]}, "by_path_suffix": {}}}
+            (extra,) = _collect_emulator_extras(
+                config, "emulators", db, {"x.dat"}, "", profiles
+            )
+            self.assertEqual(
+                (extra["source_name"], extra["source_path"], extra["category"]),
+                ("x.dat", "core/x.dat", "game_data"),
+            )
+            kinds: dict[str, str] = {}
+            hidden_extras(config, "emulators", db, profiles, {"x.dat": str(held)},
+                          report=[], placed_kind=kinds)
+        self.assertEqual(kinds, {"core/x.dat": "game_data"})
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_region_held.py

@@ -41,9 +41,9 @@ class ARefusedDumpHoldsNothing(unittest.TestCase):
                         "by_path_suffix": {}},
         }
         entry = {"name": "us.bin", "md5": "0" * 32}
-        self.assertFalse(_ships({"verification_mode": "md5"}, entry, db, "us.bin"))
-        self.assertTrue(_ships({"verification_mode": "existence"}, entry, db, "us.bin"))
-        self.assertTrue(_ships({"verification_mode": "md5"}, {"name": "us.bin"}, db, "us.bin"))
+        self.assertFalse(_ships({"verification_mode": "md5"}, entry, db))
+        self.assertTrue(_ships({"verification_mode": "existence"}, entry, db))
+        self.assertTrue(_ships({"verification_mode": "md5"}, {"name": "us.bin"}, db))
 
 
 if __name__ == "__main__":

tests/test_verify_emulator_evidence.py

@@ -52,6 +52,23 @@ def _db(path: Path, name: str) -> dict:
     }
 
 
+def _pack_files(root: Path, emulators: Path, db: dict) -> set[str]:
+    """The paths the emulator pack writes, built in memory as the pipeline does."""
+    import generate_pack
+
+    record = generate_pack.PackRecord()
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        generate_pack.generate_emulator_pack(
+            ["demo"], str(emulators), db, "bios", "", False, {}, offline=True,
+            record=record,
+        )
+    finally:
+        os.chdir(cwd)
+    return set(record.files)
+
+
 class HashMismatchIsNotCoverage(unittest.TestCase):
     def setUp(self):
         self._tmp = tempfile.TemporaryDirectory()
@@ -113,6 +130,17 @@ class HashMismatchIsNotCoverage(unittest.TestCase):
         self.assertIn("reason", detail)
         self.assertIn("hash", detail["reason"].lower())
 
+    def test_the_contradicted_copy_the_pack_writes_is_a_pack_destination(self):
+        """The pack writes it with a DISCREPANCY, so verify's list of what
+        the pack holds names it too: step 5 compared the two and failed."""
+        self._profile("f" * 32)
+        result = self._run()
+        self.assertEqual(result["pack_destinations"], ["shared.rom"])
+        self.assertEqual(
+            set(result["pack_destinations"]),
+            _pack_files(self.root, self.emulators, self.db),
+        )
+
     def test_a_matching_hash_is_still_ok(self):
         self._profile(hashlib.md5(self.rom.read_bytes()).hexdigest())
         detail = next(
@@ -230,6 +258,15 @@ class ArchiveHashMismatchIsNotCoverage(unittest.TestCase):
             "an archive whose bytes contradict its declared hash counted as covered",
         )
 
+    def test_the_contradicted_archive_the_pack_writes_is_a_pack_destination(self):
+        self._profile("f" * 32)
+        result = self._run()
+        self.assertEqual(result["pack_destinations"], ["romset.zip"])
+        self.assertEqual(
+            set(result["pack_destinations"]),
+            _pack_files(self.root, self.emulators, self.db),
+        )
+
     def test_a_matching_container_hash_is_still_ok(self):
         self._profile(hashlib.md5(self.archive.read_bytes()).hexdigest())
         self.assertEqual(set(self._statuses(self._run())), {verify.Status.OK})