diff --git a/scripts/packextras.py b/scripts/packextras.py index 2c2f49d0..b8b5c331 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -215,6 +215,7 @@ def hidden_extras( placed: set[str] | None = None, report: list[dict] | None = None, resolver: ExtraResolver | None = None, + placed_by: dict[str, tuple[str, str, str]] | None = None, ) -> dict[tuple[str, str, str], str]: """Report entries the builder holds and still leaves out, with the reason. @@ -286,6 +287,11 @@ def hidden_extras( lower.add(full.lower()) if placed is not None and local: placed.add(full) + if placed_by is not None and local: + placed_by[full] = ( + fe.get("source_emulator", ""), fe.get("source_name", ""), + fe.get("source_path", ""), + ) return hidden diff --git a/scripts/verify.py b/scripts/verify.py index 46808562..7a124ab0 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -824,6 +824,7 @@ def _mark_hidden( region_drops: set[str] | None = None, report: list[dict] | None = None, resolver: object | None = None, + placed_by: dict[str, tuple[str, str, str]] | None = None, ) -> set[str]: """What the builder holds but cannot place beside the files it packed. @@ -838,7 +839,7 @@ def _mark_hidden( config, emulators_dir, db, emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir), - shipped, target_cores, region_drops, placed, report, resolver, + shipped, target_cores, region_drops, placed, report, resolver, placed_by, ) for u in undeclared: reason = hidden.get( @@ -1200,6 +1201,7 @@ def verify_platform( _mark_unplaced(config, undeclared, profiles) from packextras import ExtraResolver + placed_by: dict[str, tuple[str, str, str]] = {} placed = _mark_hidden( config, undeclared, emulators_dir, db, emu_profiles, shipped, target_cores, region_drops, report_for_builder, @@ -1207,7 +1209,20 @@ def verify_platform( db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index, validation_owners, ), + placed_by, ) + # What the builder places from the cores, one per pack path, by category: + # several entries claim one path (rom.key from three Amiga cores) and the + # builder ships one file there. + categories = { + (u.get("emulator", ""), u.get("name", ""), u.get("path") or ""): + u.get("category", "bios") + for u in report_for_builder + } + packed_extras = {"bios": 0, "game_data": 0} + for key in placed_by.values(): + kind = "game_data" if categories.get(key) == "game_data" else "bios" + packed_extras[kind] += 1 if not reads_file_contents(mode): # An existence frontend is satisfied by any file at the path, and the # builder fills a free platform destination with a core's file. @@ -1276,6 +1291,7 @@ def verify_platform( "severity_counts": counts, "status_counts": status_counts, "excluded_files": excluded, + "packed_extras": packed_extras, "undeclared_files": undeclared, "exclusion_notes": exclusions, "region_fallbacks": sorted(region_fallbacks), @@ -1465,7 +1481,13 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: if u["required"] and not u["in_repo"] and u.get("bundled") and not u.get("hle_fallback") ] - core_in_pack = len(req_in_repo) + len(opt_in_repo) + # The builder's count, one file per pack path, when the platform pass + # replayed it. Counting entries said 1130 for RetroArch, 48 more than the + # pack holds: rom.key alone was claimed by three Amiga cores. + packed = result.get("packed_extras") + core_in_pack = ( + packed["bios"] if packed is not None else len(req_in_repo) + len(opt_in_repo) + ) core_missing_req = len(req_not_in_repo) + len(req_hle_not_in_repo) core_missing_opt = len(opt_not_in_repo) @@ -1485,9 +1507,18 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: if game_data: gd_missing = [u for u in game_data if not u["in_repo"]] - gd_present = [u for u in game_data if u["in_repo"]] - if gd_missing or gd_present: - print(f" Game data: {len(gd_present)} in pack, {len(gd_missing)} missing") + gd_packed = ( + packed["game_data"] if packed is not None + else sum(1 for u in game_data if u["in_repo"] and u.get("in_pack", True)) + ) + gd_unplaced = [u for u in game_data if u["in_repo"] and not u.get("in_pack", True)] + if gd_missing or gd_packed or gd_unplaced: + print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing") + if gd_unplaced: + print( + f" Game data held but not packed: {len(gd_unplaced)} " + f"({gd_unplaced[0].get('omitted', 'not placed')})" + ) def print_platform_result( diff --git a/tests/test_unplaced_extras.py b/tests/test_unplaced_extras.py index a17a185f..65c4acea 100644 --- a/tests/test_unplaced_extras.py +++ b/tests/test_unplaced_extras.py @@ -274,5 +274,39 @@ class UserProvidedIsNeverTakenFromTheCollection(unittest.TestCase): self.assertNotIn(Status.OK, result["status_counts"]) +class OnePathCountsOnce(unittest.TestCase): + """rom.key is claimed by three Amiga cores and the pack holds one file + there: the report said 1130 core files in the RetroArch pack, 48 more + than it carries.""" + + def test_two_claims_on_one_path_count_one_file(self): + with tempfile.TemporaryDirectory() as tmp: + previous = os.getcwd() + os.chdir(tmp) + self.addCleanup(os.chdir, previous) + for d in ("emulators", "bios/Commodore"): + Path(d).mkdir(parents=True) + Path("bios/Commodore/rom.key").write_bytes(b"key") + sha1 = hashlib.sha1(b"key").hexdigest() + files = {sha1: {"path": "bios/Commodore/rom.key", "name": "rom.key", "size": 3, + "sha1": sha1, "md5": hashlib.md5(b"key").hexdigest(), + "sha256": hashlib.sha256(b"key").hexdigest(), "crc32": "00000001"}} + db = {"files": files, "indexes": generate_db.build_indexes(files, {})} + for core, region in (("one", ""), ("two", " region: [world]\n")): + Path(f"emulators/{core}.yml").write_text( + f"emulator: {core}\ntype: libretro\nsystems: [amiga]\ncores: [{core}]\n" + f"files:\n - name: rom.key\n required: false\n{region}" + ) + common._emulator_profiles_cache.clear() + self.addCleanup(common._emulator_profiles_cache.clear) + profiles = common.load_emulator_profiles("emulators") + config = {"platform": "P", "verification_mode": "existence", "cores": ["one", "two"], + "systems": {"amiga": {"files": []}}} + report = verify_platform(config, db, "emulators", profiles) + claims = [u for u in report["undeclared_files"] if u["name"] == "rom.key"] + self.assertEqual(len(claims), 2) + self.assertEqual(report["packed_extras"], {"bios": 1, "game_data": 0}) + + if __name__ == "__main__": unittest.main()