diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 0a0e36c6f..71ac29ed9 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -48,6 +48,7 @@ from common import ( select_emulator_profiles, ) from hashing import parse_hash_list +from launchers import LauncherLayer, declared_destinations, held_record from validation import ( case_folded_slots, declaration_settles, @@ -692,6 +693,7 @@ SET_ASIDE = { "other_versions": "other versions of a slot", "not_read": "not read on this platform", "outside_scope": "outside the pack", + "launcher": "filled by the platform's launcher", } @@ -923,6 +925,19 @@ def _cross_reference_profile( } if save_root: entry["root"] = "save" + elif index.get("launcher_candidates") is not None: + # What verify hands the launcher layer for the same entry. + probe = { + **f, + "profile": emu_name, + "emulator": profile.get("emulator", emu_name), + "path": effective_path, + "in_repo": in_repo, + "bundled": entry["bundled"], + } + index["launcher_candidates"].append( + (emu_name, is_standalone, f, probe, entry) + ) gaps.append(entry) for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]): @@ -934,12 +949,7 @@ def _cross_reference_profile( "systems": systems, "total_files": len(emu_files), "platform_covered": len(covered), - "gaps": len(gaps), - "gap_in_repo": sum(1 for g in gaps if g["in_repo"]), - "gap_missing": sum(1 for g in gaps if g["source"] == "missing"), - "gap_bios": sum(1 for g in gaps if g["source"] == "bios"), - "gap_data": sum(1 for g in gaps if g["source"] == "data"), - "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"), + **_gap_counts(gaps), "gap_details": gaps, "unsourceable": unsourceable_list, # Entries, not rows: an archive's members are one gap row. @@ -950,6 +960,61 @@ def _cross_reference_profile( } +def _gap_counts(gaps: list[dict]) -> dict[str, int]: + """The per-source tallies of a profile's gap rows.""" + return { + "gaps": len(gaps), + "gap_in_repo": sum(1 for g in gaps if g["in_repo"]), + "gap_missing": sum(1 for g in gaps if g["source"] == "missing"), + "gap_bios": sum(1 for g in gaps if g["source"] == "bios"), + "gap_data": sum(1 for g in gaps if g["source"] == "data"), + "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"), + } + + +def _settle_launcher_places( + report: dict, + candidates: list[tuple[str, bool, dict, dict, dict]], + layer: LauncherLayer, + db: dict, +) -> None: + """Apply the platform's launcher places as find_undeclared_files does. + + Batocera's DuckStation generator fills the PAL slot with scph5502.bin, + yet the report listed duckstation's scph1002a.bin as a file the pack + carries, which neither the pack nor the manifest does. + """ + added, settled = layer.settle( + [(emu, standalone, f, probe) for emu, standalone, f, probe, _ in candidates], + lambda probe: held_record(probe, db), + ) + for emu, _standalone, _f, probe, gap in candidates: + if id(probe) not in settled: + continue + data = report[emu] + data["gap_details"] = [g for g in data["gap_details"] if g is not gap] + data["undeclared_files"] -= 1 + data["set_aside"]["launcher"] += 1 + for entry in added: + data = report[entry["profile"]] + data["gap_details"].append({ + "name": entry["name"], + "path": entry["path"], + "required": entry.get("required", False), + "bundled": False, + "note": "", + "source_ref": "", + "in_platform": False, + "in_repo": True, + "source": "bios", + "identity": entry.get("sha1") or "", + "launcher": entry["launcher"], + "launcher_of": entry["launcher_of"], + }) + for data in report.values(): + data.update(_gap_counts(data["gap_details"])) + + def cross_reference( profiles: dict[str, dict], declared: dict[str, set[str]], @@ -960,6 +1025,7 @@ def cross_reference( standalone_cores: set[str] | None = None, folded_slots: frozenset[str] = frozenset(), platform: str = "", + launcher_layer: LauncherLayer | None = None, ) -> dict: """Compare emulator profiles against platform declarations. @@ -998,12 +1064,15 @@ def cross_reference( "all_declared": all_declared, "folded_slots": folded_slots, "bios_dir": db.get("bios_dir"), + "launcher_candidates": [] if launcher_layer else None, } for emu_name, profile in profiles.items(): _cross_reference_profile( emu_name, profile, declared, report, index, standalone_cores, platform, ) + if launcher_layer: + _settle_launcher_places(report, index["launcher_candidates"], launcher_layer, db) return report @@ -1195,6 +1264,10 @@ def main() -> None: standalone_cores=standalone_cores, folded_slots=folded_declared_slots(args.platforms_dir, selected, db), platform=config.get("_key", args.platform) if args.platform else "", + # What the platform's launchers open, as verify and the builder read it. + launcher_layer=LauncherLayer( + config, profiles, relevant, db, declared_destinations(config) + ) if args.platform else None, ) if args.json: diff --git a/scripts/launchers.py b/scripts/launchers.py index d3147b35a..1f9ce1d56 100644 --- a/scripts/launchers.py +++ b/scripts/launchers.py @@ -20,7 +20,7 @@ import re from collections.abc import Callable, Iterable from dataclasses import dataclass -from common import declared_root, name_match_size_ok, sanitize_pack_path +from common import declared_root, name_match_size_ok, resolve_local_file, sanitize_pack_path # The kinds whose names say which file the launcher opens. A link of a named # file is one too: ROCKNIX links /storage/roms/bios/nds_bios_arm9.bin into @@ -330,6 +330,17 @@ def standalone_root( return next(iter(folders)) if len(folders) == 1 else None +def held_record(report: dict, db: dict) -> dict | None: + """The database record of the file the collection holds for a report entry.""" + local, status = resolve_local_file(report, db, dest_hint=report.get("path") or "") + if local is None or status in ("not_found", "hash_mismatch"): + return None + for record in db.get("files", {}).values(): + if record.get("path") == local: + return record + return None + + def declared_destinations(config: dict) -> set[str]: """Each destination the platform declares, relative to the core file directory. diff --git a/scripts/verify.py b/scripts/verify.py index 147a8f80d..260131e97 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -84,7 +84,7 @@ from common import ( yaml = require_yaml() from exporter.candidates import BIOS, EntryCategories -from launchers import LauncherLayer, declared_destinations +from launchers import LauncherLayer, declared_destinations, held_record from nativemode import ( SIZE_AND_ANY_HASH, digest_algorithm, @@ -489,17 +489,6 @@ def _identity_of(entry: dict) -> dict: } -def _held_record(report: dict, db: dict) -> dict | None: - """The database record of the file the collection holds for a report entry.""" - local, status = resolve_local_file(report, db, dest_hint=report.get("path") or "") - if local is None or status in ("not_found", "hash_mismatch"): - return None - for record in db.get("files", {}).values(): - if record.get("path") == local: - return record - return None - - def find_undeclared_files( config: dict, emulators_dir: str, @@ -778,7 +767,7 @@ def find_undeclared_files( if launcher_layer: added, settled = launcher_layer.settle( - launcher_candidates, lambda report: _held_record(report, db) + launcher_candidates, lambda report: held_record(report, db) ) undeclared = [u for u in undeclared if id(u) not in settled] undeclared.extend(added) diff --git a/tests/test_launcher_layer.py b/tests/test_launcher_layer.py index bb3069a88..e765fe27e 100644 --- a/tests/test_launcher_layer.py +++ b/tests/test_launcher_layer.py @@ -318,6 +318,32 @@ class LauncherLayerTest(unittest.TestCase): self.assertNotIn("a.bin", names) self.assertEqual(result["packed_extras"]["bios"], len(names - {"plain.bin"})) + def test_the_gap_report_reads_the_launcher_too(self) -> None: + """cross_reference --platform batocera listed duckstation's + scph1002a.bin as packed while the PAL slot was filled by the + launcher's scph5502.bin, which verify and the builder both knew.""" + from cross_reference import cross_reference, declared_names, load_platform_files + from launchers import declared_destinations + + selected = ["launchertest"] + declared, data_dirs = load_platform_files(self.platforms_dir, selected) + layer = LauncherLayer( + self.config, self.profiles, self.profiles, self.db, + declared_destinations(self.config), + ) + report = cross_reference( + self.profiles, declared, self.db, data_dirs, set(), + all_declared=declared_names(self.platforms_dir, selected, self.db), + standalone_cores={"psxemu", "fixemu"}, + platform="launchertest", launcher_layer=layer, + ) + gaps = sorted( + g["path"] for data in report.values() for g in data["gap_details"] + ) + self.assertEqual(gaps, sorted(u["path"] for u in self._report())) + self.assertNotIn("core.bin", gaps) + self.assertEqual(report["fixemu"]["set_aside"]["launcher"], 1) + def test_one_decision_for_verify_and_pack(self) -> None: """The layer is applied once, in the report both sides read.""" import generate_pack