From 8f7dbb1b31fdbc7f6cd194a7a3b9b5060350a84b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:31:27 +0100 Subject: [PATCH] fix: gap report reads the launcher layer cross_reference --platform listed core files the platform's launcher already settles, as Batocera's DuckStation scph1002a.bin while the PAL slot holds scph5502.bin. It now hands the launcher layer the same entries verify does, and counts what it settles as set aside. --- scripts/cross_reference.py | 85 +++++++++++++++++++++++++++++++++--- scripts/launchers.py | 13 +++++- scripts/verify.py | 15 +------ tests/test_launcher_layer.py | 26 +++++++++++ 4 files changed, 119 insertions(+), 20 deletions(-) diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 0a0e36c6f..71ac29ed9 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -48,6 +48,7 @@ from common import ( select_emulator_profiles, ) from hashing import parse_hash_list +from launchers import LauncherLayer, declared_destinations, held_record from validation import ( case_folded_slots, declaration_settles, @@ -692,6 +693,7 @@ SET_ASIDE = { "other_versions": "other versions of a slot", "not_read": "not read on this platform", "outside_scope": "outside the pack", + "launcher": "filled by the platform's launcher", } @@ -923,6 +925,19 @@ def _cross_reference_profile( } if save_root: entry["root"] = "save" + elif index.get("launcher_candidates") is not None: + # What verify hands the launcher layer for the same entry. + probe = { + **f, + "profile": emu_name, + "emulator": profile.get("emulator", emu_name), + "path": effective_path, + "in_repo": in_repo, + "bundled": entry["bundled"], + } + index["launcher_candidates"].append( + (emu_name, is_standalone, f, probe, entry) + ) gaps.append(entry) for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]): @@ -934,12 +949,7 @@ def _cross_reference_profile( "systems": systems, "total_files": len(emu_files), "platform_covered": len(covered), - "gaps": len(gaps), - "gap_in_repo": sum(1 for g in gaps if g["in_repo"]), - "gap_missing": sum(1 for g in gaps if g["source"] == "missing"), - "gap_bios": sum(1 for g in gaps if g["source"] == "bios"), - "gap_data": sum(1 for g in gaps if g["source"] == "data"), - "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"), + **_gap_counts(gaps), "gap_details": gaps, "unsourceable": unsourceable_list, # Entries, not rows: an archive's members are one gap row. @@ -950,6 +960,61 @@ def _cross_reference_profile( } +def _gap_counts(gaps: list[dict]) -> dict[str, int]: + """The per-source tallies of a profile's gap rows.""" + return { + "gaps": len(gaps), + "gap_in_repo": sum(1 for g in gaps if g["in_repo"]), + "gap_missing": sum(1 for g in gaps if g["source"] == "missing"), + "gap_bios": sum(1 for g in gaps if g["source"] == "bios"), + "gap_data": sum(1 for g in gaps if g["source"] == "data"), + "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"), + } + + +def _settle_launcher_places( + report: dict, + candidates: list[tuple[str, bool, dict, dict, dict]], + layer: LauncherLayer, + db: dict, +) -> None: + """Apply the platform's launcher places as find_undeclared_files does. + + Batocera's DuckStation generator fills the PAL slot with scph5502.bin, + yet the report listed duckstation's scph1002a.bin as a file the pack + carries, which neither the pack nor the manifest does. + """ + added, settled = layer.settle( + [(emu, standalone, f, probe) for emu, standalone, f, probe, _ in candidates], + lambda probe: held_record(probe, db), + ) + for emu, _standalone, _f, probe, gap in candidates: + if id(probe) not in settled: + continue + data = report[emu] + data["gap_details"] = [g for g in data["gap_details"] if g is not gap] + data["undeclared_files"] -= 1 + data["set_aside"]["launcher"] += 1 + for entry in added: + data = report[entry["profile"]] + data["gap_details"].append({ + "name": entry["name"], + "path": entry["path"], + "required": entry.get("required", False), + "bundled": False, + "note": "", + "source_ref": "", + "in_platform": False, + "in_repo": True, + "source": "bios", + "identity": entry.get("sha1") or "", + "launcher": entry["launcher"], + "launcher_of": entry["launcher_of"], + }) + for data in report.values(): + data.update(_gap_counts(data["gap_details"])) + + def cross_reference( profiles: dict[str, dict], declared: dict[str, set[str]], @@ -960,6 +1025,7 @@ def cross_reference( standalone_cores: set[str] | None = None, folded_slots: frozenset[str] = frozenset(), platform: str = "", + launcher_layer: LauncherLayer | None = None, ) -> dict: """Compare emulator profiles against platform declarations. @@ -998,12 +1064,15 @@ def cross_reference( "all_declared": all_declared, "folded_slots": folded_slots, "bios_dir": db.get("bios_dir"), + "launcher_candidates": [] if launcher_layer else None, } for emu_name, profile in profiles.items(): _cross_reference_profile( emu_name, profile, declared, report, index, standalone_cores, platform, ) + if launcher_layer: + _settle_launcher_places(report, index["launcher_candidates"], launcher_layer, db) return report @@ -1195,6 +1264,10 @@ def main() -> None: standalone_cores=standalone_cores, folded_slots=folded_declared_slots(args.platforms_dir, selected, db), platform=config.get("_key", args.platform) if args.platform else "", + # What the platform's launchers open, as verify and the builder read it. + launcher_layer=LauncherLayer( + config, profiles, relevant, db, declared_destinations(config) + ) if args.platform else None, ) if args.json: diff --git a/scripts/launchers.py b/scripts/launchers.py index d3147b35a..1f9ce1d56 100644 --- a/scripts/launchers.py +++ b/scripts/launchers.py @@ -20,7 +20,7 @@ import re from collections.abc import Callable, Iterable from dataclasses import dataclass -from common import declared_root, name_match_size_ok, sanitize_pack_path +from common import declared_root, name_match_size_ok, resolve_local_file, sanitize_pack_path # The kinds whose names say which file the launcher opens. A link of a named # file is one too: ROCKNIX links /storage/roms/bios/nds_bios_arm9.bin into @@ -330,6 +330,17 @@ def standalone_root( return next(iter(folders)) if len(folders) == 1 else None +def held_record(report: dict, db: dict) -> dict | None: + """The database record of the file the collection holds for a report entry.""" + local, status = resolve_local_file(report, db, dest_hint=report.get("path") or "") + if local is None or status in ("not_found", "hash_mismatch"): + return None + for record in db.get("files", {}).values(): + if record.get("path") == local: + return record + return None + + def declared_destinations(config: dict) -> set[str]: """Each destination the platform declares, relative to the core file directory. diff --git a/scripts/verify.py b/scripts/verify.py index 147a8f80d..260131e97 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -84,7 +84,7 @@ from common import ( yaml = require_yaml() from exporter.candidates import BIOS, EntryCategories -from launchers import LauncherLayer, declared_destinations +from launchers import LauncherLayer, declared_destinations, held_record from nativemode import ( SIZE_AND_ANY_HASH, digest_algorithm, @@ -489,17 +489,6 @@ def _identity_of(entry: dict) -> dict: } -def _held_record(report: dict, db: dict) -> dict | None: - """The database record of the file the collection holds for a report entry.""" - local, status = resolve_local_file(report, db, dest_hint=report.get("path") or "") - if local is None or status in ("not_found", "hash_mismatch"): - return None - for record in db.get("files", {}).values(): - if record.get("path") == local: - return record - return None - - def find_undeclared_files( config: dict, emulators_dir: str, @@ -778,7 +767,7 @@ def find_undeclared_files( if launcher_layer: added, settled = launcher_layer.settle( - launcher_candidates, lambda report: _held_record(report, db) + launcher_candidates, lambda report: held_record(report, db) ) undeclared = [u for u in undeclared if id(u) not in settled] undeclared.extend(added) diff --git a/tests/test_launcher_layer.py b/tests/test_launcher_layer.py index bb3069a88..e765fe27e 100644 --- a/tests/test_launcher_layer.py +++ b/tests/test_launcher_layer.py @@ -318,6 +318,32 @@ class LauncherLayerTest(unittest.TestCase): self.assertNotIn("a.bin", names) self.assertEqual(result["packed_extras"]["bios"], len(names - {"plain.bin"})) + def test_the_gap_report_reads_the_launcher_too(self) -> None: + """cross_reference --platform batocera listed duckstation's + scph1002a.bin as packed while the PAL slot was filled by the + launcher's scph5502.bin, which verify and the builder both knew.""" + from cross_reference import cross_reference, declared_names, load_platform_files + from launchers import declared_destinations + + selected = ["launchertest"] + declared, data_dirs = load_platform_files(self.platforms_dir, selected) + layer = LauncherLayer( + self.config, self.profiles, self.profiles, self.db, + declared_destinations(self.config), + ) + report = cross_reference( + self.profiles, declared, self.db, data_dirs, set(), + all_declared=declared_names(self.platforms_dir, selected, self.db), + standalone_cores={"psxemu", "fixemu"}, + platform="launchertest", launcher_layer=layer, + ) + gaps = sorted( + g["path"] for data in report.values() for g in data["gap_details"] + ) + self.assertEqual(gaps, sorted(u["path"] for u in self._report())) + self.assertNotIn("core.bin", gaps) + self.assertEqual(report["fixemu"]["set_aside"]["launcher"], 1) + def test_one_decision_for_verify_and_pack(self) -> None: """The layer is applied once, in the report both sides read.""" import generate_pack