From 8268f03269a05817bc3d741fba5cb14a316fa380 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:16:38 +0100 Subject: [PATCH] feat: report files only a launcher reads cross_reference and verify list the names a platform's launch code reads that the collection holds under no name. Reported apart, never scored. --- scripts/cross_reference.py | 204 ++++++++++++++++++++++++++- scripts/verify.py | 23 +++ tests/test_launcher_gaps.py | 269 ++++++++++++++++++++++++++++++++++++ 3 files changed, 494 insertions(+), 2 deletions(-) create mode 100644 tests/test_launcher_gaps.py diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 74e1938b2..af7525a77 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -51,6 +51,7 @@ from common import ( ) from hashing import parse_hash_list from launchers import LauncherLayer, declared_destinations, held_record +from launchers import _names as launcher_names from validation import ( case_folded_slots, declaration_settles, @@ -1030,6 +1031,191 @@ def _settle_launcher_places( data.update(_gap_counts(data["gap_details"])) +# The --json key of the launcher reads category, beside the emulator keys. +LAUNCHER_READS_KEY = "_launcher_reads_not_held" + +# A package record names no file: it records what a package builds. +_NAMELESS_KINDS = frozenset({"package"}) + + +def launcher_file_names(record: dict) -> list[str]: + """The concrete file names a launcher record reads, in code order. + + The names are read as the launcher layer reads them (`launchers._names`): + a label after the name is dropped ("neocd_z.rom (CDZ)"), and a name with + `<` or without an extension is not a file name (a folder the record + links, an option value such as hatari2014's "default"). A name with a + `{X}` placeholder is a template: Batocera's hatari tries tos{L}.img for + every language and ROCKNIX links one memory card per slot and region, + so its expansions say which variants the code accepts, not which file is + owed. A wildcard is a pattern the emulator matches. `search_dir` is a + folder and never a name, whatever the kind. + """ + if record.get("kind") in _NAMELESS_KINDS: + return [] + return [ + name for name in launcher_names({**record, "expands": {}}) if "*" not in name + ] + + +def _profile_names(profiles: dict[str, dict]) -> tuple[dict[str, list[dict]], dict[str, list[dict]]]: + """Profile entries by each name and alias they answer to, exact and folded.""" + exact: dict[str, list[dict]] = {} + folded: dict[str, list[dict]] = {} + for profile in profiles.values(): + if profile.get("type") in ("launcher", "alias"): + continue + for entry in profile.get("files") or []: + names = {str(entry.get("name") or "")} | {str(a) for a in entry.get("aliases") or []} + for name in names - {""}: + base = name.rsplit("/", 1)[-1] + exact.setdefault(base, []).append(entry) + folded.setdefault(base.casefold(), []).append(entry) + return exact, folded + + +class _NameHolder: + """Whether the collection holds a file under a launcher name. + + A name is held when database.json carries a file under it or under an + alias it indexes, when a profile entry that answers to the name (as its + name or an alias) resolves, or when resolve_local_file finds it in a data + directory. The database answers for files kept as release assets, absent + from a checkout. A copy known only under another casing is returned + apart: a launcher that matches names exactly does not open it. + """ + + def __init__( + self, db: dict, profiles: dict[str, dict], data_dir_registry: dict | None + ) -> None: + self.db = db + self.registry = data_dir_registry + files = db.get("files", {}) + self.by_name = { + name: sha1s + for name, sha1s in db.get("indexes", {}).get("by_name", {}).items() + if any(sha1 in files for sha1 in sha1s) + } + self.by_name_folded: dict[str, list[str]] = {} + for name in self.by_name: + self.by_name_folded.setdefault(name.casefold(), []).append(name) + self.exact, self.folded = _profile_names(profiles) + self._cache: dict[str, tuple[str, bool] | None] = {} + + def _entry_held(self, entry: dict) -> bool: + names = [str(entry.get("name") or "")] + [str(a) for a in entry.get("aliases") or []] + if any(n.rsplit("/", 1)[-1] in self.by_name for n in names if n): + return True + path, _status = resolve_local_file(entry, self.db, data_dir_registry=self.registry) + return path is not None + + def held(self, name: str) -> tuple[str, bool] | None: + """The name the collection holds *name* under, and whether its casing is exact.""" + if name in self._cache: + return self._cache[name] + base = name.rsplit("/", 1)[-1] + found: tuple[str, bool] | None = None + if base in self.by_name or any(map(self._entry_held, self.exact.get(base, []))): + found = (base, True) + else: + path, _status = resolve_local_file( + {"name": name}, self.db, data_dir_registry=self.registry + ) + if path is not None: + found = (Path(path).name, Path(path).name == base) + elif base.casefold() in self.by_name_folded: + found = (self.by_name_folded[base.casefold()][0], False) + else: + entry = next( + (e for e in self.folded.get(base.casefold(), []) if self._entry_held(e)), + None, + ) + if entry is not None: + found = (str(entry.get("name") or ""), False) + self._cache[name] = found + return found + + +def launcher_reads_not_held( + configs: dict[str, dict], + db: dict, + profiles: dict[str, dict], + data_dir_registry: dict | None = None, + emulator: str = "", +) -> list[dict]: + """The file names a platform's launch code reads that the collection lacks. + + One row per platform and name, with every record that reads it. A + launcher that matches case-insensitively, or runs on a filesystem that + folds case, is satisfied by a copy under another casing. Elsewhere such + a copy is reported with `held_as`, since the launcher does not open it + under the name it asks for. *emulator* keeps the records whose emulator + name designates that profile. + """ + holder = _NameHolder(db, profiles, data_dir_registry) + every_profile = set(profiles) + rows: list[dict] = [] + for platform, config in configs.items(): + by_key: dict[str, dict] = {} + for record in config.get("launchers") or []: + names = launcher_file_names(record) + if not names: + continue + readers = sorted({ + key + for part in str(record.get("emulator", "")).split(",") + for key in LauncherLayer._profiles_named(part.strip(), profiles, every_profile) + }) + if emulator and emulator not in readers: + continue + folds = bool(record.get("case_insensitive") or config.get("case_insensitive_fs")) + reader = { + "emulator": str(record.get("emulator", "")), + "build": str(record.get("build", "")), + "kind": str(record.get("kind", "")), + "source_ref": str(record.get("source_ref", "")), + "profiles": readers, + } + for name in names: + found = holder.held(name) + if found and (found[1] or folds): + continue + key = name.casefold() if folds else name + row = by_key.get(key) + if row is None: + row = by_key[key] = { + "platform": platform, + "name": name, + "emulators": [], + "readers": [], + } + if found: + row["held_as"] = found[0] + if reader["emulator"] not in row["emulators"]: + row["emulators"].append(reader["emulator"]) + if reader not in row["readers"]: + row["readers"].append(reader) + rows.extend(by_key[key] for key in sorted(by_key, key=str.casefold)) + return rows + + +def print_launcher_reads(rows: list[dict]) -> None: + """The launcher reads gap category, after the profile gaps.""" + platforms = sorted({r["platform"] for r in rows}) + noun = "platform" if len(platforms) == 1 else "platforms" + print(f"\nLauncher reads, not held: {len(rows)} names across {len(platforms)} {noun}") + for platform in platforms: + print(f"\n{platform}") + for row in (r for r in rows if r["platform"] == platform): + first = row["readers"][0] + more = f" (+{len(row['readers']) - 1} records)" if len(row["readers"]) > 1 else "" + casing = f", held as {row['held_as']}" if row.get("held_as") else "" + print( + f" {row['name']} [{', '.join(row['emulators'])}, {first['kind']}, " + f"{first['source_ref']}]{more}{casing}" + ) + + def cross_reference( profiles: dict[str, dict], declared: dict[str, set[str]], @@ -1210,9 +1396,12 @@ def main() -> None: # Files resolve against data/: the trees stay put until the process # ends, which releases the hold. held = contextlib.ExitStack() - hold_data_dirs(held, load_data_dir_registry(args.platforms_dir)) + registry = load_data_dir_registry(args.platforms_dir) + hold_data_dirs(held, registry) profiles = load_emulator_profiles(args.emulators_dir) + # Every profile's names and aliases, whatever --emulator or --platform keeps. + all_profiles = profiles emulator = args.emulator if emulator: # A typo answered "No emulator profiles found" with exit 0. @@ -1288,8 +1477,18 @@ def main() -> None: ) if args.platform else None, ) + # Counted apart from the profile gaps: a name only a launcher reads is + # reported, not yet judged. + launcher_reads = launcher_reads_not_held( + { + name: load_platform_config(name, args.platforms_dir) + for name in (selected or list_registered_platforms(args.platforms_dir)) + }, + db, all_profiles, registry, emulator or "", + ) + if args.json: - print(json.dumps(report, indent=2)) + print(json.dumps({**report, LAUNCHER_READS_KEY: launcher_reads}, indent=2)) else: if emulator: scope = emulator @@ -1298,6 +1497,7 @@ def main() -> None: else: scope = "all emulators" print_report(report, scope, show_clean=bool(args.emulator)) + print_launcher_reads(launcher_reads) if __name__ == "__main__": diff --git a/scripts/verify.py b/scripts/verify.py index 2282ecb77..437077555 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -91,6 +91,7 @@ from common import ( yaml = require_yaml() from exporter.candidates import BIOS, EntryCategories from launchers import LauncherLayer, declared_destinations, held_record +from cross_reference import launcher_reads_not_held from nativemode import ( SIZE_AND_ANY_HASH, digest_algorithm, @@ -1595,6 +1596,10 @@ def verify_platform( "excluded_files": excluded, "packed_extras": packed_extras, "undeclared_files": undeclared, + # Reported apart, never scored: no severity, count or exit code reads it. + "launcher_reads_not_held": launcher_reads_not_held( + {config.get("_key", platform): config}, db, profiles, data_dir_registry + ), "exclusion_notes": exclusions, "region_fallbacks": sorted(region_fallbacks), **({"left_out": left_out} if no_game_data else {}), @@ -1959,6 +1964,23 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: ) +def _print_launcher_reads(result: dict, verbose: bool) -> None: + """Names the platform's launch code reads that the collection lacks.""" + rows = result.get("launcher_reads_not_held") or [] + if not rows: + return + listed = "" if verbose else " (--verbose lists them)" + print(f" Launcher reads, not held: {len(rows)}{listed}") + if not verbose: + return + for row in rows: + kinds = ", ".join(sorted({r["kind"] for r in row["readers"]})) + casing = f", held as {row['held_as']}" if row.get("held_as") else "" + print(f" {row['name']} [{', '.join(row['emulators'])}, {kinds}{casing}]") + for reader in row["readers"]: + print(f" {reader['emulator']}: {reader['source_ref']}") + + def print_platform_result( result: dict, group: list[str], verbose: bool = False ) -> None: @@ -2026,6 +2048,7 @@ def print_platform_result( seen_details: set[str] = set() _print_detail_entries(result["details"], seen_details, verbose) _print_undeclared_section(result, verbose) + _print_launcher_reads(result, verbose) # The same notes come back for every platform running these cores and # filled 985 of the 1114 lines of verify --all: counted, listed on ask. diff --git a/tests/test_launcher_gaps.py b/tests/test_launcher_gaps.py new file mode 100644 index 000000000..b68d19770 --- /dev/null +++ b/tests/test_launcher_gaps.py @@ -0,0 +1,269 @@ +"""Names a platform's launch code reads that the collection does not hold. + +The gap analysis compared the collection with the platforms' BIOS lists and +the emulator profiles only. PICO-8's pico8_64 went unseen although four +platforms' launchers run it. The fixtures give a synthetic platform launcher +records of every shape the scrapers write: held names, a name only a profile +alias answers, templates, folders and option values, missing names read by +two emulators, and a name held under another casing. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import contextlib +import hashlib +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) + +import yaml # noqa: E402 + +import verify # noqa: E402 +from common import _emulator_profiles_cache, load_emulator_profiles, load_platform_config # noqa: E402 +from cross_reference import ( # noqa: E402 + LAUNCHER_READS_KEY, + launcher_file_names, + launcher_reads_not_held, + print_launcher_reads, +) + +ROOT = Path(__file__).resolve().parent.parent + +CONTENT = { + "held.bin": b"HELD_BY_NAME", + "core.bin": b"CORE_FILE_ALIASED", + "Mixed.ROM": b"HELD_UNDER_ONE_CASING", +} +# Kept as a release asset: in database.json, absent from the checkout. +ASSET = ("asset.bin", b"RELEASE_ASSET_BYTES") + + +def _record(emulator: str, kind: str, names: list[str], **extra: object) -> dict: + return { + "emulator": emulator, "build": "standalone", "kind": kind, "names": names, + "search_dir": "/userdata/bios", "source_ref": f"gen/{emulator}.py:1", **extra, + } + + +LAUNCHERS = [ + _record("heldemu", "names_searched", ["held.bin", "asset.bin"]), + _record("aliasemu", "fixed_name", ["sub/alias.bin"]), + _record("tplemu", "names_searched", ["tos{L}.img"], expands={"L": ["de", "fr"]}), + _record("tplemu", "scans_dir", ["", "nand*{region}*.bin", "pat*.bin"]), + _record("diremu", "links", ["EUR", "keys"]), + _record("diremu", "core_option", ["default"]), + _record("diremu", "passes_path", []), + {"emulator": "pkg", "build": "standalone", "kind": "package", "names": [], + "source_ref": "pkg.sh:1"}, + _record("missemu", "names_searched", ["held.bin", "absent.bin"]), + _record("otheremu", "copies", ["absent.bin"]), + _record("caseemu", "fixed_name", ["mixed.rom"]), + _record("foldemu", "fixed_name", ["MIXED.rom"], case_insensitive=True), + _record("labelemu", "core_option", ["labelled.rom (CDZ)"]), +] + + +class LauncherGapTest(unittest.TestCase): + def setUp(self) -> None: + _emulator_profiles_cache.clear() + self.root = tempfile.mkdtemp() + self.bios_dir = os.path.join(self.root, "bios") + self.platforms_dir = os.path.join(self.root, "platforms") + self.emulators_dir = os.path.join(self.root, "emulators") + for d in (self.bios_dir, self.platforms_dir, self.emulators_dir): + os.makedirs(d) + files: dict[str, dict] = {} + by_md5: dict[str, str] = {} + by_name: dict[str, list[str]] = {} + + def add(name: str, data: bytes, write: bool) -> None: + path = os.path.join(self.bios_dir, name) + if write: + Path(path).write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + md5 = hashlib.md5(data).hexdigest() + files[sha1] = {"path": path, "name": name, "md5": md5, "sha1": sha1, + "size": len(data), "crc32": ""} + by_md5[md5] = sha1 + by_name.setdefault(name, []).append(sha1) + + for name, data in CONTENT.items(): + add(name, data, write=True) + add(*ASSET, write=False) + self.db = { + "files": files, + "indexes": {"by_md5": by_md5, "by_name": by_name, "by_crc32": {}, + "by_path_suffix": {}}, + } + platform = { + "platform": "GapTest", + "verification_mode": "md5", + "base_destination": "bios", + "systems": { + "test-sys": { + "files": [{"name": "held.bin", "destination": "held.bin", + "md5": hashlib.md5(CONTENT["held.bin"]).hexdigest(), + "required": True}], + }, + }, + "launchers": LAUNCHERS, + } + (Path(self.platforms_dir) / "gaptest.yml").write_text(yaml.dump(platform)) + (Path(self.platforms_dir) / "_registry.yml").write_text( + yaml.dump({"platforms": {"gaptest": {"config": "gaptest.yml"}}}) + ) + profiles = { + "aliasemu": { + "emulator": "AliasEmu", "type": "standalone", "systems": ["test-sys"], + "files": [{"name": "core.bin", "aliases": ["alias.bin"], "required": True}], + }, + "missemu": { + "emulator": "MissEmu", "type": "standalone", "systems": ["test-sys"], + "files": [{"name": "absent.bin", "required": True, "md5": "0" * 32}], + }, + } + for key, profile in profiles.items(): + (Path(self.emulators_dir) / f"{key}.yml").write_text(yaml.dump(profile)) + self.config = load_platform_config("gaptest", self.platforms_dir) + self.profiles = load_emulator_profiles(self.emulators_dir) + + def tearDown(self) -> None: + shutil.rmtree(self.root) + + def _rows(self, configs: dict[str, dict] | None = None, emulator: str = "") -> dict: + rows = launcher_reads_not_held( + configs or {"gaptest": self.config}, self.db, self.profiles, {}, emulator + ) + return {(r["platform"], r["name"]): r for r in rows} + + def test_a_held_name_is_not_reported(self) -> None: + names = {name for _, name in self._rows()} + self.assertNotIn("held.bin", names) + + def test_a_release_asset_is_held_without_its_bytes(self) -> None: + """The database answers for a file the checkout does not carry.""" + self.assertFalse(os.path.exists(os.path.join(self.bios_dir, "asset.bin"))) + self.assertNotIn("asset.bin", {name for _, name in self._rows()}) + + def test_a_profile_alias_covers_a_launcher_name(self) -> None: + """alias.bin is the name core.bin's profile entry also answers to.""" + self.assertNotIn("sub/alias.bin", {name for _, name in self._rows()}) + + def test_templates_folders_and_option_values_are_skipped(self) -> None: + for record in LAUNCHERS: + if record["emulator"] in ("tplemu", "diremu", "pkg"): + self.assertEqual(launcher_file_names(record), [], record) + reported = {r["emulators"][0] for r in self._rows().values()} + self.assertFalse(reported & {"tplemu", "diremu", "pkg"}) + + def test_a_label_after_the_name_is_dropped(self) -> None: + self.assertEqual(launcher_file_names(LAUNCHERS[-1]), ["labelled.rom"]) + self.assertIn(("gaptest", "labelled.rom"), self._rows()) + + def test_a_missing_name_is_reported_once_per_platform(self) -> None: + rows = self._rows() + row = rows[("gaptest", "absent.bin")] + self.assertEqual(row["emulators"], ["missemu", "otheremu"]) + self.assertEqual( + [(r["kind"], r["source_ref"]) for r in row["readers"]], + [("names_searched", "gen/missemu.py:1"), ("copies", "gen/otheremu.py:1")], + ) + self.assertEqual(row["readers"][0]["profiles"], ["missemu"]) + self.assertNotIn("held_as", row) + self.assertEqual(sum(1 for _, name in rows if name == "absent.bin"), 1) + + def test_each_platform_reports_its_own_row(self) -> None: + rows = self._rows({"gaptest": self.config, "second": self.config}) + self.assertIn(("gaptest", "absent.bin"), rows) + self.assertIn(("second", "absent.bin"), rows) + + def test_an_emulator_keeps_its_own_records(self) -> None: + rows = self._rows(emulator="missemu") + self.assertEqual(list(rows), [("gaptest", "absent.bin")]) + self.assertEqual(rows[("gaptest", "absent.bin")]["emulators"], ["missemu"]) + + def test_another_casing_satisfies_only_a_folding_launcher(self) -> None: + rows = self._rows() + self.assertEqual(rows[("gaptest", "mixed.rom")]["held_as"], "Mixed.ROM") + self.assertEqual(rows[("gaptest", "mixed.rom")]["emulators"], ["caseemu"]) + self.assertNotIn(("gaptest", "MIXED.rom"), rows) + + def test_a_folding_filesystem_folds_every_record(self) -> None: + config = {**self.config, "case_insensitive_fs": True} + self.assertNotIn(("gaptest", "mixed.rom"), self._rows({"gaptest": config})) + + def test_the_text_report_names_the_category(self) -> None: + out = io.StringIO() + with contextlib.redirect_stdout(out): + print_launcher_reads(list(self._rows().values())) + text = out.getvalue() + self.assertIn("Launcher reads, not held: 3 names across 1 platform", text) + self.assertIn("absent.bin [missemu, otheremu, names_searched, gen/missemu.py:1]", text) + + def test_json_carries_the_category_apart(self) -> None: + db_path = Path(self.root) / "database.json" + db_path.write_text(json.dumps(self.db)) + result = subprocess.run( + [sys.executable, "scripts/cross_reference.py", "--platform", "gaptest", + "--platforms-dir", self.platforms_dir, "--emulators-dir", self.emulators_dir, + "--db", str(db_path), "--json"], + cwd=ROOT, capture_output=True, text=True, timeout=300, + ) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + names = {r["name"] for r in report[LAUNCHER_READS_KEY]} + self.assertEqual(names, {"absent.bin", "mixed.rom", "labelled.rom"}) + # The emulator rows keep their own shape and counts. + self.assertEqual(set(report) - {LAUNCHER_READS_KEY}, {"aliasemu", "missemu"}) + + def test_verify_counts_and_exit_code_are_unchanged(self) -> None: + """A launcher name is reported, not judged: nothing else moves.""" + with_rows = verify.verify_platform( + self.config, self.db, self.emulators_dir, self.profiles, data_dir_registry={} + ) + with mock.patch.object(verify, "launcher_reads_not_held", return_value=[]): + without = verify.verify_platform( + self.config, self.db, self.emulators_dir, self.profiles, + data_dir_registry={}, + ) + rows = with_rows.pop("launcher_reads_not_held") + without.pop("launcher_reads_not_held") + self.assertEqual({r["name"] for r in rows}, {"absent.bin", "mixed.rom", "labelled.rom"}) + self.assertEqual(with_rows, without) + + def exit_code(result: dict) -> int: + try: + with contextlib.redirect_stderr(io.StringIO()): + verify._exit_if_strict(True, [result]) + except SystemExit as exc: + return int(exc.code or 0) + return 0 + + self.assertEqual(exit_code({**with_rows, "launcher_reads_not_held": rows}), + exit_code(without)) + + def test_verify_prints_a_count_and_lists_on_verbose(self) -> None: + result = verify.verify_platform( + self.config, self.db, self.emulators_dir, self.profiles, data_dir_registry={} + ) + for verbose, expected in ((False, "(--verbose lists them)"), (True, "absent.bin")): + out = io.StringIO() + with contextlib.redirect_stdout(out): + verify.print_platform_result(result, ["GapTest"], verbose=verbose) + self.assertIn("Launcher reads, not held: 3", out.getvalue()) + self.assertIn(expected, out.getvalue()) + + +if __name__ == "__main__": + unittest.main()