feat: report files only a launcher reads

cross_reference and verify list the names a platform's launch code reads that the collection holds under no name. Reported apart, never scored.

scripts/cross_reference.py

@@ -51,6 +51,7 @@ from common import (
 )
 from hashing import parse_hash_list
 from launchers import LauncherLayer, declared_destinations, held_record
+from launchers import _names as launcher_names
 from validation import (
     case_folded_slots,
     declaration_settles,
@@ -1030,6 +1031,191 @@ def _settle_launcher_places(
         data.update(_gap_counts(data["gap_details"]))
 
 
+# The --json key of the launcher reads category, beside the emulator keys.
+LAUNCHER_READS_KEY = "_launcher_reads_not_held"
+
+# A package record names no file: it records what a package builds.
+_NAMELESS_KINDS = frozenset({"package"})
+
+
+def launcher_file_names(record: dict) -> list[str]:
+    """The concrete file names a launcher record reads, in code order.
+
+    The names are read as the launcher layer reads them (`launchers._names`):
+    a label after the name is dropped ("neocd_z.rom (CDZ)"), and a name with
+    `<` or without an extension is not a file name (a folder the record
+    links, an option value such as hatari2014's "default"). A name with a
+    `{X}` placeholder is a template: Batocera's hatari tries tos{L}.img for
+    every language and ROCKNIX links one memory card per slot and region,
+    so its expansions say which variants the code accepts, not which file is
+    owed. A wildcard is a pattern the emulator matches. `search_dir` is a
+    folder and never a name, whatever the kind.
+    """
+    if record.get("kind") in _NAMELESS_KINDS:
+        return []
+    return [
+        name for name in launcher_names({**record, "expands": {}}) if "*" not in name
+    ]
+
+
+def _profile_names(profiles: dict[str, dict]) -> tuple[dict[str, list[dict]], dict[str, list[dict]]]:
+    """Profile entries by each name and alias they answer to, exact and folded."""
+    exact: dict[str, list[dict]] = {}
+    folded: dict[str, list[dict]] = {}
+    for profile in profiles.values():
+        if profile.get("type") in ("launcher", "alias"):
+            continue
+        for entry in profile.get("files") or []:
+            names = {str(entry.get("name") or "")} | {str(a) for a in entry.get("aliases") or []}
+            for name in names - {""}:
+                base = name.rsplit("/", 1)[-1]
+                exact.setdefault(base, []).append(entry)
+                folded.setdefault(base.casefold(), []).append(entry)
+    return exact, folded
+
+
+class _NameHolder:
+    """Whether the collection holds a file under a launcher name.
+
+    A name is held when database.json carries a file under it or under an
+    alias it indexes, when a profile entry that answers to the name (as its
+    name or an alias) resolves, or when resolve_local_file finds it in a data
+    directory. The database answers for files kept as release assets, absent
+    from a checkout. A copy known only under another casing is returned
+    apart: a launcher that matches names exactly does not open it.
+    """
+
+    def __init__(
+        self, db: dict, profiles: dict[str, dict], data_dir_registry: dict | None
+    ) -> None:
+        self.db = db
+        self.registry = data_dir_registry
+        files = db.get("files", {})
+        self.by_name = {
+            name: sha1s
+            for name, sha1s in db.get("indexes", {}).get("by_name", {}).items()
+            if any(sha1 in files for sha1 in sha1s)
+        }
+        self.by_name_folded: dict[str, list[str]] = {}
+        for name in self.by_name:
+            self.by_name_folded.setdefault(name.casefold(), []).append(name)
+        self.exact, self.folded = _profile_names(profiles)
+        self._cache: dict[str, tuple[str, bool] | None] = {}
+
+    def _entry_held(self, entry: dict) -> bool:
+        names = [str(entry.get("name") or "")] + [str(a) for a in entry.get("aliases") or []]
+        if any(n.rsplit("/", 1)[-1] in self.by_name for n in names if n):
+            return True
+        path, _status = resolve_local_file(entry, self.db, data_dir_registry=self.registry)
+        return path is not None
+
+    def held(self, name: str) -> tuple[str, bool] | None:
+        """The name the collection holds *name* under, and whether its casing is exact."""
+        if name in self._cache:
+            return self._cache[name]
+        base = name.rsplit("/", 1)[-1]
+        found: tuple[str, bool] | None = None
+        if base in self.by_name or any(map(self._entry_held, self.exact.get(base, []))):
+            found = (base, True)
+        else:
+            path, _status = resolve_local_file(
+                {"name": name}, self.db, data_dir_registry=self.registry
+            )
+            if path is not None:
+                found = (Path(path).name, Path(path).name == base)
+            elif base.casefold() in self.by_name_folded:
+                found = (self.by_name_folded[base.casefold()][0], False)
+            else:
+                entry = next(
+                    (e for e in self.folded.get(base.casefold(), []) if self._entry_held(e)),
+                    None,
+                )
+                if entry is not None:
+                    found = (str(entry.get("name") or ""), False)
+        self._cache[name] = found
+        return found
+
+
+def launcher_reads_not_held(
+    configs: dict[str, dict],
+    db: dict,
+    profiles: dict[str, dict],
+    data_dir_registry: dict | None = None,
+    emulator: str = "",
+) -> list[dict]:
+    """The file names a platform's launch code reads that the collection lacks.
+
+    One row per platform and name, with every record that reads it. A
+    launcher that matches case-insensitively, or runs on a filesystem that
+    folds case, is satisfied by a copy under another casing. Elsewhere such
+    a copy is reported with `held_as`, since the launcher does not open it
+    under the name it asks for. *emulator* keeps the records whose emulator
+    name designates that profile.
+    """
+    holder = _NameHolder(db, profiles, data_dir_registry)
+    every_profile = set(profiles)
+    rows: list[dict] = []
+    for platform, config in configs.items():
+        by_key: dict[str, dict] = {}
+        for record in config.get("launchers") or []:
+            names = launcher_file_names(record)
+            if not names:
+                continue
+            readers = sorted({
+                key
+                for part in str(record.get("emulator", "")).split(",")
+                for key in LauncherLayer._profiles_named(part.strip(), profiles, every_profile)
+            })
+            if emulator and emulator not in readers:
+                continue
+            folds = bool(record.get("case_insensitive") or config.get("case_insensitive_fs"))
+            reader = {
+                "emulator": str(record.get("emulator", "")),
+                "build": str(record.get("build", "")),
+                "kind": str(record.get("kind", "")),
+                "source_ref": str(record.get("source_ref", "")),
+                "profiles": readers,
+            }
+            for name in names:
+                found = holder.held(name)
+                if found and (found[1] or folds):
+                    continue
+                key = name.casefold() if folds else name
+                row = by_key.get(key)
+                if row is None:
+                    row = by_key[key] = {
+                        "platform": platform,
+                        "name": name,
+                        "emulators": [],
+                        "readers": [],
+                    }
+                    if found:
+                        row["held_as"] = found[0]
+                if reader["emulator"] not in row["emulators"]:
+                    row["emulators"].append(reader["emulator"])
+                if reader not in row["readers"]:
+                    row["readers"].append(reader)
+        rows.extend(by_key[key] for key in sorted(by_key, key=str.casefold))
+    return rows
+
+
+def print_launcher_reads(rows: list[dict]) -> None:
+    """The launcher reads gap category, after the profile gaps."""
+    platforms = sorted({r["platform"] for r in rows})
+    noun = "platform" if len(platforms) == 1 else "platforms"
+    print(f"\nLauncher reads, not held: {len(rows)} names across {len(platforms)} {noun}")
+    for platform in platforms:
+        print(f"\n{platform}")
+        for row in (r for r in rows if r["platform"] == platform):
+            first = row["readers"][0]
+            more = f" (+{len(row['readers']) - 1} records)" if len(row["readers"]) > 1 else ""
+            casing = f", held as {row['held_as']}" if row.get("held_as") else ""
+            print(
+                f"    {row['name']} [{', '.join(row['emulators'])}, {first['kind']}, "
+                f"{first['source_ref']}]{more}{casing}"
+            )
+
+
 def cross_reference(
     profiles: dict[str, dict],
     declared: dict[str, set[str]],
@@ -1210,9 +1396,12 @@ def main() -> None:
     # Files resolve against data/: the trees stay put until the process
     # ends, which releases the hold.
     held = contextlib.ExitStack()
-    hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
+    registry = load_data_dir_registry(args.platforms_dir)
+    hold_data_dirs(held, registry)
 
     profiles = load_emulator_profiles(args.emulators_dir)
+    # Every profile's names and aliases, whatever --emulator or --platform keeps.
+    all_profiles = profiles
     emulator = args.emulator
     if emulator:
         # A typo answered "No emulator profiles found" with exit 0.
@@ -1288,8 +1477,18 @@ def main() -> None:
         ) if args.platform else None,
     )
 
+    # Counted apart from the profile gaps: a name only a launcher reads is
+    # reported, not yet judged.
+    launcher_reads = launcher_reads_not_held(
+        {
+            name: load_platform_config(name, args.platforms_dir)
+            for name in (selected or list_registered_platforms(args.platforms_dir))
+        },
+        db, all_profiles, registry, emulator or "",
+    )
+
     if args.json:
-        print(json.dumps(report, indent=2))
+        print(json.dumps({**report, LAUNCHER_READS_KEY: launcher_reads}, indent=2))
     else:
         if emulator:
             scope = emulator
@@ -1298,6 +1497,7 @@ def main() -> None:
         else:
             scope = "all emulators"
         print_report(report, scope, show_clean=bool(args.emulator))
+        print_launcher_reads(launcher_reads)
 
 
 if __name__ == "__main__":

scripts/verify.py

@@ -91,6 +91,7 @@ from common import (
 yaml = require_yaml()
 from exporter.candidates import BIOS, EntryCategories
 from launchers import LauncherLayer, declared_destinations, held_record
+from cross_reference import launcher_reads_not_held
 from nativemode import (
     SIZE_AND_ANY_HASH,
     digest_algorithm,
@@ -1595,6 +1596,10 @@ def verify_platform(
         "excluded_files": excluded,
         "packed_extras": packed_extras,
         "undeclared_files": undeclared,
+        # Reported apart, never scored: no severity, count or exit code reads it.
+        "launcher_reads_not_held": launcher_reads_not_held(
+            {config.get("_key", platform): config}, db, profiles, data_dir_registry
+        ),
         "exclusion_notes": exclusions,
         "region_fallbacks": sorted(region_fallbacks),
         **({"left_out": left_out} if no_game_data else {}),
@@ -1959,6 +1964,23 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
             )
 
 
+def _print_launcher_reads(result: dict, verbose: bool) -> None:
+    """Names the platform's launch code reads that the collection lacks."""
+    rows = result.get("launcher_reads_not_held") or []
+    if not rows:
+        return
+    listed = "" if verbose else " (--verbose lists them)"
+    print(f"  Launcher reads, not held: {len(rows)}{listed}")
+    if not verbose:
+        return
+    for row in rows:
+        kinds = ", ".join(sorted({r["kind"] for r in row["readers"]}))
+        casing = f", held as {row['held_as']}" if row.get("held_as") else ""
+        print(f"    {row['name']} [{', '.join(row['emulators'])}, {kinds}{casing}]")
+        for reader in row["readers"]:
+            print(f"      {reader['emulator']}: {reader['source_ref']}")
+
+
 def print_platform_result(
     result: dict, group: list[str], verbose: bool = False
 ) -> None:
@@ -2026,6 +2048,7 @@ def print_platform_result(
     seen_details: set[str] = set()
     _print_detail_entries(result["details"], seen_details, verbose)
     _print_undeclared_section(result, verbose)
+    _print_launcher_reads(result, verbose)
 
     # The same notes come back for every platform running these cores and
     # filled 985 of the 1114 lines of verify --all: counted, listed on ask.

tests/test_launcher_gaps.py

@@ -0,0 +1,269 @@
+"""Names a platform's launch code reads that the collection does not hold.
+
+The gap analysis compared the collection with the platforms' BIOS lists and
+the emulator profiles only. PICO-8's pico8_64 went unseen although four
+platforms' launchers run it. The fixtures give a synthetic platform launcher
+records of every shape the scrapers write: held names, a name only a profile
+alias answers, templates, folders and option values, missing names read by
+two emulators, and a name held under another casing.
+"""
+
+from __future__ import annotations
+
+import tests  # noqa: F401  offline, scratch under tmp/, the suite's own cache
+
+import contextlib
+import hashlib
+import io
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
+
+import yaml  # noqa: E402
+
+import verify  # noqa: E402
+from common import _emulator_profiles_cache, load_emulator_profiles, load_platform_config  # noqa: E402
+from cross_reference import (  # noqa: E402
+    LAUNCHER_READS_KEY,
+    launcher_file_names,
+    launcher_reads_not_held,
+    print_launcher_reads,
+)
+
+ROOT = Path(__file__).resolve().parent.parent
+
+CONTENT = {
+    "held.bin": b"HELD_BY_NAME",
+    "core.bin": b"CORE_FILE_ALIASED",
+    "Mixed.ROM": b"HELD_UNDER_ONE_CASING",
+}
+# Kept as a release asset: in database.json, absent from the checkout.
+ASSET = ("asset.bin", b"RELEASE_ASSET_BYTES")
+
+
+def _record(emulator: str, kind: str, names: list[str], **extra: object) -> dict:
+    return {
+        "emulator": emulator, "build": "standalone", "kind": kind, "names": names,
+        "search_dir": "/userdata/bios", "source_ref": f"gen/{emulator}.py:1", **extra,
+    }
+
+
+LAUNCHERS = [
+    _record("heldemu", "names_searched", ["held.bin", "asset.bin"]),
+    _record("aliasemu", "fixed_name", ["sub/alias.bin"]),
+    _record("tplemu", "names_searched", ["tos{L}.img"], expands={"L": ["de", "fr"]}),
+    _record("tplemu", "scans_dir", ["<first file 4-8 MiB>", "nand*{region}*.bin", "pat*.bin"]),
+    _record("diremu", "links", ["EUR", "keys"]),
+    _record("diremu", "core_option", ["default"]),
+    _record("diremu", "passes_path", []),
+    {"emulator": "pkg", "build": "standalone", "kind": "package", "names": [],
+     "source_ref": "pkg.sh:1"},
+    _record("missemu", "names_searched", ["held.bin", "absent.bin"]),
+    _record("otheremu", "copies", ["absent.bin"]),
+    _record("caseemu", "fixed_name", ["mixed.rom"]),
+    _record("foldemu", "fixed_name", ["MIXED.rom"], case_insensitive=True),
+    _record("labelemu", "core_option", ["labelled.rom (CDZ)"]),
+]
+
+
+class LauncherGapTest(unittest.TestCase):
+    def setUp(self) -> None:
+        _emulator_profiles_cache.clear()
+        self.root = tempfile.mkdtemp()
+        self.bios_dir = os.path.join(self.root, "bios")
+        self.platforms_dir = os.path.join(self.root, "platforms")
+        self.emulators_dir = os.path.join(self.root, "emulators")
+        for d in (self.bios_dir, self.platforms_dir, self.emulators_dir):
+            os.makedirs(d)
+        files: dict[str, dict] = {}
+        by_md5: dict[str, str] = {}
+        by_name: dict[str, list[str]] = {}
+
+        def add(name: str, data: bytes, write: bool) -> None:
+            path = os.path.join(self.bios_dir, name)
+            if write:
+                Path(path).write_bytes(data)
+            sha1 = hashlib.sha1(data).hexdigest()
+            md5 = hashlib.md5(data).hexdigest()
+            files[sha1] = {"path": path, "name": name, "md5": md5, "sha1": sha1,
+                           "size": len(data), "crc32": ""}
+            by_md5[md5] = sha1
+            by_name.setdefault(name, []).append(sha1)
+
+        for name, data in CONTENT.items():
+            add(name, data, write=True)
+        add(*ASSET, write=False)
+        self.db = {
+            "files": files,
+            "indexes": {"by_md5": by_md5, "by_name": by_name, "by_crc32": {},
+                        "by_path_suffix": {}},
+        }
+        platform = {
+            "platform": "GapTest",
+            "verification_mode": "md5",
+            "base_destination": "bios",
+            "systems": {
+                "test-sys": {
+                    "files": [{"name": "held.bin", "destination": "held.bin",
+                               "md5": hashlib.md5(CONTENT["held.bin"]).hexdigest(),
+                               "required": True}],
+                },
+            },
+            "launchers": LAUNCHERS,
+        }
+        (Path(self.platforms_dir) / "gaptest.yml").write_text(yaml.dump(platform))
+        (Path(self.platforms_dir) / "_registry.yml").write_text(
+            yaml.dump({"platforms": {"gaptest": {"config": "gaptest.yml"}}})
+        )
+        profiles = {
+            "aliasemu": {
+                "emulator": "AliasEmu", "type": "standalone", "systems": ["test-sys"],
+                "files": [{"name": "core.bin", "aliases": ["alias.bin"], "required": True}],
+            },
+            "missemu": {
+                "emulator": "MissEmu", "type": "standalone", "systems": ["test-sys"],
+                "files": [{"name": "absent.bin", "required": True, "md5": "0" * 32}],
+            },
+        }
+        for key, profile in profiles.items():
+            (Path(self.emulators_dir) / f"{key}.yml").write_text(yaml.dump(profile))
+        self.config = load_platform_config("gaptest", self.platforms_dir)
+        self.profiles = load_emulator_profiles(self.emulators_dir)
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.root)
+
+    def _rows(self, configs: dict[str, dict] | None = None, emulator: str = "") -> dict:
+        rows = launcher_reads_not_held(
+            configs or {"gaptest": self.config}, self.db, self.profiles, {}, emulator
+        )
+        return {(r["platform"], r["name"]): r for r in rows}
+
+    def test_a_held_name_is_not_reported(self) -> None:
+        names = {name for _, name in self._rows()}
+        self.assertNotIn("held.bin", names)
+
+    def test_a_release_asset_is_held_without_its_bytes(self) -> None:
+        """The database answers for a file the checkout does not carry."""
+        self.assertFalse(os.path.exists(os.path.join(self.bios_dir, "asset.bin")))
+        self.assertNotIn("asset.bin", {name for _, name in self._rows()})
+
+    def test_a_profile_alias_covers_a_launcher_name(self) -> None:
+        """alias.bin is the name core.bin's profile entry also answers to."""
+        self.assertNotIn("sub/alias.bin", {name for _, name in self._rows()})
+
+    def test_templates_folders_and_option_values_are_skipped(self) -> None:
+        for record in LAUNCHERS:
+            if record["emulator"] in ("tplemu", "diremu", "pkg"):
+                self.assertEqual(launcher_file_names(record), [], record)
+        reported = {r["emulators"][0] for r in self._rows().values()}
+        self.assertFalse(reported & {"tplemu", "diremu", "pkg"})
+
+    def test_a_label_after_the_name_is_dropped(self) -> None:
+        self.assertEqual(launcher_file_names(LAUNCHERS[-1]), ["labelled.rom"])
+        self.assertIn(("gaptest", "labelled.rom"), self._rows())
+
+    def test_a_missing_name_is_reported_once_per_platform(self) -> None:
+        rows = self._rows()
+        row = rows[("gaptest", "absent.bin")]
+        self.assertEqual(row["emulators"], ["missemu", "otheremu"])
+        self.assertEqual(
+            [(r["kind"], r["source_ref"]) for r in row["readers"]],
+            [("names_searched", "gen/missemu.py:1"), ("copies", "gen/otheremu.py:1")],
+        )
+        self.assertEqual(row["readers"][0]["profiles"], ["missemu"])
+        self.assertNotIn("held_as", row)
+        self.assertEqual(sum(1 for _, name in rows if name == "absent.bin"), 1)
+
+    def test_each_platform_reports_its_own_row(self) -> None:
+        rows = self._rows({"gaptest": self.config, "second": self.config})
+        self.assertIn(("gaptest", "absent.bin"), rows)
+        self.assertIn(("second", "absent.bin"), rows)
+
+    def test_an_emulator_keeps_its_own_records(self) -> None:
+        rows = self._rows(emulator="missemu")
+        self.assertEqual(list(rows), [("gaptest", "absent.bin")])
+        self.assertEqual(rows[("gaptest", "absent.bin")]["emulators"], ["missemu"])
+
+    def test_another_casing_satisfies_only_a_folding_launcher(self) -> None:
+        rows = self._rows()
+        self.assertEqual(rows[("gaptest", "mixed.rom")]["held_as"], "Mixed.ROM")
+        self.assertEqual(rows[("gaptest", "mixed.rom")]["emulators"], ["caseemu"])
+        self.assertNotIn(("gaptest", "MIXED.rom"), rows)
+
+    def test_a_folding_filesystem_folds_every_record(self) -> None:
+        config = {**self.config, "case_insensitive_fs": True}
+        self.assertNotIn(("gaptest", "mixed.rom"), self._rows({"gaptest": config}))
+
+    def test_the_text_report_names_the_category(self) -> None:
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            print_launcher_reads(list(self._rows().values()))
+        text = out.getvalue()
+        self.assertIn("Launcher reads, not held: 3 names across 1 platform", text)
+        self.assertIn("absent.bin [missemu, otheremu, names_searched, gen/missemu.py:1]", text)
+
+    def test_json_carries_the_category_apart(self) -> None:
+        db_path = Path(self.root) / "database.json"
+        db_path.write_text(json.dumps(self.db))
+        result = subprocess.run(
+            [sys.executable, "scripts/cross_reference.py", "--platform", "gaptest",
+             "--platforms-dir", self.platforms_dir, "--emulators-dir", self.emulators_dir,
+             "--db", str(db_path), "--json"],
+            cwd=ROOT, capture_output=True, text=True, timeout=300,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        report = json.loads(result.stdout)
+        names = {r["name"] for r in report[LAUNCHER_READS_KEY]}
+        self.assertEqual(names, {"absent.bin", "mixed.rom", "labelled.rom"})
+        # The emulator rows keep their own shape and counts.
+        self.assertEqual(set(report) - {LAUNCHER_READS_KEY}, {"aliasemu", "missemu"})
+
+    def test_verify_counts_and_exit_code_are_unchanged(self) -> None:
+        """A launcher name is reported, not judged: nothing else moves."""
+        with_rows = verify.verify_platform(
+            self.config, self.db, self.emulators_dir, self.profiles, data_dir_registry={}
+        )
+        with mock.patch.object(verify, "launcher_reads_not_held", return_value=[]):
+            without = verify.verify_platform(
+                self.config, self.db, self.emulators_dir, self.profiles,
+                data_dir_registry={},
+            )
+        rows = with_rows.pop("launcher_reads_not_held")
+        without.pop("launcher_reads_not_held")
+        self.assertEqual({r["name"] for r in rows}, {"absent.bin", "mixed.rom", "labelled.rom"})
+        self.assertEqual(with_rows, without)
+
+        def exit_code(result: dict) -> int:
+            try:
+                with contextlib.redirect_stderr(io.StringIO()):
+                    verify._exit_if_strict(True, [result])
+            except SystemExit as exc:
+                return int(exc.code or 0)
+            return 0
+
+        self.assertEqual(exit_code({**with_rows, "launcher_reads_not_held": rows}),
+                         exit_code(without))
+
+    def test_verify_prints_a_count_and_lists_on_verbose(self) -> None:
+        result = verify.verify_platform(
+            self.config, self.db, self.emulators_dir, self.profiles, data_dir_registry={}
+        )
+        for verbose, expected in ((False, "(--verbose lists them)"), (True, "absent.bin")):
+            out = io.StringIO()
+            with contextlib.redirect_stdout(out):
+                verify.print_platform_result(result, ["GapTest"], verbose=verbose)
+            self.assertIn("Launcher reads, not held: 3", out.getvalue())
+            self.assertIn(expected, out.getvalue())
+
+
+if __name__ == "__main__":
+    unittest.main()