feat: report entries no examined core opens

The export keeps every entry a platform declares. An entry the truth
does not pair was still read in 1189 of 2776 cases (alias, hash, a
type: directory tree, a core outside the resolved set), so absence of
a pairing proves nothing. Each report now lists, under
removal_candidates, the unpaired entries that no profile of the cores
the platform runs answers by name, path tail, hash, directory or data
directory, with the cores examined and a reason. A core the platform
names without a profile, or a ROM set under a core that opens sets by
name, leaves the verdict undetermined. The exports page states the
count and lists the entries per platform.

scripts/common.py

@@ -2294,6 +2294,26 @@ def resolve_platform_cores(
     return result
 
 
+def unresolved_platform_cores(config: dict, profiles: dict[str, dict]) -> set[str]:
+    """The core names a platform lists that no profile answers to.
+
+    resolve_platform_cores drops them without a word: ROCKNIX's `dolphin-sa`
+    or Batocera's `wine` designate no profile, so nothing is known of the
+    files they open. `retroarch` and `all_libretro` stand for every libretro
+    profile and are never unresolved. A platform without a `cores:` list
+    names no core.
+    """
+    cores_config = config.get("cores")
+    if not isinstance(cores_config, list):
+        return set()
+    index = upstream_profile_index(profiles)
+    return {
+        name
+        for name in (str(c) for c in cores_config)
+        if name not in ("retroarch", "all_libretro") and not index.get(name)
+    }
+
+
 MANUFACTURER_PREFIXES = (
     "acorn-",
     "apple-",

scripts/export_native.py

@@ -38,6 +38,7 @@ import yaml
 from common import (
     platform_arg,
     list_registered_platforms,
+    load_data_dir_registry,
     load_database,
     load_emulator_profiles,
     load_platform_config,
@@ -47,6 +48,7 @@ from common import (
 )
 from exporter import discover_exporters
 from exporter.baseline import HASH_FIELDS, _hash_values, _single_size, build_native_model
+from exporter.candidates import removal_candidates
 from truth import generate_platform_truth
 
 DEFAULT_CACHE = ".cache/upstream-native"
@@ -613,7 +615,19 @@ def export_platform(
         # A content-scanning core's names are labels: a list checked by path
         # cannot hold them.
         summary += f", {_count(report.labels, 'entry', 'entries')} found by content not listed"
+    # Reported, never removed: the platform's file keeps every entry.
+    candidates = removal_candidates(
+        systems, scraped, load_emulator_profiles("emulators"),
+        load_data_dir_registry(platforms_dir),
+    )
     messages.append(summary)
+    if candidates:
+        open_question = sum(c["verdict"] == "undetermined" for c in candidates)
+        messages.append(
+            f"{_count(len(candidates), 'entry', 'entries')} no examined core is known to open"
+            + (f" ({open_question} undetermined)" if open_question else "")
+            + ", listed in the report"
+        )
     messages.extend(f"withheld: {reason}" for reason in withheld)
     for correction in applied[:5]:
         messages.append(f"hash corrected: {correction}")
@@ -655,6 +669,7 @@ def export_platform(
         "requirements_corrected": requirements,
         "withheld": list(withheld),
         "found_by_content": report.labels,
+        "removal_candidates": candidates,
         "invalid": issues,
     })
     return True, messages

scripts/exporter/candidates.py

@@ -0,0 +1,209 @@
+"""Entries a platform declares that no core it runs is known to open.
+
+The export never removes a platform's entry. An entry the truth did not
+pair may still be read: by a name or an alias, by its content, under a
+directory a core walks, inside a data directory. Each of those is looked
+for in the profiles of the cores the platform runs, and an entry none of
+them answers is listed with what was examined. Absence of a match is not
+proof that the code ignores the file, so the list is a report, and a core
+the platform names that no profile describes leaves the verdict open.
+"""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from common import _norm_system_id, resolve_platform_cores, unresolved_platform_cores
+from exporter.baseline import HASH_FIELDS, NativeSystem, _hash_values
+
+# Batocera cuts some md5 to 29 characters: the prefix still names the dump.
+_FULL_MD5 = 32
+_CUT_MD5 = 29
+
+
+def _tails(path: str) -> list[str]:
+    """Every trailing run of segments of a path, longest first."""
+    parts = [p for p in path.casefold().strip("/").split("/") if p]
+    return ["/".join(parts[i:]) for i in range(len(parts))]
+
+
+def _entry_hashes(entry: dict) -> set[str]:
+    values = {v for f in HASH_FIELDS for v in _hash_values(entry, f)}
+    for member in entry.get("contents") or []:
+        if isinstance(member, dict):
+            values |= {v for f in HASH_FIELDS for v in _hash_values(member, f)}
+    return values
+
+
+@dataclass
+class ReadEvidence:
+    """What the examined profiles open, indexed for one lookup per entry."""
+
+    names: set[str] = field(default_factory=set)
+    tails: set[str] = field(default_factory=set)
+    hashes: set[str] = field(default_factory=set)
+    directories: set[str] = field(default_factory=set)
+    trees: set[str] = field(default_factory=set)
+    tree_members: set[str] = field(default_factory=set)
+
+    def add_profile(self, profile: dict, data_registry: dict) -> None:
+        for entry in profile.get("files") or []:
+            if not isinstance(entry, dict):
+                continue
+            locations = [
+                str(entry[key]) for key in ("path", "standalone_path", "destination")
+                if entry.get(key)
+            ]
+            if entry.get("type") == "directory":
+                self.directories |= {
+                    "/".join(p for p in loc.casefold().split("/") if p) + "/"
+                    for loc in [*locations, str(entry.get("name", ""))] if loc
+                }
+                continue
+            self.names |= {
+                str(n).casefold()
+                for n in (entry.get("name"), *(entry.get("aliases") or []))
+                if n
+            }
+            for location in locations:
+                tails = _tails(location)
+                self.tails.update(tails)
+                if tails:
+                    self.names.add(tails[-1])
+            self.hashes |= _entry_hashes(entry)
+        for tree in profile.get("data_directories") or []:
+            if not isinstance(tree, dict):
+                continue
+            destination = str(tree.get("destination") or "").casefold().strip("/")
+            if destination:
+                self.trees.add(destination + "/")
+                continue
+            # A tree laid at the root: only its own files say what it holds.
+            cache = (data_registry.get(str(tree.get("ref", ""))) or {}).get("local_cache")
+            if cache and Path(cache).is_dir():
+                self.tree_members |= {
+                    p.relative_to(cache).as_posix().casefold()
+                    for p in Path(cache).rglob("*") if p.is_file()
+                }
+
+    def proof(self, entry: dict, destination: str) -> str | None:
+        """How a profile reads the entry, or None when nothing answers."""
+        names = {
+            str(n).casefold()
+            for n in (entry.get("name"), *(entry.get("aliases") or []))
+            if n
+        }
+        tails = _tails(destination)
+        if any(t in self.tails for t in tails[:-1]):
+            return "path"
+        if names & self.names or (tails and tails[-1] in self.names):
+            return "name"
+        hashes = {
+            v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
+        }
+        if hashes & self.hashes or any(
+            len(v) < _FULL_MD5 and any(h.startswith(v) for h in self.hashes)
+            for v in hashes if len(v) >= _CUT_MD5
+        ):
+            return "hash"
+        full = "/" + (tails[0] if tails else "")
+        if any("/" + d in full for d in self.directories):
+            return "directory"
+        if any("/" + t in full for t in self.trees):
+            return "data_directory"
+        if full[1:] in self.tree_members:
+            return "data_directory"
+        return None
+
+
+def _loads_romsets(profile: dict) -> bool:
+    """A core that opens ROM sets by name: MAME pulls a device's set from
+    device_t::searchpath(), and no profile lists every device set."""
+    return any(
+        isinstance(f, dict) and f.get("category") == "bios_zip"
+        for f in profile.get("files") or []
+    )
+
+
+_ARCHIVES = (".zip", ".7z")
+
+REASONS = {
+    "candidate": "no core the platform runs opens it",
+    "unresolved": "a core the platform names has no profile yet",
+    "romset": "a ROM set, and these cores open sets their profiles do not list",
+}
+
+
+def removal_candidates(
+    systems: dict[str, NativeSystem],
+    scraped: dict | None,
+    profiles: dict[str, dict],
+    data_registry: dict | None = None,
+) -> list[dict]:
+    """The platform's unpaired entries that no examined core is known to open.
+
+    Every core the platform runs is searched. `cores_examined` names those
+    among them that emulate the entry's system, the ones a reader expects to
+    find it in. `verdict` is `candidate` when nothing examined could open the
+    entry, `undetermined` when something still could: a core the platform
+    names that no profile describes, or a core of the system that opens ROM
+    sets by name. A platform that resolves no core (BizHawk wires each
+    firmware in its own code, MiSTer's BiosDB is its authority) gives nothing
+    to examine and lists nothing. Nothing is removed from the platform's file.
+    """
+    if not scraped:
+        return []
+    examined = resolve_platform_cores(scraped, profiles)
+    if not examined:
+        return []
+    unresolved = sorted(unresolved_platform_cores(scraped, profiles))
+    evidence = ReadEvidence()
+    for key in sorted(examined):
+        evidence.add_profile(profiles[key], data_registry or {})
+
+    system_of = {
+        id(file_entry): sys_key
+        for sys_key, sys_data in (scraped.get("systems") or {}).items()
+        for file_entry in sys_data.get("files", [])
+    }
+    by_system: dict[str, list[str]] = {}
+    for key in sorted(examined):
+        for sid in profiles[key].get("systems") or []:
+            by_system.setdefault(_norm_system_id(str(sid)), []).append(key)
+
+    romset_cores = {key for key in examined if _loads_romsets(profiles[key])}
+
+    listed: list[dict] = []
+    for system in systems.values():
+        for entry in system.files:
+            if entry.platform is None or entry.truth is not None or not entry.name:
+                continue
+            if evidence.proof(entry.platform, entry.destination) is not None:
+                continue
+            slug = system_of.get(id(entry.platform), entry.native_system)
+            cores = by_system.get(_norm_system_id(slug), [])
+            readers = (
+                [c for c in cores if c in romset_cores]
+                if entry.name.casefold().endswith(_ARCHIVES) else []
+            )
+            if unresolved:
+                verdict, reason = "undetermined", REASONS["unresolved"]
+            elif readers:
+                verdict = "undetermined"
+                reason = f"{REASONS['romset']}: {', '.join(readers)}"
+            else:
+                verdict, reason = "candidate", REASONS["candidate"]
+            listed.append({
+                "system": entry.native_system,
+                "name": entry.name,
+                "destination": entry.destination,
+                "verdict": verdict,
+                "reason": reason,
+                "cores_examined": cores,
+                "unresolved_cores": unresolved,
+            })
+    return listed

scripts/siteexports.py

@@ -54,6 +54,49 @@ def change_text(report: dict) -> str:
     return ", ".join(parts)
 
 
+# Beyond this many entries a reason's list folds away.
+LONG_LIST = 10
+
+
+def candidate_lines(candidates: list[dict]) -> list[str]:
+    """The entries no emulator the platform runs is known to open.
+
+    Each stays in the updated file. The count comes first, then the entries
+    grouped by reason, a long group in a block the reader opens.
+    """
+    if not candidates:
+        return []
+    open_question = sum(c.get("verdict") == "undetermined" for c in candidates)
+    count = _plural(len(candidates), "entry", "entries")
+    sentence = (
+        f"{count} of the original {'is' if len(candidates) == 1 else 'are'} not "
+        "opened by any profiled emulator this platform runs. "
+        f"{'It stays' if len(candidates) == 1 else 'They stay'} in the updated file"
+    )
+    if open_question:
+        sentence += (
+            f", and {open_question} of them may still be read by an emulator "
+            "this project has not profiled in full"
+        )
+    lines = [sentence + ".", ""]
+    groups: dict[str, list[dict]] = {}
+    for candidate in candidates:
+        groups.setdefault(candidate.get("reason", ""), []).append(candidate)
+    for reason, entries in groups.items():
+        title = (reason[:1].upper() + reason[1:]).replace('"', "'")
+        items = [f"`{c['system']}` `{c['destination']}`" for c in entries]
+        if len(entries) > LONG_LIST:
+            lines.extend([
+                f'??? note "{title} ({len(entries)})"',
+                "",
+                *[f"    - {item}" for item in items],
+                "",
+            ])
+        else:
+            lines.extend([f"{title}:", "", *[f"- {item}" for item in items], ""])
+    return lines
+
+
 def load_export_reports(directory: str = EXPORTS_DIR) -> list[dict]:
     """Every platform report the export wrote, by platform name."""
     root = Path(directory)
@@ -95,8 +138,12 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
         "`es_bios.xml` and so on. This project reads the emulators' source "
         "code file by file, and offers each platform its own file with those "
         "findings applied: a hash is updated where the emulator source states "
-        "another, files the platform's emulators read are added, and no entry "
-        "the platform declares is removed. Where a value of an entry is "
+        "another, and files the platform's emulators read are added. An entry "
+        "that no emulator the platform runs is known to open stays in the "
+        "file and is listed as a removal candidate, with the reason, on the "
+        "platform's section. It would leave the file only once the code of "
+        "every emulator that platform runs for the system is shown not to "
+        "read it. Where a value of an entry is "
         "dropped, the platform's section lists it. Code the file carries is "
         "patched in place. With the original files on hand, the tests run "
         "the updated Batocera and ROCKNIX scripts (`--createReadme`), parse "
@@ -159,6 +206,7 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
                 "cannot be written in this format and should be checked by hand.",
                 "",
             ])
+        lines.extend(candidate_lines(report.get("removal_candidates") or []))
         if report.get("invalid"):
             lines.extend([
                 '!!! warning "The updated file does not pass the platform\'s own checks"',

tests/test_export_report.py

@@ -157,5 +157,99 @@ class ThePageAddressesMaintainersAsPeers(unittest.TestCase):
             self.assertNotIn(word, page)
 
 
+class RemovalCandidatesNeedAReadingNoCoreAnswers(unittest.TestCase):
+    """An entry the truth does not pair can still be read: by an alias, by its
+    content, under a directory a core walks. Of 2776 such entries, 1189
+    were read that way. Only what no core answers is listed, and nothing is
+    removed."""
+
+    PROFILES = {
+        "corea": {
+            "type": "libretro",
+            "systems": ["sys-one"],
+            "files": [
+                {"name": "main.rom", "aliases": ["Alt.ROM"]},
+                {"name": "other.bin", "md5": "0123456789abcdef0123456789abcdef"},
+                {"name": "rtp/2000/", "path": "rtp/2000/", "type": "directory"},
+                {"name": "set.zip", "category": "bios_zip"},
+            ],
+        },
+    }
+
+    def _scraped(self, names: list[dict], cores: list[str]) -> dict:
+        return {
+            "_key": "fake",
+            "cores": cores,
+            "systems": {"sys-one": {"files": names}},
+        }
+
+    def _candidates(self, files: list[dict], cores: list[str] | None = None) -> list[dict]:
+        from exporter.baseline import build_native_model
+        from exporter.candidates import removal_candidates
+
+        scraped = self._scraped(files, cores or ["corea"])
+        systems, _ = build_native_model({"systems": {}}, scraped)
+        found = removal_candidates(systems, scraped, self.PROFILES)
+        kept = [f.name for s in systems.values() for f in s.files]
+        self.assertEqual(kept, [f["name"] for f in files])
+        return found
+
+    def test_an_alias_is_a_reading(self):
+        self.assertEqual(self._candidates([{"name": "alt.rom"}]), [])
+
+    def test_a_hash_is_a_reading(self):
+        found = self._candidates([
+            {"name": "renamed.bin", "md5": "0123456789ABCDEF0123456789ABCDEF"},
+        ])
+        self.assertEqual(found, [])
+
+    def test_a_directory_entry_covers_its_members(self):
+        found = self._candidates([
+            {"name": "x.png", "destination": "easyrpg/rtp/2000/x.png"},
+        ])
+        self.assertEqual(found, [])
+
+    def test_an_entry_nothing_answers_is_a_candidate(self):
+        found = self._candidates([{"name": "nobody.rom"}])
+        self.assertEqual(len(found), 1)
+        self.assertEqual(found[0]["verdict"], "candidate")
+        self.assertEqual(found[0]["cores_examined"], ["corea"])
+        self.assertEqual(found[0]["unresolved_cores"], [])
+
+    def test_an_unprofiled_core_leaves_it_undetermined(self):
+        found = self._candidates([{"name": "nobody.rom"}], ["corea", "mystery-sa"])
+        self.assertEqual(found[0]["verdict"], "undetermined")
+        self.assertEqual(found[0]["unresolved_cores"], ["mystery-sa"])
+
+    def test_a_romset_under_a_romset_core_is_undetermined(self):
+        found = self._candidates([{"name": "device.zip"}])
+        self.assertEqual(found[0]["verdict"], "undetermined")
+        self.assertIn("corea", found[0]["reason"])
+
+    def test_unresolved_names_skip_the_all_libretro_markers(self):
+        from common import unresolved_platform_cores
+
+        config = {"cores": ["retroarch", "all_libretro", "corea", "nope"]}
+        self.assertEqual(unresolved_platform_cores(config, self.PROFILES), {"nope"})
+        self.assertEqual(unresolved_platform_cores({"cores": "all_libretro"}, {}), set())
+
+    def test_the_page_gives_the_count_and_each_entry(self):
+        report = _report("retroarch", ["System.dat"], removal_candidates=[
+            {"system": "Enterprise", "name": "zt18hfnt.rom",
+             "destination": "zt18hfnt.rom", "verdict": "candidate",
+             "reason": "no core the platform runs opens it",
+             "cores_examined": [], "unresolved_cores": []},
+        ])
+        page = generate_exports_page([report], {})
+        self.assertIn("1 entry of the original is not opened", page)
+        self.assertIn("- `Enterprise` `zt18hfnt.rom`", page)
+        self.assertNotIn("no entry the platform declares is removed", page)
+
+    def test_the_report_carries_the_list(self):
+        source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8")
+        body = source[source.index("def export_platform("):source.index("def write_report(")]
+        self.assertIn('"removal_candidates": candidates,', body)
+
+
 if __name__ == "__main__":
     unittest.main()

wiki/tools.md

@@ -271,6 +271,17 @@ state is reported rather than written: EmuDeck's arrays are shared
 between emulators its file does not name, so a hash is corrected in place
 and never added.
 
+No entry the platform declares is removed. One the truth does not pair is
+looked for in the profiles of every core the platform runs: by name or
+alias, by the basename or tail of a `path:`, by any hash (archive members
+included), under a `type: directory` entry or a data directory. An entry
+none of them answers is written to `exports/<platform>.json` under
+`removal_candidates`, with the cores examined for its system and a reason.
+The verdict is `undetermined` when a core the platform names has no
+profile (`common.unresolved_platform_cores`) or when the entry is a ROM set
+and a core of the system opens sets its profile does not list. BizHawk and
+MiSTer resolve no core, so nothing is examined and nothing is listed.
+
 MAME and FBNeo entries are `.zip` ROM sets: their meaningful hashes belong
 to the files inside the archive, so the exported DAT lists those entries
 without a container sha1. Anyone submitting the DAT upstream should mention