From 0b4f98ce7975d99a26b1850934f0a03c82ff5562 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:41:43 +0200 Subject: [PATCH] feat: report entries no examined core opens The export keeps every entry a platform declares. An entry the truth does not pair was still read in 1189 of 2776 cases (alias, hash, a type: directory tree, a core outside the resolved set), so absence of a pairing proves nothing. Each report now lists, under removal_candidates, the unpaired entries that no profile of the cores the platform runs answers by name, path tail, hash, directory or data directory, with the cores examined and a reason. A core the platform names without a profile, or a ROM set under a core that opens sets by name, leaves the verdict undetermined. The exports page states the count and lists the entries per platform. --- scripts/common.py | 20 ++++ scripts/export_native.py | 15 +++ scripts/exporter/candidates.py | 209 +++++++++++++++++++++++++++++++++ scripts/siteexports.py | 52 +++++++- tests/test_export_report.py | 94 +++++++++++++++ wiki/tools.md | 11 ++ 6 files changed, 399 insertions(+), 2 deletions(-) create mode 100644 scripts/exporter/candidates.py diff --git a/scripts/common.py b/scripts/common.py index 8c21fb64..94bd8124 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -2294,6 +2294,26 @@ def resolve_platform_cores( return result +def unresolved_platform_cores(config: dict, profiles: dict[str, dict]) -> set[str]: + """The core names a platform lists that no profile answers to. + + resolve_platform_cores drops them without a word: ROCKNIX's `dolphin-sa` + or Batocera's `wine` designate no profile, so nothing is known of the + files they open. `retroarch` and `all_libretro` stand for every libretro + profile and are never unresolved. A platform without a `cores:` list + names no core. + """ + cores_config = config.get("cores") + if not isinstance(cores_config, list): + return set() + index = upstream_profile_index(profiles) + return { + name + for name in (str(c) for c in cores_config) + if name not in ("retroarch", "all_libretro") and not index.get(name) + } + + MANUFACTURER_PREFIXES = ( "acorn-", "apple-", diff --git a/scripts/export_native.py b/scripts/export_native.py index da19165d..efa42ef2 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -38,6 +38,7 @@ import yaml from common import ( platform_arg, list_registered_platforms, + load_data_dir_registry, load_database, load_emulator_profiles, load_platform_config, @@ -47,6 +48,7 @@ from common import ( ) from exporter import discover_exporters from exporter.baseline import HASH_FIELDS, _hash_values, _single_size, build_native_model +from exporter.candidates import removal_candidates from truth import generate_platform_truth DEFAULT_CACHE = ".cache/upstream-native" @@ -613,7 +615,19 @@ def export_platform( # A content-scanning core's names are labels: a list checked by path # cannot hold them. summary += f", {_count(report.labels, 'entry', 'entries')} found by content not listed" + # Reported, never removed: the platform's file keeps every entry. + candidates = removal_candidates( + systems, scraped, load_emulator_profiles("emulators"), + load_data_dir_registry(platforms_dir), + ) messages.append(summary) + if candidates: + open_question = sum(c["verdict"] == "undetermined" for c in candidates) + messages.append( + f"{_count(len(candidates), 'entry', 'entries')} no examined core is known to open" + + (f" ({open_question} undetermined)" if open_question else "") + + ", listed in the report" + ) messages.extend(f"withheld: {reason}" for reason in withheld) for correction in applied[:5]: messages.append(f"hash corrected: {correction}") @@ -655,6 +669,7 @@ def export_platform( "requirements_corrected": requirements, "withheld": list(withheld), "found_by_content": report.labels, + "removal_candidates": candidates, "invalid": issues, }) return True, messages diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py new file mode 100644 index 00000000..2a944553 --- /dev/null +++ b/scripts/exporter/candidates.py @@ -0,0 +1,209 @@ +"""Entries a platform declares that no core it runs is known to open. + +The export never removes a platform's entry. An entry the truth did not +pair may still be read: by a name or an alias, by its content, under a +directory a core walks, inside a data directory. Each of those is looked +for in the profiles of the cores the platform runs, and an entry none of +them answers is listed with what was examined. Absence of a match is not +proof that the code ignores the file, so the list is a report, and a core +the platform names that no profile describes leaves the verdict open. +""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass, field +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from common import _norm_system_id, resolve_platform_cores, unresolved_platform_cores +from exporter.baseline import HASH_FIELDS, NativeSystem, _hash_values + +# Batocera cuts some md5 to 29 characters: the prefix still names the dump. +_FULL_MD5 = 32 +_CUT_MD5 = 29 + + +def _tails(path: str) -> list[str]: + """Every trailing run of segments of a path, longest first.""" + parts = [p for p in path.casefold().strip("/").split("/") if p] + return ["/".join(parts[i:]) for i in range(len(parts))] + + +def _entry_hashes(entry: dict) -> set[str]: + values = {v for f in HASH_FIELDS for v in _hash_values(entry, f)} + for member in entry.get("contents") or []: + if isinstance(member, dict): + values |= {v for f in HASH_FIELDS for v in _hash_values(member, f)} + return values + + +@dataclass +class ReadEvidence: + """What the examined profiles open, indexed for one lookup per entry.""" + + names: set[str] = field(default_factory=set) + tails: set[str] = field(default_factory=set) + hashes: set[str] = field(default_factory=set) + directories: set[str] = field(default_factory=set) + trees: set[str] = field(default_factory=set) + tree_members: set[str] = field(default_factory=set) + + def add_profile(self, profile: dict, data_registry: dict) -> None: + for entry in profile.get("files") or []: + if not isinstance(entry, dict): + continue + locations = [ + str(entry[key]) for key in ("path", "standalone_path", "destination") + if entry.get(key) + ] + if entry.get("type") == "directory": + self.directories |= { + "/".join(p for p in loc.casefold().split("/") if p) + "/" + for loc in [*locations, str(entry.get("name", ""))] if loc + } + continue + self.names |= { + str(n).casefold() + for n in (entry.get("name"), *(entry.get("aliases") or [])) + if n + } + for location in locations: + tails = _tails(location) + self.tails.update(tails) + if tails: + self.names.add(tails[-1]) + self.hashes |= _entry_hashes(entry) + for tree in profile.get("data_directories") or []: + if not isinstance(tree, dict): + continue + destination = str(tree.get("destination") or "").casefold().strip("/") + if destination: + self.trees.add(destination + "/") + continue + # A tree laid at the root: only its own files say what it holds. + cache = (data_registry.get(str(tree.get("ref", ""))) or {}).get("local_cache") + if cache and Path(cache).is_dir(): + self.tree_members |= { + p.relative_to(cache).as_posix().casefold() + for p in Path(cache).rglob("*") if p.is_file() + } + + def proof(self, entry: dict, destination: str) -> str | None: + """How a profile reads the entry, or None when nothing answers.""" + names = { + str(n).casefold() + for n in (entry.get("name"), *(entry.get("aliases") or [])) + if n + } + tails = _tails(destination) + if any(t in self.tails for t in tails[:-1]): + return "path" + if names & self.names or (tails and tails[-1] in self.names): + return "name" + hashes = { + v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f) + } + if hashes & self.hashes or any( + len(v) < _FULL_MD5 and any(h.startswith(v) for h in self.hashes) + for v in hashes if len(v) >= _CUT_MD5 + ): + return "hash" + full = "/" + (tails[0] if tails else "") + if any("/" + d in full for d in self.directories): + return "directory" + if any("/" + t in full for t in self.trees): + return "data_directory" + if full[1:] in self.tree_members: + return "data_directory" + return None + + +def _loads_romsets(profile: dict) -> bool: + """A core that opens ROM sets by name: MAME pulls a device's set from + device_t::searchpath(), and no profile lists every device set.""" + return any( + isinstance(f, dict) and f.get("category") == "bios_zip" + for f in profile.get("files") or [] + ) + + +_ARCHIVES = (".zip", ".7z") + +REASONS = { + "candidate": "no core the platform runs opens it", + "unresolved": "a core the platform names has no profile yet", + "romset": "a ROM set, and these cores open sets their profiles do not list", +} + + +def removal_candidates( + systems: dict[str, NativeSystem], + scraped: dict | None, + profiles: dict[str, dict], + data_registry: dict | None = None, +) -> list[dict]: + """The platform's unpaired entries that no examined core is known to open. + + Every core the platform runs is searched. `cores_examined` names those + among them that emulate the entry's system, the ones a reader expects to + find it in. `verdict` is `candidate` when nothing examined could open the + entry, `undetermined` when something still could: a core the platform + names that no profile describes, or a core of the system that opens ROM + sets by name. A platform that resolves no core (BizHawk wires each + firmware in its own code, MiSTer's BiosDB is its authority) gives nothing + to examine and lists nothing. Nothing is removed from the platform's file. + """ + if not scraped: + return [] + examined = resolve_platform_cores(scraped, profiles) + if not examined: + return [] + unresolved = sorted(unresolved_platform_cores(scraped, profiles)) + evidence = ReadEvidence() + for key in sorted(examined): + evidence.add_profile(profiles[key], data_registry or {}) + + system_of = { + id(file_entry): sys_key + for sys_key, sys_data in (scraped.get("systems") or {}).items() + for file_entry in sys_data.get("files", []) + } + by_system: dict[str, list[str]] = {} + for key in sorted(examined): + for sid in profiles[key].get("systems") or []: + by_system.setdefault(_norm_system_id(str(sid)), []).append(key) + + romset_cores = {key for key in examined if _loads_romsets(profiles[key])} + + listed: list[dict] = [] + for system in systems.values(): + for entry in system.files: + if entry.platform is None or entry.truth is not None or not entry.name: + continue + if evidence.proof(entry.platform, entry.destination) is not None: + continue + slug = system_of.get(id(entry.platform), entry.native_system) + cores = by_system.get(_norm_system_id(slug), []) + readers = ( + [c for c in cores if c in romset_cores] + if entry.name.casefold().endswith(_ARCHIVES) else [] + ) + if unresolved: + verdict, reason = "undetermined", REASONS["unresolved"] + elif readers: + verdict = "undetermined" + reason = f"{REASONS['romset']}: {', '.join(readers)}" + else: + verdict, reason = "candidate", REASONS["candidate"] + listed.append({ + "system": entry.native_system, + "name": entry.name, + "destination": entry.destination, + "verdict": verdict, + "reason": reason, + "cores_examined": cores, + "unresolved_cores": unresolved, + }) + return listed diff --git a/scripts/siteexports.py b/scripts/siteexports.py index bdbbb36d..51906782 100644 --- a/scripts/siteexports.py +++ b/scripts/siteexports.py @@ -54,6 +54,49 @@ def change_text(report: dict) -> str: return ", ".join(parts) +# Beyond this many entries a reason's list folds away. +LONG_LIST = 10 + + +def candidate_lines(candidates: list[dict]) -> list[str]: + """The entries no emulator the platform runs is known to open. + + Each stays in the updated file. The count comes first, then the entries + grouped by reason, a long group in a block the reader opens. + """ + if not candidates: + return [] + open_question = sum(c.get("verdict") == "undetermined" for c in candidates) + count = _plural(len(candidates), "entry", "entries") + sentence = ( + f"{count} of the original {'is' if len(candidates) == 1 else 'are'} not " + "opened by any profiled emulator this platform runs. " + f"{'It stays' if len(candidates) == 1 else 'They stay'} in the updated file" + ) + if open_question: + sentence += ( + f", and {open_question} of them may still be read by an emulator " + "this project has not profiled in full" + ) + lines = [sentence + ".", ""] + groups: dict[str, list[dict]] = {} + for candidate in candidates: + groups.setdefault(candidate.get("reason", ""), []).append(candidate) + for reason, entries in groups.items(): + title = (reason[:1].upper() + reason[1:]).replace('"', "'") + items = [f"`{c['system']}` `{c['destination']}`" for c in entries] + if len(entries) > LONG_LIST: + lines.extend([ + f'??? note "{title} ({len(entries)})"', + "", + *[f" - {item}" for item in items], + "", + ]) + else: + lines.extend([f"{title}:", "", *[f"- {item}" for item in items], ""]) + return lines + + def load_export_reports(directory: str = EXPORTS_DIR) -> list[dict]: """Every platform report the export wrote, by platform name.""" root = Path(directory) @@ -95,8 +138,12 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str: "`es_bios.xml` and so on. This project reads the emulators' source " "code file by file, and offers each platform its own file with those " "findings applied: a hash is updated where the emulator source states " - "another, files the platform's emulators read are added, and no entry " - "the platform declares is removed. Where a value of an entry is " + "another, and files the platform's emulators read are added. An entry " + "that no emulator the platform runs is known to open stays in the " + "file and is listed as a removal candidate, with the reason, on the " + "platform's section. It would leave the file only once the code of " + "every emulator that platform runs for the system is shown not to " + "read it. Where a value of an entry is " "dropped, the platform's section lists it. Code the file carries is " "patched in place. With the original files on hand, the tests run " "the updated Batocera and ROCKNIX scripts (`--createReadme`), parse " @@ -159,6 +206,7 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str: "cannot be written in this format and should be checked by hand.", "", ]) + lines.extend(candidate_lines(report.get("removal_candidates") or [])) if report.get("invalid"): lines.extend([ '!!! warning "The updated file does not pass the platform\'s own checks"', diff --git a/tests/test_export_report.py b/tests/test_export_report.py index 6b36be5a..c976dbec 100644 --- a/tests/test_export_report.py +++ b/tests/test_export_report.py @@ -157,5 +157,99 @@ class ThePageAddressesMaintainersAsPeers(unittest.TestCase): self.assertNotIn(word, page) +class RemovalCandidatesNeedAReadingNoCoreAnswers(unittest.TestCase): + """An entry the truth does not pair can still be read: by an alias, by its + content, under a directory a core walks. Of 2776 such entries, 1189 + were read that way. Only what no core answers is listed, and nothing is + removed.""" + + PROFILES = { + "corea": { + "type": "libretro", + "systems": ["sys-one"], + "files": [ + {"name": "main.rom", "aliases": ["Alt.ROM"]}, + {"name": "other.bin", "md5": "0123456789abcdef0123456789abcdef"}, + {"name": "rtp/2000/", "path": "rtp/2000/", "type": "directory"}, + {"name": "set.zip", "category": "bios_zip"}, + ], + }, + } + + def _scraped(self, names: list[dict], cores: list[str]) -> dict: + return { + "_key": "fake", + "cores": cores, + "systems": {"sys-one": {"files": names}}, + } + + def _candidates(self, files: list[dict], cores: list[str] | None = None) -> list[dict]: + from exporter.baseline import build_native_model + from exporter.candidates import removal_candidates + + scraped = self._scraped(files, cores or ["corea"]) + systems, _ = build_native_model({"systems": {}}, scraped) + found = removal_candidates(systems, scraped, self.PROFILES) + kept = [f.name for s in systems.values() for f in s.files] + self.assertEqual(kept, [f["name"] for f in files]) + return found + + def test_an_alias_is_a_reading(self): + self.assertEqual(self._candidates([{"name": "alt.rom"}]), []) + + def test_a_hash_is_a_reading(self): + found = self._candidates([ + {"name": "renamed.bin", "md5": "0123456789ABCDEF0123456789ABCDEF"}, + ]) + self.assertEqual(found, []) + + def test_a_directory_entry_covers_its_members(self): + found = self._candidates([ + {"name": "x.png", "destination": "easyrpg/rtp/2000/x.png"}, + ]) + self.assertEqual(found, []) + + def test_an_entry_nothing_answers_is_a_candidate(self): + found = self._candidates([{"name": "nobody.rom"}]) + self.assertEqual(len(found), 1) + self.assertEqual(found[0]["verdict"], "candidate") + self.assertEqual(found[0]["cores_examined"], ["corea"]) + self.assertEqual(found[0]["unresolved_cores"], []) + + def test_an_unprofiled_core_leaves_it_undetermined(self): + found = self._candidates([{"name": "nobody.rom"}], ["corea", "mystery-sa"]) + self.assertEqual(found[0]["verdict"], "undetermined") + self.assertEqual(found[0]["unresolved_cores"], ["mystery-sa"]) + + def test_a_romset_under_a_romset_core_is_undetermined(self): + found = self._candidates([{"name": "device.zip"}]) + self.assertEqual(found[0]["verdict"], "undetermined") + self.assertIn("corea", found[0]["reason"]) + + def test_unresolved_names_skip_the_all_libretro_markers(self): + from common import unresolved_platform_cores + + config = {"cores": ["retroarch", "all_libretro", "corea", "nope"]} + self.assertEqual(unresolved_platform_cores(config, self.PROFILES), {"nope"}) + self.assertEqual(unresolved_platform_cores({"cores": "all_libretro"}, {}), set()) + + def test_the_page_gives_the_count_and_each_entry(self): + report = _report("retroarch", ["System.dat"], removal_candidates=[ + {"system": "Enterprise", "name": "zt18hfnt.rom", + "destination": "zt18hfnt.rom", "verdict": "candidate", + "reason": "no core the platform runs opens it", + "cores_examined": [], "unresolved_cores": []}, + ]) + page = generate_exports_page([report], {}) + self.assertIn("1 entry of the original is not opened", page) + self.assertIn("- `Enterprise` `zt18hfnt.rom`", page) + self.assertNotIn("no entry the platform declares is removed", page) + + def test_the_report_carries_the_list(self): + source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8") + body = source[source.index("def export_platform("):source.index("def write_report(")] + self.assertIn('"removal_candidates": candidates,', body) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/tools.md b/wiki/tools.md index 30a52629..312d0cd1 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -271,6 +271,17 @@ state is reported rather than written: EmuDeck's arrays are shared between emulators its file does not name, so a hash is corrected in place and never added. +No entry the platform declares is removed. One the truth does not pair is +looked for in the profiles of every core the platform runs: by name or +alias, by the basename or tail of a `path:`, by any hash (archive members +included), under a `type: directory` entry or a data directory. An entry +none of them answers is written to `exports/.json` under +`removal_candidates`, with the cores examined for its system and a reason. +The verdict is `undetermined` when a core the platform names has no +profile (`common.unresolved_platform_cores`) or when the entry is a ROM set +and a core of the system opens sets its profile does not list. BizHawk and +MiSTer resolve no core, so nothing is examined and nothing is listed. + MAME and FBNeo entries are `.zip` ROM sets: their meaningful hashes belong to the files inside the archive, so the exported DAT lists those entries without a container sha1. Anyone submitting the DAT upstream should mention