From d5e61308e14121030f46c7e8bd06181cedef2a10 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:57:39 +0100 Subject: [PATCH] fix: additions join the native id their cores read A platform system spanning several native ids with no native_id filed every addition under the id that sorts first: RetroDECK's arcade wrote MAME device archives as FBNeo BIOS, RetroBat's Amiga put A500 kickstarts under amiga1200. An addition now joins the id whose declared files its cores read, and one the cores do not settle is counted and not written. --- scripts/export_native.py | 3 +++ scripts/exporter/baseline.py | 37 ++++++++++++++++++++++++++++++------ tests/test_exporters.py | 22 +++++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/scripts/export_native.py b/scripts/export_native.py index 3491ce675..6a1330a95 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -776,6 +776,8 @@ def export_platform( # A content-scanning core's names are labels: a list checked # by path cannot hold them. (report.labels, "entry", "entries", "found by content"), + (len(report.unattributed), "file", "", + "no single native system of the platform takes"), ) if number ] @@ -833,6 +835,7 @@ def export_platform( "requirements_corrected": requirements, "withheld": list(withheld), "found_by_content": report.labels, + "unattributed": report.unattributed, "removal_candidates": candidates, "doubled": doubled, "invalid": issues, diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index 5420c2efe..aa10ed78e 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -356,6 +356,9 @@ class Report: required_corrected: list[str] = field(default_factory=list) # Entries only content-scanning cores name: never written as files. labels: int = 0 + # Additions to a system spanning several native ids that their cores do + # not tell apart, as truth system/name: not written. + unattributed: list[str] = field(default_factory=list) @property def corrections(self) -> int: @@ -551,7 +554,9 @@ def build_native_model( for system in systems.values(): native_by_norm.setdefault(_norm_system_id(system.native_id), system.native_id) - def _target_native_ids(truth_sid: str) -> list[str]: + def _target_native_ids(truth_sid: str) -> tuple[list[str], bool]: + """The native ids a truth system spans, and whether the first one + is where an addition belongs without asking which core reads it.""" scraped_key = truth_sid if truth_sid in scraped_systems else None if scraped_key is None: scraped_key = norm_to_scraped.get(_norm_system_id(truth_sid)) @@ -562,7 +567,7 @@ def build_native_model( for fe in sys_data.get("files", []) } if not ids: - return [sys_data.get("native_id") or scraped_key] + return [sys_data.get("native_id") or scraped_key], True # A file the platform does not declare joins the system's primary # id, not whichever of its native ids sorts first: Recalbox files # pcengine, pcenginecd and supergrafx under one slug, and an @@ -572,9 +577,9 @@ def build_native_model( if primary in ids: ordered.remove(primary) ordered.insert(0, primary) - return ordered + return ordered, primary in ids or len(ordered) == 1 direct = native_by_norm.get(_norm_system_id(truth_sid)) - return [direct] if direct else [truth_sid] + return ([direct] if direct else [truth_sid]), True # Pass 2: apply the truth onto that grouping. for truth_sid in sorted(truth.get("systems", {})): @@ -583,7 +588,7 @@ def build_native_model( if not truth_files: continue - target_ids = _target_native_ids(truth_sid) + target_ids, settled_id = _target_native_ids(truth_sid) truth_files = [ entry for entry in truth_files if (name := str(entry.get("name", ""))) @@ -596,6 +601,13 @@ def build_native_model( if candidate.truth is None ] pairs = pair_entries(truth_files, [c.platform or {} for c in candidates]) + # The cores that read what the platform files under each native id, + # from the truth entries its declarations pair with. + readers: dict[str, set[str]] = {} + for t_index, index in pairs.items(): + readers.setdefault(candidates[index].native_system, set()).update( + truth_files[t_index].get("_cores") or () + ) for candidate in ( c for native_id in target_ids @@ -660,8 +672,21 @@ def build_native_model( if platform_declares(truth_entry, declared): continue - # The truth knows a file the platform does not declare. + # The truth knows a file the platform does not declare. Across + # several native ids and no primary, it joins the one whose files + # its cores read: RetroDECK's arcade spans mame and fbneo, and + # MAME device archives were filed as FBNeo BIOS because fbneo + # sorts first. With no single answer it is not written. native_id = target_ids[0] + if not settled_id: + wanted = set(truth_entry.get("_cores") or ()) + shared = {nid: len(readers.get(nid, set()) & wanted) for nid in target_ids} + best = max(shared.values()) + winners = [nid for nid, count in shared.items() if count == best] + if not best or len(winners) > 1: + report.unattributed.append(f"{truth_sid}/{name}") + continue + native_id = winners[0] system = systems.get(native_id) if system is None: system = NativeSystem(native_id=native_id, from_platform=False) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f84ee542f..cdce8452b 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -499,6 +499,28 @@ class Reconciliation(unittest.TestCase): systems, _ = build_native_model({"systems": {}}, scraped) self.assertEqual(sorted(systems), ["pcengine", "supergrafx"]) + def test_an_addition_joins_the_native_id_its_cores_read(self): + """RetroDECK's arcade spans mame and fbneo with no primary id. Every + addition went under fbneo, the first sorted, MAME device archives + included. One the cores do not settle is not written.""" + scraped = {"systems": {"arcade": {"files": [ + {"name": "neogeo.zip", "destination": "neogeo.zip", "native_system": "mame"}, + {"name": "pgm.zip", "destination": "pgm.zip", "native_system": "fbneo"}, + ]}}} + truth = {"systems": {"arcade": {"files": [ + {"name": "neogeo.zip", "_cores": ["mame"]}, + {"name": "pgm.zip", "_cores": ["fbneo"]}, + {"name": "a2autokbd.zip", "_cores": ["mame"]}, + {"name": "dkong.zip", "_cores": ["mame", "fbneo"]}, + ]}}} + systems, report = build_native_model(truth, scraped) + self.assertEqual( + [fe.name for fe in systems["mame"].files if fe.platform is None], + ["a2autokbd.zip"], + ) + self.assertFalse([fe for fe in systems["fbneo"].files if fe.platform is None]) + self.assertEqual(report.unattributed, ["arcade/dkong.zip"]) + def test_an_entry_the_platform_declares_without_a_hash_is_still_written(self): """Batocera and RetroBat both ship entries with an empty md5.""" scraped = {