diff --git a/scripts/export_native.py b/scripts/export_native.py index 3491ce675..6a1330a95 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -776,6 +776,8 @@ def export_platform( # A content-scanning core's names are labels: a list checked # by path cannot hold them. (report.labels, "entry", "entries", "found by content"), + (len(report.unattributed), "file", "", + "no single native system of the platform takes"), ) if number ] @@ -833,6 +835,7 @@ def export_platform( "requirements_corrected": requirements, "withheld": list(withheld), "found_by_content": report.labels, + "unattributed": report.unattributed, "removal_candidates": candidates, "doubled": doubled, "invalid": issues, diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index 5420c2efe..aa10ed78e 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -356,6 +356,9 @@ class Report: required_corrected: list[str] = field(default_factory=list) # Entries only content-scanning cores name: never written as files. labels: int = 0 + # Additions to a system spanning several native ids that their cores do + # not tell apart, as truth system/name: not written. + unattributed: list[str] = field(default_factory=list) @property def corrections(self) -> int: @@ -551,7 +554,9 @@ def build_native_model( for system in systems.values(): native_by_norm.setdefault(_norm_system_id(system.native_id), system.native_id) - def _target_native_ids(truth_sid: str) -> list[str]: + def _target_native_ids(truth_sid: str) -> tuple[list[str], bool]: + """The native ids a truth system spans, and whether the first one + is where an addition belongs without asking which core reads it.""" scraped_key = truth_sid if truth_sid in scraped_systems else None if scraped_key is None: scraped_key = norm_to_scraped.get(_norm_system_id(truth_sid)) @@ -562,7 +567,7 @@ def build_native_model( for fe in sys_data.get("files", []) } if not ids: - return [sys_data.get("native_id") or scraped_key] + return [sys_data.get("native_id") or scraped_key], True # A file the platform does not declare joins the system's primary # id, not whichever of its native ids sorts first: Recalbox files # pcengine, pcenginecd and supergrafx under one slug, and an @@ -572,9 +577,9 @@ def build_native_model( if primary in ids: ordered.remove(primary) ordered.insert(0, primary) - return ordered + return ordered, primary in ids or len(ordered) == 1 direct = native_by_norm.get(_norm_system_id(truth_sid)) - return [direct] if direct else [truth_sid] + return ([direct] if direct else [truth_sid]), True # Pass 2: apply the truth onto that grouping. for truth_sid in sorted(truth.get("systems", {})): @@ -583,7 +588,7 @@ def build_native_model( if not truth_files: continue - target_ids = _target_native_ids(truth_sid) + target_ids, settled_id = _target_native_ids(truth_sid) truth_files = [ entry for entry in truth_files if (name := str(entry.get("name", ""))) @@ -596,6 +601,13 @@ def build_native_model( if candidate.truth is None ] pairs = pair_entries(truth_files, [c.platform or {} for c in candidates]) + # The cores that read what the platform files under each native id, + # from the truth entries its declarations pair with. + readers: dict[str, set[str]] = {} + for t_index, index in pairs.items(): + readers.setdefault(candidates[index].native_system, set()).update( + truth_files[t_index].get("_cores") or () + ) for candidate in ( c for native_id in target_ids @@ -660,8 +672,21 @@ def build_native_model( if platform_declares(truth_entry, declared): continue - # The truth knows a file the platform does not declare. + # The truth knows a file the platform does not declare. Across + # several native ids and no primary, it joins the one whose files + # its cores read: RetroDECK's arcade spans mame and fbneo, and + # MAME device archives were filed as FBNeo BIOS because fbneo + # sorts first. With no single answer it is not written. native_id = target_ids[0] + if not settled_id: + wanted = set(truth_entry.get("_cores") or ()) + shared = {nid: len(readers.get(nid, set()) & wanted) for nid in target_ids} + best = max(shared.values()) + winners = [nid for nid, count in shared.items() if count == best] + if not best or len(winners) > 1: + report.unattributed.append(f"{truth_sid}/{name}") + continue + native_id = winners[0] system = systems.get(native_id) if system is None: system = NativeSystem(native_id=native_id, from_platform=False) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f84ee542f..cdce8452b 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -499,6 +499,28 @@ class Reconciliation(unittest.TestCase): systems, _ = build_native_model({"systems": {}}, scraped) self.assertEqual(sorted(systems), ["pcengine", "supergrafx"]) + def test_an_addition_joins_the_native_id_its_cores_read(self): + """RetroDECK's arcade spans mame and fbneo with no primary id. Every + addition went under fbneo, the first sorted, MAME device archives + included. One the cores do not settle is not written.""" + scraped = {"systems": {"arcade": {"files": [ + {"name": "neogeo.zip", "destination": "neogeo.zip", "native_system": "mame"}, + {"name": "pgm.zip", "destination": "pgm.zip", "native_system": "fbneo"}, + ]}}} + truth = {"systems": {"arcade": {"files": [ + {"name": "neogeo.zip", "_cores": ["mame"]}, + {"name": "pgm.zip", "_cores": ["fbneo"]}, + {"name": "a2autokbd.zip", "_cores": ["mame"]}, + {"name": "dkong.zip", "_cores": ["mame", "fbneo"]}, + ]}}} + systems, report = build_native_model(truth, scraped) + self.assertEqual( + [fe.name for fe in systems["mame"].files if fe.platform is None], + ["a2autokbd.zip"], + ) + self.assertFalse([fe for fe in systems["fbneo"].files if fe.platform is None]) + self.assertEqual(report.unattributed, ["arcade/dkong.zip"]) + def test_an_entry_the_platform_declares_without_a_hash_is_still_written(self): """Batocera and RetroBat both ship entries with an empty md5.""" scraped = {