diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index bc6fa4a3..053bb6b8 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -233,6 +233,21 @@ "bios_identical_to": { "type": "string" }, + "found_by": { + "description": "How the code finds the file: by the name it opens, or by content in a folder it scans, the name then being a label. Keyed by build when the libretro and standalone builds differ.", + "oneOf": [ + {"type": "string", "enum": ["name", "content"]}, + { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "libretro": {"type": "string", "enum": ["name", "content"]}, + "standalone": {"type": "string", "enum": ["name", "content"]} + } + } + ] + }, "bios_mode": { "type": "string" }, @@ -492,6 +507,21 @@ "has_builtin": { "type": "boolean" }, + "found_by": { + "description": "How the code finds the file: by the name it opens, or by content in a folder it scans, the name then being a label. Keyed by build when the libretro and standalone builds differ.", + "oneOf": [ + {"type": "string", "enum": ["name", "content"]}, + { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "libretro": {"type": "string", "enum": ["name", "content"]}, + "standalone": {"type": "string", "enum": ["name", "content"]} + } + } + ] + }, "load_from": { "description": "Where the emulator reads the file when not from the system directory, keyed by build when the libretro and standalone builds read it from different places.", "oneOf": [ diff --git a/scripts/export_native.py b/scripts/export_native.py index 105b20d2..5e0029ef 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -612,6 +612,10 @@ def export_platform( withheld = exporter.withheld() if withheld: summary += f", {_count(len(withheld), 'correction')} withheld" + if report.labels: + # A content-scanning core's names are labels: a list checked by path + # cannot hold them. + summary += f", {_count(report.labels, 'entry', 'entries')} found by content not listed" messages.append(summary) messages.extend(f"withheld: {reason}" for reason in withheld) for correction in applied[:5]: @@ -653,6 +657,7 @@ def export_platform( "hashes_cleared": cleared, "requirements_corrected": requirements, "withheld": list(withheld), + "found_by_content": report.labels, "invalid": issues, }) return True, messages diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index ad366382..3c7b1b79 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -61,6 +61,15 @@ def checked_hashes(entry: dict | None) -> dict | None: return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds} +def is_label(entry: dict) -> bool: + """A name only content-scanning cores give: no core opens that file. + + DuckStation identifies its BIOS by MD5 in a folder it scans, so + scph1001_v20.bin names nothing a platform could check by path. + """ + return bool(entry.get("_cores")) and "_named_by" in entry and not entry["_named_by"] + + def _is_placeholder(name: str) -> bool: return "<" in name or ">" in name or "*" in name @@ -282,6 +291,8 @@ class Report: files_added: int = 0 hashes_corrected: list[str] = field(default_factory=list) required_corrected: list[str] = field(default_factory=list) + # Entries only content-scanning cores name: never written as files. + labels: int = 0 @property def corrections(self) -> int: @@ -394,7 +405,8 @@ def pair_entries(truth_entries: list[dict], candidates: list[dict]) -> dict[int, claimed: set[int] = set() for test in _PAIRING_TESTS: for t_index, truth_entry in enumerate(truth_entries): - if t_index in pairs: + # A label answers for no name or path, only for its content. + if t_index in pairs or (test is not by_hash and is_label(truth_entry)): continue for index, candidate in enumerate(candidates): if index not in claimed and test(truth_entry, candidate): @@ -549,6 +561,10 @@ def build_native_model( ) continue + if is_label(truth_entry): + report.labels += 1 + continue + # A file another core's entry already claimed is still the # platform's file, not a new one: scph101.bin, declared by two # PSX cores, was written into RetroDECK's manifest a second time. diff --git a/scripts/truth.py b/scripts/truth.py index 01918d87..2bc1e7c7 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -16,7 +16,7 @@ from common import ( standalone_set as platform_standalone_set, ) from hashing import parse_md5_list -from validation import filter_files_by_mode, read_from_system_dir +from validation import filter_files_by_mode, opens_by_name, read_from_system_dir @@ -177,7 +177,15 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non def other_path(f: dict) -> str: return str(f.get("path") or "").casefold() - same_name = [f for f in files if f["name"].lower() == name_lower] + # A label and a name a core opens are one file only on a shared hash: + # pcsx_rearmed's hashless scph1001.bin took the md5 of DuckStation's + # label of that name, another dump. + labelled = not file_entry.get("_opens_by_name", True) + same_name = [ + f for f in files + if f["name"].lower() == name_lower + and (labelled == (not f.get("_named_by", True)) or _shares_hash(f, file_entry)) + ] return ( next((f for f in same_name if path and other_path(f) == path), None) # Another core's file of the same name is the same file unless their @@ -201,6 +209,13 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non ) +def _shares_hash(a: dict, b: dict) -> bool: + return any( + _declared(a, field) & _declared(b, field) + for field in ("sha1", "md5", "sha256", "crc32") + ) + + def _declared(entry: dict, field: str) -> set[str]: """The values an entry declares for a hash field, one or a list.""" value = entry.get(field) @@ -221,6 +236,8 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None: """Add one core's declaration to the entry that already stands for it.""" revision = emu_name in existing.get("_cores", ()) existing["_cores"] = existing.get("_cores", set()) | {emu_name} + if file_entry.get("_opens_by_name", True): + existing["_named_by"] = existing.get("_named_by", set()) | {emu_name} sr = file_entry.get("source_ref") if sr is not None: sr_key = _serialize_source_ref(sr) @@ -317,6 +334,7 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict: entry["crc32"] = crc.removeprefix("0x").lower() entry["_cores"] = {emu_name} entry["_required_by"] = {emu_name} if file_entry.get("required") else set() + entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set() sr = file_entry.get("source_ref") entry["_source_refs"] = {_serialize_source_ref(sr)} if sr is not None else set() if db: @@ -545,7 +563,7 @@ def generate_platform_truth( cores_profiled.add(emu_name) filtered = _archives_not_members([ - fe + {**fe, "_opens_by_name": opens_by_name(fe, profile, standalone)} for view, standalone in platform_layouts(emu_name, profile, standalone_set) for fe in _system_dir_files(view, standalone) ]) @@ -637,6 +655,7 @@ def generate_platform_truth( for fe in sys_data.get("files", []): fe["_cores"] = sorted(fe.get("_cores", set())) fe["_required_by"] = sorted(fe.get("_required_by", set())) + fe["_named_by"] = sorted(fe.get("_named_by", set())) fe["_source_refs"] = sorted(fe.get("_source_refs", set())) # Add per-system coverage cov = system_cores.get(sys_id, {}) @@ -717,7 +736,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict: correct while the export corrected hashes the diff never named. An entry never answers for two. """ - from exporter.baseline import _is_placeholder, pair_entries, platform_declares + from exporter.baseline import _is_placeholder, is_label, pair_entries, platform_declares truth_files = [ fe for fe in truth_sys.get("files", []) @@ -737,7 +756,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict: for t_index, t_entry in enumerate(truth_files): s_position = pairs.get(t_index) if s_position is None: - if not platform_declares(t_entry, scraped_files): + if not is_label(t_entry) and not platform_declares(t_entry, scraped_files): missing.append( { "name": t_entry["name"], diff --git a/scripts/validation.py b/scripts/validation.py index e3ba678c..e009c4fc 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -512,6 +512,23 @@ def declaration_settles( return False +def opens_by_name(file_entry: dict, profile: dict, standalone: bool = False) -> bool: + """Whether the code opens this entry by its name. + + A core that scans a folder and identifies firmware by content + (DuckStation by MD5, bios.cpp:208) gives its entries names that are + labels. `found_by` says which, on the file or the profile, keyed by + build when the two builds differ. + """ + for source in (file_entry, profile): + value = source.get("found_by") + if isinstance(value, dict): + value = value.get("standalone" if standalone else "libretro") + if value: + return value != "content" + return True + + def filter_files_by_mode(files: list[dict], standalone: bool) -> list[dict]: """Filter file entries by libretro/standalone mode.""" result = [] diff --git a/tests/test_exporters.py b/tests/test_exporters.py index a5c91b69..1171a053 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -330,6 +330,49 @@ class Reconciliation(unittest.TestCase): scraped_sys = {"files": [{"name": "scph1001.bin", "md5": "b" * 32}]} self.assertNotIn("hash_mismatch", _diff_system(truth_sys, scraped_sys)) + def test_a_label_is_never_added(self): + """A name only a content-scanning core gives is no file to hold.""" + truth = {"systems": {"sony-playstation": {"files": [ + {"name": "scph1001_v20.bin", "md5": "a" * 32, + "_cores": ["duckstation"], "_named_by": []}, + {"name": "scph5903.bin", "md5": "c" * 32, + "_cores": ["beetle_psx"], "_named_by": ["beetle_psx"]}, + ]}}} + scraped = {"systems": {"sony-playstation": {"native_id": "psx", "files": [ + {"name": "scph5501.bin", "destination": "scph5501.bin", + "native_system": "psx", "md5": "d" * 32}]}}} + systems, report = build_native_model(truth, scraped) + self.assertEqual([fe.name for fe in systems["psx"].files], + ["scph5501.bin", "scph5903.bin"]) + self.assertEqual(report.files_added, 1) + + def test_a_label_pairs_by_content_only(self): + """DuckStation's and rustation's scph5000.bin labels carried another + dump's md5 and, paired by name, replaced System.dat's scph5000.bin.""" + truth = {"systems": {"sony-playstation": {"files": [ + {"name": "scph5000.bin", "md5": "a" * 32, + "_cores": ["duckstation"], "_named_by": []}, + {"name": "other_label.bin", "md5": "e" * 32, "sha256": "f" * 64, + "_cores": ["duckstation"], "_named_by": []}, + ]}}} + scraped = {"systems": {"sony-playstation": {"native_id": "psx", "files": [ + {"name": "scph5000.bin", "destination": "scph5000.bin", + "native_system": "psx", "md5": "b" * 32}, + {"name": "scph7001.bin", "destination": "scph7001.bin", + "native_system": "psx", "md5": "e" * 32}, + ]}}} + systems, report = build_native_model(truth, scraped) + scph5000, scph7001 = systems["psx"].files + self.assertIsNone(scph5000.truth) + self.assertEqual(scph7001.truth["name"], "other_label.bin") + self.assertEqual(report.hashes_corrected, []) + + def test_the_diff_names_no_missing_label(self): + from truth import _diff_system + truth_sys = {"files": [{"name": "scph1001_v20.bin", "md5": "a" * 32, + "_cores": ["duckstation"], "_named_by": []}]} + self.assertNotIn("missing", _diff_system(truth_sys, {"files": []})) + def test_the_diff_pairs_by_the_same_rule(self): """diff_truth reads the same pairing, or it names a mismatch the export does not correct.""" diff --git a/tests/test_truth_layout.py b/tests/test_truth_layout.py index 1beaf92d..24625459 100644 --- a/tests/test_truth_layout.py +++ b/tests/test_truth_layout.py @@ -294,5 +294,50 @@ class ASizeOutsideARangeIsAnotherFile(unittest.TestCase): self.assertEqual(len(system["files"]), 2) +class TruthRecordsWhoOpensByName(unittest.TestCase): + """DuckStation finds its BIOS by MD5 in a folder it scans (bios.cpp:208, + 397-427): its names are labels. The export wrote them into Batocera's + list as files to hold, and a user with only scph5501.bin read 104 + missing BIOS where DuckStation boots.""" + + DUCK = {"type": "libretro", "systems": ["sony-playstation"], "found_by": "content", + "files": [{"name": "scph1001_v20.bin", "md5": "a" * 32}, + {"name": "scph1001.bin", "md5": "b" * 32}]} + PCSX = {"type": "libretro", "systems": ["sony-playstation"], + "files": [{"name": "scph1001.bin", "size": 524288}]} + + def truth(self, profiles: dict) -> dict: + config = {"cores": list(profiles), "systems": {"sony-playstation": {"files": []}}} + return generate_platform_truth("p", config, {}, profiles) + + def entries(self, truth: dict) -> list[dict]: + return truth["systems"]["sony-playstation"]["files"] + + def test_a_content_scanning_core_names_nothing(self): + truth = self.truth({"duckstation": self.DUCK}) + self.assertEqual({f["name"]: f["_named_by"] for f in self.entries(truth)}, + {"scph1001_v20.bin": [], "scph1001.bin": []}) + + def test_a_label_lends_no_hash_to_a_name_another_core_opens(self): + """pcsx_rearmed opens scph1001.bin by name, with no hash: it took the + md5 of DuckStation's label of that name.""" + truth = self.truth({"duckstation": self.DUCK, "pcsx": self.PCSX}) + named = [f for f in self.entries(truth) if f["_named_by"] == ["pcsx"]] + self.assertEqual(len(named), 1) + self.assertNotIn("md5", named[0]) + + def test_the_build_decides(self): + ymir = {"type": "standalone + libretro", "systems": ["sega-saturn"], + "found_by": {"standalone": "content"}, + "files": [{"name": "sega_101.bin", "md5": "c" * 32}]} + config = {"cores": ["ymir"], "standalone_cores": [], + "systems": {"sega-saturn": {"files": []}}} + libretro = generate_platform_truth("p", config, {}, {"ymir": ymir}) + self.assertEqual(libretro["systems"]["sega-saturn"]["files"][0]["_named_by"], ["ymir"]) + config["standalone_cores"] = ["ymir"] + standalone = generate_platform_truth("p", config, {}, {"ymir": ymir}) + self.assertEqual(standalone["systems"]["sega-saturn"]["files"][0]["_named_by"], []) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index b91fece1..b62f433a 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -404,6 +404,7 @@ partitions), and `game_count`. | `has_builtin` | true if the core falls back to a built-in copy when the file is absent | | `config_key` | configuration key the emulator exposes to point at the file | | `load_from` | directory the core reads the file from when it is not the system directory | +| `found_by` | `name` when the code opens the file by its name (the default), `content` when it scans a folder and identifies the file by hash, so the name is only a label (DuckStation by MD5, `bios.cpp:208`). Set on the profile or on one file, a dict splits `standalone` from `libretro`. A label is never written into a platform file as a name to hold, and pairs with a platform entry only by hash | | `source_ref` | source file and line number (e.g. `boot.cpp:42`), read at the profile's `source_commit` when set. A dict splits `standalone` from `libretro` when they diverge. In a profile that declares both `source` and `upstream`, each cited path opens with the one its lines are in: `source libretro.c:120` for the port's own code, `upstream emu/memory.c:46` for the emulator's, including the copy a port carries. `profile_sync --attribute-refs` writes these words from where the lines are found, and the site labels each citation with them | | `path` | destination path relative to system directory | | `standalone_path` | where the standalone build reads the file, under its own data folder, when it differs from `path` (the libretro destination). Packs for a platform running the emulator standalone use it |