From 5794ed417820ab77b5f30140b9aba32f33d26063 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:22:55 +0100 Subject: [PATCH] fix: drop the atari800 config key aliases ROM_5200_A, ROM_BASIC_A and six more are atari800.cfg keys, not names atari800 opens: MatchByName knows only atariosa.rom and its kin (sysrom.c:278-294), and the keys are config_key since 969907e29. The twelve labels the scan identifies by CRC32 are found_by: content, and a platform declaration of a file such an entry accepts by hash, in the folder the core scans, now meets it: RetroDECK's ROM_BASIC_A answers atari-basic-rev-a.rom without the alias. --- emulators/atari800.yml | 20 +++++---- scripts/common.py | 44 ++++++++++++++++++++ scripts/cross_reference.py | 13 +++++- scripts/validation.py | 13 ++++++ scripts/verify.py | 6 ++- tests/test_content_declarations.py | 66 ++++++++++++++++++++++++++++++ 6 files changed, 152 insertions(+), 10 deletions(-) create mode 100644 tests/test_content_declarations.py diff --git a/emulators/atari800.yml b/emulators/atari800.yml index f10dff6c7..cf6cc8771 100644 --- a/emulators/atari800.yml +++ b/emulators/atari800.yml @@ -53,8 +53,8 @@ files: source_ref: "source atari800/src/sysrom.c:103-104, source atari800/src/sysrom.c:290, source atari800/src/sysrom.c:138, source atari800/src/sysrom.c:616-623" - name: "atari-5200-os-2port.rom" + found_by: content path: "atari-5200-os-2port.rom" - aliases: [ROM_5200_A] required: false hle_fallback: true size: 2048 @@ -189,8 +189,8 @@ files: # -- Further revisions identified by CRC32 in the system directory scan -- - name: "atari-800-os-a-ntsc.rom" + found_by: content path: "atari-800-os-a-ntsc.rom" - aliases: [ROM_OS_A_NTSC] required: false hle_fallback: true size: 10240 @@ -206,8 +206,8 @@ files: source_ref: "source atari800/src/sysrom.c:89, source atari800/src/sysrom.c:124, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:124, source atari800/src/sysrom.c:616-623" - name: "atari-1200xl-os-rev-10.rom" + found_by: content path: "atari-1200xl-os-rev-10.rom" - aliases: [ROM_OS_AA00R10] required: false hle_fallback: true size: 16384 @@ -223,8 +223,8 @@ files: source_ref: "source atari800/src/sysrom.c:92, source atari800/src/sysrom.c:127, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:127, source atari800/src/sysrom.c:616-623" - name: "atari-1200xl-os-rev-11.rom" + found_by: content path: "atari-1200xl-os-rev-11.rom" - aliases: [ROM_OS_AA00R11] required: false hle_fallback: true size: 16384 @@ -240,8 +240,8 @@ files: source_ref: "source atari800/src/sysrom.c:93, source atari800/src/sysrom.c:128, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:128, source atari800/src/sysrom.c:616-623" - name: "atari-xl-os-ver-1.rom" + found_by: content path: "atari-xl-os-ver-1.rom" - aliases: [ROM_OS_BB00R1] required: false hle_fallback: true size: 16384 @@ -257,6 +257,7 @@ files: source_ref: "source atari800/src/sysrom.c:94, source atari800/src/sysrom.c:129, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:129, source atari800/src/sysrom.c:616-623" - name: "ROM_OS_BB02R3V4" + found_by: content path: "ROM_OS_BB02R3V4" required: false hle_fallback: true @@ -273,6 +274,7 @@ files: source_ref: "source atari800/src/sysrom.c:97, source atari800/src/sysrom.c:132, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:132, source atari800/src/sysrom.c:616-623" - name: "ROM_OS_CC01R4" + found_by: content path: "ROM_OS_CC01R4" required: false hle_fallback: true @@ -289,8 +291,8 @@ files: source_ref: "source atari800/src/sysrom.c:98, source atari800/src/sysrom.c:133, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:133, source atari800/src/sysrom.c:616-623" - name: "atari-xl-os-ver-3.rom" + found_by: content path: "atari-xl-os-ver-3.rom" - aliases: [ROM_OS_BB01R3] required: false hle_fallback: true size: 16384 @@ -306,6 +308,7 @@ files: source_ref: "source atari800/src/sysrom.c:99, source atari800/src/sysrom.c:134, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:134, source atari800/src/sysrom.c:616-623" - name: "ROM_OS_BB01R59" + found_by: content path: "ROM_OS_BB01R59" required: false hle_fallback: true @@ -322,6 +325,7 @@ files: source_ref: "source atari800/src/sysrom.c:101, source atari800/src/sysrom.c:136, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:136, source atari800/src/sysrom.c:616-623" - name: "ROM_OS_BB01R59A" + found_by: content path: "ROM_OS_BB01R59A" required: false hle_fallback: true @@ -338,8 +342,8 @@ files: source_ref: "source atari800/src/sysrom.c:102, source atari800/src/sysrom.c:137, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:137, source atari800/src/sysrom.c:616-623" - name: "atari-basic-rev-a.rom" + found_by: content path: "atari-basic-rev-a.rom" - aliases: [ROM_BASIC_A] required: false hle_fallback: true size: 8192 @@ -355,8 +359,8 @@ files: source_ref: "source atari800/src/sysrom.c:105, source atari800/src/sysrom.c:140, source atari800/src/sysrom.c:317-427, source atari800/src/atari.c:537, source atari800/src/sysrom.c:140, source atari800/src/sysrom.c:616-623" - name: "atari-basic-rev-b.rom" + found_by: content path: "atari-basic-rev-b.rom" - aliases: [ROM_BASIC_B] required: false hle_fallback: true size: 8192 diff --git a/scripts/common.py b/scripts/common.py index 3b9e25538..b62541245 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -3495,6 +3495,50 @@ def declared_root(config: dict) -> str: return root if count / len(roots) > 0.9 else "" +def platform_declared_contents(config: dict, db: dict) -> dict[str, set[str]]: + """The folders each declared file's content sits in, keyed by its hashes. + + A core that scans a folder and identifies a ROM by its checksum reads a + declared file whatever its name: RetroDECK lists atari800's Atari BASIC + Rev A as ROM_BASIC_A, and the core takes it by CRC32. Keys are + "md5:", "sha1:" and "crc32:" values of the database file the + declaration resolves to, each with the declared slot's folder. + """ + prefix = declared_root(config).lower() + by_md5 = db.get("indexes", {}).get("by_md5", {}) + files_db = db.get("files", {}) + contents: dict[str, set[str]] = {} + for system in config.get("systems", {}).values(): + for fe in system.get("files", []): + if fe.get("zipped_file"): + continue + record = files_db.get((fe.get("sha1") or "").lower()) + md5 = (fe.get("md5") or "").lower() + if record is None and md5 and "," not in md5: + sha1 = by_md5.get(md5, "") + record = files_db.get(sha1[0] if isinstance(sha1, list) else sha1) + if not record: + continue + slot = sanitize_pack_path(fe.get("destination") or fe.get("name", "")).lower() + if prefix and slot.startswith(prefix + "/"): + slot = slot[len(prefix) + 1:] + folder = slot.rpartition("/")[0] + for field in ("md5", "sha1", "crc32"): + value = str(record.get(field) or "").lower() + if value: + contents.setdefault(f"{field}:{value}", set()).add(folder) + return contents + + +def merged_platform_declared_contents(configs: list[dict], db: dict) -> dict[str, set[str]]: + """`platform_declared_contents` of several platforms, one index.""" + merged: dict[str, set[str]] = {} + for config in configs: + for key, folders in platform_declared_contents(config, db).items(): + merged.setdefault(key, set()).update(folders) + return merged + + def merged_platform_declarations( configs: list[dict], db: dict ) -> dict[str, list[tuple[int | None, str]]]: diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 7b9bf179f..bbae012ad 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -35,6 +35,7 @@ from common import ( get_mame_clone_map, list_emulator_profiles, list_registered_platforms, + merged_platform_declared_contents, load_data_dir_registry, load_database, load_emulator_profiles, @@ -867,7 +868,8 @@ def _cross_reference_profile( # A platform declares files of its BIOS tree: none answers for one # the core reads below the save directory. in_platform = not save_root and declaration_settles( - f, fname, platform_names, effective_path, index.get("folded_slots", frozenset()) + f, fname, platform_names, effective_path, index.get("folded_slots", frozenset()), + index.get("declared_contents"), ) if in_platform: @@ -1318,6 +1320,7 @@ def cross_reference( folded_slots: frozenset[str] = frozenset(), platform: str = "", launcher_layer: LauncherLayer | None = None, + declared_contents: dict[str, set[str]] | None = None, ) -> dict: """Compare emulator profiles against platform declarations. @@ -1355,6 +1358,7 @@ def cross_reference( "data_names": data_names, "all_declared": all_declared, "folded_slots": folded_slots, + "declared_contents": declared_contents, "bios_dir": db.get("bios_dir"), "launcher_candidates": [] if launcher_layer else None, } @@ -1561,6 +1565,13 @@ def main() -> None: all_declared=declared_names(args.platforms_dir, selected, db), standalone_cores=standalone_cores, folded_slots=folded_declared_slots(args.platforms_dir, selected, db), + declared_contents=merged_platform_declared_contents( + [ + load_platform_config(name, args.platforms_dir) + for name in selected or list_registered_platforms(args.platforms_dir) + ], + db, + ), platform=config.get("_key", args.platform) if args.platform else "", # What the platform's launchers open, as verify and the builder read it. launcher_layer=LauncherLayer( diff --git a/scripts/validation.py b/scripts/validation.py index f8f93819b..4e6daa4d8 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -606,9 +606,15 @@ def declaration_settles( declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str], dest: str = "", folded_slots: frozenset[str] = frozenset(), + declared_contents: Mapping[str, set[str]] | None = None, ) -> bool: """Whether a platform declaration meets a core's requirement. + An entry the core finds by content (``found_by: content`` on the entry) + is met by any declaration of a file it accepts by hash, in the folder it + scans: atari800 takes RetroDECK's ROM_BASIC_A by its CRC32, so its own + label for that ROM is not one more file to ship. + A platform declaring the name the core reads has met the requirement where the declaration fills the entry's own slot, whose content the slot arbitration decides, or elsewhere at a size the core accepts: the builder @@ -630,6 +636,13 @@ def declaration_settles( return True slot = sanitize_pack_path(dest or fname).lower() directory = slot if dest.endswith("/") else slot.rpartition("/")[0] + if declared_contents and file_entry.get("found_by") == "content": + for field in ("md5", "sha1", "crc32"): + values = file_entry.get(field) or [] + for value in values if isinstance(values, list) else str(values).split(","): + folders = declared_contents.get(f"{field}:{str(value).strip().lower()}") + if folders and directory in folders: + return True if not dest.endswith("/") and slot in folded_slots: return True for name in (fname, *file_entry.get("aliases", [])): diff --git a/scripts/verify.py b/scripts/verify.py index fa43dad47..737998d8f 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -60,6 +60,7 @@ from common import ( compute_hashes, expand_directory_entries, platform_declarations, + platform_declared_contents, group_identical_platforms, list_emulator_profiles, list_system_ids, @@ -454,6 +455,7 @@ def _candidate_verdict( declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str], dest: str = "", folded_slots: frozenset[str] = frozenset(), + declared_contents: Mapping[str, set[str]] | None = None, ) -> str: """Whether a profile entry can be a gap, and whether it is settled. @@ -469,7 +471,7 @@ def _candidate_verdict( if outside_gap_scope(file_entry, is_standalone): return "skip" if not include_all and declaration_settles( - file_entry, fname, declared_names, dest, folded_slots + file_entry, fname, declared_names, dest, folded_slots, declared_contents ): return "settled" return "keep" @@ -503,6 +505,7 @@ def find_undeclared_files( if declared_names is None: declared_names = platform_declarations(config, db) folded_slots = case_folded_slots(config, declared_names) + declared_contents = platform_declared_contents(config, db) # Whether the builder drops a file whose local copy contradicts its # declared hash, which decides if such a copy counts as held here. @@ -603,6 +606,7 @@ def find_undeclared_files( verdict = _candidate_verdict( f, fname, is_standalone, include_all, {} if save_root else declared_names, dest, folded_slots, + None if save_root else declared_contents, ) if verdict == "settled": seen_files.add(seen_key) diff --git a/tests/test_content_declarations.py b/tests/test_content_declarations.py new file mode 100644 index 000000000..21440435c --- /dev/null +++ b/tests/test_content_declarations.py @@ -0,0 +1,66 @@ +"""A platform declaration that a content-found core entry accepts by hash. + +atari800 identifies its ROMs by CRC32 in the folder it scans. RetroDECK +lists Atari BASIC Rev A as ROM_BASIC_A, a name atari800 never opens, and +the profile answered it with a false alias. Settled by content instead, the +declared file meets the entry and the label is not shipped a second time. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) + +from common import platform_declared_contents # noqa: E402 +from validation import declaration_settles # noqa: E402 + +SHA1 = "70cdf57469a208528f78a14275837352e90e20f1" +DB = { + "files": {SHA1: {"path": "bios/Atari/ROM_BASIC_A", "name": "ROM_BASIC_A", "sha1": SHA1, + "md5": "a4dc52536d526ecc51ea857b9fa2b90f", "crc32": "4bec4de2", + "size": 8192}}, + "indexes": {"by_md5": {"a4dc52536d526ecc51ea857b9fa2b90f": SHA1}}, +} +CONFIG = { + "base_destination": "", + "systems": {"atari800": {"files": [ + {"name": "ROM_BASIC_A", "destination": "bios/ROM_BASIC_A", + "md5": "a4dc52536d526ecc51ea857b9fa2b90f"}, + ]}}, +} +ENTRY = {"name": "atari-basic-rev-a.rom", "found_by": "content", "crc32": "4bec4de2"} + + +class ContentDeclarationTest(unittest.TestCase): + def setUp(self) -> None: + self.contents = platform_declared_contents(CONFIG, DB) + + def test_the_declared_file_is_indexed_by_each_hash(self) -> None: + self.assertEqual(self.contents["crc32:4bec4de2"], {""}) + self.assertEqual(self.contents[f"sha1:{SHA1}"], {""}) + + def test_a_content_found_entry_is_met_by_its_hash(self) -> None: + self.assertTrue(declaration_settles( + ENTRY, ENTRY["name"], {}, ENTRY["name"], declared_contents=self.contents, + )) + + def test_an_entry_opened_by_name_is_not(self) -> None: + entry = {key: value for key, value in ENTRY.items() if key != "found_by"} + self.assertFalse(declaration_settles( + entry, entry["name"], {}, entry["name"], declared_contents=self.contents, + )) + + def test_another_folder_is_not_the_one_scanned(self) -> None: + self.assertFalse(declaration_settles( + ENTRY, ENTRY["name"], {}, "atari800/" + ENTRY["name"], + declared_contents=self.contents, + )) + + +if __name__ == "__main__": + unittest.main()