From b550b9780c8c0424655fda3eaedd93b61b476f80 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:12:25 +0100 Subject: [PATCH 01/15] fix: judge content-found cores by any dump A core with found_by: content reads no file by the destination's name. Any dump its profile accepts for the system settles the slot: Batocera, KNULLI and ROCKNIX ship scph1001.bin v2.0, which DuckStation and rustation accept under scph1001_v20.bin. --- scripts/slots.py | 32 +++++++++++++++++++++++++++++++- tests/test_slots.py | 19 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/scripts/slots.py b/scripts/slots.py index 8e7f826cd..f3908a631 100644 --- a/scripts/slots.py +++ b/scripts/slots.py @@ -43,7 +43,12 @@ from common import ( variant_base_name, ) from packresolve import preferred_entries -from validation import outside_gap_scope, read_from_save_dir, with_archive_members +from validation import ( + opens_by_name, + outside_gap_scope, + read_from_save_dir, + with_archive_members, +) if TYPE_CHECKING: from packextras import ExtraPlacement @@ -85,6 +90,9 @@ class Claim: entry: dict = field(default_factory=dict) local_path: str | None = None status: str = "" + # A core that identifies its firmware by content reads no file by this + # name: any dump it accepts for the system answers for it. + accepts: tuple[dict, ...] = () @property def is_proven(self) -> bool: @@ -210,6 +218,12 @@ def profile_claims( zip_contents, data_dir_registry, ) ) + labels = [ + f for f in profile.get("files") or [] + if isinstance(f, dict) + and not opens_by_name(f, profile, is_standalone) + and _lists_hashes(f) + ] for entry in profile.get("files") or []: if not isinstance(entry, dict): continue @@ -260,6 +274,13 @@ def profile_claims( ) if status == "name_exact" and _size_the_code_reads(entry, local): status = SIZE_EXACT + accepts: tuple[dict, ...] = () + if not opens_by_name(entry, profile, is_standalone): + systems = set(entry_system_ids(entry, profile)) + accepts = tuple( + f for f in labels + if not systems or systems & set(entry_system_ids(f, profile)) + ) claims.append( Claim( origin="profile", @@ -269,6 +290,7 @@ def profile_claims( entry=entry, local_path=local, status=status, + accepts=accepts, ) ) return claims @@ -746,6 +768,10 @@ def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool: They agree when they resolve to the same bytes, when the platform's own hash list accepts the profile's file, or when the profile's accepts the platform's. A contradiction needs both lists to reject the other side. + A core found by content (found_by: content) reads no file by the + destination's name, so any dump it accepts for the system settles the + slot: DuckStation takes Batocera's scph1001.bin v2.0, which its profile + labels scph1001_v20.bin. """ if profile.local_path == platform.local_path: return True @@ -764,6 +790,10 @@ def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool: and _accepted_by_all(profile.local_path, [platform.entry], db) ): return True + if platform.local_path and any( + _accepted_by_all(platform.local_path, [label], db) for label in profile.accepts + ): + return True return bool( platform.local_path and _lists_hashes(profile.entry) diff --git a/tests/test_slots.py b/tests/test_slots.py index 44afbe8e0..0acf3c653 100644 --- a/tests/test_slots.py +++ b/tests/test_slots.py @@ -111,6 +111,25 @@ class TestConflicts(unittest.TestCase): conflicts = slots.find_conflicts(config, self._profile("GC/JAP/IPL.bin"), REGIONS_DB) self.assertEqual([c.destination for c in conflicts], ["bios/GC/JAP/IPL.bin"]) + def test_a_core_found_by_content_takes_any_dump_it_accepts(self): + """DuckStation identifies its BIOS by content, so the name of its + scph1001.bin entry is a label. Batocera ships the v2.0 dump there, + which the profile accepts under scph1001_v20.bin: no conflict.""" + files = [ + {"name": "IPL.bin", "path": "GC/JAP/IPL.bin", "md5": "n" * 32}, + {"name": "IPL_usa.bin", "md5": "m" * 32}, + ] + by_name = slots.find_conflicts( + self._config("m" * 32), {"dolphin": {"files": files}}, REGIONS_DB + ) + self.assertEqual(len(by_name), 1) + by_content = slots.find_conflicts( + self._config("m" * 32), + {"dolphin": {"found_by": "content", "files": files}}, + REGIONS_DB, + ) + self.assertEqual(by_content, []) + def test_wrong_region_under_a_region_slot_is_reported(self): conflicts = slots.find_conflicts( self._config("m" * 32), self._profile("GC/JAP/IPL.bin"), REGIONS_DB From 807ffa0ca98d9808e19d220afa6208bcd132772f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:16:08 +0100 Subject: [PATCH 02/15] fix: let content labels accept their region slot verify judged a found_by: content entry by its label alone and called ROCKNIX's scph1001.bin (v2.0) a rustation sha256 mismatch, though rustation declares and boots that dump. A label now accepts the dumps its core takes for the same system and declared region. --- scripts/validation.py | 69 +++++++++++++++++++++++++++++++++- tests/test_validation_scope.py | 39 +++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/scripts/validation.py b/scripts/validation.py index 544451f90..5a4272dd5 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -16,6 +16,7 @@ from common import ( adler32_of, archive_members, compute_hashes, + entry_system_ids, name_match_size_ok, sanitize_pack_path, size_fits, @@ -145,6 +146,72 @@ def _rule_failure( return None +def _found_by_content(entry: dict, profile: dict) -> bool: + """Whether every build of the core identifies this entry by content.""" + return not opens_by_name(entry, profile) and not opens_by_name( + entry, profile, standalone=True + ) + + +def _regions(entry: dict) -> set[str]: + value = entry.get("region") + return set(value) if isinstance(value, list) else {value} if value else set() + + +def content_alternatives(entry: dict, profile: dict) -> list[dict]: + """The labels a content-found core takes in place of this one. + + rustation scans its folder by sha256 (found_by: content): its + scph1001.bin is a label for v2.2, and the v2.0 dump ROCKNIX ships under + that name boots the same North American console, labelled + scph1001_v20.bin. Alternatives share the entry's system and declared + region, the slot slot.py groups by: openMSX also finds its ROMs by + sha1, but each one serves its own machine and declares no region. + """ + if not _found_by_content(entry, profile): + return [] + regions = _regions(entry) + if not regions: + return [] + systems = set(entry_system_ids(entry, profile)) + return [ + other for other in profile.get("files") or [] + if isinstance(other, dict) + and _found_by_content(other, profile) + and regions & _regions(other) + and systems == set(entry_system_ids(other, profile)) + ] + + +def _labels_widened(profile: dict) -> list[dict]: + """The profile's entries, each content-found label accepting the dumps + of its alternatives (content_alternatives).""" + widened = [] + for f in profile.get("files") or []: + if not isinstance(f, dict): + continue + siblings = content_alternatives(f, profile) + if len(siblings) < 2: + widened.append(f) + continue + merged = dict(f) + for field_name in ("crc32", "md5", "sha1", "sha256"): + if f.get(field_name): + merged[field_name] = list(dict.fromkeys( + value for other in siblings + for value in parse_hash_list(other.get(field_name)) + )) + if f.get("size") is not None: + merged["size"] = sorted({ + size for other in siblings + for size in (other["size"] if isinstance(other.get("size"), list) + else [other.get("size")]) + if size is not None + }) + widened.append(merged) + return widened + + def _build_validation_index(profiles: dict) -> dict[str, dict]: """Build per-filename validation rules from emulator profiles. @@ -170,7 +237,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: for emu_name, profile in profiles.items(): if profile.get("type") in ("launcher", "alias"): continue - for f in profile.get("files", []): + for f in _labels_widened(profile): fname = f.get("name", "") if not fname: continue diff --git a/tests/test_validation_scope.py b/tests/test_validation_scope.py index 87399f979..30c8946b4 100644 --- a/tests/test_validation_scope.py +++ b/tests/test_validation_scope.py @@ -47,5 +47,44 @@ class HleIndexScope(unittest.TestCase): self.assertIn("for profile in platform_profiles.values()", block) self.assertNotIn("profiles.values()", block.replace("platform_profiles.values()", "")) + +class ContentFoundLabelsTakeTheirRegionSlot(unittest.TestCase): + """rustation scans its folder by sha256, so its scph1001.bin entry is a + label. ROCKNIX ships the v2.0 dump under that name, which rustation + boots (labelled scph1001_v20.bin), and verify called it a mismatch.""" + + def test_an_alternative_dump_passes_under_the_label(self): + import hashlib + import sys + import tempfile + + sys.path.insert(0, str(REPO_ROOT / "scripts")) + from validation import _build_validation_index, check_file_validation + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "scph1001.bin" + path.write_bytes(b"v2.0") + v20 = hashlib.sha256(b"v2.0").hexdigest() + profile = { + "found_by": "content", + "systems": ["sony-playstation"], + "files": [ + {"name": "scph1001.bin", "region": ["north-america"], + "validation": ["sha256"], "sha256": "a" * 64}, + {"name": "scph1001_v20.bin", "region": ["north-america"], + "validation": ["sha256"], "sha256": v20}, + {"name": "scph5500.bin", "region": ["japan"], + "validation": ["sha256"], "sha256": "b" * 64}, + ], + } + index = _build_validation_index({"rustation": profile}) + self.assertIsNone(check_file_validation(str(path), "scph1001.bin", index)) + # Another region is another slot. + self.assertIsNotNone(check_file_validation(str(path), "scph5500.bin", index)) + # A core that opens the file by name rejects it. + index = _build_validation_index({"rustation": {**profile, "found_by": "name"}}) + self.assertIsNotNone(check_file_validation(str(path), "scph1001.bin", index)) + + if __name__ == "__main__": unittest.main() From 6695efeebeab687e41cc9f7d2ad92ce3377e230a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:16:08 +0100 Subject: [PATCH 03/15] fix: scope slot labels to system and region slots reads content_alternatives too, so a content-found label is settled only by a dump of its own system and region slot. --- scripts/slots.py | 25 +++++++------------------ tests/test_slots.py | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/scripts/slots.py b/scripts/slots.py index f3908a631..cb69660c1 100644 --- a/scripts/slots.py +++ b/scripts/slots.py @@ -44,7 +44,7 @@ from common import ( ) from packresolve import preferred_entries from validation import ( - opens_by_name, + content_alternatives, outside_gap_scope, read_from_save_dir, with_archive_members, @@ -91,7 +91,7 @@ class Claim: local_path: str | None = None status: str = "" # A core that identifies its firmware by content reads no file by this - # name: any dump it accepts for the system answers for it. + # name: any dump it takes for the same system and region answers for it. accepts: tuple[dict, ...] = () @property @@ -218,12 +218,6 @@ def profile_claims( zip_contents, data_dir_registry, ) ) - labels = [ - f for f in profile.get("files") or [] - if isinstance(f, dict) - and not opens_by_name(f, profile, is_standalone) - and _lists_hashes(f) - ] for entry in profile.get("files") or []: if not isinstance(entry, dict): continue @@ -274,13 +268,7 @@ def profile_claims( ) if status == "name_exact" and _size_the_code_reads(entry, local): status = SIZE_EXACT - accepts: tuple[dict, ...] = () - if not opens_by_name(entry, profile, is_standalone): - systems = set(entry_system_ids(entry, profile)) - accepts = tuple( - f for f in labels - if not systems or systems & set(entry_system_ids(f, profile)) - ) + accepts = tuple(content_alternatives(entry, profile)) claims.append( Claim( origin="profile", @@ -769,8 +757,8 @@ def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool: hash list accepts the profile's file, or when the profile's accepts the platform's. A contradiction needs both lists to reject the other side. A core found by content (found_by: content) reads no file by the - destination's name, so any dump it accepts for the system settles the - slot: DuckStation takes Batocera's scph1001.bin v2.0, which its profile + destination's name, so any dump it takes for the same system and region + settles the slot: DuckStation takes Batocera's scph1001.bin v2.0, which its profile labels scph1001_v20.bin. """ if profile.local_path == platform.local_path: @@ -791,7 +779,8 @@ def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool: ): return True if platform.local_path and any( - _accepted_by_all(platform.local_path, [label], db) for label in profile.accepts + _lists_hashes(label) and _accepted_by_all(platform.local_path, [label], db) + for label in profile.accepts ): return True return bool( diff --git a/tests/test_slots.py b/tests/test_slots.py index 0acf3c653..f5c7eafbc 100644 --- a/tests/test_slots.py +++ b/tests/test_slots.py @@ -116,8 +116,9 @@ class TestConflicts(unittest.TestCase): scph1001.bin entry is a label. Batocera ships the v2.0 dump there, which the profile accepts under scph1001_v20.bin: no conflict.""" files = [ - {"name": "IPL.bin", "path": "GC/JAP/IPL.bin", "md5": "n" * 32}, - {"name": "IPL_usa.bin", "md5": "m" * 32}, + {"name": "IPL.bin", "path": "GC/JAP/IPL.bin", "md5": "n" * 32, + "region": ["japan"]}, + {"name": "IPL_alt.bin", "md5": "m" * 32, "region": ["japan"]}, ] by_name = slots.find_conflicts( self._config("m" * 32), {"dolphin": {"files": files}}, REGIONS_DB @@ -129,6 +130,15 @@ class TestConflicts(unittest.TestCase): REGIONS_DB, ) self.assertEqual(by_content, []) + # Another region is another slot: openMSX finds each ROM by sha1 too, + # and one machine's ROM never stands for another's. + files[1]["region"] = ["north-america"] + other_slot = slots.find_conflicts( + self._config("m" * 32), + {"dolphin": {"found_by": "content", "files": files}}, + REGIONS_DB, + ) + self.assertEqual(len(other_slot), 1) def test_wrong_region_under_a_region_slot_is_reported(self): conflicts = slots.find_conflicts( From 7a03578dd96efa8a55fa3ee6fbb9931fa48e6b4a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:21:47 +0100 Subject: [PATCH 04/15] fix: count name holders with case folded A hashless profile entry adopted the only exact holder of its name as the file its aliases designate. dosbox's SC-55 ROM2.BIN found Galaksija's ROM2.BIN, and ROM2_ST.BIN was indexed against a 4 KiB BASIC ROM while SC-55's rom2.bin sat under the same folded name. The name now has to designate one file once case is folded, a primary and its own variants counted as one. --- database.json | 5 +---- scripts/generate_db.py | 27 +++++++++++++++++++++++---- tests/test_db_aliases.py | 19 +++++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/database.json b/database.json index bc7d27139..375671581 100644 --- a/database.json +++ b/database.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "generated_at": "2026-11-05T23:59:37Z", + "generated_at": "2026-11-06T07:21:27Z", "total_files": 19947, "total_size": 16926035621, "bios_dir": "bios", @@ -281813,9 +281813,6 @@ "TRIO64 (Ver. 1.5-07) [VGA] (S3 Incorporated).bin": [ "a308b15f93da34961c0141ec95b98c211557e2d5" ], - "ROM2_ST.BIN": [ - "5d5ab4313a2d0effe7572bb129193b64cab002c1" - ], "mt32_ctrl_1_04.rom": [ "5a5cb5a77d7d55ee69657c2f870416daed52dea7" ], diff --git a/scripts/generate_db.py b/scripts/generate_db.py index ade383713..32c956171 100644 --- a/scripts/generate_db.py +++ b/scripts/generate_db.py @@ -576,12 +576,26 @@ def _collect_all_aliases( # naming its content once 64DD_IPL.bin sorted first, and its aliases # 64dd_ipl_J.bin and 64DD_IPL_JP.n64 went with it. holders: dict[str, set[str]] = {} + # The files a name designates once case is folded, as the resolver's + # name step folds it, a primary and its own .variants/ counted as one + # file. dosbox's SC-55 ROM2.BIN found Galaksija's ROM2.BIN the only + # exact holder, and its alias ROM2_ST.BIN went to a 4 KiB BASIC ROM, + # while SC-55's rom2.bin sat under the same folded name. + families: dict[str, set[str]] = {} + + def _hold(name: str, path: str, sha1: str) -> None: + holders.setdefault(name, set()).add(sha1) + directory = path.rpartition("/")[0].removesuffix("/.variants") + families.setdefault(name.casefold(), set()).add( + f"{directory}/{variant_base_name(name)}".casefold() + ) + for sha1, entry in files.items(): - holders.setdefault(entry["name"], set()).add(sha1) + _hold(entry["name"], entry["path"], sha1) for sha1, copy_list in (copies or {}).items(): if sha1 in files: for copy in copy_list: - holders.setdefault(copy["name"], set()).add(sha1) + _hold(copy["name"], copy["path"], sha1) aliases = {} @@ -691,11 +705,16 @@ def _collect_all_aliases( ] if taken: shadowed.append((entry_name, set(matched), taken)) - if not matched and len(holders.get(entry_name, ())) == 1: + held = holders.get(entry_name, set()) + if ( + not matched + and len(held) == 1 + and len(families[entry_name.casefold()]) == 1 + ): # A name carried by several contents names none of them: # quasi88's disk.rom aliases went to whichever # disk.rom the scan met last, a Tandy CoCo ROM. - matched = set(holders[entry_name]) + matched = set(held) for sha in sorted(matched): for alias_name in entry_aliases: _add_alias(alias_name, sha) diff --git a/tests/test_db_aliases.py b/tests/test_db_aliases.py index 91bfd56fa..eff324b5f 100644 --- a/tests/test_db_aliases.py +++ b/tests/test_db_aliases.py @@ -56,6 +56,25 @@ class AliasesNeedAnUnambiguousName(unittest.TestCase): named = {a["name"] for entries in aliases.values() for a in entries} self.assertNotIn("n88sub.rom", named) + def test_homonyms_differing_in_case_take_no_alias(self): + """dosbox's SC-55 ROM2.BIN: Galaksija's ROM2.BIN was the only exact + holder, and ROM2_ST.BIN went to it beside SC-55's rom2.bin.""" + files = { + "a": self._file("DISK.ROM", "bios/NEC/PC-98/DISK.ROM", "m1"), + "b": self._file("disk.rom", "bios/Tandy/CoCo/disk.rom", "m2"), + } + aliases = self._aliases(files) + named = {a["name"] for entries in aliases.values() for a in entries} + self.assertNotIn("n88sub.rom", named) + + def test_a_primary_and_its_variant_are_one_file(self): + files = { + "a": self._file("disk.rom", "bios/NEC/PC-88/disk.rom", "m1"), + "b": self._file("DISK.ROM", "bios/NEC/PC-88/.variants/DISK.ROM.0123abcd", "m2"), + } + aliases = self._aliases(files) + self.assertEqual([a["name"] for a in aliases.get("a", [])], ["n88sub.rom"]) + def test_a_unique_name_still_carries_its_aliases(self): files = {"a": self._file("disk.rom", "bios/NEC/PC-88/disk.rom", "m1")} aliases = self._aliases(files) From 8dc3ac82c71e2c366921e1d16ff61e63d00791b6 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:24:49 +0100 Subject: [PATCH 05/15] fix: date each prose citation on its own realign_prose dated a whole field from the last commit that changed any of its citations. idtech4a++ corrected two notes citations by hand in the commit that advanced its pin, so the untouched Game.java:69-72 and GameResourceUrl.java:14-118 read as written under the current pin and were never offered again. Each citation is now followed through the history on its own. The two idtech4a++ ranges move to 72-75 and 15-121, read at ce49510f. --- emulators/idtech4a++.yml | 4 +- scripts/profile_sync.py | 92 ++++++++++++++------------------------ tests/test_profile_sync.py | 18 ++++++++ 3 files changed, 54 insertions(+), 60 deletions(-) diff --git a/emulators/idtech4a++.yml b/emulators/idtech4a++.yml index 6e329f943..3f3096a71 100644 --- a/emulators/idtech4a++.yml +++ b/emulators/idtech4a++.yml @@ -110,7 +110,7 @@ notes: | Q3E/src/main/jni/quake2/yquake2/src/common/filesystem.c:1801-1850, Q3E/src/main/jni/zdoom/src/d_iwad.cpp:738, Q3E/src/main/jni/ecwolf/src/wl_iwad.cpp:766, - idTech4Amm/src/main/java/com/karin/idTech4Amm/sys/GameResourceUrl.java:14-118 + idTech4Amm/src/main/java/com/karin/idTech4Amm/sys/GameResourceUrl.java:15-121 The Quake III lineage keeps id's release checksums for `baseq3` pak0 through pak8 and `missionpack` pak0 through pak3, and an archive whose value disagrees @@ -130,7 +130,7 @@ notes: | two games whose data is a file rather than a folder. ref: Q3E/src/main/jni/zdoom/src/d_iwad.cpp:287-297, 321-342, Q3E/src/main/jni/zdoom/src/version.h:100-102, - idTech4Amm/src/main/java/com/karin/idTech4Amm/sys/Game.java:69-72, + idTech4Amm/src/main/java/com/karin/idTech4Amm/sys/Game.java:72-75, Q3E/src/main/java/com/n0n3m4/q3e/Q3EInterface.java:710-713 Starting a game extracts that game's bundled resources first. A text file named diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 795fbd62d..76bfdb95a 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -3742,15 +3742,6 @@ def _git_file_at(path: Path, sha: str) -> dict | None: return document if isinstance(document, dict) else None -def _scalar_values(document: dict) -> dict[str, str]: - """String scalars of the document, keyed by field path.""" - return { - field: value - for kind, field, _, _, value in _walk_document(document) - if kind == "str" - } - - def _pin_of( document: dict, field: str, mode: str = "", repo: upstream.Repo | None = None, @@ -3788,26 +3779,6 @@ def _pin_of( return value -_CITATION_RUN_RE = re.compile( - PROSE_CITE_RE.pattern + r"(?:, *\d+(?:-\d+)?)*" -) - - -def _scalar_citations(text: str | None) -> tuple[str, ...] | None: - """The citations a scalar carries, in order, without the prose around them. - - A note reworded around unchanged citations was still written under the - pin those citations describe: comparing whole texts moved the writing - revision to the rewording commit, whose pin is current, and the stale - citations were never offered for realignment again. - """ - if text is None: - return None - return tuple( - _citation_key(m.group(0)) for m in _CITATION_RUN_RE.finditer(text) - ) - - def _writing_pins( revisions: list[tuple[str, dict]], intro_sha: str, field: str, mode: str = "", repo: upstream.Repo | None = None, @@ -4066,68 +4037,73 @@ def realign_prose( if not history: return [] - current_values = _scalar_values(document) - fields = {citation.field for citation in citations} - alive = dict.fromkeys(fields, True) - intro: dict[str, str | None] = dict.fromkeys(fields) + # Each citation is followed on its own: a field's text changes whenever + # one of its citations is corrected by hand, and dating the whole field + # from that commit put idtech4a++'s untouched Game.java:69-72 under the + # current pin, so the stale range was never offered again. + keys = [(c.field, _citation_key(c.ref)) for c in citations] + alive = [True] * len(citations) + intro: list[str | None] = [None] * len(citations) revisions: list[tuple[str, dict]] = [] - # Fields whose writing pin lies past a revision that cannot be read: + # Citations whose writing pin lies past a revision that cannot be read: # where their text came in is unknown, so they are not moved. - unknown: dict[str, str] = {} + unknown: dict[int, str] = {} # A revision that does not parse is passed over unless the pins differ - # on its two sides while the field's citations changed across it: then - # the text may have come in there under either pin. Otherwise nothing + # on its two sides while the citation is absent across it: then the + # text may have come in there under either pin. Otherwise nothing # it could hold changes the reading. lindbergh-loader # committed invalid YAML once (df7346b3), fixed the same day, and every # note older than that commit stayed blocked for good. gap: str | None = None + + def carried_by(past: dict) -> set[tuple[str, str]]: + return { + (c.field, _citation_key(c.ref)) + for c in collect_citations(past) if c.kind == "prose" + } + for sha in history: - if not any(alive.values()): + if not any(alive): break past = _git_file_at(path, sha) if past is None: gap = gap or sha continue + held = carried_by(past) if gap: newer = revisions[-1][1] if revisions else document moved = any( newer.get(pin) != past.get(pin) for pin in ("source_commit", "upstream_commit") ) - values = _scalar_values(past) - for field, live in alive.items(): - if live and moved and _scalar_citations(values.get(field)) != ( - _scalar_citations(current_values.get(field)) - ): - unknown[field] = gap - alive[field] = False + for index, live in enumerate(alive): + if live and moved and keys[index] not in held: + unknown[index] = gap + alive[index] = False gap = None revisions.append((sha, past)) - values = _scalar_values(past) - for field in fields: - if not alive[field]: + for index, live in enumerate(alive): + if not live: continue - if _scalar_citations(values.get(field)) == _scalar_citations( - current_values.get(field) - ): - intro[field] = sha + if keys[index] in held: + intro[index] = sha else: - alive[field] = False + alive[index] = False if gap: # The oldest revisions do not parse: where the text came in is past # what can be read. - unknown.update({field: gap for field, live in alive.items() if live}) + unknown.update({index: gap for index, live in enumerate(alive) if live}) messages: list[str] = [] jobs: list[tuple[Citation, dict]] = [] - for citation in citations: - if citation.field in unknown: + for index, citation in enumerate(citations): + if index in unknown: messages.append( f"read again: {citation.field}: {citation.ref} " - f"(revision {unknown[citation.field][:8]} unreadable)" + f"(revision {unknown[index][:8]} unreadable)" ) continue - intro_sha = intro.get(citation.field) + intro_sha = intro[index] if intro_sha is None: # The scalar is not committed yet: written now, under this pin. continue diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index d9574bfc9..2521591b2 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -4766,6 +4766,24 @@ class TestRealignProse(unittest.TestCase): any("pin moved under this text" in m for m in messages), messages ) + def test_a_citation_corrected_beside_another_dates_only_itself(self): + """idtech4a++ advanced its pin in the commit that corrected two + citations of its notes by hand. Dated per field, the untouched + Game.java:69-72 took that commit's current pin and was never + offered again, though its lines had moved.""" + self._repo_with_advanced_pin() + self._commit( + PROSE_SAMPLE.replace("(b.c:2,8-9)", "(b.c:3,9-10)"), + "advance the pin, b.c corrected by hand", + ) + self.files[("oldpin", "a.c")] = ["x"] * 9 + ["subject"] + self.files[("pin", "a.c")] = ["x"] * 13 + ["subject"] + self.files[("pin", "b.c")] = list("pqrstuvwxyz") + messages = profile_sync.realign_prose(self.path, self.tmp.name, dry_run=True) + flagged = [m for m in messages if "a.c:10" in m] + self.assertTrue(flagged, messages) + self.assertFalse([m for m in messages if "b.c:" in m], messages) + def test_a_citation_in_an_unmoved_upstream_needs_no_reading(self): """nethersx2 cites AetherSX2 files that only its upstream carries, at an upstream pin that never moved. The port's pin did move under From a0ecca002448268fdd9a3810489345b953cffaa3 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:33:16 +0100 Subject: [PATCH 06/15] fix: collect abbreviated prose citations A location written without its file, (:507), reads in the file named last in the scalar, either cited or named as 'path:' before a list. A dotless file cited with its directory (tools/errorgen:20) is a citation too. devilutionx, corsixth, starship, plastic, eden-nightly, theforceengine, lightspark and higan_sfc gain 33 citations that bump and realign now follow. Two read that way pointed at another file and now name theirs: devilutionx assets.cpp:511-518, corsixth app.lua:1591. --- emulators/corsixth.yml | 2 +- emulators/devilutionx.yml | 2 +- scripts/sourceref.py | 126 ++++++++++++++++++++++++++----------- tests/test_profile_sync.py | 33 ++++++++++ 4 files changed, 123 insertions(+), 40 deletions(-) diff --git a/emulators/corsixth.yml b/emulators/corsixth.yml index 019755a9d..dc366dd4f 100644 --- a/emulators/corsixth.yml +++ b/emulators/corsixth.yml @@ -54,7 +54,7 @@ notes: | upper-case names the DOS release ships and the lookup normalises to. The sound/music path uses the game's XMI tracks when a SoundFont is - available (App:findSoundFont at :1591 probes GeneralUser-GS.sf2, + available (App:findSoundFont at CorsixTH/Lua/app.lua:1591 probes GeneralUser-GS.sf2, FluidR3_GM.sf2, FluidR3.sf3 and three Linux system paths), and the SoundFont is a user preference, not game data. diff --git a/emulators/devilutionx.yml b/emulators/devilutionx.yml index 0c7dfcb13..97825919c 100644 --- a/emulators/devilutionx.yml +++ b/emulators/devilutionx.yml @@ -54,7 +54,7 @@ notes: | the mods/hf file list under the BUILD_MOD_MPQS option (CMake/Mods.cmake:11-36, 61-71). The compiled binary expects devilutionx.mpq next to the executable on every - platform except Android, Apple and 3DS/Switch (:511-518), where its + platform except Android, Apple and 3DS/Switch (Source/engine/assets.cpp:511-518), where its contents are shipped loose. After loading the core archives the engine reads the asset ASSETS_VERSION and aborts with "Failed to load UI resources" unless its content is "1\n" (Source/init.cpp:54, 79-85, diff --git a/scripts/sourceref.py b/scripts/sourceref.py index c0bf3fcd1..ec980b6e2 100644 --- a/scripts/sourceref.py +++ b/scripts/sourceref.py @@ -220,10 +220,21 @@ PROSE_EXTENSIONLESS = ( "COPYING", "CHANGELOG", ) -PROSE_CITE_RE = re.compile( +# Any other dotless file is cited with its directory, which a constant or a +# hex address never carries: lightspark's tools/errorgen:20. +_PROSE_PATH = ( r"(?P[A-Za-z0-9_][\w./+-]*\.[A-Za-z]\w*" - r"|(?:[\w./+-]*/)?(?:" + "|".join(PROSE_EXTENSIONLESS) + r"))" - r":(?P\d+(?:-\d+)?)" + r"|(?:[\w./+-]*/)?(?:" + "|".join(PROSE_EXTENSIONLESS) + r")" + r"|(?:[\w.+-]+/)+[A-Za-z_][\w+-]*)" +) +PROSE_CITE_RE = re.compile(_PROSE_PATH + r":(?P\d+(?:-\d+)?)") +# A file named before its locations, `Source/engine/assets.cpp:` then a +# list, and a location that leaves the file out, `(:507)`, which reads in +# the file named last: devilutionx, corsixth and starship cite their load +# order this way. Neither was collected, so no bump or guard followed them. +_PROSE_FILE_NAMED_RE = re.compile(_PROSE_PATH + r":(?=\s|$)") +_PROSE_ABBREVIATED_RE = re.compile( + r"(?:(?<=[(\s])|^):(?P\d+(?:-\d+)?)(?![\w:-])(?!\.\d)" ) PROSE_LIST_ITEM_RE = re.compile(r"\d+[.)](?:\s|$)") PROSE_WRAPPED_RANGE_RE = re.compile(r":\d+(?:-\d+)?(?:[,)\s]|$)") @@ -344,6 +355,12 @@ def _prose_runs( joined line, so a writer can put each span back on its own line. """ lines = text.splitlines() + # The file an abbreviated location reads in, and the role it was named + # under, carried from line to line within the scalar. + named: tuple[str, str] | None = None + # Where the run wrapped from the line before ends in this one: its + # ranges are that run's, not abbreviated locations of their own. + carried = 0 for number, own in enumerate(lines): # A literal block keeps its line breaks, so a citation wrapped after # `file:` or after a range's comma lost the ranges on the next line @@ -369,46 +386,35 @@ def _prose_runs( elif following and line.endswith(","): joined_at = len(line) + 1 line += " " + following - consumed = 0 - for match in PROSE_CITE_RE.finditer(line): + consumed, carried = carried, 0 + events = sorted( + [(m.start(), 0, m) for m in PROSE_CITE_RE.finditer(line)] + + [(m.start(), 1, m) for m in _PROSE_FILE_NAMED_RE.finditer(line)] + + [(m.start(), 2, m) for m in _PROSE_ABBREVIATED_RE.finditer(line)], + key=lambda event: (event[0], event[1]), + ) + for _start, kind, match in events: if match.start() >= boundary: break if match.start() < consumed: continue - token_start = max(line.rfind(" ", 0, match.start()), - line.rfind("\t", 0, match.start())) + 1 - token = line[token_start:].split()[0] if line[token_start:] else "" - if "://" in token: + if kind == 1: + if "://" not in line[max(0, match.start() - 3):match.end()]: + named = (match.group("path"), "") continue - path = match.group("path") - path_start = match.start("path") - # A dotfile citation opens its token with the dot: `.gitlab-ci.yml` - # after a delimiter is the filename, while the dot of `boot.cfg` - # inside a word belongs to the word before it. - if ( - path_start > 0 - and line[path_start - 1] == "." - and (path_start == 1 or line[path_start - 2] in " \t(") - ): - path_start -= 1 - path = "." + path - # A path opening its line reads its role or project word at the - # end of the line before: genesis_plus_gx_expanded wrapped - # `(Batocera` and `package/batocera/...mk:7` one line apart. - lead = line[:path_start] - if number and not lead.strip(): - lead = lines[number - 1].rstrip() + " " - role = _prose_role(lead, len(lead)) - if not role and _prose_external(lead, len(lead), known): - # Kept with its project word, the source_ref convention: the - # report looks for the project inside the declared trees - # (mupen64plus-ae vendors `rice Config.cpp`) and calls the - # citation EXTERNAL only when none carries it. - before = lead.rstrip() - word = before[max(before.rfind(" "), before.rfind("\t"), - before.rfind("(")) + 1:] - path = f"{word} {path}" - path_span = (path_start, match.end("path")) + if kind == 2: + if named is None: + continue + path, role = named + path_start = match.start() + path_span = (path_start, path_start) + else: + path, role, path_start, path_span = _prose_path( + line, lines, number, match, known + ) + if path is None: + continue + named = (path, role) spans = [(match.start("range"), match.end("range"))] end = match.end() while True: @@ -429,6 +435,9 @@ def _prose_runs( spans.append((cont.start("range"), cont.end("range"))) end = cont.end() consumed = end + if end > boundary: + indent = len(lines[number + 1]) - len(lines[number + 1].lstrip()) + carried = indent + end - joined_at parts = [] for lo, hi in spans: rng = line[lo:hi] @@ -438,3 +447,44 @@ def _prose_runs( number, path_span, tuple(spans), tuple(parts), line[path_start:end], joined_at if end > boundary else 0, ) + + +def _prose_path( + line: str, lines: list[str], number: int, match: re.Match, + known: frozenset[str] | set[str], +) -> tuple[str | None, str, int, tuple[int, int]]: + """(path, role, start, span) of one written citation, None for a URL.""" + token_start = max(line.rfind(" ", 0, match.start()), + line.rfind("\t", 0, match.start())) + 1 + token = line[token_start:].split()[0] if line[token_start:] else "" + if "://" in token: + return None, "", 0, (0, 0) + path = match.group("path") + path_start = match.start("path") + # A dotfile citation opens its token with the dot: `.gitlab-ci.yml` + # after a delimiter is the filename, while the dot of `boot.cfg` + # inside a word belongs to the word before it. + if ( + path_start > 0 + and line[path_start - 1] == "." + and (path_start == 1 or line[path_start - 2] in " \t(") + ): + path_start -= 1 + path = "." + path + # A path opening its line reads its role or project word at the + # end of the line before: genesis_plus_gx_expanded wrapped + # `(Batocera` and `package/batocera/...mk:7` one line apart. + lead = line[:path_start] + if number and not lead.strip(): + lead = lines[number - 1].rstrip() + " " + role = _prose_role(lead, len(lead)) + if not role and _prose_external(lead, len(lead), known): + # Kept with its project word, the source_ref convention: the + # report looks for the project inside the declared trees + # (mupen64plus-ae vendors `rice Config.cpp`) and calls the + # citation EXTERNAL only when none carries it. + before = lead.rstrip() + word = before[max(before.rfind(" "), before.rfind("\t"), + before.rfind("(")) + 1:] + path = f"{word} {path}" + return path, role, path_start, (path_start, match.end("path")) diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index 2521591b2..8bee4ba3d 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -5919,6 +5919,39 @@ class ARoleIsReadPerPart(unittest.TestCase): ("src/a.cpp", "source"), ("src/a.cpp", "source"), ("b.c", "upstream"), ]) + def test_an_abbreviated_location_reads_in_the_file_named_last(self): + """devilutionx writes `Source/engine/assets.cpp:` then `(:507)` for + each loader, and its pin moved under five uncollected numbers.""" + from sourceref import _prose_runs + + text = ( + "Load order is defined in Source/engine/assets.cpp:\n" + " LoadCoreArchives() (:507) - devilutionx.mpq, fonts.mpq\n" + " LoadLanguageArchive() (:523) - .mpq\n" + "Startup (src/port/Engine.cpp:65-127) loads sf64.o2r (:72, :88-89),\n" + "falls back at :90-106. Wrapped (code/q_shared.h\n" + ":44,65) and lightspark's (tools/errorgen:20).\n" + ) + cited = [ + (p.path, p.start, p.end) for run in _prose_runs(text) for p in run[3] + ] + self.assertEqual(cited, [ + ("Source/engine/assets.cpp", 507, 507), + ("Source/engine/assets.cpp", 523, 523), + ("src/port/Engine.cpp", 65, 127), + ("src/port/Engine.cpp", 72, 72), + ("src/port/Engine.cpp", 88, 89), + ("src/port/Engine.cpp", 90, 106), + ("code/q_shared.h", 44, 44), + ("code/q_shared.h", 65, 65), + ("tools/errorgen", 20, 20), + ]) + + def test_an_abbreviated_location_needs_a_file_before_it(self): + from sourceref import _prose_runs + + self.assertEqual(list(_prose_runs("read at boot (:507), devilutionx.mpq")), []) + class TheTriageCountsWhatBlocksThePin(unittest.TestCase): """Prose written under an older pin refuses the recale and the bump, yet From 63fafaf0f9e02171a02e7a20f41cd0c760b69262 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:37:12 +0100 Subject: [PATCH 07/15] fix: read teknoparrot and simple64 from mirrors teknogods/TeknoParrotUI and simple64/simple64 answer 404 on the API and the web, and neither organisation lists them now. simple64's README at the pin archives the project for gopher64. Each profile declares upstream_gone and a fork that keeps its default branch at the analysed commit: thecrixux/TeknoParrotUI master is b75872ad, simple64-vault/simple64 main is d8c969c7. realign_prose reads a source declared gone through that mirror, which unblocks five teknoparrot notes citations that anchor unchanged. --- emulators/simple64.yml | 2 ++ emulators/teknoparrot.yml | 2 ++ scripts/profile_sync.py | 24 ++++++++++++++++++++---- tests/test_profile_sync.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/emulators/simple64.yml b/emulators/simple64.yml index f09be5b1d..9df8dfdad 100644 --- a/emulators/simple64.yml +++ b/emulators/simple64.yml @@ -5,6 +5,8 @@ source: "https://github.com/simple64/simple64" upstream: "https://github.com/simple64/simple64" profiled_date: "2026-08-08" source_commit: "d8c969c7b932e3d76e6a25549d76348839dbaefd" +upstream_gone: "simple64/simple64 answers 404 on the API and the web, and the simple64 organisation lists no repository. The README at the pin archives the project in favour of gopher64. Observed 2026-11-06. simple64-vault/simple64 keeps main at the analysed commit." +source_mirror: "https://github.com/simple64-vault/simple64" core_version: "v2024.12.1-5-gd8c969c" version_basis: kind: tag diff --git a/emulators/teknoparrot.yml b/emulators/teknoparrot.yml index 59a2a4a5f..8b6ed2cf4 100644 --- a/emulators/teknoparrot.yml +++ b/emulators/teknoparrot.yml @@ -6,6 +6,8 @@ upstream: closed-source author: "TeknoGods" profiled_date: "2026-10-20" source_commit: "b75872addc1d5cce81a62f4c1468eab59e2d3cbe" +upstream_gone: "teknogods/TeknoParrotUI answers 404 on the API and the web, and the teknogods organisation no longer lists it. Observed 2026-11-06. thecrixux/TeknoParrotUI keeps master at the analysed commit." +source_mirror: "https://github.com/thecrixux/TeknoParrotUI" core_version: "TeknoParrotUI 7ffcdf1" version_basis: kind: unverified diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 76bfdb95a..9c4471dfd 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -3914,7 +3914,8 @@ def _realign_part( def _writing_pairs( - document: dict, repos: list, revisions: list[tuple[str, dict]], intro_sha: str + document: dict, repos: list, revisions: list[tuple[str, dict]], intro_sha: str, + served: dict | None = None, ) -> tuple[list[list], list[str]]: """Every reading of where the text was written, and the ambiguities. @@ -3944,8 +3945,9 @@ def _writing_pairs( ambiguous.append(f"{pin_field}_commit {' -> '.join(written)}") # The current pin among the candidates reads as "no move" (None) for # that repository, which the other candidates must then agree with. + reader = (served or {}).get(repo, repo) options.append([ - (repo, pin, current) if pin != current else None for pin in written + (reader, pin, current) if pin != current else None for pin in written ]) readings = [ [pair for pair in combo if pair is not None] @@ -4026,8 +4028,20 @@ def realign_prose( ] if not repos: return [] + # A source declared gone is read through its mirror, which carries the + # same commits: teknoparrot's notes stayed blocked once + # teknogods/TeknoParrotUI answered 404, every reading of the old pin + # calling the files absent. + mirror = ( + upstream.parse_repo(str(document.get("source_mirror") or "")) + if document.get("upstream_gone") else None + ) + served = { + repo: mirror for field, _mode, repo in repos + if mirror is not None and field == "source" + } pinned = [ - (repo, pin) + (served.get(repo, repo), pin) for field, mode, repo in repos if (pin := _pin_of(document, f"{field}_commit", mode)) ] @@ -4107,7 +4121,9 @@ def realign_prose( if intro_sha is None: # The scalar is not committed yet: written now, under this pin. continue - readings, ambiguous = _writing_pairs(document, repos, revisions, intro_sha) + readings, ambiguous = _writing_pairs( + document, repos, revisions, intro_sha, served + ) if not any(readings): continue moves, blocked = _realign_citation( diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index 8bee4ba3d..9a0fb5241 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -4784,6 +4784,34 @@ class TestRealignProse(unittest.TestCase): self.assertTrue(flagged, messages) self.assertFalse([m for m in messages if "b.c:" in m], messages) + def test_a_source_declared_gone_is_read_through_its_mirror(self): + """teknogods/TeknoParrotUI answers 404. Read there, every citation + written under the old pin was absent and blocked the profile.""" + self._repo_with_advanced_pin() + self._commit( + PROSE_SAMPLE.replace( + 'source_commit: "pin"\n', + 'source_commit: "pin"\n' + 'upstream_gone: "o/n answers 404."\n' + 'source_mirror: "https://github.com/m/n"\n', + ), + "advance the pin, declare the forge gone", + ) + self.files[("oldpin", "a.c")] = ["x"] * 9 + ["subject"] + self.files[("pin", "a.c")] = ["x"] * 13 + ["subject"] + self.files[("oldpin", "b.c")] = list("pqrstuvwx") + self.files[("pin", "b.c")] = list("pqrstuvwx") + served = profile_sync.upstream.fetch_file + profile_sync.upstream.fetch_file = ( + lambda repo, sha, path, cache_dir, offline=False: ( + served(repo, sha, path, cache_dir, offline) + if repo.slug == "m/n" else None + ) + ) + messages = profile_sync.realign_prose(self.path, self.tmp.name, dry_run=True) + # b.c did not move between the two pins: nothing to read again. + self.assertFalse([m for m in messages if "b.c:" in m], messages) + def test_a_citation_in_an_unmoved_upstream_needs_no_reading(self): """nethersx2 cites AetherSX2 files that only its upstream carries, at an upstream pin that never moved. The port's pin did move under From 420162c79b78ae698f25d0d6cd9aea47bb07ff4b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:37:36 +0100 Subject: [PATCH 08/15] fix: rank uncheckable profiles first in triage --strict fails on a profile no declared repository answers for, yet --triage scored it 0 under the clean profiles and left it out of the count to review. It now weighs as a pin to set again and is counted. --- scripts/profile_sync.py | 5 ++++- tests/test_profile_sync.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index 9c4471dfd..5a90476ae 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -2665,6 +2665,9 @@ def drift_score( # set again before any ref can be read, so each outranks a GONE ref: # pcem ranked last at 0 with an orphaned upstream_commit. score += PIN_WEIGHT * (len(report.orphaned) + len(report.moved_repos)) + # A profile no repository answers for cannot be read at all, and + # --strict fails on it: it ranked at 0 under the clean profiles. + score += PIN_WEIGHT * report.uncheckable() score += min(commits, 20) // 4 return score @@ -5034,7 +5037,7 @@ def _print_triage( state = triage_state(report) if refused: state += f", {refused}" - pending += bool(report.needs_review()) + pending += bool(report.needs_review() or report.uncheckable()) print(f"{score:5d} {report.name:30s} {state}") print(f"{len(ranked)} profiles, {pending} with something to review") diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index 9a0fb5241..95879c00a 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -6027,6 +6027,25 @@ class TheTriageCountsWhatBlocksThePin(unittest.TestCase): self.assertEqual(lines[-1], "2 profiles, 1 with something to review") self.assertEqual(len(err.getvalue().splitlines()), 1, err.getvalue()) + def test_an_uncheckable_profile_ranks_first_and_counts(self): + """simple64 and teknoparrot answered 404 and --strict failed on them, + while the triage put them at 0 under the clean profiles.""" + gone = ProfileReport( + name="gone", pin="p", skipped="no declared repository answers (o/n)", + ) + reports = [self._report("clean", 1), gone] + selected = {r.name: {"source": "https://github.com/o/n"} for r in reports} + args = argparse.Namespace(cache_dir="unused", offline=True) + out = io.StringIO() + with mock.patch.object(profile_sync, "check_version", return_value=None), \ + mock.patch.object(profile_sync.upstream, "commits_touching", + return_value=0), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + profile_sync._print_triage(args, selected, reports) + lines = out.getvalue().splitlines() + self.assertIn("gone", lines[1]) + self.assertEqual(lines[-1], "2 profiles, 1 with something to review") + class TestBranchProfiles(unittest.TestCase): """--changed and --added name the profiles a branch brings, not by hand.""" From 63be8cc3bccfbb9d2a5ac8d942cdfc29a4f76cf6 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:39:40 +0100 Subject: [PATCH 09/15] fix: find tab-indented batocera-systems entries The entry pattern required four spaces of indent. EmuELEC indents 23 systems with one tab, so render appended each again whole and the published export carried 21 duplicated keys, the md5 fills only in the second copy. Any indent opens an entry now, and an entry written again keeps its own. The EmuELEC export is regenerated: one copy of each system, the amiga entry also gaining the eight WHDLoad Kickstarts the profiles added since 2026-10-22. --- exports/emuelec.json | 98 +++++++++++++++++- exports/emuelec/batocera-systems | 138 +++++++++++++------------- scripts/exporter/batocera_exporter.py | 18 ++-- tests/test_exporters.py | 18 ++++ 4 files changed, 191 insertions(+), 81 deletions(-) diff --git a/exports/emuelec.json b/exports/emuelec.json index 25e4cdf5f..36da198f3 100644 --- a/exports/emuelec.json +++ b/exports/emuelec.json @@ -6,14 +6,102 @@ "files": [ "batocera-systems" ], - "summary": "590 kept, 39 files added, 47 hashes added to an existing entry", + "summary": "590 kept, 47 files added, 47 hashes added to an existing entry", "counts": { "kept": 590, - "added": 39, - "not_added": 1117, + "added": 47, + "not_added": 1712, "not_kept": 0 }, "changes": [ + { + "kind": "added", + "system": "amiga", + "name": "kick33180.A500.RTB", + "destination": "whdboot/save-data/Kickstarts/kick33180.A500.RTB", + "field": null, + "old": null, + "new": { + "md5": "c993feb1eeddec1d2e687aa514231028" + } + }, + { + "kind": "added", + "system": "amiga", + "name": "kick34005.A500.RTB", + "destination": "whdboot/save-data/Kickstarts/kick34005.A500.RTB", + "field": null, + "old": null, + "new": { + "md5": "7eb13d974af0992d672b67c317a7e0a1" + } + }, + { + "kind": "added", + "system": "amiga", + "name": "kick37175.A500.RTB", + "destination": "whdboot/save-data/Kickstarts/kick37175.A500.RTB", + "field": null, + "old": null, + "new": { + "md5": "e057054449b5e21591ce27d2eb9ac720" + } + }, + { + "kind": "added", + "system": "amiga", + "name": "kick36143.A3000.RTB", + "destination": "whdboot/save-data/Kickstarts/kick36143.A3000.RTB", + "field": null, + "old": null, + "new": { + "md5": "2a2652daba6be3ef1d0bdf5ddd4347c6" + } + }, + { + "kind": "added", + "system": "amiga", + "name": "kick39106.A1200.RTB", + "destination": "whdboot/save-data/Kickstarts/kick39106.A1200.RTB", + "field": null, + "old": null, + "new": { + "md5": "af8b2e2227f01a142e20c8ab463f14ed" + } + }, + { + "kind": "added", + "system": "amiga", + "name": "kick40063.A600.RTB", + "destination": "whdboot/save-data/Kickstarts/kick40063.A600.RTB", + "field": null, + "old": null, + "new": { + "md5": "3f97c1963d82c808ef29e7c984c4c688" + } + }, + { + "kind": "added", + "system": "amiga", + "name": "kick40068.A1200.RTB", + "destination": "whdboot/save-data/Kickstarts/kick40068.A1200.RTB", + "field": null, + "old": null, + "new": { + "md5": "547b42b7b070c7f227ea45e0986d57a1" + } + }, + { + "kind": "added", + "system": "amiga", + "name": "kick40068.A4000.RTB", + "destination": "whdboot/save-data/Kickstarts/kick40068.A4000.RTB", + "field": null, + "old": null, + "new": { + "md5": "f89f19b858ea6b1a04327c6f71ee1600" + } + }, { "kind": "added", "system": "amiga", @@ -922,7 +1010,8 @@ "hashes_cleared": [], "requirements_corrected": 0, "withheld": [], - "found_by_content": 107, + "found_by_content": 312, + "unattributed": [], "removal_candidates": [ { "system": "colecovision", @@ -4421,5 +4510,6 @@ "unresolved_cores": [] } ], + "doubled": [], "invalid": [] } diff --git a/exports/emuelec/batocera-systems b/exports/emuelec/batocera-systems index 8d5bbb68f..06688a758 100755 --- a/exports/emuelec/batocera-systems +++ b/exports/emuelec/batocera-systems @@ -51,7 +51,25 @@ systems = { { "md5": "e40a5dfb3d017ba8779faba30cbd1c8e", "file": "bios/kick40063.A600" }, { "md5": "646773759326fbac3b2311fd8c8793ee", "file": "bios/kick40068.A1200" }, { "md5": "9bdedde6a4f33555b4a270c8ca53297d", "file": "bios/kick40068.A4000" }, - { "md5": "bb72565701b1b6faece07d68ea5da639", "file": "bios/CD32 Extended-ROM r40.60 (1993)(Commodore)(CD32).rom" } ] }, + { "md5": "bb72565701b1b6faece07d68ea5da639", "file": "bios/CD32 Extended-ROM r40.60 (1993)(Commodore)(CD32).rom" }, + { "md5": "c993feb1eeddec1d2e687aa514231028", "file": "whdboot/save-data/Kickstarts/kick33180.A500.RTB" }, + { "md5": "7eb13d974af0992d672b67c317a7e0a1", "file": "whdboot/save-data/Kickstarts/kick34005.A500.RTB" }, + { "md5": "e057054449b5e21591ce27d2eb9ac720", "file": "whdboot/save-data/Kickstarts/kick37175.A500.RTB" }, + { "md5": "2a2652daba6be3ef1d0bdf5ddd4347c6", "file": "whdboot/save-data/Kickstarts/kick36143.A3000.RTB" }, + { "md5": "af8b2e2227f01a142e20c8ab463f14ed", "file": "whdboot/save-data/Kickstarts/kick39106.A1200.RTB" }, + { "md5": "3f97c1963d82c808ef29e7c984c4c688", "file": "whdboot/save-data/Kickstarts/kick40063.A600.RTB" }, + { "md5": "547b42b7b070c7f227ea45e0986d57a1", "file": "whdboot/save-data/Kickstarts/kick40068.A1200.RTB" }, + { "md5": "f89f19b858ea6b1a04327c6f71ee1600", "file": "whdboot/save-data/Kickstarts/kick40068.A4000.RTB" }, + { "md5": "6c5b44c1a2bcee664129f850bfe4ed8a", "file": "uae_data/drive_click_A500.wav" }, + { "md5": "7401f854a75f9ede18769168ab896a2a", "file": "uae_data/drive_spin_A500.wav" }, + { "md5": "f9d954879fee8e96baa93d6139077cdb", "file": "uae_data/drive_spinnd_A500.wav" }, + { "md5": "aebeacc7e7bc52ef890785c835bec3d0", "file": "uae_data/drive_startup_A500.wav" }, + { "md5": "1a4b2d6254918ff769f26beeede75655", "file": "uae_data/drive_snatch_A500.wav" }, + { "md5": "b64bee23f356635f67fb08b543df9797", "file": "uae_data/drive_click_LOUD.wav" }, + { "md5": "56d8e36a2f04b5e8c5a1f1530cfc886c", "file": "uae_data/drive_spin_LOUD.wav" }, + { "md5": "43ab05c6d969cb40805970a5b2352875", "file": "uae_data/drive_spinnd_LOUD.wav" }, + { "md5": "743b4b9f0ca645c4c2a34953b5e01e45", "file": "uae_data/drive_startup_LOUD.wav" }, + { "md5": "40cdf89f35b68cfefd4df5fb0256e02f", "file": "uae_data/drive_snatch_LOUD.wav" } ] }, # ---------- Fujistu FM-Towns ---------- # # https://github.com/captainys/TOWNSEMU @@ -492,7 +510,9 @@ systems = { # SNK "neogeo": { "name": "NeoGeo", "biosFiles": [ { "md5": "", "file": "neogeo/neogeo.zip" } ] }, "neogeocd": { "name": "NeoGeo CD", "biosFiles": [ { "md5": "", "file": "neogeo/neogeo.zip" }, - { "md5": "f39572af7584cb5b3f70ae8cc848aba2", "file": "bios/neocd/neocd_z.rom"} ] }, + { "md5": "f39572af7584cb5b3f70ae8cc848aba2", "file": "bios/neocd/neocd_z.rom"}, + { "md5": "b31a246b69493c35194fe0bac120dc37", "file": "neocdz.zip"}, + { "md5": "006a881532f2caffc3fe9fb5593f2ed5", "file": "neocd.zip"} ] }, # Sony Computer Entertainment @@ -656,7 +676,7 @@ systems = { { "md5": "", "file": "ports/dinothawr/xbr-lv2-a.glslp" }, { "md5": "", "file": "ports/dinothawr/xbr-lv2-pass1.glsl" } ] }, - "DOOM1": { "name": "DOOM1 (PrBoom)", "biosFiles": [ { "md5": "", "file": "ports/doom/doom1.wad" } ] }, + "DOOM1": { "name": "DOOM1 (PrBoom)", "biosFiles": [ { "md5": "", "file": "ports/doom/doom1.wad" }, { "md5": "5772f907fdee4284f6332d89c0c5609c", "file": "gzdoom.pk3" }, { "md5": "094baa608b85fbe636933b4098e7e1f1", "file": "game_widescreen_gfx.pk3" } ] }, "DOOM2": { "name": "DOOM2 (PrBoom)", "biosFiles": [ { "md5": "", "file": "ports/doom2/doom2.wad" } ] }, @@ -749,59 +769,60 @@ systems = { { "md5": "", "file": "ports/opentyrian/newshi.shp" }, { "md5": "", "file": "ports/opentyrian/shapes).dat" } ] }, - "Cannonball": { "name": "Cannonball", "biosFiles": [ { "md5": "", "file": "ports/cannonball/epr-10187.88" }, - { "md5": "", "file": "ports/cannonball/epr-10327.76" }, - { "md5": "", "file": "ports/cannonball/epr-10327a.76" }, - { "md5": "", "file": "ports/cannonball/epr-10328.75" }, - { "md5": "", "file": "ports/cannonball/epr-10328a.75" }, - { "md5": "", "file": "ports/cannonball/epr-10329.58" }, - { "md5": "", "file": "ports/cannonball/epr-10329a.58" }, - { "md5": "", "file": "ports/cannonball/epr-10330.57" }, - { "md5": "", "file": "ports/cannonball/epr-10330a.57" }, - { "md5": "", "file": "ports/cannonball/epr-10380.133" }, - { "md5": "", "file": "ports/cannonball/epr-10380b.133" }, - { "md5": "", "file": "ports/cannonball/epr-10381.132" }, - { "md5": "", "file": "ports/cannonball/epr-10381b.132" }, - { "md5": "", "file": "ports/cannonball/epr-10382.118" }, - { "md5": "", "file": "ports/cannonball/epr-10382b.118" }, - { "md5": "", "file": "ports/cannonball/epr-10383.117" }, - { "md5": "", "file": "ports/cannonball/epr-10383b.117" }, - { "md5": "", "file": "ports/cannonball/mpr-10371.9" }, - { "md5": "", "file": "ports/cannonball/mpr-10372.13" }, - { "md5": "", "file": "ports/cannonball/mpr-10373.10" }, - { "md5": "", "file": "ports/cannonball/mpr-10374.14" }, - { "md5": "", "file": "ports/cannonball/mpr-10375.11" }, - { "md5": "", "file": "ports/cannonball/mpr-10376.15" }, - { "md5": "", "file": "ports/cannonball/mpr-10377.12" }, - { "md5": "", "file": "ports/cannonball/mpr-10378.16" }, - { "md5": "", "file": "ports/cannonball/opr-10185.11" }, - { "md5": "", "file": "ports/cannonball/opr-10186.47" }, - { "md5": "", "file": "ports/cannonball/opr-10188.71" }, - { "md5": "", "file": "ports/cannonball/opr-10189.70" }, - { "md5": "", "file": "ports/cannonball/opr-10190.69" }, - { "md5": "", "file": "ports/cannonball/opr-10191.68" }, - { "md5": "", "file": "ports/cannonball/opr-10192.67" }, - { "md5": "", "file": "ports/cannonball/opr-10193.66" }, - { "md5": "", "file": "ports/cannonball/opr-10230.104" }, - { "md5": "", "file": "ports/cannonball/opr-10231.103" }, - { "md5": "", "file": "ports/cannonball/opr-10232.102" }, - { "md5": "", "file": "ports/cannonball/opr-10266.101" }, - { "md5": "", "file": "ports/cannonball/opr-10267.100" }, - { "md5": "", "file": "ports/cannonball/opr-10268.99" } ] }, + "Cannonball": { "name": "Cannonball", "biosFiles": [ { "md5": "0cc087bbf1d5ad46319756b5b17ab9be", "file": "ports/cannonball/epr-10187.88" }, + { "md5": "eddae6f176ad5a3438e4dae1b8c4d4fb", "file": "ports/cannonball/epr-10327.76" }, + { "md5": "35bdcb3d8eaae6505af896a55f26dda1", "file": "ports/cannonball/epr-10327a.76" }, + { "md5": "73486b2da791109e322c71cbaf51338b", "file": "ports/cannonball/epr-10328.75" }, + { "md5": "5c82ed7dd7450f5c037b4dcaa0103218", "file": "ports/cannonball/epr-10328a.75" }, + { "md5": "7197dfbdf83029abd56e72b98b274c63", "file": "ports/cannonball/epr-10329.58" }, + { "md5": "f8729ace245eb8fc4854819f783f6042", "file": "ports/cannonball/epr-10329a.58" }, + { "md5": "9556c737b5506241a5a58635e58bdbad", "file": "ports/cannonball/epr-10330.57" }, + { "md5": "f0f9e2cce2d64dbb28650b11679f745b", "file": "ports/cannonball/epr-10330a.57" }, + { "md5": "591daabf7c4b9005171f9af83068f3a1", "file": "ports/cannonball/epr-10380.133" }, + { "md5": "94bb116334e7076aeeee5fcdcb536f87", "file": "ports/cannonball/epr-10380b.133" }, + { "md5": "ab1a9fe4feb5d5bd867552e3740bdc07", "file": "ports/cannonball/epr-10381.132" }, + { "md5": "ab1a9fe4feb5d5bd867552e3740bdc07", "file": "ports/cannonball/epr-10381b.132" }, + { "md5": "bcf3cae054791aa0c2c9d3c144bb1682", "file": "ports/cannonball/epr-10382.118" }, + { "md5": "8ff3212b3a2e37a37ba73a998bbc424f", "file": "ports/cannonball/epr-10382b.118" }, + { "md5": "161ceb196fcc9825bc981f933f305b5e", "file": "ports/cannonball/epr-10383.117" }, + { "md5": "2d2b6149f1bccaaebf9c792f292e36d0", "file": "ports/cannonball/epr-10383b.117" }, + { "md5": "9bc7fcfda9dea3a6d1c527b3146a4a95", "file": "ports/cannonball/mpr-10371.9" }, + { "md5": "7da2895e20d2152ebddbdd2df29f3bd4", "file": "ports/cannonball/mpr-10372.13" }, + { "md5": "5ed55433e64a4daecb9a0dbfc8771131", "file": "ports/cannonball/mpr-10373.10" }, + { "md5": "5d09103268ac520609fb10ba186374b1", "file": "ports/cannonball/mpr-10374.14" }, + { "md5": "187a6496e67c95e119ce97600902d178", "file": "ports/cannonball/mpr-10375.11" }, + { "md5": "107a513ce97fb3b88d97425f19054471", "file": "ports/cannonball/mpr-10376.15" }, + { "md5": "4a41f5ad01d78f52e8b6187c971c7270", "file": "ports/cannonball/mpr-10377.12" }, + { "md5": "f6b7defe9860aa8a2f2e2915f1d9b94d", "file": "ports/cannonball/mpr-10378.16" }, + { "md5": "866f1a596cc943bc5a416ff3e63cc9b0", "file": "ports/cannonball/opr-10185.11" }, + { "md5": "866f1a596cc943bc5a416ff3e63cc9b0", "file": "ports/cannonball/opr-10186.47" }, + { "md5": "ce4c62deb59049edb9972a5214ab39a1", "file": "ports/cannonball/opr-10188.71" }, + { "md5": "c23c9998ed1b32918cdb3b0a80547aac", "file": "ports/cannonball/opr-10189.70" }, + { "md5": "e555e967fcbb98d01c2da5730e9f6a28", "file": "ports/cannonball/opr-10190.69" }, + { "md5": "edfdde9f0a1f2ea98139c9823a190e13", "file": "ports/cannonball/opr-10191.68" }, + { "md5": "e36e9675a128d78acb191781e2271ae5", "file": "ports/cannonball/opr-10192.67" }, + { "md5": "4980cb65d6ef5d4c4910d28fc200d369", "file": "ports/cannonball/opr-10193.66" }, + { "md5": "e58109f14a0a4891afed803580577833", "file": "ports/cannonball/opr-10230.104" }, + { "md5": "faf3f323326434cebbe3536359724985", "file": "ports/cannonball/opr-10231.103" }, + { "md5": "7d6cae678c6a36bf0712958ee0e7711a", "file": "ports/cannonball/opr-10232.102" }, + { "md5": "786da8c971d530bd51400da1904d5a3d", "file": "ports/cannonball/opr-10266.101" }, + { "md5": "7ca240eb19818825f9d59d26b3037f69", "file": "ports/cannonball/opr-10267.100" }, + { "md5": "080b53f5d3952a8a07981f9f66997887", "file": "ports/cannonball/opr-10268.99" }, + { "md5": "7300090310ef7dc5cf4b7ebe06a38e66", "file": "cannonball/opr-10188.71f" } ] }, - "CaveStory": { "name": "Cave Story (nxengine)", "biosFiles": [ { "md5": "", "file": "ports/CaveStory/Doukutsu.exe" } ] }, + "CaveStory": { "name": "Cave Story (nxengine)", "biosFiles": [ { "md5": "", "file": "ports/CaveStory/Doukutsu.exe" }, { "md5": "38695d3d69d7a0ada8178072dad4c58b", "file": "nxengine/Doukutsu.exe" } ] }, "Dinothawr": { "name": "Dinothawr", "biosFiles": [ { "md5": "", "file": "ports/dinothawr/dinothawr.game" } ] }, - "Quake": { "name": "Quake (tryquake)", "biosFiles": [ { "md5": "", "file": "ports/quake/id1/pak0.pak" } ] }, + "Quake": { "name": "Quake (tryquake)", "biosFiles": [ { "md5": "", "file": "ports/quake/id1/pak0.pak" }, { "md5": "00480ea1008b9fe2e134896a015a31fd", "file": "quoth/pak0.pak" }, { "md5": "7e67039da2527d95f61c215ef6f879cd", "file": "quoth/pak1.pak" }, { "md5": "ae0a13119175f9b26b039d9bae508591", "file": "quoth/pak2.pak" } ] }, "REminiscence": { "name": "Flashback (REminiscence)", "biosFiles": [ { "md5": "", "file": "ports/reminiscence/level1.map" } ] }, - "RickDangerous": { "name": "Rick Dangerous (xrick)", "biosFiles": [ { "md5": "", "file": "ports/xrick/data.zip" } ] }, + "RickDangerous": { "name": "Rick Dangerous (xrick)", "biosFiles": [ { "md5": "a471e64e9f69afbe59c10cc94ed1b184", "file": "ports/xrick/data.zip" } ] }, "VVVVVV": { "name": "VVVVVV", "biosFiles": [ { "md5": "", "file": "ports/VVVVVV/data.zip" } ] }, - "eduke32": { "name": "Duke Nukem 3D", "biosFiles": [ { "md5": "", "file": "ports/eduke/duke3d.grp" } ] }, + "eduke32": { "name": "Duke Nukem 3D", "biosFiles": [ { "md5": "", "file": "ports/eduke/duke3d.grp" }, { "md5": "f8a7f96a84fc0f401abe2b1d617d1c76", "file": "penthous.zip" } ] }, "sonic1": { "name": "Sonic 1", "biosFiles": [ { "md5": "", "file": "ports/sonic1/Data.rsdk" } ] }, "sonic2": { "name": "Sonic 2", "biosFiles": [ { "md5": "", "file": "ports/sonic2/Data.rsdk" } ] }, "soniccd": { "name": "Sonic CD", "biosFiles": [ { "md5": "", "file": "ports/soniccd/Data.rsdk" } ] }, @@ -818,30 +839,7 @@ systems = { "cdi": { "name": "Phillips CDI", "biosFiles": [ { "md5": "", "file": "bios/same_cdi/bios/cdimono1.zip" }, { "md5": "", "file": "bios/same_cdi/bios/cdimono2.zip" }, { "md5": "", "file": "bios/same_cdi/bios/cdibios.zip" } ] }, - - "Cannonball": { "name": "Cannonball", "biosFiles": [ { "md5": "0cc087bbf1d5ad46319756b5b17ab9be", "file": "ports/cannonball/epr-10187.88" }, { "md5": "eddae6f176ad5a3438e4dae1b8c4d4fb", "file": "ports/cannonball/epr-10327.76" }, { "md5": "35bdcb3d8eaae6505af896a55f26dda1", "file": "ports/cannonball/epr-10327a.76" }, { "md5": "73486b2da791109e322c71cbaf51338b", "file": "ports/cannonball/epr-10328.75" }, { "md5": "5c82ed7dd7450f5c037b4dcaa0103218", "file": "ports/cannonball/epr-10328a.75" }, { "md5": "7197dfbdf83029abd56e72b98b274c63", "file": "ports/cannonball/epr-10329.58" }, { "md5": "f8729ace245eb8fc4854819f783f6042", "file": "ports/cannonball/epr-10329a.58" }, { "md5": "9556c737b5506241a5a58635e58bdbad", "file": "ports/cannonball/epr-10330.57" }, { "md5": "f0f9e2cce2d64dbb28650b11679f745b", "file": "ports/cannonball/epr-10330a.57" }, { "md5": "591daabf7c4b9005171f9af83068f3a1", "file": "ports/cannonball/epr-10380.133" }, { "md5": "94bb116334e7076aeeee5fcdcb536f87", "file": "ports/cannonball/epr-10380b.133" }, { "md5": "ab1a9fe4feb5d5bd867552e3740bdc07", "file": "ports/cannonball/epr-10381.132" }, { "md5": "ab1a9fe4feb5d5bd867552e3740bdc07", "file": "ports/cannonball/epr-10381b.132" }, { "md5": "bcf3cae054791aa0c2c9d3c144bb1682", "file": "ports/cannonball/epr-10382.118" }, { "md5": "8ff3212b3a2e37a37ba73a998bbc424f", "file": "ports/cannonball/epr-10382b.118" }, { "md5": "161ceb196fcc9825bc981f933f305b5e", "file": "ports/cannonball/epr-10383.117" }, { "md5": "2d2b6149f1bccaaebf9c792f292e36d0", "file": "ports/cannonball/epr-10383b.117" }, { "md5": "9bc7fcfda9dea3a6d1c527b3146a4a95", "file": "ports/cannonball/mpr-10371.9" }, { "md5": "7da2895e20d2152ebddbdd2df29f3bd4", "file": "ports/cannonball/mpr-10372.13" }, { "md5": "5ed55433e64a4daecb9a0dbfc8771131", "file": "ports/cannonball/mpr-10373.10" }, { "md5": "5d09103268ac520609fb10ba186374b1", "file": "ports/cannonball/mpr-10374.14" }, { "md5": "187a6496e67c95e119ce97600902d178", "file": "ports/cannonball/mpr-10375.11" }, { "md5": "107a513ce97fb3b88d97425f19054471", "file": "ports/cannonball/mpr-10376.15" }, { "md5": "4a41f5ad01d78f52e8b6187c971c7270", "file": "ports/cannonball/mpr-10377.12" }, { "md5": "f6b7defe9860aa8a2f2e2915f1d9b94d", "file": "ports/cannonball/mpr-10378.16" }, { "md5": "866f1a596cc943bc5a416ff3e63cc9b0", "file": "ports/cannonball/opr-10185.11" }, { "md5": "866f1a596cc943bc5a416ff3e63cc9b0", "file": "ports/cannonball/opr-10186.47" }, { "md5": "ce4c62deb59049edb9972a5214ab39a1", "file": "ports/cannonball/opr-10188.71" }, { "md5": "c23c9998ed1b32918cdb3b0a80547aac", "file": "ports/cannonball/opr-10189.70" }, { "md5": "e555e967fcbb98d01c2da5730e9f6a28", "file": "ports/cannonball/opr-10190.69" }, { "md5": "edfdde9f0a1f2ea98139c9823a190e13", "file": "ports/cannonball/opr-10191.68" }, { "md5": "e36e9675a128d78acb191781e2271ae5", "file": "ports/cannonball/opr-10192.67" }, { "md5": "4980cb65d6ef5d4c4910d28fc200d369", "file": "ports/cannonball/opr-10193.66" }, { "md5": "e58109f14a0a4891afed803580577833", "file": "ports/cannonball/opr-10230.104" }, { "md5": "faf3f323326434cebbe3536359724985", "file": "ports/cannonball/opr-10231.103" }, { "md5": "7d6cae678c6a36bf0712958ee0e7711a", "file": "ports/cannonball/opr-10232.102" }, { "md5": "786da8c971d530bd51400da1904d5a3d", "file": "ports/cannonball/opr-10266.101" }, { "md5": "7ca240eb19818825f9d59d26b3037f69", "file": "ports/cannonball/opr-10267.100" }, { "md5": "080b53f5d3952a8a07981f9f66997887", "file": "ports/cannonball/opr-10268.99" }, { "md5": "7300090310ef7dc5cf4b7ebe06a38e66", "file": "cannonball/opr-10188.71f" } ] }, - "CaveStory": { "name": "Cave Story (nxengine)", "biosFiles": [ { "md5": "", "file": "ports/CaveStory/Doukutsu.exe" }, { "md5": "38695d3d69d7a0ada8178072dad4c58b", "file": "nxengine/Doukutsu.exe" } ] }, - "DOOM1": { "name": "DOOM1 (PrBoom)", "biosFiles": [ { "md5": "", "file": "ports/doom/doom1.wad" }, { "md5": "5772f907fdee4284f6332d89c0c5609c", "file": "gzdoom.pk3" }, { "md5": "094baa608b85fbe636933b4098e7e1f1", "file": "game_widescreen_gfx.pk3" } ] }, - "DOOM2": { "name": "DOOM2 (PrBoom)", "biosFiles": [ { "md5": "", "file": "ports/doom2/doom2.wad" } ] }, - "Diablo": { "name": "Diablo 1 (devilutionx) ", "biosFiles": [ { "md5": "68f049866b44688a7af65ba766bef75a", "file": "ports/diablo/diabdat.mpq" } ] }, - "Dinothawr": { "name": "Dinothawr", "biosFiles": [ { "md5": "", "file": "ports/dinothawr/dinothawr.game" } ] }, - "OpenTyrian": { "name": "OpenTyrian", "biosFiles": [ { "md5": "", "file": "ports/opentyrian/cubetxt1.dat" }, { "md5": "", "file": "ports/opentyrian/cubetxt2.dat" }, { "md5": "", "file": "ports/opentyrian/cubetxt3.dat" }, { "md5": "", "file": "ports/opentyrian/cubetxt4.dat" }, { "md5": "", "file": "ports/opentyrian/demo.1" }, { "md5": "", "file": "ports/opentyrian/demo.2" }, { "md5": "", "file": "ports/opentyrian/demo.3" }, { "md5": "", "file": "ports/opentyrian/demo.4" }, { "md5": "", "file": "ports/opentyrian/demo.5" }, { "md5": "", "file": "ports/opentyrian/dpmi16bi.ovl" }, { "md5": "", "file": "ports/opentyrian/estsc.shp" }, { "md5": "", "file": "ports/opentyrian/exitmsg.bin" }, { "md5": "", "file": "ports/opentyrian/levels1.dat" }, { "md5": "", "file": "ports/opentyrian/levels2.dat" }, { "md5": "", "file": "ports/opentyrian/levels3.dat" }, { "md5": "", "file": "ports/opentyrian/levels4.dat" }, { "md5": "", "file": "ports/opentyrian/loudness.awe" }, { "md5": "", "file": "ports/opentyrian/modems.txt" }, { "md5": "", "file": "ports/opentyrian/music.mus" }, { "md5": "", "file": "ports/opentyrian/netarena.pcx" }, { "md5": "", "file": "ports/opentyrian/netfont1.pcx" }, { "md5": "", "file": "ports/opentyrian/netfont2.pcx" }, { "md5": "", "file": "ports/opentyrian/netmega.pcx" }, { "md5": "", "file": "ports/opentyrian/netset.pcx" }, { "md5": "", "file": "ports/opentyrian/newsh#.shp" }, { "md5": "", "file": "ports/opentyrian/newsh0.shp" }, { "md5": "", "file": "ports/opentyrian/newsh1.shp" }, { "md5": "", "file": "ports/opentyrian/newsh2.shp" }, { "md5": "", "file": "ports/opentyrian/newsh3.shp" }, { "md5": "", "file": "ports/opentyrian/newsh4.shp" }, { "md5": "", "file": "ports/opentyrian/newsh5.shp" }, { "md5": "", "file": "ports/opentyrian/newsh6.shp" }, { "md5": "", "file": "ports/opentyrian/newsh7.shp" }, { "md5": "", "file": "ports/opentyrian/newsh8.shp" }, { "md5": "", "file": "ports/opentyrian/newsh9.shp" }, { "md5": "", "file": "ports/opentyrian/newsha.shp" }, { "md5": "", "file": "ports/opentyrian/newshb.shp" }, { "md5": "", "file": "ports/opentyrian/newshc.shp" }, { "md5": "", "file": "ports/opentyrian/newshd.shp" }, { "md5": "", "file": "ports/opentyrian/newshe.shp" }, { "md5": "", "file": "ports/opentyrian/newshf.shp" }, { "md5": "", "file": "ports/opentyrian/newshg.shp" }, { "md5": "", "file": "ports/opentyrian/newshh.shp" }, { "md5": "", "file": "ports/opentyrian/newshj.shp" }, { "md5": "", "file": "ports/opentyrian/newshk.shp" }, { "md5": "", "file": "ports/opentyrian/newshl.shp" }, { "md5": "", "file": "ports/opentyrian/newshm.shp" }, { "md5": "", "file": "ports/opentyrian/newshn.shp" }, { "md5": "", "file": "ports/opentyrian/newsho.shp" }, { "md5": "", "file": "ports/opentyrian/newshp.shp" }, { "md5": "", "file": "ports/opentyrian/newshr.shp" }, { "md5": "", "file": "ports/opentyrian/newshs.shp" }, { "md5": "", "file": "ports/opentyrian/newsht.shp" }, { "md5": "", "file": "ports/opentyrian/newshu.shp" }, { "md5": "", "file": "ports/opentyrian/newshv.shp" }, { "md5": "", "file": "ports/opentyrian/newsh^.shp" }, { "md5": "", "file": "ports/opentyrian/newsh~.shp" }, { "md5": "", "file": "ports/opentyrian/order.tfp" }, { "md5": "", "file": "ports/opentyrian/palette.dat" }, { "md5": "", "file": "ports/opentyrian/setup.box" }, { "md5": "", "file": "ports/opentyrian/setup.ini" }, { "md5": "", "file": "ports/opentyrian/setup.int" }, { "md5": "", "file": "ports/opentyrian/shapesw.dat" }, { "md5": "", "file": "ports/opentyrian/shapesx.dat" }, { "md5": "", "file": "ports/opentyrian/shapesy.dat" }, { "md5": "", "file": "ports/opentyrian/shapesz.dat" }, { "md5": "", "file": "ports/opentyrian/shipedit.pcx" }, { "md5": "", "file": "ports/opentyrian/tshp2.pcx" }, { "md5": "", "file": "ports/opentyrian/tyrend.anm" }, { "md5": "", "file": "ports/opentyrian/tyrian.cdt" }, { "md5": "", "file": "ports/opentyrian/tyrian.hdt" }, { "md5": "", "file": "ports/opentyrian/tyrian.ico" }, { "md5": "", "file": "ports/opentyrian/tyrian.pic" }, { "md5": "", "file": "ports/opentyrian/tyrian.shp" }, { "md5": "", "file": "ports/opentyrian/tyrian.snd" }, { "md5": "", "file": "ports/opentyrian/tyrian1.lvl" }, { "md5": "", "file": "ports/opentyrian/tyrian2.lvl" }, { "md5": "", "file": "ports/opentyrian/tyrian3.lvl" }, { "md5": "", "file": "ports/opentyrian/tyrian4.lvl" }, { "md5": "", "file": "ports/opentyrian/tyrianc.shp" }, { "md5": "", "file": "ports/opentyrian/tyrset.pcx" }, { "md5": "", "file": "ports/opentyrian/user1.shp" }, { "md5": "", "file": "ports/opentyrian/user2.shp" }, { "md5": "", "file": "ports/opentyrian/voices.snd" }, { "md5": "", "file": "ports/opentyrian/voicesc.snd" }, { "md5": "", "file": "ports/opentyrian/netterm.int" }, { "md5": "", "file": "ports/opentyrian/newshi.shp" }, { "md5": "", "file": "ports/opentyrian/shapes).dat" } ] }, - "Quake": { "name": "Quake (tryquake)", "biosFiles": [ { "md5": "", "file": "ports/quake/id1/pak0.pak" }, { "md5": "00480ea1008b9fe2e134896a015a31fd", "file": "quoth/pak0.pak" }, { "md5": "7e67039da2527d95f61c215ef6f879cd", "file": "quoth/pak1.pak" }, { "md5": "ae0a13119175f9b26b039d9bae508591", "file": "quoth/pak2.pak" } ] }, - "REminiscence": { "name": "Flashback (REminiscence)", "biosFiles": [ { "md5": "", "file": "ports/reminiscence/level1.map" } ] }, - "RickDangerous": { "name": "Rick Dangerous (xrick)", "biosFiles": [ { "md5": "a471e64e9f69afbe59c10cc94ed1b184", "file": "ports/xrick/data.zip" } ] }, - "VVVVVV": { "name": "VVVVVV", "biosFiles": [ { "md5": "", "file": "ports/VVVVVV/data.zip" } ] }, - "amiga": { "name": "Amiga/CD32", "biosFiles": [ { "md5": "89da1838a24460e4b93f4f0c5d92d48d", "file": "bios/CDTV Extended-ROM v1.0 (1991)(Commodore)(CDTV)[!].rom" }, { "md5": "85ad74194e87c08904327de1a9443b7a", "file": "bios/Kickstart v1.2 r33.180 (1986)(Commodore)(A500-A1000-A2000)[!].rom" }, { "md5": "68c9c0826f6c0ca20546d588ee77391c", "file": "bios/Kickstart v1.2 rev 33.166 (1986)(Commodore)(A1000).rom" }, { "md5": "192d6d950d0ed3df8040b788502831c2", "file": "bios/Kickstart v1.3 r34.5 (1987)(Commodore)(A500-A1000-A2000-CDTV)[o].rom" }, { "md5": "82a21c1890cae844b3df741f2762d48d", "file": "bios/Kickstart v1.3 r34.5 (1987)(Commodore)(A500-A1000-A2000-CDTV)[!].rom" }, { "md5": "dc10d7bdd1b6f450773dfb558477c230", "file": "bios/Kickstart v2.04 r37.175 (1991)(Commodore)(A500+)[!].rom" }, { "md5": "465646c9b6729f77eea5314d1f057951", "file": "bios/Kickstart v2.05 r37.350 (1992)(Commodore)(A600HD)[!].rom" }, { "md5": "5f8924d013dd57a89cf349f4cdedc6b1", "file": "bios/Kickstart v3.1 r40.60 (1993)(Commodore)(CD32).rom" }, { "md5": "646773759326fbac3b2311fd8c8793ee", "file": "bios/Kickstart v3.1 r40.68 (1993)(Commodore)(A1200)[!].rom" }, { "md5": "413590e50098a056cfec418d3df0212d", "file": "bios/Kickstart v3.1 r40.68 (1993)(Commodore)(A3000).rom" }, { "md5": "9bdedde6a4f33555b4a270c8ca53297d", "file": "bios/Kickstart v3.1 r40.68 (1993)(Commodore)(A4000).rom" }, { "md5": "85ad74194e87c08904327de1a9443b7a", "file": "bios/kick33180.A500" }, { "md5": "82a21c1890cae844b3df741f2762d48d", "file": "bios/kick34005.A500" }, { "md5": "dc10d7bdd1b6f450773dfb558477c230", "file": "bios/kick37175.A500" }, { "md5": "5f8924d013dd57a89cf349f4cdedc6b1", "file": "bios/kick40060.CD32" }, { "md5": "bb72565701b1b6faece07d68ea5da639", "file": "bios/kick40060.CD32.ext" }, { "md5": "e40a5dfb3d017ba8779faba30cbd1c8e", "file": "bios/kick40063.A600" }, { "md5": "646773759326fbac3b2311fd8c8793ee", "file": "bios/kick40068.A1200" }, { "md5": "9bdedde6a4f33555b4a270c8ca53297d", "file": "bios/kick40068.A4000" }, { "md5": "bb72565701b1b6faece07d68ea5da639", "file": "bios/CD32 Extended-ROM r40.60 (1993)(Commodore)(CD32).rom" }, { "md5": "6c5b44c1a2bcee664129f850bfe4ed8a", "file": "uae_data/drive_click_A500.wav" }, { "md5": "7401f854a75f9ede18769168ab896a2a", "file": "uae_data/drive_spin_A500.wav" }, { "md5": "f9d954879fee8e96baa93d6139077cdb", "file": "uae_data/drive_spinnd_A500.wav" }, { "md5": "aebeacc7e7bc52ef890785c835bec3d0", "file": "uae_data/drive_startup_A500.wav" }, { "md5": "1a4b2d6254918ff769f26beeede75655", "file": "uae_data/drive_snatch_A500.wav" }, { "md5": "b64bee23f356635f67fb08b543df9797", "file": "uae_data/drive_click_LOUD.wav" }, { "md5": "56d8e36a2f04b5e8c5a1f1530cfc886c", "file": "uae_data/drive_spin_LOUD.wav" }, { "md5": "43ab05c6d969cb40805970a5b2352875", "file": "uae_data/drive_spinnd_LOUD.wav" }, { "md5": "743b4b9f0ca645c4c2a34953b5e01e45", "file": "uae_data/drive_startup_LOUD.wav" }, { "md5": "40cdf89f35b68cfefd4df5fb0256e02f", "file": "uae_data/drive_snatch_LOUD.wav" } ] }, - "atari7800": { "name": "Atari 7800", "biosFiles": [ { "md5": "0763f1ffb006ddbe32e52d497ee848ae", "file": "bios/7800 BIOS (U).rom" } ] }, - "atari800": { "name": "Atari 800", "biosFiles": [ { "md5": "06daac977823773a3eea3422fd26a703", "file": "bios/ATARIXL.ROM" }, { "md5": "0bac0c6a50104045d902df4503a4c30b", "file": "bios/ATARIBAS.ROM" }, { "md5": "eb1f32f5d9f382db1bbfb8d7f9cb343a", "file": "bios/ATARIOSA.ROM" }, { "md5": "a3e8d617c95d08031fe1b20d541434b2", "file": "bios/ATARIOSB.ROM" } ] }, - "colecovision": { "name": "Colecovision", "biosFiles": [ { "md5": "", "file": "bios/Machines/COL - Bit Corporation Dina/config.ini" }, { "md5": "1de922acdd742d31349c2801e9768c35", "file": "bios/Machines/COL - Bit Corporation Dina/czz50-1.rom" }, { "md5": "72b089dc55b7fe7ffb5028f365e8c045", "file": "bios/Machines/COL - Bit Corporation Dina/czz50-2.rom" }, { "md5": "2c66f5911e5b42b8ebe113403548eee7", "file": "bios/Machines/COL - ColecoVision/coleco.rom" }, { "md5": "", "file": "bios/Machines/COL - ColecoVision/config.ini" }, { "md5": "2c66f5911e5b42b8ebe113403548eee7", "file": "bios/Machines/COL - ColecoVision with Opcode Memory Extension/coleco.rom" }, { "md5": "", "file": "bios/Machines/COL - ColecoVision with Opcode Memory Extension/config.ini" }, { "md5": "", "file": "bios/Machines/COL - Spectravideo SVI-603 Coleco/config.ini" }, { "md5": "c60a2e85572c0ccb69505a7646d5c1b6", "file": "bios/Machines/COL - Spectravideo SVI-603 Coleco/SVI603.ROM" }, { "md5": "", "file": "bios/Databases/colecodb.xml" } ] }, - "eduke32": { "name": "Duke Nukem 3D", "biosFiles": [ { "md5": "", "file": "ports/eduke/duke3d.grp" }, { "md5": "f8a7f96a84fc0f401abe2b1d617d1c76", "file": "penthous.zip" } ] }, - "fds": { "name": "Nintendo Family Computer Disk System", "biosFiles": [ { "md5": "7bfe8c0540ed4bd6a0f1e2a0f0118ced", "file": "bios/NstDatabase.xml" }, { "md5": "ca30b50f880eb660a320674ed365ef7a", "file": "bios/disksys.rom" } ] }, - "gbc": { "name": "Game Boy Color", "biosFiles": [ { "md5": "32fbbd84168d3482956eb3c5051637f5", "file": "bios/gb_bios.bin" }, { "md5": "dbfce9db9deaa2567f6a84fde55f9680", "file": "bios/gbc_bios.bin" } ] }, - "neogeo": { "name": "NeoGeo", "biosFiles": [ { "md5": "", "file": "neogeo/neogeo.zip" } ] }, - "neogeocd": { "name": "NeoGeo CD", "biosFiles": [ { "md5": "", "file": "neogeo/neogeo.zip" }, { "md5": "f39572af7584cb5b3f70ae8cc848aba2", "file": "bios/neocd/neocd_z.rom" }, { "md5": "b31a246b69493c35194fe0bac120dc37", "file": "neocdz.zip" }, { "md5": "006a881532f2caffc3fe9fb5593f2ed5", "file": "neocd.zip" } ] }, - "o2em": { "name": "Odyssey 2", "biosFiles": [ { "md5": "562d5ebf9e030a40d6fabfc2f33139fd", "file": "bios/o2rom.bin" }, { "md5": "f1071cdb0b6b10dde94d3bc8a6146387", "file": "bios/c52.bin" }, { "md5": "c500ff71236068e0dc0d0603d265ae76", "file": "bios/g7400.bin" }, { "md5": "279008e4a0db2dc5f1c048853b033828", "file": "bios/jopac.bin" } ] }, - "pcengine": { "name": "PC Engine", "biosFiles": [ { "md5": "ff1a674273fe3540ccef576376407d1d", "file": "bios/syscard3.pce" } ] }, - + } class BiosStatus: diff --git a/scripts/exporter/batocera_exporter.py b/scripts/exporter/batocera_exporter.py index 6d877263f..bbf9e9de2 100644 --- a/scripts/exporter/batocera_exporter.py +++ b/scripts/exporter/batocera_exporter.py @@ -19,7 +19,9 @@ SOURCE_URL = ( "/package/batocera/core/batocera-scripts/scripts/batocera-systems" ) -_ENTRY_START = re.compile(r'^(\s{4})"([^"]+)":\s*\{') +# Any indent: Batocera writes four spaces, EmuELEC one tab for 23 systems, +# and an entry the pattern missed was appended again whole. +_ENTRY_START = re.compile(r'^(\s+)"([^"]+)":\s*\{') _INDENT = " " * 4 # One BIOS item: a dict literal with no nested braces. _ITEM = re.compile(r"\{[^{}\n]*\}") @@ -382,12 +384,13 @@ class Exporter(BaseExporter): return inner if isinstance(inner, dict) else {} def _entry_line( - self, system: NativeSystem, files: list[NativeFile], before: dict | None = None + self, system: NativeSystem, files: list[NativeFile], before: dict | None = None, + indent: str = _INDENT, ) -> str: own = self._own_entry(system, before) runner = "".join(f"{key}, " for key in self._runner_keys(own)) return ( - f'{_INDENT}"{system.native_id}": ' + f'{indent}"{system.native_id}": ' f'{{ "name": "{self.display_name(system)}", {runner}' f'"biosFiles": [ {self._bios_files(files, own)} ] }},' ) @@ -440,16 +443,17 @@ class Exporter(BaseExporter): before = self._parse_entry(original) own = self._own_entry(system, before) items = self._bios_items(files, own) + indent = _ENTRY_START.match(original[0]).group(1) one_line = len(original) == 1 and not trailing and not leading if one_line: - return [self._entry_line(system, files, before)] + return [self._entry_line(system, files, before, indent)] runner = "".join(f"{key}, " for key in self._runner_keys(own)) head = ( - f'{_INDENT}"{system.native_id}": ' + f'{indent}"{system.native_id}": ' f'{{ "name": "{self.display_name(system)}", {runner}"biosFiles": [' ) - pad = " " * (len(_INDENT) + 4) + pad = indent + " " * 4 lines = [head] for index, item in enumerate(items): key = _note_key(item) @@ -458,7 +462,7 @@ class Exporter(BaseExporter): note = trailing.get(key, "") suffix = f" {note}" if note else "" lines.append(f"{pad}{item}{separator}{suffix}") - lines.append(f"{_INDENT}] }},") + lines.append(f"{indent}] }},") return lines def _patched_entry( diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f84b4e624..9f7f9565b 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -2909,6 +2909,24 @@ class BatoceraNotesStayWithTheirItem(unittest.TestCase): self.assertEqual(trailing, {("bios/bk0010.zip", "two.rom"): "# tail"}) +class BatoceraEntriesAtAnyIndent(unittest.TestCase): + """EmuELEC indents 23 of its systems with one tab. The entry pattern + required four spaces, so each was appended again whole and the export + carried 21 duplicated keys that ast.literal_eval kept silently.""" + + def test_a_tab_indented_entry_is_located(self): + from exporter.batocera_exporter import Exporter + + body = [ + ' "atari5200": { "name": "5200", "biosFiles": [ { "md5": "a", "file": "bios/5200.rom" } ] },', + '\t"amiga": { "name": "Amiga", "biosFiles": [ { "md5": "b", "file": "bios/kick.rom" },', + '\t\t{ "md5": "c", "file": "bios/kick2.rom" } ] },', + ] + self.assertEqual( + Exporter._entry_spans(body), {"atari5200": (0, 0), "amiga": (1, 2)} + ) + + class RecalboxKeepsUnchangedText(unittest.TestCase): """Every element the model held was re-rendered: attribute order, alignment, line breaks inside md5 and digest case changed in 222 From 9a805fd2401b7ff0ff322de1c27701e305aebbd4 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:50:29 +0100 Subject: [PATCH 10/15] docs: name the methods that count an export adding-a-platform listed what a format can hold and none of what counts the written file. lands, states, fills, clears, names_anew and withheld are what seven exporters override so the summary matches the diff; a test reads the page for every one an exporter overrides. --- tests/test_exporters.py | 20 ++++++++++++++++++++ wiki/adding-a-platform.md | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 9f7f9565b..251b31966 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -2909,6 +2909,26 @@ class BatoceraNotesStayWithTheirItem(unittest.TestCase): self.assertEqual(trailing, {("bios/bk0010.zip", "two.rom"): "# tail"}) +class TheWikiNamesWhatCountsAnExport(unittest.TestCase): + """adding-a-platform listed the methods that say what a format can hold + and none of those that count what render wrote, which seven exporters + override so the report matches the diff.""" + + def test_every_overridden_contract_method_is_documented(self): + documented = ("carries", "requires", "can_add", "keys_by_system", + "lands", "states", "fills", "clears", "names_anew", "withheld") + overridden = { + name + for cls in discover_exporters().values() + for name in documented + if name in vars(cls) + } + wiki = (REPO_ROOT / "wiki" / "adding-a-platform.md").read_text(encoding="utf-8") + for name in sorted(overridden): + with self.subTest(method=name): + self.assertIn(f"`{name}(", wiki) + + class BatoceraEntriesAtAnyIndent(unittest.TestCase): """EmuELEC indents 23 of its systems with one tab. The entry pattern required four spaces, so each was appended again whole and the export diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index 8da75e437..add851786 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -606,6 +606,28 @@ of files: own system, so an entry filed under one of our system ids is never written. - `needs_original()` - `True` as soon as `native_sources()` names a file. +Those methods say what the format could hold. The report counts what `render()` +actually wrote, through a second set, read entry by entry: + +- `lands(fe)` - whether the entry is in the written file. Defaults to + `writable()`. Override it when `render()` drops an entry the format could + hold: RetroBat merges an addition into a path it already declares, and + System.dat leaves out a file with no section name. +- `states(fe, field)` - whether the written entry carries the corrected field. + Defaults to `field in carries()`. +- `fills(fe, field)` and `clears(fe, field)` - whether a hash only the truth + names is written, and whether a hash the truth contradicts leaves the entry. + Both default to `states()`. +- `names_anew(fe)` - whether the render wrote, for the first time, a file the + platform already declares. Only a format whose unit is not the entry needs it + (a RetroPie help sentence). +- `withheld()` - corrections the render proved and still did not write, named + so the run does not read as nothing to do. + +An exporter whose `render()` skips anything the defaults count must override +the matching method. Otherwise the summary announces a change the maintainer +will not find in the diff. + ### Round-trip validation The scraper and the exporter are checked against each other as From 46a4a26a546259c07c69532458f83ac711a5ddfc Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:50:29 +0100 Subject: [PATCH 11/15] fix: keep bundled game data out of missing verify counted game data installed with its emulator as missing, which cross_reference files as bundled: 24 of Batocera's 427. Bundled firmware already left the lists, and the line naming it never printed. Every bundled entry is now counted on that line, required firmware named and the rest under --verbose. --- scripts/verify.py | 26 ++++++++++++++++++-------- tests/test_verify_listing.py | 27 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/scripts/verify.py b/scripts/verify.py index f47c7f011..b12991d90 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1845,6 +1845,13 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: game_data, bios_files, unplaced = _split_undeclared(undeclared) + # A file installed with its emulator is never collected, firmware or + # game data, required or not: cross_reference counts it apart. The + # firmware already left the pack lists (_mark_unplaced), the game data + # did not, and Abuse's three bundled data files read as missing. + bundled = [u for u in undeclared if not u["in_repo"] and u.get("bundled")] + game_data = [u for u in game_data if u["in_repo"] or not u.get("bundled")] + req_not_in_repo = _core_required_missing(bios_files) req_hle_not_in_repo = [ u @@ -1854,12 +1861,6 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: req_in_repo = [u for u in bios_files if u["required"] and u["in_repo"]] opt_in_repo = [u for u in bios_files if not u["required"] and u["in_repo"]] opt_not_in_repo = [u for u in bios_files if not u["required"] and not u["in_repo"]] - - bundled = [ - u for u in bios_files - if u["required"] and not u["in_repo"] and u.get("bundled") - and not u.get("hle_fallback") - ] # The builder's count, one file per pack path, when the platform pass # replayed it. Counting entries said 1130 for RetroArch, 48 more than the # pack holds: rom.key alone was claimed by three Amiga cores. @@ -1886,8 +1887,17 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: for u in opt_not_in_repo: _print_undeclared_entry(u, "MISSING (optional)", verbose, repeated) if bundled: - names = ", ".join(sorted({u["name"] for u in bundled})) - print(f" Bundled with their emulator, not collected: {names}") + # The required firmware is named, as a missing one would be; the + # rest (shaders, runtime libraries, engine data) on request. + shown = bundled if verbose else [ + u for u in bundled + if u["required"] and u.get("category", "bios") != "game_data" + ] + names = ", ".join(sorted({u["name"] for u in shown})) + print( + f" Bundled with their emulator, not collected: {len(bundled)}" + + (f" ({names})" if names else "") + ) if unplaced: print(f" Core files held but not packed: {len(unplaced)} " f"({_omission_reasons(unplaced)})") diff --git a/tests/test_verify_listing.py b/tests/test_verify_listing.py index c825b52a5..209a8dae1 100644 --- a/tests/test_verify_listing.py +++ b/tests/test_verify_listing.py @@ -200,6 +200,33 @@ class EveryGapCountedIsNamed(unittest.TestCase): self.assertIn("MISSING (game data): engine needs level.dat", verbose.getvalue()) +class BundledGameDataIsNotMissing(unittest.TestCase): + """cross_reference put Abuse's abuse.lsp, defaults.prp and english.lsp + under bundled while verify called them missing game data: 24 of the 427 + Batocera counted.""" + + def test_bundled_game_data_is_counted_apart(self): + import contextlib + import io + + sys.path.insert(0, str(REPO_ROOT / "scripts")) + import verify # noqa: PLC0415 + + result = {"undeclared_files": [ + {"name": "abuse.lsp", "emulator": "Abuse", "required": True, + "in_repo": False, "category": "game_data", "bundled": True, + "in_pack": False}, + {"name": "level.dat", "emulator": "engine", "required": True, + "in_repo": False, "category": "game_data"}, + ]} + with contextlib.redirect_stdout(io.StringIO()) as out: + verify._print_undeclared_section(result, verbose=True) + self.assertIn("Game data: 0 in pack, 1 missing", out.getvalue()) + self.assertNotIn("needs abuse.lsp", out.getvalue()) + self.assertIn("Bundled with their emulator, not collected: 1 (abuse.lsp)", + out.getvalue()) + + class CoreCopiesWithoutAGapEntryAreCounted(unittest.TestCase): """verify --platform recalbox --system nec-pc-engine printed no Core files line while the pack held six geargrafx syscard copies: the gap From 810756463171990723f91878227d1e317780c989 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:59:15 +0100 Subject: [PATCH 12/15] fix: gearcoleco warns on an unknown CRC32 At be1a3716 find_firmware rejects a candidate by size alone (platforms/libretro/libretro.cpp:811-816), and an OS-7, EOS or SmartWriter image whose CRC32 differs is logged and loaded (lines 875-876, 928-935). The three entries keep crc32 in validation with on_mismatch warn, so another 8 KiB OS-7 revision such as colecoa.rom no longer reads as rejected. --- emulators/gearcoleco.yml | 3 +++ tests/test_validation_scope.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/emulators/gearcoleco.yml b/emulators/gearcoleco.yml index 72dd6fc61..c48ae5414 100644 --- a/emulators/gearcoleco.yml +++ b/emulators/gearcoleco.yml @@ -67,6 +67,7 @@ files: size: 8192 crc32: "3aa93ef3" validation: [size, crc32] + on_mismatch: {crc32: warn} source_ref: "platforms/libretro/libretro.cpp:779-840 (find_firmware), platforms/libretro/libretro.cpp:858-885 (load_colecovision_firmware), platforms/libretro/libretro.cpp:2453-2457, platforms/libretro/libretro.cpp:2258-2259, src/Adam.cpp:28-31, src/Adam.h:52, src/Memory.cpp:310-319, src/Memory.cpp:246-249" note: > Loaded for both machines. Any 8192-byte image is accepted, and a CRC32 @@ -80,6 +81,7 @@ files: size: 8192 crc32: "05a37a34" validation: [size, crc32] + on_mismatch: {crc32: warn} source_ref: "platforms/libretro/libretro.cpp:887-948 (load_adam_firmware), platforms/libretro/libretro.cpp:2426-2427, platforms/libretro/libretro.cpp:2483-2487, platforms/libretro/libretro.cpp:2500-2501, platforms/libretro/libretro.cpp:2584-2588, src/Adam.cpp:32-35, src/Adam.h:53, src/Adam.cpp:137-146, src/Adam.cpp:166-188, src/GearcolecoCore.cpp:414-419, src/Adam.cpp:519-522" note: > The 8 KiB EOS bank, the first half of the 16 KiB u22 chip. A CRC32 @@ -94,6 +96,7 @@ files: size: 32768 crc32: "58d86a2a" validation: [size, crc32] + on_mismatch: {crc32: warn} source_ref: "platforms/libretro/libretro.cpp:887-948 (load_adam_firmware), platforms/libretro/libretro.cpp:2426-2427, platforms/libretro/libretro.cpp:2483-2487, platforms/libretro/libretro.cpp:2500-2501, platforms/libretro/libretro.cpp:2584-2588, src/Adam.cpp:36-39, src/Adam.h:54, src/Adam.cpp:137-146, src/Adam.cpp:166-188, src/GearcolecoCore.cpp:414-419, src/Adam.cpp:513-522" note: > A CRC32 other than 58d86a2a logs "Unknown SmartWriter revision" and diff --git a/tests/test_validation_scope.py b/tests/test_validation_scope.py index 30c8946b4..423f7b81b 100644 --- a/tests/test_validation_scope.py +++ b/tests/test_validation_scope.py @@ -86,5 +86,35 @@ class ContentFoundLabelsTakeTheirRegionSlot(unittest.TestCase): self.assertIsNotNone(check_file_validation(str(path), "scph1001.bin", index)) +class GearcolecoWarnsOnAnUnknownRevision(unittest.TestCase): + """Gearcoleco rejects a firmware by size only and logs an unknown CRC32 + before loading it (platforms/libretro/libretro.cpp:811-816, 875-876, + 928-935 at be1a3716): another OS-7 revision of 8 KiB runs.""" + + def test_another_revision_passes(self): + import sys + import tempfile + + import yaml + + sys.path.insert(0, str(REPO_ROOT / "scripts")) + from validation import _build_validation_index, check_file_validation + + profile = yaml.safe_load( + (REPO_ROOT / "emulators" / "gearcoleco.yml").read_text(encoding="utf-8") + ) + index = _build_validation_index({"gearcoleco": profile}) + with tempfile.TemporaryDirectory() as tmp: + for name, size in (("colecovision.rom", 8192), ("eos.rom", 8192), + ("writer.rom", 32768)): + path = Path(tmp) / name + path.write_bytes(b"\x01" * size) + with self.subTest(name=name): + self.assertIsNone(check_file_validation(str(path), name, index)) + path.write_bytes(b"\x01" * (size - 1)) + with self.subTest(name=name, size="short"): + self.assertIsNotNone(check_file_validation(str(path), name, index)) + + if __name__ == "__main__": unittest.main() From 43cb0f84ab6d79aca99ccd2ea300cd7fc77439a8 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:04:51 +0100 Subject: [PATCH 13/15] feat: add AmberELEC hardware targets AmberELEC's metapackage adds LIBRETRO_CORES_EXTRA (bsnes, Mesen, melonDS, geolith) only on RG552, RG353 and RG503, and deletes those cores from es_systems.cfg on every other device (packages/amberelec/package.mk:23-31, 45-58 at a21975bf). The new target scraper reads the Makefile's device images, that test and es_systems.cfg the way the platform scraper does: RG351P, RG351V and RG351MP list 125 cores, the other three 136. The installer reads the device from COREELEC_DEVICE in os-release (scripts/image:168). --- install.ps1 | 2 +- install.py | 5 +- install.sh | 2 +- install/launchers/index.json | 16 +- install/targets/amberelec.json | 797 +++++++++++++++++ platforms/_registry.yml | 4 +- platforms/targets/amberelec.yml | 805 ++++++++++++++++++ .../targets/amberelec_targets_scraper.py | 157 ++++ tests/test_amberelec_targets.py | 102 +++ tests/test_install.py | 22 +- wiki/tools.md | 1 + 11 files changed, 1895 insertions(+), 18 deletions(-) create mode 100644 install/targets/amberelec.json create mode 100644 platforms/targets/amberelec.yml create mode 100644 scripts/scraper/targets/amberelec_targets_scraper.py create mode 100644 tests/test_amberelec_targets.py diff --git a/install.ps1 b/install.ps1 index 896cdaf76..4ddc9ef79 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858" + $defaultInstallSha256 = "f90d3dc9f71f7d4b309f2a9a564907291eb694eb6f88dd651b861e54f7414e38" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 01c0b9425..e3b82854b 100644 --- a/install.py +++ b/install.py @@ -109,7 +109,7 @@ TARGETS_URL = f"{BASE_URL}/install/targets/{{platform}}.json" # has no hardware targets, where the run sent the user after a list that # exists nowhere. TARGET_PLATFORMS = frozenset({ - "batocera", "emudeck", "knulli", "lakka", "recalbox", "retroarch", "retrobat", + "amberelec", "batocera", "emudeck", "knulli", "lakka", "recalbox", "retroarch", "retrobat", "retrodeck", "retropie", "rocknix", "romm", }) RAW_FILE_URL = f"{BASE_URL}/{{path}}" @@ -1472,7 +1472,10 @@ def _parse_bash_var(path: Path, key: str) -> str | None: # /usr/share/batocera to /usr/share/knulli (knulli-linux 57afa662, # board/scripts/post-build-script.sh:264-278) and writes KNULLI_SYSTEM_ARCH # to knulli.arch (package/system/knulli-system/knulli-system.mk:98-99). +# AmberELEC's scripts/image writes COREELEC_DEVICE (a21975bf, line 168), the +# DEVICE its Makefile builds (RG351P). _BOARD_SOURCES: dict[str, tuple[str, str]] = { + "amberelec": ("os_release", "COREELEC_DEVICE"), "batocera": ("file", "/usr/share/batocera/batocera.arch"), "recalbox": ("file", "/recalbox/recalbox.arch"), "rocknix": ("os_release", "HW_DEVICE"), diff --git a/install.sh b/install.sh index b085a4cbb..d5f49f5e2 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858" +DEFAULT_INSTALL_SHA256="f90d3dc9f71f7d4b309f2a9a564907291eb694eb6f88dd651b861e54f7414e38" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index 358ee41c9..771cf18b9 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1103292, + "sha256": "f90d3dc9f71f7d4b309f2a9a564907291eb694eb6f88dd651b861e54f7414e38", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1156738, - "sha256": "35c0920008baadca997d0f778e99ce46d702bbe3d3f0d33104d7158422b292aa" + "size": 1156919, + "sha256": "2efccc205141dfffc016b4ef9988e11877ef249634052835069ca1dd8921cdce" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1103292, + "sha256": "f90d3dc9f71f7d4b309f2a9a564907291eb694eb6f88dd651b861e54f7414e38", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1103292, + "sha256": "f90d3dc9f71f7d4b309f2a9a564907291eb694eb6f88dd651b861e54f7414e38", "executable": false }, { diff --git a/install/targets/amberelec.json b/install/targets/amberelec.json new file mode 100644 index 000000000..ff186677e --- /dev/null +++ b/install/targets/amberelec.json @@ -0,0 +1,797 @@ +{ + "rg351mp": [ + "81", + "AMIBERRY", + "AdvanceMame", + "DoubleCherryGB", + "HATARISA", + "OPENBOR", + "PPSSPPSDL", + "SCUMMVMSA", + "a5200", + "arduous", + "atari800", + "beetle_gba", + "beetle_lynx", + "beetle_ngp", + "beetle_pce", + "beetle_pce_fast", + "beetle_pcfx", + "beetle_supafaust", + "beetle_supergrafx", + "beetle_vb", + "beetle_wswan", + "bluemsx", + "cap32", + "crocods", + "dosbox_core", + "dosbox_pure", + "drastic", + "duckstation", + "easyrpg", + "ecwolf", + "emuscv", + "ep128emu_core", + "fake08", + "fbalpha2012", + "fbalpha2019", + "fbneo", + "fceumm", + "flycast", + "flycast2021", + "flycast2021le", + "fmsx", + "freechaf", + "freeintv", + "freej2me", + "freej2me-plus", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_wide", + "gpsp", + "gw", + "gzdoom", + "handy", + "hatari", + "hypseus_singe", + "jaxe", + "lowresnx", + "lzdoom", + "mame", + "mame2000", + "mame2003_plus", + "mame2010", + "mame2015", + "mame2016", + "mgba", + "mojozork", + "mupen64plus_next", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "o2em", + "opera", + "parallel_n64", + "pcsx_rearmed", + "pico8", + "picodrive", + "piemu", + "pokemini", + "potator", + "ppsspp", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "quasi88", + "quicknes", + "race", + "raze", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "smsplus", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "solarus", + "stella", + "stella2014", + "swanstation", + "tgbdual", + "theodore", + "tic80", + "uae4arm", + "uw8", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_xplus4", + "vice_xvic", + "vircon32", + "virtualjaguar", + "wasm4", + "x1", + "yabasanshiroSA" + ], + "rg351p": [ + "81", + "AMIBERRY", + "AdvanceMame", + "DoubleCherryGB", + "HATARISA", + "OPENBOR", + "PPSSPPSDL", + "SCUMMVMSA", + "a5200", + "arduous", + "atari800", + "beetle_gba", + "beetle_lynx", + "beetle_ngp", + "beetle_pce", + "beetle_pce_fast", + "beetle_pcfx", + "beetle_supafaust", + "beetle_supergrafx", + "beetle_vb", + "beetle_wswan", + "bluemsx", + "cap32", + "crocods", + "dosbox_core", + "dosbox_pure", + "drastic", + "duckstation", + "easyrpg", + "ecwolf", + "emuscv", + "ep128emu_core", + "fake08", + "fbalpha2012", + "fbalpha2019", + "fbneo", + "fceumm", + "flycast", + "flycast2021", + "flycast2021le", + "fmsx", + "freechaf", + "freeintv", + "freej2me", + "freej2me-plus", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_wide", + "gpsp", + "gw", + "gzdoom", + "handy", + "hatari", + "hypseus_singe", + "jaxe", + "lowresnx", + "lzdoom", + "mame", + "mame2000", + "mame2003_plus", + "mame2010", + "mame2015", + "mame2016", + "mgba", + "mojozork", + "mupen64plus_next", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "o2em", + "opera", + "parallel_n64", + "pcsx_rearmed", + "pico8", + "picodrive", + "piemu", + "pokemini", + "potator", + "ppsspp", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "quasi88", + "quicknes", + "race", + "raze", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "smsplus", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "solarus", + "stella", + "stella2014", + "swanstation", + "tgbdual", + "theodore", + "tic80", + "uae4arm", + "uw8", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_xplus4", + "vice_xvic", + "vircon32", + "virtualjaguar", + "wasm4", + "x1", + "yabasanshiroSA" + ], + "rg351v": [ + "81", + "AMIBERRY", + "AdvanceMame", + "DoubleCherryGB", + "HATARISA", + "OPENBOR", + "PPSSPPSDL", + "SCUMMVMSA", + "a5200", + "arduous", + "atari800", + "beetle_gba", + "beetle_lynx", + "beetle_ngp", + "beetle_pce", + "beetle_pce_fast", + "beetle_pcfx", + "beetle_supafaust", + "beetle_supergrafx", + "beetle_vb", + "beetle_wswan", + "bluemsx", + "cap32", + "crocods", + "dosbox_core", + "dosbox_pure", + "drastic", + "duckstation", + "easyrpg", + "ecwolf", + "emuscv", + "ep128emu_core", + "fake08", + "fbalpha2012", + "fbalpha2019", + "fbneo", + "fceumm", + "flycast", + "flycast2021", + "flycast2021le", + "fmsx", + "freechaf", + "freeintv", + "freej2me", + "freej2me-plus", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_wide", + "gpsp", + "gw", + "gzdoom", + "handy", + "hatari", + "hypseus_singe", + "jaxe", + "lowresnx", + "lzdoom", + "mame", + "mame2000", + "mame2003_plus", + "mame2010", + "mame2015", + "mame2016", + "mgba", + "mojozork", + "mupen64plus_next", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "o2em", + "opera", + "parallel_n64", + "pcsx_rearmed", + "pico8", + "picodrive", + "piemu", + "pokemini", + "potator", + "ppsspp", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "quasi88", + "quicknes", + "race", + "raze", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "smsplus", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "solarus", + "stella", + "stella2014", + "swanstation", + "tgbdual", + "theodore", + "tic80", + "uae4arm", + "uw8", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_xplus4", + "vice_xvic", + "vircon32", + "virtualjaguar", + "wasm4", + "x1", + "yabasanshiroSA" + ], + "rg353": [ + "81", + "AMIBERRY", + "AdvanceMame", + "DoubleCherryGB", + "HATARISA", + "OPENBOR", + "PPSSPPSDL", + "SCUMMVMSA", + "a5200", + "arduous", + "atari800", + "beetle_gba", + "beetle_lynx", + "beetle_ngp", + "beetle_pce", + "beetle_pce_fast", + "beetle_pcfx", + "beetle_snes", + "beetle_supafaust", + "beetle_supergrafx", + "beetle_vb", + "beetle_wswan", + "bluemsx", + "bsnes", + "bsnes2014_balanced", + "bsnes2014_performance", + "bsnes_mercury_balanced", + "bsnes_mercury_performance", + "cap32", + "crocods", + "dosbox_core", + "dosbox_pure", + "drastic", + "duckstation", + "easyrpg", + "ecwolf", + "emuscv", + "ep128emu_core", + "fake08", + "fbalpha2012", + "fbalpha2019", + "fbneo", + "fceumm", + "flycast", + "flycast2021", + "flycast2021le", + "fmsx", + "freechaf", + "freeintv", + "freej2me", + "freej2me-plus", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_wide", + "geolith", + "gpsp", + "gw", + "gzdoom", + "handy", + "hatari", + "hypseus_singe", + "jaxe", + "lowresnx", + "lzdoom", + "mame", + "mame2000", + "mame2003_plus", + "mame2010", + "mame2015", + "mame2016", + "melonds", + "melondsds", + "mesen", + "mesen-s", + "mgba", + "mojozork", + "mupen64plus_next", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "o2em", + "opera", + "parallel_n64", + "pcsx_rearmed", + "pico8", + "picodrive", + "piemu", + "pokemini", + "potator", + "ppsspp", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "quasi88", + "quicknes", + "race", + "raze", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "smsplus", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "solarus", + "stella", + "stella2014", + "swanstation", + "tgbdual", + "theodore", + "tic80", + "uae4arm", + "uw8", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_xplus4", + "vice_xvic", + "vircon32", + "virtualjaguar", + "wasm4", + "x1", + "yabasanshiroSA" + ], + "rg503": [ + "81", + "AMIBERRY", + "AdvanceMame", + "DoubleCherryGB", + "HATARISA", + "OPENBOR", + "PPSSPPSDL", + "SCUMMVMSA", + "a5200", + "arduous", + "atari800", + "beetle_gba", + "beetle_lynx", + "beetle_ngp", + "beetle_pce", + "beetle_pce_fast", + "beetle_pcfx", + "beetle_snes", + "beetle_supafaust", + "beetle_supergrafx", + "beetle_vb", + "beetle_wswan", + "bluemsx", + "bsnes", + "bsnes2014_balanced", + "bsnes2014_performance", + "bsnes_mercury_balanced", + "bsnes_mercury_performance", + "cap32", + "crocods", + "dosbox_core", + "dosbox_pure", + "drastic", + "duckstation", + "easyrpg", + "ecwolf", + "emuscv", + "ep128emu_core", + "fake08", + "fbalpha2012", + "fbalpha2019", + "fbneo", + "fceumm", + "flycast", + "flycast2021", + "flycast2021le", + "fmsx", + "freechaf", + "freeintv", + "freej2me", + "freej2me-plus", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_wide", + "geolith", + "gpsp", + "gw", + "gzdoom", + "handy", + "hatari", + "hypseus_singe", + "jaxe", + "lowresnx", + "lzdoom", + "mame", + "mame2000", + "mame2003_plus", + "mame2010", + "mame2015", + "mame2016", + "melonds", + "melondsds", + "mesen", + "mesen-s", + "mgba", + "mojozork", + "mupen64plus_next", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "o2em", + "opera", + "parallel_n64", + "pcsx_rearmed", + "pico8", + "picodrive", + "piemu", + "pokemini", + "potator", + "ppsspp", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "quasi88", + "quicknes", + "race", + "raze", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "smsplus", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "solarus", + "stella", + "stella2014", + "swanstation", + "tgbdual", + "theodore", + "tic80", + "uae4arm", + "uw8", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_xplus4", + "vice_xvic", + "vircon32", + "virtualjaguar", + "wasm4", + "x1", + "yabasanshiroSA" + ], + "rg552": [ + "81", + "AMIBERRY", + "AdvanceMame", + "DoubleCherryGB", + "HATARISA", + "OPENBOR", + "PPSSPPSDL", + "SCUMMVMSA", + "a5200", + "arduous", + "atari800", + "beetle_gba", + "beetle_lynx", + "beetle_ngp", + "beetle_pce", + "beetle_pce_fast", + "beetle_pcfx", + "beetle_snes", + "beetle_supafaust", + "beetle_supergrafx", + "beetle_vb", + "beetle_wswan", + "bluemsx", + "bsnes", + "bsnes2014_balanced", + "bsnes2014_performance", + "bsnes_mercury_balanced", + "bsnes_mercury_performance", + "cap32", + "crocods", + "dosbox_core", + "dosbox_pure", + "drastic", + "duckstation", + "easyrpg", + "ecwolf", + "emuscv", + "ep128emu_core", + "fake08", + "fbalpha2012", + "fbalpha2019", + "fbneo", + "fceumm", + "flycast", + "flycast2021", + "flycast2021le", + "fmsx", + "freechaf", + "freeintv", + "freej2me", + "freej2me-plus", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_wide", + "geolith", + "gpsp", + "gw", + "gzdoom", + "handy", + "hatari", + "hypseus_singe", + "jaxe", + "lowresnx", + "lzdoom", + "mame", + "mame2000", + "mame2003_plus", + "mame2010", + "mame2015", + "mame2016", + "melonds", + "melondsds", + "mesen", + "mesen-s", + "mgba", + "mojozork", + "mupen64plus_next", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "o2em", + "opera", + "parallel_n64", + "pcsx_rearmed", + "pico8", + "picodrive", + "piemu", + "pokemini", + "potator", + "ppsspp", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "quasi88", + "quicknes", + "race", + "raze", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "smsplus", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "solarus", + "stella", + "stella2014", + "swanstation", + "tgbdual", + "theodore", + "tic80", + "uae4arm", + "uw8", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_xplus4", + "vice_xvic", + "vircon32", + "virtualjaguar", + "wasm4", + "x1", + "yabasanshiroSA" + ] +} diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 891b01a95..e93abe5a7 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -868,8 +868,8 @@ platforms: # setsettings.py leaves it: saves sit beside each game, no single place. save_directory: null schedule: monthly - target_scraper: null - target_source: null + target_scraper: amberelec_targets + target_source: https://github.com/AmberELEC/AmberELEC/blob/dev/packages/amberelec/package.mk install: detect: # AmberELEC 5a3882cb scripts/image:150-152, options:17. CHAos diff --git a/platforms/targets/amberelec.yml b/platforms/targets/amberelec.yml new file mode 100644 index 000000000..1515bed13 --- /dev/null +++ b/platforms/targets/amberelec.yml @@ -0,0 +1,805 @@ +platform: amberelec +source: https://github.com/AmberELEC/AmberELEC/blob/a21975bfb1a3af68f66e07a931b994b523e1e6e2/packages/amberelec/package.mk +scraped_at: '2026-11-06T07:53:33Z' +targets: + rg351mp: + architecture: aarch64 + cores: + - '81' + - AMIBERRY + - AdvanceMame + - DoubleCherryGB + - HATARISA + - OPENBOR + - PPSSPPSDL + - SCUMMVMSA + - a5200 + - arduous + - atari800 + - beetle_gba + - beetle_lynx + - beetle_ngp + - beetle_pce + - beetle_pce_fast + - beetle_pcfx + - beetle_supafaust + - beetle_supergrafx + - beetle_vb + - beetle_wswan + - bluemsx + - cap32 + - crocods + - dosbox_core + - dosbox_pure + - drastic + - duckstation + - easyrpg + - ecwolf + - emuscv + - ep128emu_core + - fake08 + - fbalpha2012 + - fbalpha2019 + - fbneo + - fceumm + - flycast + - flycast2021 + - flycast2021le + - fmsx + - freechaf + - freeintv + - freej2me + - freej2me-plus + - fuse + - gambatte + - gearboy + - gearcoleco + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_wide + - gpsp + - gw + - gzdoom + - handy + - hatari + - hypseus_singe + - jaxe + - lowresnx + - lzdoom + - mame + - mame2000 + - mame2003_plus + - mame2010 + - mame2015 + - mame2016 + - mgba + - mojozork + - mupen64plus_next + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - o2em + - opera + - parallel_n64 + - pcsx_rearmed + - pico8 + - picodrive + - piemu + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - puae + - puae2021 + - px68k + - quasi88 + - quicknes + - race + - raze + - same_cdi + - sameboy + - sameduck + - scummvm + - smsplus + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - solarus + - stella + - stella2014 + - swanstation + - tgbdual + - theodore + - tic80 + - uae4arm + - uw8 + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_xplus4 + - vice_xvic + - vircon32 + - virtualjaguar + - wasm4 + - x1 + - yabasanshiroSA + rg351p: + architecture: aarch64 + cores: + - '81' + - AMIBERRY + - AdvanceMame + - DoubleCherryGB + - HATARISA + - OPENBOR + - PPSSPPSDL + - SCUMMVMSA + - a5200 + - arduous + - atari800 + - beetle_gba + - beetle_lynx + - beetle_ngp + - beetle_pce + - beetle_pce_fast + - beetle_pcfx + - beetle_supafaust + - beetle_supergrafx + - beetle_vb + - beetle_wswan + - bluemsx + - cap32 + - crocods + - dosbox_core + - dosbox_pure + - drastic + - duckstation + - easyrpg + - ecwolf + - emuscv + - ep128emu_core + - fake08 + - fbalpha2012 + - fbalpha2019 + - fbneo + - fceumm + - flycast + - flycast2021 + - flycast2021le + - fmsx + - freechaf + - freeintv + - freej2me + - freej2me-plus + - fuse + - gambatte + - gearboy + - gearcoleco + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_wide + - gpsp + - gw + - gzdoom + - handy + - hatari + - hypseus_singe + - jaxe + - lowresnx + - lzdoom + - mame + - mame2000 + - mame2003_plus + - mame2010 + - mame2015 + - mame2016 + - mgba + - mojozork + - mupen64plus_next + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - o2em + - opera + - parallel_n64 + - pcsx_rearmed + - pico8 + - picodrive + - piemu + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - puae + - puae2021 + - px68k + - quasi88 + - quicknes + - race + - raze + - same_cdi + - sameboy + - sameduck + - scummvm + - smsplus + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - solarus + - stella + - stella2014 + - swanstation + - tgbdual + - theodore + - tic80 + - uae4arm + - uw8 + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_xplus4 + - vice_xvic + - vircon32 + - virtualjaguar + - wasm4 + - x1 + - yabasanshiroSA + rg351v: + architecture: aarch64 + cores: + - '81' + - AMIBERRY + - AdvanceMame + - DoubleCherryGB + - HATARISA + - OPENBOR + - PPSSPPSDL + - SCUMMVMSA + - a5200 + - arduous + - atari800 + - beetle_gba + - beetle_lynx + - beetle_ngp + - beetle_pce + - beetle_pce_fast + - beetle_pcfx + - beetle_supafaust + - beetle_supergrafx + - beetle_vb + - beetle_wswan + - bluemsx + - cap32 + - crocods + - dosbox_core + - dosbox_pure + - drastic + - duckstation + - easyrpg + - ecwolf + - emuscv + - ep128emu_core + - fake08 + - fbalpha2012 + - fbalpha2019 + - fbneo + - fceumm + - flycast + - flycast2021 + - flycast2021le + - fmsx + - freechaf + - freeintv + - freej2me + - freej2me-plus + - fuse + - gambatte + - gearboy + - gearcoleco + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_wide + - gpsp + - gw + - gzdoom + - handy + - hatari + - hypseus_singe + - jaxe + - lowresnx + - lzdoom + - mame + - mame2000 + - mame2003_plus + - mame2010 + - mame2015 + - mame2016 + - mgba + - mojozork + - mupen64plus_next + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - o2em + - opera + - parallel_n64 + - pcsx_rearmed + - pico8 + - picodrive + - piemu + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - puae + - puae2021 + - px68k + - quasi88 + - quicknes + - race + - raze + - same_cdi + - sameboy + - sameduck + - scummvm + - smsplus + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - solarus + - stella + - stella2014 + - swanstation + - tgbdual + - theodore + - tic80 + - uae4arm + - uw8 + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_xplus4 + - vice_xvic + - vircon32 + - virtualjaguar + - wasm4 + - x1 + - yabasanshiroSA + rg353: + architecture: aarch64 + cores: + - '81' + - AMIBERRY + - AdvanceMame + - DoubleCherryGB + - HATARISA + - OPENBOR + - PPSSPPSDL + - SCUMMVMSA + - a5200 + - arduous + - atari800 + - beetle_gba + - beetle_lynx + - beetle_ngp + - beetle_pce + - beetle_pce_fast + - beetle_pcfx + - beetle_snes + - beetle_supafaust + - beetle_supergrafx + - beetle_vb + - beetle_wswan + - bluemsx + - bsnes + - bsnes2014_balanced + - bsnes2014_performance + - bsnes_mercury_balanced + - bsnes_mercury_performance + - cap32 + - crocods + - dosbox_core + - dosbox_pure + - drastic + - duckstation + - easyrpg + - ecwolf + - emuscv + - ep128emu_core + - fake08 + - fbalpha2012 + - fbalpha2019 + - fbneo + - fceumm + - flycast + - flycast2021 + - flycast2021le + - fmsx + - freechaf + - freeintv + - freej2me + - freej2me-plus + - fuse + - gambatte + - gearboy + - gearcoleco + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - gzdoom + - handy + - hatari + - hypseus_singe + - jaxe + - lowresnx + - lzdoom + - mame + - mame2000 + - mame2003_plus + - mame2010 + - mame2015 + - mame2016 + - melonds + - melondsds + - mesen + - mesen-s + - mgba + - mojozork + - mupen64plus_next + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - o2em + - opera + - parallel_n64 + - pcsx_rearmed + - pico8 + - picodrive + - piemu + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - puae + - puae2021 + - px68k + - quasi88 + - quicknes + - race + - raze + - same_cdi + - sameboy + - sameduck + - scummvm + - smsplus + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - solarus + - stella + - stella2014 + - swanstation + - tgbdual + - theodore + - tic80 + - uae4arm + - uw8 + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_xplus4 + - vice_xvic + - vircon32 + - virtualjaguar + - wasm4 + - x1 + - yabasanshiroSA + rg503: + architecture: aarch64 + cores: + - '81' + - AMIBERRY + - AdvanceMame + - DoubleCherryGB + - HATARISA + - OPENBOR + - PPSSPPSDL + - SCUMMVMSA + - a5200 + - arduous + - atari800 + - beetle_gba + - beetle_lynx + - beetle_ngp + - beetle_pce + - beetle_pce_fast + - beetle_pcfx + - beetle_snes + - beetle_supafaust + - beetle_supergrafx + - beetle_vb + - beetle_wswan + - bluemsx + - bsnes + - bsnes2014_balanced + - bsnes2014_performance + - bsnes_mercury_balanced + - bsnes_mercury_performance + - cap32 + - crocods + - dosbox_core + - dosbox_pure + - drastic + - duckstation + - easyrpg + - ecwolf + - emuscv + - ep128emu_core + - fake08 + - fbalpha2012 + - fbalpha2019 + - fbneo + - fceumm + - flycast + - flycast2021 + - flycast2021le + - fmsx + - freechaf + - freeintv + - freej2me + - freej2me-plus + - fuse + - gambatte + - gearboy + - gearcoleco + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - gzdoom + - handy + - hatari + - hypseus_singe + - jaxe + - lowresnx + - lzdoom + - mame + - mame2000 + - mame2003_plus + - mame2010 + - mame2015 + - mame2016 + - melonds + - melondsds + - mesen + - mesen-s + - mgba + - mojozork + - mupen64plus_next + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - o2em + - opera + - parallel_n64 + - pcsx_rearmed + - pico8 + - picodrive + - piemu + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - puae + - puae2021 + - px68k + - quasi88 + - quicknes + - race + - raze + - same_cdi + - sameboy + - sameduck + - scummvm + - smsplus + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - solarus + - stella + - stella2014 + - swanstation + - tgbdual + - theodore + - tic80 + - uae4arm + - uw8 + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_xplus4 + - vice_xvic + - vircon32 + - virtualjaguar + - wasm4 + - x1 + - yabasanshiroSA + rg552: + architecture: aarch64 + cores: + - '81' + - AMIBERRY + - AdvanceMame + - DoubleCherryGB + - HATARISA + - OPENBOR + - PPSSPPSDL + - SCUMMVMSA + - a5200 + - arduous + - atari800 + - beetle_gba + - beetle_lynx + - beetle_ngp + - beetle_pce + - beetle_pce_fast + - beetle_pcfx + - beetle_snes + - beetle_supafaust + - beetle_supergrafx + - beetle_vb + - beetle_wswan + - bluemsx + - bsnes + - bsnes2014_balanced + - bsnes2014_performance + - bsnes_mercury_balanced + - bsnes_mercury_performance + - cap32 + - crocods + - dosbox_core + - dosbox_pure + - drastic + - duckstation + - easyrpg + - ecwolf + - emuscv + - ep128emu_core + - fake08 + - fbalpha2012 + - fbalpha2019 + - fbneo + - fceumm + - flycast + - flycast2021 + - flycast2021le + - fmsx + - freechaf + - freeintv + - freej2me + - freej2me-plus + - fuse + - gambatte + - gearboy + - gearcoleco + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - gzdoom + - handy + - hatari + - hypseus_singe + - jaxe + - lowresnx + - lzdoom + - mame + - mame2000 + - mame2003_plus + - mame2010 + - mame2015 + - mame2016 + - melonds + - melondsds + - mesen + - mesen-s + - mgba + - mojozork + - mupen64plus_next + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - o2em + - opera + - parallel_n64 + - pcsx_rearmed + - pico8 + - picodrive + - piemu + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - puae + - puae2021 + - px68k + - quasi88 + - quicknes + - race + - raze + - same_cdi + - sameboy + - sameduck + - scummvm + - smsplus + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - solarus + - stella + - stella2014 + - swanstation + - tgbdual + - theodore + - tic80 + - uae4arm + - uw8 + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_xplus4 + - vice_xvic + - vircon32 + - virtualjaguar + - wasm4 + - x1 + - yabasanshiroSA diff --git a/scripts/scraper/targets/amberelec_targets_scraper.py b/scripts/scraper/targets/amberelec_targets_scraper.py new file mode 100644 index 000000000..72b3b557f --- /dev/null +++ b/scripts/scraper/targets/amberelec_targets_scraper.py @@ -0,0 +1,157 @@ +"""Scraper for AmberELEC per-device emulator availability. + +Sources (AmberELEC/AmberELEC, at the commit platforms/amberelec.yml names): + - Makefile -- one `DEVICE= ARCH= ./scripts/build_distro` target + per image (RG351P, RG351V, RG351MP, RG552, RG353, RG503 at a21975bf) + - packages/amberelec/package.mk -- LIBRETRO_CORES_EXTRA joins PKG_EMUS + only on the devices its `if` names (lines 23-31 at a21975bf: RG552, + RG353, RG503), and on every other device makeinstall_target deletes + each of those cores from es_systems.cfg (lines 45-58) + - packages/ui/emulationstation/config/es_systems.cfg -- read the way the + platform scraper reads it, after that deletion + +An RG351P image carries neither melonDS nor bsnes: without a target list, +its pack shipped the Nintendo DS firmware anyway. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from datetime import datetime, timezone + +import yaml + +from . import BaseTargetScraper + +PLATFORM_NAME = "amberelec" + +RAW_BASE = "https://raw.githubusercontent.com/AmberELEC/AmberELEC/{ref}" +META_MK = "packages/amberelec/package.mk" + +try: + from ..amberelec_scraper import BUILDS_OF, ES_SYSTEMS_TEMPLATE, RUNNERS + from ..base_scraper import fetch_bytes + from ..batocera_scraper import _not_cores + from ..es_emulators import es_cores, xml_systems +except ImportError: + from scraper.amberelec_scraper import BUILDS_OF, ES_SYSTEMS_TEMPLATE, RUNNERS + from scraper.base_scraper import fetch_bytes + from scraper.batocera_scraper import _not_cores + from scraper.es_emulators import es_cores, xml_systems + +_DEVICE_RE = re.compile( + r"^(\w+):\n\tDEVICE=(\w+) ARCH=(\w+) \./scripts/build_distro$", re.MULTILINE +) +_EXTRA_RE = re.compile(r'LIBRETRO_CORES_EXTRA="([^"]*)"') +# The `if` whose body adds the extra cores to PKG_EMUS. +_EXTRA_IF_RE = re.compile( + r"if \[\[ (?P[^\n]*?) \]\]; then\s*\n\s*PKG_EMUS\+=\" \$\{LIBRETRO_CORES_EXTRA\}\"" +) +# What makeinstall_target does to es_systems.cfg on the other devices. +_REMOVAL_LINES = ( + "for CORE in ${LIBRETRO_CORES_EXTRA}; do", + 'sed -i "s|$CORE||g"', + r"sed -i -e '//!ba;/melonds/d}'", +) +_EMULATOR_BLOCK_RE = re.compile(r"[^\n]*[^\n]*\n?", re.DOTALL) + + +def devices(makefile: str) -> dict[str, str]: + """{device: architecture} of each image the Makefile builds.""" + found = {device: arch for target, device, arch in _DEVICE_RE.findall(makefile) + if target == device} + if not found: + raise RuntimeError("the AmberELEC Makefile builds no device image") + return found + + +def extra_cores(package_mk: str) -> tuple[list[str], set[str]]: + """(the extra cores, the devices that keep them) from the metapackage.""" + listed = _EXTRA_RE.search(package_mk) + test = _EXTRA_IF_RE.search(package_mk) + if listed is None or test is None: + raise RuntimeError(f"{META_MK}: no LIBRETRO_CORES_EXTRA or no test adding it") + missing = [line for line in _REMOVAL_LINES if line not in package_mk] + if missing: + raise RuntimeError(f"{META_MK}: makeinstall_target no longer reads {missing[0]!r}") + keep = set(re.findall(r'"\$\{DEVICE\}" == "(\w+)"', test.group("test"))) + for alternatives in re.findall(r'"\$\{DEVICE\}" =~ \^\(([\w|]+)\)\$', test.group("test")): + keep.update(alternatives.split("|")) + if not keep: + raise RuntimeError(f"{META_MK}: the test adding the extra cores names no device") + return listed.group(1).split(), keep + + +def without_cores(es_systems: str, cores: list[str]) -> str: + """es_systems.cfg as makeinstall_target leaves it on a device without them.""" + for core in cores: + if core == "melonds": + es_systems = _EMULATOR_BLOCK_RE.sub( + lambda block: "" if "melonds" in block.group(0) else block.group(0), + es_systems, + ) + es_systems = es_systems.replace(f"{core}", "") + return es_systems + + +def device_cores(es_systems: str, extra: list[str], keeps_extra: bool) -> list[str]: + """Every core and standalone emulator one device's es_systems.cfg offers.""" + text = es_systems if keeps_extra else without_cores(es_systems, extra) + cores, _standalone, _dual = es_cores( + xml_systems(text), RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME) + ) + return sorted(set(cores)) + + +class Scraper(BaseTargetScraper): + """Per-device core lists from AmberELEC's metapackage.""" + + def __init__(self, url: str = "https://github.com/AmberELEC/AmberELEC") -> None: + super().__init__(url=url) + self.ref = "" + + def _raw(self, path: str) -> str: + return fetch_bytes(f"{RAW_BASE.format(ref=self.ref)}/{path}").decode("utf-8") + + def fetch_targets(self) -> dict: + self.ref = self.pinned_revision(PLATFORM_NAME) + extra, keep = extra_cores(self._raw(META_MK)) + es_systems = fetch_bytes(ES_SYSTEMS_TEMPLATE.format(ref=self.ref)).decode("utf-8") + targets: dict[str, dict] = {} + for device, arch in sorted(devices(self._raw("Makefile")).items()): + cores = device_cores(es_systems, extra, device in keep) + targets[device.lower()] = {"architecture": arch, "cores": cores} + print(f" {device.lower()} ({arch}): {len(cores)} cores", file=sys.stderr) + return { + "platform": PLATFORM_NAME, + "source": f"{self.url}/blob/{self.ref}/{META_MK}", + "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "targets": targets, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + allow_abbrev=False, description="Scrape AmberELEC per-device cores" + ) + parser.add_argument("--dry-run", action="store_true", help="Show target summary") + parser.add_argument("--output", "-o", help="Output YAML file") + args = parser.parse_args() + + scraper = Scraper() + data = scraper.fetch_targets() + if args.dry_run: + for name, info in data["targets"].items(): + print(f" {name} ({info['architecture']}): {len(info['cores'])} cores") + return + if args.output: + scraper.write_output(data, args.output) + print(f"Written to {args.output}") + return + print(yaml.dump(data, default_flow_style=False, sort_keys=False)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_amberelec_targets.py b/tests/test_amberelec_targets.py new file mode 100644 index 000000000..3724fd0d5 --- /dev/null +++ b/tests/test_amberelec_targets.py @@ -0,0 +1,102 @@ +"""AmberELEC's images differ by device, and only its metapackage says how. + +packages/amberelec/package.mk adds LIBRETRO_CORES_EXTRA (melonDS, bsnes, +Mesen, geolith) on RG552, RG353 and RG503 and deletes those cores from +es_systems.cfg everywhere else. With no target list, an RG351P pack carried +the firmware of cores its image never had. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from scraper.targets import amberelec_targets_scraper as scraper # noqa: E402 + +MAKEFILE = """RG351P: +\tDEVICE=RG351P ARCH=aarch64 ./scripts/build_distro + +RG552: +\tDEVICE=RG552 ARCH=aarch64 ./scripts/build_distro + +toolchain-RK3326: +\tDEVICE=RK3326 ARCH=aarch64 ./scripts/build toolchain +""" + +PACKAGE_MK = """ LIBRETRO_CORES_EXTRA="bsnes melonds" + + PKG_EMUS="${LIBRETRO_CORES}" + + if [[ "${DEVICE}" == "RG552" || "${DEVICE}" =~ ^(RG353|RG503)$ ]]; then + PKG_EMUS+=" ${LIBRETRO_CORES_EXTRA}" + fi + +makeinstall_target() { + for CORE in ${LIBRETRO_CORES_EXTRA}; do + if [[ $CORE == "melonds" ]]; then + sed -i -e '//!ba;/melonds/d}' es_systems.cfg + fi + sed -i "s|$CORE||g" es_systems.cfg + done +} +""" + +ES_SYSTEMS = """ + + snes + + + + snes9x + bsnes + + + + + + nds + + + + melonds + + + + + drastic + + + + + +""" + + +class TheMetapackageDecidesEachDevice(unittest.TestCase): + def test_devices_are_the_images_the_makefile_builds(self): + self.assertEqual(scraper.devices(MAKEFILE), {"RG351P": "aarch64", "RG552": "aarch64"}) + + def test_the_extra_cores_and_their_devices(self): + extra, keep = scraper.extra_cores(PACKAGE_MK) + self.assertEqual(extra, ["bsnes", "melonds"]) + self.assertEqual(keep, {"RG552", "RG353", "RG503"}) + + def test_a_device_without_them_loses_their_cores(self): + extra, _keep = scraper.extra_cores(PACKAGE_MK) + full = scraper.device_cores(ES_SYSTEMS, extra, True) + lean = scraper.device_cores(ES_SYSTEMS, extra, False) + self.assertEqual(set(full) - set(lean), {"bsnes", "melonds"}) + self.assertIn("drastic", lean) + + def test_a_changed_removal_stops_the_scrape(self): + changed = PACKAGE_MK.replace('sed -i "s|$CORE||g"', "true") + with self.assertRaises(RuntimeError): + scraper.extra_cores(changed) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_install.py b/tests/test_install.py index f36389a1e..0ea26705f 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3420,7 +3420,8 @@ class ACardPreparedOnAComputerGetsItsBoard(unittest.TestCase): class APlatformWithoutTargetsSaysSoOffline(unittest.TestCase): """Offline, --target on AmberELEC asked for install/targets/amberelec.json, - which exists nowhere: online the same run says it has no targets.""" + which existed nowhere: online the same run said it had no targets. + EmuELEC has none still.""" def test_the_list_matches_the_published_targets(self): published = {path.stem for path in (REPO_ROOT / "install" / "targets").glob("*.json")} @@ -3428,16 +3429,16 @@ class APlatformWithoutTargetsSaysSoOffline(unittest.TestCase): def test_offline_says_there_are_no_targets(self): dest = Path(_tempdir(self)) - for extra in ([], ["--manifest", str(REPO_ROOT / "install" / "amberelec.json")]): + for extra in ([], ["--manifest", str(REPO_ROOT / "install" / "emuelec.json")]): with self.subTest(manifest=bool(extra)): proc = subprocess.run( - [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "amberelec", + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "emuelec", "--dest", str(dest), "--target", "rk3326", "--offline", "--no-input", "--dry-run", *extra], capture_output=True, text=True, timeout=120, stdin=subprocess.DEVNULL) self.assertEqual(proc.returncode, 2, proc.stderr) - self.assertIn("AmberELEC has no hardware targets", proc.stderr) - self.assertNotIn("targets/amberelec.json", proc.stderr) + self.assertIn("EmuELEC has no hardware targets", proc.stderr) + self.assertNotIn("targets/emuelec.json", proc.stderr) class TheBoardTheSystemNamesIsTheTarget(unittest.TestCase): @@ -3492,6 +3493,17 @@ class TheBoardTheSystemNamesIsTheTarget(unittest.TestCase): .read_text(encoding="utf-8")) self.assertLessEqual({*install._RPI_MODELS.values(), "x86_64"}, set(targets)) + def test_amberelec_names_its_device(self): + # An RG351P image carries no melonDS or bsnes, and the full list was + # installed: AmberELEC had no target list to read the device against. + with unittest.mock.patch.object( + install, "_parse_os_release", return_value={"COREELEC_DEVICE": "RG351P"} + ): + self.assertEqual(install.detect_board("amberelec"), "rg351p") + targets = json.loads((REPO_ROOT / "install" / "targets" / "amberelec.json") + .read_text(encoding="utf-8")) + self.assertIn("rg351p", targets) + def test_lakka_names_its_device_or_its_project(self): # An RPi4 Lakka installed 5.7 GB of every board's files: its board # was in os-release and never read. diff --git a/wiki/tools.md b/wiki/tools.md index dab60e5b0..a42c26adb 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1701,6 +1701,7 @@ Located in `scripts/scraper/targets/`. Each inherits `BaseTargetScraper` and imp | `knulli_targets_scraper` | the same, read from KNULLI's tree and Buildroot fork | board | | `recalbox_targets_scraper` | board defconfigs + system.ini cores | board | | `rocknix_targets_scraper` | emulators package.mk + device options | device | +| `amberelec_targets_scraper` | Makefile devices + metapackage extra cores + es_systems.cfg | device | | `lakka_targets_scraper` | libretro_cores package.mk + project and device options | device image | | `emudeck_targets_scraper` | EmuScripts (SteamOS and Windows) | operating system | | `retropie_targets_scraper` | scriptmodules + rp_module_flags | platform flag | From f9f7ad3b922a1a39edc75f38a6edc6b748c1bed2 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:46:37 +0100 Subject: [PATCH 14/15] docs: quote the bundled line in the verify excerpt reading-reports quotes a RetroDECK run, which now prints the bundled count and 30 missing game data instead of 38. The comment introducing the bundled names also loses its semicolon. --- scripts/verify.py | 2 +- wiki/reading-reports.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/verify.py b/scripts/verify.py index b12991d90..f63d70d4a 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1887,7 +1887,7 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: for u in opt_not_in_repo: _print_undeclared_entry(u, "MISSING (optional)", verbose, repeated) if bundled: - # The required firmware is named, as a missing one would be; the + # The required firmware is named, as a missing one would be. The # rest (shaders, runtime libraries, engine data) on request. shown = bundled if verbose else [ u for u in bundled diff --git a/wiki/reading-reports.md b/wiki/reading-reports.md index 09345c90c..a3bfd6765 100644 --- a/wiki/reading-reports.md +++ b/wiki/reading-reports.md @@ -110,7 +110,8 @@ RetroDECK: 2007/2008 OK, 1 untested [md5] Core files: 1764 in pack, 1 required missing, 26 optional missing (--verbose lists them) Core files held but not packed: 154 (142 installed with its emulator, 8 read below a save folder per system and no system runs the core there, 4 a packed file occupies its directory) MISSING (required): FinalBurn Neo needs sfiii2a.zip (1 file) - Game data: 5099 in pack, 38 missing (--verbose lists them) + Bundled with their emulator, not collected: 20 + Game data: 5099 in pack, 30 missing (--verbose lists them) Game data held but not packed: 30 (installed with its emulator) No external files: 78 cores (--verbose lists them) Ground truth: 97/2008 files have emulator validation (5%) From 3672c6de4a7bd8ce18eee84f82c97a2414b5c8fb Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:46:37 +0100 Subject: [PATCH 15/15] docs: list the AmberELEC target scraper --- wiki/adding-a-scraper.md | 1 + wiki/testing-guide.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/wiki/adding-a-scraper.md b/wiki/adding-a-scraper.md index df8db46d0..7dc29ef32 100644 --- a/wiki/adding-a-scraper.md +++ b/wiki/adding-a-scraper.md @@ -338,6 +338,7 @@ Register the target scraper in `_registry.yml`: | `lakka_targets` | Lakka-LibreELEC `package.mk` and project options | Core list minus each project and device's exclusions | | `recalbox_targets` | board defconfigs + `system.ini` | Buildroot packages each image enables, cores per system | | `rocknix_targets` | emulators `package.mk` + device options | Cores per SoC block and 32-bit support per device | +| `amberelec_targets` | Makefile devices + metapackage `package.mk` + `es_systems.cfg` | Extra cores kept only on the devices the metapackage names | ## Parser modules diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 3ef512d15..27866d49b 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -54,7 +54,7 @@ Every module in `tests/`, by the area it guards. | Verification, cross-reference and profile checks | synthetic + profiles | `test_bios_layout`, `test_cache_independent_counts`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_names`, `test_profile_paths`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | | Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_core_data_trees`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | -| Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | +| Hardware targets and systems | synthetic + target files | `test_amberelec_targets`, `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_games`, `test_install_offline`, `test_install_selection`, `test_install_tui` | | Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` | | Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` |