From 53f9746bfa6f02a87a0592a27d1839f989ebb076 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:44:12 +0200 Subject: [PATCH] fix: count a core file only where it is packed verify kept its own copy of the placement rule and missed the flat RomM folder, so the report still counted 543 unreachable files as packed. It now asks the builder's ExtraPlacement. The README fraction counted every held core file, including those a platform cannot place or that a same-named packed file hides. It now counts what the pack holds. --- README.md | 20 ++++++------- scripts/generate_readme.py | 6 +++- scripts/packextras.py | 54 +++++++++++++++++++++-------------- scripts/verify.py | 13 +++++++-- tests/test_unplaced_extras.py | 25 +++++++++++++++- 5 files changed, 83 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index f7d83aaf..f88e300e 100644 --- a/README.md +++ b/README.md @@ -87,17 +87,17 @@ Full list with per-file details: **[https://abdess.github.io/retrobios/](https:/ | Platform | On its BIOS list | Files its emulators load | Checked by | |----------|-----------------:|-------------------------:|------------| | Batocera | 353/353 | 12,354/12,788 | MD5 hash | -| BizHawk | 118/118 | 755/756 | SHA1 hash | -| EmuDeck | 168/168 | 5,810/5,840 | MD5 hash | -| Lakka | 532/532 | 5,267/5,297 | file presence | +| BizHawk | 118/118 | 754/756 | SHA1 hash | +| EmuDeck | 168/168 | 5,804/5,840 | MD5 hash | +| Lakka | 532/532 | 5,265/5,297 | file presence | | MiSTer FPGA | 81/81 | - | MD5 hash | -| ROCKNIX | 38/38 | 5,926/6,003 | MD5 hash | +| ROCKNIX | 38/38 | 5,920/6,003 | MD5 hash | | Recalbox | 351/351 | 2,788/2,806 | MD5 hash | -| RetroArch | 526/526 | 5,279/5,309 | file presence | -| RetroBat | 348/348 | 7,258/7,417 | MD5 hash | -| RetroDECK | 2,008/2,008 | 5,327/5,377 | MD5 hash | -| RetroPie * | 532/532 | 9,197/9,339 | file presence | -| RomM | 383/383 | 774/781 | MD5 hash | +| RetroArch | 526/526 | 5,277/5,309 | file presence | +| RetroBat | 348/348 | 7,255/7,417 | MD5 hash | +| RetroDECK | 2,008/2,008 | 5,323/5,377 | MD5 hash | +| RetroPie * | 532/532 | 9,195/9,339 | file presence | +| RomM | 383/383 | 61/788 | MD5 hash | Each fraction is what the pack has over what is needed, counting required and optional files alike since both ship. The first column is the BIOS list the platform publishes. The second counts files its emulators load that this list never mentions, found by reading their source code, and it is routinely several times larger. A short fraction means files are still missing, and they are named in the [gap analysis](https://abdess.github.io/retrobios/gaps/). That second number is a floor, not a ceiling: an emulator that accepts any file handed to it names none in its code, so nothing there can be counted. @@ -180,4 +180,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in the [FAQ](https://abdess.github.io/retrobios/wiki/faq/#is-this-legal). -*Auto-generated on 2026-10-19T02:14:14Z* +*Auto-generated on 2026-10-19T05:42:39Z* diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index ab8634d7..c419b8dd 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -63,7 +63,11 @@ def compute_coverage( present = ok + untested pct = (present / total * 100) if total > 0 else 0 undeclared = result.get("undeclared_files", []) - core_present = sum(1 for u in undeclared if u.get("in_repo")) + # Held is not packed: a file the platform has no folder for, or one below + # a folder it lists flat, stays out of the pack the fraction describes. + core_present = sum( + 1 for u in undeclared if u.get("in_repo") and u.get("in_pack", True) + ) core_missing = len(undeclared) - core_present # Files a profile marks unsourceable never reach the undeclared list, so # the gap they represent has to be counted back in. diff --git a/scripts/packextras.py b/scripts/packextras.py index 2879ea12..ec8a4a9c 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -224,27 +224,41 @@ def hidden_extras( return hidden -def unplaceable_extras(config: dict, undeclared: list[dict], profiles: dict) -> set: - """Keys of the report entries a slug-based platform cannot place. +def _extra_dest(u: dict) -> str: + """Where a report entry lands before any platform prefix. + + An archive entry goes by the archive's name, and a directory path takes + the file's name after it ("cafeLibs/" + "snd_user.rpl"). + """ + archive = u.get("archive") + raw = archive if archive else (u.get("path") or u["name"]) + return f"{raw}{u['name']}" if raw.endswith("/") else raw + + +def unplaceable_extras( + config: dict, undeclared: list[dict], profiles: dict +) -> dict[tuple[str, str, str], str]: + """The report entries a slug-based platform cannot place, with the reason. RomM files every BIOS under the slug of its system and knows no slug - for DOS or for the arcade sets MAME 2003 reads. The builder skipped - those entries and the report counted them in the pack. + for DOS or for the arcade sets MAME 2003 reads, and it lists each folder + flat. The builder skipped those entries and the report counted them in + the pack. The answer comes from the builder's own placement. """ - from common import _norm_system_id - - is_slug_based, sys_to_slug = _detect_slug_structure(config) - if not is_slug_based: - return set() - platform_systems = set(config.get("systems", {}).keys()) - norm_map = {_norm_system_id(sid): sid for sid in platform_systems} - native = native_slug_systems(config) - return { - (u.get("emulator", ""), u.get("name", ""), u.get("path") or "") - for u in undeclared - if u.get("in_repo") - and not _slug_for(u, profiles, platform_systems, norm_map, sys_to_slug, native) - } + placement = ExtraPlacement(config, "", profiles) + if not placement.slug_based: + return {} + unplaced: dict[tuple[str, str, str], str] = {} + for u in undeclared: + if not u.get("in_repo"): + continue + dest = _extra_dest(u) + if placement.place(u, dest) is None: + key = (u.get("emulator", ""), u.get("name", ""), u.get("path") or "") + unplaced[key] = ( + "below_platform_folder" if placement.nested(dest) else "no_platform_slug" + ) + return unplaced def _agnostic_scan_extras( @@ -795,9 +809,7 @@ def _collect_emulator_extras( # For archive entries, use the archive name for resolution archive = u.get("archive") name = archive if archive else u["name"] - raw_dest = archive if archive else (u.get("path") or u["name"]) - # Directory path: append filename (e.g. "cafeLibs/" + "snd_user.rpl") - dest = f"{raw_dest}{u['name']}" if raw_dest.endswith("/") else raw_dest + dest = _extra_dest(u) # Slug-based platforms: prefix dest with system slug placed = placement.place(u, dest) diff --git a/scripts/verify.py b/scripts/verify.py index ac180d8b..9c8c6c9a 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -762,15 +762,24 @@ def find_exclusion_notes( # Platform verification +_UNPLACED_REASONS = { + "no_platform_slug": "no platform slug for its system", + "below_platform_folder": "below the folder the platform lists", +} + + def _mark_unplaced(config: dict, undeclared: list[dict], profiles: dict) -> None: """What the builder cannot place is not in the pack, held or not.""" from packextras import unplaceable_extras unplaced = unplaceable_extras(config, undeclared, profiles) for u in undeclared: - if (u.get("emulator", ""), u.get("name", ""), u.get("path") or "") in unplaced: + reason = unplaced.get( + (u.get("emulator", ""), u.get("name", ""), u.get("path") or "") + ) + if reason: u["in_pack"] = False - u["omitted"] = "no platform slug for its system" + u["omitted"] = _UNPLACED_REASONS[reason] _HIDDEN_REASONS = { diff --git a/tests/test_unplaced_extras.py b/tests/test_unplaced_extras.py index 01e34014..46c5037d 100644 --- a/tests/test_unplaced_extras.py +++ b/tests/test_unplaced_extras.py @@ -92,6 +92,17 @@ class SlugPlatform(unittest.TestCase): self.assertTrue(entry["in_repo"]) self.assertFalse(entry.get("in_pack", True)) + def test_the_readme_fraction_does_not_count_it(self): + # The README's second column is what the pack has: a held file the + # platform has no folder for was counted there. + import generate_readme + + coverage = generate_readme.compute_coverage( + "slugs", "platforms", self.db, emulators_dir="emulators" + ) + self.assertEqual(coverage["core_present"], 0) + self.assertEqual(coverage["core_missing"], 1) + def test_the_manifest_says_why_it_is_absent(self): manifest = builder.generate_manifest( "slugs", "platforms", self.db, "bios", "platforms/_registry.yml", @@ -148,7 +159,7 @@ class FlatFirmwareFolder(unittest.TestCase): " - name: deep.bin\n path: core/bios/deep.bin\n required: false\n" ) - def _manifest(self, flat: bool) -> dict: + def _manifest(self, flat: bool, report: bool = False) -> dict: with tempfile.TemporaryDirectory() as tmp: previous = os.getcwd() os.chdir(tmp) @@ -188,6 +199,11 @@ class FlatFirmwareFolder(unittest.TestCase): common._platform_config_cache.clear() common._emulator_profiles_cache.clear() profiles = common.load_emulator_profiles("emulators") + if report: + return verify_platform( + common.load_platform_config("flat", "platforms"), db, + "emulators", profiles, + ) return builder.generate_manifest( "flat", "platforms", db, "bios", "platforms/_registry.yml", emulators_dir="emulators", emu_profiles=profiles, offline=True, @@ -205,6 +221,13 @@ class FlatFirmwareFolder(unittest.TestCase): reasons = {o["name"]: o["reason"] for o in manifest["omitted_files"]} self.assertEqual(reasons.get("deep.bin"), "below_platform_folder") + def test_the_report_agrees_with_the_builder(self): + report = self._manifest(flat=True, report=True) + entries = {u["name"]: u for u in report["undeclared_files"]} + self.assertFalse(entries["deep.bin"].get("in_pack", True)) + self.assertEqual(entries["deep.bin"]["omitted"], "below the folder the platform lists") + self.assertTrue(entries["flat.bin"].get("in_pack", True)) + def test_a_folder_read_recursively_keeps_it(self): dests = {entry["dest"] for entry in self._manifest(flat=False)["files"]} self.assertIn("psx/core/bios/deep.bin", dests)