diff --git a/emulators/freeintv_ts_overlay.yml b/emulators/freeintv_ts_overlay.yml index 9435413b..f95d5bca 100644 --- a/emulators/freeintv_ts_overlay.yml +++ b/emulators/freeintv_ts_overlay.yml @@ -31,6 +31,7 @@ bios_identical_to: freeintv data_directories: - ref: freeintv-overlays destination: freeintv_overlays + required: false files: - name: "exec.bin" diff --git a/platforms/retroarch.yml b/platforms/retroarch.yml index 50e32746..7bdc9399 100644 --- a/platforms/retroarch.yml +++ b/platforms/retroarch.yml @@ -1234,6 +1234,7 @@ systems: data_directories: - ref: freeintv-overlays destination: freeintv_overlays + required: false microsoft-msx: files: - name: CARTS.SHA diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 4adcd508..c53b6bdf 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -279,6 +279,10 @@ "destination": { "type": "string" }, + "required": { + "type": "boolean", + "description": "false when the core runs without the tree; --required-only leaves it out" + }, "mode": { "type": "string", "enum": [ diff --git a/schemas/platform.schema.json b/schemas/platform.schema.json index 8a5dda57..f45be8a6 100644 --- a/schemas/platform.schema.json +++ b/schemas/platform.schema.json @@ -297,6 +297,10 @@ }, "destination": { "type": "string" + }, + "required": { + "type": "boolean", + "description": "false when the core runs without the tree; --required-only leaves it out" } } } diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index fa5533ff..ff3cc9ce 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -383,6 +383,7 @@ def _data_directory_members( seen_lower: set, seen_parents: set, refs_out: dict | None = None, + required_only: bool = False, ) -> Iterator[tuple[str, str]]: """Yield (source, destination) for the cached data directories a platform's systems declare. @@ -394,6 +395,8 @@ def _data_directory_members( trees an emulator reads, refreshed from upstream, so what ships is whatever the cache holds. The pack and the install manifest both read this, the first to write the files and the second to count them. + ``required_only`` leaves out a tree declared ``required: false``, one the + core runs without, as it leaves out such a file. """ # Data directories from _data_dirs.yml for sys_id, system in sorted(pack_systems.items()): @@ -401,6 +404,8 @@ def _data_directory_members( ref_key = dd.get("ref", "") if not ref_key or not data_registry or ref_key not in data_registry: continue + if required_only and dd.get("required") is False: + continue if ref_key in installed_data_refs(platform_name): continue entry = data_registry[ref_key] @@ -455,12 +460,14 @@ def _pack_data_directories( seen_destinations: set, seen_lower: set, seen_parents: set, + required_only: bool = False, ) -> int: """Add the data directories to the pack. Returns how many files went in.""" added = 0 for src, full in _data_directory_members( pack_systems, data_registry, platform_name, base_dest, case_insensitive, seen_destinations, seen_lower, seen_parents, + required_only=required_only, ): _add_pack_member(zf, src, _flat(full, base_dest, flatten)) added += 1 @@ -1142,7 +1149,7 @@ def generate_pack( total_files += _pack_data_directories( zf, pack_systems, data_registry, platform_name, base_dest, flatten, case_insensitive, seen_destinations, seen_lower, - seen_parents, + seen_parents, required_only=required_only, ) # README.txt for users: personalized step-by-step per platform @@ -1682,6 +1689,8 @@ def generate_emulator_pack( for dd in deliverable( profile.get("data_directories", []), standalone ): + if required_only and dd.get("required") is False: + continue ref_key = dd.get("ref", "") if not ref_key or not data_registry or ref_key not in data_registry: if ref_key: @@ -4264,7 +4273,7 @@ def generate_manifest( data_members = list(_data_directory_members( pack_systems, data_registry, platform_name, base_dest, case_insensitive, seen_destinations, seen_lower, seen_parents, - data_refs, + data_refs, required_only=required_only, )) data_sizes = pack_only_sizes + [os.path.getsize(src) for src, _dest in data_members] data_directories = _manifest_data_directories( diff --git a/scripts/scraper/libretro_scraper.py b/scripts/scraper/libretro_scraper.py index 1e2a8c59..74571c11 100644 --- a/scripts/scraper/libretro_scraper.py +++ b/scripts/scraper/libretro_scraper.py @@ -495,8 +495,15 @@ class Scraper(BaseScraper): # so final path is system/freeintv_overlays/FreeIntvTS_Overlays/.png # but core expects system/freeintv_overlays/.png # fix: point destination into the subfolder + # Optional: read only with the multi-screen option on, and a + # missing PNG falls back to a drawn placeholder keypad + # (FreeIntv/src/libretro.c:294-296, 350-369). "mattel-intellivision": [ - {"ref": "freeintv-overlays", "destination": "freeintv_overlays"}, + { + "ref": "freeintv-overlays", + "destination": "freeintv_overlays", + "required": False, + }, ], } for sys_id, data_dirs in SYSTEM_DATA_DIRS.items(): diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 2c500274..38750698 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5794,5 +5794,59 @@ class AFileTheLauncherNeverOpens(unittest.TestCase): self.assertIn('unread_on(f, config.get("_key", ""))', source) +class RequiredOnlyReachesDataDirectories(unittest.TestCase): + """--required-only filtered files and packed every data tree anyway: the + Required RetroArch pack still carried the 90 FreeIntv overlays, which the + core reads only with its multi-screen option and replaces by a drawn + keypad when absent (FreeIntv src/libretro.c:294-296, 350-369).""" + + def _members(self, required_only: bool) -> list[str]: + from scripts import generate_pack as gp + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp: + for key in ("needed", "optional"): + Path(tmp, key).mkdir() + Path(tmp, key, f"{key}.bin").write_bytes(b"x") + registry = { + key: {"local_cache": str(Path(tmp, key))} + for key in ("needed", "optional") + } + systems = { + "sys": { + "data_directories": [ + {"ref": "needed", "destination": "a"}, + {"ref": "optional", "destination": "b", "required": False}, + ] + } + } + return [ + dest + for _src, dest in gp._data_directory_members( + systems, registry, "retroarch", "", False, set(), set(), set(), + required_only=required_only, + ) + ] + + def test_an_optional_tree_leaves_the_required_pack(self): + self.assertEqual(self._members(False), ["a/needed.bin", "b/optional.bin"]) + self.assertEqual(self._members(True), ["a/needed.bin"]) + + def test_every_tree_walk_reads_the_flag(self): + source = (ROOT / "scripts" / "generate_pack.py").read_text(encoding="utf-8") + self.assertEqual( + source.count('if required_only and dd.get("required") is False:'), 2 + ) + self.assertIn("seen_parents, required_only=required_only,", source) + self.assertIn("data_refs, required_only=required_only,", source) + + def test_the_overlays_are_declared_optional(self): + retroarch = yaml.safe_load((ROOT / "platforms" / "retroarch.yml").read_text()) + trees = retroarch["systems"]["mattel-intellivision"]["data_directories"] + self.assertEqual(trees, [ + {"ref": "freeintv-overlays", "destination": "freeintv_overlays", + "required": False}, + ]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index d0d17163..34d87d1a 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -493,7 +493,7 @@ which CI validates every profile against. | `mode` | no | default mode: `standalone`, `libretro`, or `both` | | `verification` | no | how the core verifies BIOS: `existence`, `md5`, `sha1`, `crc32` | | `files` | yes, unless `type: alias` | list of file entries | -| `data_directories` | no | whole directory trees the core needs, referencing `_data_dirs.yml` keys. An entry may carry `mode:` when only one build reads the tree | +| `data_directories` | no | whole directory trees the core needs, referencing `_data_dirs.yml` keys. An entry may carry `mode:` when only one build reads the tree, and `required: false` when the core runs without it (`--required-only` then leaves it out) | | `notes` | no | free-form technical notes | | `note` | no | single-paragraph variant of `notes` | | `exclusion_note` | no | why the profile has no files despite .info declaring firmware | diff --git a/wiki/tools.md b/wiki/tools.md index ebe0395e..709cfa06 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -213,7 +213,8 @@ would have loaded. ### Granular options - `--system` with `--platform`: filter to specific systems within a platform pack -- `--required-only`: exclude optional files, keep only required +- `--required-only`: exclude optional files and the data directories declared + `required: false`, keep only required - `--split`: generate one ZIP per system instead of one big pack - `--split --group-by manufacturer`: group split packs by manufacturer (Sony, Nintendo, Sega...) - `--from-md5`: look up a hash in the database, or build a custom pack with `--platform`/`--emulator`