fix: apply --required-only to data directories

The Required packs and manifests carried every data tree, the FreeIntv
overlays included. A data_directories entry now accepts required: false,
and the platform, emulator and manifest walks leave such a tree out under
--required-only. FreeIntv reads its overlays only with the multi-screen
option and draws a placeholder keypad without one (src/libretro.c:294-296,
350-369).

emulators/freeintv_ts_overlay.yml

@@ -31,6 +31,7 @@ bios_identical_to: freeintv
 data_directories:
   - ref: freeintv-overlays
     destination: freeintv_overlays
+    required: false
 
 files:
   - name: "exec.bin"

platforms/retroarch.yml

@@ -1234,6 +1234,7 @@ systems:
     data_directories:
     - ref: freeintv-overlays
       destination: freeintv_overlays
+      required: false
   microsoft-msx:
     files:
     - name: CARTS.SHA

schemas/emulator.schema.json

@@ -279,6 +279,10 @@
           "destination": {
             "type": "string"
           },
+          "required": {
+            "type": "boolean",
+            "description": "false when the core runs without the tree; --required-only leaves it out"
+          },
           "mode": {
             "type": "string",
             "enum": [

schemas/platform.schema.json

@@ -297,6 +297,10 @@
               },
               "destination": {
                 "type": "string"
+              },
+              "required": {
+                "type": "boolean",
+                "description": "false when the core runs without the tree; --required-only leaves it out"
               }
             }
           }

scripts/generate_pack.py

@@ -383,6 +383,7 @@ def _data_directory_members(
     seen_lower: set,
     seen_parents: set,
     refs_out: dict | None = None,
+    required_only: bool = False,
 ) -> Iterator[tuple[str, str]]:
     """Yield (source, destination) for the cached data directories a
     platform's systems declare.
@@ -394,6 +395,8 @@ def _data_directory_members(
     trees an emulator reads, refreshed from upstream, so what ships is
     whatever the cache holds. The pack and the install manifest both read
     this, the first to write the files and the second to count them.
+    ``required_only`` leaves out a tree declared ``required: false``, one the
+    core runs without, as it leaves out such a file.
     """
     # Data directories from _data_dirs.yml
     for sys_id, system in sorted(pack_systems.items()):
@@ -401,6 +404,8 @@ def _data_directory_members(
             ref_key = dd.get("ref", "")
             if not ref_key or not data_registry or ref_key not in data_registry:
                 continue
+            if required_only and dd.get("required") is False:
+                continue
             if ref_key in installed_data_refs(platform_name):
                 continue
             entry = data_registry[ref_key]
@@ -455,12 +460,14 @@ def _pack_data_directories(
     seen_destinations: set,
     seen_lower: set,
     seen_parents: set,
+    required_only: bool = False,
 ) -> int:
     """Add the data directories to the pack. Returns how many files went in."""
     added = 0
     for src, full in _data_directory_members(
         pack_systems, data_registry, platform_name, base_dest,
         case_insensitive, seen_destinations, seen_lower, seen_parents,
+        required_only=required_only,
     ):
         _add_pack_member(zf, src, _flat(full, base_dest, flatten))
         added += 1
@@ -1142,7 +1149,7 @@ def generate_pack(
       total_files += _pack_data_directories(
           zf, pack_systems, data_registry, platform_name, base_dest,
           flatten, case_insensitive, seen_destinations, seen_lower,
-          seen_parents,
+          seen_parents, required_only=required_only,
       )
 
       # README.txt for users: personalized step-by-step per platform
@@ -1682,6 +1689,8 @@ def generate_emulator_pack(
             for dd in deliverable(
                 profile.get("data_directories", []), standalone
             ):
+                if required_only and dd.get("required") is False:
+                    continue
                 ref_key = dd.get("ref", "")
                 if not ref_key or not data_registry or ref_key not in data_registry:
                     if ref_key:
@@ -4264,7 +4273,7 @@ def generate_manifest(
     data_members = list(_data_directory_members(
         pack_systems, data_registry, platform_name, base_dest,
         case_insensitive, seen_destinations, seen_lower, seen_parents,
-        data_refs,
+        data_refs, required_only=required_only,
     ))
     data_sizes = pack_only_sizes + [os.path.getsize(src) for src, _dest in data_members]
     data_directories = _manifest_data_directories(

scripts/scraper/libretro_scraper.py

@@ -495,8 +495,15 @@ class Scraper(BaseScraper):
             # so final path is system/freeintv_overlays/FreeIntvTS_Overlays/<rom>.png
             # but core expects system/freeintv_overlays/<rom>.png
             # fix: point destination into the subfolder
+            # Optional: read only with the multi-screen option on, and a
+            # missing PNG falls back to a drawn placeholder keypad
+            # (FreeIntv/src/libretro.c:294-296, 350-369).
             "mattel-intellivision": [
-                {"ref": "freeintv-overlays", "destination": "freeintv_overlays"},
+                {
+                    "ref": "freeintv-overlays",
+                    "destination": "freeintv_overlays",
+                    "required": False,
+                },
             ],
         }
         for sys_id, data_dirs in SYSTEM_DATA_DIRS.items():

tests/test_audit_regressions.py

@@ -5794,5 +5794,59 @@ class AFileTheLauncherNeverOpens(unittest.TestCase):
         self.assertIn('unread_on(f, config.get("_key", ""))', source)
 
 
+class RequiredOnlyReachesDataDirectories(unittest.TestCase):
+    """--required-only filtered files and packed every data tree anyway: the
+    Required RetroArch pack still carried the 90 FreeIntv overlays, which the
+    core reads only with its multi-screen option and replaces by a drawn
+    keypad when absent (FreeIntv src/libretro.c:294-296, 350-369)."""
+
+    def _members(self, required_only: bool) -> list[str]:
+        from scripts import generate_pack as gp
+
+        with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp:
+            for key in ("needed", "optional"):
+                Path(tmp, key).mkdir()
+                Path(tmp, key, f"{key}.bin").write_bytes(b"x")
+            registry = {
+                key: {"local_cache": str(Path(tmp, key))}
+                for key in ("needed", "optional")
+            }
+            systems = {
+                "sys": {
+                    "data_directories": [
+                        {"ref": "needed", "destination": "a"},
+                        {"ref": "optional", "destination": "b", "required": False},
+                    ]
+                }
+            }
+            return [
+                dest
+                for _src, dest in gp._data_directory_members(
+                    systems, registry, "retroarch", "", False, set(), set(), set(),
+                    required_only=required_only,
+                )
+            ]
+
+    def test_an_optional_tree_leaves_the_required_pack(self):
+        self.assertEqual(self._members(False), ["a/needed.bin", "b/optional.bin"])
+        self.assertEqual(self._members(True), ["a/needed.bin"])
+
+    def test_every_tree_walk_reads_the_flag(self):
+        source = (ROOT / "scripts" / "generate_pack.py").read_text(encoding="utf-8")
+        self.assertEqual(
+            source.count('if required_only and dd.get("required") is False:'), 2
+        )
+        self.assertIn("seen_parents, required_only=required_only,", source)
+        self.assertIn("data_refs, required_only=required_only,", source)
+
+    def test_the_overlays_are_declared_optional(self):
+        retroarch = yaml.safe_load((ROOT / "platforms" / "retroarch.yml").read_text())
+        trees = retroarch["systems"]["mattel-intellivision"]["data_directories"]
+        self.assertEqual(trees, [
+            {"ref": "freeintv-overlays", "destination": "freeintv_overlays",
+             "required": False},
+        ])
+
+
 if __name__ == "__main__":
     unittest.main()

wiki/profiling.md

@@ -493,7 +493,7 @@ which CI validates every profile against.
 | `mode` | no | default mode: `standalone`, `libretro`, or `both` |
 | `verification` | no | how the core verifies BIOS: `existence`, `md5`, `sha1`, `crc32` |
 | `files` | yes, unless `type: alias` | list of file entries |
-| `data_directories` | no | whole directory trees the core needs, referencing `_data_dirs.yml` keys. An entry may carry `mode:` when only one build reads the tree |
+| `data_directories` | no | whole directory trees the core needs, referencing `_data_dirs.yml` keys. An entry may carry `mode:` when only one build reads the tree, and `required: false` when the core runs without it (`--required-only` then leaves it out) |
 | `notes` | no | free-form technical notes |
 | `note` | no | single-paragraph variant of `notes` |
 | `exclusion_note` | no | why the profile has no files despite .info declaring firmware |

wiki/tools.md

@@ -213,7 +213,8 @@ would have loaded.
 ### Granular options
 
 - `--system` with `--platform`: filter to specific systems within a platform pack
-- `--required-only`: exclude optional files, keep only required
+- `--required-only`: exclude optional files and the data directories declared
+  `required: false`, keep only required
 - `--split`: generate one ZIP per system instead of one big pack
 - `--split --group-by manufacturer`: group split packs by manufacturer (Sony, Nintendo, Sega...)
 - `--from-md5`: look up a hash in the database, or build a custom pack with `--platform`/`--emulator`