feat: pack the files a directory entry names

scripts/common.py

@@ -1116,6 +1116,101 @@ def group_identical_platforms(
     return result
 
 
+_directory_members_cache: dict[int, tuple[dict, dict[str, dict[str, str]]]] = {}
+
+
+def directory_members(db: dict, directory: str) -> dict[str, str]:
+    """Files the collection holds under a directory an emulator reads whole.
+
+    Returns {destination: sha1}, each destination starting with the directory
+    as the profile writes it. The collection and the destination meet on a
+    tail, so the members are the indexed tails that begin with it.
+
+    A generic directory name can sit in two unrelated trees (``voice/``,
+    ``data/``). Members that do not share one parent, or a destination two
+    contents claim, would merge them into one emulator's directory: nothing
+    is returned and the entry stays a reported gap.
+    """
+    tail = directory.strip("/")
+    if not tail:
+        return {}
+    by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
+    cached = _directory_members_cache.get(id(by_path_suffix))
+    if cached is None or cached[0] is not by_path_suffix:
+        cached = (by_path_suffix, {})
+        _directory_members_cache[id(by_path_suffix)] = cached
+    if tail in cached[1]:
+        return cached[1][tail]
+
+    prefix = f"{tail}/"
+    infix = f"/{prefix}"
+    held: dict[str, list[str]] = {}
+    parent: dict[str, str] = {}
+    for key, sha1s in by_path_suffix.items():
+        if key.startswith(prefix):
+            held[key] = sha1s
+            continue
+        at = key.find(infix)
+        if at > 0:
+            member = key[at + 1:]
+            if len(key[:at]) > len(parent.get(member, "")):
+                parent[member] = key[:at]
+    held = {
+        dest: sha1s for dest, sha1s in held.items() if "/.variants/" not in dest
+    }
+    one_tree = len({parent.get(dest, "") for dest in held}) <= 1
+    one_content = all(len(sha1s) == 1 for sha1s in held.values())
+    members = (
+        {dest: held[dest][0] for dest in sorted(held)}
+        if one_tree and one_content
+        else {}
+    )
+    cached[1][tail] = members
+    return members
+
+
+def expand_directory_entries(
+    files: list[dict], db: dict, standalone: bool = False
+) -> list[dict]:
+    """Replace each ``type: directory`` entry by the files held under it.
+
+    A profile declares the directory its code walks; what a pack can carry
+    is the files. Each member keeps the entry's fields and is pinned to the
+    content the collection holds at that destination. A directory nothing
+    answers is kept as declared, one gap, and a member the profile also
+    names as a file is left to that entry.
+    """
+    def destination(entry: dict) -> str:
+        chosen = entry.get("standalone_path") if standalone else None
+        return chosen or entry.get("path") or entry.get("name", "")
+
+    if not any(entry.get("type") == "directory" for entry in files):
+        return files
+    named = {
+        destination(entry) for entry in files if entry.get("type") != "directory"
+    }
+    expanded: list[dict] = []
+    for entry in files:
+        if entry.get("type") != "directory":
+            expanded.append(entry)
+            continue
+        members = directory_members(db, destination(entry))
+        if not members:
+            expanded.append(entry)
+            continue
+        inherited = {
+            key: value
+            for key, value in entry.items()
+            if key not in ("type", "standalone_path")
+        }
+        expanded.extend(
+            {**inherited, "name": dest, "path": dest, "sha1": sha1}
+            for dest, sha1 in members.items()
+            if dest not in named
+        )
+    return expanded
+
+
 def runs_standalone(
     emu_name: str, profile: dict, standalone_cores: set[str]
 ) -> bool:

scripts/generate_pack.py

@@ -36,6 +36,7 @@ from common import (
     build_zip_contents_index,
     check_inside_zip,
     compute_hashes,
+    expand_directory_entries,
     expand_platform_declared_names,
     fetch_large_file,
     filter_systems_by_target,
@@ -1288,7 +1289,11 @@ def generate_emulator_pack(
     with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
         for emu_name, profile in sorted(selected):
             pack_structure = profile.get("pack_structure")
-            files = filter_files_by_mode(profile.get("files", []), standalone)
+            files = expand_directory_entries(
+                filter_files_by_mode(profile.get("files", []), standalone),
+                db,
+                standalone,
+            )
             # A tree only one build reads follows that build, like a file.
             for dd in filter_files_by_mode(
                 profile.get("data_directories", []), standalone

scripts/verify.py

@@ -38,6 +38,7 @@ from common import (
     build_zip_contents_index,
     check_inside_zip,
     compute_hashes,
+    expand_directory_entries,
     expand_platform_declared_names,
     filter_systems_by_target,
     group_identical_platforms,
@@ -458,7 +459,9 @@ def find_undeclared_files(
 
         is_standalone = runs_standalone(emu_name, profile, standalone_set)
 
-        for f in profile.get("files", []):
+        for f in expand_directory_entries(
+            profile.get("files", []), db, is_standalone
+        ):
             fname = f.get("name", "")
             effective_path = (
                 f.get("standalone_path") if is_standalone else f.get("path")
@@ -1363,7 +1366,11 @@ def verify_emulator(
         )
 
     for emu_name, profile in selected:
-        files = filter_files_by_mode(profile.get("files", []), standalone)
+        files = expand_directory_entries(
+            filter_files_by_mode(profile.get("files", []), standalone),
+            db,
+            standalone,
+        )
         if region_drops:
             files = [
                 fe