diff --git a/install/amberelec.json b/install/amberelec.json index c1390a71e..ea9555055 100644 --- a/install/amberelec.json +++ b/install/amberelec.json @@ -63651,7 +63651,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "fbneo-cheats", @@ -77382,7 +77385,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -77617,7 +77623,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -77991,7 +78000,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] } ] } \ No newline at end of file diff --git a/install/batocera.json b/install/batocera.json index c1f5c14e2..bc8f45064 100644 --- a/install/batocera.json +++ b/install/batocera.json @@ -225292,7 +225292,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -225527,7 +225530,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] } ], "save_directory": { diff --git a/install/emudeck.json b/install/emudeck.json index 3f01646ff..bf431183d 100644 --- a/install/emudeck.json +++ b/install/emudeck.json @@ -100487,7 +100487,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "dinothawr", @@ -100936,7 +100939,10 @@ } }, "strip_components": 0, - "archive_size": 5752250 + "archive_size": 5752250, + "cores": [ + "dinothawr" + ] }, { "ref": "dirksimple", @@ -100993,7 +100999,10 @@ } }, "strip_components": 0, - "archive_size": 175384 + "archive_size": 175384, + "cores": [ + "dirksimple" + ] }, { "ref": "dolphin-sys", @@ -109670,7 +109679,10 @@ "exclude": [ "Themes" ], - "archive_size": 3197914 + "archive_size": 3197914, + "cores": [ + "ishiiruka" + ] }, { "ref": "fbneo-cheats", @@ -123401,7 +123413,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -123636,7 +123651,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -124010,7 +124028,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] }, { "ref": "lrps2", @@ -124027,7 +124048,10 @@ } }, "strip_components": 0, - "archive_size": 318896 + "archive_size": 318896, + "cores": [ + "lrps2" + ] }, { "ref": "yaps2-resources", @@ -124488,7 +124512,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ], "save_directory": { diff --git a/install/emuelec.json b/install/emuelec.json index 35ec58389..a1697c473 100644 --- a/install/emuelec.json +++ b/install/emuelec.json @@ -69460,7 +69460,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "dinothawr", @@ -69909,7 +69912,10 @@ } }, "strip_components": 0, - "archive_size": 5752250 + "archive_size": 5752250, + "cores": [ + "dinothawr" + ] }, { "ref": "fbneo-cheats", @@ -83640,7 +83646,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -83875,7 +83884,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -84249,7 +84261,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] } ] } \ No newline at end of file diff --git a/install/knulli.json b/install/knulli.json index fd6a9e45b..d2acf136a 100644 --- a/install/knulli.json +++ b/install/knulli.json @@ -157934,7 +157934,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -158169,7 +158172,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "lrps2", @@ -158186,7 +158192,10 @@ } }, "strip_components": 0, - "archive_size": 318896 + "archive_size": 318896, + "cores": [ + "lrps2" + ] }, { "ref": "ppsspp-assets", @@ -158875,7 +158884,10 @@ } }, "strip_components": 1, - "archive_size": 9844388 + "archive_size": 9844388, + "cores": [ + "ppsspp" + ] } ], "save_directory": { diff --git a/install/lakka.json b/install/lakka.json index 09f9de198..6126a09cd 100644 --- a/install/lakka.json +++ b/install/lakka.json @@ -116890,7 +116890,10 @@ } }, "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", - "archive_size": 21360579 + "archive_size": 21360579, + "cores": [ + "cemu" + ] }, { "ref": "dirksimple", @@ -116947,7 +116950,10 @@ } }, "strip_components": 0, - "archive_size": 175384 + "archive_size": 175384, + "cores": [ + "dirksimple" + ] }, { "ref": "fbneo-cheats", @@ -130678,7 +130684,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -130913,7 +130922,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -131748,7 +131760,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ], "save_directory": { diff --git a/install/myminui.json b/install/myminui.json index 7cb542e4e..1ce0f0c13 100644 --- a/install/myminui.json +++ b/install/myminui.json @@ -18959,7 +18959,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-cheats", @@ -32690,7 +32693,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -32925,7 +32931,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -33160,7 +33169,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] } ] } \ No newline at end of file diff --git a/install/nextui.json b/install/nextui.json index 933ec3af5..a348f9b8b 100644 --- a/install/nextui.json +++ b/install/nextui.json @@ -5636,7 +5636,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "fbneo-cheats", @@ -19367,7 +19370,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -19602,7 +19608,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] } ] } \ No newline at end of file diff --git a/install/recalbox.json b/install/recalbox.json index 6db9f7daa..e5c2410b5 100644 --- a/install/recalbox.json +++ b/install/recalbox.json @@ -62835,7 +62835,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -63070,7 +63073,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] } ], "save_directory": { diff --git a/install/retroarch.json b/install/retroarch.json index 5e89fef85..9d4a736ad 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -117881,7 +117881,10 @@ } }, "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", - "archive_size": 21360579 + "archive_size": 21360579, + "cores": [ + "cemu" + ] }, { "ref": "dinothawr", @@ -118330,7 +118333,10 @@ } }, "strip_components": 0, - "archive_size": 5752250 + "archive_size": 5752250, + "cores": [ + "dinothawr" + ] }, { "ref": "dirksimple", @@ -118387,7 +118393,10 @@ } }, "strip_components": 0, - "archive_size": 175384 + "archive_size": 175384, + "cores": [ + "dirksimple" + ] }, { "ref": "dolphin-sys", @@ -140790,7 +140799,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -141025,7 +141037,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -142549,7 +142564,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ], "save_directory": { diff --git a/install/retrobat.json b/install/retrobat.json index aec7d58a0..b3a4c47e3 100644 --- a/install/retrobat.json +++ b/install/retrobat.json @@ -89077,7 +89077,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -89312,7 +89315,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -89686,7 +89692,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] } ], "save_directory": { diff --git a/install/retrodeck.json b/install/retrodeck.json index 2938c951d..ff33b80e0 100644 --- a/install/retrodeck.json +++ b/install/retrodeck.json @@ -139411,7 +139411,10 @@ } }, "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", - "archive_size": 21360579 + "archive_size": 21360579, + "cores": [ + "cemu" + ] }, { "ref": "fbneo-cheats", @@ -153142,7 +153145,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -153377,7 +153383,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -153751,7 +153760,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] }, { "ref": "yaps2-resources", @@ -154216,7 +154228,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ], "save_directory": { diff --git a/install/retropie.json b/install/retropie.json index b6ab50768..1eaa3801f 100644 --- a/install/retropie.json +++ b/install/retropie.json @@ -124072,7 +124072,10 @@ } }, "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", - "archive_size": 21360579 + "archive_size": 21360579, + "cores": [ + "cemu" + ] }, { "ref": "dolphin-sys", @@ -146475,7 +146478,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -146710,7 +146716,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -147545,7 +147554,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ] } \ No newline at end of file diff --git a/install/rocknix.json b/install/rocknix.json index e15cda7e5..715a44c25 100644 --- a/install/rocknix.json +++ b/install/rocknix.json @@ -86539,7 +86539,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "dinothawr", @@ -86988,7 +86991,10 @@ } }, "strip_components": 0, - "archive_size": 5752250 + "archive_size": 5752250, + "cores": [ + "dinothawr" + ] }, { "ref": "dirksimple", @@ -87045,7 +87051,10 @@ } }, "strip_components": 0, - "archive_size": 175384 + "archive_size": 175384, + "cores": [ + "dirksimple" + ] }, { "ref": "dolphin-sys", @@ -95723,7 +95732,11 @@ "exclude": [ "Themes" ], - "archive_size": 3197914 + "archive_size": 3197914, + "cores": [ + "dolphin", + "ishiiruka" + ] }, { "ref": "fbneo-cheats", @@ -109454,7 +109467,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -109689,7 +109705,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -110063,7 +110082,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] }, { "ref": "lrps2", @@ -110080,7 +110102,10 @@ } }, "strip_components": 0, - "archive_size": 318896 + "archive_size": 318896, + "cores": [ + "lrps2" + ] }, { "ref": "ppsspp-assets", @@ -110769,7 +110794,10 @@ } }, "strip_components": 1, - "archive_size": 9844388 + "archive_size": 9844388, + "cores": [ + "ppsspp" + ] }, { "ref": "yaps2-resources", @@ -111230,7 +111258,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ] } \ No newline at end of file diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index a72245722..0dbcecd40 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -457,6 +457,14 @@ "const": true, "description": "Every core reading the tree runs without it (required: false in its profile): --required-only leaves it out." }, + "cores": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "The cores that read the tree, when no platform system declares it: an install for the games of a card keeps it only when its plan runs one of them." + }, "files": { "type": "object", "additionalProperties": { diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index c0a20376b..a425cbf06 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -435,7 +435,7 @@ def core_data_trees( # the trees go there as the core extras do. root = declared_root(config) trees: list[dict] = [] - seen: set[tuple[str, str]] = set() + seen: dict[tuple[str, str], int] = {} for core in sorted(_platform_cores(config, profiles, target_cores)): profile = profiles[core] if all(standalone for _view, standalone in platform_layouts(core, profile, handed)): @@ -460,13 +460,19 @@ def core_data_trees( if part.strip("/") ), root) or root if (ref, destination) in seen: + # One tree, read by a second core: the manifest says both + # (tree_readers), the pack writes it once. + first = trees[seen[(ref, destination)]] + if core not in first["readers"]: + first["readers"].append(core) continue - seen.add((ref, destination)) + seen[(ref, destination)] = len(trees) trees.append({ "ref": ref, "destination": destination, "required": tree.get("required"), "core": core, + "readers": [core], "source_systems": entry_system_ids(tree, profile), }) return trees @@ -4914,16 +4920,14 @@ def generate_manifest( # whose hash the list below records, the same tree the pack carries. data_refs: dict[str, tuple[str, tuple[str, ...], str, bool]] = {} reclaim = {full: path for full, (path, _size) in cache_held.items()} + core_trees = ([] if source == "platform" + else core_data_trees(config, emu_profiles, platform_name, data_registry, + target_cores)) data_members = list(_data_directory_members( pack_systems, data_registry, platform_name, base_dest, case_insensitive, seen_destinations, seen_lower, seen_parents, data_refs, required_only=required_only, - core_trees=( - [] if source == "platform" - else core_data_trees( - config, emu_profiles, platform_name, data_registry, target_cores - ) - ), + core_trees=core_trees, reclaim=reclaim, )) # A platform entry found in a tree the pack carries is that tree's member: @@ -4936,7 +4940,8 @@ def generate_manifest( omitted_by_destination.pop(full_dest, None) data_sizes = pack_only_sizes + [os.path.getsize(src) for src, _dest in data_members] data_directories = _manifest_data_directories( - data_members, data_refs, data_registry or {}, base_dest + data_members, data_refs, data_registry or {}, base_dest, + tree_readers(pack_systems, core_trees), ) now = _build_timestamp(db) @@ -5021,11 +5026,28 @@ def generate_manifest( return result +def tree_readers(pack_systems: dict, core_trees: Iterable[dict]) -> dict[str, list[str]]: + """The cores that read each tree, for a tree only cores read. A tree a + platform system declares is the system's, whatever core runs it, and + gets no entry. Every reference counts: two cores reading one tree yield + its files once.""" + cores: dict[str, set[str]] = {} + declared: set[str] = set() + for _systems, dd in _tree_references(pack_systems, core_trees): + ref = str(dd.get("ref") or "") + if dd.get("core"): + cores.setdefault(ref, set()).update(str(core) for core in dd.get("readers") or [dd["core"]]) + else: + declared.add(ref) + return {ref: sorted(names) for ref, names in cores.items() if ref not in declared} + + def _manifest_data_directories( members: list[tuple[str, str]], refs: dict[str, tuple[str, tuple[str, ...], str, bool]], registry: dict, base_dest: str, + readers: dict[str, list[str]] | None = None, ) -> list[dict]: """Each data directory the pack carries, as the installer rebuilds it. @@ -5038,7 +5060,9 @@ def _manifest_data_directories( the tree costs: a tarball whose tree is 10 KB is a download of 21 MB. ``optional`` marks a tree every core reading it runs without (``required: false``): install.py --required-only leaves it out, as the - pack built with --required-only does. + pack built with --required-only does. ``cores`` names the cores that + read a tree no platform system declares (``readers``): an install for + the games of a card keeps it only when its plan runs one of them. """ grouped: dict[tuple[str, str], dict] = {} versions: dict[str, dict[str, dict]] = {} @@ -5088,6 +5112,8 @@ def _manifest_data_directories( group = grouped[key] if not group.pop("needed", False): group["optional"] = True + if readers and group["ref"] in readers: + group["cores"] = readers[group["ref"]] trees.append(group) return trees diff --git a/tests/test_core_data_trees.py b/tests/test_core_data_trees.py index 2fb6e7f3e..6d352552b 100644 --- a/tests/test_core_data_trees.py +++ b/tests/test_core_data_trees.py @@ -87,6 +87,24 @@ class CoreDataTreesTest(unittest.TestCase): def test_standalone_build_brings_no_tree(self) -> None: self.assertNotIn("solo", {t["core"] for t in self.trees("other")}) + def test_every_reader_of_a_shared_tree_is_recorded(self) -> None: + """The second core reading a tree was dropped with the duplicate, and an + install for its games would have lost the tree.""" + twin = {**PROFILES["demo"], "emulator": "Twin", "cores": ["twin"]} + profiles = {**PROFILES, "twin": twin} + config = {**CONFIG, "cores": ["demo", "solo", "twin"]} + trees = builder.core_data_trees(config, profiles, "plat", REGISTRY) + shared = [t for t in trees if t["ref"] == "shared"] + self.assertEqual(len(shared), 1) + self.assertEqual(shared[0]["readers"], ["demo", "twin"]) + self.assertEqual(builder.tree_readers({}, trees)["shared"], ["demo", "twin"]) + + def test_a_tree_a_system_declares_names_no_core(self) -> None: + trees = self.trees() + declared = {"demo-system": {"data_directories": [{"ref": "shared"}]}} + self.assertNotIn("shared", builder.tree_readers(declared, trees)) + self.assertIn("shared", builder.tree_readers({}, trees)) + def test_tree_below_the_declared_bios_folder(self) -> None: config = { **CONFIG,