diff --git a/install.ps1 b/install.ps1 index 46ef83dfa..0b44aa3b4 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "ee24f4f154e68d7d503f12e4816e3f1ff38f80159432093e4c6ed86411783ed7" + $defaultInstallSha256 = "bba0f7cd7677fb7f4d767504412843cf3809c03b91808c7c7772f17758893d50" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 6613a0554..a7a272964 100644 --- a/install.py +++ b/install.py @@ -3813,6 +3813,9 @@ def _check_data_directory(entry: object, where: str) -> None: raise ValueError(f"{where}.{field} is invalid") if entry.get("optional", True) is not True: raise ValueError(f"{where}.optional is invalid") + cores = entry.get("cores", ["-"]) + if not isinstance(cores, list) or not cores or not all(isinstance(c, str) and c for c in cores): + raise ValueError(f"{where}.cores is invalid") files = entry.get("files") if not isinstance(files, dict) or not files or len(files) > MAX_MANIFEST_FILES: raise ValueError(f"{where}.files is invalid") @@ -6201,7 +6204,7 @@ _CHOICE_OPTIONS = ("--system", "--core", "--region", "--for-games", "--required- # An option not named here comes after them. _COMMAND_ORDER = ( "--manifest", "--from-pack", "--system", "--core", "--emulator", "--region", - "--for-games", "--required-only", "--target", "--no-game-data", "--installed-cores", "--cores-dir", "--every-board", + "--for-games", "--required-only", "--all-files", "--target", "--no-game-data", "--installed-cores", "--cores-dir", "--every-board", "--declared-only", "--standalone-copies", "--save-dir", "--jobs", "--from", "--add-app", "--offline", "--check", "--dry-run", ) @@ -6238,6 +6241,7 @@ class Invocation: options.append(("--for-games", None if games is True else str(games))) for flag, on in ( ("--required-only", getattr(args, "required_only", False)), + ("--all-files", getattr(args, "all_files", False)), ("--every-board", args.every_board), ("--installed-cores", args.installed_cores and not args.cores_dir), ("--no-game-data", args.no_game_data), @@ -7551,13 +7555,20 @@ def _install_data_member( def select_data_directories(manifest: dict, files: list[dict], narrowed: bool, - required_only: bool = False) -> list[dict]: + required_only: bool = False, + games: GamesPlan | None = None) -> list[dict]: """The data trees this install keeps: all, or those of a kept system. --required-only leaves out a tree every core reading it runs without (`optional`): a GBA and a SNES game took FBNeo's cheats and samples, - 3,472 files no game opened.""" + 3,472 files no game opened. For the games of a card, a tree only cores + read (`cores`) is kept when the plan runs one of them, or runs a system + of the tree on a core no setup names.""" entries = [e for e in manifest.get("data_directories") or [] if not (required_only and e.get("optional"))] + if games is not None: + entries = [e for e in entries if not e.get("cores") + or set(e["cores"]) & games.profiles + or set(e.get("systems") or []) & games.any_core] if not narrowed: return entries kept = {system for f in files for system in entry_systems(f)} @@ -8989,7 +9000,8 @@ def _prompt_custom_selection( kept = menu_choice(files, omitted, installed, current, False)[0] kept_trees = select_data_directories({"data_directories": trees}, kept, narrowed or current is not None, - isinstance(current, GamesChoice) and current.required) + isinstance(current, GamesChoice) and current.required, + current.plan if isinstance(current, GamesChoice) else None) if kept and all_in_place(kept, kept_trees, bios_path): # Read before the menu, as the screen reads before it asks: a # rerun was offered an install that installs nothing, then told @@ -9119,7 +9131,8 @@ def _print_menu_head( # and PPSSPP were counted for every --system tried. kept_trees = select_data_directories({"data_directories": trees}, shown, narrowed or current is not None, - isinstance(current, GamesChoice) and current.required) + isinstance(current, GamesChoice) and current.required, + current.plan if isinstance(current, GamesChoice) else None) data = sum(len(entry.get("files") or {}) for entry in kept_trees) data_size = sum(meta.get("size", 0) for entry in kept_trees for meta in (entry.get("files") or {}).values()) @@ -10683,6 +10696,10 @@ class GamesPlan: self.required: set[str] = set() self.unknown_core: list[str] = [] self.truncated = False + # The cores the games run on (a folder's default and its boards'), + # and the systems of a folder no setup names a core for. + self.profiles: set[str] = set() + self.any_core: set[str] = set() @property def fallback(self) -> list[GameFacts]: @@ -10910,6 +10927,11 @@ def plan_games(plat: str, bios_path: Path, manifest: dict, folder: Path | None = cache.save() for game_folder, group in read: if group: + for profile in _folder_profiles(game_folder.row): + if profile: + plan.profiles.add(str(profile)) + else: + plan.any_core.update(str(s) for s in game_folder.row.get("systems") or []) arcade = game_folder.row.get("arcade") sets = (arcade_index(arcade) or {}).get("support", set()) \ if isinstance(arcade, str) else set() @@ -13514,6 +13536,13 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="with --for-games, only the files the games cannot start without", ) + parser.add_argument( + "--all-files", + action="store_true", + help="install the whole list even when the card holds games. A run that " + "asks nothing (--yes, --no-input, --events) otherwise installs only " + "what the card's games need, the plan's first choice", + ) parser.add_argument( "--no-game-data", action="store_true", @@ -13713,6 +13742,8 @@ def main() -> None: parser.error("--quiet applies to an install, not to --check, --dry-run or a listing") if args.tui and args.plain: parser.error("--tui and --plain ask for two different interfaces") + if args.all_files and args.for_games is not None: + parser.error("--all-files and --for-games ask for two different lists") if args.required_only and args.for_games is None: parser.error("--required-only applies to --for-games: the list alone does not " "say which files a game cannot start without") @@ -15694,8 +15725,18 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # some, what they need is the menu's first choice, and --for-games # narrows the list here, before the systems are counted. will_ask = interactive and not args.check and not listing + # A run that asks nothing takes the plan's first choice: the games of + # the card. `--yes` alone installed 6.4 GB on a dArkOS card holding + # two games, while the screen offers what they need first. Not for a + # run given its list (the screen's own runs, the choice already + # made), nor one narrowed by name. + unasked = (not interactive and not args.check and not listing + and not getattr(args, "all_files", False) + and LOCAL.manifest is None and LOCAL.pack is None + and not (args.installed_cores or args.no_game_data or args.system + or args.core or args.region or args.emulator or args.declared_only)) games_plan = None - if args.for_games is not None or (will_ask and not ( + if args.for_games is not None or unasked or (will_ask and not ( args.installed_cores or args.no_game_data or LOCAL.pack is not None)): games_plan = read_games(args, plat_name, bios_path, manifest, term, os_type) games_applied = False @@ -15708,6 +15749,9 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: games_applied = True print(f" For the games: {_plural(len(files), 'file')} of the list" + (f", {GAMES_REQUIRED_WORDS}" if args.required_only else "")) + if args.for_games is None: + print(" The card holds games, so only what they need is installed. " + "--all-files installs the whole list.") version_notes += [line.strip() for line in fallback_lines(games_plan)] # --no-game-data leaves the game data out once the list is narrowed # to the choice: taken out first, its line named the 5099 game data @@ -15848,7 +15892,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: or games_applied ) data_dirs = select_data_directories(manifest, files, narrowed, - bool(args.required_only)) + bool(args.required_only), + games_plan if games_applied else None) # The kinds as the plan and the menu head say them (parts_words), # the data files of the folders kept among them: this head said # BIOS files and game data beside a plan of three kinds, neither diff --git a/install.sh b/install.sh index d5ee07167..1e47b7f91 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="ee24f4f154e68d7d503f12e4816e3f1ff38f80159432093e4c6ed86411783ed7" +DEFAULT_INSTALL_SHA256="bba0f7cd7677fb7f4d767504412843cf3809c03b91808c7c7772f17758893d50" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/amberelec.json b/install/amberelec.json index 71782316f..fb516f651 100644 --- a/install/amberelec.json +++ b/install/amberelec.json @@ -63651,7 +63651,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "fbneo-cheats", @@ -77382,7 +77385,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -77617,7 +77623,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -77991,7 +78000,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] } ] } \ No newline at end of file diff --git a/install/batocera.json b/install/batocera.json index 3e8742ea8..d19b0d64e 100644 --- a/install/batocera.json +++ b/install/batocera.json @@ -226706,7 +226706,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -226941,7 +226944,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] } ], "save_directory": { diff --git a/install/emudeck.json b/install/emudeck.json index ef736b334..88a028ad1 100644 --- a/install/emudeck.json +++ b/install/emudeck.json @@ -102220,7 +102220,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "dinothawr", @@ -102669,7 +102672,10 @@ } }, "strip_components": 0, - "archive_size": 5752250 + "archive_size": 5752250, + "cores": [ + "dinothawr" + ] }, { "ref": "dirksimple", @@ -102726,7 +102732,10 @@ } }, "strip_components": 0, - "archive_size": 175384 + "archive_size": 175384, + "cores": [ + "dirksimple" + ] }, { "ref": "dolphin-sys", @@ -111403,7 +111412,10 @@ "exclude": [ "Themes" ], - "archive_size": 3197914 + "archive_size": 3197914, + "cores": [ + "ishiiruka" + ] }, { "ref": "fbneo-cheats", @@ -125134,7 +125146,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -125369,7 +125384,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -125743,7 +125761,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] }, { "ref": "lrps2", @@ -125760,7 +125781,10 @@ } }, "strip_components": 0, - "archive_size": 318896 + "archive_size": 318896, + "cores": [ + "lrps2" + ] }, { "ref": "yaps2-resources", @@ -126221,7 +126245,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ], "save_directory": { diff --git a/install/emuelec.json b/install/emuelec.json index 3e930db89..3c86a3205 100644 --- a/install/emuelec.json +++ b/install/emuelec.json @@ -69460,7 +69460,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "dinothawr", @@ -69909,7 +69912,10 @@ } }, "strip_components": 0, - "archive_size": 5752250 + "archive_size": 5752250, + "cores": [ + "dinothawr" + ] }, { "ref": "fbneo-cheats", @@ -83640,7 +83646,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -83875,7 +83884,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -84249,7 +84261,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] } ] } \ No newline at end of file diff --git a/install/knulli.json b/install/knulli.json index 803284e04..49e7ebd8f 100644 --- a/install/knulli.json +++ b/install/knulli.json @@ -157969,7 +157969,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -158204,7 +158207,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "lrps2", @@ -158221,7 +158227,10 @@ } }, "strip_components": 0, - "archive_size": 318896 + "archive_size": 318896, + "cores": [ + "lrps2" + ] }, { "ref": "ppsspp-assets", @@ -158910,7 +158919,10 @@ } }, "strip_components": 1, - "archive_size": 9844388 + "archive_size": 9844388, + "cores": [ + "ppsspp" + ] } ], "save_directory": { diff --git a/install/lakka.json b/install/lakka.json index 047ec53e3..c9b1dd78e 100644 --- a/install/lakka.json +++ b/install/lakka.json @@ -116877,7 +116877,10 @@ } }, "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", - "archive_size": 21360579 + "archive_size": 21360579, + "cores": [ + "cemu" + ] }, { "ref": "dirksimple", @@ -116934,7 +116937,10 @@ } }, "strip_components": 0, - "archive_size": 175384 + "archive_size": 175384, + "cores": [ + "dirksimple" + ] }, { "ref": "fbneo-cheats", @@ -130665,7 +130671,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -130900,7 +130909,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -131735,7 +131747,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ], "save_directory": { diff --git a/install/launchers/index.json b/install/launchers/index.json index 44d04c39f..8db6e297a 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1122422, - "sha256": "ee24f4f154e68d7d503f12e4816e3f1ff38f80159432093e4c6ed86411783ed7", + "size": 1125268, + "sha256": "bba0f7cd7677fb7f4d767504412843cf3809c03b91808c7c7772f17758893d50", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1176049, - "sha256": "9ac4845d2a448ece2d421db3137c7aea6d636d0ecf6c92ed777630f47dd61021" + "size": 1178895, + "sha256": "814d80f123e59f5192631842d53a4808bf373599ced857f4622612ffbc25322b" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1122422, - "sha256": "ee24f4f154e68d7d503f12e4816e3f1ff38f80159432093e4c6ed86411783ed7", + "size": 1125268, + "sha256": "bba0f7cd7677fb7f4d767504412843cf3809c03b91808c7c7772f17758893d50", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1122422, - "sha256": "ee24f4f154e68d7d503f12e4816e3f1ff38f80159432093e4c6ed86411783ed7", + "size": 1125268, + "sha256": "bba0f7cd7677fb7f4d767504412843cf3809c03b91808c7c7772f17758893d50", "executable": false }, { diff --git a/install/myminui.json b/install/myminui.json index 108f1c677..883fc5b48 100644 --- a/install/myminui.json +++ b/install/myminui.json @@ -18959,7 +18959,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-cheats", @@ -32690,7 +32693,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -32925,7 +32931,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -33160,7 +33169,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] } ] } \ No newline at end of file diff --git a/install/nextui.json b/install/nextui.json index 9804f5110..c84614823 100644 --- a/install/nextui.json +++ b/install/nextui.json @@ -5636,7 +5636,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "fbneo-cheats", @@ -19367,7 +19370,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -19602,7 +19608,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] } ] } \ No newline at end of file diff --git a/install/recalbox.json b/install/recalbox.json index 7bb16ef45..478bfb5c2 100644 --- a/install/recalbox.json +++ b/install/recalbox.json @@ -64189,7 +64189,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -64424,7 +64427,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] } ], "save_directory": { diff --git a/install/retroarch.json b/install/retroarch.json index 59a748017..1b052c818 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -119822,7 +119822,10 @@ } }, "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", - "archive_size": 21360579 + "archive_size": 21360579, + "cores": [ + "cemu" + ] }, { "ref": "dinothawr", @@ -120271,7 +120274,10 @@ } }, "strip_components": 0, - "archive_size": 5752250 + "archive_size": 5752250, + "cores": [ + "dinothawr" + ] }, { "ref": "dirksimple", @@ -120328,7 +120334,10 @@ } }, "strip_components": 0, - "archive_size": 175384 + "archive_size": 175384, + "cores": [ + "dirksimple" + ] }, { "ref": "dolphin-sys", @@ -142731,7 +142740,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -142966,7 +142978,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -144490,7 +144505,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ], "save_directory": { diff --git a/install/retrobat.json b/install/retrobat.json index 5a159ad3d..6f7912e0f 100644 --- a/install/retrobat.json +++ b/install/retrobat.json @@ -90410,7 +90410,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -90645,7 +90648,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -91019,7 +91025,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] } ], "save_directory": { diff --git a/install/retrodeck.json b/install/retrodeck.json index caa70af85..cdc184a37 100644 --- a/install/retrodeck.json +++ b/install/retrodeck.json @@ -141323,7 +141323,10 @@ } }, "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", - "archive_size": 21360579 + "archive_size": 21360579, + "cores": [ + "cemu" + ] }, { "ref": "fbneo-cheats", @@ -155054,7 +155057,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -155289,7 +155295,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -155663,7 +155672,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] }, { "ref": "yaps2-resources", @@ -156128,7 +156140,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ], "save_directory": { diff --git a/install/retropie.json b/install/retropie.json index eb54566d4..a9fab43cd 100644 --- a/install/retropie.json +++ b/install/retropie.json @@ -124049,7 +124049,10 @@ } }, "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", - "archive_size": 21360579 + "archive_size": 21360579, + "cores": [ + "cemu" + ] }, { "ref": "dolphin-sys", @@ -146452,7 +146455,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -146687,7 +146693,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -147522,7 +147531,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ] } \ No newline at end of file diff --git a/install/rocknix.json b/install/rocknix.json index 2fd9be176..6bab702de 100644 --- a/install/rocknix.json +++ b/install/rocknix.json @@ -86513,7 +86513,10 @@ } }, "strip_components": 0, - "archive_size": 4465709 + "archive_size": 4465709, + "cores": [ + "bluemsx" + ] }, { "ref": "dinothawr", @@ -86962,7 +86965,10 @@ } }, "strip_components": 0, - "archive_size": 5752250 + "archive_size": 5752250, + "cores": [ + "dinothawr" + ] }, { "ref": "dirksimple", @@ -87019,7 +87025,10 @@ } }, "strip_components": 0, - "archive_size": 175384 + "archive_size": 175384, + "cores": [ + "dirksimple" + ] }, { "ref": "dolphin-sys", @@ -95697,7 +95706,11 @@ "exclude": [ "Themes" ], - "archive_size": 3197914 + "archive_size": 3197914, + "cores": [ + "dolphin", + "ishiiruka" + ] }, { "ref": "fbneo-cheats", @@ -109428,7 +109441,10 @@ }, "strip_components": 1, "archive_size": 5083767, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "fbneo-samples", @@ -109663,7 +109679,10 @@ }, "strip_components": 0, "archive_size": 53006540, - "optional": true + "optional": true, + "cores": [ + "fbneo" + ] }, { "ref": "freeintv-overlays", @@ -110037,7 +110056,10 @@ }, "strip_components": 1, "archive_size": 21311170, - "optional": true + "optional": true, + "cores": [ + "freeintv" + ] }, { "ref": "lrps2", @@ -110054,7 +110076,10 @@ } }, "strip_components": 0, - "archive_size": 318896 + "archive_size": 318896, + "cores": [ + "lrps2" + ] }, { "ref": "ppsspp-assets", @@ -110743,7 +110768,10 @@ } }, "strip_components": 1, - "archive_size": 9844388 + "archive_size": 9844388, + "cores": [ + "ppsspp" + ] }, { "ref": "yaps2-resources", @@ -111204,7 +111232,10 @@ } }, "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", - "archive_size": 21181421 + "archive_size": 21181421, + "cores": [ + "yaps2" + ] } ] } \ No newline at end of file diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index a72245722..0dbcecd40 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -457,6 +457,14 @@ "const": true, "description": "Every core reading the tree runs without it (required: false in its profile): --required-only leaves it out." }, + "cores": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "The cores that read the tree, when no platform system declares it: an install for the games of a card keeps it only when its plan runs one of them." + }, "files": { "type": "object", "additionalProperties": { diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 4b4c4f726..25b9f97ca 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -448,7 +448,7 @@ def core_data_trees( # the trees go there as the core extras do. root = declared_root(config) trees: list[dict] = [] - seen: set[tuple[str, str]] = set() + seen: dict[tuple[str, str], int] = {} for core in sorted(_platform_cores(config, profiles, target_cores)): profile = profiles[core] if all(standalone for _view, standalone in platform_layouts(core, profile, handed)): @@ -473,13 +473,19 @@ def core_data_trees( if part.strip("/") ), root) or root if (ref, destination) in seen: + # One tree, read by a second core: the manifest says both + # (tree_readers), the pack writes it once. + first = trees[seen[(ref, destination)]] + if core not in first["readers"]: + first["readers"].append(core) continue - seen.add((ref, destination)) + seen[(ref, destination)] = len(trees) trees.append({ "ref": ref, "destination": destination, "required": tree.get("required"), "core": core, + "readers": [core], "source_systems": entry_system_ids(tree, profile), }) return trees @@ -5086,16 +5092,14 @@ def generate_manifest( # whose hash the list below records, the same tree the pack carries. data_refs: dict[str, tuple[str, tuple[str, ...], str, bool]] = {} reclaim = {full: path for full, (path, _size) in cache_held.items()} + core_trees = ([] if source == "platform" + else core_data_trees(config, emu_profiles, platform_name, data_registry, + target_cores)) data_members = list(_data_directory_members( pack_systems, data_registry, platform_name, base_dest, case_insensitive, seen_destinations, seen_lower, seen_parents, data_refs, required_only=required_only, - core_trees=( - [] if source == "platform" - else core_data_trees( - config, emu_profiles, platform_name, data_registry, target_cores - ) - ), + core_trees=core_trees, reclaim=reclaim, )) # A platform entry found in a tree the pack carries is that tree's member: @@ -5108,7 +5112,8 @@ def generate_manifest( omitted_by_destination.pop(full_dest, None) data_sizes = pack_only_sizes + [os.path.getsize(src) for src, _dest in data_members] data_directories = _manifest_data_directories( - data_members, data_refs, data_registry or {}, base_dest + data_members, data_refs, data_registry or {}, base_dest, + tree_readers(pack_systems, core_trees), ) now = _build_timestamp(db) @@ -5192,11 +5197,28 @@ def generate_manifest( return result +def tree_readers(pack_systems: dict, core_trees: Iterable[dict]) -> dict[str, list[str]]: + """The cores that read each tree, for a tree only cores read. A tree a + platform system declares is the system's, whatever core runs it, and + gets no entry. Every reference counts: two cores reading one tree yield + its files once.""" + cores: dict[str, set[str]] = {} + declared: set[str] = set() + for _systems, dd in _tree_references(pack_systems, core_trees): + ref = str(dd.get("ref") or "") + if dd.get("core"): + cores.setdefault(ref, set()).update(str(core) for core in dd.get("readers") or [dd["core"]]) + else: + declared.add(ref) + return {ref: sorted(names) for ref, names in cores.items() if ref not in declared} + + def _manifest_data_directories( members: list[tuple[str, str]], refs: dict[str, tuple[str, tuple[str, ...], str, bool]], registry: dict, base_dest: str, + readers: dict[str, list[str]] | None = None, ) -> list[dict]: """Each data directory the pack carries, as the installer rebuilds it. @@ -5209,7 +5231,9 @@ def _manifest_data_directories( the tree costs: a tarball whose tree is 10 KB is a download of 21 MB. ``optional`` marks a tree every core reading it runs without (``required: false``): install.py --required-only leaves it out, as the - pack built with --required-only does. + pack built with --required-only does. ``cores`` names the cores that + read a tree no platform system declares (``readers``): an install for + the games of a card keeps it only when its plan runs one of them. """ grouped: dict[tuple[str, str], dict] = {} versions: dict[str, dict[str, dict]] = {} @@ -5259,6 +5283,8 @@ def _manifest_data_directories( group = grouped[key] if not group.pop("needed", False): group["optional"] = True + if readers and group["ref"] in readers: + group["cores"] = readers[group["ref"]] trees.append(group) return trees diff --git a/scripts/siteagents.py b/scripts/siteagents.py index ffed73260..1776cb992 100644 --- a/scripts/siteagents.py +++ b/scripts/siteagents.py @@ -82,8 +82,8 @@ def llms_txt(platforms: list[tuple[str, str]]) -> str: "Most people want to play now, with the fewest files. The installer does that in one " "command: it finds the setup and its BIOS folder, reads the games on the card, and with " "`--for-games --required-only` installs only the files those games cannot start " - "without. Without these two options it installs every file of the setup's list " - "(gigabytes), even with `--yes`.", + "without. `--yes` alone installs what the card's games need, optional files " + "included, and `--all-files` the setup's whole list (gigabytes).", "", f"1. Preview, nothing written: `{SH} --for-games --required-only --dry-run`", f"2. Install, no questions: `{SH} --for-games --required-only --yes`", diff --git a/scripts/sitediagrams.py b/scripts/sitediagrams.py index d552657b3..dc75524d6 100644 --- a/scripts/sitediagrams.py +++ b/scripts/sitediagrams.py @@ -899,7 +899,7 @@ GLANCE_HEADING = "## Options at a glance" GLANCE_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = ( ("Where and for what", ("--platform", "--dest", "--target", "--every-board", "--save-dir", "--romm-firmware", "--standalone-copies", "--add-app")), - ("Fewer files", ("--for-games", "--required-only", "--system", "--core", "--emulator", + ("Fewer files", ("--for-games", "--required-only", "--all-files", "--system", "--core", "--emulator", "--region", "--no-game-data", "--declared-only", "--installed-cores", "--cores-dir")), ("Look first", ("--check", "--dry-run", "--list-platforms", "--list-targets", diff --git a/tests/test_core_data_trees.py b/tests/test_core_data_trees.py index 2fb6e7f3e..6d352552b 100644 --- a/tests/test_core_data_trees.py +++ b/tests/test_core_data_trees.py @@ -87,6 +87,24 @@ class CoreDataTreesTest(unittest.TestCase): def test_standalone_build_brings_no_tree(self) -> None: self.assertNotIn("solo", {t["core"] for t in self.trees("other")}) + def test_every_reader_of_a_shared_tree_is_recorded(self) -> None: + """The second core reading a tree was dropped with the duplicate, and an + install for its games would have lost the tree.""" + twin = {**PROFILES["demo"], "emulator": "Twin", "cores": ["twin"]} + profiles = {**PROFILES, "twin": twin} + config = {**CONFIG, "cores": ["demo", "solo", "twin"]} + trees = builder.core_data_trees(config, profiles, "plat", REGISTRY) + shared = [t for t in trees if t["ref"] == "shared"] + self.assertEqual(len(shared), 1) + self.assertEqual(shared[0]["readers"], ["demo", "twin"]) + self.assertEqual(builder.tree_readers({}, trees)["shared"], ["demo", "twin"]) + + def test_a_tree_a_system_declares_names_no_core(self) -> None: + trees = self.trees() + declared = {"demo-system": {"data_directories": [{"ref": "shared"}]}} + self.assertNotIn("shared", builder.tree_readers(declared, trees)) + self.assertIn("shared", builder.tree_readers({}, trees)) + def test_tree_below_the_declared_bios_folder(self) -> None: config = { **CONFIG, diff --git a/tests/test_install_games.py b/tests/test_install_games.py index 8388202d0..829f5da1e 100644 --- a/tests/test_install_games.py +++ b/tests/test_install_games.py @@ -912,6 +912,78 @@ class CommandLine(unittest.TestCase): self.assertTrue((self.card.cache / install.GAMES_SCAN_FILE).is_file()) +class ARunThatAsksNothing(unittest.TestCase): + """`--yes` alone installed the whole list (6.4 GB on a dArkOS card that + held two games), while the screen offers what the games need first.""" + + def install(self, *options: str) -> tuple[list[str], str]: + card = Card() + self.addCleanup(card.close) + card.game("SEGACD", "sonic.iso", mcd_iso(100)) + card.game("MD", "eu.md", MD_EU) + root = card.root / "repo" + (root / "install").mkdir(parents=True) + shutil.copytree(card.lists / "games", root / "install" / "games") + listed = manifest() + (root / "bios").mkdir() + for entry in listed["files"]: + payload = entry["dest"].encode().ljust(entry["size"], b".")[:entry["size"]] + (root / entry["repo_path"]).write_bytes(payload) + (root / "install" / "retroarch.json").write_text(json.dumps(listed)) + handler = functools.partial(_Quiet, directory=str(root)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + proc = run_cli(card, *options, offline=False, + base=f"http://127.0.0.1:{server.server_address[1]}") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + return sorted(p.relative_to(card.bios).as_posix() + for p in card.bios.rglob("*") if p.is_file()), proc.stdout + + def test_it_installs_what_the_games_need(self): + unasked, said = self.install("--yes") + self.assertEqual(unasked, self.install("--for-games", "--yes")[0]) + self.assertIn("--all-files installs the whole list", said) + + def test_all_files_installs_the_whole_list(self): + everything, said = self.install("--all-files", "--yes") + self.assertEqual(everything, sorted(entry["dest"] for entry in manifest()["files"])) + self.assertNotIn("For the games", said) + + def test_all_files_and_for_games_are_refused_together(self): + card = Card() + self.addCleanup(card.close) + proc = run_cli(card, "--all-files", "--for-games", "--check") + self.assertEqual(proc.returncode, 2) + self.assertIn("--all-files and --for-games ask for two different lists", proc.stderr) + + +class DataTreesFollowTheGamesCores(unittest.TestCase): + """One GBA and one SNES game took FBNeo's cheats and samples: a tree + only cores read is kept when the plan runs one of them.""" + + def test_a_tree_of_a_core_the_plan_does_not_run_is_left_out(self): + plan = install.GamesPlan(Path("card"), True) + plan.profiles = {"snes9x"} + fbneo = {"ref": "fbneo-cheats", "systems": ["nintendo-snes"], "cores": ["fbneo"]} + dolphin = {"ref": "dolphin-sys", "systems": ["nintendo-snes"]} + trees = {"data_directories": [fbneo, dolphin]} + files = [{"dest": "a.bin", "systems": ["nintendo-snes"]}] + self.assertEqual(install.select_data_directories(trees, files, True, games=plan), [dolphin]) + plan.profiles.add("fbneo") + self.assertEqual(install.select_data_directories(trees, files, True, games=plan), + [fbneo, dolphin]) + + def test_a_system_with_no_named_core_keeps_every_cores_tree(self): + plan = install.GamesPlan(Path("card"), True) + plan.any_core = {"nintendo-gba"} + fbneo = {"ref": "fbneo-cheats", "systems": ["nintendo-gba"], "cores": ["fbneo"]} + self.assertEqual(install.select_data_directories( + {"data_directories": [fbneo]}, [{"systems": ["nintendo-gba"]}], True, games=plan), + [fbneo]) + + class _Quiet(http.server.SimpleHTTPRequestHandler): def log_message(self, format, *args): # noqa: A002 - stdlib signature return diff --git a/wiki/installer.md b/wiki/installer.md index 0ecd328ec..894674a5d 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -866,7 +866,10 @@ games on this card: yes**, and Install takes only what those games need. One press on the row installs everything instead. Line mode asks the same: Enter installs for the games, `e` everything, `r` only what starts. On the command line it is `--for-games`, and `--for-games FOLDER` reads a games folder on a -computer, one subfolder per system. +computer, one subfolder per system. A run that asks nothing (`--yes`, +`--no-input`, `--events`) takes the same first choice when it finds games on +the card, and says so. `--all-files` installs the whole list instead, as does +a run narrowed by name (`--system`, `--core`, `--region`, `--emulator`). The rule is that every game must start. Per system, the run keeps what the system's default core reads. One region is kept only where the game's own @@ -983,6 +986,7 @@ what the options name, everything by default. | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | | `--for-games FOLDER` | Install only what the games on the card need. See [For the games on this card](#for-the-games-on-this-card) | | `--required-only` | With `--for-games`, only the files the games cannot start without | +| `--all-files` | The whole list, even when the card holds games. A run that asks nothing (`--yes`, `--no-input`, `--events`) otherwise installs what the card's games need, the plan's first choice. See [For the games on this card](#for-the-games-on-this-card) | | `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | | `--declared-only` | Keep only the files the platform's own list names (`cores: null` in the manifest), not the extra files cores load. With `--check`, the answer is about the list the platform itself checks. Refused with `--core`, the listings it does not change, `--standalone-copies` and `--tui` | | `--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart. `--system`, `--core`, `--region`, `--target` and `--installed-cores` narrow it, as they narrow an install. A run prints one list: two listing options are refused |