From 725325f0a159a39a7dea64102edb8c37659814ac Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:59:10 +0100 Subject: [PATCH] fix: announce a data tree at its archive size A tarball tree was announced at its extracted size: yaps2 at 6.5 MB for a 21.2 MB download, Cemu's game profiles at 9.9 KB for 21.4 MB. refresh_data_dirs records the size of each archive it downloads (a tree current from before is measured, not refetched), the install lists carry it as archive_size, and the plan, the menu and the data event count it. --- install.ps1 | 2 +- install.py | 32 ++++++++++--- install.sh | 2 +- install/amberelec.json | 12 +++-- install/batocera.json | 6 ++- install/emudeck.json | 27 +++++++---- install/emuelec.json | 15 ++++--- install/knulli.json | 12 +++-- install/lakka.json | 21 ++++++--- install/launchers/index.json | 16 +++---- install/myminui.json | 14 +++--- install/nextui.json | 11 +++-- install/recalbox.json | 6 ++- install/retroarch.json | 30 ++++++++----- install/retrobat.json | 9 ++-- install/retrodeck.json | 15 ++++--- install/retropie.json | 18 +++++--- install/rocknix.json | 30 ++++++++----- schemas/install-manifest.schema.json | 5 +++ scripts/generate_pack.py | 16 ++++++- scripts/refresh_data_dirs.py | 67 +++++++++++++++++++++++++--- tests/test_install.py | 31 +++++++++++++ tests/test_refresh_data_dirs.py | 49 +++++++++++++++++++- wiki/installer.md | 9 ++-- 24 files changed, 352 insertions(+), 103 deletions(-) diff --git a/install.ps1 b/install.ps1 index 412a6456e..7225221c0 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51" + $defaultInstallSha256 = "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index feabcc09e..8a4f6cb3c 100644 --- a/install.py +++ b/install.py @@ -3708,6 +3708,9 @@ def _check_data_directory(entry: object, where: str) -> None: strip = entry.get("strip_components", 0) if not isinstance(strip, int) or isinstance(strip, bool) or not 0 <= strip <= 16: raise ValueError(f"{where}.strip_components is invalid") + size = entry.get("archive_size", 1) + if not isinstance(size, int) or isinstance(size, bool) or size < 1: + raise ValueError(f"{where}.archive_size is invalid") if kind == "tarball" and ( not isinstance(entry.get("source_path"), str) or not entry["source_path"] ): @@ -6873,11 +6876,12 @@ def _print_plan( # files, and 2636 data files" read as two kinds of one thing. The kinds # follow, adding up to it. total = counts.files + tree_files - parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_counts.size)})"] + tree_bytes = sum(data_fetch_bytes(entry, needed) for entry, needed in trees) + parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_bytes)})"] if total else []) split = parts_words(counts if (manifest or {}).get("categorized") else tally({"size": e.get("size")} for e in to_download), - tree_files, tree_counts.size) + tree_files, tree_bytes) if split and trees: split += f", the data files extracted from {_plural(len(trees), 'archive')}" # A pack is copied from, not downloaded: "Would download 23 files" was @@ -6952,8 +6956,11 @@ def _print_plan( "every system and its files.") for entry, needed in trees: source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"] + fetched = data_fetch_bytes(entry, needed) + archive = ("" if fetched == sum(meta.get("size", 0) for meta in needed.values()) + else f" (a {format_size(fetched)} archive)") print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, " - f"from {source}") + f"from {source}{archive}") for folder, (count, size) in sorted((kept or {}).items()): print(f"\n Would keep {_plural(count, 'file')} you have ({format_size(size)}), " f"whose contents differ, in {folder}") @@ -7184,6 +7191,19 @@ def data_directory_state(entry: dict, bios_path: Path, return root, needed, present +def data_fetch_bytes(entry: dict, needed: dict[str, dict]) -> int: + """What getting a tree's missing files costs: its source archive, which + comes whole for one file, or from a pack the files themselves. A list + that records no archive size gives the files' own sizes: a tarball whose + tree is 10 KB was announced as 10 KB and was a download of 21 MB.""" + if not needed: + return 0 + size = entry.get("archive_size") + if LOCAL.pack is None and isinstance(size, int) and size > 0: + return size + return sum(meta.get("size", 0) for meta in needed.values()) + + def install_data_directory(entry: dict, bios_path: Path, state: tuple[Path, dict[str, dict], int] | None = None, progress: Progress | None = None) -> dict: @@ -8963,7 +8983,7 @@ def found_head(files: list[dict], trees: list[dict], bios_path: Path) -> str: for tree in trees: _root, wanted, _present = data_directory_state(tree, bios_path, remember=True) data += len(wanted) - data_bytes += sum(meta.get("size", 0) for meta in wanted.values()) + data_bytes += data_fetch_bytes(tree, wanted) left = [entry for entry in needed if id(entry) not in copies] rest = len(left) + data where = ", ".join(f"{count} found in {_last_name(label)}" @@ -15683,8 +15703,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: plat_name, manifest, bios_path, files, up_to_date, mismatched, sum(len(state[1]) for _entry, state in data_states), len(omitted_files), chosen=selection[0] if selection else None, - data_bytes=sum(meta.get("size", 0) for _entry, state in data_states - for meta in state[1].values()), + data_bytes=sum(data_fetch_bytes(entry, state[1]) + for entry, state in data_states), ) emit_copies(plat_name, to_download, copies, bios_index, bios_path, files) emit_unreadable(plat_name, unread) diff --git a/install.sh b/install.sh index c8cba304a..fe3d4fd6e 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51" +DEFAULT_INSTALL_SHA256="567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/amberelec.json b/install/amberelec.json index 745e5792b..51f5a9028 100644 --- a/install/amberelec.json +++ b/install/amberelec.json @@ -63489,7 +63489,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "fbneo-cheats", @@ -77218,7 +77219,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -77451,7 +77453,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -77823,7 +77826,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ] } \ No newline at end of file diff --git a/install/batocera.json b/install/batocera.json index 8c24297c0..a5ecfcb19 100644 --- a/install/batocera.json +++ b/install/batocera.json @@ -225291,7 +225291,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -225524,7 +225525,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ], "save_directory": { diff --git a/install/emudeck.json b/install/emudeck.json index c1d09d585..32f4ed6a2 100644 --- a/install/emudeck.json +++ b/install/emudeck.json @@ -100486,7 +100486,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -100934,7 +100935,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -100990,7 +100992,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -109666,7 +109669,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -123395,7 +123399,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -123628,7 +123633,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -124000,7 +124006,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "lrps2", @@ -124016,7 +124023,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "yaps2-resources", @@ -124476,7 +124484,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/emuelec.json b/install/emuelec.json index 8135201bd..e9709f639 100644 --- a/install/emuelec.json +++ b/install/emuelec.json @@ -69309,7 +69309,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -69757,7 +69758,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "fbneo-cheats", @@ -83486,7 +83488,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -83719,7 +83722,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -84091,7 +84095,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ] } \ No newline at end of file diff --git a/install/knulli.json b/install/knulli.json index 25b3ccce4..4e70dab7f 100644 --- a/install/knulli.json +++ b/install/knulli.json @@ -157933,7 +157933,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -158166,7 +158167,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "lrps2", @@ -158182,7 +158184,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "ppsspp-assets", @@ -158870,7 +158873,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 } ], "save_directory": { diff --git a/install/lakka.json b/install/lakka.json index 093e07996..5866fe5db 100644 --- a/install/lakka.json +++ b/install/lakka.json @@ -115932,7 +115932,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "cemu-gameprofiles", @@ -116888,7 +116889,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dirksimple", @@ -116944,7 +116946,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "fbneo-cheats", @@ -130673,7 +130676,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -130906,7 +130910,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -131278,7 +131283,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -131738,7 +131744,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/launchers/index.json b/install/launchers/index.json index cd32da514..acfb61130 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1107258, - "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", + "size": 1108265, + "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1160885, - "sha256": "5ccc4ec3df34a1fef96caea9e84218de1fb92331664047c36d6863c11973dc3d" + "size": 1161892, + "sha256": "1031750dcd599e0c4dd2e6c39296b3bcada68494a4837ad442150da64b67a55c" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1107258, - "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", + "size": 1108265, + "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1107258, - "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", + "size": 1108265, + "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", "executable": false }, { diff --git a/install/myminui.json b/install/myminui.json index 919ecf4b1..85de7b9bc 100644 --- a/install/myminui.json +++ b/install/myminui.json @@ -6,7 +6,7 @@ "platform": "myminui", "display_name": "MyMinUI", "version": "2026-08-09", - "generated": "2026-11-05T18:33:47Z", + "generated": "2026-11-05T23:59:37Z", "base_destination": "", "detect": [ { @@ -18957,7 +18957,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-cheats", @@ -32686,7 +32687,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -32919,7 +32921,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "fbneo-samples", @@ -33152,7 +33155,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ] } \ No newline at end of file diff --git a/install/nextui.json b/install/nextui.json index d38175219..4518d3d5c 100644 --- a/install/nextui.json +++ b/install/nextui.json @@ -6,7 +6,7 @@ "platform": "nextui", "display_name": "NextUI", "version": "2026-10-20", - "generated": "2026-11-05T18:33:47Z", + "generated": "2026-11-05T23:59:37Z", "base_destination": "", "detect": [ { @@ -5635,7 +5635,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "fbneo-cheats", @@ -19364,7 +19365,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -19597,7 +19599,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ] } \ No newline at end of file diff --git a/install/recalbox.json b/install/recalbox.json index b947571f5..9b97c9d50 100644 --- a/install/recalbox.json +++ b/install/recalbox.json @@ -62833,7 +62833,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -63066,7 +63067,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ], "save_directory": { diff --git a/install/retroarch.json b/install/retroarch.json index 166d197d3..11b60b965 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -116923,7 +116923,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "cemu-gameprofiles", @@ -117879,7 +117880,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dinothawr", @@ -118327,7 +118329,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -118383,7 +118386,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -127054,7 +127058,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -140783,7 +140788,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -141016,7 +141022,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -141388,7 +141395,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "ppsspp-assets", @@ -142076,7 +142084,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 }, { "ref": "yaps2-resources", @@ -142536,7 +142545,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/retrobat.json b/install/retrobat.json index 3a2e41f89..ca7459771 100644 --- a/install/retrobat.json +++ b/install/retrobat.json @@ -89075,7 +89075,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -89308,7 +89309,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -89680,7 +89682,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ], "save_directory": { diff --git a/install/retrodeck.json b/install/retrodeck.json index b70189e03..e94e2c442 100644 --- a/install/retrodeck.json +++ b/install/retrodeck.json @@ -137751,7 +137751,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "fbneo-cheats", @@ -151480,7 +151481,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -151713,7 +151715,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -152085,7 +152088,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -152549,7 +152553,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/retropie.json b/install/retropie.json index c255b9d1f..3db3da56e 100644 --- a/install/retropie.json +++ b/install/retropie.json @@ -124071,7 +124071,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dolphin-sys", @@ -132742,7 +132743,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -146471,7 +146473,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -146704,7 +146707,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -147076,7 +147080,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -147536,7 +147541,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ] } \ No newline at end of file diff --git a/install/rocknix.json b/install/rocknix.json index 793ca6670..f06bd3144 100644 --- a/install/rocknix.json +++ b/install/rocknix.json @@ -86538,7 +86538,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -86986,7 +86987,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -87042,7 +87044,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -95719,7 +95722,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -109448,7 +109452,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -109681,7 +109686,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -110053,7 +110059,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "lrps2", @@ -110069,7 +110076,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "ppsspp-assets", @@ -110757,7 +110765,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 }, { "ref": "yaps2-resources", @@ -111217,7 +111226,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ] } \ No newline at end of file diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index 4e7d1ec20..264c08558 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -429,6 +429,11 @@ "minimum": 0, "description": "With type zip: leading path parts dropped from each member." }, + "archive_size": { + "type": "integer", + "minimum": 1, + "description": "Bytes of the source archive when it was last downloaded: what fetching the tree costs. Absent when not recorded." + }, "source_path": { "type": "string", "description": "With type tarball: only members below this path are kept, relative to it." diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 354964481..8a62186a3 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -4949,9 +4949,12 @@ def _manifest_data_directories( (strip_components for a ZIP, source_path for a tarball, exclude) come from _data_dirs.yml, the same refresh_data_dirs applies. Every file carries the sha1 and size the cache holds, so a member the source - changed since is refused rather than installed. + changed since is refused rather than installed. archive_size is the + size of the archive refresh_data_dirs last downloaded, what fetching + the tree costs: a tarball whose tree is 10 KB is a download of 21 MB. """ grouped: dict[tuple[str, str], dict] = {} + versions: dict[str, dict[str, dict]] = {} for src, full in members: ref, systems, prefix = refs[full] entry = registry.get(ref, {}) @@ -4976,6 +4979,17 @@ def _manifest_data_directories( group["source_path"] = str(entry.get("source_path", "")).format(version=version) if entry.get("exclude"): group["exclude"] = [str(e) for e in entry["exclude"]] + cache = entry.get("local_cache") + if cache: + recorded = Path(cache).parent / ".versions.json" + if str(recorded) not in versions: + try: + versions[str(recorded)] = json.loads(recorded.read_text()) + except FileNotFoundError: + versions[str(recorded)] = {} + size = versions[str(recorded)].get(ref, {}).get("archive_size") + if isinstance(size, int) and size > 0: + group["archive_size"] = size group["systems"].extend(s for s in systems if s not in group["systems"]) rel = full[len(prefix):].lstrip("/") if prefix else full hashes = compute_hashes(src, frozenset({"sha1"})) diff --git a/scripts/refresh_data_dirs.py b/scripts/refresh_data_dirs.py index 7b74f7be1..7c3d63562 100644 --- a/scripts/refresh_data_dirs.py +++ b/scripts/refresh_data_dirs.py @@ -24,7 +24,7 @@ import time import urllib.error import urllib.request import zipfile -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path from artifacts import file_lock as _file_lock, set_plain_mode from common import InputMissing, github_token, platform_arg, require_platform, suggest_names, yaml_load @@ -270,10 +270,12 @@ def _download_and_extract( source_path: str, local_cache: str, exclude: list[str] | None = None, + measured: Callable[[int], None] | None = None, ) -> int: """Download tarball, extract source_path subtree to local_cache. - Returns the number of files extracted. + Returns the number of files extracted. measured receives the size of + the archive downloaded. """ exclude = exclude or [] cache_dir = Path(local_cache) @@ -290,6 +292,8 @@ def _download_and_extract( if not chunk: break f.write(chunk) + if measured is not None: + measured(tarball_path.stat().st_size) log.info("extracting %s -> %s", source_path, local_cache) @@ -347,6 +351,7 @@ def _download_and_extract_zip( local_cache: str, exclude: list[str] | None = None, strip_components: int = 0, + measured: Callable[[int], None] | None = None, ) -> int: """Download ZIP, extract to local_cache. Returns file count. @@ -369,6 +374,8 @@ def _download_and_extract_zip( if not chunk: break f.write(chunk) + if measured is not None: + measured(zip_path.stat().st_size) extract_dir = Path(tmpdir) / "extract" extract_dir.mkdir() @@ -408,6 +415,46 @@ def _download_and_extract_zip( return file_count +def _measure_archive(source_url: str) -> int | None: + """The size of the archive at source_url, read by streaming it: a + tarball GitHub builds on request is sent without a Content-Length.""" + req = urllib.request.Request(source_url, headers={"User-Agent": USER_AGENT}) + size = 0 + try: + with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp: + while chunk := resp.read(65536): + size += len(chunk) + except _TRANSIENT as exc: + log.warning("could not measure %s: %s", source_url, exc) + return None + return size or None + + +def _record_archive_size(key: str, entry: dict, versions_path: str) -> None: + """Record the archive size of a tree current since before it was kept. + + The installer announces a tree's download as that size: a tarball + whose tree is 10 KB is a download of 21 MB. Measured, not refetched, + so the tree the packs carry stays the one recorded. + """ + path = Path(versions_path) + with _file_lock(path.with_name(f".{path.name}.lock")): + versions = _load_versions(versions_path) + record = versions.get(key) + if record is None or "archive_size" in record: + return + version = entry.get("version", "master") + size = _measure_archive(entry["source_url"].format(version=version)) + if size is None: + return + with _file_lock(path.with_name(f".{path.name}.lock")): + versions = _load_versions(versions_path) + if key in versions and "archive_size" not in versions[key]: + versions[key]["archive_size"] = size + _save_versions(versions, versions_path) + log.info("[%s] archive size recorded: %d bytes", key, size) + + def _get_remote_etag(source_url: str) -> str | None: """HEAD request to get ETag or Last-Modified for freshness check.""" try: @@ -456,6 +503,7 @@ def refresh_entry( return None if not needs: log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag)) + _record_archive_size(key, entry, versions_path) return False with cache_lock(entry["local_cache"]): return _refresh_entry(key, entry, force, dry_run, versions_path) @@ -516,13 +564,19 @@ def _refresh_entry( else: remote_tag = get_remote_sha(entry["source_url"], version) + archive_sizes: list[int] = [] + def fetch() -> int: + archive_sizes.clear() if source_type == "zip": return _download_and_extract_zip( - source_url, local_cache, exclude, entry.get("strip_components", 0) + source_url, local_cache, exclude, entry.get("strip_components", 0), + measured=archive_sizes.append, ) source_path = entry["source_path"].format(version=version) - return _download_and_extract(source_url, source_path, local_cache, exclude) + return _download_and_extract( + source_url, source_path, local_cache, exclude, measured=archive_sizes.append + ) # An archive that reads but holds nothing under the path is the same on # every try: only a failed or unreadable download is tried again. @@ -540,7 +594,10 @@ def _refresh_entry( log.warning("[%s] download failed (%s), trying again in %d s", key, exc, wait) time.sleep(wait) - _record_version(key, {"sha": remote_tag or "", "version": version}, versions_path) + record = {"sha": remote_tag or "", "version": version} + if archive_sizes: + record["archive_size"] = archive_sizes[-1] + _record_version(key, record, versions_path) log.info("[%s] refreshed: %d files extracted to %s", key, file_count, local_cache) return True diff --git a/tests/test_install.py b/tests/test_install.py index b8c72ae82..a918d7d06 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3888,6 +3888,37 @@ class DataDirectoriesAreInstalled(unittest.TestCase): self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) self.assertIn(f"{install.format_size(bios + archive)} downloaded in", proc.stdout) + def test_the_plan_announces_the_archive_not_the_tree(self): + """yaps2's tarball is 21.2 MB for a 6.5 MB tree, Cemu's 21.4 MB for + 9.9 KB: the plan announced the tree, and the run downloaded far more.""" + from test_install_selection import FakeRepository # noqa: PLC0415 + + member = b"profile" + archive_size = 21_360_579 + with FakeRepository() as repo: + listed = json.loads((repo.root / "install" / "retroarch.json").read_text()) + listed["data_directories"] = [{ + "ref": "cemu-gameprofiles", "dest": "cemu", "type": "tarball", + "url": "https://example.invalid/Cemu.tar.gz", "source_path": "Cemu-1/bin", + "systems": ["nintendo-gba"], "archive_size": archive_size, + "files": {"gameProfiles/a.ini": {"sha1": hashlib.sha1(member).hexdigest(), + "size": len(member)}}}] + (repo.root / "install" / "retroarch.json").write_text(json.dumps(listed)) + proc = repo.run("--no-input", "--dry-run") + bios = sum(len(payload) for payload, *_ in FakeRepository.PAYLOADS.values()) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn(f"({install.format_size(bios + archive_size)})", proc.stdout) + self.assertIn(f"(a {install.format_size(archive_size)} archive)", proc.stdout) + + def test_a_pack_or_an_old_list_counts_the_files_themselves(self): + needed = {"a": {"size": 7}} + entry = {"archive_size": 21_360_579} + self.assertEqual(install.data_fetch_bytes(entry, needed), 21_360_579) + self.assertEqual(install.data_fetch_bytes({}, needed), 7) + self.assertEqual(install.data_fetch_bytes(entry, {}), 0) + with unittest.mock.patch.object(install.LOCAL, "pack", object()): + self.assertEqual(install.data_fetch_bytes(entry, needed), 7) + def test_verified_members_land_the_rest_are_said(self): work = Path(_tempdir(self)) archive = self._zip(work) diff --git a/tests/test_refresh_data_dirs.py b/tests/test_refresh_data_dirs.py index 42f21d122..0a4af04fd 100644 --- a/tests/test_refresh_data_dirs.py +++ b/tests/test_refresh_data_dirs.py @@ -93,6 +93,26 @@ class RefreshConcurrency(unittest.TestCase): ) self.assertEqual(seen, [True]) + def test_the_archive_size_is_recorded_without_replacing_the_tree(self): + """The installer announces a tree's download as the archive's size: + a tarball extracted to 10 KB is a download of 21 MB. A tree current + before the size was kept is measured, its files left as they are.""" + payload = _zip_bytes({"a.bin": b"x" * 100}) + entry = {"source_type": "zip", "source_url": "https://x/pak.zip", + "local_cache": str(self.cache)} + with mock.patch.object(rdd.urllib.request, "urlopen", + lambda *a, **k: _Response(payload)), \ + mock.patch.object(rdd, "_get_remote_etag", return_value="e1"): + self.assertTrue(rdd.refresh_entry("pak", entry, versions_path=self.versions)) + self.assertEqual(rdd._load_versions(self.versions)["pak"]["archive_size"], + len(payload)) + rdd._record_version("pak", {"sha": "e1"}, self.versions) + (self.cache / "a.bin").write_bytes(b"kept") + self.assertFalse(rdd.refresh_entry("pak", entry, versions_path=self.versions)) + self.assertEqual(rdd._load_versions(self.versions)["pak"], + {"sha": "e1", "archive_size": len(payload)}) + self.assertEqual((self.cache / "a.bin").read_bytes(), b"kept") + def test_a_current_tree_is_not_waited_for(self): """A pipeline reads every tree for its whole run. A build beside it waited on the exclusive lock before asking the remote, which would @@ -100,7 +120,7 @@ class RefreshConcurrency(unittest.TestCase): import fcntl # noqa: PLC0415 self.cache.mkdir(parents=True) - rdd._record_version("pak", {"sha": "e1"}, self.versions) + rdd._record_version("pak", {"sha": "e1", "archive_size": 1}, self.versions) entry = {"source_type": "zip", "source_url": "https://x/pak.zip", "local_cache": str(self.cache)} result: list = [] @@ -229,7 +249,7 @@ class DryRunReport(unittest.TestCase): def test_the_remote_tag_is_shown_beside_the_cached_one(self): with tempfile.TemporaryDirectory() as tmp: versions = Path(tmp) / "versions.json" - versions.write_text(json.dumps({"k": {"etag": '"2725165595"'}})) + versions.write_text(json.dumps({"k": {"etag": '"2725165595"', "archive_size": 1}})) entry = {"source_type": "zip", "source_url": "https://x.invalid/a.zip", "local_cache": tmp} with mock.patch.object(rdd, "_get_remote_etag", return_value='"2685251384"'), \ @@ -249,5 +269,30 @@ class DryRunReport(unittest.TestCase): ) +class TheManifestCarriesTheArchiveSize(unittest.TestCase): + """The size refresh_data_dirs recorded reaches the install list, which + the installer announces as the tree's download.""" + + def test_recorded_size_is_written_and_an_unrecorded_one_left_out(self): + from generate_pack import _manifest_data_directories # noqa: PLC0415 + + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + data = Path(tmp) / "data" + for ref in ("pak", "old"): + (data / ref).mkdir(parents=True) + (data / ref / "a.bin").write_bytes(b"abc") + (data / ".versions.json").write_text(json.dumps( + {"pak": {"sha": "e1", "archive_size": 21_360_579}, "old": {"sha": "e2"}})) + registry = {ref: {"source_type": "zip", "source_url": f"https://x/{ref}.zip", + "local_cache": str(data / ref)} for ref in ("pak", "old")} + members = [(str(data / ref / "a.bin"), f"{ref}/a.bin") for ref in ("pak", "old")] + refs = {f"{ref}/a.bin": (ref, ("s",), ref) for ref in ("pak", "old")} + listed = {entry["ref"]: entry for entry in + _manifest_data_directories(members, refs, registry, "")} + self.assertEqual(listed["pak"]["archive_size"], 21_360_579) + self.assertNotIn("archive_size", listed["old"]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index 0d6e5141b..796938117 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1620,8 +1620,11 @@ built (`strip_components` for a ZIP, `source_path` for a tarball, `exclude`), and writes a file only when its hash matches the list. A file the source has changed since is left out and counted, and the pack still holds it. `--check` counts the data files missing, `--dry-run` names each tree it would fetch, and -narrowing with `--system`, `--core` or `--target` keeps only the trees of the -systems that stay. +the size announced for a tree is its archive's (`archive_size`, recorded when +`refresh_data_dirs.py` last downloaded it): one missing file fetches the whole +archive, and a tarball can be a thousand times its tree. Narrowing with +`--system`, `--core` or `--target` keeps only the trees of the systems that +stay. ## What the bootstrap does @@ -1802,7 +1805,7 @@ other schemas under `api/v1/schemas/`. | `games_unavailable` | `platform`, `reason`, `message` | With `--for-games`, what the games need could not be read (`offline`, `network`, `invalid`, `stale`, `no_sets`, `no_folder`): the run keeps every file of the list, except for a `FOLDER` that does not exist, where it stops | | `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply | | `declared` | `platform`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `dests` | Right after `check`, the same counts over the files the platform's own list names (`cores: null`), and their `dests` in the order of the `file` events. A setup that checks its own list reads its answer here, with or without `--declared-only` | -| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes they write once in place | +| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes fetching them costs: each tree's archive, or the files themselves from a pack or a list that records no archive size | | `found` | `platform`, `dest`, `folder`, `path`, `member`, `reason` | Each file to install that this machine already holds, after `check` and before anything is written, in `--check` and `--dry-run` too. `folder` is the `--from` folder as given, or the `bios_dir` of the check when the BIOS folder holds the file at another place, and then `reason` says where in words (`nested folder: ...` for a misplaced file, `the same file is at ...` for one the list installs there too). `path` is the file, and `member` the ZIP member holding the bytes when `path` is an archive, an empty string otherwise. Its copy is reported by a `downloaded` event | | `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) | | `saves` | `platform`, `files`, `folder` | After `choices` and before `checking`, when the files the run installs (once `--system` and the other choices narrowed the list) hold some read from the frontend's save directory and that directory is known: how many, and the `folder` they go to, where `` and `` stand for one folder per system or per core (the screen and `--plain` say it in words: "a folder per core under /sdcard/RetroArch/saves"). When it is not known, the files are counted in the summary's `skipped` instead (`no_save_directory`) |