kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-06 10:03:49 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix: announce a data tree at its archive size

Browse Source 
A tarball tree was announced at its extracted size: yaps2 at 6.5 MB for a 21.2 MB download, Cemu's game profiles at 9.9 KB for 21.4 MB. refresh_data_dirs records the size of each archive it downloads (a tree current from before is measured, not refetched), the install lists carry it as archive_size, and the plan, the menu and the data event count it.
This commit is contained in:
Abdessamad Derraz committed 2026-11-06 08:59:10 +01:00
1 parent eb99131a6e
commit 725325f0a1
24 files changed
+352 -103
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

install.ps1
+1 -1
View File
@@ -16,7 +16,7 @@
# and Invoke-WebRequest throws on a cut transfer before the hash is read:
# the site's build rewrites this line there (scripts/sitebootstrap.py).
$mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that."
$defaultInstallSha256 = "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51"
$defaultInstallSha256 = "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f"
$maximumInstallerBytes = 2MB
$installerExit = 0
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
install.py
+26 -6
View File
@@ -3708,6 +3708,9 @@ def _check_data_directory(entry: object, where: str) -> None:
strip = entry.get("strip_components", 0)
if not isinstance(strip, int) or isinstance(strip, bool) or not 0 <= strip <= 16:
raise ValueError(f"{where}.strip_components is invalid")
size = entry.get("archive_size", 1)
if not isinstance(size, int) or isinstance(size, bool) or size < 1:
raise ValueError(f"{where}.archive_size is invalid")
if kind == "tarball" and (
not isinstance(entry.get("source_path"), str) or not entry["source_path"]
):
@@ -6873,11 +6876,12 @@ def _print_plan(
# files, and 2636 data files" read as two kinds of one thing. The kinds
# follow, adding up to it.
total = counts.files + tree_files
parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_counts.size)})"]
tree_bytes = sum(data_fetch_bytes(entry, needed) for entry, needed in trees)
parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_bytes)})"]
if total else [])
split = parts_words(counts if (manifest or {}).get("categorized")
else tally({"size": e.get("size")} for e in to_download),
tree_files, tree_counts.size)
tree_files, tree_bytes)
if split and trees:
split += f", the data files extracted from {_plural(len(trees), 'archive')}"
# A pack is copied from, not downloaded: "Would download 23 files" was
@@ -6952,8 +6956,11 @@ def _print_plan(
"every system and its files.")
for entry, needed in trees:
source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"]
fetched = data_fetch_bytes(entry, needed)
archive = ("" if fetched == sum(meta.get("size", 0) for meta in needed.values())
else f" (a {format_size(fetched)} archive)")
print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, "
f"from {source}")
f"from {source}{archive}")
for folder, (count, size) in sorted((kept or {}).items()):
print(f"\n Would keep {_plural(count, 'file')} you have ({format_size(size)}), "
f"whose contents differ, in {folder}")
@@ -7184,6 +7191,19 @@ def data_directory_state(entry: dict, bios_path: Path,
return root, needed, present
def data_fetch_bytes(entry: dict, needed: dict[str, dict]) -> int:
"""What getting a tree's missing files costs: its source archive, which
comes whole for one file, or from a pack the files themselves. A list
that records no archive size gives the files' own sizes: a tarball whose
tree is 10 KB was announced as 10 KB and was a download of 21 MB."""
if not needed:
return 0
size = entry.get("archive_size")
if LOCAL.pack is None and isinstance(size, int) and size > 0:
return size
return sum(meta.get("size", 0) for meta in needed.values())
def install_data_directory(entry: dict, bios_path: Path,
state: tuple[Path, dict[str, dict], int] | None = None,
progress: Progress | None = None) -> dict:
@@ -8963,7 +8983,7 @@ def found_head(files: list[dict], trees: list[dict], bios_path: Path) -> str:
for tree in trees:
_root, wanted, _present = data_directory_state(tree, bios_path, remember=True)
data += len(wanted)
data_bytes += sum(meta.get("size", 0) for meta in wanted.values())
data_bytes += data_fetch_bytes(tree, wanted)
left = [entry for entry in needed if id(entry) not in copies]
rest = len(left) + data
where = ", ".join(f"{count} found in {_last_name(label)}"
@@ -15683,8 +15703,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
plat_name, manifest, bios_path, files, up_to_date, mismatched,
sum(len(state[1]) for _entry, state in data_states), len(omitted_files),
chosen=selection[0] if selection else None,
data_bytes=sum(meta.get("size", 0) for _entry, state in data_states
for meta in state[1].values()),
data_bytes=sum(data_fetch_bytes(entry, state[1])
for entry, state in data_states),
)
emit_copies(plat_name, to_download, copies, bios_index, bios_path, files)
emit_unreadable(plat_name, unread)
install.sh
+1 -1
View File
@@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins
# curl or wget stop on a cut transfer before the hash is read: the site's
# build rewrites this line there (scripts/sitebootstrap.py).
MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that."
DEFAULT_INSTALL_SHA256="3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51"
DEFAULT_INSTALL_SHA256="567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f"
MAX_INSTALLER_BYTES=2097152
cleanup() {
install/amberelec.json
+8 -4
View File
@@ -63489,7 +63489,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "fbneo-cheats",
@@ -77218,7 +77219,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -77451,7 +77453,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -77823,7 +77826,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
}
]
}
install/batocera.json
+4 -2
View File
@@ -225291,7 +225291,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -225524,7 +225525,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
}
],
"save_directory": {
install/emudeck.json
+18 -9
View File
@@ -100486,7 +100486,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "dinothawr",
@@ -100934,7 +100935,8 @@
"size": 105
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 5752250
},
{
"ref": "dirksimple",
@@ -100990,7 +100992,8 @@
"size": 12988
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 175384
},
{
"ref": "dolphin-sys",
@@ -109666,7 +109669,8 @@
"strip_components": 2,
"exclude": [
"Themes"
]
],
"archive_size": 3197914
},
{
"ref": "fbneo-cheats",
@@ -123395,7 +123399,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -123628,7 +123633,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -124000,7 +124006,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "lrps2",
@@ -124016,7 +124023,8 @@
"size": 1814032
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 318896
},
{
"ref": "yaps2-resources",
@@ -124476,7 +124484,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
],
"save_directory": {
install/emuelec.json
+10 -5
View File
@@ -69309,7 +69309,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "dinothawr",
@@ -69757,7 +69758,8 @@
"size": 105
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 5752250
},
{
"ref": "fbneo-cheats",
@@ -83486,7 +83488,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -83719,7 +83722,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -84091,7 +84095,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
}
]
}
install/knulli.json
+8 -4
View File
@@ -157933,7 +157933,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -158166,7 +158167,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "lrps2",
@@ -158182,7 +158184,8 @@
"size": 1814032
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 318896
},
{
"ref": "ppsspp-assets",
@@ -158870,7 +158873,8 @@
"size": 15005
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 9844388
}
],
"save_directory": {
install/lakka.json
+14 -7
View File
@@ -115932,7 +115932,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "cemu-gameprofiles",
@@ -116888,7 +116889,8 @@
"size": 36
}
},
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default"
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default",
"archive_size": 21360579
},
{
"ref": "dirksimple",
@@ -116944,7 +116946,8 @@
"size": 12988
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 175384
},
{
"ref": "fbneo-cheats",
@@ -130673,7 +130676,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -130906,7 +130910,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -131278,7 +131283,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "yaps2-resources",
@@ -131738,7 +131744,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
],
"save_directory": {
install/launchers/index.json
+8 -8
View File
@@ -27,8 +27,8 @@
{
"dest": "roms/ports/retrobios/install.py",
"repo_path": "install.py",
"size": 1107258,
"sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51",
"size": 1108265,
"sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f",
"executable": false
},
{
@@ -71,8 +71,8 @@
"muos": {
"archive": {
"dest": "ARCHIVE/RetroBIOS.muxapp",
"size": 1160885,
"sha256": "5ccc4ec3df34a1fef96caea9e84218de1fb92331664047c36d6863c11973dc3d"
"size": 1161892,
"sha256": "1031750dcd599e0c4dd2e6c39296b3bcada68494a4837ad442150da64b67a55c"
},
"members": [
{
@@ -92,8 +92,8 @@
{
"name": "RetroBIOS/install.py",
"repo_path": "install.py",
"size": 1107258,
"sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51",
"size": 1108265,
"sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f",
"executable": false
},
{
@@ -166,8 +166,8 @@
{
"dest": ".config/modules/retrobios/install.py",
"repo_path": "install.py",
"size": 1107258,
"sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51",
"size": 1108265,
"sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f",
"executable": false
},
{
install/myminui.json
+9 -5
View File
@@ -6,7 +6,7 @@
"platform": "myminui",
"display_name": "MyMinUI",
"version": "2026-08-09",
"generated": "2026-11-05T18:33:47Z",
"generated": "2026-11-05T23:59:37Z",
"base_destination": "",
"detect": [
{
@@ -18957,7 +18957,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-cheats",
@@ -32686,7 +32687,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -32919,7 +32921,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "fbneo-samples",
@@ -33152,7 +33155,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
}
]
}
install/nextui.json
+7 -4
View File
@@ -6,7 +6,7 @@
"platform": "nextui",
"display_name": "NextUI",
"version": "2026-10-20",
"generated": "2026-11-05T18:33:47Z",
"generated": "2026-11-05T23:59:37Z",
"base_destination": "",
"detect": [
{
@@ -5635,7 +5635,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "fbneo-cheats",
@@ -19364,7 +19365,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -19597,7 +19599,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
}
]
}
install/recalbox.json
+4 -2
View File
@@ -62833,7 +62833,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -63066,7 +63067,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
}
],
"save_directory": {
install/retroarch.json
+20 -10
View File
@@ -116923,7 +116923,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "cemu-gameprofiles",
@@ -117879,7 +117880,8 @@
"size": 36
}
},
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default"
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default",
"archive_size": 21360579
},
{
"ref": "dinothawr",
@@ -118327,7 +118329,8 @@
"size": 105
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 5752250
},
{
"ref": "dirksimple",
@@ -118383,7 +118386,8 @@
"size": 12988
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 175384
},
{
"ref": "dolphin-sys",
@@ -127054,7 +127058,8 @@
"strip_components": 2,
"exclude": [
"Themes"
]
],
"archive_size": 3197914
},
{
"ref": "fbneo-cheats",
@@ -140783,7 +140788,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -141016,7 +141022,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -141388,7 +141395,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "ppsspp-assets",
@@ -142076,7 +142084,8 @@
"size": 15005
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 9844388
},
{
"ref": "yaps2-resources",
@@ -142536,7 +142545,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
],
"save_directory": {
install/retrobat.json
+6 -3
View File
@@ -89075,7 +89075,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -89308,7 +89309,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -89680,7 +89682,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
}
],
"save_directory": {
install/retrodeck.json
+10 -5
View File
@@ -137751,7 +137751,8 @@
"size": 36
}
},
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default"
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default",
"archive_size": 21360579
},
{
"ref": "fbneo-cheats",
@@ -151480,7 +151481,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -151713,7 +151715,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -152085,7 +152088,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "yaps2-resources",
@@ -152549,7 +152553,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
],
"save_directory": {
install/retropie.json
+12 -6
View File
@@ -124071,7 +124071,8 @@
"size": 36
}
},
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default"
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default",
"archive_size": 21360579
},
{
"ref": "dolphin-sys",
@@ -132742,7 +132743,8 @@
"strip_components": 2,
"exclude": [
"Themes"
]
],
"archive_size": 3197914
},
{
"ref": "fbneo-cheats",
@@ -146471,7 +146473,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -146704,7 +146707,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -147076,7 +147080,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "yaps2-resources",
@@ -147536,7 +147541,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
]
}
install/rocknix.json
+20 -10
View File
@@ -86538,7 +86538,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "dinothawr",
@@ -86986,7 +86987,8 @@
"size": 105
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 5752250
},
{
"ref": "dirksimple",
@@ -87042,7 +87044,8 @@
"size": 12988
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 175384
},
{
"ref": "dolphin-sys",
@@ -95719,7 +95722,8 @@
"strip_components": 2,
"exclude": [
"Themes"
]
],
"archive_size": 3197914
},
{
"ref": "fbneo-cheats",
@@ -109448,7 +109452,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -109681,7 +109686,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -110053,7 +110059,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "lrps2",
@@ -110069,7 +110076,8 @@
"size": 1814032
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 318896
},
{
"ref": "ppsspp-assets",
@@ -110757,7 +110765,8 @@
"size": 15005
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 9844388
},
{
"ref": "yaps2-resources",
@@ -111217,7 +111226,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
]
}
schemas/install-manifest.schema.json
+5
View File
@@ -429,6 +429,11 @@
"minimum": 0,
"description": "With type zip: leading path parts dropped from each member."
},
"archive_size": {
"type": "integer",
"minimum": 1,
"description": "Bytes of the source archive when it was last downloaded: what fetching the tree costs. Absent when not recorded."
},
"source_path": {
"type": "string",
"description": "With type tarball: only members below this path are kept, relative to it."
scripts/generate_pack.py
+15 -1
View File
@@ -4949,9 +4949,12 @@ def _manifest_data_directories(
(strip_components for a ZIP, source_path for a tarball, exclude) come
from _data_dirs.yml, the same refresh_data_dirs applies. Every file
carries the sha1 and size the cache holds, so a member the source
changed since is refused rather than installed.
changed since is refused rather than installed. archive_size is the
size of the archive refresh_data_dirs last downloaded, what fetching
the tree costs: a tarball whose tree is 10 KB is a download of 21 MB.
"""
grouped: dict[tuple[str, str], dict] = {}
versions: dict[str, dict[str, dict]] = {}
for src, full in members:
ref, systems, prefix = refs[full]
entry = registry.get(ref, {})
@@ -4976,6 +4979,17 @@ def _manifest_data_directories(
group["source_path"] = str(entry.get("source_path", "")).format(version=version)
if entry.get("exclude"):
group["exclude"] = [str(e) for e in entry["exclude"]]
cache = entry.get("local_cache")
if cache:
recorded = Path(cache).parent / ".versions.json"
if str(recorded) not in versions:
try:
versions[str(recorded)] = json.loads(recorded.read_text())
except FileNotFoundError:
versions[str(recorded)] = {}
size = versions[str(recorded)].get(ref, {}).get("archive_size")
if isinstance(size, int) and size > 0:
group["archive_size"] = size
group["systems"].extend(s for s in systems if s not in group["systems"])
rel = full[len(prefix):].lstrip("/") if prefix else full
hashes = compute_hashes(src, frozenset({"sha1"}))
scripts/refresh_data_dirs.py
+62 -5
View File
@@ -24,7 +24,7 @@ import time
import urllib.error
import urllib.request
import zipfile
from collections.abc import Iterator
from collections.abc import Callable, Iterator
from pathlib import Path
from artifacts import file_lock as _file_lock, set_plain_mode
from common import InputMissing, github_token, platform_arg, require_platform, suggest_names, yaml_load
@@ -270,10 +270,12 @@ def _download_and_extract(
source_path: str,
local_cache: str,
exclude: list[str] | None = None,
measured: Callable[[int], None] | None = None,
) -> int:
"""Download tarball, extract source_path subtree to local_cache.
Returns the number of files extracted.
Returns the number of files extracted. measured receives the size of
the archive downloaded.
"""
exclude = exclude or []
cache_dir = Path(local_cache)
@@ -290,6 +292,8 @@ def _download_and_extract(
if not chunk:
break
f.write(chunk)
if measured is not None:
measured(tarball_path.stat().st_size)
log.info("extracting %s -> %s", source_path, local_cache)
@@ -347,6 +351,7 @@ def _download_and_extract_zip(
local_cache: str,
exclude: list[str] | None = None,
strip_components: int = 0,
measured: Callable[[int], None] | None = None,
) -> int:
"""Download ZIP, extract to local_cache. Returns file count.
@@ -369,6 +374,8 @@ def _download_and_extract_zip(
if not chunk:
break
f.write(chunk)
if measured is not None:
measured(zip_path.stat().st_size)
extract_dir = Path(tmpdir) / "extract"
extract_dir.mkdir()
@@ -408,6 +415,46 @@ def _download_and_extract_zip(
return file_count
def _measure_archive(source_url: str) -> int | None:
"""The size of the archive at source_url, read by streaming it: a
tarball GitHub builds on request is sent without a Content-Length."""
req = urllib.request.Request(source_url, headers={"User-Agent": USER_AGENT})
size = 0
try:
with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp:
while chunk := resp.read(65536):
size += len(chunk)
except _TRANSIENT as exc:
log.warning("could not measure %s: %s", source_url, exc)
return None
return size or None
def _record_archive_size(key: str, entry: dict, versions_path: str) -> None:
"""Record the archive size of a tree current since before it was kept.
The installer announces a tree's download as that size: a tarball
whose tree is 10 KB is a download of 21 MB. Measured, not refetched,
so the tree the packs carry stays the one recorded.
"""
path = Path(versions_path)
with _file_lock(path.with_name(f".{path.name}.lock")):
versions = _load_versions(versions_path)
record = versions.get(key)
if record is None or "archive_size" in record:
return
version = entry.get("version", "master")
size = _measure_archive(entry["source_url"].format(version=version))
if size is None:
return
with _file_lock(path.with_name(f".{path.name}.lock")):
versions = _load_versions(versions_path)
if key in versions and "archive_size" not in versions[key]:
versions[key]["archive_size"] = size
_save_versions(versions, versions_path)
log.info("[%s] archive size recorded: %d bytes", key, size)
def _get_remote_etag(source_url: str) -> str | None:
"""HEAD request to get ETag or Last-Modified for freshness check."""
try:
@@ -456,6 +503,7 @@ def refresh_entry(
return None
if not needs:
log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag))
_record_archive_size(key, entry, versions_path)
return False
with cache_lock(entry["local_cache"]):
return _refresh_entry(key, entry, force, dry_run, versions_path)
@@ -516,13 +564,19 @@ def _refresh_entry(
else:
remote_tag = get_remote_sha(entry["source_url"], version)
archive_sizes: list[int] = []
def fetch() -> int:
archive_sizes.clear()
if source_type == "zip":
return _download_and_extract_zip(
source_url, local_cache, exclude, entry.get("strip_components", 0)
source_url, local_cache, exclude, entry.get("strip_components", 0),
measured=archive_sizes.append,
)
source_path = entry["source_path"].format(version=version)
return _download_and_extract(source_url, source_path, local_cache, exclude)
return _download_and_extract(
source_url, source_path, local_cache, exclude, measured=archive_sizes.append
)
# An archive that reads but holds nothing under the path is the same on
# every try: only a failed or unreadable download is tried again.
@@ -540,7 +594,10 @@ def _refresh_entry(
log.warning("[%s] download failed (%s), trying again in %d s", key, exc, wait)
time.sleep(wait)
_record_version(key, {"sha": remote_tag or "", "version": version}, versions_path)
record = {"sha": remote_tag or "", "version": version}
if archive_sizes:
record["archive_size"] = archive_sizes[-1]
_record_version(key, record, versions_path)
log.info("[%s] refreshed: %d files extracted to %s", key, file_count, local_cache)
return True
tests/test_install.py
+31
View File
@@ -3888,6 +3888,37 @@ class DataDirectoriesAreInstalled(unittest.TestCase):
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertIn(f"{install.format_size(bios + archive)} downloaded in", proc.stdout)
def test_the_plan_announces_the_archive_not_the_tree(self):
"""yaps2's tarball is 21.2 MB for a 6.5 MB tree, Cemu's 21.4 MB for
9.9 KB: the plan announced the tree, and the run downloaded far more."""
from test_install_selection import FakeRepository # noqa: PLC0415
member = b"profile"
archive_size = 21_360_579
with FakeRepository() as repo:
listed = json.loads((repo.root / "install" / "retroarch.json").read_text())
listed["data_directories"] = [{
"ref": "cemu-gameprofiles", "dest": "cemu", "type": "tarball",
"url": "https://example.invalid/Cemu.tar.gz", "source_path": "Cemu-1/bin",
"systems": ["nintendo-gba"], "archive_size": archive_size,
"files": {"gameProfiles/a.ini": {"sha1": hashlib.sha1(member).hexdigest(),
"size": len(member)}}}]
(repo.root / "install" / "retroarch.json").write_text(json.dumps(listed))
proc = repo.run("--no-input", "--dry-run")
bios = sum(len(payload) for payload, *_ in FakeRepository.PAYLOADS.values())
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertIn(f"({install.format_size(bios + archive_size)})", proc.stdout)
self.assertIn(f"(a {install.format_size(archive_size)} archive)", proc.stdout)
def test_a_pack_or_an_old_list_counts_the_files_themselves(self):
needed = {"a": {"size": 7}}
entry = {"archive_size": 21_360_579}
self.assertEqual(install.data_fetch_bytes(entry, needed), 21_360_579)
self.assertEqual(install.data_fetch_bytes({}, needed), 7)
self.assertEqual(install.data_fetch_bytes(entry, {}), 0)
with unittest.mock.patch.object(install.LOCAL, "pack", object()):
self.assertEqual(install.data_fetch_bytes(entry, needed), 7)
def test_verified_members_land_the_rest_are_said(self):
work = Path(_tempdir(self))
archive = self._zip(work)
tests/test_refresh_data_dirs.py
+47 -2
View File
@@ -93,6 +93,26 @@ class RefreshConcurrency(unittest.TestCase):
)
self.assertEqual(seen, [True])
def test_the_archive_size_is_recorded_without_replacing_the_tree(self):
"""The installer announces a tree's download as the archive's size:
a tarball extracted to 10 KB is a download of 21 MB. A tree current
before the size was kept is measured, its files left as they are."""
payload = _zip_bytes({"a.bin": b"x" * 100})
entry = {"source_type": "zip", "source_url": "https://x/pak.zip",
"local_cache": str(self.cache)}
with mock.patch.object(rdd.urllib.request, "urlopen",
lambda *a, **k: _Response(payload)), \
mock.patch.object(rdd, "_get_remote_etag", return_value="e1"):
self.assertTrue(rdd.refresh_entry("pak", entry, versions_path=self.versions))
self.assertEqual(rdd._load_versions(self.versions)["pak"]["archive_size"],
len(payload))
rdd._record_version("pak", {"sha": "e1"}, self.versions)
(self.cache / "a.bin").write_bytes(b"kept")
self.assertFalse(rdd.refresh_entry("pak", entry, versions_path=self.versions))
self.assertEqual(rdd._load_versions(self.versions)["pak"],
{"sha": "e1", "archive_size": len(payload)})
self.assertEqual((self.cache / "a.bin").read_bytes(), b"kept")
def test_a_current_tree_is_not_waited_for(self):
"""A pipeline reads every tree for its whole run. A build beside it
waited on the exclusive lock before asking the remote, which would
@@ -100,7 +120,7 @@ class RefreshConcurrency(unittest.TestCase):
import fcntl # noqa: PLC0415
self.cache.mkdir(parents=True)
rdd._record_version("pak", {"sha": "e1"}, self.versions)
rdd._record_version("pak", {"sha": "e1", "archive_size": 1}, self.versions)
entry = {"source_type": "zip", "source_url": "https://x/pak.zip",
"local_cache": str(self.cache)}
result: list = []
@@ -229,7 +249,7 @@ class DryRunReport(unittest.TestCase):
def test_the_remote_tag_is_shown_beside_the_cached_one(self):
with tempfile.TemporaryDirectory() as tmp:
versions = Path(tmp) / "versions.json"
versions.write_text(json.dumps({"k": {"etag": '"2725165595"'}}))
versions.write_text(json.dumps({"k": {"etag": '"2725165595"', "archive_size": 1}}))
entry = {"source_type": "zip", "source_url": "https://x.invalid/a.zip",
"local_cache": tmp}
with mock.patch.object(rdd, "_get_remote_etag", return_value='"2685251384"'), \
@@ -249,5 +269,30 @@ class DryRunReport(unittest.TestCase):
)
class TheManifestCarriesTheArchiveSize(unittest.TestCase):
"""The size refresh_data_dirs recorded reaches the install list, which
the installer announces as the tree's download."""
def test_recorded_size_is_written_and_an_unrecorded_one_left_out(self):
from generate_pack import _manifest_data_directories # noqa: PLC0415
(REPO_ROOT / "tmp").mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
data = Path(tmp) / "data"
for ref in ("pak", "old"):
(data / ref).mkdir(parents=True)
(data / ref / "a.bin").write_bytes(b"abc")
(data / ".versions.json").write_text(json.dumps(
{"pak": {"sha": "e1", "archive_size": 21_360_579}, "old": {"sha": "e2"}}))
registry = {ref: {"source_type": "zip", "source_url": f"https://x/{ref}.zip",
"local_cache": str(data / ref)} for ref in ("pak", "old")}
members = [(str(data / ref / "a.bin"), f"{ref}/a.bin") for ref in ("pak", "old")]
refs = {f"{ref}/a.bin": (ref, ("s",), ref) for ref in ("pak", "old")}
listed = {entry["ref"]: entry for entry in
_manifest_data_directories(members, refs, registry, "")}
self.assertEqual(listed["pak"]["archive_size"], 21_360_579)
self.assertNotIn("archive_size", listed["old"])
if __name__ == "__main__":
unittest.main()
wiki/installer.md
+6 -3
View File
@@ -1620,8 +1620,11 @@ built (`strip_components` for a ZIP, `source_path` for a tarball, `exclude`),
and writes a file only when its hash matches the list. A file the source has
changed since is left out and counted, and the pack still holds it. `--check`
counts the data files missing, `--dry-run` names each tree it would fetch, and
narrowing with `--system`, `--core` or `--target` keeps only the trees of the
systems that stay.
the size announced for a tree is its archive's (`archive_size`, recorded when
`refresh_data_dirs.py` last downloaded it): one missing file fetches the whole
archive, and a tarball can be a thousand times its tree. Narrowing with
`--system`, `--core` or `--target` keeps only the trees of the systems that
stay.
## What the bootstrap does
@@ -1802,7 +1805,7 @@ other schemas under `api/v1/schemas/`.
| `games_unavailable` | `platform`, `reason`, `message` | With `--for-games`, what the games need could not be read (`offline`, `network`, `invalid`, `stale`, `no_sets`, `no_folder`): the run keeps every file of the list, except for a `FOLDER` that does not exist, where it stops |
| `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply |
| `declared` | `platform`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `dests` | Right after `check`, the same counts over the files the platform's own list names (`cores: null`), and their `dests` in the order of the `file` events. A setup that checks its own list reads its answer here, with or without `--declared-only` |
| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes they write once in place |
| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes fetching them costs: each tree's archive, or the files themselves from a pack or a list that records no archive size |
| `found` | `platform`, `dest`, `folder`, `path`, `member`, `reason` | Each file to install that this machine already holds, after `check` and before anything is written, in `--check` and `--dry-run` too. `folder` is the `--from` folder as given, or the `bios_dir` of the check when the BIOS folder holds the file at another place, and then `reason` says where in words (`nested folder: ...` for a misplaced file, `the same file is at ...` for one the list installs there too). `path` is the file, and `member` the ZIP member holding the bytes when `path` is an archive, an empty string otherwise. Its copy is reported by a `downloaded` event |
| `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) |
| `saves` | `platform`, `files`, `folder` | After `choices` and before `checking`, when the files the run installs (once `--system` and the other choices narrowed the list) hold some read from the frontend's save directory and that directory is known: how many, and the `folder` they go to, where `<system>` and `<core>` stand for one folder per system or per core (the screen and `--plain` say it in words: "a folder per core under /sdcard/RetroArch/saves"). When it is not known, the files are counted in the summary's `skipped` instead (`no_save_directory`) |