diff --git a/install.ps1 b/install.ps1 index 896cdaf76..c8d2c3399 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858" + $defaultInstallSha256 = "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 01c0b9425..16c011ea6 100644 --- a/install.py +++ b/install.py @@ -291,7 +291,6 @@ def _os_default_dests(os_type: str) -> dict[str, Path]: the command line but not found must not land in ~/bios, which none of them reads. """ - home = Path.home() if os_type == "windows": return { # RetroArch's system directory defaults to \system @@ -307,13 +306,13 @@ def _os_default_dests(os_type: str) -> dict[str, Path]: # RetroArch may be the Linux one inside the distribution, while RetroBat # only exists on the Windows side, reached through /mnt/c. return { - "retroarch": home / ".config" / "retroarch" / "system", + "retroarch": _unix_retroarch_base() / "system", "retrobat": _host_path("C:/RetroBat/bios"), } if os_type == "darwin": return {"retroarch": _macos_retroarch_base() / "system"} return { - "retroarch": home / ".config" / "retroarch" / "system", + "retroarch": _unix_retroarch_base() / "system", } @@ -681,8 +680,9 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { # found nowhere and no folder to assume) or from (a folder named with --from, # reason: not_found, not_a_folder, unreadable, or system for the root # of the system and /proc, /sys, /dev) or dest (reason: unreadable, a - # folder above it this user cannot open, or not_writable, the BIOS - # folder cannot be created, a summary following) or option (reason: + # folder above it this user cannot open, not_a_folder, the BIOS folder + # is a file or lies under one, or not_writable, the BIOS folder cannot + # be created, a summary following) or option (reason: # unknown, a platform, target, system, core or emulator the run cannot # use, refused before any download). "error": ("stage", "reason", "platform", "message"), @@ -1385,6 +1385,17 @@ def _macos_retroarch_base() -> Path: return Path.home() / "Documents" / "RetroArch" +def _unix_retroarch_base() -> Path: + """RetroArch's base folder on Linux: $XDG_CONFIG_HOME/retroarch when the + variable is set, else ~/.config/retroarch (fill_pathname_application_data + in file_path_special.c, frontend_unix_get_env in platform_unix.c). The + code tests only that the variable exists, so an empty one counts.""" + xdg = os.environ.get("XDG_CONFIG_HOME") + if xdg is not None: + return Path(xdg) / "retroarch" + return Path.home() / ".config" / "retroarch" + + # System directory -> the retroarch.cfg it was read from and the directory # ':' expands to there, for finding the cores of the same installation. _RETROARCH_CONFIGS: dict[Path, tuple[Path, Path]] = {} @@ -2773,11 +2784,16 @@ def detect_platforms(os_type: str) -> list[tuple[str, Path]]: if ra_dir: found.append(("retroarch", ra_dir)) - # RetroArch native - native_cfg = home / ".config" / "retroarch" / "retroarch.cfg" - ra_dir = _parse_retroarch_system_dir(native_cfg) - if ra_dir: - found.append(("retroarch", ra_dir)) + # RetroArch native: the config under its base folder, then + # ~/.retroarch.cfg when that one is missing (open_default_config_file + # in configuration.c). Either way the system folder defaults to the + # base folder's. + native_base = _unix_retroarch_base() + for native_cfg in (native_base / "retroarch.cfg", home / ".retroarch.cfg"): + ra_dir = _parse_retroarch_system_dir(native_cfg, native_base) + if ra_dir: + found.append(("retroarch", ra_dir)) + break if os_type == "darwin": # open_default_config_file reads /RetroArch/config @@ -3692,6 +3708,9 @@ def _check_data_directory(entry: object, where: str) -> None: strip = entry.get("strip_components", 0) if not isinstance(strip, int) or isinstance(strip, bool) or not 0 <= strip <= 16: raise ValueError(f"{where}.strip_components is invalid") + size = entry.get("archive_size", 1) + if not isinstance(size, int) or isinstance(size, bool) or size < 1: + raise ValueError(f"{where}.archive_size is invalid") if kind == "tarball" and ( not isinstance(entry.get("source_path"), str) or not entry["source_path"] ): @@ -6440,7 +6459,8 @@ def _report_outcome( kept = _kept_lines() + ([ f"{_plural(len(unplaced), 'file')} read from " f"{unplaced[0].get('frontend') or 'the frontend'}'s save directory left out: " - f"its place is not known. {LINE_SAVE_REMEDY}"] if unplaced else []) + + (f"{unplaced[0]['save_why']}." if unplaced[0].get("save_why") + else f"its place is not known. {LINE_SAVE_REMEDY}")] if unplaced else []) kept += list(notes or []) if interrupted: say(f"\nStopped. {_plural(downloaded, 'file')} installed.") @@ -6856,11 +6876,12 @@ def _print_plan( # files, and 2636 data files" read as two kinds of one thing. The kinds # follow, adding up to it. total = counts.files + tree_files - parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_counts.size)})"] + tree_bytes = sum(data_fetch_bytes(entry, needed) for entry, needed in trees) + parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_bytes)})"] if total else []) split = parts_words(counts if (manifest or {}).get("categorized") else tally({"size": e.get("size")} for e in to_download), - tree_files, tree_counts.size) + tree_files, tree_bytes) if split and trees: split += f", the data files extracted from {_plural(len(trees), 'archive')}" # A pack is copied from, not downloaded: "Would download 23 files" was @@ -6935,8 +6956,11 @@ def _print_plan( "every system and its files.") for entry, needed in trees: source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"] + fetched = data_fetch_bytes(entry, needed) + archive = ("" if fetched == sum(meta.get("size", 0) for meta in needed.values()) + else f" (a {format_size(fetched)} archive)") print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, " - f"from {source}") + f"from {source}{archive}") for folder, (count, size) in sorted((kept or {}).items()): print(f"\n Would keep {_plural(count, 'file')} you have ({format_size(size)}), " f"whose contents differ, in {folder}") @@ -7062,20 +7086,72 @@ def download_files( return failed, interrupted -def _fetch_archive(url: str, directory: Path, progress: Progress | None = None) -> Path: +def _archive_validator(headers: object) -> str: + """What names this revision of an archive to its server: an If-Range + with it gets the rest of the same bytes, or the whole new archive.""" + if not headers: + return "" + etag = headers.get("ETag", "") or "" + # A weak tag never answers a range (RFC 9110 13.1.5). + return etag if etag and not etag.startswith("W/") else headers.get("Last-Modified", "") or "" + + +def _fetch_archive(url: str, directory: Path, progress: Progress | None = None, + keep: Path | None = None) -> Path: """A data directory's source archive, in *directory*, retried like files. - progress draws its bytes as they come, an attempt that failed taken back.""" + progress draws its bytes as they come, an attempt that failed taken back. + + With keep, the archive is written there and kept when the transfer is + cut, beside a note of its address and the validator its server sent: + the next attempt, or the next run, asks for the rest with Range and + If-Range, and a server that changed the archive sends it whole. A cut + 50 MB archive started again from zero, and a link that drops more often + than one archive takes never installed the tree. + """ _network_allowed(url) + note = keep.with_name(keep.name + ".source") if keep is not None else None reason: Exception | None = None for attempt in range(1, MAX_RETRIES + 1): - handle = tempfile.NamedTemporaryFile(dir=directory, suffix=".archive", delete=False) + offset = 0 + validator = "" + if keep is not None: + try: + recorded = note.read_text(encoding="utf-8").split("\n") + if recorded[0] == url and len(recorded) > 1 and recorded[1]: + validator = recorded[1] + offset = keep.stat().st_size + except (OSError, UnicodeError): + offset = 0 + request = urllib.request.Request(url) + if offset: + request.add_header("Range", f"bytes={offset}-") + request.add_header("If-Range", validator) + handle = (open(keep, "r+b" if offset else "wb") if keep is not None else + tempfile.NamedTemporaryFile(dir=directory, suffix=".archive", delete=False)) received = 0 + resumed = 0 try: - with handle, urllib.request.urlopen(url, timeout=60) as resp: + with handle, urllib.request.urlopen(request, timeout=60) as resp: + headers = resp.headers or {} + ranged = (offset and getattr(resp, "status", 200) == 206 and str( + headers.get("Content-Range", "")).startswith(f"bytes {offset}-")) + if not ranged: + offset = 0 + handle.seek(offset) + handle.truncate() + if keep is not None and not offset: + # Written before the first byte: a cut transfer is then + # resumed only against the revision it holds. + note.write_text(f"{url}\n{_archive_validator(headers)}\n", + encoding="utf-8") if progress is not None: - length = resp.headers.get("Content-Length", "") + length = headers.get("Content-Length", "") if length.isdigit(): - progress.expect(int(length)) + progress.expect(offset + int(length)) + if offset: + progress.add(offset, resumed=True) + resumed = offset + total = offset while True: if _stop.is_set(): raise _Interrupted() @@ -7083,30 +7159,51 @@ def _fetch_archive(url: str, directory: Path, progress: Progress | None = None) if not chunk: break received += len(chunk) - if received > MAX_DATA_ARCHIVE_SIZE: + total += len(chunk) + if total > MAX_DATA_ARCHIVE_SIZE: raise ValueError("archive exceeds the size limit") handle.write(chunk) if progress is not None: progress.add(len(chunk)) - return Path(handle.name) + return keep if keep is not None else Path(handle.name) except urllib.error.HTTPError as exc: - Path(handle.name).unlink(missing_ok=True) - if exc.code < 500 and exc.code != 429: + if keep is None or exc.code == 416 or exc.code < 500 and exc.code != 429: + # Refused, or a range past an archive that shrank: no + # attempt resumes it. + _drop_archive(Path(handle.name), note) + if exc.code < 500 and exc.code not in (416, 429): raise reason = exc except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: - Path(handle.name).unlink(missing_ok=True) + if keep is None: + Path(handle.name).unlink(missing_ok=True) reason = exc if isinstance(exc, OSError) else ConnectionError(repr(exc)) - except BaseException: - Path(handle.name).unlink(missing_ok=True) + except ValueError: + _drop_archive(Path(handle.name), note) raise - if progress is not None and received: - progress.add(-received) + except BaseException: + # Stopped: a kept archive waits for the next run. + if keep is None: + Path(handle.name).unlink(missing_ok=True) + raise + if progress is not None: + if received: + progress.add(-received) + if resumed: + progress.add(-resumed) + progress.forget_resumed(resumed) if attempt < MAX_RETRIES: time.sleep(_retry_delay(attempt, None)) raise reason if reason else OSError("download failed") +def _drop_archive(path: Path, note: Path | None) -> None: + """Remove an archive and the note of where it came from.""" + path.unlink(missing_ok=True) + if note is not None: + note.unlink(missing_ok=True) + + def _archive_members( archive: Path, entry: dict ) -> Iterator[tuple[str, int, Callable[[], BinaryIO]]]: @@ -7167,6 +7264,19 @@ def data_directory_state(entry: dict, bios_path: Path, return root, needed, present +def data_fetch_bytes(entry: dict, needed: dict[str, dict]) -> int: + """What getting a tree's missing files costs: its source archive, which + comes whole for one file, or from a pack the files themselves. A list + that records no archive size gives the files' own sizes: a tarball whose + tree is 10 KB was announced as 10 KB and was a download of 21 MB.""" + if not needed: + return 0 + size = entry.get("archive_size") + if LOCAL.pack is None and isinstance(size, int) and size > 0: + return size + return sum(meta.get("size", 0) for meta in needed.values()) + + def install_data_directory(entry: dict, bios_path: Path, state: tuple[Path, dict[str, dict], int] | None = None, progress: Progress | None = None) -> dict: @@ -7186,6 +7296,7 @@ def install_data_directory(entry: dict, bios_path: Path, if not needed: return counts _writing.set() + fetched: Path | None = None try: root.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=root) as scratch: @@ -7193,7 +7304,11 @@ def install_data_directory(entry: dict, bios_path: Path, # The pack holds the tree where it extracts, already mapped. members = LOCAL.pack.tree(entry.get("dest", "")) else: - archive = _fetch_archive(entry["url"], Path(scratch), progress) + # Beside the tree, outside the scratch folder: a transfer + # cut short is resumed by the next attempt or the next run. + kept = root / f".{entry['ref']}.archive.part" + archive = _fetch_archive(entry["url"], Path(scratch), progress, kept) + fetched = archive members = _archive_members(archive, entry) for rel, declared, opener in members: meta = needed.pop(rel, None) @@ -7227,6 +7342,10 @@ def install_data_directory(entry: dict, bios_path: Path, except _MEMBER_ERRORS as exc: # A damaged, encrypted or unsupported member, in words. counts["error"] = unreadable_words(exc) + finally: + if fetched is not None: + # Read whole: the next run fetches the archive afresh. + _drop_archive(fetched, fetched.with_name(fetched.name + ".source")) counts["absent"] = len(needed) return counts @@ -8620,6 +8739,13 @@ def saves_line(display: str, count: int, where: str) -> str: f"{shown_path(folder)}") +def unlisted_saves_words(display: str) -> str: + """Why a file list holds none of the files read from the save + directory: the platform keeps it in no fixed place, and the list leaves + them out with no hash or address, which --save-dir cannot change.""" + return f"{display} keeps it in no fixed place, and its file list holds none of them" + + def unplaced_line(display: str, count: int, remedy: str) -> str: """The files read from an unknown save directory, left out.""" return (f"{_plural(count, 'file')} read from {display}'s save directory left out: " @@ -8939,7 +9065,7 @@ def found_head(files: list[dict], trees: list[dict], bios_path: Path) -> str: for tree in trees: _root, wanted, _present = data_directory_state(tree, bios_path, remember=True) data += len(wanted) - data_bytes += sum(meta.get("size", 0) for meta in wanted.values()) + data_bytes += data_fetch_bytes(tree, wanted) left = [entry for entry in needed if id(entry) not in copies] rest = len(left) + data where = ", ".join(f"{count} found in {_last_name(label)}" @@ -13413,6 +13539,29 @@ def main() -> None: if (args.list_platforms or args.list_targets or args.list_systems or args.list_cores or args.list_emulators): parser.error("--for-games narrows an install, not a listing") + # Applied or refused, never ignored: a run prints one list, and the + # narrowing options apply to --list-systems and --list-cores only, past + # --target and --installed-cores, which --list-emulators also reads. + listings = [flag for flag, given in ( + ("--list-platforms", args.list_platforms), ("--list-targets", args.list_targets), + ("--list-systems", args.list_systems), ("--list-cores", args.list_cores), + ("--list-emulators", args.list_emulators)) if given] + if len(listings) > 1: + parser.error(f"{listings[0]} and {listings[1]} each print a list: run them " + "one at a time") + if listings and listings[0] in ("--list-platforms", "--list-targets", + "--list-emulators"): + unread = (("--system", args.system), ("--core", args.core), + ("--region", args.region), ("--emulator", args.emulator)) + if listings[0] != "--list-emulators": + unread += (("--target", args.target), ("--every-board", args.every_board), + ("--installed-cores", args.installed_cores)) + if listings[0] == "--list-platforms": + unread += (("--platform", args.platform),) + given = next((flag for flag, value in unread if value), "") + if given: + parser.error(f"{listings[0]} prints the whole list, which {given} " + "does not narrow") if args.declared_only: # Applied or refused, never ignored. for given, reason in ( @@ -13758,6 +13907,27 @@ def _refuse_closed(dest: Path, named: str) -> None: sys.exit(1) +def _refuse_file(dest: Path, named: str) -> None: + """End the run on a BIOS folder that is a file, or lies under one. + + mkdir then failed on EEXIST or ENOTDIR, and the run blamed permissions + and offered sudo, which fails the same way. + """ + for folder in (dest, *dest.parents): + if _exists(folder): + break + else: + return + if _is_dir(folder): + return + where = (f"{shown_path(dest)} is a file, not a folder" if folder == dest + else f"{shown_path(dest)} cannot be a folder: {shown_path(folder)} is a file") + EVENTS.emit("error", stage="dest", reason="not_a_folder", platform="", message=where) + print(f"Error: {screen_text(where)}. Name the folder the BIOS files go in, not " + f"the file {named}.", file=sys.stderr) + sys.exit(1) + + def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest", again: bool = False) -> bool: """Say that a --dest folder does not exist before anything goes there. @@ -13772,6 +13942,7 @@ def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest", a card's folder ended the run where it meant "not that card". """ _refuse_closed(dest, named) + _refuse_file(dest, named) if dest.exists(): return True nearest = dest.absolute() @@ -15099,6 +15270,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: else f"{found_in[0]} names" if found_in else "detected here") # One setup by now: two found without questions stopped above. _refuse_closed(path, f"{named} (name another with --dest)") + _refuse_file(path, f"{named} (name another with --dest)") total_downloaded = 0 total_up_to_date = 0 @@ -15137,7 +15309,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # --installed-cores is below: EmuDeck installed 3.2 GB and only then was # steamos refused for RetroArch, with no summary of what had landed. target_lists: dict[str, dict] = {} - if args.target and not args.list_targets: + if args.target: for plat_name, _bios_path in platforms: targets = fetch_targets(plat_name) if targets is None: @@ -15333,6 +15505,10 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: display = display_name(plat_name, manifest) files, unplaced = place_save_entries( files, bios_path, manifest, args.save_dir, display, say=False) + if not manifest.get("save_directory"): + # Its own list leaves these files out: naming the folder with + # --save-dir installs none of them, and was advised in a loop. + _SAVE_WHY[bios_path] = unlisted_saves_words(display) # The games of the card, read before the choices: where it holds # some, what they need is the menu's first choice, and --for-games # narrows the list here, before the systems are counted. @@ -15535,7 +15711,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: total_omitted += len(omitted_files) # Named for the end of the run: ROCKNIX's own list leaves them out, # and the end said "the frontend's save directory". - omitted_seen.extend({**entry, "frontend": display} + omitted_seen.extend({**entry, "frontend": display, + "save_why": unlisted_saves_words(display)} if entry.get("reason") == "no_save_directory" and not entry.get("frontend") else entry for entry in omitted_files) @@ -15608,8 +15785,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: plat_name, manifest, bios_path, files, up_to_date, mismatched, sum(len(state[1]) for _entry, state in data_states), len(omitted_files), chosen=selection[0] if selection else None, - data_bytes=sum(meta.get("size", 0) for _entry, state in data_states - for meta in state[1].values()), + data_bytes=sum(data_fetch_bytes(entry, state[1]) + for entry, state in data_states), ) emit_copies(plat_name, to_download, copies, bios_index, bios_path, files) emit_unreadable(plat_name, unread) @@ -21150,6 +21327,9 @@ def _unplaced_lines(summary: dict | None, state: TuiState, if not count: return [] name = display_name(state.platform) if state.platform else "the frontend" + if state.manifest and not state.manifest.get("save_directory"): + return [f"{_plural(count, 'file')} read from {name}'s save directory left out: " + f"{unlisted_saves_words(name)}."] return [f"{_plural(count, 'file')} read from {name}'s save directory left out: " f"its place is not known. {remedy or TUI_SAVE_REMEDY}"] diff --git a/install.sh b/install.sh index b085a4cbb..4e6c1560d 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858" +DEFAULT_INSTALL_SHA256="fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/amberelec.json b/install/amberelec.json index 745e5792b..51f5a9028 100644 --- a/install/amberelec.json +++ b/install/amberelec.json @@ -63489,7 +63489,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "fbneo-cheats", @@ -77218,7 +77219,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -77451,7 +77453,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -77823,7 +77826,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ] } \ No newline at end of file diff --git a/install/batocera.json b/install/batocera.json index 8c24297c0..a5ecfcb19 100644 --- a/install/batocera.json +++ b/install/batocera.json @@ -225291,7 +225291,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -225524,7 +225525,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ], "save_directory": { diff --git a/install/emudeck.json b/install/emudeck.json index c1d09d585..32f4ed6a2 100644 --- a/install/emudeck.json +++ b/install/emudeck.json @@ -100486,7 +100486,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -100934,7 +100935,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -100990,7 +100992,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -109666,7 +109669,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -123395,7 +123399,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -123628,7 +123633,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -124000,7 +124006,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "lrps2", @@ -124016,7 +124023,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "yaps2-resources", @@ -124476,7 +124484,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/emuelec.json b/install/emuelec.json index 8135201bd..e9709f639 100644 --- a/install/emuelec.json +++ b/install/emuelec.json @@ -69309,7 +69309,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -69757,7 +69758,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "fbneo-cheats", @@ -83486,7 +83488,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -83719,7 +83722,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -84091,7 +84095,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ] } \ No newline at end of file diff --git a/install/knulli.json b/install/knulli.json index 25b3ccce4..4e70dab7f 100644 --- a/install/knulli.json +++ b/install/knulli.json @@ -157933,7 +157933,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -158166,7 +158167,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "lrps2", @@ -158182,7 +158184,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "ppsspp-assets", @@ -158870,7 +158873,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 } ], "save_directory": { diff --git a/install/lakka.json b/install/lakka.json index 093e07996..5866fe5db 100644 --- a/install/lakka.json +++ b/install/lakka.json @@ -115932,7 +115932,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "cemu-gameprofiles", @@ -116888,7 +116889,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dirksimple", @@ -116944,7 +116946,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "fbneo-cheats", @@ -130673,7 +130676,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -130906,7 +130910,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -131278,7 +131283,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -131738,7 +131744,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/launchers/index.json b/install/launchers/index.json index 358ee41c9..2f435fcb1 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1112094, + "sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1156738, - "sha256": "35c0920008baadca997d0f778e99ce46d702bbe3d3f0d33104d7158422b292aa" + "size": 1165721, + "sha256": "1b6ea28a5fc2f413aa7444ddb9ebe00b4b89c29ef639d62e3e1cf4d10793ed16" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1112094, + "sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1112094, + "sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1", "executable": false }, { diff --git a/install/myminui.json b/install/myminui.json index 919ecf4b1..85de7b9bc 100644 --- a/install/myminui.json +++ b/install/myminui.json @@ -6,7 +6,7 @@ "platform": "myminui", "display_name": "MyMinUI", "version": "2026-08-09", - "generated": "2026-11-05T18:33:47Z", + "generated": "2026-11-05T23:59:37Z", "base_destination": "", "detect": [ { @@ -18957,7 +18957,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-cheats", @@ -32686,7 +32687,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -32919,7 +32921,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "fbneo-samples", @@ -33152,7 +33155,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ] } \ No newline at end of file diff --git a/install/nextui.json b/install/nextui.json index d38175219..4518d3d5c 100644 --- a/install/nextui.json +++ b/install/nextui.json @@ -6,7 +6,7 @@ "platform": "nextui", "display_name": "NextUI", "version": "2026-10-20", - "generated": "2026-11-05T18:33:47Z", + "generated": "2026-11-05T23:59:37Z", "base_destination": "", "detect": [ { @@ -5635,7 +5635,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "fbneo-cheats", @@ -19364,7 +19365,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -19597,7 +19599,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ] } \ No newline at end of file diff --git a/install/recalbox.json b/install/recalbox.json index b947571f5..9b97c9d50 100644 --- a/install/recalbox.json +++ b/install/recalbox.json @@ -62833,7 +62833,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -63066,7 +63067,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ], "save_directory": { diff --git a/install/retroarch.json b/install/retroarch.json index 024fb6396..11b60b965 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -21,12 +21,24 @@ "config": "$HOME/snap/retroarch/current/.config/retroarch/retroarch.cfg", "parse_key": "system_directory" }, + { + "os": "linux", + "method": "config_file", + "config": "$XDG_CONFIG_HOME/retroarch/retroarch.cfg", + "parse_key": "system_directory" + }, { "os": "linux", "method": "config_file", "config": "$HOME/.config/retroarch/retroarch.cfg", "parse_key": "system_directory" }, + { + "os": "linux", + "method": "config_file", + "config": "$HOME/.retroarch.cfg", + "parse_key": "system_directory" + }, { "os": "darwin", "method": "config_file", @@ -116911,7 +116923,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "cemu-gameprofiles", @@ -117867,7 +117880,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dinothawr", @@ -118315,7 +118329,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -118371,7 +118386,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -127042,7 +127058,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -140771,7 +140788,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -141004,7 +141022,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -141376,7 +141395,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "ppsspp-assets", @@ -142064,7 +142084,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 }, { "ref": "yaps2-resources", @@ -142524,7 +142545,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/retrobat.json b/install/retrobat.json index 3a2e41f89..ca7459771 100644 --- a/install/retrobat.json +++ b/install/retrobat.json @@ -89075,7 +89075,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -89308,7 +89309,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -89680,7 +89682,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ], "save_directory": { diff --git a/install/retrodeck.json b/install/retrodeck.json index b70189e03..e94e2c442 100644 --- a/install/retrodeck.json +++ b/install/retrodeck.json @@ -137751,7 +137751,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "fbneo-cheats", @@ -151480,7 +151481,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -151713,7 +151715,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -152085,7 +152088,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -152549,7 +152553,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/retropie.json b/install/retropie.json index c255b9d1f..3db3da56e 100644 --- a/install/retropie.json +++ b/install/retropie.json @@ -124071,7 +124071,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dolphin-sys", @@ -132742,7 +132743,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -146471,7 +146473,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -146704,7 +146707,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -147076,7 +147080,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -147536,7 +147541,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ] } \ No newline at end of file diff --git a/install/rocknix.json b/install/rocknix.json index 793ca6670..f06bd3144 100644 --- a/install/rocknix.json +++ b/install/rocknix.json @@ -86538,7 +86538,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -86986,7 +86987,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -87042,7 +87044,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -95719,7 +95722,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -109448,7 +109452,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -109681,7 +109686,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -110053,7 +110059,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "lrps2", @@ -110069,7 +110076,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "ppsspp-assets", @@ -110757,7 +110765,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 }, { "ref": "yaps2-resources", @@ -111217,7 +111226,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ] } \ No newline at end of file diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 891b01a95..a2da8d55c 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -219,10 +219,23 @@ platforms: method: config_file config: $HOME/snap/retroarch/current/.config/retroarch/retroarch.cfg parse_key: system_directory + # Native: $XDG_CONFIG_HOME/retroarch when the variable is set, else + # $HOME/.config/retroarch, then $HOME/.retroarch.cfg when that file is + # missing (file_path_special.c fill_pathname_application_data, + # configuration.c open_default_config_file). A value left at default + # is the system folder of the first of the two folders. + - os: linux + method: config_file + config: $XDG_CONFIG_HOME/retroarch/retroarch.cfg + parse_key: system_directory - os: linux method: config_file config: $HOME/.config/retroarch/retroarch.cfg parse_key: system_directory + - os: linux + method: config_file + config: $HOME/.retroarch.cfg + parse_key: system_directory # configuration.c reads /RetroArch/config, and a value # left at default is ~/Documents/RetroArch/system (platform_darwin.m). # Older builds kept the file one level up. diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index 4e7d1ec20..264c08558 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -429,6 +429,11 @@ "minimum": 0, "description": "With type zip: leading path parts dropped from each member." }, + "archive_size": { + "type": "integer", + "minimum": 1, + "description": "Bytes of the source archive when it was last downloaded: what fetching the tree costs. Absent when not recorded." + }, "source_path": { "type": "string", "description": "With type tarball: only members below this path are kept, relative to it." diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 354964481..8a62186a3 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -4949,9 +4949,12 @@ def _manifest_data_directories( (strip_components for a ZIP, source_path for a tarball, exclude) come from _data_dirs.yml, the same refresh_data_dirs applies. Every file carries the sha1 and size the cache holds, so a member the source - changed since is refused rather than installed. + changed since is refused rather than installed. archive_size is the + size of the archive refresh_data_dirs last downloaded, what fetching + the tree costs: a tarball whose tree is 10 KB is a download of 21 MB. """ grouped: dict[tuple[str, str], dict] = {} + versions: dict[str, dict[str, dict]] = {} for src, full in members: ref, systems, prefix = refs[full] entry = registry.get(ref, {}) @@ -4976,6 +4979,17 @@ def _manifest_data_directories( group["source_path"] = str(entry.get("source_path", "")).format(version=version) if entry.get("exclude"): group["exclude"] = [str(e) for e in entry["exclude"]] + cache = entry.get("local_cache") + if cache: + recorded = Path(cache).parent / ".versions.json" + if str(recorded) not in versions: + try: + versions[str(recorded)] = json.loads(recorded.read_text()) + except FileNotFoundError: + versions[str(recorded)] = {} + size = versions[str(recorded)].get(ref, {}).get("archive_size") + if isinstance(size, int) and size > 0: + group["archive_size"] = size group["systems"].extend(s for s in systems if s not in group["systems"]) rel = full[len(prefix):].lstrip("/") if prefix else full hashes = compute_hashes(src, frozenset({"sha1"})) diff --git a/scripts/refresh_data_dirs.py b/scripts/refresh_data_dirs.py index 31def57cc..575b73917 100644 --- a/scripts/refresh_data_dirs.py +++ b/scripts/refresh_data_dirs.py @@ -24,7 +24,7 @@ import time import urllib.error import urllib.request import zipfile -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path from artifacts import file_lock as _file_lock, set_plain_mode from common import InputMissing, github_token, platform_arg, require_platform, suggest_names, yaml_load @@ -288,10 +288,12 @@ def _download_and_extract( source_path: str, local_cache: str, exclude: list[str] | None = None, + measured: Callable[[int], None] | None = None, ) -> int: """Download tarball, extract source_path subtree to local_cache. - Returns the number of files extracted. + Returns the number of files extracted. measured receives the size of + the archive downloaded. """ exclude = exclude or [] cache_dir = Path(local_cache) @@ -308,6 +310,8 @@ def _download_and_extract( if not chunk: break f.write(chunk) + if measured is not None: + measured(tarball_path.stat().st_size) log.info("extracting %s -> %s", source_path, local_cache) @@ -365,6 +369,7 @@ def _download_and_extract_zip( local_cache: str, exclude: list[str] | None = None, strip_components: int = 0, + measured: Callable[[int], None] | None = None, ) -> int: """Download ZIP, extract to local_cache. Returns file count. @@ -387,6 +392,8 @@ def _download_and_extract_zip( if not chunk: break f.write(chunk) + if measured is not None: + measured(zip_path.stat().st_size) extract_dir = Path(tmpdir) / "extract" extract_dir.mkdir() @@ -426,6 +433,46 @@ def _download_and_extract_zip( return file_count +def _measure_archive(source_url: str) -> int | None: + """The size of the archive at source_url, read by streaming it: a + tarball GitHub builds on request is sent without a Content-Length.""" + req = urllib.request.Request(source_url, headers={"User-Agent": USER_AGENT}) + size = 0 + try: + with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp: + while chunk := resp.read(65536): + size += len(chunk) + except _TRANSIENT as exc: + log.warning("could not measure %s: %s", source_url, exc) + return None + return size or None + + +def _record_archive_size(key: str, entry: dict, versions_path: str) -> None: + """Record the archive size of a tree current since before it was kept. + + The installer announces a tree's download as that size: a tarball + whose tree is 10 KB is a download of 21 MB. Measured, not refetched, + so the tree the packs carry stays the one recorded. + """ + path = Path(versions_path) + with _file_lock(path.with_name(f".{path.name}.lock")): + versions = _load_versions(versions_path) + record = versions.get(key) + if record is None or "archive_size" in record: + return + version = entry.get("version", "master") + size = _measure_archive(entry["source_url"].format(version=version)) + if size is None: + return + with _file_lock(path.with_name(f".{path.name}.lock")): + versions = _load_versions(versions_path) + if key in versions and "archive_size" not in versions[key]: + versions[key]["archive_size"] = size + _save_versions(versions, versions_path) + log.info("[%s] archive size recorded: %d bytes", key, size) + + def _get_remote_etag(source_url: str) -> str | None: """HEAD request to get ETag or Last-Modified for freshness check.""" try: @@ -474,6 +521,7 @@ def refresh_entry( return None if not needs: log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag)) + _record_archive_size(key, entry, versions_path) return False with cache_lock(entry["local_cache"]): return _refresh_entry(key, entry, force, dry_run, versions_path) @@ -534,13 +582,19 @@ def _refresh_entry( else: remote_tag = get_remote_sha(entry["source_url"], version) + archive_sizes: list[int] = [] + def fetch() -> int: + archive_sizes.clear() if source_type == "zip": return _download_and_extract_zip( - source_url, local_cache, exclude, entry.get("strip_components", 0) + source_url, local_cache, exclude, entry.get("strip_components", 0), + measured=archive_sizes.append, ) source_path = entry["source_path"].format(version=version) - return _download_and_extract(source_url, source_path, local_cache, exclude) + return _download_and_extract( + source_url, source_path, local_cache, exclude, measured=archive_sizes.append + ) # An archive that reads but holds nothing under the path is the same on # every try: only a failed or unreadable download is tried again. @@ -558,7 +612,10 @@ def _refresh_entry( log.warning("[%s] download failed (%s), trying again in %d s", key, exc, wait) time.sleep(wait) - _record_version(key, {"sha": remote_tag or "", "version": version}, versions_path) + record = {"sha": remote_tag or "", "version": version} + if archive_sizes: + record["archive_size"] = archive_sizes[-1] + _record_version(key, record, versions_path) log.info("[%s] refreshed: %d files extracted to %s", key, file_count, local_cache) return True diff --git a/tests/test_install.py b/tests/test_install.py index f128216e6..5c7b22409 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -451,6 +451,64 @@ class AMissingDestIsSaid(unittest.TestCase): self.assertIn("does not exist", out) +class ADestThatIsAFileIsNamed(unittest.TestCase): + """--dest naming a file, or a path under one, failed at mkdir with + EEXIST or ENOTDIR, and the run said this user cannot write there and + offered a sudo command that fails the same way.""" + + def _run(self, dest: Path) -> subprocess.CompletedProcess: + env = dict(os.environ, RETROBIOS_OS="linux") + return subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", str(dest), "--manifest", str(REPO_ROOT / "install" / "retroarch.json"), + "--system", "nintendo-gba", "--yes", "--events", "--offline"], + env=env, capture_output=True, text=True, timeout=120, check=False, + stdin=subprocess.DEVNULL) + + def test_the_file_is_named_and_sudo_is_not_offered(self): + root = Path(_tempdir(self)) + (root / "scph5501.bin").write_bytes(b"x") + for dest, words in ( + (root / "scph5501.bin", f"{root / 'scph5501.bin'} is a file, not a folder"), + (root / "scph5501.bin" / "sub", + f"cannot be a folder: {root / 'scph5501.bin'} is a file"), + ): + with self.subTest(dest=dest): + proc = self._run(dest) + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertIn(words, proc.stderr) + self.assertNotIn("sudo", proc.stderr) + errors = [json.loads(line) for line in proc.stdout.splitlines() + if line.startswith("{") and '"error"' in line] + self.assertEqual([(e["stage"], e["reason"]) for e in errors], + [("dest", "not_a_folder")]) + self.assertEqual((root / "scph5501.bin").read_bytes(), b"x") + + +class SaveFilesAListHoldsNoneOfAreNotPromised(unittest.TestCase): + """RetroPie, ROCKNIX, AmberELEC, EmuELEC and RomM keep their save + directory in no fixed place: their lists leave those files out with no + hash or address. The run said "Name it with --save-dir", and said it + again with --save-dir given, which installs none of them.""" + + def test_the_reason_is_said_and_no_option_is_offered(self): + manifest = json.loads((REPO_ROOT / "install" / "retropie.json").read_text()) + self.assertIsNone(manifest.get("save_directory")) + root = Path(_tempdir(self)) + env = dict(os.environ, RETROBIOS_OS="linux") + for extra in ([], ["--save-dir", str(root / "saves")]): + with self.subTest(extra=extra): + proc = _install_run( + env, "--platform", "retropie", "--dest", str(root / "bios"), + "--manifest", "install/retropie.json", "--offline", "--no-input", + "--yes", "--emulator", "azahar", *extra) + out = proc.stdout + proc.stderr + self.assertIn("read from RetroPie's save directory left out: RetroPie " + "keeps it in no fixed place, and its file list holds none " + "of them.", out) + self.assertNotIn("Name it with --save-dir", out) + + class AFolderThatRefusesWritesIsNamed(unittest.TestCase): """A read-only BIOS folder ended the run on a traceback (a subfolder to create), or on three retries per file and "once that is resolved", and a @@ -2673,6 +2731,52 @@ class RetroArchWhereItReallyLives(unittest.TestCase): self.assertIn(("retroarch", support / "system"), found) +class LinuxRetroArchFollowsXdgConfigHome(unittest.TestCase): + """RetroArch on Linux keeps its config under $XDG_CONFIG_HOME/retroarch + when the variable is set, then falls back to ~/.retroarch.cfg + (file_path_special.c fill_pathname_application_data, configuration.c + open_default_config_file). Only ~/.config/retroarch was read: a home + with its own XDG layout detected nothing.""" + + def _env(self) -> tuple[Path, Path]: + home = Path(_tempdir(self)) + xdg = Path(_tempdir(self)) + patcher = unittest.mock.patch.dict(os.environ, {"HOME": str(home)}, clear=True) + patcher.start() + self.addCleanup(patcher.stop) + return home, xdg + + def test_the_config_under_xdg_config_home_is_read(self): + home, xdg = self._env() + (xdg / "retroarch").mkdir() + (xdg / "retroarch" / "retroarch.cfg").write_text('system_directory = "/srv/bios"\n') + # A config under ~/.config is not the one this RetroArch opens. + (home / ".config" / "retroarch").mkdir(parents=True) + (home / ".config" / "retroarch" / "retroarch.cfg").write_text( + 'system_directory = "/elsewhere"\n') + os.environ["XDG_CONFIG_HOME"] = str(xdg) + found = install.detect_platforms("linux") + self.assertIn(("retroarch", Path("/srv/bios")), found) + self.assertNotIn(("retroarch", Path("/elsewhere")), found) + self.assertEqual(install._os_default_dests("linux")["retroarch"], + xdg / "retroarch" / "system") + + def test_home_retroarch_cfg_is_the_fallback(self): + home, xdg = self._env() + os.environ["XDG_CONFIG_HOME"] = str(xdg) + (home / ".retroarch.cfg").write_text('system_directory = "default"\n') + found = install.detect_platforms("linux") + self.assertIn(("retroarch", xdg / "retroarch" / "system"), found) + + def test_without_the_variable_the_folder_is_under_dot_config(self): + home, _ = self._env() + (home / ".config" / "retroarch").mkdir(parents=True) + (home / ".config" / "retroarch" / "retroarch.cfg").write_text( + 'system_directory = "default"\n') + found = install.detect_platforms("linux") + self.assertIn(("retroarch", home / ".config" / "retroarch" / "system"), found) + + class NoPythonNamesThisSystemsCommand(unittest.TestCase): """Without Python the bootstrap said `sudo apt install python3` on every system: Termux has no sudo and installs with `pkg install python`, and @@ -3790,6 +3894,96 @@ class DataDirectoriesAreInstalled(unittest.TestCase): (len(body), len(body))) emit.assert_not_called() + def test_a_cut_archive_resumes_where_it_stopped(self): + """A data archive cut at 90% was deleted and fetched again from + zero, on the retry and on the next run, while the run said that + nothing was lost.""" + work = Path(_tempdir(self)) + body = self._zip(work).read_bytes() + keep = work / ".dolphin-sys.archive.part" + requests: list[dict] = [] + + class Response(io.BytesIO): + def __init__(self, data, status, headers): + super().__init__(data) + self.status, self.headers = status, headers + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def urlopen(request, timeout=None): + sent = dict(request.header_items()) + requests.append(sent) + if "Range" in sent: + start = int(sent["Range"][len("bytes="):-1]) + if sent.get("If-range") != '"v1"': + return Response(body, 200, {"ETag": '"v2"'}) + return Response(body[start:], 206, { + "Content-Range": f"bytes {start}-{len(body) - 1}/{len(body)}", + "Content-Length": str(len(body) - start), "ETag": '"v1"'}) + cut = len(body) * 9 // 10 + + class Cut(Response): + def read(self, size=-1): + if self.tell() >= cut: + raise ConnectionResetError("cut") + return super().read(min(size, cut - self.tell())) + return Cut(body, 200, {"Content-Length": str(len(body)), "ETag": '"v1"'}) + + with unittest.mock.patch.object(install.urllib.request, "urlopen", urlopen), \ + unittest.mock.patch.object(install, "_network_allowed", lambda url: None), \ + unittest.mock.patch.object(install, "_retry_delay", lambda *a: 0): + # The next run, after a stop: what the cut attempt wrote is kept. + with unittest.mock.patch.object(install, "MAX_RETRIES", 1), \ + self.assertRaises(ConnectionResetError): + install._fetch_archive("https://example.invalid/D.zip", work, None, keep) + self.assertEqual(keep.stat().st_size, len(body) * 9 // 10) + path = install._fetch_archive("https://example.invalid/D.zip", work, None, keep) + self.assertEqual(path.read_bytes(), body) + self.assertEqual(requests[-1]["Range"], f"bytes={len(body) * 9 // 10}-") + self.assertEqual(requests[-1]["If-range"], '"v1"') + + def test_a_changed_archive_is_fetched_whole(self): + work = Path(_tempdir(self)) + body = self._zip(work).read_bytes() + keep = work / ".dolphin-sys.archive.part" + keep.write_bytes(b"older revision") + keep.with_name(keep.name + ".source").write_text( + 'https://example.invalid/D.zip\n"v0"\n') + + class Whole(io.BytesIO): + status, headers = 200, {"ETag": '"v1"'} + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + with unittest.mock.patch.object(install.urllib.request, "urlopen", + lambda request, timeout=None: Whole(body)), \ + unittest.mock.patch.object(install, "_network_allowed", lambda url: None): + path = install._fetch_archive("https://example.invalid/D.zip", work, None, keep) + self.assertEqual(path.read_bytes(), body) + + def test_a_tree_read_whole_leaves_no_archive(self): + work = Path(_tempdir(self)) + archive = self._zip(work) + bios = work / "system" + bios.mkdir() + + def fetch(url, directory, progress=None, keep=None): + shutil.copy(archive, keep) + keep.with_name(keep.name + ".source").write_text(f"{url}\n\"v1\"\n") + return keep + + with unittest.mock.patch.object(install, "_fetch_archive", fetch): + install.install_data_directory(self._entry(), bios) + self.assertEqual([p.name for p in (bios / "dolphin-emu").iterdir()], ["Sys"]) + def test_the_run_ends_on_the_archive_bytes_too(self): from test_install_selection import FakeRepository # noqa: PLC0415 @@ -3811,13 +4005,44 @@ class DataDirectoriesAreInstalled(unittest.TestCase): self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) self.assertIn(f"{install.format_size(bios + archive)} downloaded in", proc.stdout) + def test_the_plan_announces_the_archive_not_the_tree(self): + """yaps2's tarball is 21.2 MB for a 6.5 MB tree, Cemu's 21.4 MB for + 9.9 KB: the plan announced the tree, and the run downloaded far more.""" + from test_install_selection import FakeRepository # noqa: PLC0415 + + member = b"profile" + archive_size = 21_360_579 + with FakeRepository() as repo: + listed = json.loads((repo.root / "install" / "retroarch.json").read_text()) + listed["data_directories"] = [{ + "ref": "cemu-gameprofiles", "dest": "cemu", "type": "tarball", + "url": "https://example.invalid/Cemu.tar.gz", "source_path": "Cemu-1/bin", + "systems": ["nintendo-gba"], "archive_size": archive_size, + "files": {"gameProfiles/a.ini": {"sha1": hashlib.sha1(member).hexdigest(), + "size": len(member)}}}] + (repo.root / "install" / "retroarch.json").write_text(json.dumps(listed)) + proc = repo.run("--no-input", "--dry-run") + bios = sum(len(payload) for payload, *_ in FakeRepository.PAYLOADS.values()) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn(f"({install.format_size(bios + archive_size)})", proc.stdout) + self.assertIn(f"(a {install.format_size(archive_size)} archive)", proc.stdout) + + def test_a_pack_or_an_old_list_counts_the_files_themselves(self): + needed = {"a": {"size": 7}} + entry = {"archive_size": 21_360_579} + self.assertEqual(install.data_fetch_bytes(entry, needed), 21_360_579) + self.assertEqual(install.data_fetch_bytes({}, needed), 7) + self.assertEqual(install.data_fetch_bytes(entry, {}), 0) + with unittest.mock.patch.object(install.LOCAL, "pack", object()): + self.assertEqual(install.data_fetch_bytes(entry, needed), 7) + def test_verified_members_land_the_rest_are_said(self): work = Path(_tempdir(self)) archive = self._zip(work) bios = work / "system" bios.mkdir() - def fetch(url, directory, progress=None): + def fetch(url, directory, progress=None, keep=None): copy = Path(directory) / "a.zip" shutil.copy(archive, copy) return copy @@ -3853,8 +4078,9 @@ class DataDirectoriesAreInstalled(unittest.TestCase): def members(_archive, _entry): yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Endless()) - with unittest.mock.patch.object(install, "_fetch_archive", - lambda url, d, progress=None: Path(d) / "a"), \ + with unittest.mock.patch.object( + install, "_fetch_archive", + lambda url, d, progress=None, keep=None: Path(d) / "a"), \ unittest.mock.patch.object(install, "_archive_members", members): counts = install.install_data_directory(entry, work) self.assertEqual(counts["changed"], 1) @@ -3881,8 +4107,9 @@ class DataDirectoriesAreInstalled(unittest.TestCase): def members(_archive, _entry): yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Broken()) - with unittest.mock.patch.object(install, "_fetch_archive", - lambda url, d, progress=None: Path(d) / "a"), \ + with unittest.mock.patch.object( + install, "_fetch_archive", + lambda url, d, progress=None, keep=None: Path(d) / "a"), \ unittest.mock.patch.object(install, "_archive_members", members): counts = install.install_data_directory(entry, work) self.assertTrue(counts["error"]) @@ -4422,6 +4649,50 @@ def _install_run(env: dict, *argv: str, **kwargs) -> subprocess.CompletedProcess ) +class AListingAppliesItsOptionsOrRefusesThem(unittest.TestCase): + """--list-systems --list-cores printed the systems alone, and + --list-platforms, --list-targets and --list-emulators printed the same + list with --system, --region or --core as without, code 0.""" + + def test_two_listings_or_an_option_a_listing_ignores_are_refused(self): + env = dict(os.environ, RETROBIOS_OS="linux") + given = ["--platform", "retroarch", "--offline", "--no-input", + "--manifest", "install/retroarch.json"] + for argv, words in ( + ([*given, "--list-systems", "--list-cores"], + "--list-systems and --list-cores each print a list"), + ([*given, "--list-cores", "--list-emulators"], + "--list-cores and --list-emulators each print a list"), + (["--list-platforms", "--list-systems", "--platform", "retroarch"], + "--list-platforms and --list-systems each print a list"), + (["--list-platforms", "--region", "us"], + "--list-platforms prints the whole list, which --region does not narrow"), + (["--list-platforms", "--platform", "retroarch"], + "--list-platforms prints the whole list, which --platform does not narrow"), + ([*given, "--list-targets", "--system", "sony-playstation"], + "--list-targets prints the whole list, which --system does not narrow"), + ([*given, "--list-targets", "--target", "switch"], + "--list-targets prints the whole list, which --target does not narrow"), + ([*given, "--list-emulators", "--system", "sony-playstation"], + "--list-emulators prints the whole list, which --system does not narrow"), + ([*given, "--list-emulators", "--core", "beetle_psx"], + "--list-emulators prints the whole list, which --core does not narrow"), + ): + with self.subTest(argv=argv): + proc = _install_run(env, *argv) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn(words, proc.stderr) + + def test_the_options_a_listing_reads_still_apply(self): + env = dict(os.environ, RETROBIOS_OS="linux") + given = ["--platform", "retroarch", "--offline", "--no-input", + "--manifest", "install/retroarch.json"] + whole = _install_run(env, *given, "--list-cores") + narrowed = _install_run(env, *given, "--list-cores", "--system", "nintendo-gba") + self.assertEqual((whole.returncode, narrowed.returncode), (0, 0), narrowed.stderr) + self.assertLess(len(narrowed.stdout), len(whole.stdout)) + + class AMissingPackNamesTheOptionAndThePacksBeside(unittest.TestCase): """A typo in --from-pack said "Error: X does not exist": not the option, not the pack sitting next to the name typed.""" @@ -6562,7 +6833,7 @@ class AFileYouHadIsKept(unittest.TestCase): "type": "zip", "files": {"Sys/a.ini": { "sha1": hashlib.sha1(self.NEW).hexdigest(), "size": len(self.NEW)}}} - def fetch(url, directory, progress=None): + def fetch(url, directory, progress=None, keep=None): copy = Path(directory) / "a.zip" shutil.copy(archive, copy) return copy diff --git a/tests/test_install_offline.py b/tests/test_install_offline.py index b46ea29be..08d9194d2 100644 --- a/tests/test_install_offline.py +++ b/tests/test_install_offline.py @@ -778,7 +778,12 @@ class OfflineMakesNoRequest(Scratch): self.assertIn("--offline", proc.stderr) def test_a_listing_and_a_pack_need_no_network_anyway(self): - proc = self.run_installer("--offline", "--list-platforms") + # --platform alone with it is refused: the listing names every one. + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--offline", "--list-platforms"], + env=_offline_env(), capture_output=True, text=True, timeout=120, + check=False, stdin=subprocess.DEVNULL, + ) self.assertEqual(proc.returncode, 0, proc.stderr) diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 24bbd3485..988b7234e 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -2967,12 +2967,44 @@ class SaveDirectoryLeftOut(unittest.TestCase): UNPLACED = {"reason": "no_save_directory", "label": "no save directory", "files": 2, "required": 0} + def _ended(self, manifest: dict) -> tuple: + state = fresh_state() + state.stack = ["setup"] + with mock.patch.object(install.Path, "exists", return_value=True): + install.tui_update(state, Key("enter")) + install.tui_receive(state, ("manifest", "retroarch", manifest)) + summary = {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 0, + "pending": 5, "failed": [], "skipped": [self.UNPLACED], "left_out": [], + "kept": [], "interrupted": False} + for event in [*FILES, *SYSTEMS, CHECK, summary]: + install.tui_receive(state, ("event", state.run_id, event)) + with mock.patch.object(install, "_tui_folder_facts", return_value={ + "free": 50 * 1024 ** 3, "writable": True, "cores": []}): + install.tui_receive(state, ("ended", state.run_id, 1, [])) + return state, summary + + def test_a_list_that_holds_none_of_them_offers_no_folder(self): + """RetroPie's list leaves its save files out with no hash or address: + --save-dir installs none of them, and was advised again each run.""" + state, summary = self._ended(MANIFEST) + line = ("2 files read from RetroArch's save directory left out: RetroArch keeps " + "it in no fixed place, and its file list holds none of them.") + plan = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn(line, plan) + self.assertNotIn("--save-dir", plan) + state.stack.append("result") + state.code = 0 + state.summary = {**summary, "mode": "install", "installed": 5} + self.assertEqual([text for text in install.tui_outcome(state)[1] if "save" in text], + [line]) + def test_files_left_out_of_the_save_directory_are_said(self): state = fresh_state() state.stack = ["setup"] with mock.patch.object(install.Path, "exists", return_value=True): install.tui_update(state, Key("enter")) - install.tui_receive(state, ("manifest", "retroarch", MANIFEST)) + install.tui_receive(state, ("manifest", "retroarch", { + **MANIFEST, "save_directory": {"folder": "saves", "by_core": True}})) summary = {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 0, "pending": 5, "failed": [], "skipped": [self.UNPLACED], "left_out": [], "kept": [], "interrupted": False} @@ -3834,6 +3866,8 @@ class BackTicksAndSaveFolder(unittest.TestCase): def test_the_save_folder_remedy_is_the_same_on_both_screens(self): summary = {"skipped": [{"reason": "no_save_directory", "files": 1}]} state = checked_state() + state.manifest = {**state.manifest, "save_directory": {"folder": "saves", + "by_core": True}} lines = install._unplaced_lines(summary, state) self.assertIn("Options, Save folder (--save-dir)", lines[0]) @@ -5898,7 +5932,10 @@ def _card_from_rules(root: Path, setup: install.CardSetup) -> Path: (root / "bios").mkdir(exist_ok=True) for value in rule.values: (root / value).parent.mkdir(parents=True, exist_ok=True) - if rule.method == "games_part" and "." not in PurePosixPath(value).name: + # A marker the BIOS folder sits under is a folder on the card + # (muOS's MUOS/bios): written as a file, no folder fits below it. + if (rule.method == "games_part" and "." not in PurePosixPath(value).name + or str(setup.below).startswith(f"{value}/")): (root / value).mkdir(exist_ok=True) else: (root / value).write_text("", encoding="utf-8") @@ -9763,14 +9800,21 @@ class ScreenRowsAndFilters(unittest.TestCase): from test_install_offline import PAYLOAD, _offline_env, _pack - blocker = self.root / "file" - blocker.write_text("not a folder") + if os.name == "nt" or os.geteuid() == 0: + self.skipTest("a folder that refuses writes is POSIX, and root writes anyway") + # A file in the way is refused before the plan, as not_a_folder. + blocker = self.root / "closed" + blocker.mkdir() pack = _pack(self.root, "RetroArch_v1_BIOS_Pack.zip", PAYLOAD) - proc = subprocess.run( - [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", - "--dest", str(blocker / "system"), "--from-pack", str(pack), "--events"], - env=_offline_env(), capture_output=True, text=True, timeout=120, check=False, - stdin=subprocess.DEVNULL) + os.chmod(blocker, 0o500) + try: + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", str(blocker / "system"), "--from-pack", str(pack), "--events"], + env=_offline_env(), capture_output=True, text=True, timeout=120, + check=False, stdin=subprocess.DEVNULL) + finally: + os.chmod(blocker, 0o700) events = [json.loads(line) for line in proc.stdout.splitlines()] kinds = [event["event"] for event in events] self.assertEqual(kinds[-3:], ["error", "summary", "end"], proc.stderr) @@ -11836,6 +11880,8 @@ class OneChoicePerRun(unittest.TestCase): def test_the_farewell_names_the_option_not_the_screen(self): state = _installed_state(SYSTEM_DIR) + state.manifest = {**state.manifest, "save_directory": {"folder": "saves", + "by_core": True}} state.summary = {**state.summary, "skipped": [ {"reason": "no_save_directory", "label": "", "files": 1, "required": False}]} farewell = " ".join(install.tui_farewell(state)) diff --git a/tests/test_refresh_data_dirs.py b/tests/test_refresh_data_dirs.py index dc38143ac..b7fcd5617 100644 --- a/tests/test_refresh_data_dirs.py +++ b/tests/test_refresh_data_dirs.py @@ -93,6 +93,26 @@ class RefreshConcurrency(unittest.TestCase): ) self.assertEqual(seen, [True]) + def test_the_archive_size_is_recorded_without_replacing_the_tree(self): + """The installer announces a tree's download as the archive's size: + a tarball extracted to 10 KB is a download of 21 MB. A tree current + before the size was kept is measured, its files left as they are.""" + payload = _zip_bytes({"a.bin": b"x" * 100}) + entry = {"source_type": "zip", "source_url": "https://x/pak.zip", + "local_cache": str(self.cache)} + with mock.patch.object(rdd.urllib.request, "urlopen", + lambda *a, **k: _Response(payload)), \ + mock.patch.object(rdd, "_get_remote_etag", return_value="e1"): + self.assertTrue(rdd.refresh_entry("pak", entry, versions_path=self.versions)) + self.assertEqual(rdd._load_versions(self.versions)["pak"]["archive_size"], + len(payload)) + rdd._record_version("pak", {"sha": "e1"}, self.versions) + (self.cache / "a.bin").write_bytes(b"kept") + self.assertFalse(rdd.refresh_entry("pak", entry, versions_path=self.versions)) + self.assertEqual(rdd._load_versions(self.versions)["pak"], + {"sha": "e1", "archive_size": len(payload)}) + self.assertEqual((self.cache / "a.bin").read_bytes(), b"kept") + def test_a_current_tree_is_not_waited_for(self): """A pipeline reads every tree for its whole run. A build beside it waited on the exclusive lock before asking the remote, which would @@ -100,7 +120,7 @@ class RefreshConcurrency(unittest.TestCase): import fcntl # noqa: PLC0415 self.cache.mkdir(parents=True) - rdd._record_version("pak", {"sha": "e1"}, self.versions) + rdd._record_version("pak", {"sha": "e1", "archive_size": 1}, self.versions) entry = {"source_type": "zip", "source_url": "https://x/pak.zip", "local_cache": str(self.cache)} result: list = [] @@ -258,7 +278,7 @@ class DryRunReport(unittest.TestCase): def test_the_remote_tag_is_shown_beside_the_cached_one(self): with tempfile.TemporaryDirectory() as tmp: versions = Path(tmp) / "versions.json" - versions.write_text(json.dumps({"k": {"etag": '"2725165595"'}})) + versions.write_text(json.dumps({"k": {"etag": '"2725165595"', "archive_size": 1}})) entry = {"source_type": "zip", "source_url": "https://x.invalid/a.zip", "local_cache": tmp} with mock.patch.object(rdd, "_get_remote_etag", return_value='"2685251384"'), \ @@ -278,5 +298,30 @@ class DryRunReport(unittest.TestCase): ) +class TheManifestCarriesTheArchiveSize(unittest.TestCase): + """The size refresh_data_dirs recorded reaches the install list, which + the installer announces as the tree's download.""" + + def test_recorded_size_is_written_and_an_unrecorded_one_left_out(self): + from generate_pack import _manifest_data_directories # noqa: PLC0415 + + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + data = Path(tmp) / "data" + for ref in ("pak", "old"): + (data / ref).mkdir(parents=True) + (data / ref / "a.bin").write_bytes(b"abc") + (data / ".versions.json").write_text(json.dumps( + {"pak": {"sha": "e1", "archive_size": 21_360_579}, "old": {"sha": "e2"}})) + registry = {ref: {"source_type": "zip", "source_url": f"https://x/{ref}.zip", + "local_cache": str(data / ref)} for ref in ("pak", "old")} + members = [(str(data / ref / "a.bin"), f"{ref}/a.bin") for ref in ("pak", "old")] + refs = {f"{ref}/a.bin": (ref, ("s",), ref) for ref in ("pak", "old")} + listed = {entry["ref"]: entry for entry in + _manifest_data_directories(members, refs, registry, "")} + self.assertEqual(listed["pak"]["archive_size"], 21_360_579) + self.assertNotIn("archive_size", listed["old"]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index d5a930694..d92206b8e 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -352,8 +352,10 @@ file list is downloaded once and handed to each run. times, after 1, 2 then 4 seconds or the delay a `Retry-After` header asks. A file of 16 MB or more keeps its partial copy (`..part`) when a transfer stops, and the next attempt or the next run requests only the - rest. A file already there with other contents is moved aside first, see - [Files you already had](#files-you-already-had). + rest. A data tree's archive does the same beside the tree + (`..archive.part`), asking with `If-Range` so that a server whose + archive changed sends it whole. A file already there with other contents + is moved aside first, see [Files you already had](#files-you-already-had). 8. Copies into standalone emulator directories, only with `--standalone-copies`. The file list names the version of the platform it was built for (Batocera @@ -386,7 +388,10 @@ the save directory beside it (`saves`, `/storage/savefiles` on Lakka). applies, or when `savefiles_in_content_dir` or `sort_savefiles_by_content_enable` keeps saves beside each game, those files are left out and the run says why, like the other entries the list cannot -install. The full screen says so on its plan and its result. +install. The full screen says so on its plan and its result. RetroPie, ROCKNIX, +AmberELEC, EmuELEC and RomM keep the save directory in no fixed place: their +lists hold none of these files, so `--save-dir` installs none, and the run says +that instead of naming the option. Batocera, KNULLI, Recalbox, RetroBat and RetroDECK write the save directory at each launch, one folder per system (`by_system` in the file list). Their lists @@ -976,9 +981,9 @@ what the options name, everything by default. | `--required-only` | With `--for-games`, only the files the games cannot start without | | `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | | `--declared-only` | Keep only the files the platform's own list names (`cores: null` in the manifest), not the extra files cores load. With `--check`, the answer is about the list the platform itself checks. Refused with `--core`, the listings it does not change, `--standalone-copies` and `--tui` | -| `--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart | -| `--list-cores` | Print the cores whose extra files a platform installs | -| `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | +| `--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart. `--system`, `--core`, `--region`, `--target` and `--installed-cores` narrow it, as they narrow an install. A run prints one list: two listing options are refused | +| `--list-cores` | Print the cores whose extra files a platform installs. The options that narrow `--list-systems` narrow it too | +| `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for. `--target` and `--installed-cores` narrow it, and `--system`, `--core`, `--region` and `--emulator` are refused | | `--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) | | `--no-input` | Never ask a question, even at a terminal. A run whose output goes to a file or a pipe asks nothing either, and says so on stderr. It finds the setups and the mounted cards the screen finds: one is taken, as Enter takes it, two stop the run with an `error` event (`stage` `detect`, `reason` `conflict`) naming each with its `--platform` and `--dest`, and none stops it too, with `reason` `not_found` when a file list or a pack names the platform | | `--check` | Report and exit without writing | @@ -994,8 +999,8 @@ what the options name, everything by default. | `--tui` | Open the full-screen interface, or stop with the reason it cannot open here (code 2) instead of falling back to the questions. Wins over `RETROBIOS_PLAIN` and `ACCESSIBLE`. See [The full-screen interface](#the-full-screen-interface) | | `--plain` | Ask the questions line by line instead of the full-screen interface, the mode for a screen reader. See [The full-screen interface](#the-full-screen-interface) | | `--version` | Print the installer's identity, the first 12 hex digits of the SHA-256 the bootstraps pin | -| `--list-platforms` | Print the supported platforms and what was detected here | -| `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries | +| `--list-platforms` | Print the supported platforms and what was detected here. An option that narrows a list (`--platform`, `--system`, `--core`, `--region`, `--emulator`, `--target`, `--every-board`, `--installed-cores`) is refused | +| `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries. `--system`, `--core`, `--region`, `--emulator`, `--target`, `--every-board` and `--installed-cores` are refused | | `--jobs N`, `-j N` | Parallel downloads, 1 to 32, default 8 | | `--verbose`, `-v` | Print per-attempt failures and hash mismatches | | `--standalone-copies` | Also copy into detected standalone emulator directories | @@ -1617,8 +1622,11 @@ built (`strip_components` for a ZIP, `source_path` for a tarball, `exclude`), and writes a file only when its hash matches the list. A file the source has changed since is left out and counted, and the pack still holds it. `--check` counts the data files missing, `--dry-run` names each tree it would fetch, and -narrowing with `--system`, `--core` or `--target` keeps only the trees of the -systems that stay. +the size announced for a tree is its archive's (`archive_size`, recorded when +`refresh_data_dirs.py` last downloaded it): one missing file fetches the whole +archive, and a tarball can be a thousand times its tree. Narrowing with +`--system`, `--core` or `--target` keeps only the trees of the systems that +stay. ## What the bootstrap does @@ -1740,7 +1748,7 @@ fields, the installer refuses to narrow rather than install everything. | Code | Meaning | |------|---------| | `0` | everything asked for is installed or already in place | -| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 with its ssl, unicodedata and zlib modules (nothing was run), an unusable base URL, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed or were left out by `--offline` while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, a `--dest` that is not the card of the system named (another system's card, a folder inside one, a drive with none of its files where its card always has them, or a folder that does not exist), `--restore` could not put a file back, or `--add-app` could not put every file of the app on the card | +| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 with its ssl, unicodedata and zlib modules (nothing was run), an unusable base URL, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed or were left out by `--offline` while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, a BIOS folder that is a file or lies under one, a `--dest` that is not the card of the system named (another system's card, a folder inside one, a drive with none of its files where its card always has them, or a folder that does not exist), `--restore` could not put a file back, or `--add-app` could not put every file of the app on the card | | `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), or `--offline` left every file out, the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board`, `--target` with `--from-pack` or `--offline` (or `RETROBIOS_INSTALL_OFFLINE` set) when no core list per board is at hand (the pack carries none and none is beside `--manifest` as `targets/.json`), or for a platform that publishes none, which is said to have no hardware targets, `--list-targets` with `--offline` and neither, `--add-app` with `--check`, `--dry-run`, a listing, `--from-pack` or `--offline`. The message then starts with `install.py: error:`. Also an unknown platform, target, system, core or emulator, said in an `Error:` or `Unknown platform` line and an `error` event whose `stage` is `option`. Also `--add-app` for a folder on no card that takes the app, said in an `Error:` line and an `app` event before anything is written. With `--restore`, a folder that is not a dated folder inside `retrobios-backup`. Also a quit chosen before the install, as `gh` ends a cancelled prompt: `q` at a question, `q` or **Quit** on the full screen, which then keeps `RetroBIOS: quit. Nothing was installed.`, and the full screen closed with no button pressed (`RETROBIOS_IDLE_EXIT`). No code but `0` closes a Windows Terminal tab started on the installer itself, so such a tab stays open on a quit | | `130` | stopped with Ctrl+C, or with Ctrl+C or Ctrl+D at a question, the terminal keeping `RetroBIOS: stopped. Nothing was installed.` before an install. Run the same command again to carry on | | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) | @@ -1787,7 +1795,7 @@ other schemas under `api/v1/schemas/`. | `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` | | `unsearched` | `folder`, `reason`, `message` | Before `detect`, once for each folder the search for setups could not look in, this user being refused it or a read failing there (a failing card, a link that loops). What it would hold counts as absent, and the other places are still searched: `detect` then lists what they hold, and an empty `platforms` after an `unsearched` line means nothing was found where the installer could look. `reason` says why in words (`Permission denied`), `message` says it in a sentence | | `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found | -| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), `no_ssl` when this Python was built without its `ssl` module and cannot use HTTPS, or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). The BIOS folder sits under a folder this user cannot open (`stage` is `dest`, `reason` `unreadable`), and the run ends with code 1 before anything is written. A platform, target, system, core or emulator named on the command line is unknown or cannot be used here (`stage` is `option`, `reason` `unknown`, `platform` the platform it was checked against or the name typed), refused before any download with code 2. The BIOS folder cannot be created (`stage` is `dest`, `reason` `not_writable`): a `summary` follows with what the run installed before it. Two systems are detected on one machine and no `--dest` names the folder (`stage` is `detect`, `reason` `conflict`). The platform named with `--platform` is found nowhere and no folder can be assumed for it: a card system with no card named, a portable emulator, Batocera or Recalbox named on a computer (their folder is the `bios` folder of their SHARE drive), or no usual folder on this system or one set aside behind a folder this user cannot open (`stage` is `detect`, `reason` `not_found`). So is a platform named by a file list or a pack, in a run without questions where it is found nowhere here. A `--dest` refused as the card of the system named carries the same `stage` and `reason`: another system's card, a folder inside a card, a drive with none of the files its card always has, or a folder that does not exist. `message` says it in words, and is the only thing that tells these cases apart | +| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), `no_ssl` when this Python was built without its `ssl` module and cannot use HTTPS, or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). The BIOS folder sits under a folder this user cannot open (`stage` is `dest`, `reason` `unreadable`), or is a file or lies under one (`reason` `not_a_folder`), and the run ends with code 1 before anything is written. A platform, target, system, core or emulator named on the command line is unknown or cannot be used here (`stage` is `option`, `reason` `unknown`, `platform` the platform it was checked against or the name typed), refused before any download with code 2. The BIOS folder cannot be created (`stage` is `dest`, `reason` `not_writable`): a `summary` follows with what the run installed before it. Two systems are detected on one machine and no `--dest` names the folder (`stage` is `detect`, `reason` `conflict`). The platform named with `--platform` is found nowhere and no folder can be assumed for it: a card system with no card named, a portable emulator, Batocera or Recalbox named on a computer (their folder is the `bios` folder of their SHARE drive), or no usual folder on this system or one set aside behind a folder this user cannot open (`stage` is `detect`, `reason` `not_found`). So is a platform named by a file list or a pack, in a run without questions where it is found nowhere here. A `--dest` refused as the card of the system named carries the same `stage` and `reason`: another system's card, a folder inside a card, a drive with none of the files its card always has, or a folder that does not exist. `message` says it in words, and is the only thing that tells these cases apart | | `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered | | `version` | `platform`, `list_version`, `installed_version`, `message` | Before `choices`, when the system here runs another version than the one the list was built for: `list_version` is the list's, `installed_version` the one the system's version file states, and `message` says it and what it means. Not written when either version is unknown. The run goes on | | `board` | `platform`, `board`, `applied`, `message` | Before `choices`, with no `--target` and no `--every-board`, when the system names its board or a Batocera or KNULLI card mounted here does (`boot/batocera.board` or `boot/knulli.board` on its boot partition, `board` then being the board its image maps to): `applied` is true when the run keeps that board's files only, false for a board the platform's list does not name, which keeps every board's files. `message` says it | @@ -1801,7 +1809,7 @@ other schemas under `api/v1/schemas/`. | `games_unavailable` | `platform`, `reason`, `message` | With `--for-games`, what the games need could not be read (`offline`, `network`, `invalid`, `stale`, `no_sets`, `no_folder`): the run keeps every file of the list, except for a `FOLDER` that does not exist, where it stops | | `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply | | `declared` | `platform`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `dests` | Right after `check`, the same counts over the files the platform's own list names (`cores: null`), and their `dests` in the order of the `file` events. A setup that checks its own list reads its answer here, with or without `--declared-only` | -| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes they write once in place | +| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes fetching them costs: each tree's archive, or the files themselves from a pack or a list that records no archive size | | `found` | `platform`, `dest`, `folder`, `path`, `member`, `reason` | Each file to install that this machine already holds, after `check` and before anything is written, in `--check` and `--dry-run` too. `folder` is the `--from` folder as given, or the `bios_dir` of the check when the BIOS folder holds the file at another place, and then `reason` says where in words (`nested folder: ...` for a misplaced file, `the same file is at ...` for one the list installs there too). `path` is the file, and `member` the ZIP member holding the bytes when `path` is an archive, an empty string otherwise. Its copy is reported by a `downloaded` event | | `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) | | `saves` | `platform`, `files`, `folder` | After `choices` and before `checking`, when the files the run installs (once `--system` and the other choices narrowed the list) hold some read from the frontend's save directory and that directory is known: how many, and the `folder` they go to, where `` and `` stand for one folder per system or per core (the screen and `--plain` say it in words: "a folder per core under /sdcard/RetroArch/saves"). When it is not known, the files are counted in the summary's `skipped` instead (`no_save_directory`) |