mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-28 06:37:24 -05:00
feat: add --offline and bound the connect time
--offline (or RETROBIOS_INSTALL_OFFLINE=1, which the runs the full-screen interface starts inherit) makes no network request. The file list stops the run at once with an error event of reason offline unless --manifest or --from-pack gives it, a file the run would download fails at once with its own advice, and board detection is skipped. --target and --list-targets refuse it. RETROBIOS_OFFLINE is not reused: CI and the test suite set it for every process, and the installer tests serve their files over loopback. A connection now has 4 s to open, then the read keeps its 30 or 60 s timeout: three tries that never connect end in about 15 s instead of 93, and a slow but open transfer is not cut. A --dry-run prints its totals first. A plan taller than the screen names the ten largest systems, and --verbose lists every system and file. Events are unchanged.
This commit is contained in:
1 parent
7143d20ef8
commit
d6782913cf
5 files changed
+322
-26
No files matched your search
+1
-1
@@ -11,7 +11,7 @@
|
|||||||
|
|
||||||
$ErrorActionPreference = "Stop"
|
$ErrorActionPreference = "Stop"
|
||||||
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||||
$defaultInstallSha256 = "240faa427aaa602deca1a3180f770c6f335095b1cf1d5bb28fc060eb55f54917"
|
$defaultInstallSha256 = "ea2777500114b796bb4a23ea9e0dd0a15bc4499bb1289e6e426943beece48865"
|
||||||
$maximumInstallerBytes = 2MB
|
$maximumInstallerBytes = 2MB
|
||||||
$installerExit = 0
|
$installerExit = 0
|
||||||
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
|
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
|
||||||
|
|||||||
+142
-21
@@ -98,6 +98,11 @@ RELEASE_URL = (
|
|||||||
"https://github.com/Abdess/retrobios/releases/download/large-files/{asset}"
|
"https://github.com/Abdess/retrobios/releases/download/large-files/{asset}"
|
||||||
)
|
)
|
||||||
MAX_RETRIES = 3
|
MAX_RETRIES = 3
|
||||||
|
# Seconds a connection has to open. A dropped network cost three tries of a
|
||||||
|
# 30-second timeout, 93 s, before the error: a connection that does not open
|
||||||
|
# now fails in this time, and once open a read waits as long as before, so
|
||||||
|
# a slow line is not cut.
|
||||||
|
CONNECT_TIMEOUT = 4.0
|
||||||
MAX_MANIFEST_BYTES = 16 * 1024 * 1024
|
MAX_MANIFEST_BYTES = 16 * 1024 * 1024
|
||||||
MAX_TARGETS_BYTES = 4 * 1024 * 1024
|
MAX_TARGETS_BYTES = 4 * 1024 * 1024
|
||||||
MAX_MANIFEST_FILES = 100_000
|
MAX_MANIFEST_FILES = 100_000
|
||||||
@@ -3150,10 +3155,13 @@ def network_words(
|
|||||||
) -> tuple[str, str]:
|
) -> tuple[str, str]:
|
||||||
"""The reason a request failed, and the same in words a beginner reads.
|
"""The reason a request failed, and the same in words a beginner reads.
|
||||||
|
|
||||||
The reason is clock, no_ca or network. Offline, the run printed
|
The reason is clock, no_ca, network, or offline for a request the run
|
||||||
|
was told not to make. Offline, the run printed
|
||||||
"[Errno -2] Name or service not known": the raw text now follows the
|
"[Errno -2] Name or service not known": the raw text now follows the
|
||||||
words only under --verbose.
|
words only under --verbose.
|
||||||
"""
|
"""
|
||||||
|
if isinstance(exc, _Offline):
|
||||||
|
return "offline", str(exc)
|
||||||
reason = getattr(exc, "reason", None)
|
reason = getattr(exc, "reason", None)
|
||||||
inner = reason if isinstance(reason, BaseException) else exc
|
inner = reason if isinstance(reason, BaseException) else exc
|
||||||
raw = str(reason or exc)
|
raw = str(reason or exc)
|
||||||
@@ -3208,6 +3216,49 @@ def _address_lines(url: str) -> list[str]:
|
|||||||
return lines
|
return lines
|
||||||
|
|
||||||
|
|
||||||
|
def _open_socket(
|
||||||
|
address: tuple[str, int], timeout: object, source_address: object = None,
|
||||||
|
) -> socket.socket:
|
||||||
|
"""A connection given CONNECT_TIMEOUT to open, then the caller's timeout."""
|
||||||
|
bounded = isinstance(timeout, (int, float))
|
||||||
|
limit = min(CONNECT_TIMEOUT, timeout) if bounded else CONNECT_TIMEOUT
|
||||||
|
sock = socket.create_connection(address, limit, source_address)
|
||||||
|
sock.settimeout(timeout if bounded else socket.getdefaulttimeout())
|
||||||
|
return sock
|
||||||
|
|
||||||
|
|
||||||
|
class _BoundedHTTPConnection(http.client.HTTPConnection):
|
||||||
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._create_connection = _open_socket
|
||||||
|
|
||||||
|
|
||||||
|
class _BoundedHTTPHandler(urllib.request.HTTPHandler):
|
||||||
|
def do_open(self, http_class: Any, req: Any, **kwargs: Any) -> Any:
|
||||||
|
return super().do_open(_BoundedHTTPConnection, req, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def install_bounded_opener() -> None:
|
||||||
|
"""Every request of the run opens its connection in CONNECT_TIMEOUT.
|
||||||
|
|
||||||
|
The proxy variables are read as before: build_opener keeps its
|
||||||
|
ProxyHandler, and only the HTTP and HTTPS handlers are replaced.
|
||||||
|
"""
|
||||||
|
handlers: list[type] = [_BoundedHTTPHandler]
|
||||||
|
if hasattr(http.client, "HTTPSConnection"):
|
||||||
|
class _BoundedHTTPSConnection(http.client.HTTPSConnection):
|
||||||
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._create_connection = _open_socket
|
||||||
|
|
||||||
|
class _BoundedHTTPSHandler(urllib.request.HTTPSHandler):
|
||||||
|
def do_open(self, http_class: Any, req: Any, **kwargs: Any) -> Any:
|
||||||
|
return super().do_open(_BoundedHTTPSConnection, req, **kwargs)
|
||||||
|
|
||||||
|
handlers.append(_BoundedHTTPSHandler)
|
||||||
|
urllib.request.install_opener(urllib.request.build_opener(*handlers))
|
||||||
|
|
||||||
|
|
||||||
def _fetch_json(
|
def _fetch_json(
|
||||||
url: str, limit: int, label: str, attempts: int = MAX_RETRIES, timeout: float = 30,
|
url: str, limit: int, label: str, attempts: int = MAX_RETRIES, timeout: float = 30,
|
||||||
) -> object:
|
) -> object:
|
||||||
@@ -3293,6 +3344,11 @@ def fetch_manifest(plat: str) -> dict:
|
|||||||
EVENTS.emit("error", stage="manifest", reason=code, platform=plat,
|
EVENTS.emit("error", stage="manifest", reason=code, platform=plat,
|
||||||
message=words if code != "network"
|
message=words if code != "network"
|
||||||
else f"the file list could not be fetched: {words}")
|
else f"the file list could not be fetched: {words}")
|
||||||
|
if code == "offline":
|
||||||
|
print(f" No file list for {plat}: {OFFLINE_WORDS}. Give it with "
|
||||||
|
"--manifest FILE (a copy of install/<platform>.json) or "
|
||||||
|
"--from-pack PACK, or run without --offline.", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
if code != "network":
|
if code != "network":
|
||||||
print(f" Could not fetch the file list for {plat}.", file=sys.stderr)
|
print(f" Could not fetch the file list for {plat}.", file=sys.stderr)
|
||||||
print(f" {words}.", file=sys.stderr)
|
print(f" {words}.", file=sys.stderr)
|
||||||
@@ -4343,7 +4399,8 @@ def _download_one(
|
|||||||
|
|
||||||
if LOCAL.pack is not None:
|
if LOCAL.pack is not None:
|
||||||
return _extract_from_pack(f, bios_path, dest, progress)
|
return _extract_from_pack(f, bios_path, dest, progress)
|
||||||
_network_allowed(f["dest"])
|
if offline_requested():
|
||||||
|
return f["dest"], False, OfflineReason(f"not fetched: {OFFLINE_WORDS}")
|
||||||
if f.get("release_asset"):
|
if f.get("release_asset"):
|
||||||
url = RELEASE_URL.format(asset=urllib.parse.quote(f["release_asset"], safe=""))
|
url = RELEASE_URL.format(asset=urllib.parse.quote(f["release_asset"], safe=""))
|
||||||
else:
|
else:
|
||||||
@@ -4776,6 +4833,7 @@ class Invocation:
|
|||||||
("--no-game-data", args.no_game_data),
|
("--no-game-data", args.no_game_data),
|
||||||
("--declared-only", args.declared_only),
|
("--declared-only", args.declared_only),
|
||||||
("--standalone-copies", args.standalone_copies),
|
("--standalone-copies", args.standalone_copies),
|
||||||
|
("--offline", getattr(args, "offline", False)),
|
||||||
("--check", args.check),
|
("--check", args.check),
|
||||||
("--dry-run", args.dry_run),
|
("--dry-run", args.dry_run),
|
||||||
):
|
):
|
||||||
@@ -4892,6 +4950,10 @@ def _plural(count: int, word: str) -> str:
|
|||||||
return f"{count} {word}" if count == 1 else f"{count} {word}s"
|
return f"{count} {word}" if count == 1 else f"{count} {word}s"
|
||||||
|
|
||||||
|
|
||||||
|
class OfflineReason(str):
|
||||||
|
"""A file not fetched because the run was told to make no request."""
|
||||||
|
|
||||||
|
|
||||||
class NetworkReason(str):
|
class NetworkReason(str):
|
||||||
"""A failure the connection caused: cut, timed out, refused by a server
|
"""A failure the connection caused: cut, timed out, refused by a server
|
||||||
or a proxy. The summary names the network instead of the folder."""
|
or a proxy. The summary names the network instead of the folder."""
|
||||||
@@ -4989,11 +5051,15 @@ def _report_outcome(
|
|||||||
"(or delete them), then run again.")
|
"(or delete them), then run again.")
|
||||||
if any(isinstance(reason, NetworkReason) for _dest, reason in failures or []):
|
if any(isinstance(reason, NetworkReason) for _dest, reason in failures or []):
|
||||||
advice.append(NETWORK_FAILURE_ADVICE)
|
advice.append(NETWORK_FAILURE_ADVICE)
|
||||||
|
if any(isinstance(reason, OfflineReason) for _dest, reason in failures or []):
|
||||||
|
advice.append(" These files come from the network, which --offline does not "
|
||||||
|
"use: run without --offline, or name a pack with --from-pack.")
|
||||||
paths = paths + [folder for folder in elsewhere if folder not in paths]
|
paths = paths + [folder for folder in elsewhere if folder not in paths]
|
||||||
|
|
||||||
if errors and not downloaded:
|
if errors and not downloaded:
|
||||||
print(f"\nNothing was installed. {_plural(errors, 'file')} failed.")
|
print(f"\nNothing was installed. {_plural(errors, 'file')} failed.")
|
||||||
if failures and all(isinstance(r, NetworkReason) for _d, r in failures):
|
if failures and all(isinstance(r, (NetworkReason, OfflineReason))
|
||||||
|
for _d, r in failures):
|
||||||
# The folder was never the problem: naming it as the place being
|
# The folder was never the problem: naming it as the place being
|
||||||
# written to sent people looking at their disk.
|
# written to sent people looking at their disk.
|
||||||
print(f"Destination: {_locations(paths)}")
|
print(f"Destination: {_locations(paths)}")
|
||||||
@@ -5104,6 +5170,12 @@ def installer_id() -> str:
|
|||||||
return "unknown"
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
# A --dry-run plan: file names per system, the systems shown when the
|
||||||
|
# listing is taller than the screen, and the lines around the listing.
|
||||||
|
PLAN_FILES_PER_SYSTEM = 8
|
||||||
|
PLAN_LARGEST_SYSTEMS = 10
|
||||||
|
PLAN_FRAME_LINES = 6
|
||||||
|
|
||||||
# Answers that stop: every menu before this one taught q as quit.
|
# Answers that stop: every menu before this one taught q as quit.
|
||||||
_REFUSALS = ("n", "q", "quit", "exit", "stop", "abort")
|
_REFUSALS = ("n", "q", "quit", "exit", "stop", "abort")
|
||||||
|
|
||||||
@@ -5119,42 +5191,61 @@ def _print_plan(
|
|||||||
trees: list[tuple[dict, dict[str, dict]]] = (),
|
trees: list[tuple[dict, dict[str, dict]]] = (),
|
||||||
kept: dict[Path, list[int]] | None = None,
|
kept: dict[Path, list[int]] | None = None,
|
||||||
copied: list[str] = (),
|
copied: list[str] = (),
|
||||||
|
verbose: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""What a real run would fetch, by system, and nothing written.
|
"""What a real run would fetch: the totals first, then by system, and
|
||||||
|
nothing written.
|
||||||
|
|
||||||
The data trees count in the total and come before the closing line:
|
The data trees count in the total and come before the closing line:
|
||||||
printed after it, they left 2636 RetroArch files out of the figure a
|
printed after it, they left 2636 RetroArch files out of the figure a
|
||||||
user plans an SD card from.
|
user plans an SD card from. A listing taller than the screen pushed the
|
||||||
|
total out of sight (599 lines for RetroArch): it gives way to the
|
||||||
|
largest systems, and --verbose lists every file.
|
||||||
"""
|
"""
|
||||||
total = sum(f.get("size", 0) for f in to_download)
|
total = sum(f.get("size", 0) for f in to_download)
|
||||||
trees = [(entry, needed) for entry, needed in trees if needed]
|
trees = [(entry, needed) for entry, needed in trees if needed]
|
||||||
tree_files = sum(len(needed) for _entry, needed in trees)
|
tree_files = sum(len(needed) for _entry, needed in trees)
|
||||||
tree_size = sum(meta.get("size", 0) for _entry, needed in trees for meta in needed.values())
|
tree_size = sum(meta.get("size", 0) for _entry, needed in trees for meta in needed.values())
|
||||||
for line in copied:
|
|
||||||
print(line)
|
|
||||||
if not to_download and not trees:
|
|
||||||
if not copied:
|
|
||||||
print("\n All files up to date. A real run would download nothing.")
|
|
||||||
return
|
|
||||||
print("\n Nothing to download.")
|
|
||||||
parts = []
|
parts = []
|
||||||
if to_download:
|
if to_download:
|
||||||
parts.append(f"{_plural(len(to_download), 'file')} ({format_size(total)})")
|
parts.append(f"{_plural(len(to_download), 'file')} ({format_size(total)})")
|
||||||
if trees:
|
if trees:
|
||||||
parts.append(f"{_plural(tree_files, 'data file')} ({format_size(tree_size)}, "
|
parts.append(f"{_plural(tree_files, 'data file')} ({format_size(tree_size)}, "
|
||||||
f"extracted from {_plural(len(trees), 'archive')})")
|
f"extracted from {_plural(len(trees), 'archive')})")
|
||||||
|
if not parts and not copied:
|
||||||
|
print("\n All files up to date. A real run would download nothing.")
|
||||||
|
return
|
||||||
if parts:
|
if parts:
|
||||||
print(f"\nWould download {' and '.join(parts)} to {bios_path}:")
|
print(f"\nWould download {' and '.join(parts)} to {bios_path}:")
|
||||||
|
for line in copied:
|
||||||
|
print(line)
|
||||||
|
if not parts:
|
||||||
|
print("\n Nothing to download.")
|
||||||
by_system: dict[str, list[dict]] = {}
|
by_system: dict[str, list[dict]] = {}
|
||||||
for entry in to_download:
|
for entry in to_download:
|
||||||
system = ", ".join(entry_systems(entry)) or "other"
|
system = ", ".join(entry_systems(entry)) or "other"
|
||||||
by_system.setdefault(system, []).append(entry)
|
by_system.setdefault(system, []).append(entry)
|
||||||
|
shown = None if verbose else PLAN_FILES_PER_SYSTEM
|
||||||
|
listing: list[str] = []
|
||||||
for system in sorted(by_system):
|
for system in sorted(by_system):
|
||||||
print(f" {term.bold(system)} {_sized(by_system[system])}")
|
entries = sorted(by_system[system], key=lambda e: e["dest"])
|
||||||
for entry in sorted(by_system[system], key=lambda e: e["dest"])[:8]:
|
listing.append(f" {term.bold(system)} {_sized(entries)}")
|
||||||
print(f" {entry['dest']} {term.dim(format_size(entry.get('size', 0)))}")
|
listing += [f" {entry['dest']} {term.dim(format_size(entry.get('size', 0)))}"
|
||||||
if len(by_system[system]) > 8:
|
for entry in entries[:shown]]
|
||||||
print(f" ... and {len(by_system[system]) - 8} more")
|
if shown is not None and len(entries) > shown:
|
||||||
|
listing.append(f" ... and {len(entries) - shown} more")
|
||||||
|
screen = shutil.get_terminal_size((80, 24)).lines
|
||||||
|
if verbose or len(listing) + len(trees) + PLAN_FRAME_LINES <= screen:
|
||||||
|
for line in listing:
|
||||||
|
print(line)
|
||||||
|
else:
|
||||||
|
largest = sorted(by_system, key=lambda s: (-_bytes(by_system[s]), s))
|
||||||
|
print(f" {_plural(len(largest), 'system')}, the largest first:")
|
||||||
|
for system in largest[:PLAN_LARGEST_SYSTEMS]:
|
||||||
|
print(f" {term.bold(system)} {_sized(by_system[system])}")
|
||||||
|
if len(largest) > PLAN_LARGEST_SYSTEMS:
|
||||||
|
print(f" ... and {len(largest) - PLAN_LARGEST_SYSTEMS} more systems: "
|
||||||
|
"--verbose lists every system and its files.")
|
||||||
for entry, needed in trees:
|
for entry, needed in trees:
|
||||||
source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"]
|
source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"]
|
||||||
print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, "
|
print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, "
|
||||||
@@ -5573,7 +5664,8 @@ def _install_standalone_folders(
|
|||||||
pending += len(to_download)
|
pending += len(to_download)
|
||||||
continue
|
continue
|
||||||
if args.dry_run:
|
if args.dry_run:
|
||||||
_print_plan(to_download, folder, term, kept=plan_kept(mismatched, folder))
|
_print_plan(to_download, folder, term, kept=plan_kept(mismatched, folder),
|
||||||
|
verbose=args.verbose)
|
||||||
continue
|
continue
|
||||||
if not to_download:
|
if not to_download:
|
||||||
print(" All files up to date.")
|
print(" All files up to date.")
|
||||||
@@ -6596,7 +6688,8 @@ class LocalSources:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def offline(self) -> bool:
|
def offline(self) -> bool:
|
||||||
return self.pack is not None
|
"""No network request in this run: a pack is read, or --offline."""
|
||||||
|
return self.pack is not None or offline_requested()
|
||||||
|
|
||||||
def reset(self) -> None:
|
def reset(self) -> None:
|
||||||
if self.pack is not None:
|
if self.pack is not None:
|
||||||
@@ -6614,9 +6707,22 @@ class _Offline(OSError):
|
|||||||
"""A network request in a run told to make none."""
|
"""A network request in a run told to make none."""
|
||||||
|
|
||||||
|
|
||||||
|
# --offline, or this variable set to anything but 0, which every run the
|
||||||
|
# full-screen interface starts inherits. RETROBIOS_OFFLINE stays the build
|
||||||
|
# scripts' own: CI and the test suite set it for every process they start.
|
||||||
|
OFFLINE_VARIABLE = "RETROBIOS_INSTALL_OFFLINE"
|
||||||
|
OFFLINE_WORDS = "this run makes no network request (--offline)"
|
||||||
|
|
||||||
|
|
||||||
|
def offline_requested() -> bool:
|
||||||
|
return os.environ.get(OFFLINE_VARIABLE, "") not in ("", "0")
|
||||||
|
|
||||||
|
|
||||||
def _network_allowed(what: str) -> None:
|
def _network_allowed(what: str) -> None:
|
||||||
if LOCAL.offline:
|
if LOCAL.pack is not None:
|
||||||
raise _Offline(f"{what}: this run reads a pack and makes no network request")
|
raise _Offline(f"{what}: this run reads a pack and makes no network request")
|
||||||
|
if offline_requested():
|
||||||
|
raise _Offline(f"{what}: {OFFLINE_WORDS}")
|
||||||
|
|
||||||
|
|
||||||
def read_local_manifest(path: Path, plat: str) -> dict:
|
def read_local_manifest(path: Path, plat: str) -> dict:
|
||||||
@@ -7980,6 +8086,13 @@ def main() -> None:
|
|||||||
"(the others are found beside it) or the folder holding them. "
|
"(the others are found beside it) or the folder holding them. "
|
||||||
"Makes no network request, and checks every file's hash",
|
"Makes no network request, and checks every file's hash",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--offline",
|
||||||
|
action="store_true",
|
||||||
|
help="make no network request: --check, --from, --from-pack and --manifest "
|
||||||
|
"work without one, and a file the run would download fails at once "
|
||||||
|
f"({OFFLINE_VARIABLE}=1 does the same)",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--from",
|
"--from",
|
||||||
action="append",
|
action="append",
|
||||||
@@ -8088,6 +8201,13 @@ def main() -> None:
|
|||||||
sys.exit(restore_kept(args.restore, args.dry_run))
|
sys.exit(restore_kept(args.restore, args.dry_run))
|
||||||
if args.every_board and args.target:
|
if args.every_board and args.target:
|
||||||
parser.error("--every-board installs every board; --target names one")
|
parser.error("--every-board installs every board; --target names one")
|
||||||
|
if args.offline:
|
||||||
|
# The runs the full-screen interface starts read it from here.
|
||||||
|
os.environ[OFFLINE_VARIABLE] = "1"
|
||||||
|
if offline_requested() and (args.target or args.list_targets):
|
||||||
|
parser.error("--target and --list-targets read a core list from the network, "
|
||||||
|
"which --offline does not use")
|
||||||
|
install_bounded_opener()
|
||||||
if args.from_pack and args.target:
|
if args.from_pack and args.target:
|
||||||
parser.error("--target reads a core list from the network, which --from-pack "
|
parser.error("--target reads a core list from the network, which --from-pack "
|
||||||
"does not use")
|
"does not use")
|
||||||
@@ -9050,7 +9170,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
|||||||
], bios_path)
|
], bios_path)
|
||||||
_print_plan([f for f in to_download if id(f) not in copies], bios_path, term,
|
_print_plan([f for f in to_download if id(f) not in copies], bios_path, term,
|
||||||
trees, kept, copy_plan_lines(to_download, copies, bios_index,
|
trees, kept, copy_plan_lines(to_download, copies, bios_index,
|
||||||
bios_path, files, args.verbose, True))
|
bios_path, files, args.verbose, True),
|
||||||
|
verbose=args.verbose)
|
||||||
for line in unreadable_lines(unread, args.verbose):
|
for line in unreadable_lines(unread, args.verbose):
|
||||||
print(line)
|
print(line)
|
||||||
for line in hint:
|
for line in hint:
|
||||||
|
|||||||
+1
-1
@@ -18,7 +18,7 @@ esac
|
|||||||
TEMP_INSTALLER=""
|
TEMP_INSTALLER=""
|
||||||
TEMP_DIRECTORY=""
|
TEMP_DIRECTORY=""
|
||||||
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||||
DEFAULT_INSTALL_SHA256="240faa427aaa602deca1a3180f770c6f335095b1cf1d5bb28fc060eb55f54917"
|
DEFAULT_INSTALL_SHA256="ea2777500114b796bb4a23ea9e0dd0a15bc4499bb1289e6e426943beece48865"
|
||||||
MAX_INSTALLER_BYTES=2097152
|
MAX_INSTALLER_BYTES=2097152
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import sys
|
|||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
|
import unittest.mock
|
||||||
import zipfile
|
import zipfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -589,5 +590,171 @@ class TheSummaryNamesTheBulk(unittest.TestCase):
|
|||||||
self.assertLess(text.index("Done."), text.index("arcade pulls"))
|
self.assertLess(text.index("Done."), text.index("arcade pulls"))
|
||||||
|
|
||||||
|
|
||||||
|
class OfflineMakesNoRequest(Scratch):
|
||||||
|
"""A handheld with no Wi-Fi ran --check against a loopback address to
|
||||||
|
learn its platform, since nothing said "make no request"."""
|
||||||
|
|
||||||
|
def _served(self, *options: str, env: dict | None = None):
|
||||||
|
import test_install_selection # noqa: PLC0415
|
||||||
|
|
||||||
|
hits: list[str] = []
|
||||||
|
with FakeRepository() as repo, unittest.mock.patch.object(
|
||||||
|
test_install_selection._QuietHandler, "log_message",
|
||||||
|
lambda handler, *args: hits.append(handler.path)):
|
||||||
|
started = time.monotonic()
|
||||||
|
proc = subprocess.run(
|
||||||
|
repo.command(*options), env={**repo.env(), **(env or {})},
|
||||||
|
capture_output=True, text=True, timeout=60, check=False,
|
||||||
|
stdin=subprocess.DEVNULL)
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
installed = repo.installed()
|
||||||
|
return proc, hits, elapsed, installed
|
||||||
|
|
||||||
|
def test_check_names_the_platform_and_stops_at_the_file_list(self):
|
||||||
|
for options, env in (((("--offline",)), None),
|
||||||
|
((), {"RETROBIOS_INSTALL_OFFLINE": "1"})):
|
||||||
|
with self.subTest(options=options, env=env):
|
||||||
|
proc, hits, elapsed, installed = self._served(
|
||||||
|
*options, "--check", "--events", env=env)
|
||||||
|
self.assertEqual(hits, [])
|
||||||
|
self.assertLess(elapsed, 30)
|
||||||
|
self.assertEqual(proc.returncode, 1, proc.stderr)
|
||||||
|
events = [json.loads(line) for line in proc.stdout.splitlines()]
|
||||||
|
self.assertEqual([e["event"] for e in events],
|
||||||
|
["start", "detect", "error", "end"])
|
||||||
|
self.assertEqual(events[1]["platforms"][0]["platform"], "retroarch")
|
||||||
|
self.assertEqual((events[2]["stage"], events[2]["reason"]),
|
||||||
|
("manifest", "offline"))
|
||||||
|
self.assertIn("no network request", events[2]["message"])
|
||||||
|
self.assertIn("--manifest", proc.stderr)
|
||||||
|
self.assertNotIn("attempts", proc.stderr)
|
||||||
|
self.assertEqual(installed, [])
|
||||||
|
|
||||||
|
def test_a_copy_lands_and_the_rest_fails_at_once(self):
|
||||||
|
old = self.root / "old"
|
||||||
|
old.mkdir()
|
||||||
|
(old / "scph5501.bin").write_bytes(US)
|
||||||
|
proc = self.run_installer("--offline", "--manifest", str(self.list),
|
||||||
|
"--from", str(old), "--no-input")
|
||||||
|
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
|
||||||
|
self.assertEqual((self.dest / "scph5501.bin").read_bytes(), US)
|
||||||
|
self.assertFalse((self.dest / "gba_bios.bin").exists())
|
||||||
|
self.assertIn("no network request", proc.stdout + proc.stderr)
|
||||||
|
self.assertIn("run without --offline", proc.stdout + proc.stderr)
|
||||||
|
self.assertNotIn("trying again", proc.stderr)
|
||||||
|
|
||||||
|
def test_modes_that_need_the_network_refuse_it(self):
|
||||||
|
for extra in (["--target", "switch"], ["--list-targets"]):
|
||||||
|
with self.subTest(extra=extra):
|
||||||
|
proc = self.run_installer("--offline", *extra)
|
||||||
|
self.assertEqual(proc.returncode, 2, proc.stderr)
|
||||||
|
self.assertIn("--offline", proc.stderr)
|
||||||
|
|
||||||
|
def test_a_listing_and_a_pack_need_no_network_anyway(self):
|
||||||
|
proc = self.run_installer("--offline", "--list-platforms")
|
||||||
|
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
class ANetworkThatDroppedFailsFast(unittest.TestCase):
|
||||||
|
"""A dropped network cost 93 s before the line-mode error: three tries
|
||||||
|
of a 30-second timeout spent waiting for a connection that never opened."""
|
||||||
|
|
||||||
|
def test_the_connection_is_bounded_and_the_read_is_not(self):
|
||||||
|
calls: list[tuple] = []
|
||||||
|
|
||||||
|
class Sock:
|
||||||
|
def settimeout(self, value):
|
||||||
|
calls.append(("read", value))
|
||||||
|
|
||||||
|
def create(address, timeout, source_address=None):
|
||||||
|
calls.append(("connect", timeout))
|
||||||
|
return Sock()
|
||||||
|
|
||||||
|
with unittest.mock.patch.object(install.socket, "create_connection", create):
|
||||||
|
install._open_socket(("example.org", 443), 60)
|
||||||
|
install._open_socket(("example.org", 443), 2)
|
||||||
|
self.assertEqual(calls, [("connect", install.CONNECT_TIMEOUT), ("read", 60),
|
||||||
|
("connect", 2), ("read", 2)])
|
||||||
|
|
||||||
|
def test_three_tries_that_never_connect_end_within_fifteen_seconds(self):
|
||||||
|
waited = sum(install._retry_delay(n, None) for n in range(1, install.MAX_RETRIES))
|
||||||
|
self.assertLessEqual(install.MAX_RETRIES * install.CONNECT_TIMEOUT + waited, 15)
|
||||||
|
|
||||||
|
def test_a_slow_answer_on_an_open_connection_still_arrives(self):
|
||||||
|
import http.server # noqa: PLC0415
|
||||||
|
import threading # noqa: PLC0415
|
||||||
|
|
||||||
|
delay = install.CONNECT_TIMEOUT + 1
|
||||||
|
|
||||||
|
class Slow(http.server.BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self): # noqa: N802 - stdlib name
|
||||||
|
time.sleep(delay)
|
||||||
|
body = b'{"ok": true}'
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def log_message(self, *args):
|
||||||
|
return
|
||||||
|
|
||||||
|
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Slow)
|
||||||
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||||
|
self.addCleanup(server.server_close)
|
||||||
|
self.addCleanup(server.shutdown)
|
||||||
|
install.install_bounded_opener()
|
||||||
|
self.addCleanup(install.urllib.request.install_opener, None)
|
||||||
|
url = f"http://127.0.0.1:{server.server_address[1]}/list.json"
|
||||||
|
started = time.monotonic()
|
||||||
|
self.assertEqual(install._fetch_json(url, 1024, "file list", attempts=1), {"ok": True})
|
||||||
|
self.assertGreaterEqual(time.monotonic() - started, delay - 0.5)
|
||||||
|
|
||||||
|
|
||||||
|
class ADryRunOpensOnItsTotals(Scratch):
|
||||||
|
"""A full --dry-run printed 599 lines, the total scrolled away above
|
||||||
|
eight file names for each of 66 systems."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super().setUp()
|
||||||
|
files = [
|
||||||
|
{"dest": f"sys{n:02d}/file{k}.bin", "sha1": _sha1(b"%d-%d" % (n, k)),
|
||||||
|
"size": 1000 * (n + 1), "repo_path": f"bios/sys{n:02d}_{k}", "cores": None,
|
||||||
|
"systems": [f"system-{n:02d}"]}
|
||||||
|
for n in range(40) for k in range(3)
|
||||||
|
]
|
||||||
|
self.list.write_text(json.dumps({"manifest_version": 2, "platform": "retroarch",
|
||||||
|
"files": files}))
|
||||||
|
|
||||||
|
def _plan(self, *options: str) -> list[str]:
|
||||||
|
proc = subprocess.run(
|
||||||
|
[sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch",
|
||||||
|
"--dest", str(self.dest), "--manifest", str(self.list), "--dry-run",
|
||||||
|
"--no-input", *options],
|
||||||
|
env={**_offline_env(), "LINES": "24", "COLUMNS": "80"},
|
||||||
|
capture_output=True, text=True, timeout=120, check=False,
|
||||||
|
stdin=subprocess.DEVNULL)
|
||||||
|
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
||||||
|
lines = proc.stdout.splitlines()
|
||||||
|
return lines[next(i for i, line in enumerate(lines)
|
||||||
|
if line.startswith("Would download")):]
|
||||||
|
|
||||||
|
def test_the_total_comes_first_and_the_largest_systems_after(self):
|
||||||
|
plan = self._plan()
|
||||||
|
self.assertTrue(plan[0].startswith("Would download 120 files"), plan[0])
|
||||||
|
self.assertLessEqual(len(plan), 24, "\n".join(plan))
|
||||||
|
self.assertIn("system-39", plan[2])
|
||||||
|
self.assertTrue(any("30 more systems" in line and "--verbose" in line
|
||||||
|
for line in plan), "\n".join(plan))
|
||||||
|
self.assertFalse(any("file0.bin" in line for line in plan))
|
||||||
|
self.assertEqual(plan[-1].strip(), "Nothing was written. Run without --dry-run to install.")
|
||||||
|
|
||||||
|
def test_verbose_lists_every_system_and_its_files(self):
|
||||||
|
plan = self._plan("--verbose")
|
||||||
|
self.assertTrue(plan[0].startswith("Would download 120 files"), plan[0])
|
||||||
|
for n in range(40):
|
||||||
|
self.assertTrue(any(line.strip().startswith(f"system-{n:02d}") for line in plan), n)
|
||||||
|
self.assertTrue(any("sys00/file0.bin" in line for line in plan))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
+11
-3
@@ -648,12 +648,13 @@ what the options name, everything by default.
|
|||||||
| `--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) |
|
| `--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) |
|
||||||
| `--no-input` | Never ask a question, even at a terminal |
|
| `--no-input` | Never ask a question, even at a terminal |
|
||||||
| `--check` | Report and exit without writing |
|
| `--check` | Report and exit without writing |
|
||||||
| `--dry-run` | Show what would be downloaded, by system and with sizes, without writing anything |
|
| `--dry-run` | Show what would be downloaded, the totals first, then by system and with sizes, without writing anything. A plan taller than the screen names the ten largest systems, and `--verbose` lists every system and file |
|
||||||
| `--yes`, `-y` | Do not ask before a download of 1 GB or more |
|
| `--yes`, `-y` | Do not ask before a download of 1 GB or more |
|
||||||
| `--quiet`, `-q` | Print only failures and the final summary. Implies `--no-input` |
|
| `--quiet`, `-q` | Print only failures and the final summary. Implies `--no-input` |
|
||||||
| `--events` | Write the run as JSON events on standard output, one object per line, and the usual text on standard error. Implies `--no-input`. See [Machine-readable events](#machine-readable-events) |
|
| `--events` | Write the run as JSON events on standard output, one object per line, and the usual text on standard error. Implies `--no-input`. See [Machine-readable events](#machine-readable-events) |
|
||||||
| `--from FOLDER` | Copy the BIOS files a folder holds (an old SD card, a backup, a ZIP) instead of downloading them, each one recognised by its hash and copied, never moved. Repeat for several folders. See [Copy from an old card](#copy-from-an-old-card) |
|
| `--from FOLDER` | Copy the BIOS files a folder holds (an old SD card, a backup, a ZIP) instead of downloading them, each one recognised by its hash and copied, never moved. Repeat for several folders. See [Copy from an old card](#copy-from-an-old-card) |
|
||||||
| `--from-pack PATH` | Install from a pack already downloaded: its ZIP, one of its parts (the others are found in the same folder), or the folder holding them. Makes no network request and checks every file's hash. See [Without a network](#without-a-network-offline-install) |
|
| `--from-pack PATH` | Install from a pack already downloaded: its ZIP, one of its parts (the others are found in the same folder), or the folder holding them. Makes no network request and checks every file's hash. See [Without a network](#without-a-network-offline-install) |
|
||||||
|
| `--offline` | Make no network request. `--check`, `--from`, `--from-pack` and `--manifest` work without one, the file list stops the run at once when none of them gives it, and a file the run would download fails at once. `RETROBIOS_INSTALL_OFFLINE=1` does the same. Refused with `--target` and `--list-targets`, which read a core list from the network |
|
||||||
| `--manifest FILE` | Read the file list from FILE, a copy of `install/<platform>.json`, instead of downloading it. See [Without a network](#without-a-network-offline-install) |
|
| `--manifest FILE` | Read the file list from FILE, a copy of `install/<platform>.json`, instead of downloading it. See [Without a network](#without-a-network-offline-install) |
|
||||||
| `--restore FOLDER` | Put back the files a run replaced and kept in FOLDER, a dated folder inside `retrobios-backup`. Takes no other option but `--dry-run` and `--no-input`. See [Files you already had](#files-you-already-had) |
|
| `--restore FOLDER` | Put back the files a run replaced and kept in FOLDER, a dated folder inside `retrobios-backup`. Takes no other option but `--dry-run` and `--no-input`. See [Files you already had](#files-you-already-had) |
|
||||||
| `--tui` | Open the full-screen interface, or stop with the reason it cannot open here (code 2) instead of falling back to the questions. Wins over `RETROBIOS_PLAIN` and `ACCESSIBLE`. See [The full-screen interface](#the-full-screen-interface) |
|
| `--tui` | Open the full-screen interface, or stop with the reason it cannot open here (code 2) instead of falling back to the questions. Wins over `RETROBIOS_PLAIN` and `ACCESSIBLE`. See [The full-screen interface](#the-full-screen-interface) |
|
||||||
@@ -1095,6 +1096,10 @@ pack: `--system`, `--core`, `--region` and `--no-game-data` then apply as
|
|||||||
they do online, and a file of the list the pack does not hold fails as
|
they do online, and a file of the list the pack does not hold fails as
|
||||||
`not in the pack`. The list's platform has to be the one installed for.
|
`not in the pack`. The list's platform has to be the one installed for.
|
||||||
|
|
||||||
|
`--offline` forbids every request, for a device with no Wi-Fi: `--check`
|
||||||
|
still names the platform and its folder, then stops at the file list unless
|
||||||
|
`--manifest` or `--from-pack` gives it.
|
||||||
|
|
||||||
## Where the files come from
|
## Where the files come from
|
||||||
|
|
||||||
Each manifest entry names either `repo_path`, served from the repository at the
|
Each manifest entry names either `repo_path`, served from the repository at the
|
||||||
@@ -1247,7 +1252,7 @@ other schemas under `api/v1/schemas/`.
|
|||||||
|------|------|------|
|
|------|------|------|
|
||||||
| `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` |
|
| `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` |
|
||||||
| `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found |
|
| `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found |
|
||||||
| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). `message` says it in words |
|
| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). `message` says it in words |
|
||||||
| `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered |
|
| `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered |
|
||||||
| `checking` | `files_done`, `files_total` | While the files already in the folder are read and hashed, at most every half second and once at the end. `files_total` counts the files found in place |
|
| `checking` | `files_done`, `files_total` | While the files already in the folder are read and hashed, at most every half second and once at the end. `files_total` counts the files found in place |
|
||||||
| `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` |
|
| `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` |
|
||||||
@@ -1274,6 +1279,7 @@ no event at all. A program reads a missing `end` as that case.
|
|||||||
|----------|--------|
|
|----------|--------|
|
||||||
| `RETROBIOS_REF` | Branch or tag the file list and the files committed to the repository are read from. Defaults to `main`. Files the manifest marks `release_asset` always come from the `large-files` release, whatever the ref, and that release replaces an asset in place when it is rebuilt. Each release deletes the previous release tags, so a tag holds a file list until the next release. For contributors: [release process](release-process.md#cutting-a-release) |
|
| `RETROBIOS_REF` | Branch or tag the file list and the files committed to the repository are read from. Defaults to `main`. Files the manifest marks `release_asset` always come from the `large-files` release, whatever the ref, and that release replaces an asset in place when it is rebuilt. Each release deletes the previous release tags, so a tag holds a file list until the next release. For contributors: [release process](release-process.md#cutting-a-release) |
|
||||||
| `RETROBIOS_BASE_URL` | Base URL for the manifest and the files it declares. HTTPS only, plain HTTP allowed to loopback for end-to-end tests. Whoever controls this base controls the expected hashes, hence the check. A file list that cannot be fetched names the address, and this variable when it is set |
|
| `RETROBIOS_BASE_URL` | Base URL for the manifest and the files it declares. HTTPS only, plain HTTP allowed to loopback for end-to-end tests. Whoever controls this base controls the expected hashes, hence the check. A file list that cannot be fetched names the address, and this variable when it is set |
|
||||||
|
| `RETROBIOS_INSTALL_OFFLINE` | Set to anything but `0`: no network request, as with `--offline`. The runs the full-screen interface starts inherit it. `RETROBIOS_OFFLINE` is the build scripts' own and leaves the installer alone |
|
||||||
| `RETROBIOS_OS` | Names the host outright (`linux`, `wsl`, `windows`, `darwin`, `android`) instead of detecting it |
|
| `RETROBIOS_OS` | Names the host outright (`linux`, `wsl`, `windows`, `darwin`, `android`) instead of detecting it |
|
||||||
| `EXTERNAL_STORAGE` | Android only, the shared storage root RetroArch builds its default directories from. Defaults to `/storage/emulated/0` |
|
| `EXTERNAL_STORAGE` | Android only, the shared storage root RetroArch builds its default directories from. Defaults to `/storage/emulated/0` |
|
||||||
| `RETROBIOS_INSTALL_URL` | Where the bootstrap fetches `install.py`. HTTPS only |
|
| `RETROBIOS_INSTALL_URL` | Where the bootstrap fetches `install.py`. HTTPS only |
|
||||||
@@ -1295,7 +1301,9 @@ The common cases, with the short answer, are on
|
|||||||
The installer fetches its manifest, then downloads only the repository or
|
The installer fetches its manifest, then downloads only the repository or
|
||||||
`large-files` release objects it needs. When a download fails, check that
|
`large-files` release objects it needs. When a download fails, check that
|
||||||
`https://github.com` opens. Behind a proxy, set `HTTPS_PROXY` in the
|
`https://github.com` opens. Behind a proxy, set `HTTPS_PROXY` in the
|
||||||
environment. A file that fails three attempts is counted and reported at the
|
environment. A connection that does not open within 4 seconds counts as
|
||||||
|
none, so a dropped network gives its error in about 15 seconds. Once a
|
||||||
|
connection is open, a slow transfer keeps the full timeout. A file that fails three attempts is counted and reported at the
|
||||||
end, and the next run fetches only what is still missing.
|
end, and the next run fetches only what is still missing.
|
||||||
|
|
||||||
The reason is said in words: no internet connection or a name that could not
|
The reason is said in words: no internet connection or a name that could not
|
||||||
|
|||||||
Reference in new issue
Block a user