mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-24 04:39:06 -05:00
feat: add --offline and bound the connect time
--offline (or RETROBIOS_INSTALL_OFFLINE=1, which the runs the full-screen interface starts inherit) makes no network request. The file list stops the run at once with an error event of reason offline unless --manifest or --from-pack gives it, a file the run would download fails at once with its own advice, and board detection is skipped. --target and --list-targets refuse it. RETROBIOS_OFFLINE is not reused: CI and the test suite set it for every process, and the installer tests serve their files over loopback. A connection now has 4 s to open, then the read keeps its 30 or 60 s timeout: three tries that never connect end in about 15 s instead of 93, and a slow but open transfer is not cut. A --dry-run prints its totals first. A plan taller than the screen names the ten largest systems, and --verbose lists every system and file. Events are unchanged.
This commit is contained in:
1 parent
7143d20ef8
commit
d6782913cf
5 files changed
+322
-26
No files matched your search
+1
-1
@@ -11,7 +11,7 @@
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
$defaultInstallSha256 = "240faa427aaa602deca1a3180f770c6f335095b1cf1d5bb28fc060eb55f54917"
|
||||
$defaultInstallSha256 = "ea2777500114b796bb4a23ea9e0dd0a15bc4499bb1289e6e426943beece48865"
|
||||
$maximumInstallerBytes = 2MB
|
||||
$installerExit = 0
|
||||
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
|
||||
|
||||
+142
-21
@@ -98,6 +98,11 @@ RELEASE_URL = (
|
||||
"https://github.com/Abdess/retrobios/releases/download/large-files/{asset}"
|
||||
)
|
||||
MAX_RETRIES = 3
|
||||
# Seconds a connection has to open. A dropped network cost three tries of a
|
||||
# 30-second timeout, 93 s, before the error: a connection that does not open
|
||||
# now fails in this time, and once open a read waits as long as before, so
|
||||
# a slow line is not cut.
|
||||
CONNECT_TIMEOUT = 4.0
|
||||
MAX_MANIFEST_BYTES = 16 * 1024 * 1024
|
||||
MAX_TARGETS_BYTES = 4 * 1024 * 1024
|
||||
MAX_MANIFEST_FILES = 100_000
|
||||
@@ -3150,10 +3155,13 @@ def network_words(
|
||||
) -> tuple[str, str]:
|
||||
"""The reason a request failed, and the same in words a beginner reads.
|
||||
|
||||
The reason is clock, no_ca or network. Offline, the run printed
|
||||
The reason is clock, no_ca, network, or offline for a request the run
|
||||
was told not to make. Offline, the run printed
|
||||
"[Errno -2] Name or service not known": the raw text now follows the
|
||||
words only under --verbose.
|
||||
"""
|
||||
if isinstance(exc, _Offline):
|
||||
return "offline", str(exc)
|
||||
reason = getattr(exc, "reason", None)
|
||||
inner = reason if isinstance(reason, BaseException) else exc
|
||||
raw = str(reason or exc)
|
||||
@@ -3208,6 +3216,49 @@ def _address_lines(url: str) -> list[str]:
|
||||
return lines
|
||||
|
||||
|
||||
def _open_socket(
|
||||
address: tuple[str, int], timeout: object, source_address: object = None,
|
||||
) -> socket.socket:
|
||||
"""A connection given CONNECT_TIMEOUT to open, then the caller's timeout."""
|
||||
bounded = isinstance(timeout, (int, float))
|
||||
limit = min(CONNECT_TIMEOUT, timeout) if bounded else CONNECT_TIMEOUT
|
||||
sock = socket.create_connection(address, limit, source_address)
|
||||
sock.settimeout(timeout if bounded else socket.getdefaulttimeout())
|
||||
return sock
|
||||
|
||||
|
||||
class _BoundedHTTPConnection(http.client.HTTPConnection):
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._create_connection = _open_socket
|
||||
|
||||
|
||||
class _BoundedHTTPHandler(urllib.request.HTTPHandler):
|
||||
def do_open(self, http_class: Any, req: Any, **kwargs: Any) -> Any:
|
||||
return super().do_open(_BoundedHTTPConnection, req, **kwargs)
|
||||
|
||||
|
||||
def install_bounded_opener() -> None:
|
||||
"""Every request of the run opens its connection in CONNECT_TIMEOUT.
|
||||
|
||||
The proxy variables are read as before: build_opener keeps its
|
||||
ProxyHandler, and only the HTTP and HTTPS handlers are replaced.
|
||||
"""
|
||||
handlers: list[type] = [_BoundedHTTPHandler]
|
||||
if hasattr(http.client, "HTTPSConnection"):
|
||||
class _BoundedHTTPSConnection(http.client.HTTPSConnection):
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._create_connection = _open_socket
|
||||
|
||||
class _BoundedHTTPSHandler(urllib.request.HTTPSHandler):
|
||||
def do_open(self, http_class: Any, req: Any, **kwargs: Any) -> Any:
|
||||
return super().do_open(_BoundedHTTPSConnection, req, **kwargs)
|
||||
|
||||
handlers.append(_BoundedHTTPSHandler)
|
||||
urllib.request.install_opener(urllib.request.build_opener(*handlers))
|
||||
|
||||
|
||||
def _fetch_json(
|
||||
url: str, limit: int, label: str, attempts: int = MAX_RETRIES, timeout: float = 30,
|
||||
) -> object:
|
||||
@@ -3293,6 +3344,11 @@ def fetch_manifest(plat: str) -> dict:
|
||||
EVENTS.emit("error", stage="manifest", reason=code, platform=plat,
|
||||
message=words if code != "network"
|
||||
else f"the file list could not be fetched: {words}")
|
||||
if code == "offline":
|
||||
print(f" No file list for {plat}: {OFFLINE_WORDS}. Give it with "
|
||||
"--manifest FILE (a copy of install/<platform>.json) or "
|
||||
"--from-pack PACK, or run without --offline.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if code != "network":
|
||||
print(f" Could not fetch the file list for {plat}.", file=sys.stderr)
|
||||
print(f" {words}.", file=sys.stderr)
|
||||
@@ -4343,7 +4399,8 @@ def _download_one(
|
||||
|
||||
if LOCAL.pack is not None:
|
||||
return _extract_from_pack(f, bios_path, dest, progress)
|
||||
_network_allowed(f["dest"])
|
||||
if offline_requested():
|
||||
return f["dest"], False, OfflineReason(f"not fetched: {OFFLINE_WORDS}")
|
||||
if f.get("release_asset"):
|
||||
url = RELEASE_URL.format(asset=urllib.parse.quote(f["release_asset"], safe=""))
|
||||
else:
|
||||
@@ -4776,6 +4833,7 @@ class Invocation:
|
||||
("--no-game-data", args.no_game_data),
|
||||
("--declared-only", args.declared_only),
|
||||
("--standalone-copies", args.standalone_copies),
|
||||
("--offline", getattr(args, "offline", False)),
|
||||
("--check", args.check),
|
||||
("--dry-run", args.dry_run),
|
||||
):
|
||||
@@ -4892,6 +4950,10 @@ def _plural(count: int, word: str) -> str:
|
||||
return f"{count} {word}" if count == 1 else f"{count} {word}s"
|
||||
|
||||
|
||||
class OfflineReason(str):
|
||||
"""A file not fetched because the run was told to make no request."""
|
||||
|
||||
|
||||
class NetworkReason(str):
|
||||
"""A failure the connection caused: cut, timed out, refused by a server
|
||||
or a proxy. The summary names the network instead of the folder."""
|
||||
@@ -4989,11 +5051,15 @@ def _report_outcome(
|
||||
"(or delete them), then run again.")
|
||||
if any(isinstance(reason, NetworkReason) for _dest, reason in failures or []):
|
||||
advice.append(NETWORK_FAILURE_ADVICE)
|
||||
if any(isinstance(reason, OfflineReason) for _dest, reason in failures or []):
|
||||
advice.append(" These files come from the network, which --offline does not "
|
||||
"use: run without --offline, or name a pack with --from-pack.")
|
||||
paths = paths + [folder for folder in elsewhere if folder not in paths]
|
||||
|
||||
if errors and not downloaded:
|
||||
print(f"\nNothing was installed. {_plural(errors, 'file')} failed.")
|
||||
if failures and all(isinstance(r, NetworkReason) for _d, r in failures):
|
||||
if failures and all(isinstance(r, (NetworkReason, OfflineReason))
|
||||
for _d, r in failures):
|
||||
# The folder was never the problem: naming it as the place being
|
||||
# written to sent people looking at their disk.
|
||||
print(f"Destination: {_locations(paths)}")
|
||||
@@ -5104,6 +5170,12 @@ def installer_id() -> str:
|
||||
return "unknown"
|
||||
|
||||
|
||||
# A --dry-run plan: file names per system, the systems shown when the
|
||||
# listing is taller than the screen, and the lines around the listing.
|
||||
PLAN_FILES_PER_SYSTEM = 8
|
||||
PLAN_LARGEST_SYSTEMS = 10
|
||||
PLAN_FRAME_LINES = 6
|
||||
|
||||
# Answers that stop: every menu before this one taught q as quit.
|
||||
_REFUSALS = ("n", "q", "quit", "exit", "stop", "abort")
|
||||
|
||||
@@ -5119,42 +5191,61 @@ def _print_plan(
|
||||
trees: list[tuple[dict, dict[str, dict]]] = (),
|
||||
kept: dict[Path, list[int]] | None = None,
|
||||
copied: list[str] = (),
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
"""What a real run would fetch, by system, and nothing written.
|
||||
"""What a real run would fetch: the totals first, then by system, and
|
||||
nothing written.
|
||||
|
||||
The data trees count in the total and come before the closing line:
|
||||
printed after it, they left 2636 RetroArch files out of the figure a
|
||||
user plans an SD card from.
|
||||
user plans an SD card from. A listing taller than the screen pushed the
|
||||
total out of sight (599 lines for RetroArch): it gives way to the
|
||||
largest systems, and --verbose lists every file.
|
||||
"""
|
||||
total = sum(f.get("size", 0) for f in to_download)
|
||||
trees = [(entry, needed) for entry, needed in trees if needed]
|
||||
tree_files = sum(len(needed) for _entry, needed in trees)
|
||||
tree_size = sum(meta.get("size", 0) for _entry, needed in trees for meta in needed.values())
|
||||
for line in copied:
|
||||
print(line)
|
||||
if not to_download and not trees:
|
||||
if not copied:
|
||||
print("\n All files up to date. A real run would download nothing.")
|
||||
return
|
||||
print("\n Nothing to download.")
|
||||
parts = []
|
||||
if to_download:
|
||||
parts.append(f"{_plural(len(to_download), 'file')} ({format_size(total)})")
|
||||
if trees:
|
||||
parts.append(f"{_plural(tree_files, 'data file')} ({format_size(tree_size)}, "
|
||||
f"extracted from {_plural(len(trees), 'archive')})")
|
||||
if not parts and not copied:
|
||||
print("\n All files up to date. A real run would download nothing.")
|
||||
return
|
||||
if parts:
|
||||
print(f"\nWould download {' and '.join(parts)} to {bios_path}:")
|
||||
for line in copied:
|
||||
print(line)
|
||||
if not parts:
|
||||
print("\n Nothing to download.")
|
||||
by_system: dict[str, list[dict]] = {}
|
||||
for entry in to_download:
|
||||
system = ", ".join(entry_systems(entry)) or "other"
|
||||
by_system.setdefault(system, []).append(entry)
|
||||
shown = None if verbose else PLAN_FILES_PER_SYSTEM
|
||||
listing: list[str] = []
|
||||
for system in sorted(by_system):
|
||||
print(f" {term.bold(system)} {_sized(by_system[system])}")
|
||||
for entry in sorted(by_system[system], key=lambda e: e["dest"])[:8]:
|
||||
print(f" {entry['dest']} {term.dim(format_size(entry.get('size', 0)))}")
|
||||
if len(by_system[system]) > 8:
|
||||
print(f" ... and {len(by_system[system]) - 8} more")
|
||||
entries = sorted(by_system[system], key=lambda e: e["dest"])
|
||||
listing.append(f" {term.bold(system)} {_sized(entries)}")
|
||||
listing += [f" {entry['dest']} {term.dim(format_size(entry.get('size', 0)))}"
|
||||
for entry in entries[:shown]]
|
||||
if shown is not None and len(entries) > shown:
|
||||
listing.append(f" ... and {len(entries) - shown} more")
|
||||
screen = shutil.get_terminal_size((80, 24)).lines
|
||||
if verbose or len(listing) + len(trees) + PLAN_FRAME_LINES <= screen:
|
||||
for line in listing:
|
||||
print(line)
|
||||
else:
|
||||
largest = sorted(by_system, key=lambda s: (-_bytes(by_system[s]), s))
|
||||
print(f" {_plural(len(largest), 'system')}, the largest first:")
|
||||
for system in largest[:PLAN_LARGEST_SYSTEMS]:
|
||||
print(f" {term.bold(system)} {_sized(by_system[system])}")
|
||||
if len(largest) > PLAN_LARGEST_SYSTEMS:
|
||||
print(f" ... and {len(largest) - PLAN_LARGEST_SYSTEMS} more systems: "
|
||||
"--verbose lists every system and its files.")
|
||||
for entry, needed in trees:
|
||||
source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"]
|
||||
print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, "
|
||||
@@ -5573,7 +5664,8 @@ def _install_standalone_folders(
|
||||
pending += len(to_download)
|
||||
continue
|
||||
if args.dry_run:
|
||||
_print_plan(to_download, folder, term, kept=plan_kept(mismatched, folder))
|
||||
_print_plan(to_download, folder, term, kept=plan_kept(mismatched, folder),
|
||||
verbose=args.verbose)
|
||||
continue
|
||||
if not to_download:
|
||||
print(" All files up to date.")
|
||||
@@ -6596,7 +6688,8 @@ class LocalSources:
|
||||
|
||||
@property
|
||||
def offline(self) -> bool:
|
||||
return self.pack is not None
|
||||
"""No network request in this run: a pack is read, or --offline."""
|
||||
return self.pack is not None or offline_requested()
|
||||
|
||||
def reset(self) -> None:
|
||||
if self.pack is not None:
|
||||
@@ -6614,9 +6707,22 @@ class _Offline(OSError):
|
||||
"""A network request in a run told to make none."""
|
||||
|
||||
|
||||
# --offline, or this variable set to anything but 0, which every run the
|
||||
# full-screen interface starts inherits. RETROBIOS_OFFLINE stays the build
|
||||
# scripts' own: CI and the test suite set it for every process they start.
|
||||
OFFLINE_VARIABLE = "RETROBIOS_INSTALL_OFFLINE"
|
||||
OFFLINE_WORDS = "this run makes no network request (--offline)"
|
||||
|
||||
|
||||
def offline_requested() -> bool:
|
||||
return os.environ.get(OFFLINE_VARIABLE, "") not in ("", "0")
|
||||
|
||||
|
||||
def _network_allowed(what: str) -> None:
|
||||
if LOCAL.offline:
|
||||
if LOCAL.pack is not None:
|
||||
raise _Offline(f"{what}: this run reads a pack and makes no network request")
|
||||
if offline_requested():
|
||||
raise _Offline(f"{what}: {OFFLINE_WORDS}")
|
||||
|
||||
|
||||
def read_local_manifest(path: Path, plat: str) -> dict:
|
||||
@@ -7980,6 +8086,13 @@ def main() -> None:
|
||||
"(the others are found beside it) or the folder holding them. "
|
||||
"Makes no network request, and checks every file's hash",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--offline",
|
||||
action="store_true",
|
||||
help="make no network request: --check, --from, --from-pack and --manifest "
|
||||
"work without one, and a file the run would download fails at once "
|
||||
f"({OFFLINE_VARIABLE}=1 does the same)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--from",
|
||||
action="append",
|
||||
@@ -8088,6 +8201,13 @@ def main() -> None:
|
||||
sys.exit(restore_kept(args.restore, args.dry_run))
|
||||
if args.every_board and args.target:
|
||||
parser.error("--every-board installs every board; --target names one")
|
||||
if args.offline:
|
||||
# The runs the full-screen interface starts read it from here.
|
||||
os.environ[OFFLINE_VARIABLE] = "1"
|
||||
if offline_requested() and (args.target or args.list_targets):
|
||||
parser.error("--target and --list-targets read a core list from the network, "
|
||||
"which --offline does not use")
|
||||
install_bounded_opener()
|
||||
if args.from_pack and args.target:
|
||||
parser.error("--target reads a core list from the network, which --from-pack "
|
||||
"does not use")
|
||||
@@ -9050,7 +9170,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
], bios_path)
|
||||
_print_plan([f for f in to_download if id(f) not in copies], bios_path, term,
|
||||
trees, kept, copy_plan_lines(to_download, copies, bios_index,
|
||||
bios_path, files, args.verbose, True))
|
||||
bios_path, files, args.verbose, True),
|
||||
verbose=args.verbose)
|
||||
for line in unreadable_lines(unread, args.verbose):
|
||||
print(line)
|
||||
for line in hint:
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ esac
|
||||
TEMP_INSTALLER=""
|
||||
TEMP_DIRECTORY=""
|
||||
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
DEFAULT_INSTALL_SHA256="240faa427aaa602deca1a3180f770c6f335095b1cf1d5bb28fc060eb55f54917"
|
||||
DEFAULT_INSTALL_SHA256="ea2777500114b796bb4a23ea9e0dd0a15bc4499bb1289e6e426943beece48865"
|
||||
MAX_INSTALLER_BYTES=2097152
|
||||
|
||||
cleanup() {
|
||||
|
||||
@@ -21,6 +21,7 @@ import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import unittest.mock
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
@@ -589,5 +590,171 @@ class TheSummaryNamesTheBulk(unittest.TestCase):
|
||||
self.assertLess(text.index("Done."), text.index("arcade pulls"))
|
||||
|
||||
|
||||
class OfflineMakesNoRequest(Scratch):
|
||||
"""A handheld with no Wi-Fi ran --check against a loopback address to
|
||||
learn its platform, since nothing said "make no request"."""
|
||||
|
||||
def _served(self, *options: str, env: dict | None = None):
|
||||
import test_install_selection # noqa: PLC0415
|
||||
|
||||
hits: list[str] = []
|
||||
with FakeRepository() as repo, unittest.mock.patch.object(
|
||||
test_install_selection._QuietHandler, "log_message",
|
||||
lambda handler, *args: hits.append(handler.path)):
|
||||
started = time.monotonic()
|
||||
proc = subprocess.run(
|
||||
repo.command(*options), env={**repo.env(), **(env or {})},
|
||||
capture_output=True, text=True, timeout=60, check=False,
|
||||
stdin=subprocess.DEVNULL)
|
||||
elapsed = time.monotonic() - started
|
||||
installed = repo.installed()
|
||||
return proc, hits, elapsed, installed
|
||||
|
||||
def test_check_names_the_platform_and_stops_at_the_file_list(self):
|
||||
for options, env in (((("--offline",)), None),
|
||||
((), {"RETROBIOS_INSTALL_OFFLINE": "1"})):
|
||||
with self.subTest(options=options, env=env):
|
||||
proc, hits, elapsed, installed = self._served(
|
||||
*options, "--check", "--events", env=env)
|
||||
self.assertEqual(hits, [])
|
||||
self.assertLess(elapsed, 30)
|
||||
self.assertEqual(proc.returncode, 1, proc.stderr)
|
||||
events = [json.loads(line) for line in proc.stdout.splitlines()]
|
||||
self.assertEqual([e["event"] for e in events],
|
||||
["start", "detect", "error", "end"])
|
||||
self.assertEqual(events[1]["platforms"][0]["platform"], "retroarch")
|
||||
self.assertEqual((events[2]["stage"], events[2]["reason"]),
|
||||
("manifest", "offline"))
|
||||
self.assertIn("no network request", events[2]["message"])
|
||||
self.assertIn("--manifest", proc.stderr)
|
||||
self.assertNotIn("attempts", proc.stderr)
|
||||
self.assertEqual(installed, [])
|
||||
|
||||
def test_a_copy_lands_and_the_rest_fails_at_once(self):
|
||||
old = self.root / "old"
|
||||
old.mkdir()
|
||||
(old / "scph5501.bin").write_bytes(US)
|
||||
proc = self.run_installer("--offline", "--manifest", str(self.list),
|
||||
"--from", str(old), "--no-input")
|
||||
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
|
||||
self.assertEqual((self.dest / "scph5501.bin").read_bytes(), US)
|
||||
self.assertFalse((self.dest / "gba_bios.bin").exists())
|
||||
self.assertIn("no network request", proc.stdout + proc.stderr)
|
||||
self.assertIn("run without --offline", proc.stdout + proc.stderr)
|
||||
self.assertNotIn("trying again", proc.stderr)
|
||||
|
||||
def test_modes_that_need_the_network_refuse_it(self):
|
||||
for extra in (["--target", "switch"], ["--list-targets"]):
|
||||
with self.subTest(extra=extra):
|
||||
proc = self.run_installer("--offline", *extra)
|
||||
self.assertEqual(proc.returncode, 2, proc.stderr)
|
||||
self.assertIn("--offline", proc.stderr)
|
||||
|
||||
def test_a_listing_and_a_pack_need_no_network_anyway(self):
|
||||
proc = self.run_installer("--offline", "--list-platforms")
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
|
||||
|
||||
class ANetworkThatDroppedFailsFast(unittest.TestCase):
|
||||
"""A dropped network cost 93 s before the line-mode error: three tries
|
||||
of a 30-second timeout spent waiting for a connection that never opened."""
|
||||
|
||||
def test_the_connection_is_bounded_and_the_read_is_not(self):
|
||||
calls: list[tuple] = []
|
||||
|
||||
class Sock:
|
||||
def settimeout(self, value):
|
||||
calls.append(("read", value))
|
||||
|
||||
def create(address, timeout, source_address=None):
|
||||
calls.append(("connect", timeout))
|
||||
return Sock()
|
||||
|
||||
with unittest.mock.patch.object(install.socket, "create_connection", create):
|
||||
install._open_socket(("example.org", 443), 60)
|
||||
install._open_socket(("example.org", 443), 2)
|
||||
self.assertEqual(calls, [("connect", install.CONNECT_TIMEOUT), ("read", 60),
|
||||
("connect", 2), ("read", 2)])
|
||||
|
||||
def test_three_tries_that_never_connect_end_within_fifteen_seconds(self):
|
||||
waited = sum(install._retry_delay(n, None) for n in range(1, install.MAX_RETRIES))
|
||||
self.assertLessEqual(install.MAX_RETRIES * install.CONNECT_TIMEOUT + waited, 15)
|
||||
|
||||
def test_a_slow_answer_on_an_open_connection_still_arrives(self):
|
||||
import http.server # noqa: PLC0415
|
||||
import threading # noqa: PLC0415
|
||||
|
||||
delay = install.CONNECT_TIMEOUT + 1
|
||||
|
||||
class Slow(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self): # noqa: N802 - stdlib name
|
||||
time.sleep(delay)
|
||||
body = b'{"ok": true}'
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
return
|
||||
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Slow)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
self.addCleanup(server.server_close)
|
||||
self.addCleanup(server.shutdown)
|
||||
install.install_bounded_opener()
|
||||
self.addCleanup(install.urllib.request.install_opener, None)
|
||||
url = f"http://127.0.0.1:{server.server_address[1]}/list.json"
|
||||
started = time.monotonic()
|
||||
self.assertEqual(install._fetch_json(url, 1024, "file list", attempts=1), {"ok": True})
|
||||
self.assertGreaterEqual(time.monotonic() - started, delay - 0.5)
|
||||
|
||||
|
||||
class ADryRunOpensOnItsTotals(Scratch):
|
||||
"""A full --dry-run printed 599 lines, the total scrolled away above
|
||||
eight file names for each of 66 systems."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
files = [
|
||||
{"dest": f"sys{n:02d}/file{k}.bin", "sha1": _sha1(b"%d-%d" % (n, k)),
|
||||
"size": 1000 * (n + 1), "repo_path": f"bios/sys{n:02d}_{k}", "cores": None,
|
||||
"systems": [f"system-{n:02d}"]}
|
||||
for n in range(40) for k in range(3)
|
||||
]
|
||||
self.list.write_text(json.dumps({"manifest_version": 2, "platform": "retroarch",
|
||||
"files": files}))
|
||||
|
||||
def _plan(self, *options: str) -> list[str]:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch",
|
||||
"--dest", str(self.dest), "--manifest", str(self.list), "--dry-run",
|
||||
"--no-input", *options],
|
||||
env={**_offline_env(), "LINES": "24", "COLUMNS": "80"},
|
||||
capture_output=True, text=True, timeout=120, check=False,
|
||||
stdin=subprocess.DEVNULL)
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
||||
lines = proc.stdout.splitlines()
|
||||
return lines[next(i for i, line in enumerate(lines)
|
||||
if line.startswith("Would download")):]
|
||||
|
||||
def test_the_total_comes_first_and_the_largest_systems_after(self):
|
||||
plan = self._plan()
|
||||
self.assertTrue(plan[0].startswith("Would download 120 files"), plan[0])
|
||||
self.assertLessEqual(len(plan), 24, "\n".join(plan))
|
||||
self.assertIn("system-39", plan[2])
|
||||
self.assertTrue(any("30 more systems" in line and "--verbose" in line
|
||||
for line in plan), "\n".join(plan))
|
||||
self.assertFalse(any("file0.bin" in line for line in plan))
|
||||
self.assertEqual(plan[-1].strip(), "Nothing was written. Run without --dry-run to install.")
|
||||
|
||||
def test_verbose_lists_every_system_and_its_files(self):
|
||||
plan = self._plan("--verbose")
|
||||
self.assertTrue(plan[0].startswith("Would download 120 files"), plan[0])
|
||||
for n in range(40):
|
||||
self.assertTrue(any(line.strip().startswith(f"system-{n:02d}") for line in plan), n)
|
||||
self.assertTrue(any("sys00/file0.bin" in line for line in plan))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+11
-3
@@ -648,12 +648,13 @@ what the options name, everything by default.
|
||||
| `--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) |
|
||||
| `--no-input` | Never ask a question, even at a terminal |
|
||||
| `--check` | Report and exit without writing |
|
||||
| `--dry-run` | Show what would be downloaded, by system and with sizes, without writing anything |
|
||||
| `--dry-run` | Show what would be downloaded, the totals first, then by system and with sizes, without writing anything. A plan taller than the screen names the ten largest systems, and `--verbose` lists every system and file |
|
||||
| `--yes`, `-y` | Do not ask before a download of 1 GB or more |
|
||||
| `--quiet`, `-q` | Print only failures and the final summary. Implies `--no-input` |
|
||||
| `--events` | Write the run as JSON events on standard output, one object per line, and the usual text on standard error. Implies `--no-input`. See [Machine-readable events](#machine-readable-events) |
|
||||
| `--from FOLDER` | Copy the BIOS files a folder holds (an old SD card, a backup, a ZIP) instead of downloading them, each one recognised by its hash and copied, never moved. Repeat for several folders. See [Copy from an old card](#copy-from-an-old-card) |
|
||||
| `--from-pack PATH` | Install from a pack already downloaded: its ZIP, one of its parts (the others are found in the same folder), or the folder holding them. Makes no network request and checks every file's hash. See [Without a network](#without-a-network-offline-install) |
|
||||
| `--offline` | Make no network request. `--check`, `--from`, `--from-pack` and `--manifest` work without one, the file list stops the run at once when none of them gives it, and a file the run would download fails at once. `RETROBIOS_INSTALL_OFFLINE=1` does the same. Refused with `--target` and `--list-targets`, which read a core list from the network |
|
||||
| `--manifest FILE` | Read the file list from FILE, a copy of `install/<platform>.json`, instead of downloading it. See [Without a network](#without-a-network-offline-install) |
|
||||
| `--restore FOLDER` | Put back the files a run replaced and kept in FOLDER, a dated folder inside `retrobios-backup`. Takes no other option but `--dry-run` and `--no-input`. See [Files you already had](#files-you-already-had) |
|
||||
| `--tui` | Open the full-screen interface, or stop with the reason it cannot open here (code 2) instead of falling back to the questions. Wins over `RETROBIOS_PLAIN` and `ACCESSIBLE`. See [The full-screen interface](#the-full-screen-interface) |
|
||||
@@ -1095,6 +1096,10 @@ pack: `--system`, `--core`, `--region` and `--no-game-data` then apply as
|
||||
they do online, and a file of the list the pack does not hold fails as
|
||||
`not in the pack`. The list's platform has to be the one installed for.
|
||||
|
||||
`--offline` forbids every request, for a device with no Wi-Fi: `--check`
|
||||
still names the platform and its folder, then stops at the file list unless
|
||||
`--manifest` or `--from-pack` gives it.
|
||||
|
||||
## Where the files come from
|
||||
|
||||
Each manifest entry names either `repo_path`, served from the repository at the
|
||||
@@ -1247,7 +1252,7 @@ other schemas under `api/v1/schemas/`.
|
||||
|------|------|------|
|
||||
| `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` |
|
||||
| `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found |
|
||||
| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). `message` says it in words |
|
||||
| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). `message` says it in words |
|
||||
| `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered |
|
||||
| `checking` | `files_done`, `files_total` | While the files already in the folder are read and hashed, at most every half second and once at the end. `files_total` counts the files found in place |
|
||||
| `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` |
|
||||
@@ -1274,6 +1279,7 @@ no event at all. A program reads a missing `end` as that case.
|
||||
|----------|--------|
|
||||
| `RETROBIOS_REF` | Branch or tag the file list and the files committed to the repository are read from. Defaults to `main`. Files the manifest marks `release_asset` always come from the `large-files` release, whatever the ref, and that release replaces an asset in place when it is rebuilt. Each release deletes the previous release tags, so a tag holds a file list until the next release. For contributors: [release process](release-process.md#cutting-a-release) |
|
||||
| `RETROBIOS_BASE_URL` | Base URL for the manifest and the files it declares. HTTPS only, plain HTTP allowed to loopback for end-to-end tests. Whoever controls this base controls the expected hashes, hence the check. A file list that cannot be fetched names the address, and this variable when it is set |
|
||||
| `RETROBIOS_INSTALL_OFFLINE` | Set to anything but `0`: no network request, as with `--offline`. The runs the full-screen interface starts inherit it. `RETROBIOS_OFFLINE` is the build scripts' own and leaves the installer alone |
|
||||
| `RETROBIOS_OS` | Names the host outright (`linux`, `wsl`, `windows`, `darwin`, `android`) instead of detecting it |
|
||||
| `EXTERNAL_STORAGE` | Android only, the shared storage root RetroArch builds its default directories from. Defaults to `/storage/emulated/0` |
|
||||
| `RETROBIOS_INSTALL_URL` | Where the bootstrap fetches `install.py`. HTTPS only |
|
||||
@@ -1295,7 +1301,9 @@ The common cases, with the short answer, are on
|
||||
The installer fetches its manifest, then downloads only the repository or
|
||||
`large-files` release objects it needs. When a download fails, check that
|
||||
`https://github.com` opens. Behind a proxy, set `HTTPS_PROXY` in the
|
||||
environment. A file that fails three attempts is counted and reported at the
|
||||
environment. A connection that does not open within 4 seconds counts as
|
||||
none, so a dropped network gives its error in about 15 seconds. Once a
|
||||
connection is open, a slow transfer keeps the full timeout. A file that fails three attempts is counted and reported at the
|
||||
end, and the next run fetches only what is still missing.
|
||||
|
||||
The reason is said in words: no internet connection or a name that could not
|
||||
|
||||
Reference in new issue
Block a user