diff --git a/install.ps1 b/install.ps1 index a4e777f1..98a97b42 100644 --- a/install.ps1 +++ b/install.ps1 @@ -8,7 +8,7 @@ param( $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -$defaultInstallSha256 = "b5749a3aea8679657a6d48f04ef18879f2fe04cc55d1c1a6510fd6b5d74068e1" +$defaultInstallSha256 = "3ea3a3e21012cc2dbf24c08a0ad314233ade04e62c0d2164b76146f1b361d714" $maximumInstallerBytes = 2MB $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } $temporary = $null diff --git a/install.py b/install.py index 857a68eb..d91be472 100644 --- a/install.py +++ b/install.py @@ -23,6 +23,7 @@ import re import shutil import sys import tempfile +import threading import urllib.error import urllib.parse import urllib.request @@ -104,6 +105,14 @@ DEFAULT_DESTS = { "retropie": Path.home() / "RetroPie" / "BIOS", } +# Set when the run is interrupted, so queued downloads end instead of draining +# the pool: every future is submitted before the first one completes. +_stop = threading.Event() + +class _Interrupted(Exception): + """Raised inside a worker once the run has been stopped.""" + + # The three RetroArch packages published for Android, newest ABI first. ANDROID_RETROARCH_PACKAGES = ( "com.retroarch.aarch64", @@ -1052,6 +1061,8 @@ def _download_one( f: dict, bios_path: Path, verbose: bool = False ) -> tuple[str, bool]: """Download a single file. Returns (dest, success).""" + if _stop.is_set(): + return str(f.get("dest", "?")), False try: dest = _destination_path(bios_path, f["dest"]) except ValueError: @@ -1083,6 +1094,8 @@ def _download_one( tmp_path = Path(out.name) downloaded = 0 while True: + if _stop.is_set(): + raise _Interrupted() chunk = resp.read(1024 * 1024) if not chunk: break @@ -1113,6 +1126,13 @@ def _download_one( os.replace(tmp_path, dest) return f["dest"], True + except _Interrupted: + # Stopped mid-transfer: drop the scratch file rather than leaving + # it beside the destination, and do not retry. + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) + return f["dest"], False + except ( urllib.error.URLError, urllib.error.HTTPError, @@ -1127,12 +1147,94 @@ def _download_one( return f["dest"], False +def _locations(paths: list[Path]) -> str: + """Where the run wrote, naming every destination when it served several.""" + return ", ".join(str(p) for p in paths) if paths else "nowhere" + + +def _plural(count: int, word: str) -> str: + """Count and noun, so a run of one file does not read as machine output.""" + return f"{count} {word}" if count == 1 else f"{count} {word}s" + + +def _report_outcome( + paths: list[Path], + downloaded: int, + up_to_date: int, + errors: int, + omitted: int, + interrupted: bool, +) -> None: + """Close the run on what happened, and leave with a status that says so. + + A run that installed nothing used to end on the word Done and a zero exit, + which reads as success to a person and to a script alike. + """ + if interrupted: + print(f"\nStopped. {_plural(downloaded, 'file')} installed.") + print("Run the same command again to carry on from here. Nothing is lost.") + sys.exit(130) + + if errors and not downloaded: + print(f"\nNothing was installed. {_plural(errors, 'file')} failed.") + print(f"Location that was being written to: {_locations(paths)}") + print("Run the same command again once that is resolved. " + "Files already installed are kept.") + sys.exit(2) + + if errors: + print(f"\nFinished with errors. {_plural(downloaded, 'file')} installed, " + f"{_plural(errors, 'file')} failed.") + print(f"Location: {_locations(paths)}") + print("Run the same command again to retry only those.") + sys.exit(1) + + print(f"\nDone. {_plural(downloaded, 'file')} installed, " + f"{_plural(up_to_date, 'file')} already up to date.") + print(f"Location: {_locations(paths)}") + if omitted: + print(f"{_plural(omitted, 'file')} are not in the collection yet " + "and were skipped.") + print("To check this later, run the same command with --check.") + + +def _check_free_space(bios_path: Path, needed: int) -> None: + """Stop before writing when the volume cannot hold the download. + + Reported against the nearest existing ancestor, since the destination + itself may have just been created on a volume that is already full. + """ + probe = bios_path + while not probe.exists() and probe != probe.parent: + probe = probe.parent + try: + free = shutil.disk_usage(probe).free + except OSError: + return + if free >= needed: + return + print(f"\nNot enough space on the drive holding {bios_path}.", file=sys.stderr) + print(f" Needed: {format_size(needed)}. Free: {format_size(free)}.", + file=sys.stderr) + print(f" Free up {format_size(needed - free)}, or use --dest to install " + "to another drive.", file=sys.stderr) + sys.exit(1) + + def download_files( files: list[dict], bios_path: Path, jobs: int = 8, verbose: bool = False -) -> list[str]: - """Download files in parallel. Returns list of failed file names.""" +) -> tuple[list[str], bool]: + """Download files in parallel. + + Returns the destinations that failed and whether the run was interrupted. + Every future is submitted up front, so a bare KeyboardInterrupt would let + the pool drain the queue in full: the stop flag is what makes Ctrl+C end + the run rather than only stop the reporting. + """ failed: list[str] = [] total = len(files) + interrupted = False + _stop.clear() with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool: future_map = { @@ -1140,15 +1242,20 @@ def download_files( for f in files } done_count = 0 - for future in concurrent.futures.as_completed(future_map): - done_count += 1 - dest, success = future.result() - status = "ok" if success else "FAILED" - print(f" [{done_count}/{total}] {dest} {status}") - if not success: - failed.append(dest) + try: + for future in concurrent.futures.as_completed(future_map): + done_count += 1 + dest, success = future.result() + status = "ok" if success else "FAILED" + print(f" [{done_count}/{total}] {dest} {status}") + if not success: + failed.append(dest) + except KeyboardInterrupt: + interrupted = True + _stop.set() + print("\n Stopping. Files already installed are kept.") - return failed + return failed, interrupted def do_standalone_copies( @@ -1460,8 +1567,12 @@ def main() -> None: total_up_to_date = 0 total_errors = 0 total_omitted = 0 + total_interrupted = False + installed_paths: list[Path] = [] for plat_name, bios_path in platforms: + if bios_path not in installed_paths: + installed_paths.append(bios_path) print(f"\nFetching file index for {plat_name}...") manifest = fetch_manifest(plat_name) files = manifest.get("files", []) @@ -1554,12 +1665,26 @@ def main() -> None: if to_download: dl_size = sum(f.get("size", 0) for f in to_download) print(f"\nDownloading {len(to_download)} files ({format_size(dl_size)})...") - bios_path.mkdir(parents=True, exist_ok=True) - failed = download_files( + print(" Press Ctrl+C to stop. Running the same command again " + "carries on from here.") + try: + bios_path.mkdir(parents=True, exist_ok=True) + except OSError as exc: + print(f"\nCannot create {bios_path}.", file=sys.stderr) + print(f" {exc.strerror}.", file=sys.stderr) + print(" On Batocera, Recalbox and ROCKNIX the BIOS folder " + "belongs to root: run the same command with sudo.", + file=sys.stderr) + sys.exit(1) + _check_free_space(bios_path, dl_size) + failed, interrupted = download_files( to_download, bios_path, jobs=args.jobs, verbose=args.verbose ) total_downloaded += len(to_download) - len(failed) total_errors += len(failed) + if interrupted: + total_interrupted = True + break else: print("\n All files up to date.") @@ -1586,10 +1711,13 @@ def main() -> None: ) if not args.check and not args.list_targets: - print( - f"\nDone. {total_downloaded} downloaded, " - f"{total_up_to_date} up to date, {total_errors} errors, " - f"{total_omitted} safely omitted." + _report_outcome( + installed_paths, + total_downloaded, + total_up_to_date, + total_errors, + total_omitted, + total_interrupted, ) diff --git a/install.sh b/install.sh index 4482f9b8..fbce14a7 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="b5749a3aea8679657a6d48f04ef18879f2fe04cc55d1c1a6510fd6b5d74068e1" +DEFAULT_INSTALL_SHA256="3ea3a3e21012cc2dbf24c08a0ad314233ade04e62c0d2164b76146f1b361d714" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index b14793fe..51cfaf4f 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1,10 +1,12 @@ """Tests for install.py platform detection and config parsing.""" from __future__ import annotations +import contextlib import functools import hashlib import http.server import importlib.util +import io import json import os import re @@ -13,6 +15,7 @@ import subprocess import tempfile import threading import unittest +from types import SimpleNamespace import unittest.mock from pathlib import Path @@ -214,6 +217,121 @@ class TestEmbeddedDetection(unittest.TestCase): self.assertEqual(self._detect_with(set()), []) +class TestRunOutcome(unittest.TestCase): + """A run says what happened and leaves with a status that matches.""" + + def _report(self, **kw): + defaults = dict( + paths=[Path("/userdata/bios")], + downloaded=0, + up_to_date=0, + errors=0, + omitted=0, + interrupted=False, + ) + defaults.update(kw) + buf = io.StringIO() + code = 0 + with contextlib.redirect_stdout(buf): + try: + install._report_outcome(**defaults) + except SystemExit as exc: + code = exc.code + return buf.getvalue(), code + + def test_success_names_the_location_and_the_way_to_check(self): + out, code = self._report(downloaded=12, up_to_date=3) + self.assertEqual(code, 0) + self.assertIn("12 files installed", out) + self.assertIn("/userdata/bios", out) + self.assertIn("--check", out) + + def test_total_failure_does_not_say_done_and_exits_two(self): + out, code = self._report(errors=7) + self.assertEqual(code, 2) + self.assertNotIn("Done.", out) + self.assertIn("Nothing was installed", out) + + def test_partial_failure_exits_one(self): + out, code = self._report(downloaded=5, errors=2) + self.assertEqual(code, 1) + self.assertIn("5 files installed", out) + self.assertIn("2 files failed", out) + + def test_interruption_is_resumable_and_exits_130(self): + out, code = self._report(downloaded=431, interrupted=True) + self.assertEqual(code, 130) + self.assertIn("carry on", out) + self.assertIn("Nothing is lost", out) + + def test_one_file_reads_as_one_file(self): + out, _ = self._report(downloaded=1) + self.assertIn("1 file installed", out) + self.assertNotIn("1 files", out) + + def test_every_destination_is_named(self): + out, _ = self._report( + paths=[Path("/a/bios"), Path("/b/bios")], downloaded=2 + ) + self.assertIn("/a/bios", out) + self.assertIn("/b/bios", out) + + +class TestFreeSpaceGuard(unittest.TestCase): + """The run refuses before writing when the volume cannot hold it.""" + + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.real = install.shutil.disk_usage + + def tearDown(self): + install.shutil.disk_usage = self.real + shutil.rmtree(self.root, ignore_errors=True) + + def _run(self, free: int, needed: int, path: Path | None = None): + install.shutil.disk_usage = lambda _p: SimpleNamespace(free=free) + buf = io.StringIO() + code = 0 + with contextlib.redirect_stderr(buf): + try: + install._check_free_space(path or self.root, needed) + except SystemExit as exc: + code = exc.code + return buf.getvalue(), code + + def test_enough_space_says_nothing(self): + out, code = self._run(free=10_000, needed=1_000) + self.assertEqual((out, code), ("", 0)) + + def test_short_of_space_names_both_numbers_and_stops(self): + out, code = self._run(free=1_000, needed=5_000) + self.assertEqual(code, 1) + self.assertIn("Needed", out) + self.assertIn("Free", out) + self.assertIn("--dest", out) + + def test_probe_climbs_to_an_existing_ancestor(self): + missing = self.root / "not" / "created" / "yet" + out, code = self._run(free=1, needed=2, path=missing) + self.assertEqual(code, 1) + + +class TestStopFlag(unittest.TestCase): + """Ctrl+C ends the run instead of letting the queue drain.""" + + def tearDown(self): + install._stop.clear() + + def test_worker_returns_immediately_once_stopped(self): + install._stop.set() + dest, ok = install._download_one({"dest": "x.bin"}, Path("/nowhere")) + self.assertEqual((dest, ok), ("x.bin", False)) + + def test_download_files_reports_no_interruption_normally(self): + failed, interrupted = install.download_files([], Path("/nowhere")) + self.assertEqual((failed, interrupted), ([], False)) + + class TestAndroidDetection(unittest.TestCase): """Android reports itself as Linux, and RetroArch is the frontend there."""