diff --git a/install.ps1 b/install.ps1 index 83cb19d8..abe19638 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa" + $defaultInstallSha256 = "6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index af1c5cf2..df45a790 100644 --- a/install.py +++ b/install.py @@ -106,6 +106,10 @@ OMISSION_REASONS = { "no_platform_slug": "no platform folder", "below_platform_folder": "below the platform folder", } +# What a file is when it is not firmware, as the manifest's category field +# says, and as the installer words it. Firmware carries no category. +GAME_DATA = "game_data" +CATEGORY_LABELS = {GAME_DATA: "game data"} _SHA1_RE = re.compile(r"^[0-9a-fA-F]{40}$") _SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") @@ -479,9 +483,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { "downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total", "bytes_done", "bytes_total"), # failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files", - # "required"}], the files the collection cannot supply. + # "required"}], the files the collection cannot supply. left_out: + # [{"category", "label", "files", "dests"}], what --no-game-data left + # out, empty without it. "summary": ("mode", "installed", "up_to_date", "pending", "failed", "skipped", - "interrupted"), + "left_out", "interrupted"), "end": ("code",), } # Seconds between two progress events while bytes arrive. @@ -578,6 +584,49 @@ def _emit_check( ) +def leave_out_game_data( + files: list[dict], omitted: list[dict] +) -> tuple[list[dict], list[dict], list[dict]]: + """The files and omissions without game data, and what was left out. + + The same cut as a pack built with --no-game-data: the manifest marks + each file that is a game's data, not firmware. + """ + left_out = [e for e in files + omitted if e.get("category") == GAME_DATA] + return ( + [e for e in files if e.get("category") != GAME_DATA], + [e for e in omitted if e.get("category") != GAME_DATA], + left_out, + ) + + +def _left_out_lines(left_out: list[dict], every: bool) -> list[str]: + """What --no-game-data left out, named per category, ten each unless every.""" + lines = [] + for group in _left_out_events(left_out): + dests = group["dests"] + shown = dests if every else dests[:10] + more = (f" ... and {len(dests) - len(shown)} more (--verbose lists them all)" + if len(shown) < len(dests) else "") + lines.append(f" Left out as {group['label']} (--no-game-data): " + f"{_plural(len(dests), 'file')}: {', '.join(shown)}{more}") + return lines + + +def _left_out_events(left_out: list[dict]) -> list[dict]: + """What --no-game-data left out, per category, every file named.""" + groups: dict[str, dict] = {} + for entry in left_out: + category = entry.get("category", "") + group = groups.setdefault(category, { + "category": category, "label": CATEGORY_LABELS.get(category, category), + "files": 0, "dests": [], + }) + group["files"] += 1 + group["dests"].append(entry["dest"]) + return [groups[category] for category in sorted(groups)] + + def _skipped(omitted: list[dict]) -> list[dict]: """The files the collection cannot supply, grouped by reason.""" groups: dict[str, dict] = {} @@ -3812,6 +3861,7 @@ class Invocation: for flag, on in ( ("--every-board", args.every_board), ("--installed-cores", args.installed_cores and not args.cores_dir), + ("--no-game-data", args.no_game_data), ("--standalone-copies", args.standalone_copies), ("--check", args.check), ("--dry-run", args.dry_run), @@ -5332,6 +5382,12 @@ def main() -> None: help="preferred regions, best first, e.g. us,eu,jp: keep one BIOS per " "region where a system has several", ) + parser.add_argument( + "--no-game-data", + action="store_true", + help="leave out game data (files a game reads, not firmware) and " + "install the firmware only", + ) parser.add_argument( "--list-systems", action="store_true", @@ -5410,6 +5466,9 @@ def main() -> None: parser.error("--quiet applies to an install, not to --check, --dry-run or a listing") if args.every_board and args.target: parser.error("--every-board installs every board; --target names one") + if args.no_game_data and (args.list_platforms or args.list_targets): + parser.error("--no-game-data narrows a file list, not --list-platforms " + "or --list-targets") if args.events: # The events own stdout. The text a person reads goes to stderr, so a # program never has to tell one from the other. @@ -5725,6 +5784,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: total_errors = 0 total_omitted = 0 omitted_seen: list[dict] = [] + left_out_seen: list[dict] = [] total_interrupted = False total_bytes = 0 total_seconds = 0.0 @@ -5806,6 +5866,23 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: f"case and holds another file under the same name ({shown})" ) + game_data_cut = 0 + if args.no_game_data: + if not manifest.get("categorized"): + # Without the field nothing is marked, and the run would + # install the game data it was asked to leave out. + print(f"Error: the {plat_name} file list records no file category, " + "which --no-game-data needs.", file=sys.stderr) + print(_older_list_advice(), file=sys.stderr) + sys.exit(1) + before = len(files) + files, omitted_files, left_out = leave_out_game_data(files, omitted_files) + game_data_cut = before - len(files) + left_out_seen.extend(left_out) + if left_out: + for line in _left_out_lines(left_out, args.verbose): + print(line) + if args.list_targets: targets = fetch_targets(plat_name) if targets is None: @@ -5912,12 +5989,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: if selection == INSTALLED_ONLY: files, omitted_files = installed print("\n Same choice without questions: " - f"--platform {plat_name} --installed-cores") + f"--platform {plat_name} --installed-cores" + + (" --no-game-data" if args.no_game_data else "")) _INVOCATION.chosen[plat_name] = [("--installed-cores", None)] selection = None elif selection is not None: print("\n Same choice without questions: " - + _same_choice_hint(plat_name, *selection)) + + _same_choice_hint(plat_name, *selection) + + (" --no-game-data" if args.no_game_data else "")) systems, cores, regions = selection _INVOCATION.chosen[plat_name] = [ *(("--system", name) for name in systems), @@ -5994,7 +6073,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: narrowed = bool( target or args.system or args.core or args.region or args.installed_cores - or len(files) != len(manifest.get("files", [])) + len( + or len(files) + game_data_cut != len(manifest.get("files", [])) + len( variants if case_kept else [] ) ) @@ -6173,7 +6252,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: up_to_date=total_verified if args.check or args.dry_run else total_up_to_date, pending=check_pending if args.check else files_pending, failed=[{"dest": dest, "reason": str(reason)} for dest, reason in failures], - skipped=_skipped(omitted_seen), interrupted=total_interrupted, + skipped=_skipped(omitted_seen), left_out=_left_out_events(left_out_seen), + interrupted=total_interrupted, ) # A check scripted as `--check && launch` reads the exit code only. if args.check and check_pending: diff --git a/install.sh b/install.sh index ba3ef375..6cfd570e 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa" +DEFAULT_INSTALL_SHA256="6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 67814009..c5016eb0 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3797,7 +3797,7 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase): args = SimpleNamespace( system=None, core=None, emulator=None, region=None, target=None, cores_dir=None, every_board=False, installed_cores=False, - standalone_copies=False, check=False, dry_run=False, + standalone_copies=False, check=False, dry_run=False, no_game_data=False, ) flags = list(argv) while flags: @@ -4966,7 +4966,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): installer = events[0]["installer"] self.assertEqual(events, self._head("check", installer) + [ {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 1, - "pending": 3, "failed": [], "skipped": self.SKIPPED, "interrupted": False}, + "pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False}, {"event": "end", "code": 1}, ]) self.assertIn("2/4 present (1 verified, 1 wrong hash)", proc.stderr) @@ -4989,7 +4989,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): {"event": "download", "platform": "retroarch", "files": 3, "bytes": 245}, *done, {"event": "summary", "mode": "install", "installed": 3, "up_to_date": 1, - "pending": 0, "failed": [], "skipped": self.SKIPPED, "interrupted": False}, + "pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False}, {"event": "end", "code": 0}, ]) self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"]) diff --git a/tests/test_no_game_data.py b/tests/test_no_game_data.py new file mode 100644 index 00000000..5e348c0b --- /dev/null +++ b/tests/test_no_game_data.py @@ -0,0 +1,355 @@ +"""--no-game-data: the firmware without the files a game reads. + +A platform list can name a game's data beside the firmware (EmuELEC lists the +Doom WADs), and a core profile marks its game data `category: game_data`. The +manifest marks both, the builder leaves both out under --no-game-data, verify +reports on the same file set, and the installer makes the same cut from the +full manifest. +""" + +from __future__ import annotations + +import contextlib +import functools +import hashlib +import http.server +import io +import json +import os +import subprocess +import sys +import tempfile +import threading +import unittest +import zipfile +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) +sys.path.insert(0, str(REPO_ROOT)) + +import common # noqa: E402 +import generate_db # noqa: E402 +import generate_pack as builder # noqa: E402 +import install # noqa: E402 +import packverify # noqa: E402 +import verify # noqa: E402 + +PAYLOADS = { + "System/firmware.rom": b"firmware of the console", + "System/corefw.bin": b"firmware the core loads", + "Doom/doom1.wad": b"shareware episode", + "Doom/engine.pk3": b"engine resources", +} + +PROFILES = { + "emu": { + "emulator": "Emu", "type": "libretro", "cores": ["emu"], + "systems": ["console"], + "files": [ + {"name": "firmware.rom", "required": True}, + {"name": "corefw.bin", "required": True}, + ], + }, + "doomcore": { + "emulator": "DoomCore", "type": "libretro", "cores": ["doomcore"], + "systems": ["doom"], + "files": [ + {"name": "doom1.wad", "path": "doom/doom1.wad", "category": "game_data", + "required": False}, + {"name": "engine.pk3", "category": "game_data", "required": True}, + ], + }, +} + +FIRMWARE = {"firmware.rom", "corefw.bin"} +GAME = {"doom/doom1.wad", "engine.pk3"} + + +class _Fixture(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.bios = self.root / "bios" + self.platforms = self.root / "platforms" + self.emulators = self.root / "emulators" + files = {} + for rel, data in PAYLOADS.items(): + path = self.bios / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + files[sha1] = { + "path": str(path), "name": path.name, "size": len(data), "sha1": sha1, + "md5": hashlib.md5(data).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), "crc32": "00000000", + } + self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})} + self.platforms.mkdir() + self.emulators.mkdir() + for key, profile in PROFILES.items(): + (self.emulators / f"{key}.yml").write_text(yaml.dump(profile)) + platform = { + "platform": "Handheld", "verification_mode": "existence", + "cores": ["emu", "doomcore"], + "systems": { + "console": {"files": [{"name": "firmware.rom", "destination": "firmware.rom"}]}, + "doom": {"files": [{"name": "doom1.wad", "destination": "doom/doom1.wad"}]}, + }, + } + (self.platforms / "handheld.yml").write_text(yaml.dump(platform)) + (self.platforms / "_registry.yml").write_text( + yaml.dump({"platforms": {"handheld": {"status": "active"}}}) + ) + common._platform_config_cache.clear() + common._emulator_profiles_cache.clear() + self.out = self.root / "dist" + self.out.mkdir() + + def tearDown(self): + common._platform_config_cache.clear() + common._emulator_profiles_cache.clear() + self._tmp.cleanup() + + def pack(self, no_game_data: bool) -> tuple[str, set[str], str]: + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + zip_path = builder.generate_pack( + "handheld", str(self.platforms), self.db, str(self.bios), str(self.out), + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + offline=True, no_game_data=no_game_data, + ) + with zipfile.ZipFile(zip_path) as archive: + members = set(archive.namelist()) - set(builder.PACK_DOCUMENTS) + return zip_path, members, printed.getvalue() + + def manifest(self, no_game_data: bool = False) -> dict: + return builder.generate_manifest( + "handheld", str(self.platforms), self.db, str(self.bios), + str(self.platforms / "_registry.yml"), emulators_dir=str(self.emulators), + emu_profiles=PROFILES, offline=True, no_game_data=no_game_data, + ) + + +class ManifestCategory(_Fixture): + def test_game_data_is_marked_and_firmware_is_not(self): + manifest = self.manifest() + self.assertIs(manifest["categorized"], True) + category = {entry["dest"]: entry.get("category") for entry in manifest["files"]} + self.assertEqual(set(category), FIRMWARE | GAME) + # A platform entry only game data reads, and a core's game data file. + self.assertEqual(category["doom/doom1.wad"], "game_data") + self.assertEqual(category["engine.pk3"], "game_data") + # Firmware carries no field at all. + for entry in manifest["files"]: + if entry["dest"] in FIRMWARE: + self.assertNotIn("category", entry) + + def test_a_narrowed_manifest_is_the_full_one_without_game_data(self): + full = self.manifest() + narrowed = self.manifest(no_game_data=True) + self.assertIs(narrowed["no_game_data"], True) + self.assertEqual( + {(e["dest"], e["sha1"]) for e in narrowed["files"]}, + {(e["dest"], e["sha1"]) for e in full["files"] if e.get("category") != "game_data"}, + ) + + def test_an_omitted_game_data_file_is_marked(self): + (self.bios / "Doom" / "engine.pk3").unlink() + files = {sha1: e for sha1, e in self.db["files"].items() if e["name"] != "engine.pk3"} + self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})} + manifest = self.manifest() + omitted = {e["dest"]: e.get("category") for e in manifest["omitted_files"]} + self.assertEqual(omitted.get("engine.pk3"), "game_data") + + +class PackNarrowing(_Fixture): + def test_the_pack_leaves_game_data_out_and_names_it(self): + _full_path, full, _ = self.pack(no_game_data=False) + zip_path, narrowed, printed = self.pack(no_game_data=True) + self.assertEqual(full, FIRMWARE | GAME) + self.assertEqual(narrowed, FIRMWARE) + self.assertIn("_NoGameData_BIOS_Pack.zip", os.path.basename(zip_path)) + with zipfile.ZipFile(zip_path) as archive: + readme = archive.read("README.txt").decode() + self.assertIn("PACK TYPE: Narrowed", readme) + self.assertIn("firmware only, game data left out", readme) + # Named, with the reason, never only counted. + self.assertIn("doom/doom1.wad (game data of doomcore)", printed) + self.assertIn("doomcore: doom1.wad, engine.pk3", printed) + + def test_verify_reports_the_pack_the_builder_wrote(self): + zip_path, members, _ = self.pack(no_game_data=True) + config = common.load_platform_config("handheld", str(self.platforms)) + result = verify.verify_platform( + config, self.db, str(self.emulators), PROFILES, no_game_data=True, + ) + self.assertEqual(set(result["pack_destinations"]), members) + self.assertTrue(any("doom/doom1.wad" in line for line in result["left_out"])) + checked = packverify.verify_pack_against_platform( + zip_path, "handheld", str(self.platforms), db=self.db, + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + no_game_data=True, + ) + self.assertTrue(checked[0], checked[3]) + # Held to the full list, the narrowed pack is short of the game data. + full = packverify.verify_pack_against_platform( + zip_path, "handheld", str(self.platforms), db=self.db, + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + ) + self.assertFalse(full[0]) + + def test_the_full_report_is_unchanged(self): + config = common.load_platform_config("handheld", str(self.platforms)) + result = verify.verify_platform(config, self.db, str(self.emulators), PROFILES) + self.assertNotIn("left_out", result) + self.assertEqual(set(result["pack_destinations"]), FIRMWARE | GAME) + + def test_split_parts_add_up_to_the_narrowed_pack(self): + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + parts = builder.generate_split_packs( + "handheld", str(self.platforms), self.db, str(self.bios), str(self.out), + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + offline=True, no_game_data=True, + ) + held: set[str] = set() + for part in parts: + self.assertIn("_NoGameData", part) + with zipfile.ZipFile(part) as archive: + held |= set(archive.namelist()) - set(builder.PACK_DOCUMENTS) + self.assertEqual(held, FIRMWARE) + + +class _Quiet(http.server.SimpleHTTPRequestHandler): + def log_message(self, *args) -> None: + return + + +class InstallerLeavesGameDataOut(unittest.TestCase): + """The installer cut, in each mode, from a manifest served over loopback.""" + + FILES = { + "bios/a.bin": (b"alpha" * 20, None), + "bios/game.wad": (b"wad" * 10, "game_data"), + } + + def _tempdir(self) -> Path: + handle = tempfile.TemporaryDirectory() + self.addCleanup(handle.cleanup) + return Path(handle.name) + + def serve(self, categorized: bool = True) -> None: + root = self._tempdir() + files = [] + for path, (data, category) in self.FILES.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(data) + entry = {"dest": Path(path).name, "repo_path": path, "size": len(data), + "sha1": hashlib.sha1(data).hexdigest(), "cores": None, + "systems": ["sys-a"]} + if category: + entry["category"] = category + files.append(entry) + manifest = { + "manifest_version": 2, "platform": "retroarch", "display_name": "RetroArch", + "files": files, + "omitted_files": [{"dest": "level.dat", "name": "level.dat", "system": "sys-a", + "required": True, "reason": "not_found", + "systems": ["sys-a"], "category": "game_data"}], + } + if categorized: + manifest["categorized"] = True + (root / "install").mkdir() + (root / "install" / "retroarch.json").write_text(json.dumps(manifest)) + handler = functools.partial(_Quiet, directory=str(root)) + httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + self.addCleanup(httpd.server_close) + self.addCleanup(httpd.shutdown) + home = self._tempdir() + self.bios = home / "bios" + self.bios.mkdir() + self.env = {**os.environ, "HOME": str(home), "RETROBIOS_OS": "linux", + "RETROBIOS_BASE_URL": f"http://127.0.0.1:{httpd.server_address[1]}"} + + def run_installer(self, *argv: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "install.py", "--platform", "retroarch", "--dest", + str(self.bios), "--no-input", "--jobs", "1", *argv], + cwd=REPO_ROOT, env=self.env, capture_output=True, text=True, timeout=120, + check=False, + ) + + def test_install_leaves_it_out_and_names_it(self): + self.serve() + proc = self.run_installer("--no-game-data") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue((self.bios / "a.bin").exists()) + self.assertFalse((self.bios / "game.wad").exists()) + self.assertIn("Left out as game data (--no-game-data): 2 files: game.wad, level.dat", + proc.stdout) + + def test_check_and_dry_run_leave_it_out(self): + self.serve() + for mode, listed in (("--check", "1 file to download:"), + ("--dry-run", "Would download 1 file")): + with self.subTest(mode=mode): + proc = self.run_installer("--no-game-data", mode) + self.assertIn( + "Left out as game data (--no-game-data): 2 files: game.wad, level.dat", + proc.stdout, + ) + self.assertIn(listed, proc.stdout) + self.assertNotIn("missing: game.wad", proc.stdout) + self.assertFalse((self.bios / "a.bin").exists()) + + def test_the_default_installs_everything(self): + self.serve() + proc = self.run_installer() + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue((self.bios / "game.wad").exists()) + self.assertNotIn("Left out as game data", proc.stdout) + + def test_events_name_what_was_left_out(self): + self.serve() + proc = self.run_installer("--no-game-data", "--check", "--events") + events = [json.loads(line) for line in proc.stdout.splitlines()] + summary = next(e for e in events if e["event"] == "summary") + self.assertEqual(list(summary), ["event", *install.EVENT_KEYS["summary"]]) + self.assertEqual(summary["left_out"], [{ + "category": "game_data", "label": "game data", "files": 2, + "dests": ["game.wad", "level.dat"], + }]) + # The omission is left out, not counted as a file the collection lacks. + self.assertEqual(summary["skipped"], []) + check = next(e for e in events if e["event"] == "check") + self.assertEqual(check["files"], 1) + + def test_a_list_without_categories_is_refused(self): + self.serve(categorized=False) + proc = self.run_installer("--no-game-data", "--check") + self.assertEqual(proc.returncode, 1) + self.assertIn("records no file category", proc.stderr) + + def test_listings_that_read_no_file_list_refuse_it(self): + proc = subprocess.run( + [sys.executable, "install.py", "--list-platforms", "--no-game-data"], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=60, check=False, + ) + self.assertEqual(proc.returncode, 2) + self.assertIn("--no-game-data", proc.stderr) + + def test_the_cut_function(self): + files = [{"dest": "a"}, {"dest": "b", "category": "game_data"}] + omitted = [{"dest": "c", "category": "game_data"}, {"dest": "d"}] + kept, kept_omitted, left = install.leave_out_game_data(files, omitted) + self.assertEqual([e["dest"] for e in kept], ["a"]) + self.assertEqual([e["dest"] for e in kept_omitted], ["d"]) + self.assertEqual([e["dest"] for e in left], ["b", "c"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_port.py b/tests/test_port.py index b752d1b0..60e43f42 100644 --- a/tests/test_port.py +++ b/tests/test_port.py @@ -141,7 +141,7 @@ class EventRecords(unittest.TestCase): json.dumps({"event": "summary", "mode": "install", "installed": 0, "up_to_date": 4, "pending": 1, "failed": [{"dest": "x.bin", "reason": "HTTP 404"}], - "skipped": [], "interrupted": False}), + "skipped": [], "left_out": [], "interrupted": False}), json.dumps({"event": "end", "code": 2}), ] out, summary = io.StringIO(), io.StringIO() @@ -228,7 +228,8 @@ if check: present=3 if done else 1, missing=0 if done else 1, wrong_hash=0 if done else 1, bytes_needed=0 if done else 300, data_missing=0, not_installable=1) emit("summary", mode="check", installed=0, up_to_date=3 if done else 1, - pending=0 if done else 2, failed=[], skipped=[], interrupted=False) + pending=0 if done else 2, failed=[], skipped=[], left_out=[], + interrupted=False) end(0 if done else 1) emit("download", platform="retroarch", files=2, bytes=300) for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1): @@ -237,7 +238,7 @@ for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1): (state / "installed").write_text("") emit("summary", mode="install", installed=2, up_to_date=1, pending=0, failed=[], skipped=[{"reason": "not_found", "label": "not found", "files": 1, "required": 1}], - interrupted=False) + left_out=[], interrupted=False) end(0) ''' diff --git a/wiki/installer.md b/wiki/installer.md index 06a32891..75521a80 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -170,8 +170,28 @@ python install.py --platform retroarch --list-systems python install.py --platform retroarch --system sony-playstation,nintendo-gba python install.py --platform retroarch --system sony-playstation --core pcsx_rearmed python install.py --platform recalbox --region us,eu,jp +python install.py --platform emuelec --no-game-data ``` +### Firmware without game data + +Some platform lists name the files a game reads beside the firmware its +emulator needs (the Doom WADs in EmuELEC's list), and the cores' profiles mark +the data files of the games they run `category: game_data`: MAME's software +lists, a game engine's own data. The manifest marks each of these files +`"category": "game_data"`, and firmware carries no category. + +`--no-game-data` leaves them out of the install, `--check` and `--dry-run`, +and names what it left out with its category (ten names, `--verbose` for all): + +```text + Left out as game data (--no-game-data): 2436 files: ports/doom/doom1.wad, ports/doom2/doom2.wad, ... +``` + +It is the cut `generate_pack.py --no-game-data` makes, and a test holds the +two to the same file list. A file list that records no category is refused +rather than read as having no game data. + ### Installed cores `--installed-cores` reads the cores a RetroArch already has and installs what @@ -242,6 +262,7 @@ what the options name, everything by default. | `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names | | `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included | | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | +| `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | | `--list-systems` | Print a platform's systems with the files and size each holds | | `--list-cores` | Print the cores whose extra files a platform installs | | `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | @@ -288,7 +309,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is | `download` | `platform`, `files`, `bytes` | Before the downloads start | | `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive | | `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file. `reason` says why one failed, and is empty otherwise | -| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply. Not written for a listing | +| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. Not written for a listing | | `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) | An option refused before the run starts (`install.py: error:`, code 2) writes