feat: let the installer skip game data

install.py --no-game-data leaves out the files and omissions the manifest
marks game data, in an install, --check and --dry-run, and names them with
their category. The --events summary gains left_out, one group per
category with every destination. A list without categories is refused.

install.ps1

@@ -11,7 +11,7 @@
 
     $ErrorActionPreference = "Stop"
     $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-    $defaultInstallSha256 = "abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa"
+    $defaultInstallSha256 = "6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a"
     $maximumInstallerBytes = 2MB
     $installerExit = 0
     $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }

install.py

@@ -106,6 +106,10 @@ OMISSION_REASONS = {
     "no_platform_slug": "no platform folder",
     "below_platform_folder": "below the platform folder",
 }
+# What a file is when it is not firmware, as the manifest's category field
+# says, and as the installer words it. Firmware carries no category.
+GAME_DATA = "game_data"
+CATEGORY_LABELS = {GAME_DATA: "game data"}
 _SHA1_RE = re.compile(r"^[0-9a-fA-F]{40}$")
 _SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
 
@@ -479,9 +483,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = {
     "downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total",
                    "bytes_done", "bytes_total"),
     # failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files",
-    # "required"}], the files the collection cannot supply.
+    # "required"}], the files the collection cannot supply. left_out:
+    # [{"category", "label", "files", "dests"}], what --no-game-data left
+    # out, empty without it.
     "summary": ("mode", "installed", "up_to_date", "pending", "failed", "skipped",
-                "interrupted"),
+                "left_out", "interrupted"),
     "end": ("code",),
 }
 # Seconds between two progress events while bytes arrive.
@@ -578,6 +584,49 @@ def _emit_check(
     )
 
 
+def leave_out_game_data(
+    files: list[dict], omitted: list[dict]
+) -> tuple[list[dict], list[dict], list[dict]]:
+    """The files and omissions without game data, and what was left out.
+
+    The same cut as a pack built with --no-game-data: the manifest marks
+    each file that is a game's data, not firmware.
+    """
+    left_out = [e for e in files + omitted if e.get("category") == GAME_DATA]
+    return (
+        [e for e in files if e.get("category") != GAME_DATA],
+        [e for e in omitted if e.get("category") != GAME_DATA],
+        left_out,
+    )
+
+
+def _left_out_lines(left_out: list[dict], every: bool) -> list[str]:
+    """What --no-game-data left out, named per category, ten each unless every."""
+    lines = []
+    for group in _left_out_events(left_out):
+        dests = group["dests"]
+        shown = dests if every else dests[:10]
+        more = (f" ... and {len(dests) - len(shown)} more (--verbose lists them all)"
+                if len(shown) < len(dests) else "")
+        lines.append(f"  Left out as {group['label']} (--no-game-data): "
+                     f"{_plural(len(dests), 'file')}: {', '.join(shown)}{more}")
+    return lines
+
+
+def _left_out_events(left_out: list[dict]) -> list[dict]:
+    """What --no-game-data left out, per category, every file named."""
+    groups: dict[str, dict] = {}
+    for entry in left_out:
+        category = entry.get("category", "")
+        group = groups.setdefault(category, {
+            "category": category, "label": CATEGORY_LABELS.get(category, category),
+            "files": 0, "dests": [],
+        })
+        group["files"] += 1
+        group["dests"].append(entry["dest"])
+    return [groups[category] for category in sorted(groups)]
+
+
 def _skipped(omitted: list[dict]) -> list[dict]:
     """The files the collection cannot supply, grouped by reason."""
     groups: dict[str, dict] = {}
@@ -3812,6 +3861,7 @@ class Invocation:
         for flag, on in (
             ("--every-board", args.every_board),
             ("--installed-cores", args.installed_cores and not args.cores_dir),
+            ("--no-game-data", args.no_game_data),
             ("--standalone-copies", args.standalone_copies),
             ("--check", args.check),
             ("--dry-run", args.dry_run),
@@ -5332,6 +5382,12 @@ def main() -> None:
         help="preferred regions, best first, e.g. us,eu,jp: keep one BIOS per "
              "region where a system has several",
     )
+    parser.add_argument(
+        "--no-game-data",
+        action="store_true",
+        help="leave out game data (files a game reads, not firmware) and "
+             "install the firmware only",
+    )
     parser.add_argument(
         "--list-systems",
         action="store_true",
@@ -5410,6 +5466,9 @@ def main() -> None:
         parser.error("--quiet applies to an install, not to --check, --dry-run or a listing")
     if args.every_board and args.target:
         parser.error("--every-board installs every board; --target names one")
+    if args.no_game_data and (args.list_platforms or args.list_targets):
+        parser.error("--no-game-data narrows a file list, not --list-platforms "
+                     "or --list-targets")
     if args.events:
         # The events own stdout. The text a person reads goes to stderr, so a
         # program never has to tell one from the other.
@@ -5725,6 +5784,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
     total_errors = 0
     total_omitted = 0
     omitted_seen: list[dict] = []
+    left_out_seen: list[dict] = []
     total_interrupted = False
     total_bytes = 0
     total_seconds = 0.0
@@ -5806,6 +5866,23 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
                     f"case and holds another file under the same name ({shown})"
                 )
 
+        game_data_cut = 0
+        if args.no_game_data:
+            if not manifest.get("categorized"):
+                # Without the field nothing is marked, and the run would
+                # install the game data it was asked to leave out.
+                print(f"Error: the {plat_name} file list records no file category, "
+                      "which --no-game-data needs.", file=sys.stderr)
+                print(_older_list_advice(), file=sys.stderr)
+                sys.exit(1)
+            before = len(files)
+            files, omitted_files, left_out = leave_out_game_data(files, omitted_files)
+            game_data_cut = before - len(files)
+            left_out_seen.extend(left_out)
+            if left_out:
+                for line in _left_out_lines(left_out, args.verbose):
+                    print(line)
+
         if args.list_targets:
             targets = fetch_targets(plat_name)
             if targets is None:
@@ -5912,12 +5989,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
             if selection == INSTALLED_ONLY:
                 files, omitted_files = installed
                 print("\n  Same choice without questions: "
-                      f"--platform {plat_name} --installed-cores")
+                      f"--platform {plat_name} --installed-cores"
+                      + (" --no-game-data" if args.no_game_data else ""))
                 _INVOCATION.chosen[plat_name] = [("--installed-cores", None)]
                 selection = None
             elif selection is not None:
                 print("\n  Same choice without questions: "
-                      + _same_choice_hint(plat_name, *selection))
+                      + _same_choice_hint(plat_name, *selection)
+                      + (" --no-game-data" if args.no_game_data else ""))
                 systems, cores, regions = selection
                 _INVOCATION.chosen[plat_name] = [
                     *(("--system", name) for name in systems),
@@ -5994,7 +6073,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
 
         narrowed = bool(
             target or args.system or args.core or args.region or args.installed_cores
-            or len(files) != len(manifest.get("files", [])) + len(
+            or len(files) + game_data_cut != len(manifest.get("files", [])) + len(
                 variants if case_kept else []
             )
         )
@@ -6173,7 +6252,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
             up_to_date=total_verified if args.check or args.dry_run else total_up_to_date,
             pending=check_pending if args.check else files_pending,
             failed=[{"dest": dest, "reason": str(reason)} for dest, reason in failures],
-            skipped=_skipped(omitted_seen), interrupted=total_interrupted,
+            skipped=_skipped(omitted_seen), left_out=_left_out_events(left_out_seen),
+            interrupted=total_interrupted,
         )
     # A check scripted as `--check && launch` reads the exit code only.
     if args.check and check_pending:

install.sh

@@ -18,7 +18,7 @@ esac
 TEMP_INSTALLER=""
 TEMP_DIRECTORY=""
 DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-DEFAULT_INSTALL_SHA256="abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa"
+DEFAULT_INSTALL_SHA256="6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a"
 MAX_INSTALLER_BYTES=2097152
 
 cleanup() {

tests/test_install.py

@@ -3797,7 +3797,7 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase):
         args = SimpleNamespace(
             system=None, core=None, emulator=None, region=None, target=None,
             cores_dir=None, every_board=False, installed_cores=False,
-            standalone_copies=False, check=False, dry_run=False,
+            standalone_copies=False, check=False, dry_run=False, no_game_data=False,
         )
         flags = list(argv)
         while flags:
@@ -4966,7 +4966,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
         installer = events[0]["installer"]
         self.assertEqual(events, self._head("check", installer) + [
             {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 1,
-             "pending": 3, "failed": [], "skipped": self.SKIPPED, "interrupted": False},
+             "pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False},
             {"event": "end", "code": 1},
         ])
         self.assertIn("2/4 present (1 verified, 1 wrong hash)", proc.stderr)
@@ -4989,7 +4989,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
             {"event": "download", "platform": "retroarch", "files": 3, "bytes": 245},
             *done,
             {"event": "summary", "mode": "install", "installed": 3, "up_to_date": 1,
-             "pending": 0, "failed": [], "skipped": self.SKIPPED, "interrupted": False},
+             "pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False},
             {"event": "end", "code": 0},
         ])
         self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])

tests/test_no_game_data.py

@@ -0,0 +1,355 @@
+"""--no-game-data: the firmware without the files a game reads.
+
+A platform list can name a game's data beside the firmware (EmuELEC lists the
+Doom WADs), and a core profile marks its game data `category: game_data`. The
+manifest marks both, the builder leaves both out under --no-game-data, verify
+reports on the same file set, and the installer makes the same cut from the
+full manifest.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import functools
+import hashlib
+import http.server
+import io
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import threading
+import unittest
+import zipfile
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+sys.path.insert(0, str(REPO_ROOT))
+
+import common  # noqa: E402
+import generate_db  # noqa: E402
+import generate_pack as builder  # noqa: E402
+import install  # noqa: E402
+import packverify  # noqa: E402
+import verify  # noqa: E402
+
+PAYLOADS = {
+    "System/firmware.rom": b"firmware of the console",
+    "System/corefw.bin": b"firmware the core loads",
+    "Doom/doom1.wad": b"shareware episode",
+    "Doom/engine.pk3": b"engine resources",
+}
+
+PROFILES = {
+    "emu": {
+        "emulator": "Emu", "type": "libretro", "cores": ["emu"],
+        "systems": ["console"],
+        "files": [
+            {"name": "firmware.rom", "required": True},
+            {"name": "corefw.bin", "required": True},
+        ],
+    },
+    "doomcore": {
+        "emulator": "DoomCore", "type": "libretro", "cores": ["doomcore"],
+        "systems": ["doom"],
+        "files": [
+            {"name": "doom1.wad", "path": "doom/doom1.wad", "category": "game_data",
+             "required": False},
+            {"name": "engine.pk3", "category": "game_data", "required": True},
+        ],
+    },
+}
+
+FIRMWARE = {"firmware.rom", "corefw.bin"}
+GAME = {"doom/doom1.wad", "engine.pk3"}
+
+
+class _Fixture(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self._tmp.name)
+        self.bios = self.root / "bios"
+        self.platforms = self.root / "platforms"
+        self.emulators = self.root / "emulators"
+        files = {}
+        for rel, data in PAYLOADS.items():
+            path = self.bios / rel
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(data)
+            sha1 = hashlib.sha1(data).hexdigest()
+            files[sha1] = {
+                "path": str(path), "name": path.name, "size": len(data), "sha1": sha1,
+                "md5": hashlib.md5(data).hexdigest(),
+                "sha256": hashlib.sha256(data).hexdigest(), "crc32": "00000000",
+            }
+        self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+        self.platforms.mkdir()
+        self.emulators.mkdir()
+        for key, profile in PROFILES.items():
+            (self.emulators / f"{key}.yml").write_text(yaml.dump(profile))
+        platform = {
+            "platform": "Handheld", "verification_mode": "existence",
+            "cores": ["emu", "doomcore"],
+            "systems": {
+                "console": {"files": [{"name": "firmware.rom", "destination": "firmware.rom"}]},
+                "doom": {"files": [{"name": "doom1.wad", "destination": "doom/doom1.wad"}]},
+            },
+        }
+        (self.platforms / "handheld.yml").write_text(yaml.dump(platform))
+        (self.platforms / "_registry.yml").write_text(
+            yaml.dump({"platforms": {"handheld": {"status": "active"}}})
+        )
+        common._platform_config_cache.clear()
+        common._emulator_profiles_cache.clear()
+        self.out = self.root / "dist"
+        self.out.mkdir()
+
+    def tearDown(self):
+        common._platform_config_cache.clear()
+        common._emulator_profiles_cache.clear()
+        self._tmp.cleanup()
+
+    def pack(self, no_game_data: bool) -> tuple[str, set[str], str]:
+        printed = io.StringIO()
+        with contextlib.redirect_stdout(printed):
+            zip_path = builder.generate_pack(
+                "handheld", str(self.platforms), self.db, str(self.bios), str(self.out),
+                emulators_dir=str(self.emulators), emu_profiles=PROFILES,
+                offline=True, no_game_data=no_game_data,
+            )
+        with zipfile.ZipFile(zip_path) as archive:
+            members = set(archive.namelist()) - set(builder.PACK_DOCUMENTS)
+        return zip_path, members, printed.getvalue()
+
+    def manifest(self, no_game_data: bool = False) -> dict:
+        return builder.generate_manifest(
+            "handheld", str(self.platforms), self.db, str(self.bios),
+            str(self.platforms / "_registry.yml"), emulators_dir=str(self.emulators),
+            emu_profiles=PROFILES, offline=True, no_game_data=no_game_data,
+        )
+
+
+class ManifestCategory(_Fixture):
+    def test_game_data_is_marked_and_firmware_is_not(self):
+        manifest = self.manifest()
+        self.assertIs(manifest["categorized"], True)
+        category = {entry["dest"]: entry.get("category") for entry in manifest["files"]}
+        self.assertEqual(set(category), FIRMWARE | GAME)
+        # A platform entry only game data reads, and a core's game data file.
+        self.assertEqual(category["doom/doom1.wad"], "game_data")
+        self.assertEqual(category["engine.pk3"], "game_data")
+        # Firmware carries no field at all.
+        for entry in manifest["files"]:
+            if entry["dest"] in FIRMWARE:
+                self.assertNotIn("category", entry)
+
+    def test_a_narrowed_manifest_is_the_full_one_without_game_data(self):
+        full = self.manifest()
+        narrowed = self.manifest(no_game_data=True)
+        self.assertIs(narrowed["no_game_data"], True)
+        self.assertEqual(
+            {(e["dest"], e["sha1"]) for e in narrowed["files"]},
+            {(e["dest"], e["sha1"]) for e in full["files"] if e.get("category") != "game_data"},
+        )
+
+    def test_an_omitted_game_data_file_is_marked(self):
+        (self.bios / "Doom" / "engine.pk3").unlink()
+        files = {sha1: e for sha1, e in self.db["files"].items() if e["name"] != "engine.pk3"}
+        self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+        manifest = self.manifest()
+        omitted = {e["dest"]: e.get("category") for e in manifest["omitted_files"]}
+        self.assertEqual(omitted.get("engine.pk3"), "game_data")
+
+
+class PackNarrowing(_Fixture):
+    def test_the_pack_leaves_game_data_out_and_names_it(self):
+        _full_path, full, _ = self.pack(no_game_data=False)
+        zip_path, narrowed, printed = self.pack(no_game_data=True)
+        self.assertEqual(full, FIRMWARE | GAME)
+        self.assertEqual(narrowed, FIRMWARE)
+        self.assertIn("_NoGameData_BIOS_Pack.zip", os.path.basename(zip_path))
+        with zipfile.ZipFile(zip_path) as archive:
+            readme = archive.read("README.txt").decode()
+        self.assertIn("PACK TYPE: Narrowed", readme)
+        self.assertIn("firmware only, game data left out", readme)
+        # Named, with the reason, never only counted.
+        self.assertIn("doom/doom1.wad (game data of doomcore)", printed)
+        self.assertIn("doomcore: doom1.wad, engine.pk3", printed)
+
+    def test_verify_reports_the_pack_the_builder_wrote(self):
+        zip_path, members, _ = self.pack(no_game_data=True)
+        config = common.load_platform_config("handheld", str(self.platforms))
+        result = verify.verify_platform(
+            config, self.db, str(self.emulators), PROFILES, no_game_data=True,
+        )
+        self.assertEqual(set(result["pack_destinations"]), members)
+        self.assertTrue(any("doom/doom1.wad" in line for line in result["left_out"]))
+        checked = packverify.verify_pack_against_platform(
+            zip_path, "handheld", str(self.platforms), db=self.db,
+            emulators_dir=str(self.emulators), emu_profiles=PROFILES,
+            no_game_data=True,
+        )
+        self.assertTrue(checked[0], checked[3])
+        # Held to the full list, the narrowed pack is short of the game data.
+        full = packverify.verify_pack_against_platform(
+            zip_path, "handheld", str(self.platforms), db=self.db,
+            emulators_dir=str(self.emulators), emu_profiles=PROFILES,
+        )
+        self.assertFalse(full[0])
+
+    def test_the_full_report_is_unchanged(self):
+        config = common.load_platform_config("handheld", str(self.platforms))
+        result = verify.verify_platform(config, self.db, str(self.emulators), PROFILES)
+        self.assertNotIn("left_out", result)
+        self.assertEqual(set(result["pack_destinations"]), FIRMWARE | GAME)
+
+    def test_split_parts_add_up_to_the_narrowed_pack(self):
+        printed = io.StringIO()
+        with contextlib.redirect_stdout(printed):
+            parts = builder.generate_split_packs(
+                "handheld", str(self.platforms), self.db, str(self.bios), str(self.out),
+                emulators_dir=str(self.emulators), emu_profiles=PROFILES,
+                offline=True, no_game_data=True,
+            )
+        held: set[str] = set()
+        for part in parts:
+            self.assertIn("_NoGameData", part)
+            with zipfile.ZipFile(part) as archive:
+                held |= set(archive.namelist()) - set(builder.PACK_DOCUMENTS)
+        self.assertEqual(held, FIRMWARE)
+
+
+class _Quiet(http.server.SimpleHTTPRequestHandler):
+    def log_message(self, *args) -> None:
+        return
+
+
+class InstallerLeavesGameDataOut(unittest.TestCase):
+    """The installer cut, in each mode, from a manifest served over loopback."""
+
+    FILES = {
+        "bios/a.bin": (b"alpha" * 20, None),
+        "bios/game.wad": (b"wad" * 10, "game_data"),
+    }
+
+    def _tempdir(self) -> Path:
+        handle = tempfile.TemporaryDirectory()
+        self.addCleanup(handle.cleanup)
+        return Path(handle.name)
+
+    def serve(self, categorized: bool = True) -> None:
+        root = self._tempdir()
+        files = []
+        for path, (data, category) in self.FILES.items():
+            (root / path).parent.mkdir(parents=True, exist_ok=True)
+            (root / path).write_bytes(data)
+            entry = {"dest": Path(path).name, "repo_path": path, "size": len(data),
+                     "sha1": hashlib.sha1(data).hexdigest(), "cores": None,
+                     "systems": ["sys-a"]}
+            if category:
+                entry["category"] = category
+            files.append(entry)
+        manifest = {
+            "manifest_version": 2, "platform": "retroarch", "display_name": "RetroArch",
+            "files": files,
+            "omitted_files": [{"dest": "level.dat", "name": "level.dat", "system": "sys-a",
+                               "required": True, "reason": "not_found",
+                               "systems": ["sys-a"], "category": "game_data"}],
+        }
+        if categorized:
+            manifest["categorized"] = True
+        (root / "install").mkdir()
+        (root / "install" / "retroarch.json").write_text(json.dumps(manifest))
+        handler = functools.partial(_Quiet, directory=str(root))
+        httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
+        threading.Thread(target=httpd.serve_forever, daemon=True).start()
+        self.addCleanup(httpd.server_close)
+        self.addCleanup(httpd.shutdown)
+        home = self._tempdir()
+        self.bios = home / "bios"
+        self.bios.mkdir()
+        self.env = {**os.environ, "HOME": str(home), "RETROBIOS_OS": "linux",
+                    "RETROBIOS_BASE_URL": f"http://127.0.0.1:{httpd.server_address[1]}"}
+
+    def run_installer(self, *argv: str) -> subprocess.CompletedProcess:
+        return subprocess.run(
+            [sys.executable, "install.py", "--platform", "retroarch", "--dest",
+             str(self.bios), "--no-input", "--jobs", "1", *argv],
+            cwd=REPO_ROOT, env=self.env, capture_output=True, text=True, timeout=120,
+            check=False,
+        )
+
+    def test_install_leaves_it_out_and_names_it(self):
+        self.serve()
+        proc = self.run_installer("--no-game-data")
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+        self.assertTrue((self.bios / "a.bin").exists())
+        self.assertFalse((self.bios / "game.wad").exists())
+        self.assertIn("Left out as game data (--no-game-data): 2 files: game.wad, level.dat",
+                      proc.stdout)
+
+    def test_check_and_dry_run_leave_it_out(self):
+        self.serve()
+        for mode, listed in (("--check", "1 file to download:"),
+                             ("--dry-run", "Would download 1 file")):
+            with self.subTest(mode=mode):
+                proc = self.run_installer("--no-game-data", mode)
+                self.assertIn(
+                    "Left out as game data (--no-game-data): 2 files: game.wad, level.dat",
+                    proc.stdout,
+                )
+                self.assertIn(listed, proc.stdout)
+                self.assertNotIn("missing: game.wad", proc.stdout)
+                self.assertFalse((self.bios / "a.bin").exists())
+
+    def test_the_default_installs_everything(self):
+        self.serve()
+        proc = self.run_installer()
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+        self.assertTrue((self.bios / "game.wad").exists())
+        self.assertNotIn("Left out as game data", proc.stdout)
+
+    def test_events_name_what_was_left_out(self):
+        self.serve()
+        proc = self.run_installer("--no-game-data", "--check", "--events")
+        events = [json.loads(line) for line in proc.stdout.splitlines()]
+        summary = next(e for e in events if e["event"] == "summary")
+        self.assertEqual(list(summary), ["event", *install.EVENT_KEYS["summary"]])
+        self.assertEqual(summary["left_out"], [{
+            "category": "game_data", "label": "game data", "files": 2,
+            "dests": ["game.wad", "level.dat"],
+        }])
+        # The omission is left out, not counted as a file the collection lacks.
+        self.assertEqual(summary["skipped"], [])
+        check = next(e for e in events if e["event"] == "check")
+        self.assertEqual(check["files"], 1)
+
+    def test_a_list_without_categories_is_refused(self):
+        self.serve(categorized=False)
+        proc = self.run_installer("--no-game-data", "--check")
+        self.assertEqual(proc.returncode, 1)
+        self.assertIn("records no file category", proc.stderr)
+
+    def test_listings_that_read_no_file_list_refuse_it(self):
+        proc = subprocess.run(
+            [sys.executable, "install.py", "--list-platforms", "--no-game-data"],
+            cwd=REPO_ROOT, capture_output=True, text=True, timeout=60, check=False,
+        )
+        self.assertEqual(proc.returncode, 2)
+        self.assertIn("--no-game-data", proc.stderr)
+
+    def test_the_cut_function(self):
+        files = [{"dest": "a"}, {"dest": "b", "category": "game_data"}]
+        omitted = [{"dest": "c", "category": "game_data"}, {"dest": "d"}]
+        kept, kept_omitted, left = install.leave_out_game_data(files, omitted)
+        self.assertEqual([e["dest"] for e in kept], ["a"])
+        self.assertEqual([e["dest"] for e in kept_omitted], ["d"])
+        self.assertEqual([e["dest"] for e in left], ["b", "c"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_port.py

@@ -141,7 +141,7 @@ class EventRecords(unittest.TestCase):
             json.dumps({"event": "summary", "mode": "install", "installed": 0,
                         "up_to_date": 4, "pending": 1,
                         "failed": [{"dest": "x.bin", "reason": "HTTP 404"}],
-                        "skipped": [], "interrupted": False}),
+                        "skipped": [], "left_out": [], "interrupted": False}),
             json.dumps({"event": "end", "code": 2}),
         ]
         out, summary = io.StringIO(), io.StringIO()
@@ -228,7 +228,8 @@ if check:
          present=3 if done else 1, missing=0 if done else 1, wrong_hash=0 if done else 1,
          bytes_needed=0 if done else 300, data_missing=0, not_installable=1)
     emit("summary", mode="check", installed=0, up_to_date=3 if done else 1,
-         pending=0 if done else 2, failed=[], skipped=[], interrupted=False)
+         pending=0 if done else 2, failed=[], skipped=[], left_out=[],
+         interrupted=False)
     end(0 if done else 1)
 emit("download", platform="retroarch", files=2, bytes=300)
 for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1):
@@ -237,7 +238,7 @@ for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1):
 (state / "installed").write_text("")
 emit("summary", mode="install", installed=2, up_to_date=1, pending=0, failed=[],
      skipped=[{"reason": "not_found", "label": "not found", "files": 1, "required": 1}],
-     interrupted=False)
+     left_out=[], interrupted=False)
 end(0)
 '''
 

wiki/installer.md

@@ -170,8 +170,28 @@ python install.py --platform retroarch --list-systems
 python install.py --platform retroarch --system sony-playstation,nintendo-gba
 python install.py --platform retroarch --system sony-playstation --core pcsx_rearmed
 python install.py --platform recalbox --region us,eu,jp
+python install.py --platform emuelec --no-game-data
 ```
 
+### Firmware without game data
+
+Some platform lists name the files a game reads beside the firmware its
+emulator needs (the Doom WADs in EmuELEC's list), and the cores' profiles mark
+the data files of the games they run `category: game_data`: MAME's software
+lists, a game engine's own data. The manifest marks each of these files
+`"category": "game_data"`, and firmware carries no category.
+
+`--no-game-data` leaves them out of the install, `--check` and `--dry-run`,
+and names what it left out with its category (ten names, `--verbose` for all):
+
+```text
+  Left out as game data (--no-game-data): 2436 files: ports/doom/doom1.wad, ports/doom2/doom2.wad, ...
+```
+
+It is the cut `generate_pack.py --no-game-data` makes, and a test holds the
+two to the same file list. A file list that records no category is refused
+rather than read as having no game data.
+
 ### Installed cores
 
 `--installed-cores` reads the cores a RetroArch already has and installs what
@@ -242,6 +262,7 @@ what the options name, everything by default.
 | `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names |
 | `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included |
 | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` |
+| `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) |
 | `--list-systems` | Print a platform's systems with the files and size each holds |
 | `--list-cores` | Print the cores whose extra files a platform installs |
 | `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for |
@@ -288,7 +309,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is
 | `download` | `platform`, `files`, `bytes` | Before the downloads start |
 | `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive |
 | `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file. `reason` says why one failed, and is empty otherwise |
-| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply. Not written for a listing |
+| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. Not written for a listing |
 | `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) |
 
 An option refused before the run starts (`install.py: error:`, code 2) writes