From 8b952968d666053d7317629b2846ea8def414b0f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 06:19:05 +0200 Subject: [PATCH 1/3] feat: add a --no-game-data pack narrowing The install manifest marks each game data file "category": "game_data": a platform entry only game data profile entries read, or a core file its profile declares game data. Firmware carries no category. Every manifest says "categorized": true so the installer can refuse an older list. generate_pack.py --no-game-data cuts those files from the platform list and the core profiles before the build, names each one with the cores that read it as game data, and tags the pack _NoGameData through _narrowings. verify.py and the pack check take the same cut, so --verify-packs and the report describe the pack the builder wrote. Modes without a platform (--emulator, --system alone, --from-md5) and --all-variants refuse it. --- .github/workflows/deploy-site.yml | 2 + schemas/install-manifest.schema.json | 16 +++ scripts/gamedata.py | 159 +++++++++++++++++++++++++++ scripts/generate_pack.py | 145 ++++++++++++++++++++++-- scripts/packverify.py | 9 +- scripts/verify.py | 32 ++++++ tests/test_e2e.py | 15 +++ tests/test_verify_listing.py | 4 + 8 files changed, 371 insertions(+), 11 deletions(-) create mode 100644 scripts/gamedata.py diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 1703638b..183deaba 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -20,6 +20,8 @@ on: - "scripts/deterministic_zip.py" - "scripts/dumpcatalog.py" - "scripts/exporter/baseline.py" + - "scripts/exporter/candidates.py" + - "scripts/gamedata.py" - "scripts/generate_pack.py" - "scripts/generate_readme.py" - "scripts/generate_site.py" diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index cca54788..1dd9e05f 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -44,6 +44,14 @@ "minLength": 1, "description": "Hardware target this manifest was filtered for, absent when unfiltered" }, + "categorized": { + "description": "Every game data file carries category: game_data. Without it the list records no category, and the installer refuses --no-game-data.", + "const": true + }, + "no_game_data": { + "description": "Built without game data, absent when the game data is listed", + "const": true + }, "platform": { "type": "string", "minLength": 1 @@ -170,6 +178,10 @@ "below_platform_folder" ] }, + "category": { + "description": "game_data: a game's own data, not firmware. Absent on firmware.", + "const": "game_data" + }, "held": { "description": "The pack builder holds this file and the installer cannot download it. Region narrowing counts it held, as the builder does, so it displaces the same fallback.", "const": true @@ -685,6 +697,10 @@ } } }, + "category": { + "description": "game_data: a game's own data, not firmware. Absent on firmware. The installer's --no-game-data leaves these out, as a pack built with --no-game-data does.", + "const": "game_data" + }, "executable": { "description": "The repository keeps the file executable, and the installer gives it the execute bits the umask allows.", "type": "boolean" diff --git a/scripts/gamedata.py b/scripts/gamedata.py new file mode 100644 index 00000000..2035c9a9 --- /dev/null +++ b/scripts/gamedata.py @@ -0,0 +1,159 @@ +"""Leaving a game's own data out of a pack, a manifest or a report. + +A platform list can name the files a game reads beside the firmware its +emulator needs, and a core profile marks the data files of the games it runs +`category: game_data`. `--no-game-data` keeps the firmware: the platform's +entries categorized as game data and the profiles' game data entries are left +out, and each one is named with the reason. + +Every caller cuts from the full platform config and the full profiles. The +category of a platform entry is read from the profiles' game data entries, so +cutting twice would find no evidence the second time. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from common import resolve_platform_cores +from exporter.candidates import GAME_DATA, EntryCategories + + +@dataclass(frozen=True) +class LeftOut: + """One game data file left out, and who reads it as game data.""" + + layer: str # "platform" (the platform's list) or "core" (a core profile) + system: str + name: str + readers: tuple[str, ...] + + def describe(self) -> str: + where = f"{self.system}/{self.name}" if self.system else self.name + if not self.readers: + return where + return f"{where} (game data of {', '.join(self.readers)})" + + +@dataclass +class GameDataCut: + """The config and profiles without game data, and what was left out.""" + + config: dict + profiles: dict[str, dict] + left_out: list[LeftOut] + + def report_lines(self) -> list[str]: + """What the cut left out, named, one line per layer. + + The platform layer names each entry and the cores that read it as + game data. The core layer names each core with the files its profile + declares `category: game_data`, held by the collection or not. + """ + lines = [] + platform = [item.describe() for item in self.left_out if item.layer == "platform"] + if platform: + lines.append( + f"Left out as game data, {len(platform)} entries of the platform's " + f"list: {', '.join(platform)}" + ) + by_core: dict[str, list[str]] = {} + for item in self.left_out: + if item.layer == "core": + names = by_core.setdefault(item.readers[0], []) + if item.name not in names: + names.append(item.name) + if by_core: + count = sum(len(names) for names in by_core.values()) + named = "; ".join( + f"{core}: {', '.join(names)}" for core, names in sorted(by_core.items()) + ) + lines.append( + f"Left out as game data, {count} files the cores declare " + f"category: game_data ({named})" + ) + return lines + + +def is_game_data(entry: dict) -> bool: + return entry.get("category") == GAME_DATA + + +def _entry_names(entry: dict) -> set[str]: + names = {str(entry.get("name") or "")} + names.update(str(a) for a in entry.get("aliases") or []) + path = str(entry.get("path") or "") + if path: + names.add(path.rstrip("/").rsplit("/", 1)[-1]) + return {n.casefold() for n in names if n} + + +def platform_game_data( + config: dict, profiles: dict[str, dict], data_registry: dict | None +) -> set[int]: + """The `id()` of every platform entry categorized as game data.""" + categories = EntryCategories(config, profiles, data_registry) + return { + id(fe) + for sys_id, system in (config.get("systems") or {}).items() + for fe in system.get("files") or [] + if isinstance(fe, dict) + and categories.category( + fe, str(fe.get("destination", fe.get("name", ""))), sys_id + ) == GAME_DATA + } + + +def without_game_data( + config: dict, + profiles: dict[str, dict], + data_registry: dict | None = None, + target_cores: set[str] | None = None, +) -> GameDataCut: + """The platform config and the profiles with every game data entry removed. + + The profiles lose their `category: game_data` files, so no core extra + of that category reaches the pack, the manifest or the report. The + platform keeps every entry that is not categorized game data. + """ + dropped = platform_game_data(config, profiles, data_registry) + examined = sorted(resolve_platform_cores(config, profiles, target_cores=target_cores)) + readers_by_name: dict[str, set[str]] = {} + for key in examined: + for fe in profiles[key].get("files") or []: + if isinstance(fe, dict) and is_game_data(fe): + for name in _entry_names(fe): + readers_by_name.setdefault(name, set()).add(key) + + left_out: list[LeftOut] = [] + systems: dict[str, dict] = {} + for sys_id, system in (config.get("systems") or {}).items(): + files = system.get("files") or [] + kept = [fe for fe in files if id(fe) not in dropped] + for fe in files: + if id(fe) in dropped: + name = str(fe.get("name") or "") + left_out.append(LeftOut( + "platform", sys_id, name, + tuple(sorted(readers_by_name.get(name.casefold(), ()))), + )) + systems[sys_id] = {**system, "files": kept} if len(kept) != len(files) else system + + for key in examined: + for fe in profiles[key].get("files") or []: + if isinstance(fe, dict) and is_game_data(fe): + left_out.append(LeftOut("core", "", str(fe.get("name") or ""), (key,))) + + stripped = { + key: ( + {**profile, "files": [ + fe for fe in profile.get("files") or [] + if not (isinstance(fe, dict) and is_game_data(fe)) + ]} + if any(isinstance(fe, dict) and is_game_data(fe) + for fe in profile.get("files") or []) + else profile + ) + for key, profile in profiles.items() + } + return GameDataCut({**config, "systems": systems}, stripped, left_out) diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index bb297cf5..dc3b3cc6 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -79,6 +79,7 @@ from manifestselect import SelectionIndex import slot as slot_mod import slots import split_pack +from gamedata import GAME_DATA, GameDataCut, is_game_data, platform_game_data, without_game_data from deterministic_zip import _FIXED_DATE_TIME, rebuild_zip_deterministic from largefiles import asset_name, load_registered_paths, published_asset_name from nativemode import ( @@ -193,6 +194,7 @@ def _narrowings( one_per_slot: bool, required_only: bool, standalone: bool = False, + no_game_data: bool = False, ) -> list[tuple[str, str]]: """Every dimension that narrows a pack, as (filename tag, plain label). @@ -223,6 +225,8 @@ def _narrowings( applied.append(("_Standalone", "standalone mode files only")) if required_only: applied.append(("_Required", "required files only")) + if no_game_data: + applied.append(("_NoGameData", "firmware only, game data left out")) return applied @@ -608,6 +612,22 @@ def _matches_an_alternative(fe: dict, local_path: str, profiles: dict) -> bool: return False +def _report_left_out(cut: GameDataCut, systems: set[str], cores: bool) -> None: + """Name the game data a --no-game-data build left out, and why. + + ``systems`` limits the platform layer to the systems this pack covers, + and ``cores`` says whether this pack collected the core files itself: + a split part does not, its run names them once. + """ + kept = [ + item for item in cut.left_out + if (item.layer == "platform" and item.system in systems) + or (item.layer == "core" and cores) + ] + for line in GameDataCut(cut.config, cut.profiles, kept).report_lines(): + print(f" {line}") + + def generate_pack( platform_name: str, platforms_dir: str, @@ -630,9 +650,13 @@ def generate_pack( one_per_slot: bool = False, offline: bool | None = None, pack_name: str | None = None, + no_game_data: bool = False, ) -> str | None: """Generate a ZIP pack for a platform. + ``no_game_data`` leaves out the platform's entries categorized as game + data and the cores' ``category: game_data`` files, and names each one. + ``pack_name`` replaces the derived file name: a split part is named after its group, and a name built from every system of a hundred-system group is longer than a file name can be. @@ -646,13 +670,22 @@ def generate_pack( # The pipeline always passes them, but a caller that does not must get the # same pack, not one built without validation, slots or agnostic cores. emu_profiles = load_emulator_profiles(emulators_dir) + # Cut from the full config and profiles: the category of a platform + # entry is read from the profiles' game data entries. + cut = ( + without_game_data(config, emu_profiles, data_registry, target_cores) + if no_game_data else None + ) + if cut is not None: + config, emu_profiles = cut.config, cut.profiles verification_mode = config.get("verification_mode", "existence") platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") narrowings = _narrowings( - source, regions, target_name, one_per_slot, required_only + source, regions, target_name, one_per_slot, required_only, + no_game_data=no_game_data, ) zip_path = os.path.join( output_dir, @@ -1164,6 +1197,12 @@ def generate_pack( zip_path, file_status, total_files, core_count, source, verification_mode, ) + if cut is not None: + _report_left_out( + cut, + systems=set(pack_systems) if source != "truth" else set(), + cores=source != "platform" and precomputed_extras is None, + ) for key, reason in sorted(file_reasons.items()): @@ -1884,6 +1923,7 @@ def generate_split_packs( target_name: str | None = None, one_per_slot: bool = False, offline: bool | None = None, + no_game_data: bool = False, ) -> list[str]: """Generate split packs (one ZIP per system or manufacturer).""" config = load_platform_config(platform_name, platforms_dir) @@ -1893,7 +1933,8 @@ def generate_split_packs( split_tags = "".join( tag for tag, _label in _narrowings( - source, regions, target_name, one_per_slot, required_only + source, regions, target_name, one_per_slot, required_only, + no_game_data=no_game_data, ) ) # Two groupings write different files. Without the tag they accumulate in @@ -1920,15 +1961,24 @@ def generate_split_packs( if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) base_dest = config.get("base_destination", "") + # Each part cuts the platform's own entries from the full profiles. The + # extras are collected once here, already without the cores' game data. + cut = ( + without_game_data(config, emu_profiles, data_registry, target_cores) + if no_game_data else None + ) if source == "platform": all_extras = [] elif emu_profiles: all_extras = _collect_emulator_extras( - config, emulators_dir, db, set(), base_dest, emu_profiles, + cut.config if cut else config, emulators_dir, db, set(), base_dest, + cut.profiles if cut else emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ) else: all_extras = [] + if cut is not None and source != "platform": + _report_left_out(cut, systems=set(), cores=True) results = [] for group_name, group_system_ids in sorted(groups.items()): group_extras = _extras_for_systems(all_extras, group_system_ids) @@ -1952,6 +2002,7 @@ def generate_split_packs( one_per_slot=one_per_slot, offline=offline, pack_name=f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip", + no_game_data=no_game_data, ) if zip_path: results.append(zip_path) @@ -1975,6 +2026,7 @@ def generate_split_packs( precomputed_extras=undistributed, extras_only=True, source=source, regions=regions, target_name=target_name, one_per_slot=one_per_slot, offline=offline, pack_name=f"{part_prefix}_Other_Cores_BIOS_Pack.zip", + no_game_data=no_game_data, ) if zip_path: results.append(zip_path) @@ -2411,6 +2463,12 @@ def _validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> if args.split: parser.error("--system is incompatible with --split") + # Game data is categorized against a platform's list and the cores it + # runs: a pack of one emulator or of a system's emulators has neither. + if getattr(args, "no_game_data", False) and not (has_platform or has_all): + parser.error("--no-game-data requires --platform or --all") + if getattr(args, "no_game_data", False) and has_from_md5: + parser.error("--no-game-data is incompatible with --from-md5") if getattr(args, "one_per_slot", False): if has_from_md5: parser.error("--one-per-slot and --from-md5 are mutually exclusive") @@ -2483,6 +2541,7 @@ def _run_manifest_mode( ), offline=args.offline, required_only=required_only, + no_game_data=getattr(args, "no_game_data", False), ) narrow_suffix = "".join( tag.lower() @@ -2492,6 +2551,7 @@ def _run_manifest_mode( _platform_target(representative, args.target, args.platforms_dir), False, required_only, + no_game_data=getattr(args, "no_game_data", False), ) ) out_path = os.path.join( @@ -2591,6 +2651,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None: verify_regions, args.target, ALL_VARIANTS if args.all_variants else (("full", False),), + no_game_data=getattr(args, "no_game_data", False), ) on_disk = ( set(os.listdir(args.output_dir)) if os.path.isdir(args.output_dir) else set() @@ -2641,6 +2702,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None: target_cores=tc, source=source, required_only=required_only, + no_game_data=getattr(args, "no_game_data", False), ) (ok, _checked, _present, errors, bl_checked, bl_present, core_checked, core_present, bl_excluded, bl_absent) = result @@ -2788,6 +2850,7 @@ def _run_platform_packs( target_name=named_target, one_per_slot=args.one_per_slot, offline=args.offline, + no_game_data=getattr(args, "no_game_data", False), ) print(f" Split into {len(zip_paths)} packs") else: @@ -2806,6 +2869,7 @@ def _run_platform_packs( named_target, args.one_per_slot, required_only, + no_game_data=getattr(args, "no_game_data", False), ) ) stem = _platform_pack_stem( @@ -2834,6 +2898,7 @@ def _run_platform_packs( one_per_slot=args.one_per_slot, offline=args.offline, pack_name=group_name, + no_game_data=getattr(args, "no_game_data", False), ) except (FileNotFoundError, OSError, yaml.YAMLError) as e: print(f" ERROR: {e}", file=sys.stderr) @@ -2856,6 +2921,7 @@ def _run_platform_packs( data_registry=data_registry, regions=getattr(args, "regions", None), target_name=args.target, + no_game_data=getattr(args, "no_game_data", False), ) if args.split: for entry in os.listdir(args.output_dir): @@ -2890,6 +2956,7 @@ def _refuse_for_all_variants(args: argparse.Namespace, parser: argparse.Argument for flag, given in ( ("--source", args.source is not None), ("--required-only", args.required_only), + ("--no-game-data", getattr(args, "no_game_data", False)), ("--emulator", args.emulator), ("--from-md5", args.from_md5 or args.from_md5_file), ): @@ -2978,7 +3045,10 @@ def _refuse_unapplied_flags(args: argparse.Namespace, parser: argparse.ArgumentP QUICK_MODE_FLAGS: dict[str, set[str]] = { # Checks packs already on disk against the platform's own list, narrowed # by region and target and by nothing else. - "--verify-packs": {"--platform", "--all", "--target", "--region", "--include-archived"}, + "--verify-packs": { + "--platform", "--all", "--target", "--region", "--include-archived", + "--no-game-data", + }, # One manifest per hardware target, from the target files themselves. "--manifest-targets": set(), "--list": set(), @@ -3005,6 +3075,7 @@ def _given_flags(args: argparse.Namespace) -> set[str]: "--from-md5": args.from_md5, "--from-md5-file": args.from_md5_file, "--required-only": args.required_only, + "--no-game-data": getattr(args, "no_game_data", False), "--source": args.source is not None, "--target": args.target, "--region": args.region, @@ -3107,6 +3178,12 @@ def _main(held: contextlib.ExitStack) -> None: action="store_true", help="Only include required files, skip optional", ) + parser.add_argument( + "--no-game-data", + action="store_true", + help="Leave out game data (the platform's entries a core reads as a " + "game's files, and the cores' game data files): firmware only", + ) parser.add_argument( "--source", choices=["platform", "truth", "full"], @@ -3860,6 +3937,10 @@ def _manifest_core_entries( pack_only_sizes.append(file_size) continue + if is_game_data(fe) and all( + is_game_data(claim) for claim in fe.get("other_claims", []) + ): + entry["category"] = GAME_DATA _attach_claims( entry, fe.get("other_claims", []), db, bios_dir, zip_contents, offline, required_only, validation_index, owners, dest, @@ -4027,12 +4108,17 @@ def generate_manifest( required_only: bool = False, data_registry: dict | None = None, fold_case: bool | None = None, + no_game_data: bool = False, ) -> dict: """Generate a JSON manifest for a platform (same resolution as generate_pack). Returns a dict ready for JSON serialization with file inventory, install hints, and download metadata. ``fold_case`` overrides the platform's ``case_insensitive_fs``, for the case variants below. + + A file that is a game's data, not firmware, carries ``"category": + "game_data"``, so the installer can leave it out as the builder does + under ``no_game_data``. Firmware carries no category. """ config = load_platform_config(platform_name, platforms_dir) if fold_case is not None: @@ -4041,6 +4127,15 @@ def generate_manifest( zip_contents = {} if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) + if data_registry is None: + data_registry = load_data_dir_registry(platforms_dir) + full_profiles = emu_profiles + game_entries: set[int] = set() + if no_game_data: + cut = without_game_data(config, emu_profiles, data_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles + else: + game_entries = platform_game_data(config, emu_profiles, data_registry) platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") @@ -4073,6 +4168,16 @@ def generate_manifest( platform_cores=plat_cores, ) + # A destination is game data when every declaration of it is: a firmware + # declaration of the same path keeps it in a pack built without game data. + declared_kinds: dict[str, set[bool]] = {} + for system in pack_systems.values(): + for fe in system.get("files", []): + bare = sanitize_pack_path(fe.get("destination", fe.get("name", ""))) + if bare: + declared_kinds.setdefault(bare, set()).add(id(fe) in game_entries) + game_destinations = {d for d, kinds in declared_kinds.items() if kinds == {True}} + seen_destinations: set[str] = set() # Destination -> (entry, full destination, manifest destination) of each # platform file, for the core claims a target build would fall back on. @@ -4090,8 +4195,6 @@ def generate_manifest( # collection does not index them, a data directory cache answered. pack_only_sizes: list[int] = [] rename_notes: set[str] = set() - if data_registry is None: - data_registry = load_data_dir_registry(platforms_dir) slot_overrides = slots.pack_overrides( config, emu_profiles, db, zip_contents, data_registry, target_cores=target_cores, source=source, @@ -4134,6 +4237,11 @@ def generate_manifest( # drops the same fallback. **selection.fields(bare, str(file_entry.get("name") or "")), **({"held": True} if bare in builder_held else {}), + **( + {"category": GAME_DATA} + if (bare in game_destinations if cores is None else is_game_data(file_entry)) + else {} + ), } # Read by the region grouping and by phase 2 with the same arguments. @@ -4299,6 +4407,8 @@ def generate_manifest( local_path, dest, None, db, repo_root, selection.fields(dest, file_entry["name"]), ) + if entry is not None and dest in game_destinations: + entry["category"] = GAME_DATA if entry is not None and target_cores is None: order = declaration_order(full_dest) if full_dest in slot_choices or len(order) > 1: @@ -4364,6 +4474,10 @@ def generate_manifest( if declared and declared != set(entry.get("systems") or []): entry["declared_in"] = sorted(declared) claims = baseline_claims.get(key, {}).get("other_claims") + if claims and any(not is_game_data(claim) for claim in claims): + # A core reads this path as firmware: a pack without game data + # still fills it. + entry.pop("category", None) if claims: _attach_claims( entry, claims, db, bios_dir, zip_contents, offline, required_only, @@ -4409,6 +4523,10 @@ def generate_manifest( # Absent on an unfiltered manifest rather than empty, to leave the # twelve default files byte-identical. **({"target": target_name} if target_name else {}), + # Every game data file below says so: the installer's --no-game-data + # relies on it, and refuses a list that does not record categories. + "categorized": True, + **({"no_game_data": True} if no_game_data else {}), "platform": platform_name, "display_name": platform_display, "version": "1.0", @@ -4448,9 +4566,9 @@ def generate_manifest( # or macOS too, stays folded. kept = generate_manifest( platform_name, platforms_dir, db, bios_dir, registry_path, - emulators_dir, zip_contents, emu_profiles, target_cores, source, + emulators_dir, zip_contents, full_profiles, target_cores, source, regions, target_name, offline, required_only, data_registry, - fold_case=False, + fold_case=False, no_game_data=no_game_data, ) folded = {entry["dest"] for entry in manifest_files} folded_lower = {dest.lower() for dest in folded} @@ -4624,6 +4742,7 @@ def _expected_pack_names( regions: list[str] | None, target_name: str | None, variants: tuple[tuple[str, bool], ...] = (("full", False),), + no_game_data: bool = False, ) -> dict[str, set[str]]: """Exact pack file names --verify-packs may check, per platform. @@ -4638,7 +4757,9 @@ def _expected_pack_names( return [ "".join( tag - for tag, _label in _narrowings(source, regions, named, False, required) + for tag, _label in _narrowings( + source, regions, named, False, required, no_game_data=no_game_data + ) ) for source, required in variants ] @@ -4767,6 +4888,7 @@ def verify_and_finalize_packs( regions: list[str] | None = None, target_name: str | None = None, emulators_dir: str = "emulators", + no_game_data: bool = False, ) -> bool: """Verify all packs, inject manifests, generate SHA256SUMS. @@ -4783,7 +4905,9 @@ def verify_and_finalize_packs( # substring: a _Custom or regional pack left by another run was judged # against this run's expectation and failed it. registered = list_registered_platforms(platforms_dir, include_archived=True) - expected = _expected_pack_names(registered, platforms_dir, regions, target_name) + expected = _expected_pack_names( + registered, platforms_dir, regions, target_name, no_game_data=no_game_data + ) pack_to_platform: dict[str, list[str]] = {} for name in _pack_archives(output_dir): for pname in registered: @@ -4837,6 +4961,7 @@ def verify_and_finalize_packs( regions=regions, data_registry=data_registry, target_cores=_target_cores_for(pname, target_name, platforms_dir), + no_game_data=no_game_data, ) status = "OK" if p_ok else "FAILED" exclusion_note = "".join( diff --git a/scripts/packverify.py b/scripts/packverify.py index d4aa29bf..c52f466b 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -32,6 +32,7 @@ from nativemode import reads_file_contents import region as region_mod from packresolve import resolve_file from exporter.candidates import BIOS, EntryCategories +from gamedata import without_game_data from common import resolve_platform_cores from common import sanitize_pack_path import zipfile @@ -634,6 +635,7 @@ def verify_pack_against_platform( target_cores: set[str] | None = None, source: str = "full", required_only: bool = False, + no_game_data: bool = False, ) -> tuple[bool, int, int, list[str], int, int, int, int, int, int]: """Verify a pack ZIP against its platform config and core requirements. @@ -644,6 +646,8 @@ def verify_pack_against_platform( ``required_only`` are the variant the pack was built as: a platform-only pack is held to the platform's list alone, a truth pack to the cores' files alone, and a required-only pack to the required entries of both. + ``no_game_data`` holds the pack to the firmware alone, cut as the + builder cuts it. Checks: 1. Every baseline file declared by the platform exists in the ZIP @@ -657,7 +661,7 @@ def verify_pack_against_platform( baseline_present, core_checked, core_present, baseline_excluded, baseline_absent)``. ``baseline_absent`` counts destinations the collection holds no file for that are a game's data or that no core the platform - runs opens; verify.py counts them apart from firmware. + runs opens. verify.py counts them apart from firmware. """ config = load_platform_config(platform_name, platforms_dir) @@ -668,6 +672,9 @@ def verify_pack_against_platform( if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) + if no_game_data: + cut = without_game_data(config, emu_profiles, data_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles expected_systems = config.get("systems", {}) if target_cores is not None: diff --git a/scripts/verify.py b/scripts/verify.py index 26187b60..6b6b654c 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1121,14 +1121,33 @@ def verify_platform( data_dir_registry: dict | None = None, regions: list[str] | None = None, system_filter: list[str] | None = None, + no_game_data: bool = False, ) -> dict: """Verify all BIOS files for a platform, including cross-reference gaps. + ``no_game_data`` reports on the pack built with --no-game-data: the + game data cut as the builder cuts it, and named in ``left_out``. + A region priority list narrows the report to the files a pack built with the same list would carry, using the same selection function as the builder. ``system_filter`` does the same for a pack built with --system: the systems it keeps and the core extras those systems own. """ + left_out: list[str] = [] + if no_game_data: + from gamedata import without_game_data + + if emu_profiles is None: + emu_profiles = load_emulator_profiles(emulators_dir) + cut = without_game_data(config, emu_profiles, data_dir_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles + systems = config.get("systems", {}) + in_scope = set(select_systems(systems, system_filter) if system_filter else systems) + cut.left_out = [ + item for item in cut.left_out + if item.layer == "core" or item.system in in_scope + ] + left_out = cut.report_lines() # Normalized once: a typo in a platform YAML must not verify with one # mode and be scored with another. mode = normalize_mode(config.get("verification_mode")) @@ -1524,6 +1543,7 @@ def verify_platform( "undeclared_files": undeclared, "exclusion_notes": exclusions, "region_fallbacks": sorted(region_fallbacks), + **({"left_out": left_out} if no_game_data else {}), "details": details, "ground_truth_coverage": { "with_validation": with_validation, @@ -1833,6 +1853,8 @@ def print_platform_result( # them missing, as an offline build leaves them out. shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "") print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}") + for line in result.get("left_out") or []: + print(f" {line}") fallbacks = result.get("region_fallbacks") or [] if fallbacks: from packreadme import _group_label @@ -2539,6 +2561,7 @@ def _refuse_listing_narrowings( ("--verbose", args.verbose), ("--destinations", args.destinations), ("--offline", args.offline), + ("--no-game-data", getattr(args, "no_game_data", False)), ): if on: parser.error(f"{flag} is incompatible with {listing}") @@ -2562,6 +2585,9 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser "--destinations requires --platform or --all"), (args.offline and not (args.platform or args.all), "--offline requires --platform or --all"), + # Game data is categorized against a platform's list and its cores. + (getattr(args, "no_game_data", False) and not (args.platform or args.all), + "--no-game-data requires --platform or --all"), (args.destinations and getattr(args, "region", None), "--destinations describes the full pack and is incompatible with --region"), ): @@ -2622,6 +2648,11 @@ def _main(held: contextlib.ExitStack) -> None: action="store_true", help="List available targets for the platform", ) + parser.add_argument( + "--no-game-data", action="store_true", + help="Report on the pack built with --no-game-data: game data left " + "out and named, firmware only", + ) parser.add_argument( "--db", default=DEFAULT_DB, help="database file to read (default: %(default)s)", @@ -2783,6 +2814,7 @@ def _main(held: contextlib.ExitStack) -> None: data_dir_registry=data_registry, regions=requested_regions, system_filter=platform_systems, + no_game_data=getattr(args, "no_game_data", False), ) names = [ load_platform_config(p, args.platforms_dir).get("platform", p) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 566d9c13..c744c2e5 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -6003,6 +6003,7 @@ struct BurnDriver BurnDrvneogeo = { {"target_name": "switch"}, {"one_per_slot": True}, {"required_only": True}, + {"no_game_data": True}, ] for kwargs in dimensions: applied = _narrowings( @@ -6011,6 +6012,7 @@ struct BurnDriver BurnDrvneogeo = { kwargs.get("target_name"), kwargs.get("one_per_slot", False), kwargs.get("required_only", False), + no_game_data=kwargs.get("no_game_data", False), ) self.assertEqual(len(applied), 1, kwargs) tag, label = applied[0] @@ -6136,6 +6138,19 @@ struct BurnDriver BurnDrvneogeo = { # Documented as adding core files, read by nothing. (["--emulator", "handy"], ["--include-extras"], "refuse"), (["--system", "atari-lynx"], ["--include-extras"], "refuse"), + # Game data is categorized against a platform's list and the + # cores it runs: the modes without a platform refuse it, and + # --all-variants builds its own fixed set. + (["--emulator", "handy"], ["--no-game-data"], "refuse"), + (["--system", "atari-lynx"], ["--no-game-data"], "refuse"), + (["--platform", "retroarch", "--from-md5", "d8f1"], ["--no-game-data"], + "refuse"), + (["--from-md5", "d8f1"], ["--no-game-data"], "refuse"), + (["--manifest-targets"], ["--no-game-data"], "refuse"), + (["--platform", "retroarch", "--all-variants"], ["--no-game-data"], + "refuse"), + (["--list-systems"], ["--no-game-data"], "refuse"), + (["--list"], ["--no-game-data"], "refuse"), ] for mode, flag, expected in matrix: with self.subTest(mode=mode, flag=flag): diff --git a/tests/test_verify_listing.py b/tests/test_verify_listing.py index 893ccec5..e58b0292 100644 --- a/tests/test_verify_listing.py +++ b/tests/test_verify_listing.py @@ -44,6 +44,10 @@ class VerifyRefusesWhatAModeDoesNotRead(unittest.TestCase): (("--platform", "retroarch"), "--include-archived"), (("--emulator", "handy"), "--include-archived"), (("--platform", "retroarch"), "--standalone"), + # Game data is categorized against a platform's list and its cores. + (("--emulator", "handy"), "--no-game-data"), + (("--system", "atari-lynx"), "--no-game-data"), + (("--list-systems",), "--no-game-data"), ) def test_each_combination_is_refused(self): From 48a9a6181962fb81b6e3a7ad15bc0db1e31fe359 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 06:19:10 +0200 Subject: [PATCH 2/3] feat: let the installer skip game data install.py --no-game-data leaves out the files and omissions the manifest marks game data, in an install, --check and --dry-run, and names them with their category. The --events summary gains left_out, one group per category with every destination. A list without categories is refused. --- install.ps1 | 2 +- install.py | 92 +++++++++- install.sh | 2 +- tests/test_install.py | 6 +- tests/test_no_game_data.py | 355 +++++++++++++++++++++++++++++++++++++ tests/test_port.py | 7 +- wiki/installer.md | 23 ++- 7 files changed, 472 insertions(+), 15 deletions(-) create mode 100644 tests/test_no_game_data.py diff --git a/install.ps1 b/install.ps1 index 83cb19d8..abe19638 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa" + $defaultInstallSha256 = "6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index af1c5cf2..df45a790 100644 --- a/install.py +++ b/install.py @@ -106,6 +106,10 @@ OMISSION_REASONS = { "no_platform_slug": "no platform folder", "below_platform_folder": "below the platform folder", } +# What a file is when it is not firmware, as the manifest's category field +# says, and as the installer words it. Firmware carries no category. +GAME_DATA = "game_data" +CATEGORY_LABELS = {GAME_DATA: "game data"} _SHA1_RE = re.compile(r"^[0-9a-fA-F]{40}$") _SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") @@ -479,9 +483,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { "downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total", "bytes_done", "bytes_total"), # failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files", - # "required"}], the files the collection cannot supply. + # "required"}], the files the collection cannot supply. left_out: + # [{"category", "label", "files", "dests"}], what --no-game-data left + # out, empty without it. "summary": ("mode", "installed", "up_to_date", "pending", "failed", "skipped", - "interrupted"), + "left_out", "interrupted"), "end": ("code",), } # Seconds between two progress events while bytes arrive. @@ -578,6 +584,49 @@ def _emit_check( ) +def leave_out_game_data( + files: list[dict], omitted: list[dict] +) -> tuple[list[dict], list[dict], list[dict]]: + """The files and omissions without game data, and what was left out. + + The same cut as a pack built with --no-game-data: the manifest marks + each file that is a game's data, not firmware. + """ + left_out = [e for e in files + omitted if e.get("category") == GAME_DATA] + return ( + [e for e in files if e.get("category") != GAME_DATA], + [e for e in omitted if e.get("category") != GAME_DATA], + left_out, + ) + + +def _left_out_lines(left_out: list[dict], every: bool) -> list[str]: + """What --no-game-data left out, named per category, ten each unless every.""" + lines = [] + for group in _left_out_events(left_out): + dests = group["dests"] + shown = dests if every else dests[:10] + more = (f" ... and {len(dests) - len(shown)} more (--verbose lists them all)" + if len(shown) < len(dests) else "") + lines.append(f" Left out as {group['label']} (--no-game-data): " + f"{_plural(len(dests), 'file')}: {', '.join(shown)}{more}") + return lines + + +def _left_out_events(left_out: list[dict]) -> list[dict]: + """What --no-game-data left out, per category, every file named.""" + groups: dict[str, dict] = {} + for entry in left_out: + category = entry.get("category", "") + group = groups.setdefault(category, { + "category": category, "label": CATEGORY_LABELS.get(category, category), + "files": 0, "dests": [], + }) + group["files"] += 1 + group["dests"].append(entry["dest"]) + return [groups[category] for category in sorted(groups)] + + def _skipped(omitted: list[dict]) -> list[dict]: """The files the collection cannot supply, grouped by reason.""" groups: dict[str, dict] = {} @@ -3812,6 +3861,7 @@ class Invocation: for flag, on in ( ("--every-board", args.every_board), ("--installed-cores", args.installed_cores and not args.cores_dir), + ("--no-game-data", args.no_game_data), ("--standalone-copies", args.standalone_copies), ("--check", args.check), ("--dry-run", args.dry_run), @@ -5332,6 +5382,12 @@ def main() -> None: help="preferred regions, best first, e.g. us,eu,jp: keep one BIOS per " "region where a system has several", ) + parser.add_argument( + "--no-game-data", + action="store_true", + help="leave out game data (files a game reads, not firmware) and " + "install the firmware only", + ) parser.add_argument( "--list-systems", action="store_true", @@ -5410,6 +5466,9 @@ def main() -> None: parser.error("--quiet applies to an install, not to --check, --dry-run or a listing") if args.every_board and args.target: parser.error("--every-board installs every board; --target names one") + if args.no_game_data and (args.list_platforms or args.list_targets): + parser.error("--no-game-data narrows a file list, not --list-platforms " + "or --list-targets") if args.events: # The events own stdout. The text a person reads goes to stderr, so a # program never has to tell one from the other. @@ -5725,6 +5784,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: total_errors = 0 total_omitted = 0 omitted_seen: list[dict] = [] + left_out_seen: list[dict] = [] total_interrupted = False total_bytes = 0 total_seconds = 0.0 @@ -5806,6 +5866,23 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: f"case and holds another file under the same name ({shown})" ) + game_data_cut = 0 + if args.no_game_data: + if not manifest.get("categorized"): + # Without the field nothing is marked, and the run would + # install the game data it was asked to leave out. + print(f"Error: the {plat_name} file list records no file category, " + "which --no-game-data needs.", file=sys.stderr) + print(_older_list_advice(), file=sys.stderr) + sys.exit(1) + before = len(files) + files, omitted_files, left_out = leave_out_game_data(files, omitted_files) + game_data_cut = before - len(files) + left_out_seen.extend(left_out) + if left_out: + for line in _left_out_lines(left_out, args.verbose): + print(line) + if args.list_targets: targets = fetch_targets(plat_name) if targets is None: @@ -5912,12 +5989,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: if selection == INSTALLED_ONLY: files, omitted_files = installed print("\n Same choice without questions: " - f"--platform {plat_name} --installed-cores") + f"--platform {plat_name} --installed-cores" + + (" --no-game-data" if args.no_game_data else "")) _INVOCATION.chosen[plat_name] = [("--installed-cores", None)] selection = None elif selection is not None: print("\n Same choice without questions: " - + _same_choice_hint(plat_name, *selection)) + + _same_choice_hint(plat_name, *selection) + + (" --no-game-data" if args.no_game_data else "")) systems, cores, regions = selection _INVOCATION.chosen[plat_name] = [ *(("--system", name) for name in systems), @@ -5994,7 +6073,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: narrowed = bool( target or args.system or args.core or args.region or args.installed_cores - or len(files) != len(manifest.get("files", [])) + len( + or len(files) + game_data_cut != len(manifest.get("files", [])) + len( variants if case_kept else [] ) ) @@ -6173,7 +6252,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: up_to_date=total_verified if args.check or args.dry_run else total_up_to_date, pending=check_pending if args.check else files_pending, failed=[{"dest": dest, "reason": str(reason)} for dest, reason in failures], - skipped=_skipped(omitted_seen), interrupted=total_interrupted, + skipped=_skipped(omitted_seen), left_out=_left_out_events(left_out_seen), + interrupted=total_interrupted, ) # A check scripted as `--check && launch` reads the exit code only. if args.check and check_pending: diff --git a/install.sh b/install.sh index ba3ef375..6cfd570e 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa" +DEFAULT_INSTALL_SHA256="6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 67814009..c5016eb0 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3797,7 +3797,7 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase): args = SimpleNamespace( system=None, core=None, emulator=None, region=None, target=None, cores_dir=None, every_board=False, installed_cores=False, - standalone_copies=False, check=False, dry_run=False, + standalone_copies=False, check=False, dry_run=False, no_game_data=False, ) flags = list(argv) while flags: @@ -4966,7 +4966,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): installer = events[0]["installer"] self.assertEqual(events, self._head("check", installer) + [ {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 1, - "pending": 3, "failed": [], "skipped": self.SKIPPED, "interrupted": False}, + "pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False}, {"event": "end", "code": 1}, ]) self.assertIn("2/4 present (1 verified, 1 wrong hash)", proc.stderr) @@ -4989,7 +4989,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): {"event": "download", "platform": "retroarch", "files": 3, "bytes": 245}, *done, {"event": "summary", "mode": "install", "installed": 3, "up_to_date": 1, - "pending": 0, "failed": [], "skipped": self.SKIPPED, "interrupted": False}, + "pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False}, {"event": "end", "code": 0}, ]) self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"]) diff --git a/tests/test_no_game_data.py b/tests/test_no_game_data.py new file mode 100644 index 00000000..5e348c0b --- /dev/null +++ b/tests/test_no_game_data.py @@ -0,0 +1,355 @@ +"""--no-game-data: the firmware without the files a game reads. + +A platform list can name a game's data beside the firmware (EmuELEC lists the +Doom WADs), and a core profile marks its game data `category: game_data`. The +manifest marks both, the builder leaves both out under --no-game-data, verify +reports on the same file set, and the installer makes the same cut from the +full manifest. +""" + +from __future__ import annotations + +import contextlib +import functools +import hashlib +import http.server +import io +import json +import os +import subprocess +import sys +import tempfile +import threading +import unittest +import zipfile +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) +sys.path.insert(0, str(REPO_ROOT)) + +import common # noqa: E402 +import generate_db # noqa: E402 +import generate_pack as builder # noqa: E402 +import install # noqa: E402 +import packverify # noqa: E402 +import verify # noqa: E402 + +PAYLOADS = { + "System/firmware.rom": b"firmware of the console", + "System/corefw.bin": b"firmware the core loads", + "Doom/doom1.wad": b"shareware episode", + "Doom/engine.pk3": b"engine resources", +} + +PROFILES = { + "emu": { + "emulator": "Emu", "type": "libretro", "cores": ["emu"], + "systems": ["console"], + "files": [ + {"name": "firmware.rom", "required": True}, + {"name": "corefw.bin", "required": True}, + ], + }, + "doomcore": { + "emulator": "DoomCore", "type": "libretro", "cores": ["doomcore"], + "systems": ["doom"], + "files": [ + {"name": "doom1.wad", "path": "doom/doom1.wad", "category": "game_data", + "required": False}, + {"name": "engine.pk3", "category": "game_data", "required": True}, + ], + }, +} + +FIRMWARE = {"firmware.rom", "corefw.bin"} +GAME = {"doom/doom1.wad", "engine.pk3"} + + +class _Fixture(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.bios = self.root / "bios" + self.platforms = self.root / "platforms" + self.emulators = self.root / "emulators" + files = {} + for rel, data in PAYLOADS.items(): + path = self.bios / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + files[sha1] = { + "path": str(path), "name": path.name, "size": len(data), "sha1": sha1, + "md5": hashlib.md5(data).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), "crc32": "00000000", + } + self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})} + self.platforms.mkdir() + self.emulators.mkdir() + for key, profile in PROFILES.items(): + (self.emulators / f"{key}.yml").write_text(yaml.dump(profile)) + platform = { + "platform": "Handheld", "verification_mode": "existence", + "cores": ["emu", "doomcore"], + "systems": { + "console": {"files": [{"name": "firmware.rom", "destination": "firmware.rom"}]}, + "doom": {"files": [{"name": "doom1.wad", "destination": "doom/doom1.wad"}]}, + }, + } + (self.platforms / "handheld.yml").write_text(yaml.dump(platform)) + (self.platforms / "_registry.yml").write_text( + yaml.dump({"platforms": {"handheld": {"status": "active"}}}) + ) + common._platform_config_cache.clear() + common._emulator_profiles_cache.clear() + self.out = self.root / "dist" + self.out.mkdir() + + def tearDown(self): + common._platform_config_cache.clear() + common._emulator_profiles_cache.clear() + self._tmp.cleanup() + + def pack(self, no_game_data: bool) -> tuple[str, set[str], str]: + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + zip_path = builder.generate_pack( + "handheld", str(self.platforms), self.db, str(self.bios), str(self.out), + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + offline=True, no_game_data=no_game_data, + ) + with zipfile.ZipFile(zip_path) as archive: + members = set(archive.namelist()) - set(builder.PACK_DOCUMENTS) + return zip_path, members, printed.getvalue() + + def manifest(self, no_game_data: bool = False) -> dict: + return builder.generate_manifest( + "handheld", str(self.platforms), self.db, str(self.bios), + str(self.platforms / "_registry.yml"), emulators_dir=str(self.emulators), + emu_profiles=PROFILES, offline=True, no_game_data=no_game_data, + ) + + +class ManifestCategory(_Fixture): + def test_game_data_is_marked_and_firmware_is_not(self): + manifest = self.manifest() + self.assertIs(manifest["categorized"], True) + category = {entry["dest"]: entry.get("category") for entry in manifest["files"]} + self.assertEqual(set(category), FIRMWARE | GAME) + # A platform entry only game data reads, and a core's game data file. + self.assertEqual(category["doom/doom1.wad"], "game_data") + self.assertEqual(category["engine.pk3"], "game_data") + # Firmware carries no field at all. + for entry in manifest["files"]: + if entry["dest"] in FIRMWARE: + self.assertNotIn("category", entry) + + def test_a_narrowed_manifest_is_the_full_one_without_game_data(self): + full = self.manifest() + narrowed = self.manifest(no_game_data=True) + self.assertIs(narrowed["no_game_data"], True) + self.assertEqual( + {(e["dest"], e["sha1"]) for e in narrowed["files"]}, + {(e["dest"], e["sha1"]) for e in full["files"] if e.get("category") != "game_data"}, + ) + + def test_an_omitted_game_data_file_is_marked(self): + (self.bios / "Doom" / "engine.pk3").unlink() + files = {sha1: e for sha1, e in self.db["files"].items() if e["name"] != "engine.pk3"} + self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})} + manifest = self.manifest() + omitted = {e["dest"]: e.get("category") for e in manifest["omitted_files"]} + self.assertEqual(omitted.get("engine.pk3"), "game_data") + + +class PackNarrowing(_Fixture): + def test_the_pack_leaves_game_data_out_and_names_it(self): + _full_path, full, _ = self.pack(no_game_data=False) + zip_path, narrowed, printed = self.pack(no_game_data=True) + self.assertEqual(full, FIRMWARE | GAME) + self.assertEqual(narrowed, FIRMWARE) + self.assertIn("_NoGameData_BIOS_Pack.zip", os.path.basename(zip_path)) + with zipfile.ZipFile(zip_path) as archive: + readme = archive.read("README.txt").decode() + self.assertIn("PACK TYPE: Narrowed", readme) + self.assertIn("firmware only, game data left out", readme) + # Named, with the reason, never only counted. + self.assertIn("doom/doom1.wad (game data of doomcore)", printed) + self.assertIn("doomcore: doom1.wad, engine.pk3", printed) + + def test_verify_reports_the_pack_the_builder_wrote(self): + zip_path, members, _ = self.pack(no_game_data=True) + config = common.load_platform_config("handheld", str(self.platforms)) + result = verify.verify_platform( + config, self.db, str(self.emulators), PROFILES, no_game_data=True, + ) + self.assertEqual(set(result["pack_destinations"]), members) + self.assertTrue(any("doom/doom1.wad" in line for line in result["left_out"])) + checked = packverify.verify_pack_against_platform( + zip_path, "handheld", str(self.platforms), db=self.db, + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + no_game_data=True, + ) + self.assertTrue(checked[0], checked[3]) + # Held to the full list, the narrowed pack is short of the game data. + full = packverify.verify_pack_against_platform( + zip_path, "handheld", str(self.platforms), db=self.db, + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + ) + self.assertFalse(full[0]) + + def test_the_full_report_is_unchanged(self): + config = common.load_platform_config("handheld", str(self.platforms)) + result = verify.verify_platform(config, self.db, str(self.emulators), PROFILES) + self.assertNotIn("left_out", result) + self.assertEqual(set(result["pack_destinations"]), FIRMWARE | GAME) + + def test_split_parts_add_up_to_the_narrowed_pack(self): + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + parts = builder.generate_split_packs( + "handheld", str(self.platforms), self.db, str(self.bios), str(self.out), + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + offline=True, no_game_data=True, + ) + held: set[str] = set() + for part in parts: + self.assertIn("_NoGameData", part) + with zipfile.ZipFile(part) as archive: + held |= set(archive.namelist()) - set(builder.PACK_DOCUMENTS) + self.assertEqual(held, FIRMWARE) + + +class _Quiet(http.server.SimpleHTTPRequestHandler): + def log_message(self, *args) -> None: + return + + +class InstallerLeavesGameDataOut(unittest.TestCase): + """The installer cut, in each mode, from a manifest served over loopback.""" + + FILES = { + "bios/a.bin": (b"alpha" * 20, None), + "bios/game.wad": (b"wad" * 10, "game_data"), + } + + def _tempdir(self) -> Path: + handle = tempfile.TemporaryDirectory() + self.addCleanup(handle.cleanup) + return Path(handle.name) + + def serve(self, categorized: bool = True) -> None: + root = self._tempdir() + files = [] + for path, (data, category) in self.FILES.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(data) + entry = {"dest": Path(path).name, "repo_path": path, "size": len(data), + "sha1": hashlib.sha1(data).hexdigest(), "cores": None, + "systems": ["sys-a"]} + if category: + entry["category"] = category + files.append(entry) + manifest = { + "manifest_version": 2, "platform": "retroarch", "display_name": "RetroArch", + "files": files, + "omitted_files": [{"dest": "level.dat", "name": "level.dat", "system": "sys-a", + "required": True, "reason": "not_found", + "systems": ["sys-a"], "category": "game_data"}], + } + if categorized: + manifest["categorized"] = True + (root / "install").mkdir() + (root / "install" / "retroarch.json").write_text(json.dumps(manifest)) + handler = functools.partial(_Quiet, directory=str(root)) + httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + self.addCleanup(httpd.server_close) + self.addCleanup(httpd.shutdown) + home = self._tempdir() + self.bios = home / "bios" + self.bios.mkdir() + self.env = {**os.environ, "HOME": str(home), "RETROBIOS_OS": "linux", + "RETROBIOS_BASE_URL": f"http://127.0.0.1:{httpd.server_address[1]}"} + + def run_installer(self, *argv: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "install.py", "--platform", "retroarch", "--dest", + str(self.bios), "--no-input", "--jobs", "1", *argv], + cwd=REPO_ROOT, env=self.env, capture_output=True, text=True, timeout=120, + check=False, + ) + + def test_install_leaves_it_out_and_names_it(self): + self.serve() + proc = self.run_installer("--no-game-data") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue((self.bios / "a.bin").exists()) + self.assertFalse((self.bios / "game.wad").exists()) + self.assertIn("Left out as game data (--no-game-data): 2 files: game.wad, level.dat", + proc.stdout) + + def test_check_and_dry_run_leave_it_out(self): + self.serve() + for mode, listed in (("--check", "1 file to download:"), + ("--dry-run", "Would download 1 file")): + with self.subTest(mode=mode): + proc = self.run_installer("--no-game-data", mode) + self.assertIn( + "Left out as game data (--no-game-data): 2 files: game.wad, level.dat", + proc.stdout, + ) + self.assertIn(listed, proc.stdout) + self.assertNotIn("missing: game.wad", proc.stdout) + self.assertFalse((self.bios / "a.bin").exists()) + + def test_the_default_installs_everything(self): + self.serve() + proc = self.run_installer() + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue((self.bios / "game.wad").exists()) + self.assertNotIn("Left out as game data", proc.stdout) + + def test_events_name_what_was_left_out(self): + self.serve() + proc = self.run_installer("--no-game-data", "--check", "--events") + events = [json.loads(line) for line in proc.stdout.splitlines()] + summary = next(e for e in events if e["event"] == "summary") + self.assertEqual(list(summary), ["event", *install.EVENT_KEYS["summary"]]) + self.assertEqual(summary["left_out"], [{ + "category": "game_data", "label": "game data", "files": 2, + "dests": ["game.wad", "level.dat"], + }]) + # The omission is left out, not counted as a file the collection lacks. + self.assertEqual(summary["skipped"], []) + check = next(e for e in events if e["event"] == "check") + self.assertEqual(check["files"], 1) + + def test_a_list_without_categories_is_refused(self): + self.serve(categorized=False) + proc = self.run_installer("--no-game-data", "--check") + self.assertEqual(proc.returncode, 1) + self.assertIn("records no file category", proc.stderr) + + def test_listings_that_read_no_file_list_refuse_it(self): + proc = subprocess.run( + [sys.executable, "install.py", "--list-platforms", "--no-game-data"], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=60, check=False, + ) + self.assertEqual(proc.returncode, 2) + self.assertIn("--no-game-data", proc.stderr) + + def test_the_cut_function(self): + files = [{"dest": "a"}, {"dest": "b", "category": "game_data"}] + omitted = [{"dest": "c", "category": "game_data"}, {"dest": "d"}] + kept, kept_omitted, left = install.leave_out_game_data(files, omitted) + self.assertEqual([e["dest"] for e in kept], ["a"]) + self.assertEqual([e["dest"] for e in kept_omitted], ["d"]) + self.assertEqual([e["dest"] for e in left], ["b", "c"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_port.py b/tests/test_port.py index b752d1b0..60e43f42 100644 --- a/tests/test_port.py +++ b/tests/test_port.py @@ -141,7 +141,7 @@ class EventRecords(unittest.TestCase): json.dumps({"event": "summary", "mode": "install", "installed": 0, "up_to_date": 4, "pending": 1, "failed": [{"dest": "x.bin", "reason": "HTTP 404"}], - "skipped": [], "interrupted": False}), + "skipped": [], "left_out": [], "interrupted": False}), json.dumps({"event": "end", "code": 2}), ] out, summary = io.StringIO(), io.StringIO() @@ -228,7 +228,8 @@ if check: present=3 if done else 1, missing=0 if done else 1, wrong_hash=0 if done else 1, bytes_needed=0 if done else 300, data_missing=0, not_installable=1) emit("summary", mode="check", installed=0, up_to_date=3 if done else 1, - pending=0 if done else 2, failed=[], skipped=[], interrupted=False) + pending=0 if done else 2, failed=[], skipped=[], left_out=[], + interrupted=False) end(0 if done else 1) emit("download", platform="retroarch", files=2, bytes=300) for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1): @@ -237,7 +238,7 @@ for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1): (state / "installed").write_text("") emit("summary", mode="install", installed=2, up_to_date=1, pending=0, failed=[], skipped=[{"reason": "not_found", "label": "not found", "files": 1, "required": 1}], - interrupted=False) + left_out=[], interrupted=False) end(0) ''' diff --git a/wiki/installer.md b/wiki/installer.md index 06a32891..75521a80 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -170,8 +170,28 @@ python install.py --platform retroarch --list-systems python install.py --platform retroarch --system sony-playstation,nintendo-gba python install.py --platform retroarch --system sony-playstation --core pcsx_rearmed python install.py --platform recalbox --region us,eu,jp +python install.py --platform emuelec --no-game-data ``` +### Firmware without game data + +Some platform lists name the files a game reads beside the firmware its +emulator needs (the Doom WADs in EmuELEC's list), and the cores' profiles mark +the data files of the games they run `category: game_data`: MAME's software +lists, a game engine's own data. The manifest marks each of these files +`"category": "game_data"`, and firmware carries no category. + +`--no-game-data` leaves them out of the install, `--check` and `--dry-run`, +and names what it left out with its category (ten names, `--verbose` for all): + +```text + Left out as game data (--no-game-data): 2436 files: ports/doom/doom1.wad, ports/doom2/doom2.wad, ... +``` + +It is the cut `generate_pack.py --no-game-data` makes, and a test holds the +two to the same file list. A file list that records no category is refused +rather than read as having no game data. + ### Installed cores `--installed-cores` reads the cores a RetroArch already has and installs what @@ -242,6 +262,7 @@ what the options name, everything by default. | `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names | | `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included | | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | +| `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | | `--list-systems` | Print a platform's systems with the files and size each holds | | `--list-cores` | Print the cores whose extra files a platform installs | | `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | @@ -288,7 +309,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is | `download` | `platform`, `files`, `bytes` | Before the downloads start | | `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive | | `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file. `reason` says why one failed, and is empty otherwise | -| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply. Not written for a listing | +| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. Not written for a listing | | `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) | An option refused before the run starts (`install.py: error:`, code 2) writes From aed958ccd1f64ce2429ce7f06a5e0efa7c3a0f61 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 06:19:13 +0200 Subject: [PATCH 3/3] docs: document the game data filter --- scripts/generate_readme.py | 3 +++ scripts/generate_site.py | 5 ++++- wiki/advanced-usage.md | 33 +++++++++++++++++++++++++++++++++ wiki/testing-guide.md | 4 ++-- wiki/tools.md | 11 ++++++++++- wiki/verification-modes.md | 5 +++++ 6 files changed, 57 insertions(+), 4 deletions(-) diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index fd2ffd86..b0cd1e7a 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -617,6 +617,9 @@ def generate_readme(db: dict, platforms_dir: str) -> str: "python scripts/generate_pack.py --platform retroarch --region us,eu,jp", "python scripts/generate_pack.py --platform retroarch --region us --one-per-slot", "", + "# Firmware only: the game data a list or a core names is left out, each file named", + "python scripts/generate_pack.py --platform emuelec --no-game-data", + "", "# List available emulators and systems", "python scripts/generate_pack.py --list-emulators", "python scripts/generate_pack.py --list-systems", diff --git a/scripts/generate_site.py b/scripts/generate_site.py index fb08e885..8060ca1f 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -5318,7 +5318,10 @@ everything or to choose systems, cores and regions, and `--system`, `--core`, [installer reference](wiki/installer.md#choosing-what-to-install)). From a clone of the repository, `python scripts/generate_pack.py --platform retroarch --region us` builds the same narrowing as a pack and `--required-only` the bare -minimum each core needs to start, and `--help` lists every way to build your own. +minimum each core needs to start. `--no-game-data`, for the pack and the +installer alike, keeps the firmware and leaves out the files a game reads +([details](wiki/advanced-usage.md#firmware-without-game-data)), and `--help` +lists every way to build your own. --- diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md index f9374e5e..2780d730 100644 --- a/wiki/advanced-usage.md +++ b/wiki/advanced-usage.md @@ -235,6 +235,39 @@ visible. Variants compose with `--split`, `--target` and `--manifest`. +## Firmware without game data + +The firmware an emulator needs comes first. A platform list can also name the +files a game reads (EmuELEC lists the Doom WADs), and a core profile marks the +data of the games it runs `category: game_data`: MAME's software lists, the +RPG Maker run-time packages, a game engine's own files. `--no-game-data` +leaves both out and keeps the firmware: + +```bash +python scripts/generate_pack.py --platform emuelec --no-game-data +python scripts/generate_pack.py --platform emuelec --no-game-data --verify-packs +python scripts/verify.py --platform emuelec --no-game-data +``` + +A platform entry is game data when the only cores of the platform that read it +read it as game data (see [Verification modes](verification-modes.md) for how +entries are categorized). An entry that some core reads as firmware stays. +The build names every file it left out and why: the platform's entries with +the cores that read them as game data, then each core with the game data +files its profile declares. + +The pack is named `EmuELEC_4.8_NoGameData_BIOS_Pack.zip` and its README says +it was narrowed. `verify.py --no-game-data` reports on the same file set, and +`--verify-packs --no-game-data` checks the pack against it. The flag composes +with `--region`, `--target`, `--required-only`, `--source`, `--one-per-slot`, +`--split`, `--system` and `--manifest` +(`install/emuelec_nogamedata.json`). It needs `--platform` or `--all` and is +refused with `--emulator`, `--system` alone, `--from-md5` and +`--all-variants`: game data is categorized against a platform's list and the +cores it runs. The installer's `--no-game-data` makes the same cut from the +full manifest, where each game data file carries `"category": "game_data"`. + + ## Hardware target filtering ### What targets are diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 61aeecdf..4861e474 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -39,8 +39,8 @@ Every module in `tests/`, by the area it guards. |------|----------|---------| | Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` | | File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` | -| Verification, cross-reference and profile checks | synthetic + profiles | `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | -| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | +| Verification, cross-reference and profile checks | synthetic + profiles | `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_entry_categories`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | +| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` | diff --git a/wiki/tools.md b/wiki/tools.md index 29478e40..ce21306b 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -125,6 +125,7 @@ python scripts/verify.py --system atari-lynx # single system python scripts/verify.py --platform batocera --system psx # one system of a platform pack python scripts/verify.py --platform retroarch --target switch # filter by hardware python scripts/verify.py --platform recalbox --region us # regional priority list +python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build python scripts/verify.py --list-emulators # list all emulators @@ -153,7 +154,9 @@ Verification modes per platform: Full details and severity mapping: [verification modes](verification-modes.md). `--region` narrows the report to the file set a regional pack would carry, and a mode that cannot apply it refuses it rather than ignoring it: see -[region filtering](advanced-usage.md#region-filtering). +[region filtering](advanced-usage.md#region-filtering). `--no-game-data` narrows +it to the pack built with the same flag and names what it left out. It needs +`--platform` or `--all`. `--db`, `--platforms-dir` and `--emulators-dir` point the run at another database or source tree, which is how the tests drive it against fixtures. @@ -173,6 +176,7 @@ python scripts/generate_pack.py --system atari-lynx python scripts/generate_pack.py --platform retroarch --system sony-playstation python scripts/generate_pack.py --platform batocera --system psx python scripts/generate_pack.py --platform batocera --required-only +python scripts/generate_pack.py --platform emuelec --no-game-data python scripts/generate_pack.py --platform retroarch --split python scripts/generate_pack.py --platform retroarch --split --group-by manufacturer @@ -232,6 +236,11 @@ own verification mode ([discrepancy detection](verification-modes.md#discrepancy takes our ids only. - `--required-only`: exclude optional files and the data directories declared `required: false`, keep only required +- `--no-game-data`: firmware only. Leaves out the platform's entries + categorized game data and the cores' `category: game_data` files, and names + each one. Tags the pack `_NoGameData`. `verify.py --no-game-data` and + `--verify-packs --no-game-data` check the same file set + ([details](advanced-usage.md#firmware-without-game-data)) - `--split`: generate one ZIP per system instead of one big pack - `--split --group-by manufacturer`: group split packs by manufacturer (Sony, Nintendo, Sega...) - `--from-md5`: look up a hash in the database, or build a custom pack with diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index e81e3091..f888698d 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -299,6 +299,11 @@ pack errors. A core the platform names that has no profile yet makes nothing unr The same goes for a system none of the examined cores emulates: in both cases the absence of a reader proves nothing. +The same category filters: `--no-game-data` builds, verifies and installs the +firmware without the game data, and the install manifest marks each game data +file `"category": "game_data"` +([details](advanced-usage.md#firmware-without-game-data)). + ## File resolution chain