diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 1703638b..183deaba 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -20,6 +20,8 @@ on: - "scripts/deterministic_zip.py" - "scripts/dumpcatalog.py" - "scripts/exporter/baseline.py" + - "scripts/exporter/candidates.py" + - "scripts/gamedata.py" - "scripts/generate_pack.py" - "scripts/generate_readme.py" - "scripts/generate_site.py" diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index cca54788..1dd9e05f 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -44,6 +44,14 @@ "minLength": 1, "description": "Hardware target this manifest was filtered for, absent when unfiltered" }, + "categorized": { + "description": "Every game data file carries category: game_data. Without it the list records no category, and the installer refuses --no-game-data.", + "const": true + }, + "no_game_data": { + "description": "Built without game data, absent when the game data is listed", + "const": true + }, "platform": { "type": "string", "minLength": 1 @@ -170,6 +178,10 @@ "below_platform_folder" ] }, + "category": { + "description": "game_data: a game's own data, not firmware. Absent on firmware.", + "const": "game_data" + }, "held": { "description": "The pack builder holds this file and the installer cannot download it. Region narrowing counts it held, as the builder does, so it displaces the same fallback.", "const": true @@ -685,6 +697,10 @@ } } }, + "category": { + "description": "game_data: a game's own data, not firmware. Absent on firmware. The installer's --no-game-data leaves these out, as a pack built with --no-game-data does.", + "const": "game_data" + }, "executable": { "description": "The repository keeps the file executable, and the installer gives it the execute bits the umask allows.", "type": "boolean" diff --git a/scripts/gamedata.py b/scripts/gamedata.py new file mode 100644 index 00000000..2035c9a9 --- /dev/null +++ b/scripts/gamedata.py @@ -0,0 +1,159 @@ +"""Leaving a game's own data out of a pack, a manifest or a report. + +A platform list can name the files a game reads beside the firmware its +emulator needs, and a core profile marks the data files of the games it runs +`category: game_data`. `--no-game-data` keeps the firmware: the platform's +entries categorized as game data and the profiles' game data entries are left +out, and each one is named with the reason. + +Every caller cuts from the full platform config and the full profiles. The +category of a platform entry is read from the profiles' game data entries, so +cutting twice would find no evidence the second time. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from common import resolve_platform_cores +from exporter.candidates import GAME_DATA, EntryCategories + + +@dataclass(frozen=True) +class LeftOut: + """One game data file left out, and who reads it as game data.""" + + layer: str # "platform" (the platform's list) or "core" (a core profile) + system: str + name: str + readers: tuple[str, ...] + + def describe(self) -> str: + where = f"{self.system}/{self.name}" if self.system else self.name + if not self.readers: + return where + return f"{where} (game data of {', '.join(self.readers)})" + + +@dataclass +class GameDataCut: + """The config and profiles without game data, and what was left out.""" + + config: dict + profiles: dict[str, dict] + left_out: list[LeftOut] + + def report_lines(self) -> list[str]: + """What the cut left out, named, one line per layer. + + The platform layer names each entry and the cores that read it as + game data. The core layer names each core with the files its profile + declares `category: game_data`, held by the collection or not. + """ + lines = [] + platform = [item.describe() for item in self.left_out if item.layer == "platform"] + if platform: + lines.append( + f"Left out as game data, {len(platform)} entries of the platform's " + f"list: {', '.join(platform)}" + ) + by_core: dict[str, list[str]] = {} + for item in self.left_out: + if item.layer == "core": + names = by_core.setdefault(item.readers[0], []) + if item.name not in names: + names.append(item.name) + if by_core: + count = sum(len(names) for names in by_core.values()) + named = "; ".join( + f"{core}: {', '.join(names)}" for core, names in sorted(by_core.items()) + ) + lines.append( + f"Left out as game data, {count} files the cores declare " + f"category: game_data ({named})" + ) + return lines + + +def is_game_data(entry: dict) -> bool: + return entry.get("category") == GAME_DATA + + +def _entry_names(entry: dict) -> set[str]: + names = {str(entry.get("name") or "")} + names.update(str(a) for a in entry.get("aliases") or []) + path = str(entry.get("path") or "") + if path: + names.add(path.rstrip("/").rsplit("/", 1)[-1]) + return {n.casefold() for n in names if n} + + +def platform_game_data( + config: dict, profiles: dict[str, dict], data_registry: dict | None +) -> set[int]: + """The `id()` of every platform entry categorized as game data.""" + categories = EntryCategories(config, profiles, data_registry) + return { + id(fe) + for sys_id, system in (config.get("systems") or {}).items() + for fe in system.get("files") or [] + if isinstance(fe, dict) + and categories.category( + fe, str(fe.get("destination", fe.get("name", ""))), sys_id + ) == GAME_DATA + } + + +def without_game_data( + config: dict, + profiles: dict[str, dict], + data_registry: dict | None = None, + target_cores: set[str] | None = None, +) -> GameDataCut: + """The platform config and the profiles with every game data entry removed. + + The profiles lose their `category: game_data` files, so no core extra + of that category reaches the pack, the manifest or the report. The + platform keeps every entry that is not categorized game data. + """ + dropped = platform_game_data(config, profiles, data_registry) + examined = sorted(resolve_platform_cores(config, profiles, target_cores=target_cores)) + readers_by_name: dict[str, set[str]] = {} + for key in examined: + for fe in profiles[key].get("files") or []: + if isinstance(fe, dict) and is_game_data(fe): + for name in _entry_names(fe): + readers_by_name.setdefault(name, set()).add(key) + + left_out: list[LeftOut] = [] + systems: dict[str, dict] = {} + for sys_id, system in (config.get("systems") or {}).items(): + files = system.get("files") or [] + kept = [fe for fe in files if id(fe) not in dropped] + for fe in files: + if id(fe) in dropped: + name = str(fe.get("name") or "") + left_out.append(LeftOut( + "platform", sys_id, name, + tuple(sorted(readers_by_name.get(name.casefold(), ()))), + )) + systems[sys_id] = {**system, "files": kept} if len(kept) != len(files) else system + + for key in examined: + for fe in profiles[key].get("files") or []: + if isinstance(fe, dict) and is_game_data(fe): + left_out.append(LeftOut("core", "", str(fe.get("name") or ""), (key,))) + + stripped = { + key: ( + {**profile, "files": [ + fe for fe in profile.get("files") or [] + if not (isinstance(fe, dict) and is_game_data(fe)) + ]} + if any(isinstance(fe, dict) and is_game_data(fe) + for fe in profile.get("files") or []) + else profile + ) + for key, profile in profiles.items() + } + return GameDataCut({**config, "systems": systems}, stripped, left_out) diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index bb297cf5..dc3b3cc6 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -79,6 +79,7 @@ from manifestselect import SelectionIndex import slot as slot_mod import slots import split_pack +from gamedata import GAME_DATA, GameDataCut, is_game_data, platform_game_data, without_game_data from deterministic_zip import _FIXED_DATE_TIME, rebuild_zip_deterministic from largefiles import asset_name, load_registered_paths, published_asset_name from nativemode import ( @@ -193,6 +194,7 @@ def _narrowings( one_per_slot: bool, required_only: bool, standalone: bool = False, + no_game_data: bool = False, ) -> list[tuple[str, str]]: """Every dimension that narrows a pack, as (filename tag, plain label). @@ -223,6 +225,8 @@ def _narrowings( applied.append(("_Standalone", "standalone mode files only")) if required_only: applied.append(("_Required", "required files only")) + if no_game_data: + applied.append(("_NoGameData", "firmware only, game data left out")) return applied @@ -608,6 +612,22 @@ def _matches_an_alternative(fe: dict, local_path: str, profiles: dict) -> bool: return False +def _report_left_out(cut: GameDataCut, systems: set[str], cores: bool) -> None: + """Name the game data a --no-game-data build left out, and why. + + ``systems`` limits the platform layer to the systems this pack covers, + and ``cores`` says whether this pack collected the core files itself: + a split part does not, its run names them once. + """ + kept = [ + item for item in cut.left_out + if (item.layer == "platform" and item.system in systems) + or (item.layer == "core" and cores) + ] + for line in GameDataCut(cut.config, cut.profiles, kept).report_lines(): + print(f" {line}") + + def generate_pack( platform_name: str, platforms_dir: str, @@ -630,9 +650,13 @@ def generate_pack( one_per_slot: bool = False, offline: bool | None = None, pack_name: str | None = None, + no_game_data: bool = False, ) -> str | None: """Generate a ZIP pack for a platform. + ``no_game_data`` leaves out the platform's entries categorized as game + data and the cores' ``category: game_data`` files, and names each one. + ``pack_name`` replaces the derived file name: a split part is named after its group, and a name built from every system of a hundred-system group is longer than a file name can be. @@ -646,13 +670,22 @@ def generate_pack( # The pipeline always passes them, but a caller that does not must get the # same pack, not one built without validation, slots or agnostic cores. emu_profiles = load_emulator_profiles(emulators_dir) + # Cut from the full config and profiles: the category of a platform + # entry is read from the profiles' game data entries. + cut = ( + without_game_data(config, emu_profiles, data_registry, target_cores) + if no_game_data else None + ) + if cut is not None: + config, emu_profiles = cut.config, cut.profiles verification_mode = config.get("verification_mode", "existence") platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") narrowings = _narrowings( - source, regions, target_name, one_per_slot, required_only + source, regions, target_name, one_per_slot, required_only, + no_game_data=no_game_data, ) zip_path = os.path.join( output_dir, @@ -1164,6 +1197,12 @@ def generate_pack( zip_path, file_status, total_files, core_count, source, verification_mode, ) + if cut is not None: + _report_left_out( + cut, + systems=set(pack_systems) if source != "truth" else set(), + cores=source != "platform" and precomputed_extras is None, + ) for key, reason in sorted(file_reasons.items()): @@ -1884,6 +1923,7 @@ def generate_split_packs( target_name: str | None = None, one_per_slot: bool = False, offline: bool | None = None, + no_game_data: bool = False, ) -> list[str]: """Generate split packs (one ZIP per system or manufacturer).""" config = load_platform_config(platform_name, platforms_dir) @@ -1893,7 +1933,8 @@ def generate_split_packs( split_tags = "".join( tag for tag, _label in _narrowings( - source, regions, target_name, one_per_slot, required_only + source, regions, target_name, one_per_slot, required_only, + no_game_data=no_game_data, ) ) # Two groupings write different files. Without the tag they accumulate in @@ -1920,15 +1961,24 @@ def generate_split_packs( if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) base_dest = config.get("base_destination", "") + # Each part cuts the platform's own entries from the full profiles. The + # extras are collected once here, already without the cores' game data. + cut = ( + without_game_data(config, emu_profiles, data_registry, target_cores) + if no_game_data else None + ) if source == "platform": all_extras = [] elif emu_profiles: all_extras = _collect_emulator_extras( - config, emulators_dir, db, set(), base_dest, emu_profiles, + cut.config if cut else config, emulators_dir, db, set(), base_dest, + cut.profiles if cut else emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ) else: all_extras = [] + if cut is not None and source != "platform": + _report_left_out(cut, systems=set(), cores=True) results = [] for group_name, group_system_ids in sorted(groups.items()): group_extras = _extras_for_systems(all_extras, group_system_ids) @@ -1952,6 +2002,7 @@ def generate_split_packs( one_per_slot=one_per_slot, offline=offline, pack_name=f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip", + no_game_data=no_game_data, ) if zip_path: results.append(zip_path) @@ -1975,6 +2026,7 @@ def generate_split_packs( precomputed_extras=undistributed, extras_only=True, source=source, regions=regions, target_name=target_name, one_per_slot=one_per_slot, offline=offline, pack_name=f"{part_prefix}_Other_Cores_BIOS_Pack.zip", + no_game_data=no_game_data, ) if zip_path: results.append(zip_path) @@ -2411,6 +2463,12 @@ def _validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> if args.split: parser.error("--system is incompatible with --split") + # Game data is categorized against a platform's list and the cores it + # runs: a pack of one emulator or of a system's emulators has neither. + if getattr(args, "no_game_data", False) and not (has_platform or has_all): + parser.error("--no-game-data requires --platform or --all") + if getattr(args, "no_game_data", False) and has_from_md5: + parser.error("--no-game-data is incompatible with --from-md5") if getattr(args, "one_per_slot", False): if has_from_md5: parser.error("--one-per-slot and --from-md5 are mutually exclusive") @@ -2483,6 +2541,7 @@ def _run_manifest_mode( ), offline=args.offline, required_only=required_only, + no_game_data=getattr(args, "no_game_data", False), ) narrow_suffix = "".join( tag.lower() @@ -2492,6 +2551,7 @@ def _run_manifest_mode( _platform_target(representative, args.target, args.platforms_dir), False, required_only, + no_game_data=getattr(args, "no_game_data", False), ) ) out_path = os.path.join( @@ -2591,6 +2651,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None: verify_regions, args.target, ALL_VARIANTS if args.all_variants else (("full", False),), + no_game_data=getattr(args, "no_game_data", False), ) on_disk = ( set(os.listdir(args.output_dir)) if os.path.isdir(args.output_dir) else set() @@ -2641,6 +2702,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None: target_cores=tc, source=source, required_only=required_only, + no_game_data=getattr(args, "no_game_data", False), ) (ok, _checked, _present, errors, bl_checked, bl_present, core_checked, core_present, bl_excluded, bl_absent) = result @@ -2788,6 +2850,7 @@ def _run_platform_packs( target_name=named_target, one_per_slot=args.one_per_slot, offline=args.offline, + no_game_data=getattr(args, "no_game_data", False), ) print(f" Split into {len(zip_paths)} packs") else: @@ -2806,6 +2869,7 @@ def _run_platform_packs( named_target, args.one_per_slot, required_only, + no_game_data=getattr(args, "no_game_data", False), ) ) stem = _platform_pack_stem( @@ -2834,6 +2898,7 @@ def _run_platform_packs( one_per_slot=args.one_per_slot, offline=args.offline, pack_name=group_name, + no_game_data=getattr(args, "no_game_data", False), ) except (FileNotFoundError, OSError, yaml.YAMLError) as e: print(f" ERROR: {e}", file=sys.stderr) @@ -2856,6 +2921,7 @@ def _run_platform_packs( data_registry=data_registry, regions=getattr(args, "regions", None), target_name=args.target, + no_game_data=getattr(args, "no_game_data", False), ) if args.split: for entry in os.listdir(args.output_dir): @@ -2890,6 +2956,7 @@ def _refuse_for_all_variants(args: argparse.Namespace, parser: argparse.Argument for flag, given in ( ("--source", args.source is not None), ("--required-only", args.required_only), + ("--no-game-data", getattr(args, "no_game_data", False)), ("--emulator", args.emulator), ("--from-md5", args.from_md5 or args.from_md5_file), ): @@ -2978,7 +3045,10 @@ def _refuse_unapplied_flags(args: argparse.Namespace, parser: argparse.ArgumentP QUICK_MODE_FLAGS: dict[str, set[str]] = { # Checks packs already on disk against the platform's own list, narrowed # by region and target and by nothing else. - "--verify-packs": {"--platform", "--all", "--target", "--region", "--include-archived"}, + "--verify-packs": { + "--platform", "--all", "--target", "--region", "--include-archived", + "--no-game-data", + }, # One manifest per hardware target, from the target files themselves. "--manifest-targets": set(), "--list": set(), @@ -3005,6 +3075,7 @@ def _given_flags(args: argparse.Namespace) -> set[str]: "--from-md5": args.from_md5, "--from-md5-file": args.from_md5_file, "--required-only": args.required_only, + "--no-game-data": getattr(args, "no_game_data", False), "--source": args.source is not None, "--target": args.target, "--region": args.region, @@ -3107,6 +3178,12 @@ def _main(held: contextlib.ExitStack) -> None: action="store_true", help="Only include required files, skip optional", ) + parser.add_argument( + "--no-game-data", + action="store_true", + help="Leave out game data (the platform's entries a core reads as a " + "game's files, and the cores' game data files): firmware only", + ) parser.add_argument( "--source", choices=["platform", "truth", "full"], @@ -3860,6 +3937,10 @@ def _manifest_core_entries( pack_only_sizes.append(file_size) continue + if is_game_data(fe) and all( + is_game_data(claim) for claim in fe.get("other_claims", []) + ): + entry["category"] = GAME_DATA _attach_claims( entry, fe.get("other_claims", []), db, bios_dir, zip_contents, offline, required_only, validation_index, owners, dest, @@ -4027,12 +4108,17 @@ def generate_manifest( required_only: bool = False, data_registry: dict | None = None, fold_case: bool | None = None, + no_game_data: bool = False, ) -> dict: """Generate a JSON manifest for a platform (same resolution as generate_pack). Returns a dict ready for JSON serialization with file inventory, install hints, and download metadata. ``fold_case`` overrides the platform's ``case_insensitive_fs``, for the case variants below. + + A file that is a game's data, not firmware, carries ``"category": + "game_data"``, so the installer can leave it out as the builder does + under ``no_game_data``. Firmware carries no category. """ config = load_platform_config(platform_name, platforms_dir) if fold_case is not None: @@ -4041,6 +4127,15 @@ def generate_manifest( zip_contents = {} if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) + if data_registry is None: + data_registry = load_data_dir_registry(platforms_dir) + full_profiles = emu_profiles + game_entries: set[int] = set() + if no_game_data: + cut = without_game_data(config, emu_profiles, data_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles + else: + game_entries = platform_game_data(config, emu_profiles, data_registry) platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") @@ -4073,6 +4168,16 @@ def generate_manifest( platform_cores=plat_cores, ) + # A destination is game data when every declaration of it is: a firmware + # declaration of the same path keeps it in a pack built without game data. + declared_kinds: dict[str, set[bool]] = {} + for system in pack_systems.values(): + for fe in system.get("files", []): + bare = sanitize_pack_path(fe.get("destination", fe.get("name", ""))) + if bare: + declared_kinds.setdefault(bare, set()).add(id(fe) in game_entries) + game_destinations = {d for d, kinds in declared_kinds.items() if kinds == {True}} + seen_destinations: set[str] = set() # Destination -> (entry, full destination, manifest destination) of each # platform file, for the core claims a target build would fall back on. @@ -4090,8 +4195,6 @@ def generate_manifest( # collection does not index them, a data directory cache answered. pack_only_sizes: list[int] = [] rename_notes: set[str] = set() - if data_registry is None: - data_registry = load_data_dir_registry(platforms_dir) slot_overrides = slots.pack_overrides( config, emu_profiles, db, zip_contents, data_registry, target_cores=target_cores, source=source, @@ -4134,6 +4237,11 @@ def generate_manifest( # drops the same fallback. **selection.fields(bare, str(file_entry.get("name") or "")), **({"held": True} if bare in builder_held else {}), + **( + {"category": GAME_DATA} + if (bare in game_destinations if cores is None else is_game_data(file_entry)) + else {} + ), } # Read by the region grouping and by phase 2 with the same arguments. @@ -4299,6 +4407,8 @@ def generate_manifest( local_path, dest, None, db, repo_root, selection.fields(dest, file_entry["name"]), ) + if entry is not None and dest in game_destinations: + entry["category"] = GAME_DATA if entry is not None and target_cores is None: order = declaration_order(full_dest) if full_dest in slot_choices or len(order) > 1: @@ -4364,6 +4474,10 @@ def generate_manifest( if declared and declared != set(entry.get("systems") or []): entry["declared_in"] = sorted(declared) claims = baseline_claims.get(key, {}).get("other_claims") + if claims and any(not is_game_data(claim) for claim in claims): + # A core reads this path as firmware: a pack without game data + # still fills it. + entry.pop("category", None) if claims: _attach_claims( entry, claims, db, bios_dir, zip_contents, offline, required_only, @@ -4409,6 +4523,10 @@ def generate_manifest( # Absent on an unfiltered manifest rather than empty, to leave the # twelve default files byte-identical. **({"target": target_name} if target_name else {}), + # Every game data file below says so: the installer's --no-game-data + # relies on it, and refuses a list that does not record categories. + "categorized": True, + **({"no_game_data": True} if no_game_data else {}), "platform": platform_name, "display_name": platform_display, "version": "1.0", @@ -4448,9 +4566,9 @@ def generate_manifest( # or macOS too, stays folded. kept = generate_manifest( platform_name, platforms_dir, db, bios_dir, registry_path, - emulators_dir, zip_contents, emu_profiles, target_cores, source, + emulators_dir, zip_contents, full_profiles, target_cores, source, regions, target_name, offline, required_only, data_registry, - fold_case=False, + fold_case=False, no_game_data=no_game_data, ) folded = {entry["dest"] for entry in manifest_files} folded_lower = {dest.lower() for dest in folded} @@ -4624,6 +4742,7 @@ def _expected_pack_names( regions: list[str] | None, target_name: str | None, variants: tuple[tuple[str, bool], ...] = (("full", False),), + no_game_data: bool = False, ) -> dict[str, set[str]]: """Exact pack file names --verify-packs may check, per platform. @@ -4638,7 +4757,9 @@ def _expected_pack_names( return [ "".join( tag - for tag, _label in _narrowings(source, regions, named, False, required) + for tag, _label in _narrowings( + source, regions, named, False, required, no_game_data=no_game_data + ) ) for source, required in variants ] @@ -4767,6 +4888,7 @@ def verify_and_finalize_packs( regions: list[str] | None = None, target_name: str | None = None, emulators_dir: str = "emulators", + no_game_data: bool = False, ) -> bool: """Verify all packs, inject manifests, generate SHA256SUMS. @@ -4783,7 +4905,9 @@ def verify_and_finalize_packs( # substring: a _Custom or regional pack left by another run was judged # against this run's expectation and failed it. registered = list_registered_platforms(platforms_dir, include_archived=True) - expected = _expected_pack_names(registered, platforms_dir, regions, target_name) + expected = _expected_pack_names( + registered, platforms_dir, regions, target_name, no_game_data=no_game_data + ) pack_to_platform: dict[str, list[str]] = {} for name in _pack_archives(output_dir): for pname in registered: @@ -4837,6 +4961,7 @@ def verify_and_finalize_packs( regions=regions, data_registry=data_registry, target_cores=_target_cores_for(pname, target_name, platforms_dir), + no_game_data=no_game_data, ) status = "OK" if p_ok else "FAILED" exclusion_note = "".join( diff --git a/scripts/packverify.py b/scripts/packverify.py index d4aa29bf..c52f466b 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -32,6 +32,7 @@ from nativemode import reads_file_contents import region as region_mod from packresolve import resolve_file from exporter.candidates import BIOS, EntryCategories +from gamedata import without_game_data from common import resolve_platform_cores from common import sanitize_pack_path import zipfile @@ -634,6 +635,7 @@ def verify_pack_against_platform( target_cores: set[str] | None = None, source: str = "full", required_only: bool = False, + no_game_data: bool = False, ) -> tuple[bool, int, int, list[str], int, int, int, int, int, int]: """Verify a pack ZIP against its platform config and core requirements. @@ -644,6 +646,8 @@ def verify_pack_against_platform( ``required_only`` are the variant the pack was built as: a platform-only pack is held to the platform's list alone, a truth pack to the cores' files alone, and a required-only pack to the required entries of both. + ``no_game_data`` holds the pack to the firmware alone, cut as the + builder cuts it. Checks: 1. Every baseline file declared by the platform exists in the ZIP @@ -657,7 +661,7 @@ def verify_pack_against_platform( baseline_present, core_checked, core_present, baseline_excluded, baseline_absent)``. ``baseline_absent`` counts destinations the collection holds no file for that are a game's data or that no core the platform - runs opens; verify.py counts them apart from firmware. + runs opens. verify.py counts them apart from firmware. """ config = load_platform_config(platform_name, platforms_dir) @@ -668,6 +672,9 @@ def verify_pack_against_platform( if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) + if no_game_data: + cut = without_game_data(config, emu_profiles, data_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles expected_systems = config.get("systems", {}) if target_cores is not None: diff --git a/scripts/verify.py b/scripts/verify.py index 26187b60..6b6b654c 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1121,14 +1121,33 @@ def verify_platform( data_dir_registry: dict | None = None, regions: list[str] | None = None, system_filter: list[str] | None = None, + no_game_data: bool = False, ) -> dict: """Verify all BIOS files for a platform, including cross-reference gaps. + ``no_game_data`` reports on the pack built with --no-game-data: the + game data cut as the builder cuts it, and named in ``left_out``. + A region priority list narrows the report to the files a pack built with the same list would carry, using the same selection function as the builder. ``system_filter`` does the same for a pack built with --system: the systems it keeps and the core extras those systems own. """ + left_out: list[str] = [] + if no_game_data: + from gamedata import without_game_data + + if emu_profiles is None: + emu_profiles = load_emulator_profiles(emulators_dir) + cut = without_game_data(config, emu_profiles, data_dir_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles + systems = config.get("systems", {}) + in_scope = set(select_systems(systems, system_filter) if system_filter else systems) + cut.left_out = [ + item for item in cut.left_out + if item.layer == "core" or item.system in in_scope + ] + left_out = cut.report_lines() # Normalized once: a typo in a platform YAML must not verify with one # mode and be scored with another. mode = normalize_mode(config.get("verification_mode")) @@ -1524,6 +1543,7 @@ def verify_platform( "undeclared_files": undeclared, "exclusion_notes": exclusions, "region_fallbacks": sorted(region_fallbacks), + **({"left_out": left_out} if no_game_data else {}), "details": details, "ground_truth_coverage": { "with_validation": with_validation, @@ -1833,6 +1853,8 @@ def print_platform_result( # them missing, as an offline build leaves them out. shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "") print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}") + for line in result.get("left_out") or []: + print(f" {line}") fallbacks = result.get("region_fallbacks") or [] if fallbacks: from packreadme import _group_label @@ -2539,6 +2561,7 @@ def _refuse_listing_narrowings( ("--verbose", args.verbose), ("--destinations", args.destinations), ("--offline", args.offline), + ("--no-game-data", getattr(args, "no_game_data", False)), ): if on: parser.error(f"{flag} is incompatible with {listing}") @@ -2562,6 +2585,9 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser "--destinations requires --platform or --all"), (args.offline and not (args.platform or args.all), "--offline requires --platform or --all"), + # Game data is categorized against a platform's list and its cores. + (getattr(args, "no_game_data", False) and not (args.platform or args.all), + "--no-game-data requires --platform or --all"), (args.destinations and getattr(args, "region", None), "--destinations describes the full pack and is incompatible with --region"), ): @@ -2622,6 +2648,11 @@ def _main(held: contextlib.ExitStack) -> None: action="store_true", help="List available targets for the platform", ) + parser.add_argument( + "--no-game-data", action="store_true", + help="Report on the pack built with --no-game-data: game data left " + "out and named, firmware only", + ) parser.add_argument( "--db", default=DEFAULT_DB, help="database file to read (default: %(default)s)", @@ -2783,6 +2814,7 @@ def _main(held: contextlib.ExitStack) -> None: data_dir_registry=data_registry, regions=requested_regions, system_filter=platform_systems, + no_game_data=getattr(args, "no_game_data", False), ) names = [ load_platform_config(p, args.platforms_dir).get("platform", p) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 566d9c13..c744c2e5 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -6003,6 +6003,7 @@ struct BurnDriver BurnDrvneogeo = { {"target_name": "switch"}, {"one_per_slot": True}, {"required_only": True}, + {"no_game_data": True}, ] for kwargs in dimensions: applied = _narrowings( @@ -6011,6 +6012,7 @@ struct BurnDriver BurnDrvneogeo = { kwargs.get("target_name"), kwargs.get("one_per_slot", False), kwargs.get("required_only", False), + no_game_data=kwargs.get("no_game_data", False), ) self.assertEqual(len(applied), 1, kwargs) tag, label = applied[0] @@ -6136,6 +6138,19 @@ struct BurnDriver BurnDrvneogeo = { # Documented as adding core files, read by nothing. (["--emulator", "handy"], ["--include-extras"], "refuse"), (["--system", "atari-lynx"], ["--include-extras"], "refuse"), + # Game data is categorized against a platform's list and the + # cores it runs: the modes without a platform refuse it, and + # --all-variants builds its own fixed set. + (["--emulator", "handy"], ["--no-game-data"], "refuse"), + (["--system", "atari-lynx"], ["--no-game-data"], "refuse"), + (["--platform", "retroarch", "--from-md5", "d8f1"], ["--no-game-data"], + "refuse"), + (["--from-md5", "d8f1"], ["--no-game-data"], "refuse"), + (["--manifest-targets"], ["--no-game-data"], "refuse"), + (["--platform", "retroarch", "--all-variants"], ["--no-game-data"], + "refuse"), + (["--list-systems"], ["--no-game-data"], "refuse"), + (["--list"], ["--no-game-data"], "refuse"), ] for mode, flag, expected in matrix: with self.subTest(mode=mode, flag=flag): diff --git a/tests/test_verify_listing.py b/tests/test_verify_listing.py index 893ccec5..e58b0292 100644 --- a/tests/test_verify_listing.py +++ b/tests/test_verify_listing.py @@ -44,6 +44,10 @@ class VerifyRefusesWhatAModeDoesNotRead(unittest.TestCase): (("--platform", "retroarch"), "--include-archived"), (("--emulator", "handy"), "--include-archived"), (("--platform", "retroarch"), "--standalone"), + # Game data is categorized against a platform's list and its cores. + (("--emulator", "handy"), "--no-game-data"), + (("--system", "atari-lynx"), "--no-game-data"), + (("--list-systems",), "--no-game-data"), ) def test_each_combination_is_refused(self):