diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 1703638b..183deaba 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -20,6 +20,8 @@ on: - "scripts/deterministic_zip.py" - "scripts/dumpcatalog.py" - "scripts/exporter/baseline.py" + - "scripts/exporter/candidates.py" + - "scripts/gamedata.py" - "scripts/generate_pack.py" - "scripts/generate_readme.py" - "scripts/generate_site.py" diff --git a/install.ps1 b/install.ps1 index 83cb19d8..abe19638 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa" + $defaultInstallSha256 = "6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index af1c5cf2..df45a790 100644 --- a/install.py +++ b/install.py @@ -106,6 +106,10 @@ OMISSION_REASONS = { "no_platform_slug": "no platform folder", "below_platform_folder": "below the platform folder", } +# What a file is when it is not firmware, as the manifest's category field +# says, and as the installer words it. Firmware carries no category. +GAME_DATA = "game_data" +CATEGORY_LABELS = {GAME_DATA: "game data"} _SHA1_RE = re.compile(r"^[0-9a-fA-F]{40}$") _SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") @@ -479,9 +483,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { "downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total", "bytes_done", "bytes_total"), # failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files", - # "required"}], the files the collection cannot supply. + # "required"}], the files the collection cannot supply. left_out: + # [{"category", "label", "files", "dests"}], what --no-game-data left + # out, empty without it. "summary": ("mode", "installed", "up_to_date", "pending", "failed", "skipped", - "interrupted"), + "left_out", "interrupted"), "end": ("code",), } # Seconds between two progress events while bytes arrive. @@ -578,6 +584,49 @@ def _emit_check( ) +def leave_out_game_data( + files: list[dict], omitted: list[dict] +) -> tuple[list[dict], list[dict], list[dict]]: + """The files and omissions without game data, and what was left out. + + The same cut as a pack built with --no-game-data: the manifest marks + each file that is a game's data, not firmware. + """ + left_out = [e for e in files + omitted if e.get("category") == GAME_DATA] + return ( + [e for e in files if e.get("category") != GAME_DATA], + [e for e in omitted if e.get("category") != GAME_DATA], + left_out, + ) + + +def _left_out_lines(left_out: list[dict], every: bool) -> list[str]: + """What --no-game-data left out, named per category, ten each unless every.""" + lines = [] + for group in _left_out_events(left_out): + dests = group["dests"] + shown = dests if every else dests[:10] + more = (f" ... and {len(dests) - len(shown)} more (--verbose lists them all)" + if len(shown) < len(dests) else "") + lines.append(f" Left out as {group['label']} (--no-game-data): " + f"{_plural(len(dests), 'file')}: {', '.join(shown)}{more}") + return lines + + +def _left_out_events(left_out: list[dict]) -> list[dict]: + """What --no-game-data left out, per category, every file named.""" + groups: dict[str, dict] = {} + for entry in left_out: + category = entry.get("category", "") + group = groups.setdefault(category, { + "category": category, "label": CATEGORY_LABELS.get(category, category), + "files": 0, "dests": [], + }) + group["files"] += 1 + group["dests"].append(entry["dest"]) + return [groups[category] for category in sorted(groups)] + + def _skipped(omitted: list[dict]) -> list[dict]: """The files the collection cannot supply, grouped by reason.""" groups: dict[str, dict] = {} @@ -3812,6 +3861,7 @@ class Invocation: for flag, on in ( ("--every-board", args.every_board), ("--installed-cores", args.installed_cores and not args.cores_dir), + ("--no-game-data", args.no_game_data), ("--standalone-copies", args.standalone_copies), ("--check", args.check), ("--dry-run", args.dry_run), @@ -5332,6 +5382,12 @@ def main() -> None: help="preferred regions, best first, e.g. us,eu,jp: keep one BIOS per " "region where a system has several", ) + parser.add_argument( + "--no-game-data", + action="store_true", + help="leave out game data (files a game reads, not firmware) and " + "install the firmware only", + ) parser.add_argument( "--list-systems", action="store_true", @@ -5410,6 +5466,9 @@ def main() -> None: parser.error("--quiet applies to an install, not to --check, --dry-run or a listing") if args.every_board and args.target: parser.error("--every-board installs every board; --target names one") + if args.no_game_data and (args.list_platforms or args.list_targets): + parser.error("--no-game-data narrows a file list, not --list-platforms " + "or --list-targets") if args.events: # The events own stdout. The text a person reads goes to stderr, so a # program never has to tell one from the other. @@ -5725,6 +5784,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: total_errors = 0 total_omitted = 0 omitted_seen: list[dict] = [] + left_out_seen: list[dict] = [] total_interrupted = False total_bytes = 0 total_seconds = 0.0 @@ -5806,6 +5866,23 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: f"case and holds another file under the same name ({shown})" ) + game_data_cut = 0 + if args.no_game_data: + if not manifest.get("categorized"): + # Without the field nothing is marked, and the run would + # install the game data it was asked to leave out. + print(f"Error: the {plat_name} file list records no file category, " + "which --no-game-data needs.", file=sys.stderr) + print(_older_list_advice(), file=sys.stderr) + sys.exit(1) + before = len(files) + files, omitted_files, left_out = leave_out_game_data(files, omitted_files) + game_data_cut = before - len(files) + left_out_seen.extend(left_out) + if left_out: + for line in _left_out_lines(left_out, args.verbose): + print(line) + if args.list_targets: targets = fetch_targets(plat_name) if targets is None: @@ -5912,12 +5989,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: if selection == INSTALLED_ONLY: files, omitted_files = installed print("\n Same choice without questions: " - f"--platform {plat_name} --installed-cores") + f"--platform {plat_name} --installed-cores" + + (" --no-game-data" if args.no_game_data else "")) _INVOCATION.chosen[plat_name] = [("--installed-cores", None)] selection = None elif selection is not None: print("\n Same choice without questions: " - + _same_choice_hint(plat_name, *selection)) + + _same_choice_hint(plat_name, *selection) + + (" --no-game-data" if args.no_game_data else "")) systems, cores, regions = selection _INVOCATION.chosen[plat_name] = [ *(("--system", name) for name in systems), @@ -5994,7 +6073,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: narrowed = bool( target or args.system or args.core or args.region or args.installed_cores - or len(files) != len(manifest.get("files", [])) + len( + or len(files) + game_data_cut != len(manifest.get("files", [])) + len( variants if case_kept else [] ) ) @@ -6173,7 +6252,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: up_to_date=total_verified if args.check or args.dry_run else total_up_to_date, pending=check_pending if args.check else files_pending, failed=[{"dest": dest, "reason": str(reason)} for dest, reason in failures], - skipped=_skipped(omitted_seen), interrupted=total_interrupted, + skipped=_skipped(omitted_seen), left_out=_left_out_events(left_out_seen), + interrupted=total_interrupted, ) # A check scripted as `--check && launch` reads the exit code only. if args.check and check_pending: diff --git a/install.sh b/install.sh index ba3ef375..6cfd570e 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa" +DEFAULT_INSTALL_SHA256="6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index cca54788..1dd9e05f 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -44,6 +44,14 @@ "minLength": 1, "description": "Hardware target this manifest was filtered for, absent when unfiltered" }, + "categorized": { + "description": "Every game data file carries category: game_data. Without it the list records no category, and the installer refuses --no-game-data.", + "const": true + }, + "no_game_data": { + "description": "Built without game data, absent when the game data is listed", + "const": true + }, "platform": { "type": "string", "minLength": 1 @@ -170,6 +178,10 @@ "below_platform_folder" ] }, + "category": { + "description": "game_data: a game's own data, not firmware. Absent on firmware.", + "const": "game_data" + }, "held": { "description": "The pack builder holds this file and the installer cannot download it. Region narrowing counts it held, as the builder does, so it displaces the same fallback.", "const": true @@ -685,6 +697,10 @@ } } }, + "category": { + "description": "game_data: a game's own data, not firmware. Absent on firmware. The installer's --no-game-data leaves these out, as a pack built with --no-game-data does.", + "const": "game_data" + }, "executable": { "description": "The repository keeps the file executable, and the installer gives it the execute bits the umask allows.", "type": "boolean" diff --git a/scripts/gamedata.py b/scripts/gamedata.py new file mode 100644 index 00000000..2035c9a9 --- /dev/null +++ b/scripts/gamedata.py @@ -0,0 +1,159 @@ +"""Leaving a game's own data out of a pack, a manifest or a report. + +A platform list can name the files a game reads beside the firmware its +emulator needs, and a core profile marks the data files of the games it runs +`category: game_data`. `--no-game-data` keeps the firmware: the platform's +entries categorized as game data and the profiles' game data entries are left +out, and each one is named with the reason. + +Every caller cuts from the full platform config and the full profiles. The +category of a platform entry is read from the profiles' game data entries, so +cutting twice would find no evidence the second time. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from common import resolve_platform_cores +from exporter.candidates import GAME_DATA, EntryCategories + + +@dataclass(frozen=True) +class LeftOut: + """One game data file left out, and who reads it as game data.""" + + layer: str # "platform" (the platform's list) or "core" (a core profile) + system: str + name: str + readers: tuple[str, ...] + + def describe(self) -> str: + where = f"{self.system}/{self.name}" if self.system else self.name + if not self.readers: + return where + return f"{where} (game data of {', '.join(self.readers)})" + + +@dataclass +class GameDataCut: + """The config and profiles without game data, and what was left out.""" + + config: dict + profiles: dict[str, dict] + left_out: list[LeftOut] + + def report_lines(self) -> list[str]: + """What the cut left out, named, one line per layer. + + The platform layer names each entry and the cores that read it as + game data. The core layer names each core with the files its profile + declares `category: game_data`, held by the collection or not. + """ + lines = [] + platform = [item.describe() for item in self.left_out if item.layer == "platform"] + if platform: + lines.append( + f"Left out as game data, {len(platform)} entries of the platform's " + f"list: {', '.join(platform)}" + ) + by_core: dict[str, list[str]] = {} + for item in self.left_out: + if item.layer == "core": + names = by_core.setdefault(item.readers[0], []) + if item.name not in names: + names.append(item.name) + if by_core: + count = sum(len(names) for names in by_core.values()) + named = "; ".join( + f"{core}: {', '.join(names)}" for core, names in sorted(by_core.items()) + ) + lines.append( + f"Left out as game data, {count} files the cores declare " + f"category: game_data ({named})" + ) + return lines + + +def is_game_data(entry: dict) -> bool: + return entry.get("category") == GAME_DATA + + +def _entry_names(entry: dict) -> set[str]: + names = {str(entry.get("name") or "")} + names.update(str(a) for a in entry.get("aliases") or []) + path = str(entry.get("path") or "") + if path: + names.add(path.rstrip("/").rsplit("/", 1)[-1]) + return {n.casefold() for n in names if n} + + +def platform_game_data( + config: dict, profiles: dict[str, dict], data_registry: dict | None +) -> set[int]: + """The `id()` of every platform entry categorized as game data.""" + categories = EntryCategories(config, profiles, data_registry) + return { + id(fe) + for sys_id, system in (config.get("systems") or {}).items() + for fe in system.get("files") or [] + if isinstance(fe, dict) + and categories.category( + fe, str(fe.get("destination", fe.get("name", ""))), sys_id + ) == GAME_DATA + } + + +def without_game_data( + config: dict, + profiles: dict[str, dict], + data_registry: dict | None = None, + target_cores: set[str] | None = None, +) -> GameDataCut: + """The platform config and the profiles with every game data entry removed. + + The profiles lose their `category: game_data` files, so no core extra + of that category reaches the pack, the manifest or the report. The + platform keeps every entry that is not categorized game data. + """ + dropped = platform_game_data(config, profiles, data_registry) + examined = sorted(resolve_platform_cores(config, profiles, target_cores=target_cores)) + readers_by_name: dict[str, set[str]] = {} + for key in examined: + for fe in profiles[key].get("files") or []: + if isinstance(fe, dict) and is_game_data(fe): + for name in _entry_names(fe): + readers_by_name.setdefault(name, set()).add(key) + + left_out: list[LeftOut] = [] + systems: dict[str, dict] = {} + for sys_id, system in (config.get("systems") or {}).items(): + files = system.get("files") or [] + kept = [fe for fe in files if id(fe) not in dropped] + for fe in files: + if id(fe) in dropped: + name = str(fe.get("name") or "") + left_out.append(LeftOut( + "platform", sys_id, name, + tuple(sorted(readers_by_name.get(name.casefold(), ()))), + )) + systems[sys_id] = {**system, "files": kept} if len(kept) != len(files) else system + + for key in examined: + for fe in profiles[key].get("files") or []: + if isinstance(fe, dict) and is_game_data(fe): + left_out.append(LeftOut("core", "", str(fe.get("name") or ""), (key,))) + + stripped = { + key: ( + {**profile, "files": [ + fe for fe in profile.get("files") or [] + if not (isinstance(fe, dict) and is_game_data(fe)) + ]} + if any(isinstance(fe, dict) and is_game_data(fe) + for fe in profile.get("files") or []) + else profile + ) + for key, profile in profiles.items() + } + return GameDataCut({**config, "systems": systems}, stripped, left_out) diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index bb297cf5..dc3b3cc6 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -79,6 +79,7 @@ from manifestselect import SelectionIndex import slot as slot_mod import slots import split_pack +from gamedata import GAME_DATA, GameDataCut, is_game_data, platform_game_data, without_game_data from deterministic_zip import _FIXED_DATE_TIME, rebuild_zip_deterministic from largefiles import asset_name, load_registered_paths, published_asset_name from nativemode import ( @@ -193,6 +194,7 @@ def _narrowings( one_per_slot: bool, required_only: bool, standalone: bool = False, + no_game_data: bool = False, ) -> list[tuple[str, str]]: """Every dimension that narrows a pack, as (filename tag, plain label). @@ -223,6 +225,8 @@ def _narrowings( applied.append(("_Standalone", "standalone mode files only")) if required_only: applied.append(("_Required", "required files only")) + if no_game_data: + applied.append(("_NoGameData", "firmware only, game data left out")) return applied @@ -608,6 +612,22 @@ def _matches_an_alternative(fe: dict, local_path: str, profiles: dict) -> bool: return False +def _report_left_out(cut: GameDataCut, systems: set[str], cores: bool) -> None: + """Name the game data a --no-game-data build left out, and why. + + ``systems`` limits the platform layer to the systems this pack covers, + and ``cores`` says whether this pack collected the core files itself: + a split part does not, its run names them once. + """ + kept = [ + item for item in cut.left_out + if (item.layer == "platform" and item.system in systems) + or (item.layer == "core" and cores) + ] + for line in GameDataCut(cut.config, cut.profiles, kept).report_lines(): + print(f" {line}") + + def generate_pack( platform_name: str, platforms_dir: str, @@ -630,9 +650,13 @@ def generate_pack( one_per_slot: bool = False, offline: bool | None = None, pack_name: str | None = None, + no_game_data: bool = False, ) -> str | None: """Generate a ZIP pack for a platform. + ``no_game_data`` leaves out the platform's entries categorized as game + data and the cores' ``category: game_data`` files, and names each one. + ``pack_name`` replaces the derived file name: a split part is named after its group, and a name built from every system of a hundred-system group is longer than a file name can be. @@ -646,13 +670,22 @@ def generate_pack( # The pipeline always passes them, but a caller that does not must get the # same pack, not one built without validation, slots or agnostic cores. emu_profiles = load_emulator_profiles(emulators_dir) + # Cut from the full config and profiles: the category of a platform + # entry is read from the profiles' game data entries. + cut = ( + without_game_data(config, emu_profiles, data_registry, target_cores) + if no_game_data else None + ) + if cut is not None: + config, emu_profiles = cut.config, cut.profiles verification_mode = config.get("verification_mode", "existence") platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") narrowings = _narrowings( - source, regions, target_name, one_per_slot, required_only + source, regions, target_name, one_per_slot, required_only, + no_game_data=no_game_data, ) zip_path = os.path.join( output_dir, @@ -1164,6 +1197,12 @@ def generate_pack( zip_path, file_status, total_files, core_count, source, verification_mode, ) + if cut is not None: + _report_left_out( + cut, + systems=set(pack_systems) if source != "truth" else set(), + cores=source != "platform" and precomputed_extras is None, + ) for key, reason in sorted(file_reasons.items()): @@ -1884,6 +1923,7 @@ def generate_split_packs( target_name: str | None = None, one_per_slot: bool = False, offline: bool | None = None, + no_game_data: bool = False, ) -> list[str]: """Generate split packs (one ZIP per system or manufacturer).""" config = load_platform_config(platform_name, platforms_dir) @@ -1893,7 +1933,8 @@ def generate_split_packs( split_tags = "".join( tag for tag, _label in _narrowings( - source, regions, target_name, one_per_slot, required_only + source, regions, target_name, one_per_slot, required_only, + no_game_data=no_game_data, ) ) # Two groupings write different files. Without the tag they accumulate in @@ -1920,15 +1961,24 @@ def generate_split_packs( if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) base_dest = config.get("base_destination", "") + # Each part cuts the platform's own entries from the full profiles. The + # extras are collected once here, already without the cores' game data. + cut = ( + without_game_data(config, emu_profiles, data_registry, target_cores) + if no_game_data else None + ) if source == "platform": all_extras = [] elif emu_profiles: all_extras = _collect_emulator_extras( - config, emulators_dir, db, set(), base_dest, emu_profiles, + cut.config if cut else config, emulators_dir, db, set(), base_dest, + cut.profiles if cut else emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ) else: all_extras = [] + if cut is not None and source != "platform": + _report_left_out(cut, systems=set(), cores=True) results = [] for group_name, group_system_ids in sorted(groups.items()): group_extras = _extras_for_systems(all_extras, group_system_ids) @@ -1952,6 +2002,7 @@ def generate_split_packs( one_per_slot=one_per_slot, offline=offline, pack_name=f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip", + no_game_data=no_game_data, ) if zip_path: results.append(zip_path) @@ -1975,6 +2026,7 @@ def generate_split_packs( precomputed_extras=undistributed, extras_only=True, source=source, regions=regions, target_name=target_name, one_per_slot=one_per_slot, offline=offline, pack_name=f"{part_prefix}_Other_Cores_BIOS_Pack.zip", + no_game_data=no_game_data, ) if zip_path: results.append(zip_path) @@ -2411,6 +2463,12 @@ def _validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> if args.split: parser.error("--system is incompatible with --split") + # Game data is categorized against a platform's list and the cores it + # runs: a pack of one emulator or of a system's emulators has neither. + if getattr(args, "no_game_data", False) and not (has_platform or has_all): + parser.error("--no-game-data requires --platform or --all") + if getattr(args, "no_game_data", False) and has_from_md5: + parser.error("--no-game-data is incompatible with --from-md5") if getattr(args, "one_per_slot", False): if has_from_md5: parser.error("--one-per-slot and --from-md5 are mutually exclusive") @@ -2483,6 +2541,7 @@ def _run_manifest_mode( ), offline=args.offline, required_only=required_only, + no_game_data=getattr(args, "no_game_data", False), ) narrow_suffix = "".join( tag.lower() @@ -2492,6 +2551,7 @@ def _run_manifest_mode( _platform_target(representative, args.target, args.platforms_dir), False, required_only, + no_game_data=getattr(args, "no_game_data", False), ) ) out_path = os.path.join( @@ -2591,6 +2651,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None: verify_regions, args.target, ALL_VARIANTS if args.all_variants else (("full", False),), + no_game_data=getattr(args, "no_game_data", False), ) on_disk = ( set(os.listdir(args.output_dir)) if os.path.isdir(args.output_dir) else set() @@ -2641,6 +2702,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None: target_cores=tc, source=source, required_only=required_only, + no_game_data=getattr(args, "no_game_data", False), ) (ok, _checked, _present, errors, bl_checked, bl_present, core_checked, core_present, bl_excluded, bl_absent) = result @@ -2788,6 +2850,7 @@ def _run_platform_packs( target_name=named_target, one_per_slot=args.one_per_slot, offline=args.offline, + no_game_data=getattr(args, "no_game_data", False), ) print(f" Split into {len(zip_paths)} packs") else: @@ -2806,6 +2869,7 @@ def _run_platform_packs( named_target, args.one_per_slot, required_only, + no_game_data=getattr(args, "no_game_data", False), ) ) stem = _platform_pack_stem( @@ -2834,6 +2898,7 @@ def _run_platform_packs( one_per_slot=args.one_per_slot, offline=args.offline, pack_name=group_name, + no_game_data=getattr(args, "no_game_data", False), ) except (FileNotFoundError, OSError, yaml.YAMLError) as e: print(f" ERROR: {e}", file=sys.stderr) @@ -2856,6 +2921,7 @@ def _run_platform_packs( data_registry=data_registry, regions=getattr(args, "regions", None), target_name=args.target, + no_game_data=getattr(args, "no_game_data", False), ) if args.split: for entry in os.listdir(args.output_dir): @@ -2890,6 +2956,7 @@ def _refuse_for_all_variants(args: argparse.Namespace, parser: argparse.Argument for flag, given in ( ("--source", args.source is not None), ("--required-only", args.required_only), + ("--no-game-data", getattr(args, "no_game_data", False)), ("--emulator", args.emulator), ("--from-md5", args.from_md5 or args.from_md5_file), ): @@ -2978,7 +3045,10 @@ def _refuse_unapplied_flags(args: argparse.Namespace, parser: argparse.ArgumentP QUICK_MODE_FLAGS: dict[str, set[str]] = { # Checks packs already on disk against the platform's own list, narrowed # by region and target and by nothing else. - "--verify-packs": {"--platform", "--all", "--target", "--region", "--include-archived"}, + "--verify-packs": { + "--platform", "--all", "--target", "--region", "--include-archived", + "--no-game-data", + }, # One manifest per hardware target, from the target files themselves. "--manifest-targets": set(), "--list": set(), @@ -3005,6 +3075,7 @@ def _given_flags(args: argparse.Namespace) -> set[str]: "--from-md5": args.from_md5, "--from-md5-file": args.from_md5_file, "--required-only": args.required_only, + "--no-game-data": getattr(args, "no_game_data", False), "--source": args.source is not None, "--target": args.target, "--region": args.region, @@ -3107,6 +3178,12 @@ def _main(held: contextlib.ExitStack) -> None: action="store_true", help="Only include required files, skip optional", ) + parser.add_argument( + "--no-game-data", + action="store_true", + help="Leave out game data (the platform's entries a core reads as a " + "game's files, and the cores' game data files): firmware only", + ) parser.add_argument( "--source", choices=["platform", "truth", "full"], @@ -3860,6 +3937,10 @@ def _manifest_core_entries( pack_only_sizes.append(file_size) continue + if is_game_data(fe) and all( + is_game_data(claim) for claim in fe.get("other_claims", []) + ): + entry["category"] = GAME_DATA _attach_claims( entry, fe.get("other_claims", []), db, bios_dir, zip_contents, offline, required_only, validation_index, owners, dest, @@ -4027,12 +4108,17 @@ def generate_manifest( required_only: bool = False, data_registry: dict | None = None, fold_case: bool | None = None, + no_game_data: bool = False, ) -> dict: """Generate a JSON manifest for a platform (same resolution as generate_pack). Returns a dict ready for JSON serialization with file inventory, install hints, and download metadata. ``fold_case`` overrides the platform's ``case_insensitive_fs``, for the case variants below. + + A file that is a game's data, not firmware, carries ``"category": + "game_data"``, so the installer can leave it out as the builder does + under ``no_game_data``. Firmware carries no category. """ config = load_platform_config(platform_name, platforms_dir) if fold_case is not None: @@ -4041,6 +4127,15 @@ def generate_manifest( zip_contents = {} if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) + if data_registry is None: + data_registry = load_data_dir_registry(platforms_dir) + full_profiles = emu_profiles + game_entries: set[int] = set() + if no_game_data: + cut = without_game_data(config, emu_profiles, data_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles + else: + game_entries = platform_game_data(config, emu_profiles, data_registry) platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") @@ -4073,6 +4168,16 @@ def generate_manifest( platform_cores=plat_cores, ) + # A destination is game data when every declaration of it is: a firmware + # declaration of the same path keeps it in a pack built without game data. + declared_kinds: dict[str, set[bool]] = {} + for system in pack_systems.values(): + for fe in system.get("files", []): + bare = sanitize_pack_path(fe.get("destination", fe.get("name", ""))) + if bare: + declared_kinds.setdefault(bare, set()).add(id(fe) in game_entries) + game_destinations = {d for d, kinds in declared_kinds.items() if kinds == {True}} + seen_destinations: set[str] = set() # Destination -> (entry, full destination, manifest destination) of each # platform file, for the core claims a target build would fall back on. @@ -4090,8 +4195,6 @@ def generate_manifest( # collection does not index them, a data directory cache answered. pack_only_sizes: list[int] = [] rename_notes: set[str] = set() - if data_registry is None: - data_registry = load_data_dir_registry(platforms_dir) slot_overrides = slots.pack_overrides( config, emu_profiles, db, zip_contents, data_registry, target_cores=target_cores, source=source, @@ -4134,6 +4237,11 @@ def generate_manifest( # drops the same fallback. **selection.fields(bare, str(file_entry.get("name") or "")), **({"held": True} if bare in builder_held else {}), + **( + {"category": GAME_DATA} + if (bare in game_destinations if cores is None else is_game_data(file_entry)) + else {} + ), } # Read by the region grouping and by phase 2 with the same arguments. @@ -4299,6 +4407,8 @@ def generate_manifest( local_path, dest, None, db, repo_root, selection.fields(dest, file_entry["name"]), ) + if entry is not None and dest in game_destinations: + entry["category"] = GAME_DATA if entry is not None and target_cores is None: order = declaration_order(full_dest) if full_dest in slot_choices or len(order) > 1: @@ -4364,6 +4474,10 @@ def generate_manifest( if declared and declared != set(entry.get("systems") or []): entry["declared_in"] = sorted(declared) claims = baseline_claims.get(key, {}).get("other_claims") + if claims and any(not is_game_data(claim) for claim in claims): + # A core reads this path as firmware: a pack without game data + # still fills it. + entry.pop("category", None) if claims: _attach_claims( entry, claims, db, bios_dir, zip_contents, offline, required_only, @@ -4409,6 +4523,10 @@ def generate_manifest( # Absent on an unfiltered manifest rather than empty, to leave the # twelve default files byte-identical. **({"target": target_name} if target_name else {}), + # Every game data file below says so: the installer's --no-game-data + # relies on it, and refuses a list that does not record categories. + "categorized": True, + **({"no_game_data": True} if no_game_data else {}), "platform": platform_name, "display_name": platform_display, "version": "1.0", @@ -4448,9 +4566,9 @@ def generate_manifest( # or macOS too, stays folded. kept = generate_manifest( platform_name, platforms_dir, db, bios_dir, registry_path, - emulators_dir, zip_contents, emu_profiles, target_cores, source, + emulators_dir, zip_contents, full_profiles, target_cores, source, regions, target_name, offline, required_only, data_registry, - fold_case=False, + fold_case=False, no_game_data=no_game_data, ) folded = {entry["dest"] for entry in manifest_files} folded_lower = {dest.lower() for dest in folded} @@ -4624,6 +4742,7 @@ def _expected_pack_names( regions: list[str] | None, target_name: str | None, variants: tuple[tuple[str, bool], ...] = (("full", False),), + no_game_data: bool = False, ) -> dict[str, set[str]]: """Exact pack file names --verify-packs may check, per platform. @@ -4638,7 +4757,9 @@ def _expected_pack_names( return [ "".join( tag - for tag, _label in _narrowings(source, regions, named, False, required) + for tag, _label in _narrowings( + source, regions, named, False, required, no_game_data=no_game_data + ) ) for source, required in variants ] @@ -4767,6 +4888,7 @@ def verify_and_finalize_packs( regions: list[str] | None = None, target_name: str | None = None, emulators_dir: str = "emulators", + no_game_data: bool = False, ) -> bool: """Verify all packs, inject manifests, generate SHA256SUMS. @@ -4783,7 +4905,9 @@ def verify_and_finalize_packs( # substring: a _Custom or regional pack left by another run was judged # against this run's expectation and failed it. registered = list_registered_platforms(platforms_dir, include_archived=True) - expected = _expected_pack_names(registered, platforms_dir, regions, target_name) + expected = _expected_pack_names( + registered, platforms_dir, regions, target_name, no_game_data=no_game_data + ) pack_to_platform: dict[str, list[str]] = {} for name in _pack_archives(output_dir): for pname in registered: @@ -4837,6 +4961,7 @@ def verify_and_finalize_packs( regions=regions, data_registry=data_registry, target_cores=_target_cores_for(pname, target_name, platforms_dir), + no_game_data=no_game_data, ) status = "OK" if p_ok else "FAILED" exclusion_note = "".join( diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index fd2ffd86..b0cd1e7a 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -617,6 +617,9 @@ def generate_readme(db: dict, platforms_dir: str) -> str: "python scripts/generate_pack.py --platform retroarch --region us,eu,jp", "python scripts/generate_pack.py --platform retroarch --region us --one-per-slot", "", + "# Firmware only: the game data a list or a core names is left out, each file named", + "python scripts/generate_pack.py --platform emuelec --no-game-data", + "", "# List available emulators and systems", "python scripts/generate_pack.py --list-emulators", "python scripts/generate_pack.py --list-systems", diff --git a/scripts/generate_site.py b/scripts/generate_site.py index b474a54a..b333c9c5 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -5347,7 +5347,10 @@ everything or to choose systems, cores and regions, and `--system`, `--core`, [installer reference](wiki/installer.md#choosing-what-to-install)). From a clone of the repository, `python scripts/generate_pack.py --platform retroarch --region us` builds the same narrowing as a pack and `--required-only` the bare -minimum each core needs to start, and `--help` lists every way to build your own. +minimum each core needs to start. `--no-game-data`, for the pack and the +installer alike, keeps the firmware and leaves out the files a game reads +([details](wiki/advanced-usage.md#firmware-without-game-data)), and `--help` +lists every way to build your own. --- diff --git a/scripts/packverify.py b/scripts/packverify.py index d4aa29bf..c52f466b 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -32,6 +32,7 @@ from nativemode import reads_file_contents import region as region_mod from packresolve import resolve_file from exporter.candidates import BIOS, EntryCategories +from gamedata import without_game_data from common import resolve_platform_cores from common import sanitize_pack_path import zipfile @@ -634,6 +635,7 @@ def verify_pack_against_platform( target_cores: set[str] | None = None, source: str = "full", required_only: bool = False, + no_game_data: bool = False, ) -> tuple[bool, int, int, list[str], int, int, int, int, int, int]: """Verify a pack ZIP against its platform config and core requirements. @@ -644,6 +646,8 @@ def verify_pack_against_platform( ``required_only`` are the variant the pack was built as: a platform-only pack is held to the platform's list alone, a truth pack to the cores' files alone, and a required-only pack to the required entries of both. + ``no_game_data`` holds the pack to the firmware alone, cut as the + builder cuts it. Checks: 1. Every baseline file declared by the platform exists in the ZIP @@ -657,7 +661,7 @@ def verify_pack_against_platform( baseline_present, core_checked, core_present, baseline_excluded, baseline_absent)``. ``baseline_absent`` counts destinations the collection holds no file for that are a game's data or that no core the platform - runs opens; verify.py counts them apart from firmware. + runs opens. verify.py counts them apart from firmware. """ config = load_platform_config(platform_name, platforms_dir) @@ -668,6 +672,9 @@ def verify_pack_against_platform( if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) + if no_game_data: + cut = without_game_data(config, emu_profiles, data_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles expected_systems = config.get("systems", {}) if target_cores is not None: diff --git a/scripts/verify.py b/scripts/verify.py index 97f6d06e..4d856f08 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1121,14 +1121,33 @@ def verify_platform( data_dir_registry: dict | None = None, regions: list[str] | None = None, system_filter: list[str] | None = None, + no_game_data: bool = False, ) -> dict: """Verify all BIOS files for a platform, including cross-reference gaps. + ``no_game_data`` reports on the pack built with --no-game-data: the + game data cut as the builder cuts it, and named in ``left_out``. + A region priority list narrows the report to the files a pack built with the same list would carry, using the same selection function as the builder. ``system_filter`` does the same for a pack built with --system: the systems it keeps and the core extras those systems own. """ + left_out: list[str] = [] + if no_game_data: + from gamedata import without_game_data + + if emu_profiles is None: + emu_profiles = load_emulator_profiles(emulators_dir) + cut = without_game_data(config, emu_profiles, data_dir_registry, target_cores) + config, emu_profiles = cut.config, cut.profiles + systems = config.get("systems", {}) + in_scope = set(select_systems(systems, system_filter) if system_filter else systems) + cut.left_out = [ + item for item in cut.left_out + if item.layer == "core" or item.system in in_scope + ] + left_out = cut.report_lines() # Normalized once: a typo in a platform YAML must not verify with one # mode and be scored with another. mode = normalize_mode(config.get("verification_mode")) @@ -1527,6 +1546,7 @@ def verify_platform( "undeclared_files": undeclared, "exclusion_notes": exclusions, "region_fallbacks": sorted(region_fallbacks), + **({"left_out": left_out} if no_game_data else {}), "details": details, "ground_truth_coverage": { "with_validation": with_validation, @@ -1866,6 +1886,8 @@ def print_platform_result( # them missing, as an offline build leaves them out. shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "") print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}") + for line in result.get("left_out") or []: + print(f" {line}") fallbacks = result.get("region_fallbacks") or [] if fallbacks: from packreadme import _group_label @@ -2572,6 +2594,7 @@ def _refuse_listing_narrowings( ("--verbose", args.verbose), ("--destinations", args.destinations), ("--offline", args.offline), + ("--no-game-data", getattr(args, "no_game_data", False)), ): if on: parser.error(f"{flag} is incompatible with {listing}") @@ -2595,6 +2618,9 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser "--destinations requires --platform or --all"), (args.offline and not (args.platform or args.all), "--offline requires --platform or --all"), + # Game data is categorized against a platform's list and its cores. + (getattr(args, "no_game_data", False) and not (args.platform or args.all), + "--no-game-data requires --platform or --all"), (args.destinations and getattr(args, "region", None), "--destinations describes the full pack and is incompatible with --region"), ): @@ -2655,6 +2681,11 @@ def _main(held: contextlib.ExitStack) -> None: action="store_true", help="List available targets for the platform", ) + parser.add_argument( + "--no-game-data", action="store_true", + help="Report on the pack built with --no-game-data: game data left " + "out and named, firmware only", + ) parser.add_argument( "--db", default=DEFAULT_DB, help="database file to read (default: %(default)s)", @@ -2816,6 +2847,7 @@ def _main(held: contextlib.ExitStack) -> None: data_dir_registry=data_registry, regions=requested_regions, system_filter=platform_systems, + no_game_data=getattr(args, "no_game_data", False), ) names = [ load_platform_config(p, args.platforms_dir).get("platform", p) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 566d9c13..c744c2e5 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -6003,6 +6003,7 @@ struct BurnDriver BurnDrvneogeo = { {"target_name": "switch"}, {"one_per_slot": True}, {"required_only": True}, + {"no_game_data": True}, ] for kwargs in dimensions: applied = _narrowings( @@ -6011,6 +6012,7 @@ struct BurnDriver BurnDrvneogeo = { kwargs.get("target_name"), kwargs.get("one_per_slot", False), kwargs.get("required_only", False), + no_game_data=kwargs.get("no_game_data", False), ) self.assertEqual(len(applied), 1, kwargs) tag, label = applied[0] @@ -6136,6 +6138,19 @@ struct BurnDriver BurnDrvneogeo = { # Documented as adding core files, read by nothing. (["--emulator", "handy"], ["--include-extras"], "refuse"), (["--system", "atari-lynx"], ["--include-extras"], "refuse"), + # Game data is categorized against a platform's list and the + # cores it runs: the modes without a platform refuse it, and + # --all-variants builds its own fixed set. + (["--emulator", "handy"], ["--no-game-data"], "refuse"), + (["--system", "atari-lynx"], ["--no-game-data"], "refuse"), + (["--platform", "retroarch", "--from-md5", "d8f1"], ["--no-game-data"], + "refuse"), + (["--from-md5", "d8f1"], ["--no-game-data"], "refuse"), + (["--manifest-targets"], ["--no-game-data"], "refuse"), + (["--platform", "retroarch", "--all-variants"], ["--no-game-data"], + "refuse"), + (["--list-systems"], ["--no-game-data"], "refuse"), + (["--list"], ["--no-game-data"], "refuse"), ] for mode, flag, expected in matrix: with self.subTest(mode=mode, flag=flag): diff --git a/tests/test_install.py b/tests/test_install.py index 67814009..c5016eb0 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3797,7 +3797,7 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase): args = SimpleNamespace( system=None, core=None, emulator=None, region=None, target=None, cores_dir=None, every_board=False, installed_cores=False, - standalone_copies=False, check=False, dry_run=False, + standalone_copies=False, check=False, dry_run=False, no_game_data=False, ) flags = list(argv) while flags: @@ -4966,7 +4966,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): installer = events[0]["installer"] self.assertEqual(events, self._head("check", installer) + [ {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 1, - "pending": 3, "failed": [], "skipped": self.SKIPPED, "interrupted": False}, + "pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False}, {"event": "end", "code": 1}, ]) self.assertIn("2/4 present (1 verified, 1 wrong hash)", proc.stderr) @@ -4989,7 +4989,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): {"event": "download", "platform": "retroarch", "files": 3, "bytes": 245}, *done, {"event": "summary", "mode": "install", "installed": 3, "up_to_date": 1, - "pending": 0, "failed": [], "skipped": self.SKIPPED, "interrupted": False}, + "pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False}, {"event": "end", "code": 0}, ]) self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"]) diff --git a/tests/test_no_game_data.py b/tests/test_no_game_data.py new file mode 100644 index 00000000..5e348c0b --- /dev/null +++ b/tests/test_no_game_data.py @@ -0,0 +1,355 @@ +"""--no-game-data: the firmware without the files a game reads. + +A platform list can name a game's data beside the firmware (EmuELEC lists the +Doom WADs), and a core profile marks its game data `category: game_data`. The +manifest marks both, the builder leaves both out under --no-game-data, verify +reports on the same file set, and the installer makes the same cut from the +full manifest. +""" + +from __future__ import annotations + +import contextlib +import functools +import hashlib +import http.server +import io +import json +import os +import subprocess +import sys +import tempfile +import threading +import unittest +import zipfile +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) +sys.path.insert(0, str(REPO_ROOT)) + +import common # noqa: E402 +import generate_db # noqa: E402 +import generate_pack as builder # noqa: E402 +import install # noqa: E402 +import packverify # noqa: E402 +import verify # noqa: E402 + +PAYLOADS = { + "System/firmware.rom": b"firmware of the console", + "System/corefw.bin": b"firmware the core loads", + "Doom/doom1.wad": b"shareware episode", + "Doom/engine.pk3": b"engine resources", +} + +PROFILES = { + "emu": { + "emulator": "Emu", "type": "libretro", "cores": ["emu"], + "systems": ["console"], + "files": [ + {"name": "firmware.rom", "required": True}, + {"name": "corefw.bin", "required": True}, + ], + }, + "doomcore": { + "emulator": "DoomCore", "type": "libretro", "cores": ["doomcore"], + "systems": ["doom"], + "files": [ + {"name": "doom1.wad", "path": "doom/doom1.wad", "category": "game_data", + "required": False}, + {"name": "engine.pk3", "category": "game_data", "required": True}, + ], + }, +} + +FIRMWARE = {"firmware.rom", "corefw.bin"} +GAME = {"doom/doom1.wad", "engine.pk3"} + + +class _Fixture(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.bios = self.root / "bios" + self.platforms = self.root / "platforms" + self.emulators = self.root / "emulators" + files = {} + for rel, data in PAYLOADS.items(): + path = self.bios / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + files[sha1] = { + "path": str(path), "name": path.name, "size": len(data), "sha1": sha1, + "md5": hashlib.md5(data).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), "crc32": "00000000", + } + self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})} + self.platforms.mkdir() + self.emulators.mkdir() + for key, profile in PROFILES.items(): + (self.emulators / f"{key}.yml").write_text(yaml.dump(profile)) + platform = { + "platform": "Handheld", "verification_mode": "existence", + "cores": ["emu", "doomcore"], + "systems": { + "console": {"files": [{"name": "firmware.rom", "destination": "firmware.rom"}]}, + "doom": {"files": [{"name": "doom1.wad", "destination": "doom/doom1.wad"}]}, + }, + } + (self.platforms / "handheld.yml").write_text(yaml.dump(platform)) + (self.platforms / "_registry.yml").write_text( + yaml.dump({"platforms": {"handheld": {"status": "active"}}}) + ) + common._platform_config_cache.clear() + common._emulator_profiles_cache.clear() + self.out = self.root / "dist" + self.out.mkdir() + + def tearDown(self): + common._platform_config_cache.clear() + common._emulator_profiles_cache.clear() + self._tmp.cleanup() + + def pack(self, no_game_data: bool) -> tuple[str, set[str], str]: + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + zip_path = builder.generate_pack( + "handheld", str(self.platforms), self.db, str(self.bios), str(self.out), + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + offline=True, no_game_data=no_game_data, + ) + with zipfile.ZipFile(zip_path) as archive: + members = set(archive.namelist()) - set(builder.PACK_DOCUMENTS) + return zip_path, members, printed.getvalue() + + def manifest(self, no_game_data: bool = False) -> dict: + return builder.generate_manifest( + "handheld", str(self.platforms), self.db, str(self.bios), + str(self.platforms / "_registry.yml"), emulators_dir=str(self.emulators), + emu_profiles=PROFILES, offline=True, no_game_data=no_game_data, + ) + + +class ManifestCategory(_Fixture): + def test_game_data_is_marked_and_firmware_is_not(self): + manifest = self.manifest() + self.assertIs(manifest["categorized"], True) + category = {entry["dest"]: entry.get("category") for entry in manifest["files"]} + self.assertEqual(set(category), FIRMWARE | GAME) + # A platform entry only game data reads, and a core's game data file. + self.assertEqual(category["doom/doom1.wad"], "game_data") + self.assertEqual(category["engine.pk3"], "game_data") + # Firmware carries no field at all. + for entry in manifest["files"]: + if entry["dest"] in FIRMWARE: + self.assertNotIn("category", entry) + + def test_a_narrowed_manifest_is_the_full_one_without_game_data(self): + full = self.manifest() + narrowed = self.manifest(no_game_data=True) + self.assertIs(narrowed["no_game_data"], True) + self.assertEqual( + {(e["dest"], e["sha1"]) for e in narrowed["files"]}, + {(e["dest"], e["sha1"]) for e in full["files"] if e.get("category") != "game_data"}, + ) + + def test_an_omitted_game_data_file_is_marked(self): + (self.bios / "Doom" / "engine.pk3").unlink() + files = {sha1: e for sha1, e in self.db["files"].items() if e["name"] != "engine.pk3"} + self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})} + manifest = self.manifest() + omitted = {e["dest"]: e.get("category") for e in manifest["omitted_files"]} + self.assertEqual(omitted.get("engine.pk3"), "game_data") + + +class PackNarrowing(_Fixture): + def test_the_pack_leaves_game_data_out_and_names_it(self): + _full_path, full, _ = self.pack(no_game_data=False) + zip_path, narrowed, printed = self.pack(no_game_data=True) + self.assertEqual(full, FIRMWARE | GAME) + self.assertEqual(narrowed, FIRMWARE) + self.assertIn("_NoGameData_BIOS_Pack.zip", os.path.basename(zip_path)) + with zipfile.ZipFile(zip_path) as archive: + readme = archive.read("README.txt").decode() + self.assertIn("PACK TYPE: Narrowed", readme) + self.assertIn("firmware only, game data left out", readme) + # Named, with the reason, never only counted. + self.assertIn("doom/doom1.wad (game data of doomcore)", printed) + self.assertIn("doomcore: doom1.wad, engine.pk3", printed) + + def test_verify_reports_the_pack_the_builder_wrote(self): + zip_path, members, _ = self.pack(no_game_data=True) + config = common.load_platform_config("handheld", str(self.platforms)) + result = verify.verify_platform( + config, self.db, str(self.emulators), PROFILES, no_game_data=True, + ) + self.assertEqual(set(result["pack_destinations"]), members) + self.assertTrue(any("doom/doom1.wad" in line for line in result["left_out"])) + checked = packverify.verify_pack_against_platform( + zip_path, "handheld", str(self.platforms), db=self.db, + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + no_game_data=True, + ) + self.assertTrue(checked[0], checked[3]) + # Held to the full list, the narrowed pack is short of the game data. + full = packverify.verify_pack_against_platform( + zip_path, "handheld", str(self.platforms), db=self.db, + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + ) + self.assertFalse(full[0]) + + def test_the_full_report_is_unchanged(self): + config = common.load_platform_config("handheld", str(self.platforms)) + result = verify.verify_platform(config, self.db, str(self.emulators), PROFILES) + self.assertNotIn("left_out", result) + self.assertEqual(set(result["pack_destinations"]), FIRMWARE | GAME) + + def test_split_parts_add_up_to_the_narrowed_pack(self): + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + parts = builder.generate_split_packs( + "handheld", str(self.platforms), self.db, str(self.bios), str(self.out), + emulators_dir=str(self.emulators), emu_profiles=PROFILES, + offline=True, no_game_data=True, + ) + held: set[str] = set() + for part in parts: + self.assertIn("_NoGameData", part) + with zipfile.ZipFile(part) as archive: + held |= set(archive.namelist()) - set(builder.PACK_DOCUMENTS) + self.assertEqual(held, FIRMWARE) + + +class _Quiet(http.server.SimpleHTTPRequestHandler): + def log_message(self, *args) -> None: + return + + +class InstallerLeavesGameDataOut(unittest.TestCase): + """The installer cut, in each mode, from a manifest served over loopback.""" + + FILES = { + "bios/a.bin": (b"alpha" * 20, None), + "bios/game.wad": (b"wad" * 10, "game_data"), + } + + def _tempdir(self) -> Path: + handle = tempfile.TemporaryDirectory() + self.addCleanup(handle.cleanup) + return Path(handle.name) + + def serve(self, categorized: bool = True) -> None: + root = self._tempdir() + files = [] + for path, (data, category) in self.FILES.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(data) + entry = {"dest": Path(path).name, "repo_path": path, "size": len(data), + "sha1": hashlib.sha1(data).hexdigest(), "cores": None, + "systems": ["sys-a"]} + if category: + entry["category"] = category + files.append(entry) + manifest = { + "manifest_version": 2, "platform": "retroarch", "display_name": "RetroArch", + "files": files, + "omitted_files": [{"dest": "level.dat", "name": "level.dat", "system": "sys-a", + "required": True, "reason": "not_found", + "systems": ["sys-a"], "category": "game_data"}], + } + if categorized: + manifest["categorized"] = True + (root / "install").mkdir() + (root / "install" / "retroarch.json").write_text(json.dumps(manifest)) + handler = functools.partial(_Quiet, directory=str(root)) + httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + self.addCleanup(httpd.server_close) + self.addCleanup(httpd.shutdown) + home = self._tempdir() + self.bios = home / "bios" + self.bios.mkdir() + self.env = {**os.environ, "HOME": str(home), "RETROBIOS_OS": "linux", + "RETROBIOS_BASE_URL": f"http://127.0.0.1:{httpd.server_address[1]}"} + + def run_installer(self, *argv: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "install.py", "--platform", "retroarch", "--dest", + str(self.bios), "--no-input", "--jobs", "1", *argv], + cwd=REPO_ROOT, env=self.env, capture_output=True, text=True, timeout=120, + check=False, + ) + + def test_install_leaves_it_out_and_names_it(self): + self.serve() + proc = self.run_installer("--no-game-data") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue((self.bios / "a.bin").exists()) + self.assertFalse((self.bios / "game.wad").exists()) + self.assertIn("Left out as game data (--no-game-data): 2 files: game.wad, level.dat", + proc.stdout) + + def test_check_and_dry_run_leave_it_out(self): + self.serve() + for mode, listed in (("--check", "1 file to download:"), + ("--dry-run", "Would download 1 file")): + with self.subTest(mode=mode): + proc = self.run_installer("--no-game-data", mode) + self.assertIn( + "Left out as game data (--no-game-data): 2 files: game.wad, level.dat", + proc.stdout, + ) + self.assertIn(listed, proc.stdout) + self.assertNotIn("missing: game.wad", proc.stdout) + self.assertFalse((self.bios / "a.bin").exists()) + + def test_the_default_installs_everything(self): + self.serve() + proc = self.run_installer() + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue((self.bios / "game.wad").exists()) + self.assertNotIn("Left out as game data", proc.stdout) + + def test_events_name_what_was_left_out(self): + self.serve() + proc = self.run_installer("--no-game-data", "--check", "--events") + events = [json.loads(line) for line in proc.stdout.splitlines()] + summary = next(e for e in events if e["event"] == "summary") + self.assertEqual(list(summary), ["event", *install.EVENT_KEYS["summary"]]) + self.assertEqual(summary["left_out"], [{ + "category": "game_data", "label": "game data", "files": 2, + "dests": ["game.wad", "level.dat"], + }]) + # The omission is left out, not counted as a file the collection lacks. + self.assertEqual(summary["skipped"], []) + check = next(e for e in events if e["event"] == "check") + self.assertEqual(check["files"], 1) + + def test_a_list_without_categories_is_refused(self): + self.serve(categorized=False) + proc = self.run_installer("--no-game-data", "--check") + self.assertEqual(proc.returncode, 1) + self.assertIn("records no file category", proc.stderr) + + def test_listings_that_read_no_file_list_refuse_it(self): + proc = subprocess.run( + [sys.executable, "install.py", "--list-platforms", "--no-game-data"], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=60, check=False, + ) + self.assertEqual(proc.returncode, 2) + self.assertIn("--no-game-data", proc.stderr) + + def test_the_cut_function(self): + files = [{"dest": "a"}, {"dest": "b", "category": "game_data"}] + omitted = [{"dest": "c", "category": "game_data"}, {"dest": "d"}] + kept, kept_omitted, left = install.leave_out_game_data(files, omitted) + self.assertEqual([e["dest"] for e in kept], ["a"]) + self.assertEqual([e["dest"] for e in kept_omitted], ["d"]) + self.assertEqual([e["dest"] for e in left], ["b", "c"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_port.py b/tests/test_port.py index b752d1b0..60e43f42 100644 --- a/tests/test_port.py +++ b/tests/test_port.py @@ -141,7 +141,7 @@ class EventRecords(unittest.TestCase): json.dumps({"event": "summary", "mode": "install", "installed": 0, "up_to_date": 4, "pending": 1, "failed": [{"dest": "x.bin", "reason": "HTTP 404"}], - "skipped": [], "interrupted": False}), + "skipped": [], "left_out": [], "interrupted": False}), json.dumps({"event": "end", "code": 2}), ] out, summary = io.StringIO(), io.StringIO() @@ -228,7 +228,8 @@ if check: present=3 if done else 1, missing=0 if done else 1, wrong_hash=0 if done else 1, bytes_needed=0 if done else 300, data_missing=0, not_installable=1) emit("summary", mode="check", installed=0, up_to_date=3 if done else 1, - pending=0 if done else 2, failed=[], skipped=[], interrupted=False) + pending=0 if done else 2, failed=[], skipped=[], left_out=[], + interrupted=False) end(0 if done else 1) emit("download", platform="retroarch", files=2, bytes=300) for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1): @@ -237,7 +238,7 @@ for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1): (state / "installed").write_text("") emit("summary", mode="install", installed=2, up_to_date=1, pending=0, failed=[], skipped=[{"reason": "not_found", "label": "not found", "files": 1, "required": 1}], - interrupted=False) + left_out=[], interrupted=False) end(0) ''' diff --git a/tests/test_verify_listing.py b/tests/test_verify_listing.py index 893ccec5..e58b0292 100644 --- a/tests/test_verify_listing.py +++ b/tests/test_verify_listing.py @@ -44,6 +44,10 @@ class VerifyRefusesWhatAModeDoesNotRead(unittest.TestCase): (("--platform", "retroarch"), "--include-archived"), (("--emulator", "handy"), "--include-archived"), (("--platform", "retroarch"), "--standalone"), + # Game data is categorized against a platform's list and its cores. + (("--emulator", "handy"), "--no-game-data"), + (("--system", "atari-lynx"), "--no-game-data"), + (("--list-systems",), "--no-game-data"), ) def test_each_combination_is_refused(self): diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md index f9374e5e..2780d730 100644 --- a/wiki/advanced-usage.md +++ b/wiki/advanced-usage.md @@ -235,6 +235,39 @@ visible. Variants compose with `--split`, `--target` and `--manifest`. +## Firmware without game data + +The firmware an emulator needs comes first. A platform list can also name the +files a game reads (EmuELEC lists the Doom WADs), and a core profile marks the +data of the games it runs `category: game_data`: MAME's software lists, the +RPG Maker run-time packages, a game engine's own files. `--no-game-data` +leaves both out and keeps the firmware: + +```bash +python scripts/generate_pack.py --platform emuelec --no-game-data +python scripts/generate_pack.py --platform emuelec --no-game-data --verify-packs +python scripts/verify.py --platform emuelec --no-game-data +``` + +A platform entry is game data when the only cores of the platform that read it +read it as game data (see [Verification modes](verification-modes.md) for how +entries are categorized). An entry that some core reads as firmware stays. +The build names every file it left out and why: the platform's entries with +the cores that read them as game data, then each core with the game data +files its profile declares. + +The pack is named `EmuELEC_4.8_NoGameData_BIOS_Pack.zip` and its README says +it was narrowed. `verify.py --no-game-data` reports on the same file set, and +`--verify-packs --no-game-data` checks the pack against it. The flag composes +with `--region`, `--target`, `--required-only`, `--source`, `--one-per-slot`, +`--split`, `--system` and `--manifest` +(`install/emuelec_nogamedata.json`). It needs `--platform` or `--all` and is +refused with `--emulator`, `--system` alone, `--from-md5` and +`--all-variants`: game data is categorized against a platform's list and the +cores it runs. The installer's `--no-game-data` makes the same cut from the +full manifest, where each game data file carries `"category": "game_data"`. + + ## Hardware target filtering ### What targets are diff --git a/wiki/installer.md b/wiki/installer.md index 06a32891..75521a80 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -170,8 +170,28 @@ python install.py --platform retroarch --list-systems python install.py --platform retroarch --system sony-playstation,nintendo-gba python install.py --platform retroarch --system sony-playstation --core pcsx_rearmed python install.py --platform recalbox --region us,eu,jp +python install.py --platform emuelec --no-game-data ``` +### Firmware without game data + +Some platform lists name the files a game reads beside the firmware its +emulator needs (the Doom WADs in EmuELEC's list), and the cores' profiles mark +the data files of the games they run `category: game_data`: MAME's software +lists, a game engine's own data. The manifest marks each of these files +`"category": "game_data"`, and firmware carries no category. + +`--no-game-data` leaves them out of the install, `--check` and `--dry-run`, +and names what it left out with its category (ten names, `--verbose` for all): + +```text + Left out as game data (--no-game-data): 2436 files: ports/doom/doom1.wad, ports/doom2/doom2.wad, ... +``` + +It is the cut `generate_pack.py --no-game-data` makes, and a test holds the +two to the same file list. A file list that records no category is refused +rather than read as having no game data. + ### Installed cores `--installed-cores` reads the cores a RetroArch already has and installs what @@ -242,6 +262,7 @@ what the options name, everything by default. | `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names | | `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included | | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | +| `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | | `--list-systems` | Print a platform's systems with the files and size each holds | | `--list-cores` | Print the cores whose extra files a platform installs | | `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | @@ -288,7 +309,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is | `download` | `platform`, `files`, `bytes` | Before the downloads start | | `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive | | `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file. `reason` says why one failed, and is empty otherwise | -| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply. Not written for a listing | +| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. Not written for a listing | | `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) | An option refused before the run starts (`install.py: error:`, code 2) writes diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index d5bfa9c6..fcb8793f 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -40,7 +40,7 @@ Every module in `tests/`, by the area it guards. | Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` | | File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` | | Verification, cross-reference and profile checks | synthetic + profiles | `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | -| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | +| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` | diff --git a/wiki/tools.md b/wiki/tools.md index 29478e40..ce21306b 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -125,6 +125,7 @@ python scripts/verify.py --system atari-lynx # single system python scripts/verify.py --platform batocera --system psx # one system of a platform pack python scripts/verify.py --platform retroarch --target switch # filter by hardware python scripts/verify.py --platform recalbox --region us # regional priority list +python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build python scripts/verify.py --list-emulators # list all emulators @@ -153,7 +154,9 @@ Verification modes per platform: Full details and severity mapping: [verification modes](verification-modes.md). `--region` narrows the report to the file set a regional pack would carry, and a mode that cannot apply it refuses it rather than ignoring it: see -[region filtering](advanced-usage.md#region-filtering). +[region filtering](advanced-usage.md#region-filtering). `--no-game-data` narrows +it to the pack built with the same flag and names what it left out. It needs +`--platform` or `--all`. `--db`, `--platforms-dir` and `--emulators-dir` point the run at another database or source tree, which is how the tests drive it against fixtures. @@ -173,6 +176,7 @@ python scripts/generate_pack.py --system atari-lynx python scripts/generate_pack.py --platform retroarch --system sony-playstation python scripts/generate_pack.py --platform batocera --system psx python scripts/generate_pack.py --platform batocera --required-only +python scripts/generate_pack.py --platform emuelec --no-game-data python scripts/generate_pack.py --platform retroarch --split python scripts/generate_pack.py --platform retroarch --split --group-by manufacturer @@ -232,6 +236,11 @@ own verification mode ([discrepancy detection](verification-modes.md#discrepancy takes our ids only. - `--required-only`: exclude optional files and the data directories declared `required: false`, keep only required +- `--no-game-data`: firmware only. Leaves out the platform's entries + categorized game data and the cores' `category: game_data` files, and names + each one. Tags the pack `_NoGameData`. `verify.py --no-game-data` and + `--verify-packs --no-game-data` check the same file set + ([details](advanced-usage.md#firmware-without-game-data)) - `--split`: generate one ZIP per system instead of one big pack - `--split --group-by manufacturer`: group split packs by manufacturer (Sony, Nintendo, Sega...) - `--from-md5`: look up a hash in the database, or build a custom pack with diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index f433eacc..71bbd96a 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -306,6 +306,11 @@ pack errors. A core the platform names that has no profile yet makes nothing unr The same goes for a system none of the examined cores emulates: in both cases the absence of a reader proves nothing. +The same category filters: `--no-game-data` builds, verifies and installs the +firmware without the game data, and the install manifest marks each game data +file `"category": "game_data"` +([details](advanced-usage.md#firmware-without-game-data)). + ## File resolution chain