mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
chore: merge the game data filter
This commit is contained in:
commit
8ba65de1ca
21 files changed
+899
-29
No files matched your search
@@ -20,6 +20,8 @@ on:
|
||||
- "scripts/deterministic_zip.py"
|
||||
- "scripts/dumpcatalog.py"
|
||||
- "scripts/exporter/baseline.py"
|
||||
- "scripts/exporter/candidates.py"
|
||||
- "scripts/gamedata.py"
|
||||
- "scripts/generate_pack.py"
|
||||
- "scripts/generate_readme.py"
|
||||
- "scripts/generate_site.py"
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
$defaultInstallSha256 = "abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa"
|
||||
$defaultInstallSha256 = "6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a"
|
||||
$maximumInstallerBytes = 2MB
|
||||
$installerExit = 0
|
||||
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
|
||||
|
||||
+86
-6
@@ -106,6 +106,10 @@ OMISSION_REASONS = {
|
||||
"no_platform_slug": "no platform folder",
|
||||
"below_platform_folder": "below the platform folder",
|
||||
}
|
||||
# What a file is when it is not firmware, as the manifest's category field
|
||||
# says, and as the installer words it. Firmware carries no category.
|
||||
GAME_DATA = "game_data"
|
||||
CATEGORY_LABELS = {GAME_DATA: "game data"}
|
||||
_SHA1_RE = re.compile(r"^[0-9a-fA-F]{40}$")
|
||||
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
|
||||
|
||||
@@ -479,9 +483,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = {
|
||||
"downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total",
|
||||
"bytes_done", "bytes_total"),
|
||||
# failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files",
|
||||
# "required"}], the files the collection cannot supply.
|
||||
# "required"}], the files the collection cannot supply. left_out:
|
||||
# [{"category", "label", "files", "dests"}], what --no-game-data left
|
||||
# out, empty without it.
|
||||
"summary": ("mode", "installed", "up_to_date", "pending", "failed", "skipped",
|
||||
"interrupted"),
|
||||
"left_out", "interrupted"),
|
||||
"end": ("code",),
|
||||
}
|
||||
# Seconds between two progress events while bytes arrive.
|
||||
@@ -578,6 +584,49 @@ def _emit_check(
|
||||
)
|
||||
|
||||
|
||||
def leave_out_game_data(
|
||||
files: list[dict], omitted: list[dict]
|
||||
) -> tuple[list[dict], list[dict], list[dict]]:
|
||||
"""The files and omissions without game data, and what was left out.
|
||||
|
||||
The same cut as a pack built with --no-game-data: the manifest marks
|
||||
each file that is a game's data, not firmware.
|
||||
"""
|
||||
left_out = [e for e in files + omitted if e.get("category") == GAME_DATA]
|
||||
return (
|
||||
[e for e in files if e.get("category") != GAME_DATA],
|
||||
[e for e in omitted if e.get("category") != GAME_DATA],
|
||||
left_out,
|
||||
)
|
||||
|
||||
|
||||
def _left_out_lines(left_out: list[dict], every: bool) -> list[str]:
|
||||
"""What --no-game-data left out, named per category, ten each unless every."""
|
||||
lines = []
|
||||
for group in _left_out_events(left_out):
|
||||
dests = group["dests"]
|
||||
shown = dests if every else dests[:10]
|
||||
more = (f" ... and {len(dests) - len(shown)} more (--verbose lists them all)"
|
||||
if len(shown) < len(dests) else "")
|
||||
lines.append(f" Left out as {group['label']} (--no-game-data): "
|
||||
f"{_plural(len(dests), 'file')}: {', '.join(shown)}{more}")
|
||||
return lines
|
||||
|
||||
|
||||
def _left_out_events(left_out: list[dict]) -> list[dict]:
|
||||
"""What --no-game-data left out, per category, every file named."""
|
||||
groups: dict[str, dict] = {}
|
||||
for entry in left_out:
|
||||
category = entry.get("category", "")
|
||||
group = groups.setdefault(category, {
|
||||
"category": category, "label": CATEGORY_LABELS.get(category, category),
|
||||
"files": 0, "dests": [],
|
||||
})
|
||||
group["files"] += 1
|
||||
group["dests"].append(entry["dest"])
|
||||
return [groups[category] for category in sorted(groups)]
|
||||
|
||||
|
||||
def _skipped(omitted: list[dict]) -> list[dict]:
|
||||
"""The files the collection cannot supply, grouped by reason."""
|
||||
groups: dict[str, dict] = {}
|
||||
@@ -3812,6 +3861,7 @@ class Invocation:
|
||||
for flag, on in (
|
||||
("--every-board", args.every_board),
|
||||
("--installed-cores", args.installed_cores and not args.cores_dir),
|
||||
("--no-game-data", args.no_game_data),
|
||||
("--standalone-copies", args.standalone_copies),
|
||||
("--check", args.check),
|
||||
("--dry-run", args.dry_run),
|
||||
@@ -5332,6 +5382,12 @@ def main() -> None:
|
||||
help="preferred regions, best first, e.g. us,eu,jp: keep one BIOS per "
|
||||
"region where a system has several",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-game-data",
|
||||
action="store_true",
|
||||
help="leave out game data (files a game reads, not firmware) and "
|
||||
"install the firmware only",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list-systems",
|
||||
action="store_true",
|
||||
@@ -5410,6 +5466,9 @@ def main() -> None:
|
||||
parser.error("--quiet applies to an install, not to --check, --dry-run or a listing")
|
||||
if args.every_board and args.target:
|
||||
parser.error("--every-board installs every board; --target names one")
|
||||
if args.no_game_data and (args.list_platforms or args.list_targets):
|
||||
parser.error("--no-game-data narrows a file list, not --list-platforms "
|
||||
"or --list-targets")
|
||||
if args.events:
|
||||
# The events own stdout. The text a person reads goes to stderr, so a
|
||||
# program never has to tell one from the other.
|
||||
@@ -5725,6 +5784,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
total_errors = 0
|
||||
total_omitted = 0
|
||||
omitted_seen: list[dict] = []
|
||||
left_out_seen: list[dict] = []
|
||||
total_interrupted = False
|
||||
total_bytes = 0
|
||||
total_seconds = 0.0
|
||||
@@ -5806,6 +5866,23 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
f"case and holds another file under the same name ({shown})"
|
||||
)
|
||||
|
||||
game_data_cut = 0
|
||||
if args.no_game_data:
|
||||
if not manifest.get("categorized"):
|
||||
# Without the field nothing is marked, and the run would
|
||||
# install the game data it was asked to leave out.
|
||||
print(f"Error: the {plat_name} file list records no file category, "
|
||||
"which --no-game-data needs.", file=sys.stderr)
|
||||
print(_older_list_advice(), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
before = len(files)
|
||||
files, omitted_files, left_out = leave_out_game_data(files, omitted_files)
|
||||
game_data_cut = before - len(files)
|
||||
left_out_seen.extend(left_out)
|
||||
if left_out:
|
||||
for line in _left_out_lines(left_out, args.verbose):
|
||||
print(line)
|
||||
|
||||
if args.list_targets:
|
||||
targets = fetch_targets(plat_name)
|
||||
if targets is None:
|
||||
@@ -5912,12 +5989,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
if selection == INSTALLED_ONLY:
|
||||
files, omitted_files = installed
|
||||
print("\n Same choice without questions: "
|
||||
f"--platform {plat_name} --installed-cores")
|
||||
f"--platform {plat_name} --installed-cores"
|
||||
+ (" --no-game-data" if args.no_game_data else ""))
|
||||
_INVOCATION.chosen[plat_name] = [("--installed-cores", None)]
|
||||
selection = None
|
||||
elif selection is not None:
|
||||
print("\n Same choice without questions: "
|
||||
+ _same_choice_hint(plat_name, *selection))
|
||||
+ _same_choice_hint(plat_name, *selection)
|
||||
+ (" --no-game-data" if args.no_game_data else ""))
|
||||
systems, cores, regions = selection
|
||||
_INVOCATION.chosen[plat_name] = [
|
||||
*(("--system", name) for name in systems),
|
||||
@@ -5994,7 +6073,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
|
||||
narrowed = bool(
|
||||
target or args.system or args.core or args.region or args.installed_cores
|
||||
or len(files) != len(manifest.get("files", [])) + len(
|
||||
or len(files) + game_data_cut != len(manifest.get("files", [])) + len(
|
||||
variants if case_kept else []
|
||||
)
|
||||
)
|
||||
@@ -6173,7 +6252,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
up_to_date=total_verified if args.check or args.dry_run else total_up_to_date,
|
||||
pending=check_pending if args.check else files_pending,
|
||||
failed=[{"dest": dest, "reason": str(reason)} for dest, reason in failures],
|
||||
skipped=_skipped(omitted_seen), interrupted=total_interrupted,
|
||||
skipped=_skipped(omitted_seen), left_out=_left_out_events(left_out_seen),
|
||||
interrupted=total_interrupted,
|
||||
)
|
||||
# A check scripted as `--check && launch` reads the exit code only.
|
||||
if args.check and check_pending:
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ esac
|
||||
TEMP_INSTALLER=""
|
||||
TEMP_DIRECTORY=""
|
||||
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
DEFAULT_INSTALL_SHA256="abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa"
|
||||
DEFAULT_INSTALL_SHA256="6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a"
|
||||
MAX_INSTALLER_BYTES=2097152
|
||||
|
||||
cleanup() {
|
||||
|
||||
@@ -44,6 +44,14 @@
|
||||
"minLength": 1,
|
||||
"description": "Hardware target this manifest was filtered for, absent when unfiltered"
|
||||
},
|
||||
"categorized": {
|
||||
"description": "Every game data file carries category: game_data. Without it the list records no category, and the installer refuses --no-game-data.",
|
||||
"const": true
|
||||
},
|
||||
"no_game_data": {
|
||||
"description": "Built without game data, absent when the game data is listed",
|
||||
"const": true
|
||||
},
|
||||
"platform": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -170,6 +178,10 @@
|
||||
"below_platform_folder"
|
||||
]
|
||||
},
|
||||
"category": {
|
||||
"description": "game_data: a game's own data, not firmware. Absent on firmware.",
|
||||
"const": "game_data"
|
||||
},
|
||||
"held": {
|
||||
"description": "The pack builder holds this file and the installer cannot download it. Region narrowing counts it held, as the builder does, so it displaces the same fallback.",
|
||||
"const": true
|
||||
@@ -685,6 +697,10 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"category": {
|
||||
"description": "game_data: a game's own data, not firmware. Absent on firmware. The installer's --no-game-data leaves these out, as a pack built with --no-game-data does.",
|
||||
"const": "game_data"
|
||||
},
|
||||
"executable": {
|
||||
"description": "The repository keeps the file executable, and the installer gives it the execute bits the umask allows.",
|
||||
"type": "boolean"
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Leaving a game's own data out of a pack, a manifest or a report.
|
||||
|
||||
A platform list can name the files a game reads beside the firmware its
|
||||
emulator needs, and a core profile marks the data files of the games it runs
|
||||
`category: game_data`. `--no-game-data` keeps the firmware: the platform's
|
||||
entries categorized as game data and the profiles' game data entries are left
|
||||
out, and each one is named with the reason.
|
||||
|
||||
Every caller cuts from the full platform config and the full profiles. The
|
||||
category of a platform entry is read from the profiles' game data entries, so
|
||||
cutting twice would find no evidence the second time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from common import resolve_platform_cores
|
||||
from exporter.candidates import GAME_DATA, EntryCategories
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LeftOut:
|
||||
"""One game data file left out, and who reads it as game data."""
|
||||
|
||||
layer: str # "platform" (the platform's list) or "core" (a core profile)
|
||||
system: str
|
||||
name: str
|
||||
readers: tuple[str, ...]
|
||||
|
||||
def describe(self) -> str:
|
||||
where = f"{self.system}/{self.name}" if self.system else self.name
|
||||
if not self.readers:
|
||||
return where
|
||||
return f"{where} (game data of {', '.join(self.readers)})"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GameDataCut:
|
||||
"""The config and profiles without game data, and what was left out."""
|
||||
|
||||
config: dict
|
||||
profiles: dict[str, dict]
|
||||
left_out: list[LeftOut]
|
||||
|
||||
def report_lines(self) -> list[str]:
|
||||
"""What the cut left out, named, one line per layer.
|
||||
|
||||
The platform layer names each entry and the cores that read it as
|
||||
game data. The core layer names each core with the files its profile
|
||||
declares `category: game_data`, held by the collection or not.
|
||||
"""
|
||||
lines = []
|
||||
platform = [item.describe() for item in self.left_out if item.layer == "platform"]
|
||||
if platform:
|
||||
lines.append(
|
||||
f"Left out as game data, {len(platform)} entries of the platform's "
|
||||
f"list: {', '.join(platform)}"
|
||||
)
|
||||
by_core: dict[str, list[str]] = {}
|
||||
for item in self.left_out:
|
||||
if item.layer == "core":
|
||||
names = by_core.setdefault(item.readers[0], [])
|
||||
if item.name not in names:
|
||||
names.append(item.name)
|
||||
if by_core:
|
||||
count = sum(len(names) for names in by_core.values())
|
||||
named = "; ".join(
|
||||
f"{core}: {', '.join(names)}" for core, names in sorted(by_core.items())
|
||||
)
|
||||
lines.append(
|
||||
f"Left out as game data, {count} files the cores declare "
|
||||
f"category: game_data ({named})"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def is_game_data(entry: dict) -> bool:
|
||||
return entry.get("category") == GAME_DATA
|
||||
|
||||
|
||||
def _entry_names(entry: dict) -> set[str]:
|
||||
names = {str(entry.get("name") or "")}
|
||||
names.update(str(a) for a in entry.get("aliases") or [])
|
||||
path = str(entry.get("path") or "")
|
||||
if path:
|
||||
names.add(path.rstrip("/").rsplit("/", 1)[-1])
|
||||
return {n.casefold() for n in names if n}
|
||||
|
||||
|
||||
def platform_game_data(
|
||||
config: dict, profiles: dict[str, dict], data_registry: dict | None
|
||||
) -> set[int]:
|
||||
"""The `id()` of every platform entry categorized as game data."""
|
||||
categories = EntryCategories(config, profiles, data_registry)
|
||||
return {
|
||||
id(fe)
|
||||
for sys_id, system in (config.get("systems") or {}).items()
|
||||
for fe in system.get("files") or []
|
||||
if isinstance(fe, dict)
|
||||
and categories.category(
|
||||
fe, str(fe.get("destination", fe.get("name", ""))), sys_id
|
||||
) == GAME_DATA
|
||||
}
|
||||
|
||||
|
||||
def without_game_data(
|
||||
config: dict,
|
||||
profiles: dict[str, dict],
|
||||
data_registry: dict | None = None,
|
||||
target_cores: set[str] | None = None,
|
||||
) -> GameDataCut:
|
||||
"""The platform config and the profiles with every game data entry removed.
|
||||
|
||||
The profiles lose their `category: game_data` files, so no core extra
|
||||
of that category reaches the pack, the manifest or the report. The
|
||||
platform keeps every entry that is not categorized game data.
|
||||
"""
|
||||
dropped = platform_game_data(config, profiles, data_registry)
|
||||
examined = sorted(resolve_platform_cores(config, profiles, target_cores=target_cores))
|
||||
readers_by_name: dict[str, set[str]] = {}
|
||||
for key in examined:
|
||||
for fe in profiles[key].get("files") or []:
|
||||
if isinstance(fe, dict) and is_game_data(fe):
|
||||
for name in _entry_names(fe):
|
||||
readers_by_name.setdefault(name, set()).add(key)
|
||||
|
||||
left_out: list[LeftOut] = []
|
||||
systems: dict[str, dict] = {}
|
||||
for sys_id, system in (config.get("systems") or {}).items():
|
||||
files = system.get("files") or []
|
||||
kept = [fe for fe in files if id(fe) not in dropped]
|
||||
for fe in files:
|
||||
if id(fe) in dropped:
|
||||
name = str(fe.get("name") or "")
|
||||
left_out.append(LeftOut(
|
||||
"platform", sys_id, name,
|
||||
tuple(sorted(readers_by_name.get(name.casefold(), ()))),
|
||||
))
|
||||
systems[sys_id] = {**system, "files": kept} if len(kept) != len(files) else system
|
||||
|
||||
for key in examined:
|
||||
for fe in profiles[key].get("files") or []:
|
||||
if isinstance(fe, dict) and is_game_data(fe):
|
||||
left_out.append(LeftOut("core", "", str(fe.get("name") or ""), (key,)))
|
||||
|
||||
stripped = {
|
||||
key: (
|
||||
{**profile, "files": [
|
||||
fe for fe in profile.get("files") or []
|
||||
if not (isinstance(fe, dict) and is_game_data(fe))
|
||||
]}
|
||||
if any(isinstance(fe, dict) and is_game_data(fe)
|
||||
for fe in profile.get("files") or [])
|
||||
else profile
|
||||
)
|
||||
for key, profile in profiles.items()
|
||||
}
|
||||
return GameDataCut({**config, "systems": systems}, stripped, left_out)
|
||||
+135
-10
@@ -79,6 +79,7 @@ from manifestselect import SelectionIndex
|
||||
import slot as slot_mod
|
||||
import slots
|
||||
import split_pack
|
||||
from gamedata import GAME_DATA, GameDataCut, is_game_data, platform_game_data, without_game_data
|
||||
from deterministic_zip import _FIXED_DATE_TIME, rebuild_zip_deterministic
|
||||
from largefiles import asset_name, load_registered_paths, published_asset_name
|
||||
from nativemode import (
|
||||
@@ -193,6 +194,7 @@ def _narrowings(
|
||||
one_per_slot: bool,
|
||||
required_only: bool,
|
||||
standalone: bool = False,
|
||||
no_game_data: bool = False,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Every dimension that narrows a pack, as (filename tag, plain label).
|
||||
|
||||
@@ -223,6 +225,8 @@ def _narrowings(
|
||||
applied.append(("_Standalone", "standalone mode files only"))
|
||||
if required_only:
|
||||
applied.append(("_Required", "required files only"))
|
||||
if no_game_data:
|
||||
applied.append(("_NoGameData", "firmware only, game data left out"))
|
||||
return applied
|
||||
|
||||
|
||||
@@ -608,6 +612,22 @@ def _matches_an_alternative(fe: dict, local_path: str, profiles: dict) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _report_left_out(cut: GameDataCut, systems: set[str], cores: bool) -> None:
|
||||
"""Name the game data a --no-game-data build left out, and why.
|
||||
|
||||
``systems`` limits the platform layer to the systems this pack covers,
|
||||
and ``cores`` says whether this pack collected the core files itself:
|
||||
a split part does not, its run names them once.
|
||||
"""
|
||||
kept = [
|
||||
item for item in cut.left_out
|
||||
if (item.layer == "platform" and item.system in systems)
|
||||
or (item.layer == "core" and cores)
|
||||
]
|
||||
for line in GameDataCut(cut.config, cut.profiles, kept).report_lines():
|
||||
print(f" {line}")
|
||||
|
||||
|
||||
def generate_pack(
|
||||
platform_name: str,
|
||||
platforms_dir: str,
|
||||
@@ -630,9 +650,13 @@ def generate_pack(
|
||||
one_per_slot: bool = False,
|
||||
offline: bool | None = None,
|
||||
pack_name: str | None = None,
|
||||
no_game_data: bool = False,
|
||||
) -> str | None:
|
||||
"""Generate a ZIP pack for a platform.
|
||||
|
||||
``no_game_data`` leaves out the platform's entries categorized as game
|
||||
data and the cores' ``category: game_data`` files, and names each one.
|
||||
|
||||
``pack_name`` replaces the derived file name: a split part is named after
|
||||
its group, and a name built from every system of a hundred-system group
|
||||
is longer than a file name can be.
|
||||
@@ -646,13 +670,22 @@ def generate_pack(
|
||||
# The pipeline always passes them, but a caller that does not must get the
|
||||
# same pack, not one built without validation, slots or agnostic cores.
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
# Cut from the full config and profiles: the category of a platform
|
||||
# entry is read from the profiles' game data entries.
|
||||
cut = (
|
||||
without_game_data(config, emu_profiles, data_registry, target_cores)
|
||||
if no_game_data else None
|
||||
)
|
||||
if cut is not None:
|
||||
config, emu_profiles = cut.config, cut.profiles
|
||||
|
||||
verification_mode = config.get("verification_mode", "existence")
|
||||
platform_display = config.get("platform", platform_name)
|
||||
base_dest = config.get("base_destination", "")
|
||||
|
||||
narrowings = _narrowings(
|
||||
source, regions, target_name, one_per_slot, required_only
|
||||
source, regions, target_name, one_per_slot, required_only,
|
||||
no_game_data=no_game_data,
|
||||
)
|
||||
zip_path = os.path.join(
|
||||
output_dir,
|
||||
@@ -1164,6 +1197,12 @@ def generate_pack(
|
||||
zip_path, file_status, total_files, core_count, source,
|
||||
verification_mode,
|
||||
)
|
||||
if cut is not None:
|
||||
_report_left_out(
|
||||
cut,
|
||||
systems=set(pack_systems) if source != "truth" else set(),
|
||||
cores=source != "platform" and precomputed_extras is None,
|
||||
)
|
||||
|
||||
|
||||
for key, reason in sorted(file_reasons.items()):
|
||||
@@ -1884,6 +1923,7 @@ def generate_split_packs(
|
||||
target_name: str | None = None,
|
||||
one_per_slot: bool = False,
|
||||
offline: bool | None = None,
|
||||
no_game_data: bool = False,
|
||||
) -> list[str]:
|
||||
"""Generate split packs (one ZIP per system or manufacturer)."""
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
@@ -1893,7 +1933,8 @@ def generate_split_packs(
|
||||
split_tags = "".join(
|
||||
tag
|
||||
for tag, _label in _narrowings(
|
||||
source, regions, target_name, one_per_slot, required_only
|
||||
source, regions, target_name, one_per_slot, required_only,
|
||||
no_game_data=no_game_data,
|
||||
)
|
||||
)
|
||||
# Two groupings write different files. Without the tag they accumulate in
|
||||
@@ -1920,15 +1961,24 @@ def generate_split_packs(
|
||||
if emu_profiles is None:
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
base_dest = config.get("base_destination", "")
|
||||
# Each part cuts the platform's own entries from the full profiles. The
|
||||
# extras are collected once here, already without the cores' game data.
|
||||
cut = (
|
||||
without_game_data(config, emu_profiles, data_registry, target_cores)
|
||||
if no_game_data else None
|
||||
)
|
||||
if source == "platform":
|
||||
all_extras = []
|
||||
elif emu_profiles:
|
||||
all_extras = _collect_emulator_extras(
|
||||
config, emulators_dir, db, set(), base_dest, emu_profiles,
|
||||
cut.config if cut else config, emulators_dir, db, set(), base_dest,
|
||||
cut.profiles if cut else emu_profiles,
|
||||
target_cores=target_cores, include_all=(source == "truth"),
|
||||
)
|
||||
else:
|
||||
all_extras = []
|
||||
if cut is not None and source != "platform":
|
||||
_report_left_out(cut, systems=set(), cores=True)
|
||||
results = []
|
||||
for group_name, group_system_ids in sorted(groups.items()):
|
||||
group_extras = _extras_for_systems(all_extras, group_system_ids)
|
||||
@@ -1952,6 +2002,7 @@ def generate_split_packs(
|
||||
one_per_slot=one_per_slot,
|
||||
offline=offline,
|
||||
pack_name=f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip",
|
||||
no_game_data=no_game_data,
|
||||
)
|
||||
if zip_path:
|
||||
results.append(zip_path)
|
||||
@@ -1975,6 +2026,7 @@ def generate_split_packs(
|
||||
precomputed_extras=undistributed, extras_only=True, source=source,
|
||||
regions=regions, target_name=target_name, one_per_slot=one_per_slot,
|
||||
offline=offline, pack_name=f"{part_prefix}_Other_Cores_BIOS_Pack.zip",
|
||||
no_game_data=no_game_data,
|
||||
)
|
||||
if zip_path:
|
||||
results.append(zip_path)
|
||||
@@ -2411,6 +2463,12 @@ def _validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) ->
|
||||
if args.split:
|
||||
parser.error("--system is incompatible with --split")
|
||||
|
||||
# Game data is categorized against a platform's list and the cores it
|
||||
# runs: a pack of one emulator or of a system's emulators has neither.
|
||||
if getattr(args, "no_game_data", False) and not (has_platform or has_all):
|
||||
parser.error("--no-game-data requires --platform or --all")
|
||||
if getattr(args, "no_game_data", False) and has_from_md5:
|
||||
parser.error("--no-game-data is incompatible with --from-md5")
|
||||
if getattr(args, "one_per_slot", False):
|
||||
if has_from_md5:
|
||||
parser.error("--one-per-slot and --from-md5 are mutually exclusive")
|
||||
@@ -2483,6 +2541,7 @@ def _run_manifest_mode(
|
||||
),
|
||||
offline=args.offline,
|
||||
required_only=required_only,
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
narrow_suffix = "".join(
|
||||
tag.lower()
|
||||
@@ -2492,6 +2551,7 @@ def _run_manifest_mode(
|
||||
_platform_target(representative, args.target, args.platforms_dir),
|
||||
False,
|
||||
required_only,
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
)
|
||||
out_path = os.path.join(
|
||||
@@ -2591,6 +2651,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
|
||||
verify_regions,
|
||||
args.target,
|
||||
ALL_VARIANTS if args.all_variants else (("full", False),),
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
on_disk = (
|
||||
set(os.listdir(args.output_dir)) if os.path.isdir(args.output_dir) else set()
|
||||
@@ -2641,6 +2702,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
|
||||
target_cores=tc,
|
||||
source=source,
|
||||
required_only=required_only,
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
(ok, _checked, _present, errors, bl_checked, bl_present,
|
||||
core_checked, core_present, bl_excluded, bl_absent) = result
|
||||
@@ -2788,6 +2850,7 @@ def _run_platform_packs(
|
||||
target_name=named_target,
|
||||
one_per_slot=args.one_per_slot,
|
||||
offline=args.offline,
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
print(f" Split into {len(zip_paths)} packs")
|
||||
else:
|
||||
@@ -2806,6 +2869,7 @@ def _run_platform_packs(
|
||||
named_target,
|
||||
args.one_per_slot,
|
||||
required_only,
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
)
|
||||
stem = _platform_pack_stem(
|
||||
@@ -2834,6 +2898,7 @@ def _run_platform_packs(
|
||||
one_per_slot=args.one_per_slot,
|
||||
offline=args.offline,
|
||||
pack_name=group_name,
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
except (FileNotFoundError, OSError, yaml.YAMLError) as e:
|
||||
print(f" ERROR: {e}", file=sys.stderr)
|
||||
@@ -2856,6 +2921,7 @@ def _run_platform_packs(
|
||||
data_registry=data_registry,
|
||||
regions=getattr(args, "regions", None),
|
||||
target_name=args.target,
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
if args.split:
|
||||
for entry in os.listdir(args.output_dir):
|
||||
@@ -2890,6 +2956,7 @@ def _refuse_for_all_variants(args: argparse.Namespace, parser: argparse.Argument
|
||||
for flag, given in (
|
||||
("--source", args.source is not None),
|
||||
("--required-only", args.required_only),
|
||||
("--no-game-data", getattr(args, "no_game_data", False)),
|
||||
("--emulator", args.emulator),
|
||||
("--from-md5", args.from_md5 or args.from_md5_file),
|
||||
):
|
||||
@@ -2978,7 +3045,10 @@ def _refuse_unapplied_flags(args: argparse.Namespace, parser: argparse.ArgumentP
|
||||
QUICK_MODE_FLAGS: dict[str, set[str]] = {
|
||||
# Checks packs already on disk against the platform's own list, narrowed
|
||||
# by region and target and by nothing else.
|
||||
"--verify-packs": {"--platform", "--all", "--target", "--region", "--include-archived"},
|
||||
"--verify-packs": {
|
||||
"--platform", "--all", "--target", "--region", "--include-archived",
|
||||
"--no-game-data",
|
||||
},
|
||||
# One manifest per hardware target, from the target files themselves.
|
||||
"--manifest-targets": set(),
|
||||
"--list": set(),
|
||||
@@ -3005,6 +3075,7 @@ def _given_flags(args: argparse.Namespace) -> set[str]:
|
||||
"--from-md5": args.from_md5,
|
||||
"--from-md5-file": args.from_md5_file,
|
||||
"--required-only": args.required_only,
|
||||
"--no-game-data": getattr(args, "no_game_data", False),
|
||||
"--source": args.source is not None,
|
||||
"--target": args.target,
|
||||
"--region": args.region,
|
||||
@@ -3107,6 +3178,12 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
action="store_true",
|
||||
help="Only include required files, skip optional",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-game-data",
|
||||
action="store_true",
|
||||
help="Leave out game data (the platform's entries a core reads as a "
|
||||
"game's files, and the cores' game data files): firmware only",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
choices=["platform", "truth", "full"],
|
||||
@@ -3860,6 +3937,10 @@ def _manifest_core_entries(
|
||||
pack_only_sizes.append(file_size)
|
||||
continue
|
||||
|
||||
if is_game_data(fe) and all(
|
||||
is_game_data(claim) for claim in fe.get("other_claims", [])
|
||||
):
|
||||
entry["category"] = GAME_DATA
|
||||
_attach_claims(
|
||||
entry, fe.get("other_claims", []), db, bios_dir, zip_contents,
|
||||
offline, required_only, validation_index, owners, dest,
|
||||
@@ -4027,12 +4108,17 @@ def generate_manifest(
|
||||
required_only: bool = False,
|
||||
data_registry: dict | None = None,
|
||||
fold_case: bool | None = None,
|
||||
no_game_data: bool = False,
|
||||
) -> dict:
|
||||
"""Generate a JSON manifest for a platform (same resolution as generate_pack).
|
||||
|
||||
Returns a dict ready for JSON serialization with file inventory,
|
||||
install hints, and download metadata. ``fold_case`` overrides the
|
||||
platform's ``case_insensitive_fs``, for the case variants below.
|
||||
|
||||
A file that is a game's data, not firmware, carries ``"category":
|
||||
"game_data"``, so the installer can leave it out as the builder does
|
||||
under ``no_game_data``. Firmware carries no category.
|
||||
"""
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
if fold_case is not None:
|
||||
@@ -4041,6 +4127,15 @@ def generate_manifest(
|
||||
zip_contents = {}
|
||||
if emu_profiles is None:
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
if data_registry is None:
|
||||
data_registry = load_data_dir_registry(platforms_dir)
|
||||
full_profiles = emu_profiles
|
||||
game_entries: set[int] = set()
|
||||
if no_game_data:
|
||||
cut = without_game_data(config, emu_profiles, data_registry, target_cores)
|
||||
config, emu_profiles = cut.config, cut.profiles
|
||||
else:
|
||||
game_entries = platform_game_data(config, emu_profiles, data_registry)
|
||||
|
||||
platform_display = config.get("platform", platform_name)
|
||||
base_dest = config.get("base_destination", "")
|
||||
@@ -4073,6 +4168,16 @@ def generate_manifest(
|
||||
platform_cores=plat_cores,
|
||||
)
|
||||
|
||||
# A destination is game data when every declaration of it is: a firmware
|
||||
# declaration of the same path keeps it in a pack built without game data.
|
||||
declared_kinds: dict[str, set[bool]] = {}
|
||||
for system in pack_systems.values():
|
||||
for fe in system.get("files", []):
|
||||
bare = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
|
||||
if bare:
|
||||
declared_kinds.setdefault(bare, set()).add(id(fe) in game_entries)
|
||||
game_destinations = {d for d, kinds in declared_kinds.items() if kinds == {True}}
|
||||
|
||||
seen_destinations: set[str] = set()
|
||||
# Destination -> (entry, full destination, manifest destination) of each
|
||||
# platform file, for the core claims a target build would fall back on.
|
||||
@@ -4090,8 +4195,6 @@ def generate_manifest(
|
||||
# collection does not index them, a data directory cache answered.
|
||||
pack_only_sizes: list[int] = []
|
||||
rename_notes: set[str] = set()
|
||||
if data_registry is None:
|
||||
data_registry = load_data_dir_registry(platforms_dir)
|
||||
slot_overrides = slots.pack_overrides(
|
||||
config, emu_profiles, db, zip_contents, data_registry,
|
||||
target_cores=target_cores, source=source,
|
||||
@@ -4134,6 +4237,11 @@ def generate_manifest(
|
||||
# drops the same fallback.
|
||||
**selection.fields(bare, str(file_entry.get("name") or "")),
|
||||
**({"held": True} if bare in builder_held else {}),
|
||||
**(
|
||||
{"category": GAME_DATA}
|
||||
if (bare in game_destinations if cores is None else is_game_data(file_entry))
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
# Read by the region grouping and by phase 2 with the same arguments.
|
||||
@@ -4299,6 +4407,8 @@ def generate_manifest(
|
||||
local_path, dest, None, db, repo_root,
|
||||
selection.fields(dest, file_entry["name"]),
|
||||
)
|
||||
if entry is not None and dest in game_destinations:
|
||||
entry["category"] = GAME_DATA
|
||||
if entry is not None and target_cores is None:
|
||||
order = declaration_order(full_dest)
|
||||
if full_dest in slot_choices or len(order) > 1:
|
||||
@@ -4364,6 +4474,10 @@ def generate_manifest(
|
||||
if declared and declared != set(entry.get("systems") or []):
|
||||
entry["declared_in"] = sorted(declared)
|
||||
claims = baseline_claims.get(key, {}).get("other_claims")
|
||||
if claims and any(not is_game_data(claim) for claim in claims):
|
||||
# A core reads this path as firmware: a pack without game data
|
||||
# still fills it.
|
||||
entry.pop("category", None)
|
||||
if claims:
|
||||
_attach_claims(
|
||||
entry, claims, db, bios_dir, zip_contents, offline, required_only,
|
||||
@@ -4409,6 +4523,10 @@ def generate_manifest(
|
||||
# Absent on an unfiltered manifest rather than empty, to leave the
|
||||
# twelve default files byte-identical.
|
||||
**({"target": target_name} if target_name else {}),
|
||||
# Every game data file below says so: the installer's --no-game-data
|
||||
# relies on it, and refuses a list that does not record categories.
|
||||
"categorized": True,
|
||||
**({"no_game_data": True} if no_game_data else {}),
|
||||
"platform": platform_name,
|
||||
"display_name": platform_display,
|
||||
"version": "1.0",
|
||||
@@ -4448,9 +4566,9 @@ def generate_manifest(
|
||||
# or macOS too, stays folded.
|
||||
kept = generate_manifest(
|
||||
platform_name, platforms_dir, db, bios_dir, registry_path,
|
||||
emulators_dir, zip_contents, emu_profiles, target_cores, source,
|
||||
emulators_dir, zip_contents, full_profiles, target_cores, source,
|
||||
regions, target_name, offline, required_only, data_registry,
|
||||
fold_case=False,
|
||||
fold_case=False, no_game_data=no_game_data,
|
||||
)
|
||||
folded = {entry["dest"] for entry in manifest_files}
|
||||
folded_lower = {dest.lower() for dest in folded}
|
||||
@@ -4624,6 +4742,7 @@ def _expected_pack_names(
|
||||
regions: list[str] | None,
|
||||
target_name: str | None,
|
||||
variants: tuple[tuple[str, bool], ...] = (("full", False),),
|
||||
no_game_data: bool = False,
|
||||
) -> dict[str, set[str]]:
|
||||
"""Exact pack file names --verify-packs may check, per platform.
|
||||
|
||||
@@ -4638,7 +4757,9 @@ def _expected_pack_names(
|
||||
return [
|
||||
"".join(
|
||||
tag
|
||||
for tag, _label in _narrowings(source, regions, named, False, required)
|
||||
for tag, _label in _narrowings(
|
||||
source, regions, named, False, required, no_game_data=no_game_data
|
||||
)
|
||||
)
|
||||
for source, required in variants
|
||||
]
|
||||
@@ -4767,6 +4888,7 @@ def verify_and_finalize_packs(
|
||||
regions: list[str] | None = None,
|
||||
target_name: str | None = None,
|
||||
emulators_dir: str = "emulators",
|
||||
no_game_data: bool = False,
|
||||
) -> bool:
|
||||
"""Verify all packs, inject manifests, generate SHA256SUMS.
|
||||
|
||||
@@ -4783,7 +4905,9 @@ def verify_and_finalize_packs(
|
||||
# substring: a _Custom or regional pack left by another run was judged
|
||||
# against this run's expectation and failed it.
|
||||
registered = list_registered_platforms(platforms_dir, include_archived=True)
|
||||
expected = _expected_pack_names(registered, platforms_dir, regions, target_name)
|
||||
expected = _expected_pack_names(
|
||||
registered, platforms_dir, regions, target_name, no_game_data=no_game_data
|
||||
)
|
||||
pack_to_platform: dict[str, list[str]] = {}
|
||||
for name in _pack_archives(output_dir):
|
||||
for pname in registered:
|
||||
@@ -4837,6 +4961,7 @@ def verify_and_finalize_packs(
|
||||
regions=regions,
|
||||
data_registry=data_registry,
|
||||
target_cores=_target_cores_for(pname, target_name, platforms_dir),
|
||||
no_game_data=no_game_data,
|
||||
)
|
||||
status = "OK" if p_ok else "FAILED"
|
||||
exclusion_note = "".join(
|
||||
|
||||
@@ -617,6 +617,9 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"python scripts/generate_pack.py --platform retroarch --region us,eu,jp",
|
||||
"python scripts/generate_pack.py --platform retroarch --region us --one-per-slot",
|
||||
"",
|
||||
"# Firmware only: the game data a list or a core names is left out, each file named",
|
||||
"python scripts/generate_pack.py --platform emuelec --no-game-data",
|
||||
"",
|
||||
"# List available emulators and systems",
|
||||
"python scripts/generate_pack.py --list-emulators",
|
||||
"python scripts/generate_pack.py --list-systems",
|
||||
|
||||
@@ -5347,7 +5347,10 @@ everything or to choose systems, cores and regions, and `--system`, `--core`,
|
||||
[installer reference](wiki/installer.md#choosing-what-to-install)). From a
|
||||
clone of the repository, `python scripts/generate_pack.py --platform retroarch
|
||||
--region us` builds the same narrowing as a pack and `--required-only` the bare
|
||||
minimum each core needs to start, and `--help` lists every way to build your own.
|
||||
minimum each core needs to start. `--no-game-data`, for the pack and the
|
||||
installer alike, keeps the firmware and leaves out the files a game reads
|
||||
([details](wiki/advanced-usage.md#firmware-without-game-data)), and `--help`
|
||||
lists every way to build your own.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from nativemode import reads_file_contents
|
||||
import region as region_mod
|
||||
from packresolve import resolve_file
|
||||
from exporter.candidates import BIOS, EntryCategories
|
||||
from gamedata import without_game_data
|
||||
from common import resolve_platform_cores
|
||||
from common import sanitize_pack_path
|
||||
import zipfile
|
||||
@@ -634,6 +635,7 @@ def verify_pack_against_platform(
|
||||
target_cores: set[str] | None = None,
|
||||
source: str = "full",
|
||||
required_only: bool = False,
|
||||
no_game_data: bool = False,
|
||||
) -> tuple[bool, int, int, list[str], int, int, int, int, int, int]:
|
||||
"""Verify a pack ZIP against its platform config and core requirements.
|
||||
|
||||
@@ -644,6 +646,8 @@ def verify_pack_against_platform(
|
||||
``required_only`` are the variant the pack was built as: a platform-only
|
||||
pack is held to the platform's list alone, a truth pack to the cores'
|
||||
files alone, and a required-only pack to the required entries of both.
|
||||
``no_game_data`` holds the pack to the firmware alone, cut as the
|
||||
builder cuts it.
|
||||
|
||||
Checks:
|
||||
1. Every baseline file declared by the platform exists in the ZIP
|
||||
@@ -657,7 +661,7 @@ def verify_pack_against_platform(
|
||||
baseline_present, core_checked, core_present, baseline_excluded,
|
||||
baseline_absent)``. ``baseline_absent`` counts destinations the collection
|
||||
holds no file for that are a game's data or that no core the platform
|
||||
runs opens; verify.py counts them apart from firmware.
|
||||
runs opens. verify.py counts them apart from firmware.
|
||||
"""
|
||||
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
@@ -668,6 +672,9 @@ def verify_pack_against_platform(
|
||||
|
||||
if emu_profiles is None:
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
if no_game_data:
|
||||
cut = without_game_data(config, emu_profiles, data_registry, target_cores)
|
||||
config, emu_profiles = cut.config, cut.profiles
|
||||
|
||||
expected_systems = config.get("systems", {})
|
||||
if target_cores is not None:
|
||||
|
||||
@@ -1121,14 +1121,33 @@ def verify_platform(
|
||||
data_dir_registry: dict | None = None,
|
||||
regions: list[str] | None = None,
|
||||
system_filter: list[str] | None = None,
|
||||
no_game_data: bool = False,
|
||||
) -> dict:
|
||||
"""Verify all BIOS files for a platform, including cross-reference gaps.
|
||||
|
||||
``no_game_data`` reports on the pack built with --no-game-data: the
|
||||
game data cut as the builder cuts it, and named in ``left_out``.
|
||||
|
||||
A region priority list narrows the report to the files a pack built with the
|
||||
same list would carry, using the same selection function as the builder.
|
||||
``system_filter`` does the same for a pack built with --system: the
|
||||
systems it keeps and the core extras those systems own.
|
||||
"""
|
||||
left_out: list[str] = []
|
||||
if no_game_data:
|
||||
from gamedata import without_game_data
|
||||
|
||||
if emu_profiles is None:
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
cut = without_game_data(config, emu_profiles, data_dir_registry, target_cores)
|
||||
config, emu_profiles = cut.config, cut.profiles
|
||||
systems = config.get("systems", {})
|
||||
in_scope = set(select_systems(systems, system_filter) if system_filter else systems)
|
||||
cut.left_out = [
|
||||
item for item in cut.left_out
|
||||
if item.layer == "core" or item.system in in_scope
|
||||
]
|
||||
left_out = cut.report_lines()
|
||||
# Normalized once: a typo in a platform YAML must not verify with one
|
||||
# mode and be scored with another.
|
||||
mode = normalize_mode(config.get("verification_mode"))
|
||||
@@ -1527,6 +1546,7 @@ def verify_platform(
|
||||
"undeclared_files": undeclared,
|
||||
"exclusion_notes": exclusions,
|
||||
"region_fallbacks": sorted(region_fallbacks),
|
||||
**({"left_out": left_out} if no_game_data else {}),
|
||||
"details": details,
|
||||
"ground_truth_coverage": {
|
||||
"with_validation": with_validation,
|
||||
@@ -1866,6 +1886,8 @@ def print_platform_result(
|
||||
# them missing, as an offline build leaves them out.
|
||||
shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "")
|
||||
print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}")
|
||||
for line in result.get("left_out") or []:
|
||||
print(f" {line}")
|
||||
fallbacks = result.get("region_fallbacks") or []
|
||||
if fallbacks:
|
||||
from packreadme import _group_label
|
||||
@@ -2572,6 +2594,7 @@ def _refuse_listing_narrowings(
|
||||
("--verbose", args.verbose),
|
||||
("--destinations", args.destinations),
|
||||
("--offline", args.offline),
|
||||
("--no-game-data", getattr(args, "no_game_data", False)),
|
||||
):
|
||||
if on:
|
||||
parser.error(f"{flag} is incompatible with {listing}")
|
||||
@@ -2595,6 +2618,9 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
|
||||
"--destinations requires --platform or --all"),
|
||||
(args.offline and not (args.platform or args.all),
|
||||
"--offline requires --platform or --all"),
|
||||
# Game data is categorized against a platform's list and its cores.
|
||||
(getattr(args, "no_game_data", False) and not (args.platform or args.all),
|
||||
"--no-game-data requires --platform or --all"),
|
||||
(args.destinations and getattr(args, "region", None),
|
||||
"--destinations describes the full pack and is incompatible with --region"),
|
||||
):
|
||||
@@ -2655,6 +2681,11 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
action="store_true",
|
||||
help="List available targets for the platform",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-game-data", action="store_true",
|
||||
help="Report on the pack built with --no-game-data: game data left "
|
||||
"out and named, firmware only",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db", default=DEFAULT_DB,
|
||||
help="database file to read (default: %(default)s)",
|
||||
@@ -2816,6 +2847,7 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
data_dir_registry=data_registry,
|
||||
regions=requested_regions,
|
||||
system_filter=platform_systems,
|
||||
no_game_data=getattr(args, "no_game_data", False),
|
||||
)
|
||||
names = [
|
||||
load_platform_config(p, args.platforms_dir).get("platform", p)
|
||||
|
||||
@@ -6003,6 +6003,7 @@ struct BurnDriver BurnDrvneogeo = {
|
||||
{"target_name": "switch"},
|
||||
{"one_per_slot": True},
|
||||
{"required_only": True},
|
||||
{"no_game_data": True},
|
||||
]
|
||||
for kwargs in dimensions:
|
||||
applied = _narrowings(
|
||||
@@ -6011,6 +6012,7 @@ struct BurnDriver BurnDrvneogeo = {
|
||||
kwargs.get("target_name"),
|
||||
kwargs.get("one_per_slot", False),
|
||||
kwargs.get("required_only", False),
|
||||
no_game_data=kwargs.get("no_game_data", False),
|
||||
)
|
||||
self.assertEqual(len(applied), 1, kwargs)
|
||||
tag, label = applied[0]
|
||||
@@ -6136,6 +6138,19 @@ struct BurnDriver BurnDrvneogeo = {
|
||||
# Documented as adding core files, read by nothing.
|
||||
(["--emulator", "handy"], ["--include-extras"], "refuse"),
|
||||
(["--system", "atari-lynx"], ["--include-extras"], "refuse"),
|
||||
# Game data is categorized against a platform's list and the
|
||||
# cores it runs: the modes without a platform refuse it, and
|
||||
# --all-variants builds its own fixed set.
|
||||
(["--emulator", "handy"], ["--no-game-data"], "refuse"),
|
||||
(["--system", "atari-lynx"], ["--no-game-data"], "refuse"),
|
||||
(["--platform", "retroarch", "--from-md5", "d8f1"], ["--no-game-data"],
|
||||
"refuse"),
|
||||
(["--from-md5", "d8f1"], ["--no-game-data"], "refuse"),
|
||||
(["--manifest-targets"], ["--no-game-data"], "refuse"),
|
||||
(["--platform", "retroarch", "--all-variants"], ["--no-game-data"],
|
||||
"refuse"),
|
||||
(["--list-systems"], ["--no-game-data"], "refuse"),
|
||||
(["--list"], ["--no-game-data"], "refuse"),
|
||||
]
|
||||
for mode, flag, expected in matrix:
|
||||
with self.subTest(mode=mode, flag=flag):
|
||||
|
||||
@@ -3797,7 +3797,7 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase):
|
||||
args = SimpleNamespace(
|
||||
system=None, core=None, emulator=None, region=None, target=None,
|
||||
cores_dir=None, every_board=False, installed_cores=False,
|
||||
standalone_copies=False, check=False, dry_run=False,
|
||||
standalone_copies=False, check=False, dry_run=False, no_game_data=False,
|
||||
)
|
||||
flags = list(argv)
|
||||
while flags:
|
||||
@@ -4966,7 +4966,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
|
||||
installer = events[0]["installer"]
|
||||
self.assertEqual(events, self._head("check", installer) + [
|
||||
{"event": "summary", "mode": "check", "installed": 0, "up_to_date": 1,
|
||||
"pending": 3, "failed": [], "skipped": self.SKIPPED, "interrupted": False},
|
||||
"pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False},
|
||||
{"event": "end", "code": 1},
|
||||
])
|
||||
self.assertIn("2/4 present (1 verified, 1 wrong hash)", proc.stderr)
|
||||
@@ -4989,7 +4989,7 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
|
||||
{"event": "download", "platform": "retroarch", "files": 3, "bytes": 245},
|
||||
*done,
|
||||
{"event": "summary", "mode": "install", "installed": 3, "up_to_date": 1,
|
||||
"pending": 0, "failed": [], "skipped": self.SKIPPED, "interrupted": False},
|
||||
"pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False},
|
||||
{"event": "end", "code": 0},
|
||||
])
|
||||
self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
"""--no-game-data: the firmware without the files a game reads.
|
||||
|
||||
A platform list can name a game's data beside the firmware (EmuELEC lists the
|
||||
Doom WADs), and a core profile marks its game data `category: game_data`. The
|
||||
manifest marks both, the builder leaves both out under --no-game-data, verify
|
||||
reports on the same file set, and the installer makes the same cut from the
|
||||
full manifest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import hashlib
|
||||
import http.server
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import common # noqa: E402
|
||||
import generate_db # noqa: E402
|
||||
import generate_pack as builder # noqa: E402
|
||||
import install # noqa: E402
|
||||
import packverify # noqa: E402
|
||||
import verify # noqa: E402
|
||||
|
||||
PAYLOADS = {
|
||||
"System/firmware.rom": b"firmware of the console",
|
||||
"System/corefw.bin": b"firmware the core loads",
|
||||
"Doom/doom1.wad": b"shareware episode",
|
||||
"Doom/engine.pk3": b"engine resources",
|
||||
}
|
||||
|
||||
PROFILES = {
|
||||
"emu": {
|
||||
"emulator": "Emu", "type": "libretro", "cores": ["emu"],
|
||||
"systems": ["console"],
|
||||
"files": [
|
||||
{"name": "firmware.rom", "required": True},
|
||||
{"name": "corefw.bin", "required": True},
|
||||
],
|
||||
},
|
||||
"doomcore": {
|
||||
"emulator": "DoomCore", "type": "libretro", "cores": ["doomcore"],
|
||||
"systems": ["doom"],
|
||||
"files": [
|
||||
{"name": "doom1.wad", "path": "doom/doom1.wad", "category": "game_data",
|
||||
"required": False},
|
||||
{"name": "engine.pk3", "category": "game_data", "required": True},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
FIRMWARE = {"firmware.rom", "corefw.bin"}
|
||||
GAME = {"doom/doom1.wad", "engine.pk3"}
|
||||
|
||||
|
||||
class _Fixture(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
self.bios = self.root / "bios"
|
||||
self.platforms = self.root / "platforms"
|
||||
self.emulators = self.root / "emulators"
|
||||
files = {}
|
||||
for rel, data in PAYLOADS.items():
|
||||
path = self.bios / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
sha1 = hashlib.sha1(data).hexdigest()
|
||||
files[sha1] = {
|
||||
"path": str(path), "name": path.name, "size": len(data), "sha1": sha1,
|
||||
"md5": hashlib.md5(data).hexdigest(),
|
||||
"sha256": hashlib.sha256(data).hexdigest(), "crc32": "00000000",
|
||||
}
|
||||
self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
|
||||
self.platforms.mkdir()
|
||||
self.emulators.mkdir()
|
||||
for key, profile in PROFILES.items():
|
||||
(self.emulators / f"{key}.yml").write_text(yaml.dump(profile))
|
||||
platform = {
|
||||
"platform": "Handheld", "verification_mode": "existence",
|
||||
"cores": ["emu", "doomcore"],
|
||||
"systems": {
|
||||
"console": {"files": [{"name": "firmware.rom", "destination": "firmware.rom"}]},
|
||||
"doom": {"files": [{"name": "doom1.wad", "destination": "doom/doom1.wad"}]},
|
||||
},
|
||||
}
|
||||
(self.platforms / "handheld.yml").write_text(yaml.dump(platform))
|
||||
(self.platforms / "_registry.yml").write_text(
|
||||
yaml.dump({"platforms": {"handheld": {"status": "active"}}})
|
||||
)
|
||||
common._platform_config_cache.clear()
|
||||
common._emulator_profiles_cache.clear()
|
||||
self.out = self.root / "dist"
|
||||
self.out.mkdir()
|
||||
|
||||
def tearDown(self):
|
||||
common._platform_config_cache.clear()
|
||||
common._emulator_profiles_cache.clear()
|
||||
self._tmp.cleanup()
|
||||
|
||||
def pack(self, no_game_data: bool) -> tuple[str, set[str], str]:
|
||||
printed = io.StringIO()
|
||||
with contextlib.redirect_stdout(printed):
|
||||
zip_path = builder.generate_pack(
|
||||
"handheld", str(self.platforms), self.db, str(self.bios), str(self.out),
|
||||
emulators_dir=str(self.emulators), emu_profiles=PROFILES,
|
||||
offline=True, no_game_data=no_game_data,
|
||||
)
|
||||
with zipfile.ZipFile(zip_path) as archive:
|
||||
members = set(archive.namelist()) - set(builder.PACK_DOCUMENTS)
|
||||
return zip_path, members, printed.getvalue()
|
||||
|
||||
def manifest(self, no_game_data: bool = False) -> dict:
|
||||
return builder.generate_manifest(
|
||||
"handheld", str(self.platforms), self.db, str(self.bios),
|
||||
str(self.platforms / "_registry.yml"), emulators_dir=str(self.emulators),
|
||||
emu_profiles=PROFILES, offline=True, no_game_data=no_game_data,
|
||||
)
|
||||
|
||||
|
||||
class ManifestCategory(_Fixture):
|
||||
def test_game_data_is_marked_and_firmware_is_not(self):
|
||||
manifest = self.manifest()
|
||||
self.assertIs(manifest["categorized"], True)
|
||||
category = {entry["dest"]: entry.get("category") for entry in manifest["files"]}
|
||||
self.assertEqual(set(category), FIRMWARE | GAME)
|
||||
# A platform entry only game data reads, and a core's game data file.
|
||||
self.assertEqual(category["doom/doom1.wad"], "game_data")
|
||||
self.assertEqual(category["engine.pk3"], "game_data")
|
||||
# Firmware carries no field at all.
|
||||
for entry in manifest["files"]:
|
||||
if entry["dest"] in FIRMWARE:
|
||||
self.assertNotIn("category", entry)
|
||||
|
||||
def test_a_narrowed_manifest_is_the_full_one_without_game_data(self):
|
||||
full = self.manifest()
|
||||
narrowed = self.manifest(no_game_data=True)
|
||||
self.assertIs(narrowed["no_game_data"], True)
|
||||
self.assertEqual(
|
||||
{(e["dest"], e["sha1"]) for e in narrowed["files"]},
|
||||
{(e["dest"], e["sha1"]) for e in full["files"] if e.get("category") != "game_data"},
|
||||
)
|
||||
|
||||
def test_an_omitted_game_data_file_is_marked(self):
|
||||
(self.bios / "Doom" / "engine.pk3").unlink()
|
||||
files = {sha1: e for sha1, e in self.db["files"].items() if e["name"] != "engine.pk3"}
|
||||
self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
|
||||
manifest = self.manifest()
|
||||
omitted = {e["dest"]: e.get("category") for e in manifest["omitted_files"]}
|
||||
self.assertEqual(omitted.get("engine.pk3"), "game_data")
|
||||
|
||||
|
||||
class PackNarrowing(_Fixture):
|
||||
def test_the_pack_leaves_game_data_out_and_names_it(self):
|
||||
_full_path, full, _ = self.pack(no_game_data=False)
|
||||
zip_path, narrowed, printed = self.pack(no_game_data=True)
|
||||
self.assertEqual(full, FIRMWARE | GAME)
|
||||
self.assertEqual(narrowed, FIRMWARE)
|
||||
self.assertIn("_NoGameData_BIOS_Pack.zip", os.path.basename(zip_path))
|
||||
with zipfile.ZipFile(zip_path) as archive:
|
||||
readme = archive.read("README.txt").decode()
|
||||
self.assertIn("PACK TYPE: Narrowed", readme)
|
||||
self.assertIn("firmware only, game data left out", readme)
|
||||
# Named, with the reason, never only counted.
|
||||
self.assertIn("doom/doom1.wad (game data of doomcore)", printed)
|
||||
self.assertIn("doomcore: doom1.wad, engine.pk3", printed)
|
||||
|
||||
def test_verify_reports_the_pack_the_builder_wrote(self):
|
||||
zip_path, members, _ = self.pack(no_game_data=True)
|
||||
config = common.load_platform_config("handheld", str(self.platforms))
|
||||
result = verify.verify_platform(
|
||||
config, self.db, str(self.emulators), PROFILES, no_game_data=True,
|
||||
)
|
||||
self.assertEqual(set(result["pack_destinations"]), members)
|
||||
self.assertTrue(any("doom/doom1.wad" in line for line in result["left_out"]))
|
||||
checked = packverify.verify_pack_against_platform(
|
||||
zip_path, "handheld", str(self.platforms), db=self.db,
|
||||
emulators_dir=str(self.emulators), emu_profiles=PROFILES,
|
||||
no_game_data=True,
|
||||
)
|
||||
self.assertTrue(checked[0], checked[3])
|
||||
# Held to the full list, the narrowed pack is short of the game data.
|
||||
full = packverify.verify_pack_against_platform(
|
||||
zip_path, "handheld", str(self.platforms), db=self.db,
|
||||
emulators_dir=str(self.emulators), emu_profiles=PROFILES,
|
||||
)
|
||||
self.assertFalse(full[0])
|
||||
|
||||
def test_the_full_report_is_unchanged(self):
|
||||
config = common.load_platform_config("handheld", str(self.platforms))
|
||||
result = verify.verify_platform(config, self.db, str(self.emulators), PROFILES)
|
||||
self.assertNotIn("left_out", result)
|
||||
self.assertEqual(set(result["pack_destinations"]), FIRMWARE | GAME)
|
||||
|
||||
def test_split_parts_add_up_to_the_narrowed_pack(self):
|
||||
printed = io.StringIO()
|
||||
with contextlib.redirect_stdout(printed):
|
||||
parts = builder.generate_split_packs(
|
||||
"handheld", str(self.platforms), self.db, str(self.bios), str(self.out),
|
||||
emulators_dir=str(self.emulators), emu_profiles=PROFILES,
|
||||
offline=True, no_game_data=True,
|
||||
)
|
||||
held: set[str] = set()
|
||||
for part in parts:
|
||||
self.assertIn("_NoGameData", part)
|
||||
with zipfile.ZipFile(part) as archive:
|
||||
held |= set(archive.namelist()) - set(builder.PACK_DOCUMENTS)
|
||||
self.assertEqual(held, FIRMWARE)
|
||||
|
||||
|
||||
class _Quiet(http.server.SimpleHTTPRequestHandler):
|
||||
def log_message(self, *args) -> None:
|
||||
return
|
||||
|
||||
|
||||
class InstallerLeavesGameDataOut(unittest.TestCase):
|
||||
"""The installer cut, in each mode, from a manifest served over loopback."""
|
||||
|
||||
FILES = {
|
||||
"bios/a.bin": (b"alpha" * 20, None),
|
||||
"bios/game.wad": (b"wad" * 10, "game_data"),
|
||||
}
|
||||
|
||||
def _tempdir(self) -> Path:
|
||||
handle = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(handle.cleanup)
|
||||
return Path(handle.name)
|
||||
|
||||
def serve(self, categorized: bool = True) -> None:
|
||||
root = self._tempdir()
|
||||
files = []
|
||||
for path, (data, category) in self.FILES.items():
|
||||
(root / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / path).write_bytes(data)
|
||||
entry = {"dest": Path(path).name, "repo_path": path, "size": len(data),
|
||||
"sha1": hashlib.sha1(data).hexdigest(), "cores": None,
|
||||
"systems": ["sys-a"]}
|
||||
if category:
|
||||
entry["category"] = category
|
||||
files.append(entry)
|
||||
manifest = {
|
||||
"manifest_version": 2, "platform": "retroarch", "display_name": "RetroArch",
|
||||
"files": files,
|
||||
"omitted_files": [{"dest": "level.dat", "name": "level.dat", "system": "sys-a",
|
||||
"required": True, "reason": "not_found",
|
||||
"systems": ["sys-a"], "category": "game_data"}],
|
||||
}
|
||||
if categorized:
|
||||
manifest["categorized"] = True
|
||||
(root / "install").mkdir()
|
||||
(root / "install" / "retroarch.json").write_text(json.dumps(manifest))
|
||||
handler = functools.partial(_Quiet, directory=str(root))
|
||||
httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
self.addCleanup(httpd.server_close)
|
||||
self.addCleanup(httpd.shutdown)
|
||||
home = self._tempdir()
|
||||
self.bios = home / "bios"
|
||||
self.bios.mkdir()
|
||||
self.env = {**os.environ, "HOME": str(home), "RETROBIOS_OS": "linux",
|
||||
"RETROBIOS_BASE_URL": f"http://127.0.0.1:{httpd.server_address[1]}"}
|
||||
|
||||
def run_installer(self, *argv: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, "install.py", "--platform", "retroarch", "--dest",
|
||||
str(self.bios), "--no-input", "--jobs", "1", *argv],
|
||||
cwd=REPO_ROOT, env=self.env, capture_output=True, text=True, timeout=120,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_install_leaves_it_out_and_names_it(self):
|
||||
self.serve()
|
||||
proc = self.run_installer("--no-game-data")
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
||||
self.assertTrue((self.bios / "a.bin").exists())
|
||||
self.assertFalse((self.bios / "game.wad").exists())
|
||||
self.assertIn("Left out as game data (--no-game-data): 2 files: game.wad, level.dat",
|
||||
proc.stdout)
|
||||
|
||||
def test_check_and_dry_run_leave_it_out(self):
|
||||
self.serve()
|
||||
for mode, listed in (("--check", "1 file to download:"),
|
||||
("--dry-run", "Would download 1 file")):
|
||||
with self.subTest(mode=mode):
|
||||
proc = self.run_installer("--no-game-data", mode)
|
||||
self.assertIn(
|
||||
"Left out as game data (--no-game-data): 2 files: game.wad, level.dat",
|
||||
proc.stdout,
|
||||
)
|
||||
self.assertIn(listed, proc.stdout)
|
||||
self.assertNotIn("missing: game.wad", proc.stdout)
|
||||
self.assertFalse((self.bios / "a.bin").exists())
|
||||
|
||||
def test_the_default_installs_everything(self):
|
||||
self.serve()
|
||||
proc = self.run_installer()
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
||||
self.assertTrue((self.bios / "game.wad").exists())
|
||||
self.assertNotIn("Left out as game data", proc.stdout)
|
||||
|
||||
def test_events_name_what_was_left_out(self):
|
||||
self.serve()
|
||||
proc = self.run_installer("--no-game-data", "--check", "--events")
|
||||
events = [json.loads(line) for line in proc.stdout.splitlines()]
|
||||
summary = next(e for e in events if e["event"] == "summary")
|
||||
self.assertEqual(list(summary), ["event", *install.EVENT_KEYS["summary"]])
|
||||
self.assertEqual(summary["left_out"], [{
|
||||
"category": "game_data", "label": "game data", "files": 2,
|
||||
"dests": ["game.wad", "level.dat"],
|
||||
}])
|
||||
# The omission is left out, not counted as a file the collection lacks.
|
||||
self.assertEqual(summary["skipped"], [])
|
||||
check = next(e for e in events if e["event"] == "check")
|
||||
self.assertEqual(check["files"], 1)
|
||||
|
||||
def test_a_list_without_categories_is_refused(self):
|
||||
self.serve(categorized=False)
|
||||
proc = self.run_installer("--no-game-data", "--check")
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
self.assertIn("records no file category", proc.stderr)
|
||||
|
||||
def test_listings_that_read_no_file_list_refuse_it(self):
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "install.py", "--list-platforms", "--no-game-data"],
|
||||
cwd=REPO_ROOT, capture_output=True, text=True, timeout=60, check=False,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 2)
|
||||
self.assertIn("--no-game-data", proc.stderr)
|
||||
|
||||
def test_the_cut_function(self):
|
||||
files = [{"dest": "a"}, {"dest": "b", "category": "game_data"}]
|
||||
omitted = [{"dest": "c", "category": "game_data"}, {"dest": "d"}]
|
||||
kept, kept_omitted, left = install.leave_out_game_data(files, omitted)
|
||||
self.assertEqual([e["dest"] for e in kept], ["a"])
|
||||
self.assertEqual([e["dest"] for e in kept_omitted], ["d"])
|
||||
self.assertEqual([e["dest"] for e in left], ["b", "c"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+4
-3
@@ -141,7 +141,7 @@ class EventRecords(unittest.TestCase):
|
||||
json.dumps({"event": "summary", "mode": "install", "installed": 0,
|
||||
"up_to_date": 4, "pending": 1,
|
||||
"failed": [{"dest": "x.bin", "reason": "HTTP 404"}],
|
||||
"skipped": [], "interrupted": False}),
|
||||
"skipped": [], "left_out": [], "interrupted": False}),
|
||||
json.dumps({"event": "end", "code": 2}),
|
||||
]
|
||||
out, summary = io.StringIO(), io.StringIO()
|
||||
@@ -228,7 +228,8 @@ if check:
|
||||
present=3 if done else 1, missing=0 if done else 1, wrong_hash=0 if done else 1,
|
||||
bytes_needed=0 if done else 300, data_missing=0, not_installable=1)
|
||||
emit("summary", mode="check", installed=0, up_to_date=3 if done else 1,
|
||||
pending=0 if done else 2, failed=[], skipped=[], interrupted=False)
|
||||
pending=0 if done else 2, failed=[], skipped=[], left_out=[],
|
||||
interrupted=False)
|
||||
end(0 if done else 1)
|
||||
emit("download", platform="retroarch", files=2, bytes=300)
|
||||
for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1):
|
||||
@@ -237,7 +238,7 @@ for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1):
|
||||
(state / "installed").write_text("")
|
||||
emit("summary", mode="install", installed=2, up_to_date=1, pending=0, failed=[],
|
||||
skipped=[{"reason": "not_found", "label": "not found", "files": 1, "required": 1}],
|
||||
interrupted=False)
|
||||
left_out=[], interrupted=False)
|
||||
end(0)
|
||||
'''
|
||||
|
||||
|
||||
@@ -44,6 +44,10 @@ class VerifyRefusesWhatAModeDoesNotRead(unittest.TestCase):
|
||||
(("--platform", "retroarch"), "--include-archived"),
|
||||
(("--emulator", "handy"), "--include-archived"),
|
||||
(("--platform", "retroarch"), "--standalone"),
|
||||
# Game data is categorized against a platform's list and its cores.
|
||||
(("--emulator", "handy"), "--no-game-data"),
|
||||
(("--system", "atari-lynx"), "--no-game-data"),
|
||||
(("--list-systems",), "--no-game-data"),
|
||||
)
|
||||
|
||||
def test_each_combination_is_refused(self):
|
||||
|
||||
@@ -235,6 +235,39 @@ visible.
|
||||
Variants compose with `--split`, `--target` and `--manifest`.
|
||||
|
||||
|
||||
## Firmware without game data
|
||||
|
||||
The firmware an emulator needs comes first. A platform list can also name the
|
||||
files a game reads (EmuELEC lists the Doom WADs), and a core profile marks the
|
||||
data of the games it runs `category: game_data`: MAME's software lists, the
|
||||
RPG Maker run-time packages, a game engine's own files. `--no-game-data`
|
||||
leaves both out and keeps the firmware:
|
||||
|
||||
```bash
|
||||
python scripts/generate_pack.py --platform emuelec --no-game-data
|
||||
python scripts/generate_pack.py --platform emuelec --no-game-data --verify-packs
|
||||
python scripts/verify.py --platform emuelec --no-game-data
|
||||
```
|
||||
|
||||
A platform entry is game data when the only cores of the platform that read it
|
||||
read it as game data (see [Verification modes](verification-modes.md) for how
|
||||
entries are categorized). An entry that some core reads as firmware stays.
|
||||
The build names every file it left out and why: the platform's entries with
|
||||
the cores that read them as game data, then each core with the game data
|
||||
files its profile declares.
|
||||
|
||||
The pack is named `EmuELEC_4.8_NoGameData_BIOS_Pack.zip` and its README says
|
||||
it was narrowed. `verify.py --no-game-data` reports on the same file set, and
|
||||
`--verify-packs --no-game-data` checks the pack against it. The flag composes
|
||||
with `--region`, `--target`, `--required-only`, `--source`, `--one-per-slot`,
|
||||
`--split`, `--system` and `--manifest`
|
||||
(`install/emuelec_nogamedata.json`). It needs `--platform` or `--all` and is
|
||||
refused with `--emulator`, `--system` alone, `--from-md5` and
|
||||
`--all-variants`: game data is categorized against a platform's list and the
|
||||
cores it runs. The installer's `--no-game-data` makes the same cut from the
|
||||
full manifest, where each game data file carries `"category": "game_data"`.
|
||||
|
||||
|
||||
## Hardware target filtering
|
||||
|
||||
### What targets are
|
||||
|
||||
+22
-1
@@ -170,8 +170,28 @@ python install.py --platform retroarch --list-systems
|
||||
python install.py --platform retroarch --system sony-playstation,nintendo-gba
|
||||
python install.py --platform retroarch --system sony-playstation --core pcsx_rearmed
|
||||
python install.py --platform recalbox --region us,eu,jp
|
||||
python install.py --platform emuelec --no-game-data
|
||||
```
|
||||
|
||||
### Firmware without game data
|
||||
|
||||
Some platform lists name the files a game reads beside the firmware its
|
||||
emulator needs (the Doom WADs in EmuELEC's list), and the cores' profiles mark
|
||||
the data files of the games they run `category: game_data`: MAME's software
|
||||
lists, a game engine's own data. The manifest marks each of these files
|
||||
`"category": "game_data"`, and firmware carries no category.
|
||||
|
||||
`--no-game-data` leaves them out of the install, `--check` and `--dry-run`,
|
||||
and names what it left out with its category (ten names, `--verbose` for all):
|
||||
|
||||
```text
|
||||
Left out as game data (--no-game-data): 2436 files: ports/doom/doom1.wad, ports/doom2/doom2.wad, ...
|
||||
```
|
||||
|
||||
It is the cut `generate_pack.py --no-game-data` makes, and a test holds the
|
||||
two to the same file list. A file list that records no category is refused
|
||||
rather than read as having no game data.
|
||||
|
||||
### Installed cores
|
||||
|
||||
`--installed-cores` reads the cores a RetroArch already has and installs what
|
||||
@@ -242,6 +262,7 @@ what the options name, everything by default.
|
||||
| `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names |
|
||||
| `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included |
|
||||
| `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` |
|
||||
| `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) |
|
||||
| `--list-systems` | Print a platform's systems with the files and size each holds |
|
||||
| `--list-cores` | Print the cores whose extra files a platform installs |
|
||||
| `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for |
|
||||
@@ -288,7 +309,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is
|
||||
| `download` | `platform`, `files`, `bytes` | Before the downloads start |
|
||||
| `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive |
|
||||
| `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file. `reason` says why one failed, and is empty otherwise |
|
||||
| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply. Not written for a listing |
|
||||
| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. Not written for a listing |
|
||||
| `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) |
|
||||
|
||||
An option refused before the run starts (`install.py: error:`, code 2) writes
|
||||
|
||||
@@ -40,7 +40,7 @@ Every module in `tests/`, by the area it guards.
|
||||
| Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` |
|
||||
| File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` |
|
||||
| Verification, cross-reference and profile checks | synthetic + profiles | `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` |
|
||||
| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
|
||||
| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
|
||||
| Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
|
||||
| Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
|
||||
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` |
|
||||
|
||||
+10
-1
@@ -125,6 +125,7 @@ python scripts/verify.py --system atari-lynx # single system
|
||||
python scripts/verify.py --platform batocera --system psx # one system of a platform pack
|
||||
python scripts/verify.py --platform retroarch --target switch # filter by hardware
|
||||
python scripts/verify.py --platform recalbox --region us # regional priority list
|
||||
python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named
|
||||
python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds
|
||||
python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build
|
||||
python scripts/verify.py --list-emulators # list all emulators
|
||||
@@ -153,7 +154,9 @@ Verification modes per platform:
|
||||
Full details and severity mapping: [verification modes](verification-modes.md).
|
||||
`--region` narrows the report to the file set a regional pack would carry, and a
|
||||
mode that cannot apply it refuses it rather than ignoring it: see
|
||||
[region filtering](advanced-usage.md#region-filtering).
|
||||
[region filtering](advanced-usage.md#region-filtering). `--no-game-data` narrows
|
||||
it to the pack built with the same flag and names what it left out. It needs
|
||||
`--platform` or `--all`.
|
||||
|
||||
`--db`, `--platforms-dir` and `--emulators-dir` point the run at another
|
||||
database or source tree, which is how the tests drive it against fixtures.
|
||||
@@ -173,6 +176,7 @@ python scripts/generate_pack.py --system atari-lynx
|
||||
python scripts/generate_pack.py --platform retroarch --system sony-playstation
|
||||
python scripts/generate_pack.py --platform batocera --system psx
|
||||
python scripts/generate_pack.py --platform batocera --required-only
|
||||
python scripts/generate_pack.py --platform emuelec --no-game-data
|
||||
python scripts/generate_pack.py --platform retroarch --split
|
||||
python scripts/generate_pack.py --platform retroarch --split --group-by manufacturer
|
||||
|
||||
@@ -232,6 +236,11 @@ own verification mode ([discrepancy detection](verification-modes.md#discrepancy
|
||||
takes our ids only.
|
||||
- `--required-only`: exclude optional files and the data directories declared
|
||||
`required: false`, keep only required
|
||||
- `--no-game-data`: firmware only. Leaves out the platform's entries
|
||||
categorized game data and the cores' `category: game_data` files, and names
|
||||
each one. Tags the pack `_NoGameData`. `verify.py --no-game-data` and
|
||||
`--verify-packs --no-game-data` check the same file set
|
||||
([details](advanced-usage.md#firmware-without-game-data))
|
||||
- `--split`: generate one ZIP per system instead of one big pack
|
||||
- `--split --group-by manufacturer`: group split packs by manufacturer (Sony, Nintendo, Sega...)
|
||||
- `--from-md5`: look up a hash in the database, or build a custom pack with
|
||||
|
||||
@@ -306,6 +306,11 @@ pack errors. A core the platform names that has no profile yet makes nothing unr
|
||||
The same goes for a system none of the examined cores emulates: in both cases the
|
||||
absence of a reader proves nothing.
|
||||
|
||||
The same category filters: `--no-game-data` builds, verifies and installs the
|
||||
firmware without the game data, and the install manifest marks each game data
|
||||
file `"category": "game_data"`
|
||||
([details](advanced-usage.md#firmware-without-game-data)).
|
||||
|
||||
|
||||
## File resolution chain
|
||||
|
||||
|
||||
Reference in new issue
Block a user