feat: choose systems, cores and regions to install

This commit is contained in:
Abdessamad Derraz committed 2026-10-10 04:31:57 +02:00
1 parent b0bd20f403
commit ece2c06aaf
13 files changed
+1426 -34

No files matched your search

+1 -1
View File
@@ -8,7 +8,7 @@ param(
$ErrorActionPreference = "Stop"
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
$defaultInstallSha256 = "f23eee534cfbf88187f39e47065165c8cc897efb06206a7daadc461de241f8d4"
$defaultInstallSha256 = "a1bae5feab3e85224526eaab7ed99ae0d711081c925aa520d3768532cd11a797"
$maximumInstallerBytes = 2MB
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
$temporary = $null
+471 -3
View File
@@ -10,6 +10,7 @@ Usage:
python install.py --platform retroarch --dest ~/custom/bios
python install.py --check
python install.py --list-platforms
python install.py --platform retroarch --system sony-playstation --region us
"""
from __future__ import annotations
@@ -93,6 +94,43 @@ OMISSION_REASONS = {
_SHA1_RE = re.compile(r"^[0-9a-fA-F]{40}$")
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
# The region vocabulary of scripts/region.py, which this file cannot import.
# A test holds the two copies equal.
WORLD_REGION = "world"
REGION_TREE: dict[str, frozenset] = {
"north-america": frozenset({"canada"}),
"latin-america": frozenset({"brazil", "mexico", "argentina"}),
"europe": frozenset({
"uk", "france", "germany", "italy", "spain", "netherlands", "portugal",
"greece", "poland", "russia", "sweden", "norway", "denmark", "finland",
}),
"asia": frozenset({
"japan", "south-korea", "china", "taiwan", "hong-kong", "singapore",
"india", "asia-ntsc", "asia-pal",
}),
"oceania": frozenset({"australia", "new-zealand"}),
}
REGION_ALIASES: dict[str, str] = {
"jp": "japan",
"ntsc-j": "japan",
"us": "north-america",
"usa": "north-america",
"na": "north-america",
"ntsc-u": "north-america",
"eu": "europe",
"pal": "europe",
"kr": "south-korea",
"korea": "south-korea",
"auto": WORLD_REGION,
"gb": "uk",
}
_REGION_PARENT = {
member: parent for parent, members in REGION_TREE.items() for member in members
}
REGIONS = frozenset({WORLD_REGION} | set(REGION_TREE) | set(_REGION_PARENT))
MAX_SELECTION_TAGS = 1000
SELECTION_FIELDS = ("systems", "regions", "region_groups", "region_system_groups")
# Platforms with a manifest in install/. Manifest URLs are case sensitive,
# so user input is normalized against this list before any fetch.
AVAILABLE_PLATFORMS = (
@@ -813,6 +851,19 @@ def _destination_path(root: Path, value: object) -> Path:
return root.resolve() / Path(*relative.parts)
def _check_tags(entry: dict, field: str, where: str) -> None:
"""A selection field is absent or a short list of short strings."""
value = entry.get(field)
if value is None:
return
if (
not isinstance(value, list)
or len(value) > MAX_SELECTION_TAGS
or not all(isinstance(tag, str) and 0 < len(tag) <= 256 for tag in value)
):
raise ValueError(f"invalid {field} for {where}")
def _validate_manifest(data: object, plat: str) -> dict:
"""Validate the untrusted install-manifest boundary using stdlib only."""
if not isinstance(data, dict):
@@ -871,6 +922,8 @@ def _validate_manifest(data: object, plat: str) -> dict:
not isinstance(cores, list) or not all(isinstance(core, str) for core in cores)
):
raise ValueError(f"invalid cores list for {dest}")
for field in SELECTION_FIELDS:
_check_tags(entry, field, dest)
declared_total_files = data.get("total_files")
if declared_total_files is not None and declared_total_files != len(files):
@@ -908,6 +961,8 @@ def _validate_manifest(data: object, plat: str) -> dict:
or not all(isinstance(core, str) for core in cores)
):
raise ValueError(f"invalid omitted cores list for {dest}")
for field in SELECTION_FIELDS:
_check_tags(entry, field, dest)
declared_total_omitted = data.get("total_omitted")
if (
declared_total_omitted is not None
@@ -1024,6 +1079,206 @@ def _filter_by_target(
return result
def canonical_region(raw: str) -> str:
"""One region name or alias, spelled the way manifests spell it."""
key = raw.strip().lower()
key = REGION_ALIASES.get(key, key)
if key not in REGIONS:
raise ValueError(f"unknown region: {raw.strip()}")
return key
def _region_comparable(a: str, b: str) -> bool:
return a == b or _REGION_PARENT.get(a) == b or _REGION_PARENT.get(b) == a
def _region_rank(file_regions: set[str], requested: list[str]) -> int:
for index, wanted in enumerate(requested):
if any(_region_comparable(wanted, region) for region in file_regions):
return index
return len(requested)
def region_drops(
entries: list[dict], requested: list[str], systems: "list[str] | None" = None
) -> set[str]:
"""Destinations a region priority list withdraws.
The pack builder's rule (region.resolve_region_drops), replayed over the
groups the manifest records. Per group, the best-ranked regional file
stays; a world file beats unmatched regional ones; when nothing matches
and no world file exists, every regional file stays, so a group is never
emptied. A file with no region never competes and always stays.
A core extra competes whatever systems are chosen; a platform's own
declaration competes only when its system is kept, as in the pack.
"""
if not requested:
return set()
kept_systems = set(systems) if systems else None
groups: dict[str, list[tuple[set, str]]] = {}
for entry in entries:
regions = entry.get("regions")
if not regions:
continue
joined = list(entry.get("region_groups") or [])
joined += [
group for group in entry.get("region_system_groups") or []
if kept_systems is None or group in kept_systems
]
for group in joined:
groups.setdefault(group, []).append((set(regions), entry["dest"]))
keep: set[str] = set()
drop: set[str] = set()
for members in groups.values():
world = {dest for regions, dest in members if WORLD_REGION in regions}
regional = [
(_region_rank(regions, requested), dest)
for regions, dest in members
if WORLD_REGION not in regions
]
keep |= world
if not regional:
continue
matched = [(rank, dest) for rank, dest in regional if rank < len(requested)]
if matched:
best = min(rank for rank, _dest in matched)
keep |= {dest for rank, dest in matched if rank == best}
drop |= {dest for _rank, dest in regional}
elif world:
drop |= {dest for _rank, dest in regional}
else:
keep |= {dest for _rank, dest in regional}
return drop - keep
def entry_systems(entry: dict) -> list[str]:
"""The systems a file serves: a list on files, one name on omissions."""
systems = entry.get("systems")
if systems:
return list(systems)
return [entry["system"]] if entry.get("system") else []
def narrow(
files: list[dict],
omitted: list[dict],
systems: list[str],
cores: list[str],
regions: list[str],
) -> tuple[list[dict], list[dict]]:
"""Keep what the chosen systems, cores and regions need.
A core choice keeps the platform's own list and narrows only the files
cores load, the rule --target already follows. Regions are then decided
the way the pack builder decides them under --system: over the chosen
systems' declarations and every core extra. The system filter comes last.
"""
if cores:
files = _filter_by_target(files, cores)
omitted = _filter_by_target(omitted, cores)
if regions:
dropped = region_drops(files + omitted, regions, systems)
files = [f for f in files if f["dest"] not in dropped]
omitted = [o for o in omitted if o["dest"] not in dropped]
if systems:
wanted = set(systems)
files = [f for f in files if wanted & set(entry_systems(f))]
omitted = [o for o in omitted if wanted & set(entry_systems(o))]
return files, omitted
def records_selection(files: list[dict]) -> bool:
"""Whether a manifest carries what narrowing reads; older ones do not."""
return any("systems" in entry for entry in files)
def available_choices(files: list[dict], omitted: list[dict]) -> dict[str, list[str]]:
"""The systems, cores and regions a manifest lets one narrow to."""
systems = {s for e in files + omitted for s in entry_systems(e)}
cores = {c for f in files for c in (f.get("cores") or [])}
regions = {
r for e in files + omitted for r in (e.get("regions") or [])
if r != WORLD_REGION
}
return {
"systems": sorted(systems),
"cores": sorted(cores),
"regions": sorted(regions),
}
def resolve_choices(values: list[str], known: list[str], label: str) -> list[str]:
"""Names given on the command line, matched without regard to case.
Each value is one name or a comma-separated list. A whole value that is
itself a name wins over splitting it: Recalbox names one system
`msx1,msx2,msxturbor`. An unknown name is refused, never ignored, since
carrying on would install something other than what was asked.
"""
by_lower = {name.lower(): name for name in known}
chosen: list[str] = []
unknown: list[str] = []
for value in values:
value = value.strip()
tokens = [value] if value.lower() in by_lower else value.split(",")
for token in (t.strip() for t in tokens):
if not token:
continue
name = by_lower.get(token.lower())
if name is None:
unknown.append(token)
elif name not in chosen:
chosen.append(name)
if unknown:
available = ", ".join(known) if known else "none"
raise ValueError(
f"unknown {label}: {', '.join(unknown)} (available: {available})"
)
if values and not chosen:
raise ValueError(f"--{label} needs at least one name")
return chosen
def resolve_regions(values: list[str]) -> list[str]:
"""A region priority list, best first, from names or aliases."""
chosen: list[str] = []
for value in values:
for token in value.split(","):
if token.strip():
region = canonical_region(token)
if region not in chosen:
chosen.append(region)
if values and not chosen:
raise ValueError("--region needs at least one region")
return chosen
def parse_selection(text: str, count: int, ordered: bool = False) -> list[int]:
"""Numbers typed at a prompt: `3`, `1,4`, `2-5`, or `all`, 1-based.
Returned sorted, or in the typed order when the order is a priority.
"""
if text.strip().lower() in ("all", "*"):
return list(range(1, count + 1))
chosen: list[int] = []
for token in (t.strip() for t in text.split(",")):
if not token:
continue
low, sep, high = token.partition("-")
try:
start = int(low)
end = int(high) if sep else start
except ValueError:
raise ValueError(f"not a number or range: {token}") from None
if start > end or start < 1 or end > count:
raise ValueError(f"out of range: {token} (1-{count})")
for number in range(start, end + 1):
if number not in chosen:
chosen.append(number)
return chosen if ordered else sorted(chosen)
def _digest_file(path: Path, algorithms: tuple[str, ...]) -> dict[str, str]:
"""Compute several digests of a file in a single read.
@@ -1498,6 +1753,152 @@ def _prompt_platform_choice(
return platforms
def _ask(prompt: str) -> str:
try:
return input(prompt).strip()
except (EOFError, KeyboardInterrupt):
print()
sys.exit(0)
def _sized(files: list[dict]) -> str:
return f"{_plural(len(files), 'file')}, {format_size(sum(f.get('size', 0) for f in files))}"
def _pick(title: str, rows: list[tuple[str, list[dict]]], ordered: bool = False) -> list[str]:
"""Numbered list of names; an empty answer keeps every one."""
if not rows:
return []
print(f"\n{title}")
width = len(str(len(rows)))
name_width = min(max(len(name) for name, _files in rows), 32)
for index, (name, files) in enumerate(rows, 1):
print(f" {index:>{width}}) {name:<{name_width}} {_sized(files)}")
hint = "order of preference, e.g. 2,1" if ordered else "e.g. 1,3-5"
while True:
text = _ask(f"Numbers ({hint}), Enter for all, q to quit: ")
if not text:
return []
if text.lower() == "q":
sys.exit(0)
try:
chosen = parse_selection(text, len(rows), ordered=ordered)
except ValueError as exc:
print(f" {exc}")
continue
if chosen:
return [rows[number - 1][0] for number in chosen]
def _prompt_custom_selection(
plat: str, files: list[dict], omitted: list[dict]
) -> "tuple[list[str], list[str], list[str]] | None":
"""Offer to narrow the install by system, core and region.
Enter installs everything, the default most runs want. Returns None
then, or the three choices; each empty list keeps everything on its axis.
"""
print(f"\n{plat.capitalize()}: {_sized(files)}.")
print(" Enter) install everything")
print(" c) choose systems, cores and regions")
print(" q) quit")
while True:
answer = _ask("> ").lower()
if answer == "":
return None
if answer == "q":
sys.exit(0)
if answer == "c":
break
choices = available_choices(files, omitted)
systems = _pick("Systems:", [
(name, [f for f in files if name in entry_systems(f)])
for name in choices["systems"]
])
kept, kept_omitted = narrow(files, omitted, systems, [], [])
cores: list[str] = []
core_names = available_choices(kept, kept_omitted)["cores"]
if core_names:
print("\nThe platform's own files for these systems are always kept;")
print("the extra files each core loads can be narrowed.")
cores = _pick("Cores:", [
(name, [f for f in kept if name in (f.get("cores") or [])])
for name in core_names
])
kept, kept_omitted = narrow(kept, kept_omitted, [], cores, [])
regions: list[str] = []
region_names = available_choices(kept, kept_omitted)["regions"]
if region_names:
print("\nSome systems have one BIOS per region. Keeping only the")
print("preferred region removes the others where a match exists.")
regions = _pick("Regions:", [
(name, [f for f in kept if name in (f.get("regions") or [])])
for name in region_names
], ordered=True)
return systems, cores, regions
def _same_choice_hint(plat: str, systems: list[str], cores: list[str], regions: list[str]) -> str:
"""The options that repeat a choice made at the prompts."""
def quoted(value: str) -> str:
return f'"{value}"' if any(c in value for c in " ;&|") else value
parts = [f"--platform {plat}"]
parts += [f"--system {quoted(name)}" for name in systems]
parts += [f"--core {quoted(name)}" for name in cores]
if regions:
parts.append(f"--region {','.join(regions)}")
return " ".join(parts)
def _print_choices(
plat: str, files: list[dict], omitted: list[dict], systems: bool
) -> None:
"""What --system or --core accepts for a platform, with what each holds."""
choices = available_choices(files, omitted)
names = choices["systems"] if systems else choices["cores"]
if not names:
print(f" {plat} records no {'systems' if systems else 'cores'}")
return
width = min(max(len(name) for name in names), 40)
for name in names:
held = (
[f for f in files if name in entry_systems(f)]
if systems
else [f for f in files if name in (f.get("cores") or [])]
)
print(f" {name:<{width}} {_sized(held)}")
def _requested_narrowing(
args: argparse.Namespace, plat: str, files: list[dict], omitted: list[dict]
) -> "tuple[list[str], list[str], list[str]] | None":
"""The systems, cores and regions named on the command line, checked."""
if not (args.system or args.core or args.region):
return None
if not records_selection(files):
# Without the fields, a region filter would match nothing and install
# everything: refused rather than ignored.
print(
f"Error: the {plat} file list records no systems or regions; "
"narrowing needs a newer one",
file=sys.stderr,
)
sys.exit(1)
choices = available_choices(files, omitted)
try:
systems = resolve_choices(args.system or [], choices["systems"], "system")
cores = resolve_choices(args.core or [], choices["cores"], "core")
regions = resolve_regions(args.region or [])
except ValueError as exc:
print(f"Error: {plat}: {exc}", file=sys.stderr)
sys.exit(1)
return systems, cores, regions
def main() -> None:
"""Entry point."""
parser = argparse.ArgumentParser(
@@ -1547,10 +1948,56 @@ def main() -> None:
action="store_true",
help="opt in to copies into detected standalone-emulator directories",
)
parser.add_argument(
"--system",
action="append",
metavar="NAME",
help="install only these systems (repeat or comma-separate; see --list-systems)",
)
parser.add_argument(
"--core",
action="append",
metavar="NAME",
help="keep only the extra files these cores load (see --list-cores)",
)
parser.add_argument(
"--region",
action="append",
metavar="LIST",
help="preferred regions, best first, e.g. us,eu,jp: keep one BIOS per "
"region where a system has several",
)
parser.add_argument(
"--list-systems",
action="store_true",
help="list a platform's systems with their file counts and exit",
)
parser.add_argument(
"--list-cores",
action="store_true",
help="list the cores whose extra files a platform installs and exit",
)
parser.add_argument(
"--no-input",
action="store_true",
help="never ask a question; fail where an answer is needed",
)
args = parser.parse_args()
if not 1 <= args.jobs <= 32:
parser.error("--jobs must be between 1 and 32")
if args.region:
try:
resolve_regions(args.region)
except ValueError as exc:
parser.error(f"{exc} (known: {', '.join(sorted(REGIONS))})")
for option, values in (("--system", args.system), ("--core", args.core)):
if values is not None and not any(
token.strip() for value in values for token in value.split(",")
):
parser.error(f"{option} needs at least one name")
interactive = sys.stdin.isatty() and not args.no_input
listing = args.list_targets or args.list_systems or args.list_cores
print("RetroBIOS\n")
os_type = detect_os()
@@ -1598,7 +2045,7 @@ def main() -> None:
platforms = detect_platforms(os_type)
if not platforms:
print(" No supported platform detected.")
if not sys.stdin.isatty() or args.list_targets:
if not interactive or listing:
for line in _manual_usage_hint(os_type):
print(line)
sys.exit(1)
@@ -1606,7 +2053,7 @@ def main() -> None:
for name, path in platforms:
print(f" Found {name.capitalize()} at {path}")
if len(platforms) > 1 and not args.list_targets and sys.stdin.isatty():
if len(platforms) > 1 and not listing and interactive:
platforms = _prompt_platform_choice(platforms)
total_downloaded = 0
@@ -1672,6 +2119,27 @@ def main() -> None:
omitted_files = _filter_by_target(omitted_files, target_cores)
print(f" Filtered {before} -> {len(files)} files for target {args.target}")
selection = _requested_narrowing(args, plat_name, files, omitted_files)
if (
selection is None
and interactive
and not args.check
and not listing
and records_selection(files)
):
selection = _prompt_custom_selection(plat_name, files, omitted_files)
if selection is not None:
print("\n Same choice without questions: "
+ _same_choice_hint(plat_name, *selection))
if selection is not None:
before = len(files)
files, omitted_files = narrow(files, omitted_files, *selection)
print(f" Narrowed {before} -> {len(files)} files")
if args.list_systems or args.list_cores:
_print_choices(plat_name, files, omitted_files, args.list_systems)
continue
total_size = sum(f.get("size", 0) for f in files)
print(f" {len(files)} files ({format_size(total_size)})")
if omitted_files:
@@ -1764,7 +2232,7 @@ def main() -> None:
"(use --standalone-copies to opt in)."
)
if not args.check and not args.list_targets:
if not args.check and not listing:
_report_outcome(
installed_paths,
total_downloaded,
+1 -1
View File
@@ -18,7 +18,7 @@ esac
TEMP_INSTALLER=""
TEMP_DIRECTORY=""
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
DEFAULT_INSTALL_SHA256="f23eee534cfbf88187f39e47065165c8cc897efb06206a7daadc461de241f8d4"
DEFAULT_INSTALL_SHA256="a1bae5feab3e85224526eaab7ed99ae0d711081c925aa520d3768532cd11a797"
MAX_INSTALLER_BYTES=2097152
cleanup() {
+166
View File
@@ -146,6 +146,89 @@
"uniqueItems": true
}
]
},
"regions": {
"description": "The regions the emulator code selects the file for. Absent when it never competes.",
"type": "array",
"items": {
"type": "string",
"enum": [
"argentina",
"asia",
"asia-ntsc",
"asia-pal",
"australia",
"brazil",
"canada",
"china",
"denmark",
"europe",
"finland",
"france",
"germany",
"greece",
"hong-kong",
"india",
"italy",
"japan",
"latin-america",
"mexico",
"netherlands",
"new-zealand",
"north-america",
"norway",
"oceania",
"poland",
"portugal",
"russia",
"singapore",
"south-korea",
"spain",
"sweden",
"taiwan",
"uk",
"world"
]
},
"minItems": 1,
"maxItems": 1000,
"uniqueItems": true
},
"region_groups": {
"description": "Groups the file competes in as a core extra, whatever systems are chosen.",
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 256
},
"minItems": 1,
"maxItems": 1000,
"uniqueItems": true
},
"region_system_groups": {
"description": "Groups the file competes in as the platform's own declaration: present only when that system is kept.",
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 256
},
"minItems": 1,
"maxItems": 1000,
"uniqueItems": true
},
"systems": {
"description": "Every platform system that declares the file, for the installer's --system.",
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 256
},
"minItems": 1,
"maxItems": 1000,
"uniqueItems": true
}
}
}
@@ -203,6 +286,89 @@
"release_asset": {
"type": "string",
"pattern": "^[^/\\\\]+$"
},
"systems": {
"description": "Every platform system that declares the file, for the installer's --system.",
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 256
},
"minItems": 1,
"maxItems": 1000,
"uniqueItems": true
},
"regions": {
"description": "The regions the emulator code selects the file for. Absent when it never competes.",
"type": "array",
"items": {
"type": "string",
"enum": [
"argentina",
"asia",
"asia-ntsc",
"asia-pal",
"australia",
"brazil",
"canada",
"china",
"denmark",
"europe",
"finland",
"france",
"germany",
"greece",
"hong-kong",
"india",
"italy",
"japan",
"latin-america",
"mexico",
"netherlands",
"new-zealand",
"north-america",
"norway",
"oceania",
"poland",
"portugal",
"russia",
"singapore",
"south-korea",
"spain",
"sweden",
"taiwan",
"uk",
"world"
]
},
"minItems": 1,
"maxItems": 1000,
"uniqueItems": true
},
"region_groups": {
"description": "Groups the file competes in as a core extra, whatever systems are chosen.",
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 256
},
"minItems": 1,
"maxItems": 1000,
"uniqueItems": true
},
"region_system_groups": {
"description": "Groups the file competes in as the platform's own declaration: present only when that system is kept.",
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 256
},
"minItems": 1,
"maxItems": 1000,
"uniqueItems": true
}
},
"oneOf": [
+36 -10
View File
@@ -64,6 +64,7 @@ from common import (
import packresolve
from refresh_data_dirs import cache_lock
import region as region_mod
from manifestselect import SelectionIndex
import slot as slot_mod
import slots
import split_pack
@@ -2952,6 +2953,7 @@ def _manifest_entry(
cores: list[str] | None,
db: dict,
repo_root: str,
selection_fields: dict,
) -> tuple[dict | None, int]:
"""The download record of a resolved file, and the size it occupies.
@@ -2962,6 +2964,7 @@ def _manifest_entry(
scan, a hash computed wrong) has neither a repo path nor a release
asset. None then, so the caller records an omission instead of a dead
entry that makes the installer refuse the whole manifest.
`selection_fields` carries what the installer narrows on.
"""
sha1 = ""
sha256 = ""
@@ -2982,6 +2985,7 @@ def _manifest_entry(
"size": file_size,
"repo_path": repo_path,
"cores": cores,
**selection_fields,
}
if is_release_asset:
entry["storage"] = "release"
@@ -3007,6 +3011,7 @@ def _manifest_core_entries(
omitted_by_destination: dict,
record_omission,
pack_only_sizes: list[int],
selection: SelectionIndex,
required_only: bool = False,
) -> int:
"""Add the files a platform's cores need but its list does not name.
@@ -3073,6 +3078,7 @@ def _manifest_core_entries(
entry, file_size = _manifest_entry(
local_path, manifest_dest, [source_emu] if source_emu else [],
db, repo_root,
selection.fields(dest, fe.get("name", "")),
)
if entry is None:
systems = _extra_system_ids(fe)
@@ -3095,7 +3101,7 @@ def _manifest_core_entries(
return total_size
def _manifest_region_drops(
def _manifest_selection(
config: dict,
pack_systems: dict,
emulators_dir: str,
@@ -3106,10 +3112,13 @@ def _manifest_region_drops(
source: str,
required_only: bool,
regions: list[str] | None,
) -> set[str]:
"""Destinations a region list removes, grouped as the pack groups them."""
if not regions:
return set()
) -> tuple[set[str], SelectionIndex]:
"""Destinations a region list removes, and the groups the installer replays.
Grouped as the pack groups them, so a region the installer applies to
the full manifest withdraws what `--region` withdraws from the pack.
"""
extras: list[dict] = []
region_groups, _extra_dests = platform_region_groups(
config,
pack_systems,
@@ -3121,9 +3130,16 @@ def _manifest_region_drops(
include_extras=(source != "platform"),
include_all=(source == "truth"),
required_only=required_only,
extras_out=extras,
)
return region_mod.resolve_region_drops(
region_groups, region_mod.build_region_index(emu_profiles), regions
region_index = region_mod.build_region_index(emu_profiles)
drops = (
region_mod.resolve_region_drops(region_groups, region_index, regions)
if regions
else set()
)
return drops, SelectionIndex.build(
region_groups, region_index, pack_systems, extras, required_only
)
@@ -3234,9 +3250,17 @@ def generate_manifest(
"required": bool(file_entry.get("required", True)),
"reason": reason,
"cores": cores,
# The builder counts an unavailable regional file in its groups:
# the installer must too, or it keeps a fallback the pack drops.
**selection.fields(
sanitize_pack_path(
file_entry.get("destination", file_entry.get("name", ""))
),
str(file_entry.get("name") or ""),
),
}
region_drops = _manifest_region_drops(
region_drops, selection = _manifest_selection(
config, pack_systems, emulators_dir, db, base_dest, emu_profiles,
target_cores, source, required_only, regions,
)
@@ -3325,7 +3349,8 @@ def generate_manifest(
)
entry, file_size = _manifest_entry(
local_path, dest, None, db, repo_root
local_path, dest, None, db, repo_root,
selection.fields(dest, file_entry["name"]),
)
if entry is None:
record_omission(full_dest, file_entry, sys_id, "not_found", None)
@@ -3366,7 +3391,8 @@ def generate_manifest(
core_files, config, db, bios_dir, base_dest, repo_root,
zip_contents, offline, region_drops, case_insensitive,
seen_destinations, seen_lower, seen_parents, manifest_files,
omitted_by_destination, record_omission, pack_only_sizes, required_only,
omitted_by_destination, record_omission, pack_only_sizes, selection,
required_only,
)
# Phase 3: data directories. The installer does not fetch them, so they
+1 -1
View File
@@ -372,7 +372,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
"",
"## Download BIOS packs",
"",
"One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file, and `--target` narrows it to one machine; for a region or a bare minimum, build your own pack below.",
"One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file. At a terminal it asks whether to install everything or to choose systems, cores and regions, and `--system`, `--core`, `--region` and `--target` make the same choice without questions; for a bare minimum, build your own pack below.",
"Files is how many the pack holds once extracted, the figure a file"
" manager shows for the folder. It differs from one platform to the"
" next because a pack carries only what that platform's emulators"
+7 -5
View File
@@ -3405,11 +3405,13 @@ guarantee either: source profiles can document files nobody has dumped or
that only the user can provide, all of them visible in the
[gap analysis](gaps.md).
Want less than everything? The installer takes `--target switch` to install
only what one machine's cores need. From a clone of the repository,
`python scripts/generate_pack.py --platform retroarch --region us` keeps one
BIOS per region and `--required-only` the bare minimum each core needs to
start; `--help` lists every way to build your own.
Want less than everything? At a terminal the installer asks whether to install
everything or to choose systems, cores and regions, and `--system`, `--core`,
`--region` and `--target` make the same choice without questions (see the
[installer reference](wiki/installer.md#choosing-what-to-install)). From a
clone of the repository, `python scripts/generate_pack.py --platform retroarch
--region us` builds the same narrowing as a pack and `--required-only` the bare
minimum each core needs to start; `--help` lists every way to build your own.
---
+112
View File
@@ -0,0 +1,112 @@
"""What an install manifest records about each file so the installer can narrow it.
The installer runs without the profiles or the database, yet a narrowed
install must hold the same files as a pack built with the same narrowing.
Each entry therefore carries the platform systems it belongs to and, when
the code selects it by territory, its regions and the groups in which it
competes. The installer replays region.resolve_region_drops over them.
The pack's `--system X` keeps the platform's declarations under X and every
core extra owned by X, then groups regions over the declarations it kept and
over every core extra of the platform. A group membership is therefore either
unconditional (a core extra) or present only when the declaring system is
kept (a platform declaration), and the manifest records the two apart.
"""
from __future__ import annotations
from dataclasses import dataclass
import region as region_mod
from common import _norm_system_id, sanitize_pack_path
from packextras import _extra_system_ids, _kept, extra_region_groups
def _dest(entry: dict) -> str:
return sanitize_pack_path(entry.get("destination", entry.get("name", "")))
@dataclass(frozen=True)
class SelectionIndex:
"""System and region facts for one platform, as the pack builder reads them."""
region_index: dict[str, dict]
competing: frozenset[str]
systems_by_dest: dict[str, frozenset[str]]
declared_by: dict[str, frozenset[str]]
extra_groups: dict[str, frozenset[str]]
@classmethod
def build(
cls,
region_groups: dict[str, list[tuple[str, str]]],
region_index: dict[str, dict],
pack_systems: dict[str, dict],
extras: list[dict],
required_only: bool = False,
) -> SelectionIndex:
by_norm: dict[str, set[str]] = {}
for sys_id in pack_systems:
by_norm.setdefault(_norm_system_id(sys_id), set()).add(sys_id)
declared_by: dict[str, set[str]] = {}
for sys_id, system in pack_systems.items():
for file_entry in _kept(system.get("files", []), required_only):
dest = _dest(file_entry)
if dest:
declared_by.setdefault(dest, set()).add(sys_id)
owned_by: dict[str, set[str]] = {}
extra_groups: dict[str, set[str]] = {}
for extra in extras:
dest = _dest(extra)
if not dest:
continue
extra_groups.setdefault(dest, set()).update(extra_region_groups(extra))
# A profile system the platform does not have is no system the
# pack's --system accepts: it would offer what the pack refuses.
for sys_id in _extra_system_ids(extra):
owned_by.setdefault(dest, set()).update(
by_norm.get(_norm_system_id(sys_id), ())
)
systems_by_dest = {
dest: frozenset(declared_by.get(dest, set()) | owned_by.get(dest, set()))
for dest in set(declared_by) | set(owned_by)
}
return cls(
region_index,
frozenset(
dest for members in region_groups.values() for dest, _name in members
),
systems_by_dest,
{dest: frozenset(ids) for dest, ids in declared_by.items()},
{dest: frozenset(ids) for dest, ids in extra_groups.items()},
)
def fields(self, dest: str, name: str) -> dict:
"""What an entry carries for the installer's selection.
`systems` drives --system. A regional entry adds its regions,
`region_groups` it always competes in as a core extra, and
`region_system_groups` it competes in only when that platform system
is kept. An entry that never competes carries no region field: an
untagged file always survives a region filter.
"""
out: dict = {}
systems = self.systems_by_dest.get(dest)
if systems:
out["systems"] = sorted(systems)
if dest not in self.competing:
return out
regions = region_mod.lookup_regions(self.region_index, dest, name)
if not regions:
return out
out["regions"] = sorted(regions)
always = self.extra_groups.get(dest)
if always:
out["region_groups"] = sorted(always)
declared = self.declared_by.get(dest)
if declared:
out["region_system_groups"] = sorted(declared)
return out
+18 -11
View File
@@ -685,6 +685,7 @@ def platform_region_groups(
include_extras: bool = True,
include_all: bool = False,
required_only: bool = False,
extras_out: list[dict] | None = None,
) -> tuple[dict[str, list[tuple[str, str]]], dict[tuple[str, str, str], str]]:
"""Group a platform's pack candidates the way region filtering reads them.
@@ -692,7 +693,8 @@ def platform_region_groups(
grouped under, keyed by (emulator, name). verify.py needs that mapping to
withdraw from its report exactly what the builder withdraws from the pack:
grouping the declared files here and the core extras there would let the
two answer differently on the same request.
two answer differently on the same request. `extras_out` receives the
core extras that were grouped, for the install manifest.
"""
groups: dict[str, list[tuple[str, str]]] = {}
for sys_id, system in systems.items():
@@ -733,19 +735,24 @@ def platform_region_groups(
extra.get("source_path", ""),
)
] = dest
variant = extra.get("variant_group")
if variant:
# A variant group is one slot of one core, whatever system each
# member is filed under: PicoDrive files its US Mega CD BIOS under
# sega-segacd and the EU/JP ones under sega-megacd, and find_bios
# picks one of the twelve. Keyed as emulator mode keys it.
group_ids = [f"{extra.get('source_emulator', '')}:variant:{variant}"]
else:
group_ids = _extra_system_ids(extra) or ["_extras"]
for group_id in group_ids:
for group_id in extra_region_groups(extra):
groups.setdefault(group_id, []).append((dest, name))
if extras_out is not None:
extras_out.append(extra)
return groups, extra_dests
def extra_region_groups(extra: dict) -> list[str]:
"""The region groups a core extra competes in."""
variant = extra.get("variant_group")
if variant:
# A variant group is one slot of one core, whatever system each
# member is filed under: PicoDrive files its US Mega CD BIOS under
# sega-segacd and the EU/JP ones under sega-megacd, and find_bios
# picks one of the twelve. Keyed as emulator mode keys it.
return [f"{extra.get('source_emulator', '')}:variant:{variant}"]
return _extra_system_ids(extra) or ["_extras"]
def emulator_region_drops(
selected: list[tuple[str, dict]],
standalone: bool,
+540
View File
@@ -0,0 +1,540 @@
"""The installer narrows a platform the way the pack builder does.
The one-liner can install a chosen set of systems, cores and regions. It runs
without the profiles or the database, from the manifest alone, so the
manifest records each file's systems and regional competition and the
installer replays the builder's region rule. These tests hold the two ends
to one answer.
"""
from __future__ import annotations
import functools
import hashlib
import http.server
import importlib.util
import json
import os
import random
import shlex
import shutil
import subprocess
import sys
import tempfile
import threading
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import region # noqa: E402
_spec = importlib.util.spec_from_file_location("install", REPO_ROOT / "install.py")
install = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(install)
class OneRegionVocabulary(unittest.TestCase):
"""install.py cannot import region.py, so it carries a copy."""
def test_the_copy_matches(self):
self.assertEqual(install.REGION_TREE, dict(region.REGION_TREE))
self.assertEqual(install.REGION_ALIASES, region.ALIASES)
self.assertEqual(install.WORLD_REGION, region.WORLD)
self.assertEqual(install.REGIONS, region.REGIONS)
class TheRegionRuleIsTheBuilders(unittest.TestCase):
"""Random groups, both implementations, one answer."""
POOL = ["north-america", "europe", "japan", "uk", "canada", "brazil",
"south-korea", "world", "asia", "oceania"]
def _case(self, rng: random.Random):
groups: dict[str, list[tuple[str, str]]] = {}
index: dict[str, dict] = {}
entries: dict[str, dict] = {}
for g in range(rng.randint(1, 4)):
for m in range(rng.randint(1, 5)):
dest = f"d{g}_{m}" if rng.random() < 0.8 else f"shared{m}"
if dest not in entries:
regions = (
set()
if rng.random() < 0.2
else set(rng.sample(self.POOL, rng.randint(1, 2)))
)
entries[dest] = {"dest": dest, "regions": sorted(regions),
"region_groups": []}
index[dest] = {"regions": regions,
"has_untagged": not regions, "emulators": []}
groups.setdefault(f"g{g}", []).append((dest, dest))
if f"g{g}" not in entries[dest]["region_groups"]:
entries[dest]["region_groups"].append(f"g{g}")
for entry in entries.values():
if not entry["regions"]:
del entry["regions"], entry["region_groups"]
requested = rng.sample(["north-america", "europe", "japan", "uk", "canada"],
rng.randint(1, 3))
return groups, index, list(entries.values()), requested
def test_two_thousand_random_platforms(self):
rng = random.Random(20261010)
for trial in range(2000):
groups, index, entries, requested = self._case(rng)
with self.subTest(trial=trial):
self.assertEqual(
install.region_drops(entries, requested),
region.resolve_region_drops(groups, index, requested),
)
class Narrowing(unittest.TestCase):
FILES = [
{"dest": "scph5500.bin", "size": 1, "cores": None,
"systems": ["sony-playstation"], "regions": ["japan"],
"region_system_groups": ["sony-playstation"]},
{"dest": "scph5501.bin", "size": 1, "cores": None,
"systems": ["sony-playstation"], "regions": ["north-america"],
"region_system_groups": ["sony-playstation"]},
{"dest": "gba_bios.bin", "size": 1, "cores": None,
"systems": ["nintendo-gba"]},
{"dest": "pcsx/extra.bin", "size": 1, "cores": ["pcsx_rearmed"],
"systems": ["sony-playstation"]},
{"dest": "beetle/extra.bin", "size": 1, "cores": ["beetle_psx"],
"systems": ["sony-playstation"]},
]
OMITTED = [
{"dest": "scph5502.bin", "name": "scph5502.bin", "system": "sony-playstation",
"required": True, "reason": "not_found", "cores": None,
"regions": ["europe"], "region_system_groups": ["sony-playstation"]},
]
def _dests(self, *args):
files, omitted = install.narrow(list(self.FILES), list(self.OMITTED), *args)
return sorted(f["dest"] for f in files), sorted(o["dest"] for o in omitted)
def test_nothing_chosen_keeps_everything(self):
self.assertEqual(len(self._dests([], [], [])[0]), len(self.FILES))
def test_a_system(self):
files, omitted = self._dests(["nintendo-gba"], [], [])
self.assertEqual(files, ["gba_bios.bin"])
self.assertEqual(omitted, [])
def test_a_core_keeps_the_platform_list(self):
files, _ = self._dests(["sony-playstation"], ["pcsx_rearmed"], [])
self.assertEqual(files, ["pcsx/extra.bin", "scph5500.bin", "scph5501.bin"])
def test_a_region_keeps_one_bios(self):
files, omitted = self._dests([], [], ["north-america"])
self.assertNotIn("scph5500.bin", files)
self.assertIn("scph5501.bin", files)
self.assertEqual(omitted, [])
def test_an_unavailable_match_still_wins_as_the_builder_counts_it(self):
files, omitted = self._dests([], [], ["europe"])
self.assertNotIn("scph5500.bin", files)
self.assertNotIn("scph5501.bin", files)
self.assertEqual(omitted, ["scph5502.bin"])
def test_no_match_keeps_every_regional_file(self):
files, _ = self._dests([], [], ["brazil"])
self.assertIn("scph5500.bin", files)
self.assertIn("scph5501.bin", files)
def test_a_core_variant_competes_across_systems(self):
"""PicoDrive files its US Mega CD BIOS under sega-segacd and the EU
one under sega-mega-cd, one variant group. The pack groups every core
extra before keeping a system, so under --system sega-mega-cd
--region us the US member still wins and the EU one is dropped."""
files = [
{"dest": "us_scd.bin", "size": 1, "cores": ["picodrive"],
"systems": ["sega-segacd"], "regions": ["north-america"],
"region_groups": ["picodrive:variant:mcd"]},
{"dest": "eu_mcd.bin", "size": 1, "cores": ["picodrive"],
"systems": ["sega-mega-cd"], "regions": ["europe"],
"region_groups": ["picodrive:variant:mcd"]},
{"dest": "bios_CD_U.bin", "size": 1, "cores": None,
"systems": ["sega-mega-cd"]},
]
kept, _ = install.narrow(files, [], ["sega-mega-cd"], [], ["north-america"])
self.assertEqual([f["dest"] for f in kept], ["bios_CD_U.bin"])
def test_an_unkept_systems_declaration_does_not_compete(self):
"""Group `a` holds system a's own US file and a core extra filed
under a but owned by `shared`. Under --system shared the pack builds
group a from the extra alone: nothing matches, so the extra stays.
Counting a's own file would make it win and drop the extra."""
files = [
{"dest": "a/us.bin", "size": 1, "cores": None, "systems": ["a"],
"regions": ["north-america"], "region_system_groups": ["a"]},
{"dest": "core/jp.bin", "size": 1, "cores": ["core"],
"systems": ["shared"], "regions": ["japan"], "region_groups": ["a"]},
]
kept, _ = install.narrow(files, [], ["shared"], [], ["north-america"])
self.assertEqual([f["dest"] for f in kept], ["core/jp.bin"])
class CommandLineChoices(unittest.TestCase):
KNOWN = ["msx1,msx2,msxturbor", "nintendo-gba", "sony-playstation"]
def test_comma_lists_and_repeats(self):
self.assertEqual(
install.resolve_choices(["Sony-PlayStation,nintendo-gba"], self.KNOWN, "system"),
["sony-playstation", "nintendo-gba"],
)
def test_a_name_with_commas_is_one_name(self):
self.assertEqual(
install.resolve_choices(["msx1,msx2,msxturbor"], self.KNOWN, "system"),
["msx1,msx2,msxturbor"],
)
def test_an_unknown_name_is_refused(self):
with self.assertRaises(ValueError) as ctx:
install.resolve_choices(["sega-saturn"], self.KNOWN, "system")
self.assertIn("sega-saturn", str(ctx.exception))
self.assertIn("nintendo-gba", str(ctx.exception))
def test_an_empty_list_is_refused(self):
with self.assertRaises(ValueError):
install.resolve_regions([" , "])
with self.assertRaises(ValueError):
install.resolve_choices([","], self.KNOWN, "system")
def test_region_aliases_keep_their_order(self):
self.assertEqual(
install.resolve_regions(["jp,us", "eu,jp"]),
["japan", "north-america", "europe"],
)
with self.assertRaises(ValueError):
install.resolve_regions(["mars"])
def test_prompt_numbers(self):
self.assertEqual(install.parse_selection("3,1-2", 5), [1, 2, 3])
self.assertEqual(install.parse_selection("3,1", 5, ordered=True), [3, 1])
self.assertEqual(install.parse_selection("all", 3), [1, 2, 3])
for bad in ("0", "6", "4-2", "x", "1-"):
with self.subTest(text=bad), self.assertRaises(ValueError):
install.parse_selection(bad, 5)
class TheBoundaryChecksTheNewFields(unittest.TestCase):
def _manifest(self, **extra):
entry = {"dest": "a.bin", "sha1": "a" * 40, "size": 1,
"repo_path": "bios/a.bin", **extra}
return {"manifest_version": 2, "platform": "retroarch", "files": [entry]}
def test_well_formed(self):
install._validate_manifest(
self._manifest(systems=["x"], regions=["japan"], region_groups=["x"]),
"retroarch",
)
def test_malformed(self):
for bad in ({"systems": "x"}, {"regions": [1]}, {"region_groups": [""]},
{"region_system_groups": "x"}):
with self.subTest(field=bad), self.assertRaises(ValueError):
install._validate_manifest(self._manifest(**bad), "retroarch")
class _QuietHandler(http.server.SimpleHTTPRequestHandler):
def log_message(self, format, *args): # noqa: A002 - stdlib signature
return
class FakeRepository:
"""A loopback copy of the repository: one manifest and its payloads."""
PAYLOADS = {
"scph5500.bin": (b"jp bios", ["sony-playstation"], ["japan"], None),
"scph5501.bin": (b"us bios", ["sony-playstation"], ["north-america"], None),
"gba_bios.bin": (b"gba bios", ["nintendo-gba"], None, None),
"pcsx/extra.bin": (b"pcsx", ["sony-playstation"], None, ["pcsx_rearmed"]),
"mgba/extra.bin": (b"mgba", ["nintendo-gba"], None, ["mgba"]),
}
def __init__(self, selection: bool = True):
self.selection = selection
def __enter__(self) -> FakeRepository:
self.root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
(self.root / "install").mkdir()
(self.root / "bios").mkdir()
files = []
for name, (payload, systems, regions, cores) in self.PAYLOADS.items():
stored = name.replace("/", "_")
(self.root / "bios" / stored).write_bytes(payload)
entry = {
"dest": name,
"sha1": hashlib.sha1(payload).hexdigest(),
"sha256": hashlib.sha256(payload).hexdigest(),
"size": len(payload),
"repo_path": f"bios/{stored}",
"cores": cores,
}
if self.selection:
entry["systems"] = systems
if regions:
entry.update(regions=regions, region_system_groups=systems)
files.append(entry)
manifest = {"manifest_version": 2, "platform": "retroarch", "files": files}
(self.root / "install" / "retroarch.json").write_text(json.dumps(manifest))
handler = functools.partial(_QuietHandler, directory=str(self.root))
self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
self.dest = self.root / "dest"
return self
def __exit__(self, *exc) -> None:
self.httpd.shutdown()
self.httpd.server_close()
shutil.rmtree(self.root, ignore_errors=True)
def env(self) -> dict:
env = dict(os.environ)
env["RETROBIOS_BASE_URL"] = f"http://127.0.0.1:{self.httpd.server_address[1]}"
return env
def command(self, *options: str) -> list[str]:
return [sys.executable, str(REPO_ROOT / "install.py"), "--platform",
"retroarch", "--dest", str(self.dest), *options]
def run(self, *options: str) -> subprocess.CompletedProcess:
return subprocess.run(
self.command(*options), env=self.env(), capture_output=True,
text=True, timeout=60, check=False, stdin=subprocess.DEVNULL,
)
def installed(self) -> list[str]:
if not self.dest.exists():
return []
return sorted(
p.relative_to(self.dest).as_posix()
for p in self.dest.rglob("*") if p.is_file()
)
class TheOneLinerNarrows(unittest.TestCase):
def test_options_install_only_the_choice(self):
with FakeRepository() as repo:
proc = repo.run("--system", "sony-playstation", "--region", "us")
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertIn("Narrowed 5 -> 2 files", proc.stdout)
self.assertEqual(repo.installed(), ["pcsx/extra.bin", "scph5501.bin"])
def test_a_core_choice_keeps_the_platform_list(self):
with FakeRepository() as repo:
proc = repo.run("--core", "mgba")
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertEqual(
repo.installed(),
["gba_bios.bin", "mgba/extra.bin", "scph5500.bin", "scph5501.bin"],
)
def test_an_older_file_list_is_refused_not_ignored(self):
"""Without the fields, a region filter would match nothing and the
run would install everything while the user asked for one region."""
with FakeRepository(selection=False) as repo:
proc = repo.run("--region", "us")
self.assertEqual(proc.returncode, 1)
self.assertIn("records no systems or regions", proc.stderr)
self.assertEqual(repo.installed(), [])
def test_empty_names_are_a_usage_error(self):
with FakeRepository() as repo:
for option in (["--region", " , "], ["--system", ""], ["--core", ","]):
with self.subTest(option=option):
proc = repo.run(*option)
self.assertEqual(proc.returncode, 2, proc.stderr)
self.assertIn("needs at least one", proc.stderr)
def test_listing_follows_the_narrowing(self):
with FakeRepository() as repo:
proc = repo.run("--list-cores", "--system", "nintendo-gba")
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertIn("mgba", proc.stdout)
self.assertNotIn("pcsx_rearmed", proc.stdout)
def test_an_unknown_system_is_refused_before_any_download(self):
with FakeRepository() as repo:
proc = repo.run("--system", "sega-saturn")
self.assertEqual(proc.returncode, 1)
self.assertIn("unknown system: sega-saturn", proc.stderr)
self.assertIn("nintendo-gba", proc.stderr)
self.assertEqual(repo.installed(), [])
def test_an_unknown_region_is_a_usage_error(self):
with FakeRepository() as repo:
proc = repo.run("--region", "mars")
self.assertEqual(proc.returncode, 2)
self.assertIn("unknown region: mars", proc.stderr)
def test_listing_systems_installs_nothing(self):
with FakeRepository() as repo:
proc = repo.run("--list-systems")
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertIn("nintendo-gba", proc.stdout)
self.assertIn("3 files", proc.stdout)
self.assertEqual(repo.installed(), [])
def test_no_terminal_asks_nothing_and_installs_everything(self):
with FakeRepository() as repo:
proc = repo.run()
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertNotIn("choose systems", proc.stdout)
self.assertEqual(len(repo.installed()), 5)
@unittest.skipUnless(shutil.which("script"), "script(1) provides the terminal")
def test_the_custom_mode_at_a_terminal(self):
"""Typed answers: customise, PlayStation only, no core, US first."""
with FakeRepository() as repo:
command = " ".join(shlex.quote(part) for part in repo.command())
proc = subprocess.run(
["script", "-qec", command, "/dev/null"],
input="c\n2\n\n2\n", env=repo.env(), capture_output=True,
text=True, timeout=60, check=False,
)
out = proc.stdout
self.assertEqual(proc.returncode, 0, out + proc.stderr)
self.assertIn("choose systems, cores and regions", out)
self.assertIn("pcsx_rearmed", out)
self.assertIn("--system sony-playstation --region north-america", out)
self.assertEqual(repo.installed(), ["pcsx/extra.bin", "scph5501.bin"])
@unittest.skipUnless(shutil.which("script"), "script(1) provides the terminal")
def test_enter_installs_everything_at_a_terminal(self):
with FakeRepository() as repo:
command = " ".join(shlex.quote(part) for part in repo.command())
proc = subprocess.run(
["script", "-qec", command, "/dev/null"],
input="\n", env=repo.env(), capture_output=True,
text=True, timeout=60, check=False,
)
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertEqual(len(repo.installed()), 5)
@unittest.skipUnless(shutil.which("script"), "script(1) provides the terminal")
def test_no_input_asks_nothing_at_a_terminal(self):
with FakeRepository() as repo:
command = " ".join(
shlex.quote(part) for part in repo.command("--no-input")
)
proc = subprocess.run(
["script", "-qec", command, "/dev/null"],
input="", env=repo.env(), capture_output=True,
text=True, timeout=60, check=False,
)
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertNotIn("choose systems", proc.stdout)
self.assertEqual(len(repo.installed()), 5)
class InstallMatchesThePack(unittest.TestCase):
"""Narrowing the full manifest gives what the narrowed build carries."""
PLATFORM = "recalbox"
@classmethod
def setUpClass(cls):
db_path = REPO_ROOT / "database.json"
if not db_path.exists():
raise unittest.SkipTest("database.json is not built")
import generate_pack as gp
from common import load_emulator_profiles
cls.gp = gp
cls.db = json.loads(db_path.read_text(encoding="utf-8"))
cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
cls.full = cls._manifest(None)
@classmethod
def _manifest(cls, regions):
return cls.gp.generate_manifest(
cls.PLATFORM, str(REPO_ROOT / "platforms"), cls.db,
str(REPO_ROOT / "bios"), str(REPO_ROOT / "platforms" / "_registry.yml"),
emulators_dir=str(REPO_ROOT / "emulators"), emu_profiles=cls.profiles,
regions=regions, offline=True,
)
def _narrowed(self, systems, regions):
files, omitted = install.narrow(
self.full["files"], self.full["omitted_files"], systems, [], regions
)
return {f["dest"] for f in files}, {o["dest"] for o in omitted}
def test_regions(self):
for regions in (["north-america"], ["japan", "europe"]):
with self.subTest(regions=regions):
built = self._manifest(regions)
files, omitted = self._narrowed([], regions)
self.assertEqual(files, {f["dest"] for f in built["files"]})
self.assertEqual(omitted, {o["dest"] for o in built["omitted_files"]})
class InstallMatchesASystemPack(unittest.TestCase):
"""`install.py --system X --region R` against a real `generate_pack.py
--system X --region R` build. The cases are the ones a review reproduced:
ColecoVision's extras filed at destinations another system declares, and
PicoDrive's Mega CD variant group spread over two systems."""
PLATFORM = "retroarch"
CASES = (
(["coleco-colecovision"], []),
(["sega-mega-cd"], ["north-america"]),
(["sony-playstation"], ["japan"]),
)
@classmethod
def setUpClass(cls):
db_path = REPO_ROOT / "database.json"
if not db_path.exists():
raise unittest.SkipTest("database.json is not built")
import generate_pack as gp
from common import load_emulator_profiles, load_platform_config
cls.gp = gp
cls.db = json.loads(db_path.read_text(encoding="utf-8"))
cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
cls.base = load_platform_config(
cls.PLATFORM, str(REPO_ROOT / "platforms")
).get("base_destination", "")
cls.full = gp.generate_manifest(
cls.PLATFORM, str(REPO_ROOT / "platforms"), cls.db,
str(REPO_ROOT / "bios"), str(REPO_ROOT / "platforms" / "_registry.yml"),
emulators_dir=str(REPO_ROOT / "emulators"), emu_profiles=cls.profiles,
offline=True,
)
def _packed(self, systems, regions) -> set[str]:
import zipfile
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out:
zip_path = self.gp.generate_pack(
self.PLATFORM, str(REPO_ROOT / "platforms"), self.db,
str(REPO_ROOT / "bios"), out,
emulators_dir=str(REPO_ROOT / "emulators"),
emu_profiles=self.profiles, system_filter=systems,
regions=regions or None, offline=True,
)
with zipfile.ZipFile(zip_path) as zf:
names = set(zf.namelist())
prefix = f"{self.base}/" if self.base else ""
return {n[len(prefix):] if prefix and n.startswith(prefix) else n for n in names}
def test_each_case(self):
known = {f["dest"] for f in self.full["files"]}
for systems, regions in self.CASES:
with self.subTest(systems=systems, regions=regions):
files, _omitted = install.narrow(
self.full["files"], self.full["omitted_files"],
systems, [], regions,
)
self.assertEqual(
{f["dest"] for f in files}, self._packed(systems, regions) & known
)
if __name__ == "__main__":
unittest.main()
+2 -1
View File
@@ -20,6 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import generate_pack as gp # noqa: E402
from manifestselect import SelectionIndex # noqa: E402
def _held_file(db: dict) -> tuple[str, dict] | None:
@@ -60,7 +61,7 @@ class AnEntryWithoutSourceIsAnOmission(unittest.TestCase):
gp._manifest_core_entries(
[extra], {}, self.db, str(REPO_ROOT / "bios"), "", str(REPO_ROOT),
{}, True, set(), False, set(), set(), set(), files, {},
record, pack_only,
record, pack_only, SelectionIndex.build({}, {}, {}, []),
)
return files, omitted, pack_only
+4
View File
@@ -41,6 +41,10 @@ Override detection when needed:
```bash
python install.py --platform retroarch --dest ~/custom/bios
python install.py --target switch # keep only files for that hardware
python install.py --list-systems # systems a platform holds, with sizes
python install.py --system sony-playstation,nintendo-gba
python install.py --region us,eu,jp # one BIOS per region, best first
python install.py --no-input # ask nothing, even at a terminal
python install.py --check # verify existing files, download nothing
python install.py --list-platforms # supported platforms and what was detected
python install.py --list-targets # hardware targets for a platform
+67 -1
View File
@@ -67,6 +67,11 @@ Run from a clone, `install.sh` and `install.ps1` reuse the `install.py` sitting
next to them and skip the download entirely; piped from stdin, `install.sh`
never treats the working directory as a trusted location for it.
Piped from `curl`, the shell's standard input is the script itself, so
`install.sh` hands the installer the terminal (`/dev/tty`) when there is one.
Without that, every question the installer can ask was skipped on Linux and
macOS.
## What the installer does
1. Detects the host OS and the platforms installed on it.
@@ -75,7 +80,8 @@ never treats the working directory as a trusted location for it.
always come from one commit.
3. Prints the file count and total size, then a safety notice counting the
entries the collection cannot serve, how many of those the platform marks
required, and why.
required, and why. At a terminal, it then offers to install everything or
to choose systems, cores and regions (see below).
4. Hashes what is already in the destination and reports how many entries are
present, verified, or present with the wrong contents.
5. Downloads what is missing or wrong, up to 8 files at a time.
@@ -97,6 +103,60 @@ Checking existing files...
1872 files need downloading.
```
## Choosing what to install
By default the installer fetches everything the platform runs. At a terminal,
after detection, it asks:
```
Retroarch: 5861 files, 6.0 GB.
Enter) install everything
c) choose systems, cores and regions
q) quit
```
`c` walks through three numbered lists, each answered with numbers such as
`1,4,7-9`, or Enter to keep everything on that axis.
- **Systems.** The consoles and computers the platform declares.
- **Cores.** The platform's own BIOS list for the chosen systems is always
kept; this narrows only the extra files each emulator core loads beyond it.
Asked only when the chosen systems have such files.
- **Regions.** Asked only when a chosen system has one BIOS per region. The
numbers give an order of preference: `2,1` keeps the second region where a
file for it exists, and the first one elsewhere.
The run then prints the options that repeat the same choice without
questions, for instance
`--platform retroarch --system sony-playstation --region north-america`.
The same choices on the command line:
```bash
python install.py --platform retroarch --list-systems
python install.py --platform retroarch --system sony-playstation,nintendo-gba
python install.py --platform retroarch --system sony-playstation --core pcsx_rearmed
python install.py --platform recalbox --region us,eu,jp
```
A name the platform does not have is refused with the list of the ones it
has, before anything is downloaded: carrying on would install something other
than what was asked. A system name that itself contains commas, as Recalbox's
`msx1,msx2,msxturbor` does, is read as one name.
The region rule is the one `generate_pack.py --region` applies, and a test
holds the two to the same file list. In each group of regional alternatives
the best-ranked match stays; a file declared for every region beats regional
files that match nothing asked; when nothing matches, every regional file
stays, so no system is left without its BIOS. A file the emulator does not
select by region always stays. The manifest records, for each file, the
systems that declare it, and for a regional file, its regions and the groups
it competes in; the installer reads nothing else.
The questions are asked only when standard input is a terminal. Piped,
scripted or run with `--no-input`, the installer asks nothing and installs
what the options name, everything by default.
## Options
| Option | Effect |
@@ -104,6 +164,12 @@ Checking existing files...
| `--platform NAME` | Install for this platform instead of the detected one. Unknown names are refused with the available list |
| `--dest PATH` | Destination directory, overriding detection. With `--dest` alone the file list is RetroArch's |
| `--target NAME` | Keep only the files the cores of that hardware target need. An unknown target is refused rather than ignored, since carrying on would install everything |
| `--system NAMES` | Install only these systems. Comma-separated or repeated; `--list-systems` prints the names |
| `--core NAMES` | Keep the platform's own list and only the extra files these cores load; `--list-cores` prints the names |
| `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` |
| `--list-systems` | Print a platform's systems with the files and size each holds |
| `--list-cores` | Print the cores whose extra files a platform installs |
| `--no-input` | Never ask a question, even at a terminal |
| `--check` | Report and exit without writing |
| `--list-platforms` | Print the supported platforms and what was detected here |
| `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries |