mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-26 21:57:24 -05:00
chore: merge the handheld port
This commit is contained in:
Abdessamad Derraz committed
commit
e05844e611
19 files changed
+1862
-8
No files matched your search
@@ -17,6 +17,8 @@ on:
|
||||
- "install.py"
|
||||
- "install.sh"
|
||||
- "install.ps1"
|
||||
# The handheld port bundles install.py and its tests read both.
|
||||
- "ports/**"
|
||||
# Every input of the site build, so a page that would fail the deploy
|
||||
# fails here first (tests/test_workflow_paths.py).
|
||||
- "wiki/**"
|
||||
@@ -42,6 +44,7 @@ on:
|
||||
- "install.py"
|
||||
- "install.sh"
|
||||
- "install.ps1"
|
||||
- "ports/**"
|
||||
- "wiki/**"
|
||||
- "docs_assets/**"
|
||||
- "provenance/**"
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
$defaultInstallSha256 = "5c12411406dd1c0542f8051eb5c9e5aaff022de208e4c446bfc81cc03896db8a"
|
||||
$defaultInstallSha256 = "abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa"
|
||||
$maximumInstallerBytes = 2MB
|
||||
$installerExit = 0
|
||||
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
|
||||
|
||||
+224
-4
@@ -363,6 +363,7 @@ class Progress:
|
||||
|
||||
def __init__(self, total_files: int, total_bytes: int, term: Terminal) -> None:
|
||||
self.total_files = total_files
|
||||
self.expected_bytes = total_bytes
|
||||
self.total_bytes = max(total_bytes, 1)
|
||||
self.term = term
|
||||
self.files = 0
|
||||
@@ -370,6 +371,7 @@ class Progress:
|
||||
self.resumed = 0
|
||||
self.start = time.monotonic()
|
||||
self._drawn = 0.0
|
||||
self._evented = self.start
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add(self, count: int, resumed: bool = False) -> None:
|
||||
@@ -381,6 +383,10 @@ class Progress:
|
||||
if self.term.live and not self.term.quiet and now - self._drawn >= 0.1:
|
||||
self._drawn = now
|
||||
self._draw()
|
||||
if EVENTS.on and now - self._evented >= EVENT_PROGRESS_INTERVAL:
|
||||
self._evented = now
|
||||
EVENTS.emit("progress", files_done=self.files, files_total=self.total_files,
|
||||
bytes_done=self.bytes, bytes_total=self.expected_bytes)
|
||||
|
||||
def forget_resumed(self, count: int) -> None:
|
||||
with self._lock:
|
||||
@@ -450,6 +456,142 @@ class Progress:
|
||||
print(f"\r\033[K{text[:width]}", end="", file=self.term.stream, flush=True)
|
||||
|
||||
|
||||
# --events: one JSON object per line on stdout, for a program driving the
|
||||
# installer (the handheld port shows its screens from them). Each kind has
|
||||
# exactly these keys besides "event", in this order, and nothing else.
|
||||
EVENT_SCHEMA = 1
|
||||
EVENT_KEYS: dict[str, tuple[str, ...]] = {
|
||||
# mode: install, check, dry-run or list.
|
||||
"start": ("schema", "installer", "mode"),
|
||||
# platforms: [{"platform", "name", "bios_dir"}], empty when none is found.
|
||||
"detect": ("os", "os_name", "platforms"),
|
||||
# stage: manifest (reason: network, not_found or invalid) or detect
|
||||
# (reason: conflict, two systems found on one machine).
|
||||
"error": ("stage", "reason", "platform", "message"),
|
||||
# state: present (hash verified), wrong_hash or missing.
|
||||
"file": ("platform", "dest", "systems", "size", "state"),
|
||||
"system": ("platform", "system", "name", "files", "present", "missing",
|
||||
"wrong_hash", "bytes_needed"),
|
||||
"check": ("platform", "name", "bios_dir", "files", "present", "missing",
|
||||
"wrong_hash", "bytes_needed", "data_missing", "not_installable"),
|
||||
"download": ("platform", "files", "bytes"),
|
||||
"progress": ("files_done", "files_total", "bytes_done", "bytes_total"),
|
||||
"downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total",
|
||||
"bytes_done", "bytes_total"),
|
||||
# failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files",
|
||||
# "required"}], the files the collection cannot supply.
|
||||
"summary": ("mode", "installed", "up_to_date", "pending", "failed", "skipped",
|
||||
"interrupted"),
|
||||
"end": ("code",),
|
||||
}
|
||||
# Seconds between two progress events while bytes arrive.
|
||||
EVENT_PROGRESS_INTERVAL = 0.5
|
||||
|
||||
|
||||
class EventStream:
|
||||
"""The --events output: closed (nothing written) until opened."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.stream: TextIO | None = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def open(self, stream: TextIO) -> None:
|
||||
self.stream = stream
|
||||
|
||||
@property
|
||||
def on(self) -> bool:
|
||||
return self.stream is not None
|
||||
|
||||
def emit(self, kind: str, **fields: object) -> None:
|
||||
if self.stream is None:
|
||||
return
|
||||
keys = EVENT_KEYS[kind]
|
||||
if set(fields) != set(keys):
|
||||
raise ValueError(f"event {kind}: keys {sorted(fields)} != {sorted(keys)}")
|
||||
line = json.dumps({"event": kind, **{key: fields[key] for key in keys}})
|
||||
with self._lock:
|
||||
self.stream.write(line + "\n")
|
||||
self.stream.flush()
|
||||
|
||||
|
||||
EVENTS = EventStream()
|
||||
|
||||
|
||||
def _event_mode(args: argparse.Namespace) -> str:
|
||||
if args.check:
|
||||
return "check"
|
||||
if args.dry_run:
|
||||
return "dry-run"
|
||||
if (args.list_platforms or args.list_targets or args.list_systems
|
||||
or args.list_cores or args.list_emulators):
|
||||
return "list"
|
||||
return "install"
|
||||
|
||||
|
||||
def _event_exit_code(code: object) -> int:
|
||||
"""The process status sys.exit(code) gives."""
|
||||
if code is None:
|
||||
return 0
|
||||
return code if isinstance(code, int) else 1
|
||||
|
||||
|
||||
def _os_name() -> str:
|
||||
"""The OS's own name, from os-release, else the kernel's."""
|
||||
release = _parse_os_release()
|
||||
return release.get("PRETTY_NAME") or release.get("NAME") or platform.system()
|
||||
|
||||
|
||||
def _emit_check(
|
||||
plat: str, manifest: dict, bios_path: Path, files: list[dict], up_to_date: list[dict],
|
||||
mismatched: list[dict], data_missing: int, not_installable: int,
|
||||
) -> None:
|
||||
"""One event per file, per system and per platform after a check."""
|
||||
present = {id(entry) for entry in up_to_date}
|
||||
wrong = {id(entry) for entry in mismatched}
|
||||
systems: dict[str, dict[str, int]] = {}
|
||||
for entry in files:
|
||||
state = ("present" if id(entry) in present
|
||||
else "wrong_hash" if id(entry) in wrong else "missing")
|
||||
size = entry.get("size", 0)
|
||||
EVENTS.emit("file", platform=plat, dest=entry["dest"],
|
||||
systems=entry_systems(entry), size=size, state=state)
|
||||
for system in entry_systems(entry) or [OTHER_SYSTEMS]:
|
||||
counts = systems.setdefault(system, {
|
||||
"files": 0, "present": 0, "missing": 0, "wrong_hash": 0,
|
||||
"bytes_needed": 0,
|
||||
})
|
||||
counts["files"] += 1
|
||||
counts[state] += 1
|
||||
if state != "present":
|
||||
counts["bytes_needed"] += size
|
||||
names = manifest.get("system_names") or {}
|
||||
for system in sorted(systems):
|
||||
EVENTS.emit("system", platform=plat, system=system,
|
||||
name=", ".join(names.get(system) or [system]), **systems[system])
|
||||
needed = [entry for entry in files if id(entry) not in present]
|
||||
EVENTS.emit(
|
||||
"check", platform=plat, name=display_name(plat, manifest), bios_dir=str(bios_path),
|
||||
files=len(files), present=len(up_to_date), missing=len(needed) - len(mismatched),
|
||||
wrong_hash=len(mismatched),
|
||||
bytes_needed=sum(entry.get("size", 0) for entry in needed),
|
||||
data_missing=data_missing, not_installable=not_installable,
|
||||
)
|
||||
|
||||
|
||||
def _skipped(omitted: list[dict]) -> list[dict]:
|
||||
"""The files the collection cannot supply, grouped by reason."""
|
||||
groups: dict[str, dict] = {}
|
||||
for entry in omitted:
|
||||
reason = entry.get("reason", "unknown")
|
||||
group = groups.setdefault(reason, {
|
||||
"reason": reason, "label": OMISSION_REASONS.get(reason, reason),
|
||||
"files": 0, "required": 0,
|
||||
})
|
||||
group["files"] += 1
|
||||
group["required"] += bool(entry.get("required", True))
|
||||
return [groups[reason] for reason in sorted(groups)]
|
||||
|
||||
|
||||
# The three RetroArch packages published for Android, newest ABI first.
|
||||
ANDROID_RETROARCH_PACKAGES = (
|
||||
"com.retroarch.aarch64",
|
||||
@@ -2645,9 +2787,15 @@ def fetch_manifest(plat: str) -> dict:
|
||||
try:
|
||||
return _validate_manifest(_fetch_json(url, MAX_MANIFEST_BYTES, "manifest"), plat)
|
||||
except ValueError as exc:
|
||||
EVENTS.emit("error", stage="manifest", reason="invalid", platform=plat,
|
||||
message=f"the file list is not valid: {exc}")
|
||||
print(f" The file list for {plat} is not valid: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except urllib.error.HTTPError as exc:
|
||||
EVENTS.emit(
|
||||
"error", stage="manifest", reason="not_found" if exc.code == 404 else "network",
|
||||
platform=plat, message=f"the file list could not be fetched: HTTP {exc.code}",
|
||||
)
|
||||
if exc.code == 404 and os.environ.get("RETROBIOS_BASE_URL"):
|
||||
print(f" No file list for {plat} there.", file=sys.stderr)
|
||||
for line in _address_lines(url):
|
||||
@@ -2671,6 +2819,9 @@ def fetch_manifest(plat: str) -> dict:
|
||||
print(NETWORK_ADVICE, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except (urllib.error.URLError, OSError) as exc:
|
||||
EVENTS.emit("error", stage="manifest", reason="network", platform=plat,
|
||||
message=f"the file list could not be fetched: "
|
||||
f"{getattr(exc, 'reason', None) or exc}")
|
||||
print(f" Could not fetch the file list for {plat} after {MAX_RETRIES} "
|
||||
f"attempts: {getattr(exc, 'reason', None) or exc}", file=sys.stderr)
|
||||
for line in _address_lines(url):
|
||||
@@ -4093,6 +4244,12 @@ def download_files(
|
||||
if _stop.is_set():
|
||||
continue
|
||||
progress.done(dest, success, reason)
|
||||
EVENTS.emit(
|
||||
"downloaded", dest=dest, ok=success, reason=reason,
|
||||
size=future_map[future].get("size", 0), files_done=progress.files,
|
||||
files_total=progress.total_files, bytes_done=progress.bytes,
|
||||
bytes_total=progress.expected_bytes,
|
||||
)
|
||||
if not success:
|
||||
failed.append((dest, reason))
|
||||
if reason == NOT_WRITABLE and not _folder_writable(bios_path):
|
||||
@@ -5217,6 +5374,12 @@ def main() -> None:
|
||||
action="store_true",
|
||||
help="print only errors and the final summary (implies --no-input)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--events",
|
||||
action="store_true",
|
||||
help="write the run as JSON events on stdout, one per line, and the "
|
||||
"usual text on stderr (implies --no-input)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
action="version",
|
||||
@@ -5247,6 +5410,29 @@ def main() -> None:
|
||||
parser.error("--quiet applies to an install, not to --check, --dry-run or a listing")
|
||||
if args.every_board and args.target:
|
||||
parser.error("--every-board installs every board; --target names one")
|
||||
if args.events:
|
||||
# The events own stdout. The text a person reads goes to stderr, so a
|
||||
# program never has to tell one from the other.
|
||||
args.no_input = True
|
||||
EVENTS.open(sys.stdout)
|
||||
sys.stdout = sys.stderr
|
||||
EVENTS.emit("start", schema=EVENT_SCHEMA, installer=installer_id(),
|
||||
mode=_event_mode(args))
|
||||
try:
|
||||
_run(args)
|
||||
except SystemExit as exc:
|
||||
EVENTS.emit("end", code=_event_exit_code(exc.code))
|
||||
raise
|
||||
except KeyboardInterrupt:
|
||||
EVENTS.emit("end", code=130)
|
||||
raise
|
||||
EVENTS.emit("end", code=0)
|
||||
return
|
||||
_run(args)
|
||||
|
||||
|
||||
def _run(args: argparse.Namespace) -> None:
|
||||
"""The run, its text quiet or not."""
|
||||
if not args.quiet:
|
||||
_install(args, sys.stdout)
|
||||
return
|
||||
@@ -5487,6 +5673,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
platforms = detect_platforms(os_type)
|
||||
if not platforms:
|
||||
print(" No supported platform detected.")
|
||||
EVENTS.emit("detect", os=os_type, os_name=_os_name(), platforms=[])
|
||||
# Standalone emulators read their BIOS from folders of their own,
|
||||
# which the file lists name: an ES-DE user with DuckStation and
|
||||
# PCSX2 was sent to RetroArch's folder, which nothing here reads.
|
||||
@@ -5513,11 +5700,20 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
if conflict and not interactive and not listing:
|
||||
# A machine runs one system: installing to each folder would write
|
||||
# where nothing reads, and picking one would be a guess.
|
||||
print(f"Error: {' and '.join(conflict)} are both detected here, and a "
|
||||
"machine runs one. Name the folder with --dest (and --platform for "
|
||||
message = (f"{' and '.join(conflict)} are both detected here, and a "
|
||||
"machine runs one")
|
||||
EVENTS.emit("error", stage="detect", reason="conflict", platform="",
|
||||
message=message)
|
||||
print(f"Error: {message}. Name the folder with --dest (and --platform for "
|
||||
"its file list).",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if platforms:
|
||||
EVENTS.emit("detect", os=os_type, os_name=_os_name(), platforms=[
|
||||
{"platform": name, "name": display_name(name), "bios_dir": str(path)}
|
||||
for name, path in platforms
|
||||
])
|
||||
|
||||
if len(platforms) > 1 and not listing and interactive:
|
||||
platforms = _prompt_platform_choice(platforms)
|
||||
|
||||
@@ -5534,6 +5730,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
total_seconds = 0.0
|
||||
total_declined = 0
|
||||
check_pending = 0
|
||||
total_verified = 0
|
||||
files_pending = 0
|
||||
failures: list[tuple[str, str]] = []
|
||||
installed_paths: list[Path] = []
|
||||
|
||||
@@ -5801,6 +5999,15 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
)
|
||||
)
|
||||
data_dirs = select_data_directories(manifest, files, narrowed)
|
||||
data_states = [
|
||||
(entry, data_directory_state(entry, bios_path)) for entry in data_dirs
|
||||
] if args.check or EVENTS.on else []
|
||||
if EVENTS.on:
|
||||
_emit_check(
|
||||
plat_name, manifest, bios_path, files, up_to_date, mismatched,
|
||||
sum(len(state[1]) for _entry, state in data_states), len(omitted_files),
|
||||
)
|
||||
total_verified += len(up_to_date)
|
||||
|
||||
if args.check:
|
||||
if to_download:
|
||||
@@ -5833,8 +6040,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
every=args.verbose,
|
||||
):
|
||||
print(line)
|
||||
for entry in data_dirs:
|
||||
_root, needed, present = data_directory_state(entry, bios_path)
|
||||
for entry, (_root, needed, present) in data_states:
|
||||
if needed:
|
||||
print(f" {entry['ref']}: {present}/{len(entry['files'])} data files "
|
||||
f"present, {len(needed)} to fetch")
|
||||
@@ -5842,6 +6048,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
continue
|
||||
|
||||
if args.dry_run:
|
||||
files_pending += len(to_download)
|
||||
_print_plan(to_download, bios_path, term, [
|
||||
(entry, data_directory_state(entry, bios_path)[1]) for entry in data_dirs
|
||||
])
|
||||
@@ -5861,7 +6068,9 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
"install less (see --help).")
|
||||
total_up_to_date += len(up_to_date)
|
||||
total_declined += len(to_download)
|
||||
files_pending += len(to_download)
|
||||
continue
|
||||
EVENTS.emit("download", platform=plat_name, files=len(to_download), bytes=dl_size)
|
||||
print(f"\nDownloading {len(to_download)} files ({format_size(dl_size)})...")
|
||||
print(" Press Ctrl+C to stop. Running the same command again "
|
||||
"carries on from here.")
|
||||
@@ -5889,6 +6098,9 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
term=term, progress=progress,
|
||||
)
|
||||
total_downloaded += len(to_download) - len(failed)
|
||||
files_pending += len(failed)
|
||||
if interrupted:
|
||||
files_pending += max(len(to_download) - progress.files, 0)
|
||||
total_errors += len(failed)
|
||||
failures.extend(failed)
|
||||
total_bytes += progress.transferred()
|
||||
@@ -5955,6 +6167,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
"they read into them.")
|
||||
|
||||
sys.stdout = real_stdout
|
||||
if not listing:
|
||||
EVENTS.emit(
|
||||
"summary", mode=_event_mode(args), installed=total_downloaded,
|
||||
up_to_date=total_verified if args.check or args.dry_run else total_up_to_date,
|
||||
pending=check_pending if args.check else files_pending,
|
||||
failed=[{"dest": dest, "reason": str(reason)} for dest, reason in failures],
|
||||
skipped=_skipped(omitted_seen), interrupted=total_interrupted,
|
||||
)
|
||||
# A check scripted as `--check && launch` reads the exit code only.
|
||||
if args.check and check_pending:
|
||||
sys.exit(1)
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ esac
|
||||
TEMP_INSTALLER=""
|
||||
TEMP_DIRECTORY=""
|
||||
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
DEFAULT_INSTALL_SHA256="5c12411406dd1c0542f8051eb5c9e5aaff022de208e4c446bfc81cc03896db8a"
|
||||
DEFAULT_INSTALL_SHA256="abdbe842d22bcb384824e7ef4dc6464c573662fa87a18a97021c012c617453fa"
|
||||
MAX_INSTALLER_BYTES=2097152
|
||||
|
||||
cleanup() {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
## Notes
|
||||
|
||||
RetroBIOS runs the RetroBIOS installer (`install.py`, from
|
||||
[github.com/Abdess/retrobios](https://github.com/Abdess/retrobios)) on the
|
||||
device. The installer finds the BIOS folder of the system, checks every file
|
||||
its emulators read against its hash, and downloads the missing ones. The
|
||||
screens are PortMaster's own.
|
||||
|
||||
Needs a network connection. Python 3.8 or newer: the system's, or
|
||||
PortMaster's `python_3.11` runtime on aarch64, downloaded on first use.
|
||||
|
||||
## Controls
|
||||
|
||||
| Button | Action |
|
||||
|--|--|
|
||||
| D-Pad | Move in a list |
|
||||
| A | Choose |
|
||||
| B | Back, on PortMaster releases that read it (every list also has a Back or Quit entry) |
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# RetroBIOS: checks the BIOS folder of this system and downloads the files
|
||||
# it lacks, with PortMaster's own screens.
|
||||
|
||||
XDG_DATA_HOME=${XDG_DATA_HOME:-$HOME/.local/share}
|
||||
|
||||
if [ -d "/opt/system/Tools/PortMaster/" ]; then
|
||||
controlfolder="/opt/system/Tools/PortMaster"
|
||||
elif [ -d "/opt/tools/PortMaster/" ]; then
|
||||
controlfolder="/opt/tools/PortMaster"
|
||||
elif [ -d "$XDG_DATA_HOME/PortMaster/" ]; then
|
||||
controlfolder="$XDG_DATA_HOME/PortMaster"
|
||||
else
|
||||
controlfolder="/roms/ports/PortMaster"
|
||||
fi
|
||||
|
||||
source "$controlfolder/control.txt"
|
||||
[ -f "${controlfolder}/mod_${CFW_NAME}.txt" ] && source "${controlfolder}/mod_${CFW_NAME}.txt"
|
||||
get_controls
|
||||
|
||||
GAMEDIR="/$directory/ports/retrobios"
|
||||
cd "$GAMEDIR" || exit 1
|
||||
|
||||
> "$GAMEDIR/log.txt" && exec > >(tee "$GAMEDIR/log.txt") 2>&1
|
||||
|
||||
# PortMasterDialog.txt has shipped with PortMaster since before 2024.02.26,
|
||||
# and the 8.5.23 copy JELOS and KOLOS preinstall has no funcs.txt to source
|
||||
# it: it is sourced here.
|
||||
if [ ! -f "$controlfolder/PortMasterDialog.txt" ]; then
|
||||
echo "RetroBIOS needs PortMaster 2024.02.26 or newer: update PortMaster, then start RetroBIOS again."
|
||||
exit 1
|
||||
fi
|
||||
source "$controlfolder/PortMasterDialog.txt"
|
||||
source "$GAMEDIR/screens.txt"
|
||||
|
||||
export SDL_GAMECONTROLLERCONFIG="$sdl_controllerconfig"
|
||||
|
||||
rb_main
|
||||
status=$?
|
||||
|
||||
if declare -F pm_finish > /dev/null; then
|
||||
pm_finish
|
||||
fi
|
||||
exit $status
|
||||
@@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<gameList>
|
||||
<game>
|
||||
<path>./RetroBIOS.sh</path>
|
||||
<name>RetroBIOS</name>
|
||||
<desc>Checks the BIOS folder of this system against the files its emulators read, and downloads the missing ones from the RetroBIOS collection, each verified by its hash.</desc>
|
||||
<developer>Abdess</developer>
|
||||
<publisher>Open Source</publisher>
|
||||
<genre>Utility</genre>
|
||||
</game>
|
||||
</gameList>
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"version": 4,
|
||||
"name": "retrobios.zip",
|
||||
"items": [
|
||||
"RetroBIOS.sh",
|
||||
"retrobios"
|
||||
],
|
||||
"items_opt": [],
|
||||
"attr": {
|
||||
"title": "RetroBIOS",
|
||||
"porter": [
|
||||
"Abdess"
|
||||
],
|
||||
"desc": "Checks the BIOS folder of this system against the files its emulators read, and downloads the missing ones from the RetroBIOS collection, each verified by its hash.",
|
||||
"desc_md": null,
|
||||
"inst": "Needs a network connection: the list of files and the files come from the internet. Uses the system's Python 3.8 or newer, or PortMaster's python_3.11 runtime on aarch64.",
|
||||
"inst_md": null,
|
||||
"genres": [
|
||||
"other"
|
||||
],
|
||||
"image": null,
|
||||
"rtr": true,
|
||||
"exp": false,
|
||||
"runtime": [],
|
||||
"store": [],
|
||||
"availability": "full",
|
||||
"reqs": [],
|
||||
"arch": [],
|
||||
"min_glibc": ""
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Turn install.py --events into tab-separated records for the launcher.
|
||||
|
||||
The launcher is bash and draws through PortMaster's dialog FIFO, so it reads
|
||||
one record per line with `IFS=$'\\t' read`. Text fields are cleaned for that
|
||||
FIFO: PortMasterDialog sends its arguments through an unquoted
|
||||
`echo $(printf ...)`, where a tab or newline splits a field and `*`, `?` or
|
||||
`[` expand as a file pattern.
|
||||
|
||||
rbevents.py status EVENTS records of a --check run
|
||||
rbevents.py follow SUMMARY an install run on stdin: progress
|
||||
records on stdout, the outcome in SUMMARY
|
||||
|
||||
Python 3.8 and the standard library, like install.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Iterator, TextIO
|
||||
|
||||
_UNSAFE = re.compile(r"[\x00-\x1f\x7f*?\[\]\\]+")
|
||||
_SPACES = re.compile(r" {2,}")
|
||||
# A system id becomes a bash array subscript and a menu key in the launcher.
|
||||
# Every manifest uses slugs; anything else is dropped, never cleaned into one.
|
||||
_SYSTEM_ID = re.compile(r"[A-Za-z0-9._-]+")
|
||||
|
||||
|
||||
def clean(value: object) -> str:
|
||||
"""One FIFO-safe field: no control character, pattern character or run of spaces."""
|
||||
text = _UNSAFE.sub(" ", str(value))
|
||||
return _SPACES.sub(" ", text).strip()
|
||||
|
||||
|
||||
def size_text(count: int) -> str:
|
||||
"""12 KB, 3.4 MB, 1.2 GB."""
|
||||
value = float(count)
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if value < 1024 or unit == "GB":
|
||||
return f"{value:.0f} {unit}" if unit in ("B", "KB") else f"{value:.1f} {unit}"
|
||||
value /= 1024
|
||||
return f"{value:.1f} GB"
|
||||
|
||||
|
||||
def events(lines: Iterable[str]) -> Iterator[dict]:
|
||||
"""The JSON objects of an event stream. A line that is not one is skipped."""
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(event, dict) and isinstance(event.get("event"), str):
|
||||
yield event
|
||||
|
||||
|
||||
def record(out: TextIO, *fields: object) -> None:
|
||||
# A tab is whitespace to `read`, so two in a row are one separator: an
|
||||
# empty field would shift the ones after it. "-" stands for it.
|
||||
out.write("\t".join(clean(field) or "-" for field in fields) + "\n")
|
||||
out.flush()
|
||||
|
||||
|
||||
def status(stream: Iterable[str], out: TextIO) -> None:
|
||||
"""The records the start screen and the menus read.
|
||||
|
||||
Only the first detected platform is described: a handheld runs one.
|
||||
"""
|
||||
platform = None
|
||||
ended = False
|
||||
for event in events(stream):
|
||||
kind = event["event"]
|
||||
if kind == "start":
|
||||
record(out, "start", event.get("installer", ""))
|
||||
elif kind == "detect":
|
||||
found = event.get("platforms") or []
|
||||
record(out, "detect", event.get("os_name", ""), len(found))
|
||||
if found:
|
||||
first = found[0]
|
||||
platform = first.get("platform")
|
||||
record(out, "platform", platform, first.get("name", ""),
|
||||
first.get("bios_dir", ""))
|
||||
elif kind == "error":
|
||||
record(out, "error", event.get("reason", ""), event.get("message", ""))
|
||||
elif kind == "system" and event.get("platform") == platform:
|
||||
if not _SYSTEM_ID.fullmatch(str(event.get("system", ""))):
|
||||
continue
|
||||
needed = event.get("missing", 0) + event.get("wrong_hash", 0)
|
||||
system = event.get("system", "")
|
||||
# install.py files what belongs to no single system under "other".
|
||||
name = "Files of no single system" if system == "other" else event.get("name", "")
|
||||
record(out, "system", system, name,
|
||||
event.get("files", 0), event.get("present", 0), event.get("missing", 0),
|
||||
event.get("wrong_hash", 0), size_text(event.get("bytes_needed", 0)),
|
||||
"1" if needed else "0")
|
||||
elif kind == "check" and event.get("platform") == platform:
|
||||
record(out, "check", event.get("files", 0), event.get("present", 0),
|
||||
event.get("missing", 0), event.get("wrong_hash", 0),
|
||||
size_text(event.get("bytes_needed", 0)), event.get("data_missing", 0),
|
||||
event.get("not_installable", 0))
|
||||
elif kind == "end":
|
||||
ended = True
|
||||
record(out, "end", event.get("code", 1))
|
||||
if not ended:
|
||||
# Refused before the run started, or stopped without a word.
|
||||
record(out, "end", "none")
|
||||
|
||||
|
||||
def follow(stream: Iterable[str], out: TextIO, summary: TextIO) -> None:
|
||||
"""Progress records as an install runs, and its outcome in *summary*."""
|
||||
ended = False
|
||||
for event in events(stream):
|
||||
kind = event["event"]
|
||||
if kind == "download":
|
||||
record(out, "download", event.get("files", 0), size_text(event.get("bytes", 0)))
|
||||
elif kind == "downloaded":
|
||||
record(out, "file", event.get("files_done", 0), event.get("files_total", 0),
|
||||
"ok" if event.get("ok") else "failed", event.get("dest", ""),
|
||||
size_text(event.get("bytes_done", 0)),
|
||||
size_text(event.get("bytes_total", 0)))
|
||||
elif kind == "progress":
|
||||
record(out, "progress", event.get("files_done", 0), event.get("files_total", 0),
|
||||
size_text(event.get("bytes_done", 0)),
|
||||
size_text(event.get("bytes_total", 0)))
|
||||
elif kind == "error":
|
||||
record(out, "error", event.get("reason", ""), event.get("message", ""))
|
||||
record(summary, "error", event.get("reason", ""), event.get("message", ""))
|
||||
elif kind == "summary":
|
||||
record(summary, "summary", event.get("installed", 0), event.get("up_to_date", 0),
|
||||
event.get("pending", 0), "1" if event.get("interrupted") else "0")
|
||||
for failure in event.get("failed") or []:
|
||||
record(summary, "failed", failure.get("dest", ""), failure.get("reason", ""))
|
||||
for group in event.get("skipped") or []:
|
||||
record(summary, "skipped", group.get("label", ""), group.get("files", 0),
|
||||
group.get("required", 0))
|
||||
elif kind == "end":
|
||||
ended = True
|
||||
record(summary, "end", event.get("code", 1))
|
||||
if not ended:
|
||||
record(summary, "end", "none")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) == 2 and argv[0] == "status":
|
||||
with open(argv[1], encoding="utf-8", errors="replace") as stream:
|
||||
status(stream, sys.stdout)
|
||||
return 0
|
||||
if len(argv) == 2 and argv[0] == "follow":
|
||||
with Path(argv[1]).open("w", encoding="utf-8") as summary:
|
||||
follow(sys.stdin, sys.stdout, summary)
|
||||
return 0
|
||||
sys.stderr.write("usage: rbevents.py status EVENTS | follow SUMMARY\n")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,467 @@
|
||||
#!/bin/bash
|
||||
# RetroBIOS screens, drawn by PortMaster's own GUI (pugwash fifo_control)
|
||||
# through the functions of PortMasterDialog.txt. Sourced by RetroBIOS.sh
|
||||
# after PortMasterDialog.txt.
|
||||
#
|
||||
# Only what PortMaster 2024.02.26 and the 8.5.23 copy JELOS and KOLOS ship
|
||||
# is used: PortMasterDialogInit, PortMasterDialog, PortMasterDialogResult,
|
||||
# PortMasterDialogExit, PortMasterIPCheck, and the commands
|
||||
# register_set_info, register_clear, selection_list, messages_begin,
|
||||
# message, messages_end, progress, progress_clear and check_runtime.
|
||||
# Those releases ignore --want-cancel, so every list ends on its own Back
|
||||
# or Quit entry. They also read "\n" literally, so no text has a newline,
|
||||
# and their wait for an answer gives up after 30 seconds: rb_answer keeps
|
||||
# waiting while pugwash is alive.
|
||||
#
|
||||
# Inputs: GAMEDIR, controlfolder, ESUDO, and RB_INSTALL (install.py).
|
||||
|
||||
RB_STATE="${RB_STATE:-$GAMEDIR/run}"
|
||||
RB_LOG="${RB_LOG:-$GAMEDIR/log.txt}"
|
||||
RB_INSTALL="${RB_INSTALL:-$GAMEDIR/install.py}"
|
||||
RB_EVENTS="${RB_EVENTS:-$GAMEDIR/rbevents.py}"
|
||||
RB_RUNTIME="python_3.11"
|
||||
RB_PC_ROUTE="On a computer, with this SD card mounted, the same installer fills the card: install.py --platform NAME --dest BIOS-FOLDER-ON-THE-CARD. Instructions: abdess.github.io/retrobios/wiki/installer"
|
||||
RB_PYTHON=""
|
||||
RB_PYTHONHOME=""
|
||||
RB_MOUNTED=""
|
||||
RB_PYTHON_PROBLEM=""
|
||||
|
||||
rb_arch() {
|
||||
uname -m
|
||||
}
|
||||
|
||||
rb_py() {
|
||||
if [ -n "$RB_PYTHONHOME" ]; then
|
||||
PYTHONHOME="$RB_PYTHONHOME" "$RB_PYTHON" "$@"
|
||||
else
|
||||
"$RB_PYTHON" "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Python ---------------------------------------------------------------
|
||||
|
||||
rb_python_ok() {
|
||||
"$@" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)' >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# The OS python3 when it is 3.8 or newer, else PortMaster's python_3.11
|
||||
# runtime on aarch64, mounted as PortMaster's own Python ports do. Sets
|
||||
# RB_PYTHON_PROBLEM when neither can run install.py.
|
||||
rb_select_python() {
|
||||
if command -v python3 >/dev/null 2>&1 && rb_python_ok python3; then
|
||||
RB_PYTHON="python3"
|
||||
return 0
|
||||
fi
|
||||
if [ "$(rb_arch)" != "aarch64" ]; then
|
||||
RB_PYTHON_PROBLEM="old"
|
||||
return 1
|
||||
fi
|
||||
rb_runtime
|
||||
}
|
||||
|
||||
rb_runtime() {
|
||||
local python_dir="$HOME/$RB_RUNTIME"
|
||||
local squashfs="$controlfolder/libs/$RB_RUNTIME.squashfs"
|
||||
if [ ! -f "$squashfs" ]; then
|
||||
if [ -z "$(PortMasterIPCheck)" ]; then
|
||||
RB_PYTHON_PROBLEM="runtime_offline"
|
||||
return 1
|
||||
fi
|
||||
PortMasterDialogInit "no-check"
|
||||
PortMasterDialog "messages_begin"
|
||||
PortMasterDialog "message" "This system's Python is older than 3.8. Downloading PortMaster's $RB_RUNTIME runtime."
|
||||
local answer
|
||||
answer=$(rb_answer "check_runtime" "$RB_RUNTIME.squashfs")
|
||||
PortMasterDialog "messages_end"
|
||||
PortMasterDialogExit
|
||||
if [ "$answer" != "OKAY" ] || [ ! -f "$squashfs" ]; then
|
||||
RB_PYTHON_PROBLEM="runtime_failed"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
mkdir -p "$python_dir"
|
||||
if [ "${PM_CAN_MOUNT:-Y}" != "N" ]; then
|
||||
$ESUDO umount "$python_dir" >/dev/null 2>&1
|
||||
fi
|
||||
if ! $ESUDO mount "$squashfs" "$python_dir"; then
|
||||
RB_PYTHON_PROBLEM="runtime_failed"
|
||||
return 1
|
||||
fi
|
||||
RB_MOUNTED="$python_dir"
|
||||
RB_PYTHON="$python_dir/bin/python3"
|
||||
RB_PYTHONHOME="$python_dir"
|
||||
if ! PYTHONHOME="$python_dir" rb_python_ok "$RB_PYTHON"; then
|
||||
RB_PYTHON_PROBLEM="runtime_failed"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
rb_release_python() {
|
||||
if [ -n "$RB_MOUNTED" ] && [ "${PM_CAN_MOUNT:-Y}" != "N" ]; then
|
||||
$ESUDO umount "$RB_MOUNTED" >/dev/null 2>&1
|
||||
fi
|
||||
RB_MOUNTED=""
|
||||
}
|
||||
|
||||
# --- Dialog primitives ----------------------------------------------------
|
||||
|
||||
# A command that waits for the user, and its answer. PortMasterDialogResult
|
||||
# stops waiting after 30 seconds on the older releases and then returns
|
||||
# WAIT: the answer is read again until pugwash writes one or exits.
|
||||
rb_answer() {
|
||||
local answer tries=0
|
||||
answer=$(PortMasterDialogResult "$@")
|
||||
while [ "$answer" = "WAIT" ] || [ -z "$answer" ]; do
|
||||
if [ -n "$PM_PID" ] && [ ! -d "/proc/$PM_PID" ]; then
|
||||
break
|
||||
fi
|
||||
if [ "$tries" -ge 6000 ]; then
|
||||
break
|
||||
fi
|
||||
tries=$((tries + 1))
|
||||
sleep 0.3
|
||||
answer=$(cat "$PM_DONE" 2>/dev/null)
|
||||
done
|
||||
printf '%s\n' "$answer"
|
||||
}
|
||||
|
||||
rb_plural() {
|
||||
if [ "$1" = "1" ]; then
|
||||
printf '%s %s\n' "$1" "$2"
|
||||
else
|
||||
printf '%s %ss\n' "$1" "$2"
|
||||
fi
|
||||
}
|
||||
|
||||
rb_entry() {
|
||||
PortMasterDialog "register_set_info" "$1" "$2" "title:$3" "description:$4"
|
||||
}
|
||||
|
||||
# Show a list and print the chosen key. B, where the release reads it,
|
||||
# answers like the Back or Quit entry ($2).
|
||||
rb_choose() {
|
||||
local answer
|
||||
answer=$(rb_answer "selection_list" "--register=$1" "--want-description" "--want-cancel")
|
||||
case "$answer" in
|
||||
CANCEL|FAIL|WAIT|"") printf '%s\n' "$2" ;;
|
||||
*) printf '%s\n' "$answer" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# A screen of text with one or two ways out: rb_notice TITLE TEXT KEY LABEL [KEY LABEL].
|
||||
# B answers like the last one, which leaves.
|
||||
rb_notice() {
|
||||
local title="$1" text="$2"
|
||||
shift 2
|
||||
local last="$1" answer
|
||||
PortMasterDialog "register_clear" "notice"
|
||||
# The list has no heading of its own: the first entry is the heading.
|
||||
rb_entry "notice" "info" "$title" "$text"
|
||||
rb_entry "notice" "$1" "$2" "$text"
|
||||
if [ $# -ge 4 ]; then
|
||||
rb_entry "notice" "$3" "$4" "$text"
|
||||
last="$3"
|
||||
fi
|
||||
while true; do
|
||||
answer=$(rb_choose "notice" "$last")
|
||||
if [ "$answer" != "info" ]; then
|
||||
printf '%s\n' "$answer"
|
||||
return
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# --- Reading install.py ---------------------------------------------------
|
||||
|
||||
# Run a --check and read its records into RB_* variables.
|
||||
rb_check() {
|
||||
mkdir -p "$RB_STATE"
|
||||
PortMasterDialog "messages_begin"
|
||||
PortMasterDialog "message" "Reading the BIOS folder and the file list."
|
||||
rb_py "$RB_INSTALL" --check --events "${RB_PLATFORM_ARGS[@]}" > "$RB_STATE/check.jsonl" 2>> "$RB_LOG"
|
||||
PortMasterDialog "messages_end"
|
||||
rb_py "$RB_EVENTS" status "$RB_STATE/check.jsonl" > "$RB_STATE/status.tsv" 2>> "$RB_LOG"
|
||||
rb_read_status "$RB_STATE/status.tsv"
|
||||
}
|
||||
|
||||
rb_read_status() {
|
||||
RB_OS_NAME="this system"
|
||||
RB_DETECTED=""
|
||||
RB_PLATFORM=""
|
||||
RB_PLATFORM_NAME=""
|
||||
RB_BIOS_DIR=""
|
||||
RB_ERROR=""
|
||||
RB_ERROR_TEXT=""
|
||||
RB_CHECKED=""
|
||||
RB_END="none"
|
||||
RB_SYSTEMS=()
|
||||
RB_COMPLETE=0
|
||||
local kind a b c d e f g h
|
||||
while IFS=$'\t' read -r kind a b c d e f g h; do
|
||||
case "$kind" in
|
||||
detect) RB_OS_NAME="$a"; RB_DETECTED="$b" ;;
|
||||
platform) RB_PLATFORM="$a"; RB_PLATFORM_NAME="$b"; RB_BIOS_DIR="$c" ;;
|
||||
error) RB_ERROR="$a"; RB_ERROR_TEXT="$b" ;;
|
||||
check)
|
||||
RB_CHECKED=1
|
||||
RB_FILES="$a"; RB_PRESENT="$b"; RB_MISSING="$c"; RB_WRONG="$d"
|
||||
RB_NEEDED_SIZE="$e"; RB_DATA_MISSING="$f"; RB_NOT_INSTALLABLE="$g"
|
||||
;;
|
||||
system)
|
||||
if [ "$h" = "1" ]; then
|
||||
RB_SYSTEMS+=("$a"$'\t'"$b"$'\t'"$c"$'\t'"$d"$'\t'"$e"$'\t'"$f"$'\t'"$g")
|
||||
else
|
||||
RB_COMPLETE=$((RB_COMPLETE + 1))
|
||||
fi
|
||||
;;
|
||||
end) RB_END="$a" ;;
|
||||
esac
|
||||
done < "$1"
|
||||
}
|
||||
|
||||
rb_status_text() {
|
||||
local text="$RB_OS_NAME, $RB_PLATFORM_NAME file list: $RB_PRESENT/$RB_FILES BIOS files, $RB_MISSING missing, $RB_WRONG wrong hash."
|
||||
if [ "${RB_NOT_INSTALLABLE:-0}" != "0" ]; then
|
||||
text="$text $RB_NOT_INSTALLABLE not in the collection."
|
||||
fi
|
||||
printf '%s\n' "$text"
|
||||
}
|
||||
|
||||
rb_pending() {
|
||||
printf '%s\n' $((RB_MISSING + RB_WRONG + RB_DATA_MISSING))
|
||||
}
|
||||
|
||||
# --- Screens --------------------------------------------------------------
|
||||
|
||||
rb_screen_no_python() {
|
||||
local found
|
||||
found=$(python3 --version 2>&1 | awk '{print $2}')
|
||||
local text
|
||||
case "$RB_PYTHON_PROBLEM" in
|
||||
old) text="Python 3.8 or newer is needed. This system has ${found:-none}, and PortMaster's Python runtime exists for aarch64 only, and this device is $(rb_arch)." ;;
|
||||
runtime_offline) text="Python 3.8 or newer is needed. This system has ${found:-none}, and PortMaster's $RB_RUNTIME runtime needs a connection to download." ;;
|
||||
*) text="Python 3.8 or newer is needed. This system has ${found:-none}, and PortMaster's $RB_RUNTIME runtime could not be installed or run." ;;
|
||||
esac
|
||||
rb_notice "Python 3.8 or newer is needed" "$text $RB_PC_ROUTE" "quit" "Quit" > /dev/null
|
||||
}
|
||||
|
||||
# What the check found when it found nothing to show: no platform, no
|
||||
# connection, an unreadable list. Prints check or quit.
|
||||
rb_screen_problem() {
|
||||
if [ "$RB_END" = "none" ]; then
|
||||
rb_notice "The installer stopped before checking" "install.py ended without a result. Its messages are in $RB_LOG." "check" "Check again" "quit" "Quit"
|
||||
elif [ "$RB_ERROR" = "conflict" ]; then
|
||||
rb_notice "Two systems found" "$RB_ERROR_TEXT, so the BIOS folder is not known. $RB_PC_ROUTE" "check" "Check again" "quit" "Quit"
|
||||
elif [ "$RB_DETECTED" = "0" ]; then
|
||||
rb_notice "No BIOS folder found" "install.py knows no BIOS folder on $RB_OS_NAME. $RB_PC_ROUTE" "check" "Check again" "quit" "Quit"
|
||||
elif [ "$RB_ERROR" = "network" ]; then
|
||||
rb_notice "No connection" "$RB_OS_NAME, BIOS folder $RB_BIOS_DIR. The list of files to check comes from the internet, and so do the files: without a connection nothing can be checked or downloaded. Connect to Wi-Fi, then check again." "check" "Check again" "quit" "Quit"
|
||||
elif [ "$RB_ERROR" = "invalid" ]; then
|
||||
rb_notice "The file list is newer than this port" "$RB_ERROR_TEXT. Update RetroBIOS from PortMaster, or use the installer on a computer. $RB_PC_ROUTE" "check" "Check again" "quit" "Quit"
|
||||
else
|
||||
rb_notice "The file list could not be read" "${RB_ERROR_TEXT:-install.py ended with code $RB_END}. Its messages are in $RB_LOG." "check" "Check again" "quit" "Quit"
|
||||
fi
|
||||
}
|
||||
|
||||
# The menu after a check. Prints update, choose, check or quit.
|
||||
rb_screen_menu() {
|
||||
local status pending
|
||||
status=$(rb_status_text)
|
||||
pending=$(rb_pending)
|
||||
PortMasterDialog "register_clear" "menu"
|
||||
# The list has no heading of its own: the first entry is the status.
|
||||
if [ "$pending" -gt 0 ]; then
|
||||
local data=""
|
||||
if [ "${RB_DATA_MISSING:-0}" != "0" ]; then
|
||||
data=" and $RB_DATA_MISSING data files"
|
||||
fi
|
||||
rb_entry "menu" "status" "$RB_OS_NAME: $RB_PRESENT/$RB_FILES BIOS files, $RB_MISSING missing, $RB_WRONG wrong hash" "$status BIOS folder: $RB_BIOS_DIR."
|
||||
rb_entry "menu" "update" "Update everything missing" "$status Downloads $(rb_plural $((RB_MISSING + RB_WRONG)) file)$data, $RB_NEEDED_SIZE, to $RB_BIOS_DIR."
|
||||
rb_entry "menu" "choose" "Choose systems" "$status Pick the systems to update."
|
||||
else
|
||||
rb_entry "menu" "status" "$RB_OS_NAME: all $RB_FILES BIOS files present" "$status Nothing to download. BIOS folder: $RB_BIOS_DIR."
|
||||
fi
|
||||
rb_entry "menu" "check" "Check again" "$status Reads the BIOS folder and the file list again."
|
||||
rb_entry "menu" "quit" "Quit" "$status"
|
||||
local answer
|
||||
while true; do
|
||||
answer=$(rb_choose "menu" "quit")
|
||||
if [ "$answer" != "status" ]; then
|
||||
printf '%s\n' "$answer"
|
||||
return
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Systems with files to fetch, toggled with A. Prints the chosen system
|
||||
# ids, one per line, or nothing for Back.
|
||||
rb_screen_systems() {
|
||||
local -A chosen=()
|
||||
local line id name files present missing wrong size answer count
|
||||
while true; do
|
||||
count=0
|
||||
for id in "${!chosen[@]}"; do
|
||||
count=$((count + 1))
|
||||
done
|
||||
PortMasterDialog "register_clear" "systems"
|
||||
if [ "$count" -gt 0 ]; then
|
||||
rb_entry "systems" "update" "Update the $count selected" "Downloads what the selected systems lack to $RB_BIOS_DIR."
|
||||
else
|
||||
rb_entry "systems" "update" "Update the selected" "Nothing selected yet: pick systems below with A."
|
||||
fi
|
||||
rb_entry "systems" "back" "Back" "Back to the menu. $(rb_plural "$RB_COMPLETE" system) complete, not listed."
|
||||
for line in "${RB_SYSTEMS[@]}"; do
|
||||
IFS=$'\t' read -r id name files present missing wrong size <<< "$line"
|
||||
local mark=""
|
||||
if [ -n "${chosen[$id]}" ]; then
|
||||
mark=" (selected)"
|
||||
fi
|
||||
rb_entry "systems" "sys:$id" "$name$mark" "$present/$files present, $missing missing, $wrong wrong hash, $size to download. A selects or clears it."
|
||||
done
|
||||
answer=$(rb_choose "systems" "back")
|
||||
case "$answer" in
|
||||
back) return ;;
|
||||
update)
|
||||
if [ "$count" -gt 0 ]; then
|
||||
printf '%s\n' "${!chosen[@]}" | sort
|
||||
return
|
||||
fi
|
||||
;;
|
||||
sys:*)
|
||||
id="${answer#sys:}"
|
||||
# Quoted literally so unset expands the subscript once: a double-quoted
|
||||
# "chosen[$id]" is expanded again inside unset.
|
||||
[[ "$id" =~ ^[A-Za-z0-9._-]+$ ]] || continue
|
||||
if [ -n "${chosen[$id]}" ]; then
|
||||
unset 'chosen[$id]'
|
||||
else
|
||||
chosen[$id]=1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# Download with a progress bar fed by the event stream: rb_update [SYSTEM...]
|
||||
rb_update() {
|
||||
local args=() system
|
||||
for system in "$@"; do
|
||||
args+=(--system "$system")
|
||||
done
|
||||
mkdir -p "$RB_STATE"
|
||||
PortMasterDialog "messages_begin"
|
||||
PortMasterDialog "message" "Downloading to $RB_BIOS_DIR."
|
||||
PortMasterDialog "progress" "Starting" 0 1
|
||||
local kind a b c d e f
|
||||
rb_py "$RB_INSTALL" --events "${RB_PLATFORM_ARGS[@]}" "${args[@]}" 2>> "$RB_LOG" \
|
||||
| rb_py "$RB_EVENTS" follow "$RB_STATE/summary.tsv" 2>> "$RB_LOG" \
|
||||
| while IFS=$'\t' read -r kind a b c d e f; do
|
||||
case "$kind" in
|
||||
download) PortMasterDialog "message" "$(rb_plural "$a" file), $b." ;;
|
||||
file) PortMasterDialog "progress" "$d, $e of $f" "$a" "$b" ;;
|
||||
progress) PortMasterDialog "progress" "$c of $d" "$a" "$b" ;;
|
||||
error) PortMasterDialog "message" "$b" ;;
|
||||
esac
|
||||
done
|
||||
PortMasterDialog "progress_clear"
|
||||
PortMasterDialog "messages_end"
|
||||
}
|
||||
|
||||
# What the update did. Prints back or quit.
|
||||
rb_screen_summary() {
|
||||
local kind a b c d shown=0 failed=0
|
||||
local end="none" installed=0 up_to_date=0 pending=0 interrupted=0 error=""
|
||||
while IFS=$'\t' read -r kind a b c d; do
|
||||
case "$kind" in
|
||||
summary) installed="$a"; up_to_date="$b"; pending="$c"; interrupted="$d" ;;
|
||||
failed) failed=$((failed + 1)) ;;
|
||||
error) error="$b" ;;
|
||||
end) end="$a" ;;
|
||||
esac
|
||||
done < "$RB_STATE/summary.tsv"
|
||||
PortMasterDialog "register_clear" "summary"
|
||||
# The first entry is the outcome, the heading the list does not have.
|
||||
if [ "$end" = "none" ]; then
|
||||
rb_entry "summary" "info:result" "The installer stopped" "install.py ended without a result. Its messages are in $RB_LOG."
|
||||
elif [ -n "$error" ]; then
|
||||
rb_entry "summary" "info:result" "Nothing downloaded" "$error."
|
||||
else
|
||||
rb_entry "summary" "info:result" "Installed: $(rb_plural "$installed" file)" "$up_to_date already up to date, $failed failed, $pending still to fetch."
|
||||
fi
|
||||
rb_entry "summary" "back" "Back to the menu" "Checks the BIOS folder again."
|
||||
rb_entry "summary" "quit" "Quit" "Leaves RetroBIOS."
|
||||
if [ "$interrupted" = "1" ]; then
|
||||
rb_entry "summary" "info:stopped" "Stopped before the end" "Update again to carry on. Files already installed are kept."
|
||||
fi
|
||||
while IFS=$'\t' read -r kind a b c d; do
|
||||
case "$kind" in
|
||||
failed)
|
||||
shown=$((shown + 1))
|
||||
if [ "$shown" -le 50 ]; then
|
||||
rb_entry "summary" "info:failed:$shown" "Failed: $a" "$b"
|
||||
fi
|
||||
;;
|
||||
skipped)
|
||||
rb_entry "summary" "info:skipped:$a" "Not in the collection ($a): $(rb_plural "$b" file)" "Required: $c. The RetroBIOS collection does not hold these files, so they cannot be downloaded."
|
||||
;;
|
||||
esac
|
||||
done < "$RB_STATE/summary.tsv"
|
||||
if [ "$shown" -gt 50 ]; then
|
||||
rb_entry "summary" "info:more" "And $((shown - 50)) more failed" "The full list is in $RB_LOG."
|
||||
fi
|
||||
local answer
|
||||
while true; do
|
||||
answer=$(rb_choose "summary" "back")
|
||||
case "$answer" in
|
||||
back|quit) printf '%s\n' "$answer"; return ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# --- Flow -----------------------------------------------------------------
|
||||
|
||||
rb_main() {
|
||||
if ! rb_select_python; then
|
||||
PortMasterDialogInit "no-harbour"
|
||||
rb_screen_no_python
|
||||
PortMasterDialogExit
|
||||
rb_release_python
|
||||
return 1
|
||||
fi
|
||||
PortMasterDialogInit "no-harbour"
|
||||
RB_PLATFORM_ARGS=()
|
||||
local action systems chosen
|
||||
rb_check
|
||||
while true; do
|
||||
if [ -z "$RB_CHECKED" ]; then
|
||||
action=$(rb_screen_problem)
|
||||
else
|
||||
# Later runs name the platform the first one found, so a second
|
||||
# detected platform never changes what the screens describe.
|
||||
RB_PLATFORM_ARGS=(--platform "$RB_PLATFORM")
|
||||
action=$(rb_screen_menu)
|
||||
fi
|
||||
case "$action" in
|
||||
update)
|
||||
rb_update
|
||||
[ "$(rb_screen_summary)" = "quit" ] && break
|
||||
rb_check
|
||||
;;
|
||||
choose)
|
||||
systems=$(rb_screen_systems)
|
||||
if [ -n "$systems" ]; then
|
||||
mapfile -t chosen <<< "$systems"
|
||||
rb_update "${chosen[@]}"
|
||||
[ "$(rb_screen_summary)" = "quit" ] && break
|
||||
rb_check
|
||||
fi
|
||||
;;
|
||||
check) rb_check ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
PortMasterDialogExit
|
||||
rb_release_python
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the PortMaster port in ports/RetroBIOS into dist/RetroBIOS.zip.
|
||||
|
||||
The archive has the layout PortMaster's own release build gives a port
|
||||
(PortMaster-New tools/build_release.py): the launch script at the top, and
|
||||
one folder holding port.json, gameinfo.xml, the README as <name>.md, the
|
||||
licenses and the port's files. install.py is copied from the repository
|
||||
root, and only when it is the installer the bootstraps pin, so the port
|
||||
runs the same bytes as the one-line command of the same commit. Members are
|
||||
written as the packs write theirs: fixed timestamps, sorted names, the
|
||||
executable bit read from the source, so the same tree gives the same zip.
|
||||
|
||||
Usage:
|
||||
python scripts/build_port.py
|
||||
python scripts/build_port.py --output-dir tmp/port
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from artifacts import ArtifactLockBusy, artifact_lock
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
PORT_SOURCE = REPO_ROOT / "ports" / "RetroBIOS"
|
||||
PORT_DIR = "retrobios"
|
||||
PORT_SCRIPT = "RetroBIOS.sh"
|
||||
PORT_ZIP = "RetroBIOS.zip"
|
||||
_PIN = re.compile(r'^DEFAULT_INSTALL_SHA256="([0-9a-f]{64})"$', re.MULTILINE)
|
||||
|
||||
|
||||
def pinned_installer_sha256(root: Path = REPO_ROOT) -> str:
|
||||
"""The SHA-256 install.sh accepts for install.py."""
|
||||
found = _PIN.search((root / "install.sh").read_text(encoding="utf-8"))
|
||||
if found is None:
|
||||
raise ValueError("install.sh pins no installer SHA-256")
|
||||
return found.group(1)
|
||||
|
||||
|
||||
def port_members(root: Path = REPO_ROOT) -> list[tuple[str, Path]]:
|
||||
"""(name in the archive, source file) for every member, sorted by name."""
|
||||
source = root / "ports" / "RetroBIOS"
|
||||
members = [
|
||||
(PORT_SCRIPT, source / PORT_SCRIPT),
|
||||
(f"{PORT_DIR}/port.json", source / "port.json"),
|
||||
(f"{PORT_DIR}/gameinfo.xml", source / "gameinfo.xml"),
|
||||
(f"{PORT_DIR}/{PORT_DIR}.md", source / "README.md"),
|
||||
(f"{PORT_DIR}/install.py", root / "install.py"),
|
||||
(f"{PORT_DIR}/licenses/LICENSE-retrobios.txt", root / "LICENSE"),
|
||||
(f"{PORT_DIR}/licenses/NOTICE-retrobios.txt", root / "NOTICE"),
|
||||
]
|
||||
members += [
|
||||
(f"{PORT_DIR}/{path.relative_to(source / PORT_DIR).as_posix()}", path)
|
||||
for path in sorted((source / PORT_DIR).rglob("*"))
|
||||
if path.is_file() and "__pycache__" not in path.parts
|
||||
]
|
||||
names = [name for name, _path in members]
|
||||
if len(set(names)) != len(names):
|
||||
raise ValueError("two port members share a name")
|
||||
return sorted(members)
|
||||
|
||||
|
||||
def build_port(output: Path, root: Path = REPO_ROOT) -> Path:
|
||||
"""Write the port archive to *output* and return it."""
|
||||
pinned = pinned_installer_sha256(root)
|
||||
actual = hashlib.sha256((root / "install.py").read_bytes()).hexdigest()
|
||||
if actual != pinned:
|
||||
raise ValueError(
|
||||
f"install.py is {actual[:12]}, install.sh pins {pinned[:12]}: "
|
||||
"re-pin the bootstraps before building the port"
|
||||
)
|
||||
# The packs' member writer: imported here, as it loads the whole
|
||||
# pack builder, which a caller reading PORT_ZIP does not need.
|
||||
from generate_pack import _add_pack_member # noqa: PLC0415
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
scratch = output.with_name(f".{output.name}.tmp")
|
||||
with zipfile.ZipFile(scratch, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as zf:
|
||||
for name, path in port_members(root):
|
||||
_add_pack_member(zf, str(path), name)
|
||||
scratch.replace(output)
|
||||
return output
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
allow_abbrev=False, description="Build the RetroBIOS PortMaster port"
|
||||
)
|
||||
parser.add_argument("--output-dir", type=Path, default=REPO_ROOT / "dist",
|
||||
help="where RetroBIOS.zip is written (default: dist/)")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
with artifact_lock(str(args.output_dir)):
|
||||
path = build_port(args.output_dir / PORT_ZIP)
|
||||
except (ArtifactLockBusy, OSError, ValueError) as exc:
|
||||
print(f"Error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
print(f"{path} ({path.stat().st_size} bytes, sha256 {digest})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from common import run_cli
|
||||
|
||||
run_cli(main)
|
||||
@@ -3,7 +3,8 @@
|
||||
|
||||
Run on the draft before publishing it, and on the published release at any
|
||||
time. Every pack asset release.json names must be on the release at the sha256
|
||||
SHA256SUMS.txt lists for it, the parts of a pack must add up to the download
|
||||
SHA256SUMS.txt lists for it, as must the handheld port (RetroBIOS.zip) when
|
||||
the list names it, the parts of a pack must add up to the download
|
||||
size the README prints, and nothing else may be attached. An upload that
|
||||
failed in the loop, or a part re-uploaded from an older build, otherwise
|
||||
publishes a release that `sha256sum --check` rejects for every user.
|
||||
@@ -26,6 +27,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from build_port import PORT_ZIP
|
||||
from check_release_assets import digests_of
|
||||
from largefiles import LARGE_FILES_REPO
|
||||
from release_record import RECORD, load_record
|
||||
@@ -91,7 +93,9 @@ def compare_release(
|
||||
findings.append(Finding("extra", name, "attached but neither recorded nor listed"))
|
||||
for name in sorted(set(recorded) - set(sums)):
|
||||
findings.append(Finding("unlisted", name, f"absent from {SUMS}"))
|
||||
for name in sorted(set(sums) - set(recorded)):
|
||||
# The handheld port is a release file but no pack: release.json records
|
||||
# packs, and the checksum list alone vouches for the port.
|
||||
for name in sorted(set(sums) - set(recorded) - {PORT_ZIP}):
|
||||
findings.append(Finding("unrecorded", name, f"listed in {SUMS}, absent from release.json"))
|
||||
|
||||
wanted = {**sums, **(local_files or {})}
|
||||
|
||||
@@ -5283,6 +5283,10 @@ ZIP: extract them all into the same folder.
|
||||
| [AmberELEC](https://amberelec.org/) | Linux OS for Anbernic handhelds: RG351P, RG351V, RG351MP, RG353, RG503 and RG552 | [AmberELEC]({rel}) | `/storage/roms/`, which the pack fills with `bios/` and `neogeo/` |
|
||||
| [EmuELEC](https://emuelec.org/) | Linux OS for Amlogic TV boxes, the Odroid Go Advance, GameForce and RK356x devices | [EmuELEC]({rel}) | `/storage/roms/`, which the pack fills with `bios/`, `neogeo/` and `ports/` |
|
||||
|
||||
On a handheld that runs PortMaster, the RetroBIOS port checks and fills the
|
||||
BIOS folder from the device, with gamepad screens:
|
||||
[on a handheld](wiki/installer.md#on-a-handheld-the-retrobios-port).
|
||||
|
||||
### FPGA
|
||||
|
||||
| Setup | What it is | Pack | Extract to |
|
||||
|
||||
@@ -95,6 +95,21 @@ class ReleaseMatchesWhatWasBuilt(unittest.TestCase):
|
||||
self.assertIn(("unrecorded", "Other_BIOS_Pack.zip"), kinds)
|
||||
self.assertIn(("missing", "Other_BIOS_Pack.zip"), kinds)
|
||||
|
||||
def test_the_handheld_port_is_vouched_for_by_the_checksum_list(self):
|
||||
sums = {**SUMS_LIST, "RetroBIOS.zip": A}
|
||||
assets = _assets()
|
||||
self.assertEqual(
|
||||
_kinds(compare_release(RECORD, sums, assets, "v2026.10.10")),
|
||||
{("missing", "RetroBIOS.zip")},
|
||||
)
|
||||
assets["RetroBIOS.zip"] = Asset(70, A)
|
||||
self.assertEqual(compare_release(RECORD, sums, assets, "v2026.10.10"), [])
|
||||
assets["RetroBIOS.zip"] = Asset(70, B)
|
||||
self.assertEqual(
|
||||
_kinds(compare_release(RECORD, sums, assets, "v2026.10.10")),
|
||||
{("content", "RetroBIOS.zip")},
|
||||
)
|
||||
|
||||
def test_parts_must_add_up_to_the_printed_download_size(self):
|
||||
assets = _assets()
|
||||
assets["Big_BIOS_Pack.part2of2.zip"] = Asset(99, B)
|
||||
|
||||
@@ -12,6 +12,7 @@ import os
|
||||
import shlex
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -4688,6 +4689,17 @@ class HandheldSystemsByTheirOwnFiles(unittest.TestCase):
|
||||
install.main()
|
||||
self.assertEqual(stop.exception.code, 1)
|
||||
self.assertIn("JELOS and Koriki are both detected here", err.getvalue())
|
||||
with unittest.mock.patch.object(sys, "argv", ["install.py", "--no-input", "--events"]), \
|
||||
unittest.mock.patch.object(install, "detect_os", return_value="linux"), \
|
||||
unittest.mock.patch.object(install, "detect_platforms", return_value=found), \
|
||||
unittest.mock.patch.object(install, "detect_frontends", return_value=[]), \
|
||||
contextlib.redirect_stdout(io.StringIO()) as out, \
|
||||
contextlib.redirect_stderr(io.StringIO()), \
|
||||
self.assertRaises(SystemExit):
|
||||
install.main()
|
||||
errors = [event for event in map(json.loads, out.getvalue().splitlines())
|
||||
if event["event"] == "error"]
|
||||
self.assertEqual([(e["stage"], e["reason"]) for e in errors], [("detect", "conflict")])
|
||||
|
||||
def test_the_system_is_announced_by_its_own_name(self):
|
||||
root, found, _name = self._one({self.ELEC_CFG: ""}, 'OS_NAME="JELOS"\n')
|
||||
@@ -4846,5 +4858,178 @@ class RomMFirmwareInstallsUnderItsTemplate(unittest.TestCase, _LoopbackRun):
|
||||
self.assertIn("{platform}/bios", proc.stdout)
|
||||
|
||||
|
||||
class _SyntheticTree:
|
||||
"""A served tree of four BIOS files over loopback, one with no system,
|
||||
and a BIOS folder holding one of them, another with other bytes, and
|
||||
neither of the last two."""
|
||||
|
||||
FILES = {
|
||||
"bios/A/a.bin": b"alpha" * 20,
|
||||
"bios/A/b.bin": b"bravo" * 30,
|
||||
"bios/B/c.bin": b"charlie" * 10,
|
||||
"bios/X/d.bin": b"delta" * 5,
|
||||
}
|
||||
|
||||
def serve_tree(self, test: unittest.TestCase) -> dict:
|
||||
root = Path(_tempdir(test))
|
||||
files = []
|
||||
for path, data in self.FILES.items():
|
||||
(root / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / path).write_bytes(data)
|
||||
name = Path(path).name
|
||||
systems = {"a.bin": ["sys-a"], "b.bin": ["sys-a"], "c.bin": ["sys-b"]}.get(name)
|
||||
entry = {"dest": name, "repo_path": path, "size": len(data),
|
||||
"sha1": hashlib.sha1(data).hexdigest(), "cores": None}
|
||||
if systems:
|
||||
entry["systems"] = systems
|
||||
files.append(entry)
|
||||
manifest = {
|
||||
"manifest_version": 2, "platform": "retroarch", "display_name": "RetroArch",
|
||||
"files": files, "system_names": {"sys-a": ["alpha"]},
|
||||
"omitted_files": [{"dest": "e.bin", "name": "e.bin", "system": "sys-b",
|
||||
"required": True, "reason": "not_found",
|
||||
"systems": ["sys-b"]}],
|
||||
}
|
||||
(root / "install").mkdir()
|
||||
(root / "install" / "retroarch.json").write_text(json.dumps(manifest))
|
||||
|
||||
class Quiet(http.server.SimpleHTTPRequestHandler):
|
||||
def log_message(self, *args) -> None:
|
||||
return
|
||||
|
||||
handler = functools.partial(Quiet, directory=str(root))
|
||||
httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
test.addCleanup(httpd.server_close)
|
||||
test.addCleanup(httpd.shutdown)
|
||||
self.home = Path(_tempdir(test))
|
||||
self.bios = self.home / "bios"
|
||||
self.bios.mkdir()
|
||||
(self.bios / "a.bin").write_bytes(self.FILES["bios/A/a.bin"])
|
||||
(self.bios / "b.bin").write_bytes(b"other bytes")
|
||||
return {
|
||||
**os.environ, "HOME": str(self.home), "RETROBIOS_OS": "linux",
|
||||
"RETROBIOS_BASE_URL": f"http://127.0.0.1:{httpd.server_address[1]}",
|
||||
}
|
||||
|
||||
|
||||
def _events(proc: subprocess.CompletedProcess) -> list[dict]:
|
||||
"""The --events stream, every stdout line a JSON object."""
|
||||
return [json.loads(line) for line in proc.stdout.splitlines()]
|
||||
|
||||
|
||||
class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
|
||||
"""--events: the handheld port reads the run as JSON lines on stdout and
|
||||
never parses the text, which goes to stderr."""
|
||||
|
||||
def setUp(self):
|
||||
self.env = self.serve_tree(self)
|
||||
|
||||
def _run(self, *argv: str) -> tuple[subprocess.CompletedProcess, list[dict]]:
|
||||
proc = _install_run(self.env, "--platform", "retroarch", "--dest", str(self.bios),
|
||||
"--events", "--jobs", "1", *argv)
|
||||
self.assertNotIn("Traceback", proc.stderr)
|
||||
return proc, _events(proc)
|
||||
|
||||
def _head(self, mode: str, installer: str) -> list[dict]:
|
||||
return [
|
||||
{"event": "start", "schema": install.EVENT_SCHEMA, "installer": installer,
|
||||
"mode": mode},
|
||||
{"event": "detect", "os": "linux", "os_name": install._os_name(),
|
||||
"platforms": [{"platform": "retroarch", "name": "RetroArch",
|
||||
"bios_dir": str(self.bios)}]},
|
||||
{"event": "file", "platform": "retroarch", "dest": "a.bin",
|
||||
"systems": ["sys-a"], "size": 100, "state": "present"},
|
||||
{"event": "file", "platform": "retroarch", "dest": "b.bin",
|
||||
"systems": ["sys-a"], "size": 150, "state": "wrong_hash"},
|
||||
{"event": "file", "platform": "retroarch", "dest": "c.bin",
|
||||
"systems": ["sys-b"], "size": 70, "state": "missing"},
|
||||
{"event": "file", "platform": "retroarch", "dest": "d.bin",
|
||||
"systems": [], "size": 25, "state": "missing"},
|
||||
{"event": "system", "platform": "retroarch", "system": "other", "name": "other",
|
||||
"files": 1, "present": 0, "missing": 1, "wrong_hash": 0, "bytes_needed": 25},
|
||||
{"event": "system", "platform": "retroarch", "system": "sys-a", "name": "alpha",
|
||||
"files": 2, "present": 1, "missing": 0, "wrong_hash": 1, "bytes_needed": 150},
|
||||
{"event": "system", "platform": "retroarch", "system": "sys-b", "name": "sys-b",
|
||||
"files": 1, "present": 0, "missing": 1, "wrong_hash": 0, "bytes_needed": 70},
|
||||
{"event": "check", "platform": "retroarch", "name": "RetroArch",
|
||||
"bios_dir": str(self.bios), "files": 4, "present": 1, "missing": 2,
|
||||
"wrong_hash": 1, "bytes_needed": 245, "data_missing": 0,
|
||||
"not_installable": 1},
|
||||
]
|
||||
|
||||
SKIPPED = [{"reason": "not_found", "label": "not found", "files": 1, "required": 1}]
|
||||
|
||||
def test_a_check_run(self):
|
||||
proc, events = self._run("--check")
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
installer = events[0]["installer"]
|
||||
self.assertEqual(events, self._head("check", installer) + [
|
||||
{"event": "summary", "mode": "check", "installed": 0, "up_to_date": 1,
|
||||
"pending": 3, "failed": [], "skipped": self.SKIPPED, "interrupted": False},
|
||||
{"event": "end", "code": 1},
|
||||
])
|
||||
self.assertIn("2/4 present (1 verified, 1 wrong hash)", proc.stderr)
|
||||
self.assertFalse((self.bios / "c.bin").exists())
|
||||
|
||||
def test_an_install_run(self):
|
||||
proc, events = self._run()
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
installer = events[0]["installer"]
|
||||
progress = [event for event in events if event["event"] == "progress"]
|
||||
for event in progress:
|
||||
self.assertEqual(list(event), ["event", *install.EVENT_KEYS["progress"]])
|
||||
done = [{"event": "downloaded", "dest": dest, "ok": True, "reason": "",
|
||||
"size": size, "files_done": n, "files_total": 3, "bytes_done": total,
|
||||
"bytes_total": 245}
|
||||
for n, (dest, size, total) in enumerate(
|
||||
(("c.bin", 70, 70), ("d.bin", 25, 95), ("b.bin", 150, 245)), 1)]
|
||||
self.assertEqual([e for e in events if e["event"] != "progress"],
|
||||
self._head("install", installer) + [
|
||||
{"event": "download", "platform": "retroarch", "files": 3, "bytes": 245},
|
||||
*done,
|
||||
{"event": "summary", "mode": "install", "installed": 3, "up_to_date": 1,
|
||||
"pending": 0, "failed": [], "skipped": self.SKIPPED, "interrupted": False},
|
||||
{"event": "end", "code": 0},
|
||||
])
|
||||
self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])
|
||||
self.assertIn("Done. 3 files installed", proc.stderr)
|
||||
|
||||
def test_every_event_has_its_schema_keys(self):
|
||||
_proc, events = self._run()
|
||||
for event in events:
|
||||
self.assertEqual(list(event), ["event", *install.EVENT_KEYS[event["event"]]])
|
||||
|
||||
def test_a_file_list_out_of_reach_is_a_network_error(self):
|
||||
# A port nothing listens on: the connection is refused at once.
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
closed = probe.getsockname()[1]
|
||||
env = {**self.env, "RETROBIOS_BASE_URL": f"http://127.0.0.1:{closed}"}
|
||||
proc = _install_run(env, "--platform", "retroarch", "--dest", str(self.bios),
|
||||
"--events", "--check")
|
||||
events = _events(proc)
|
||||
self.assertEqual([e["event"] for e in events], ["start", "detect", "error", "end"])
|
||||
self.assertEqual(events[2]["reason"], "network")
|
||||
self.assertEqual(events[2]["stage"], "manifest")
|
||||
self.assertEqual(events[3], {"event": "end", "code": 1})
|
||||
|
||||
def test_nothing_detected_is_an_empty_platform_list(self):
|
||||
proc = _install_run(self.env, "--events", "--check")
|
||||
events = _events(proc)
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
self.assertEqual(events[1]["platforms"], [])
|
||||
self.assertEqual(events[-1], {"event": "end", "code": 1})
|
||||
|
||||
def test_the_schema_is_documented(self):
|
||||
page = (REPO_ROOT / "wiki" / "installer.md").read_text(encoding="utf-8")
|
||||
for kind, keys in install.EVENT_KEYS.items():
|
||||
row = next((line for line in page.splitlines()
|
||||
if line.startswith(f"| `{kind}` |")), None)
|
||||
self.assertIsNotNone(row, kind)
|
||||
for key in keys:
|
||||
self.assertIn(f"`{key}`", row, f"{kind}.{key}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,469 @@
|
||||
"""The PortMaster port: its archive, its event reader and its screens.
|
||||
|
||||
The screens are driven here with PortMasterDialog's functions replaced by
|
||||
stubs that record each command and answer from a script, and with a fake
|
||||
install.py that writes canned --events streams. What a device adds (pugwash
|
||||
drawing, the pad, mount, the network) is not exercised.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
PORT = REPO_ROOT / "ports" / "RetroBIOS"
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
import build_port # noqa: E402
|
||||
|
||||
_spec = importlib.util.spec_from_file_location("install", REPO_ROOT / "install.py")
|
||||
install = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(install)
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"rbevents", PORT / "retrobios" / "rbevents.py")
|
||||
rbevents = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(rbevents)
|
||||
|
||||
SCRIPTS = [PORT / "RetroBIOS.sh", PORT / "retrobios" / "screens.txt"]
|
||||
|
||||
|
||||
def _tempdir(test: unittest.TestCase) -> Path:
|
||||
scratch = REPO_ROOT / "tmp"
|
||||
scratch.mkdir(exist_ok=True)
|
||||
path = Path(tempfile.mkdtemp(dir=str(scratch)))
|
||||
test.addCleanup(shutil.rmtree, path, True)
|
||||
return path
|
||||
|
||||
|
||||
class EveryScriptParses(unittest.TestCase):
|
||||
def test_bash_n(self):
|
||||
for script in SCRIPTS:
|
||||
with self.subTest(script=script.name):
|
||||
proc = subprocess.run(["bash", "-n", str(script)], capture_output=True,
|
||||
text=True, check=False)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
|
||||
def test_only_dialog_functions_the_oldest_releases_have(self):
|
||||
"""PortMaster 2024.02.26 and 8.5.23 define these in PortMasterDialog.txt,
|
||||
and have no funcs.txt: pm_message and the rest would be undefined."""
|
||||
known = {"PortMasterDialogInit", "PortMasterDialog", "PortMasterDialogResult",
|
||||
"PortMasterDialogExit", "PortMasterIPCheck"}
|
||||
commands = {"register_set_info", "register_clear", "selection_list",
|
||||
"messages_begin", "message", "messages_end", "progress",
|
||||
"progress_clear", "check_runtime"}
|
||||
text = "\n".join(path.read_text(encoding="utf-8") for path in SCRIPTS)
|
||||
import re # noqa: PLC0415
|
||||
used = set(re.findall(r"\b(PortMasterDialog\w*|PortMasterIPCheck)\b", text))
|
||||
self.assertLessEqual(used - {"PortMasterDialog.txt"}, known)
|
||||
self.assertNotRegex(text, r"\bpm_message\b|\bpm_platform_helper\b")
|
||||
sent = set(re.findall(r'(?:PortMasterDialog|rb_answer) "(\w+)"', text))
|
||||
self.assertLessEqual(sent, commands)
|
||||
self.assertNotIn("register_clear_info", text)
|
||||
|
||||
|
||||
class ThePortArchive(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.out = _tempdir(self)
|
||||
|
||||
def _build(self, name: str = "RetroBIOS.zip") -> Path:
|
||||
return build_port.build_port(self.out / name)
|
||||
|
||||
def test_layout_portmaster_installs(self):
|
||||
"""One launch script at the top, one folder holding port.json: what
|
||||
harbourmaster's captain.check_port accepts without a warning."""
|
||||
with zipfile.ZipFile(self._build()) as zf:
|
||||
names = zf.namelist()
|
||||
port = json.loads(zf.read("retrobios/port.json"))
|
||||
mode = zf.getinfo("RetroBIOS.sh").external_attr >> 16
|
||||
self.assertEqual(names, sorted(names))
|
||||
self.assertEqual([n for n in names if "/" not in n], ["RetroBIOS.sh"])
|
||||
self.assertEqual({n.split("/")[0] for n in names if "/" in n}, {"retrobios"})
|
||||
self.assertEqual([n for n in names if n.endswith(".sh")], ["RetroBIOS.sh"])
|
||||
for member in ("retrobios/install.py", "retrobios/rbevents.py",
|
||||
"retrobios/screens.txt", "retrobios/gameinfo.xml",
|
||||
"retrobios/retrobios.md", "retrobios/licenses/LICENSE-retrobios.txt"):
|
||||
self.assertIn(member, names)
|
||||
self.assertEqual(port["version"], 4)
|
||||
self.assertEqual(port["name"], "retrobios.zip")
|
||||
self.assertEqual(port["items"], ["RetroBIOS.sh", "retrobios"])
|
||||
self.assertEqual(port["attr"]["genres"], ["other"])
|
||||
self.assertTrue(mode & 0o111, "the launch script is executable")
|
||||
|
||||
def test_bundled_installer_is_the_pinned_one(self):
|
||||
with zipfile.ZipFile(self._build()) as zf:
|
||||
bundled = hashlib.sha256(zf.read("retrobios/install.py")).hexdigest()
|
||||
self.assertEqual(bundled, build_port.pinned_installer_sha256())
|
||||
install_sh = (REPO_ROOT / "install.sh").read_text(encoding="utf-8")
|
||||
self.assertIn(f'DEFAULT_INSTALL_SHA256="{bundled}"', install_sh)
|
||||
|
||||
def test_an_unpinned_installer_is_refused(self):
|
||||
root = _tempdir(self)
|
||||
for name in ("install.py", "install.sh", "LICENSE", "NOTICE"):
|
||||
shutil.copy(REPO_ROOT / name, root / name)
|
||||
shutil.copytree(PORT, root / "ports" / "RetroBIOS")
|
||||
with (root / "install.py").open("a") as fh:
|
||||
fh.write("\n")
|
||||
with self.assertRaises(ValueError):
|
||||
build_port.build_port(self.out / "x.zip", root)
|
||||
|
||||
def test_reproducible(self):
|
||||
first = self._build("a.zip").read_bytes()
|
||||
second = self._build("b.zip").read_bytes()
|
||||
self.assertEqual(first, second)
|
||||
|
||||
|
||||
class EventRecords(unittest.TestCase):
|
||||
"""rbevents turns --events into fields the FIFO carries intact."""
|
||||
|
||||
def test_fields_are_fifo_safe(self):
|
||||
self.assertEqual(rbevents.clean("a\tb\n[x]*? c"), "a b x c")
|
||||
out = io.StringIO()
|
||||
rbevents.record(out, "file", "", "a")
|
||||
self.assertEqual(out.getvalue(), "file\t-\ta\n")
|
||||
|
||||
def test_follow_writes_the_outcome(self):
|
||||
stream = [
|
||||
json.dumps({"event": "download", "platform": "p", "files": 1, "bytes": 2048}),
|
||||
"not json",
|
||||
json.dumps({"event": "downloaded", "dest": "x.bin", "ok": False,
|
||||
"reason": "HTTP 404", "size": 2048, "files_done": 1,
|
||||
"files_total": 1, "bytes_done": 0, "bytes_total": 2048}),
|
||||
json.dumps({"event": "summary", "mode": "install", "installed": 0,
|
||||
"up_to_date": 4, "pending": 1,
|
||||
"failed": [{"dest": "x.bin", "reason": "HTTP 404"}],
|
||||
"skipped": [], "interrupted": False}),
|
||||
json.dumps({"event": "end", "code": 2}),
|
||||
]
|
||||
out, summary = io.StringIO(), io.StringIO()
|
||||
rbevents.follow(stream, out, summary)
|
||||
self.assertEqual(out.getvalue().splitlines(), [
|
||||
"download\t1\t2 KB", "file\t1\t1\tfailed\tx.bin\t0 B\t2 KB"])
|
||||
self.assertEqual(summary.getvalue().splitlines(), [
|
||||
"summary\t0\t4\t1\t0", "failed\tx.bin\tHTTP 404", "end\t2"])
|
||||
|
||||
def test_a_system_id_that_is_no_slug_is_dropped(self):
|
||||
# The id becomes a bash array subscript in the launcher, where a
|
||||
# command substitution in it would run.
|
||||
def system(name):
|
||||
return json.dumps({"event": "system", "platform": "p", "system": name,
|
||||
"name": "N", "files": 1, "present": 0, "missing": 1,
|
||||
"wrong_hash": 0, "bytes_needed": 1})
|
||||
stream = [
|
||||
json.dumps({"event": "detect", "os": "linux", "os_name": "X",
|
||||
"platforms": [{"platform": "p", "name": "P", "bios_dir": "/b"}]}),
|
||||
system("$(touch pwned)"), system("a`id`"), system("sony-playstation"),
|
||||
]
|
||||
out = io.StringIO()
|
||||
rbevents.status(stream, out)
|
||||
ids = [line.split("\t")[1] for line in out.getvalue().splitlines()
|
||||
if line.startswith("system\t")]
|
||||
self.assertEqual(ids, ["sony-playstation"])
|
||||
|
||||
def test_launcher_never_double_expands_a_subscript(self):
|
||||
text = (PORT / "retrobios" / "screens.txt").read_text()
|
||||
self.assertNotRegex(text, r'unset\s+"[^"]*\[\$')
|
||||
|
||||
def test_a_run_without_end_says_so(self):
|
||||
out = io.StringIO()
|
||||
rbevents.status([], out)
|
||||
self.assertEqual(out.getvalue(), "end\tnone\n")
|
||||
|
||||
|
||||
FAKE_INSTALL = r'''
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
|
||||
state = Path(os.environ["FAKE_STATE"])
|
||||
scenario = os.environ["FAKE_SCENARIO"]
|
||||
keys = json.loads(os.environ["FAKE_KEYS"])
|
||||
with (state / "calls.txt").open("a") as fh:
|
||||
fh.write(" ".join(sys.argv[1:]) + "\n")
|
||||
check = "--check" in sys.argv
|
||||
|
||||
|
||||
def emit(kind, **fields):
|
||||
assert sorted(fields) == sorted(keys[kind]), kind
|
||||
print(json.dumps({"event": kind, **fields}), flush=True)
|
||||
|
||||
|
||||
def end(code):
|
||||
emit("end", code=code)
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
emit("start", schema=1, installer="fake", mode="check" if check else "install")
|
||||
if scenario == "nodetect":
|
||||
emit("detect", os="linux", os_name="Fake OS 1.0", platforms=[])
|
||||
end(1)
|
||||
if scenario == "conflict":
|
||||
emit("error", stage="detect", reason="conflict", platform="",
|
||||
message="JELOS and Koriki are both detected here, and a machine runs one")
|
||||
end(1)
|
||||
emit("detect", os="linux", os_name="Fake OS 1.0", platforms=[
|
||||
{"platform": "retroarch", "name": "RetroArch", "bios_dir": "/roms/bios"}])
|
||||
if scenario == "offline":
|
||||
emit("error", stage="manifest", reason="network", platform="retroarch",
|
||||
message="the file list could not be fetched: [Errno 101] Network is unreachable")
|
||||
end(1)
|
||||
done = scenario == "complete" or (state / "installed").exists()
|
||||
if check:
|
||||
for system, name, files, present, missing, wrong, needed in (
|
||||
("sys-a", "Alpha*", 2, 2 if done else 0, 0 if done else 1, 0 if done else 1,
|
||||
0 if done else 300),
|
||||
("sys-b", "Bravo", 1, 1, 0, 0, 0),
|
||||
):
|
||||
emit("system", platform="retroarch", system=system, name=name, files=files,
|
||||
present=present, missing=missing, wrong_hash=wrong, bytes_needed=needed)
|
||||
emit("check", platform="retroarch", name="RetroArch", bios_dir="/roms/bios", files=3,
|
||||
present=3 if done else 1, missing=0 if done else 1, wrong_hash=0 if done else 1,
|
||||
bytes_needed=0 if done else 300, data_missing=0, not_installable=1)
|
||||
emit("summary", mode="check", installed=0, up_to_date=3 if done else 1,
|
||||
pending=0 if done else 2, failed=[], skipped=[], interrupted=False)
|
||||
end(0 if done else 1)
|
||||
emit("download", platform="retroarch", files=2, bytes=300)
|
||||
for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1):
|
||||
emit("downloaded", dest=dest, ok=True, reason="", size=size, files_done=n,
|
||||
files_total=2, bytes_done=100 if n == 1 else 300, bytes_total=300)
|
||||
(state / "installed").write_text("")
|
||||
emit("summary", mode="install", installed=2, up_to_date=1, pending=0, failed=[],
|
||||
skipped=[{"reason": "not_found", "label": "not found", "files": 1, "required": 1}],
|
||||
interrupted=False)
|
||||
end(0)
|
||||
'''
|
||||
|
||||
# PortMasterDialog.txt replaced: every call is a trace line, fields split by
|
||||
# \x1f, and each selection list is answered from the answers file.
|
||||
DRIVER = r'''
|
||||
trace() {
|
||||
printf '%s' "$1"
|
||||
shift
|
||||
local field
|
||||
for field in "$@"; do printf '\x1f%s' "$field"; done
|
||||
printf '\n'
|
||||
} >> "$RB_TRACE"
|
||||
PM_PID=""
|
||||
PM_DONE="$FAKE_STATE/pm_done"
|
||||
PortMasterDialogInit() { trace INIT "$@"; }
|
||||
PortMasterDialogExit() { trace EXIT; }
|
||||
PortMasterIPCheck() { [ -n "$FAKE_IP" ] && echo 192.168.0.1; }
|
||||
PortMasterDialog() { trace CMD "$@"; }
|
||||
PortMasterDialogResult() {
|
||||
trace ASK "$@"
|
||||
if [ "$1" = "check_runtime" ]; then
|
||||
echo "${FAKE_RUNTIME:-FAIL}"
|
||||
return
|
||||
fi
|
||||
local answer
|
||||
answer=$(head -n 1 "$RB_ANSWERS")
|
||||
sed -i 1d "$RB_ANSWERS"
|
||||
case "$answer" in
|
||||
"") echo CANCEL ;;
|
||||
WAIT:*) printf '%s' "${answer#WAIT:}" > "$PM_DONE"; echo WAIT ;;
|
||||
*) echo "$answer" ;;
|
||||
esac
|
||||
}
|
||||
source "$GAMEDIR/screens.txt"
|
||||
if [ -n "$FAKE_ARCH" ]; then
|
||||
rb_arch() { echo "$FAKE_ARCH"; }
|
||||
fi
|
||||
rb_main
|
||||
trace STATUS "$?"
|
||||
'''
|
||||
|
||||
|
||||
class TheScreens(unittest.TestCase):
|
||||
"""Which screens the port shows, scenario by scenario."""
|
||||
|
||||
def run_port(self, scenario: str, answers: list[str], **env: str) -> list[tuple]:
|
||||
work = _tempdir(self)
|
||||
(work / "install.py").write_text(FAKE_INSTALL)
|
||||
(work / "driver.sh").write_text(DRIVER)
|
||||
(work / "answers").write_text("".join(f"{a}\n" for a in answers))
|
||||
bin_dir = work / "bin"
|
||||
bin_dir.mkdir()
|
||||
# The oldest Python install.py supports, where it is installed: the
|
||||
# event reader runs on whatever the device has.
|
||||
(bin_dir / "python3").symlink_to(shutil.which("python3.8") or sys.executable)
|
||||
self.work = work
|
||||
full_env = {
|
||||
**os.environ,
|
||||
"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
|
||||
"HOME": str(work),
|
||||
"GAMEDIR": str(PORT / "retrobios"),
|
||||
"controlfolder": str(work / "PortMaster"),
|
||||
"ESUDO": "",
|
||||
"RB_STATE": str(work / "run"),
|
||||
"RB_LOG": str(work / "log.txt"),
|
||||
"RB_INSTALL": str(work / "install.py"),
|
||||
"RB_TRACE": str(work / "trace"),
|
||||
"RB_ANSWERS": str(work / "answers"),
|
||||
"FAKE_STATE": str(work),
|
||||
"FAKE_SCENARIO": scenario,
|
||||
"FAKE_KEYS": json.dumps(install.EVENT_KEYS),
|
||||
**env,
|
||||
}
|
||||
proc = subprocess.run(["bash", str(work / "driver.sh")], env=full_env,
|
||||
capture_output=True, text=True, timeout=120, check=False)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
log = work / "log.txt"
|
||||
self.assertNotIn("Traceback", log.read_text() if log.exists() else "")
|
||||
return self._screens((work / "trace").read_text(encoding="utf-8"))
|
||||
|
||||
def _screens(self, trace: str) -> list[tuple]:
|
||||
"""INIT, EXIT, STATUS, each list shown with its entries, and the
|
||||
progress and message commands, in order."""
|
||||
registers: dict[str, dict[str, tuple[str, str]]] = {}
|
||||
shown: list[tuple] = []
|
||||
for line in trace.splitlines():
|
||||
kind, *fields = line.split("\x1f")
|
||||
for field in fields:
|
||||
self.assertNotRegex(field, r"[*?\[\]\t]", line)
|
||||
if kind == "CMD" and fields[0] == "register_clear":
|
||||
registers[fields[1]] = {}
|
||||
elif kind == "CMD" and fields[0] == "register_set_info":
|
||||
info = dict(item.split(":", 1) for item in fields[3:])
|
||||
registers.setdefault(fields[1], {})[fields[2]] = (
|
||||
info["title"], info["description"])
|
||||
elif kind == "ASK" and fields[0] == "selection_list":
|
||||
self.assertIn("--want-cancel", fields)
|
||||
name = fields[1].split("=", 1)[1]
|
||||
shown.append(("list", name, dict(registers[name])))
|
||||
elif kind == "ASK":
|
||||
shown.append(("ask", *fields))
|
||||
elif kind == "CMD" and fields[0] in ("progress", "message"):
|
||||
shown.append(("cmd", *fields))
|
||||
elif kind in ("INIT", "EXIT", "STATUS"):
|
||||
shown.append((kind, *fields))
|
||||
for screen in shown:
|
||||
if screen[0] == "list":
|
||||
titles = [title for title, _desc in screen[2].values()]
|
||||
self.assertTrue({"Back", "Quit", "Back to the menu"} & set(titles),
|
||||
f"{screen[1]} has no Back or Quit entry")
|
||||
return shown
|
||||
|
||||
def lists(self, shown: list[tuple]) -> list[tuple[str, list[str]]]:
|
||||
return [(s[1], list(s[2])) for s in shown if s[0] == "list"]
|
||||
|
||||
def calls(self) -> list[str]:
|
||||
return (self.work / "calls.txt").read_text().splitlines()
|
||||
|
||||
def test_all_present(self):
|
||||
shown = self.run_port("complete", ["quit"])
|
||||
self.assertEqual(self.lists(shown), [("menu", ["status", "check", "quit"])])
|
||||
menu = shown[[s[0] for s in shown].index("list")][2]
|
||||
self.assertEqual(menu["status"][0], "Fake OS 1.0: all 3 BIOS files present")
|
||||
self.assertEqual(shown[0], ("INIT", "no-harbour"))
|
||||
self.assertEqual(shown[-2:], [("EXIT",), ("STATUS", "0")])
|
||||
self.assertEqual(self.calls(), ["--check --events"])
|
||||
|
||||
def test_some_missing_update_everything(self):
|
||||
shown = self.run_port("missing", ["update", "back", "quit"])
|
||||
self.assertEqual(self.lists(shown), [
|
||||
("menu", ["status", "update", "choose", "check", "quit"]),
|
||||
("summary", ["info:result", "back", "quit", "info:skipped:not found"]),
|
||||
("menu", ["status", "check", "quit"]),
|
||||
])
|
||||
menu = shown[[s[0] for s in shown].index("list")][2]
|
||||
self.assertEqual(menu["status"][0], "Fake OS 1.0: 1/3 BIOS files, 1 missing, 1 wrong hash")
|
||||
self.assertIn("Downloads 2 files, 300 B, to /roms/bios", menu["update"][1])
|
||||
progress = [s for s in shown if s[:2] == ("cmd", "progress")]
|
||||
self.assertEqual(progress, [
|
||||
("cmd", "progress", "Starting", "0", "1"),
|
||||
("cmd", "progress", "a.bin, 100 B of 300 B", "1", "2"),
|
||||
("cmd", "progress", "b.bin, 300 B of 300 B", "2", "2"),
|
||||
])
|
||||
summary = [s for s in shown if s[:2] == ("list", "summary")][0][2]
|
||||
self.assertEqual(summary["info:result"][0], "Installed: 2 files")
|
||||
self.assertEqual(self.calls(), [
|
||||
"--check --events",
|
||||
"--events --platform retroarch",
|
||||
"--check --events --platform retroarch",
|
||||
])
|
||||
|
||||
def test_some_missing_choose_systems(self):
|
||||
shown = self.run_port("missing", [
|
||||
"choose", "update", "sys:sys-a", "sys:sys-a", "sys:sys-a", "update", "quit"])
|
||||
systems = [s[2] for s in shown if s[:2] == ("list", "systems")]
|
||||
self.assertEqual(list(systems[0]), ["update", "back", "sys:sys-a"])
|
||||
self.assertEqual(systems[0]["sys:sys-a"][0], "Alpha")
|
||||
self.assertEqual(systems[1]["sys:sys-a"][0], "Alpha")
|
||||
self.assertEqual(systems[2]["sys:sys-a"][0], "Alpha (selected)")
|
||||
self.assertEqual(systems[2]["update"][0], "Update the 1 selected")
|
||||
self.assertIn("1 system complete, not listed", systems[0]["back"][1])
|
||||
self.assertEqual(self.calls()[1], "--events --platform retroarch --system sys-a")
|
||||
self.assertEqual(self.lists(shown)[-1][0], "summary")
|
||||
self.assertEqual(shown[-1], ("STATUS", "0"))
|
||||
|
||||
def test_no_network(self):
|
||||
shown = self.run_port("offline", ["quit"])
|
||||
self.assertEqual(self.lists(shown), [("notice", ["info", "check", "quit"])])
|
||||
notice = shown[[s[0] for s in shown].index("list")][2]
|
||||
self.assertEqual(notice["info"][0], "No connection")
|
||||
self.assertIn("Connect to Wi-Fi", notice["info"][1])
|
||||
self.assertIn("/roms/bios", notice["info"][1])
|
||||
|
||||
def test_detection_failure(self):
|
||||
shown = self.run_port("nodetect", ["check", "quit"])
|
||||
self.assertEqual([name for name, _keys in self.lists(shown)], ["notice", "notice"])
|
||||
notice = shown[[s[0] for s in shown].index("list")][2]
|
||||
self.assertEqual(notice["info"][0], "No BIOS folder found")
|
||||
self.assertIn("Fake OS 1.0", notice["info"][1])
|
||||
self.assertIn("--dest", notice["info"][1])
|
||||
self.assertEqual(len(self.calls()), 2)
|
||||
|
||||
def test_two_systems_found(self):
|
||||
shown = self.run_port("conflict", ["check", "quit"])
|
||||
notice = shown[[s[0] for s in shown].index("list")][2]
|
||||
self.assertEqual(notice["info"][0], "Two systems found")
|
||||
self.assertIn("JELOS and Koriki", notice["info"][1])
|
||||
self.assertIn("--dest", notice["info"][1])
|
||||
|
||||
def test_python_too_old(self):
|
||||
for arch, env, words in (
|
||||
("armv7l", {}, ["3.7.3", "aarch64 only", "armv7l", "--dest"]),
|
||||
("aarch64", {}, ["3.7.3", "needs a connection", "--dest"]),
|
||||
("aarch64", {"FAKE_IP": "1", "FAKE_RUNTIME": "FAIL"},
|
||||
["3.7.3", "could not be installed", "--dest"]),
|
||||
):
|
||||
with self.subTest(arch=arch, env=env):
|
||||
shown = self.run_old(arch, **env)
|
||||
lists = self.lists(shown)
|
||||
self.assertEqual(lists, [("notice", ["info", "quit"])])
|
||||
notice = [s for s in shown if s[0] == "list"][0][2]
|
||||
self.assertEqual(notice["info"][0], "Python 3.8 or newer is needed")
|
||||
for word in words:
|
||||
self.assertIn(word, notice["info"][1])
|
||||
self.assertEqual(shown[-1], ("STATUS", "1"))
|
||||
self.assertFalse((self.work / "calls.txt").exists())
|
||||
asked_runtime = ("ask", "check_runtime", "python_3.11.squashfs") in shown
|
||||
self.assertEqual(asked_runtime, bool(env.get("FAKE_RUNTIME")))
|
||||
|
||||
def run_old(self, arch: str, **env: str) -> list[tuple]:
|
||||
"""A python3 on PATH that is 3.7.3, ahead of every other."""
|
||||
holder = _tempdir(self)
|
||||
fake = holder / "python3"
|
||||
fake.write_text('#!/bin/sh\n[ "$1" = "--version" ] && echo "Python 3.7.3"\nexit 1\n')
|
||||
fake.chmod(0o755)
|
||||
return self.run_port("complete", [], FAKE_ARCH=arch,
|
||||
PATH=f"{holder}{os.pathsep}{os.environ['PATH']}", **env)
|
||||
|
||||
def test_an_answer_after_the_old_30_second_wait(self):
|
||||
"""2024.02.26 and 8.5.23 stop waiting after 30 s and answer WAIT:
|
||||
the port reads the answer pugwash writes later."""
|
||||
shown = self.run_port("complete", ["WAIT:quit"])
|
||||
self.assertEqual(shown[-1], ("STATUS", "0"))
|
||||
self.assertEqual(len(self.lists(shown)), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -251,6 +251,7 @@ what the options name, everything by default.
|
||||
| `--dry-run` | Show what would be downloaded, by system and with sizes, without writing anything |
|
||||
| `--yes`, `-y` | Do not ask before a download of 1 GB or more |
|
||||
| `--quiet`, `-q` | Print only failures and the final summary. Implies `--no-input` |
|
||||
| `--events` | Write the run as JSON events on standard output, one object per line, and the usual text on standard error. Implies `--no-input`. See [Machine-readable events](#machine-readable-events) |
|
||||
| `--version` | Print the installer's identity, the first 12 hex digits of the SHA-256 the bootstraps pin |
|
||||
| `--list-platforms` | Print the supported platforms and what was detected here |
|
||||
| `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries |
|
||||
@@ -267,6 +268,32 @@ it, a file already present with the expected hash is left untouched, and one
|
||||
whose contents do not match is re-downloaded and replaced by the verified copy.
|
||||
Files the list does not name are never read, moved or deleted.
|
||||
|
||||
## Machine-readable events
|
||||
|
||||
With `--events`, standard output carries only JSON objects, one per line, and
|
||||
the text a person reads goes to standard error. A program driving the installer
|
||||
(the [handheld port](#on-a-handheld-the-retrobios-port) is one) reads the
|
||||
objects and never parses the text. Every object has an `event` key naming its
|
||||
kind, and each kind has exactly the keys below, in this order. The schema is
|
||||
`EVENT_KEYS` in `install.py`, and a test holds this table to it.
|
||||
|
||||
| Kind | Keys | When |
|
||||
|------|------|------|
|
||||
| `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` |
|
||||
| `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found |
|
||||
| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`). `reason` is `network`, `not_found` or `invalid` |
|
||||
| `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` |
|
||||
| `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id. Files with no system count under `other`. `name` is the platform's own name for it |
|
||||
| `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply |
|
||||
| `download` | `platform`, `files`, `bytes` | Before the downloads start |
|
||||
| `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive |
|
||||
| `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file. `reason` says why one failed, and is empty otherwise |
|
||||
| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply. Not written for a listing |
|
||||
| `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) |
|
||||
|
||||
An option refused before the run starts (`install.py: error:`, code 2) writes
|
||||
no event at all. A program reads a missing `end` as that case.
|
||||
|
||||
## Environment overrides
|
||||
|
||||
| Variable | Effect |
|
||||
@@ -560,6 +587,67 @@ elsewhere. A custom BIOS directory is still named directly:
|
||||
python install.py --platform retroarch --dest /storage/emulated/0/RetroArch/system
|
||||
```
|
||||
|
||||
## On a handheld: the RetroBIOS port
|
||||
|
||||
On a Linux handheld that runs [PortMaster](https://portmaster.games/) (ArkOS,
|
||||
ROCKNIX, AmberELEC, muOS, KNULLI and the others it supports), the installer
|
||||
also runs from the device itself, as a port with gamepad screens. The screens
|
||||
are PortMaster's own, so the port carries no graphics code.
|
||||
|
||||
**Getting it.** Two ways:
|
||||
|
||||
- From PortMaster's catalog, once the port is accepted there: open PortMaster,
|
||||
find RetroBIOS under the Other genre, install.
|
||||
- `RetroBIOS.zip`, attached to each [release](https://github.com/Abdess/retrobios/releases/latest)
|
||||
cut since the port was added, copied into PortMaster's
|
||||
autoinstall folder. Starting PortMaster then installs it, with or without a
|
||||
network connection. PortMaster's FAQ names the folder: `/roms/ports/PortMaster/autoinstall/`
|
||||
on AmberELEC and ROCKNIX, `/roms/tools/PortMaster/autoinstall/` on ArkOS,
|
||||
`/mmc/MUOS/PortMaster/autoinstall/` on muOS,
|
||||
`/userdata/system/.local/share/PortMaster/autoinstall/` on KNULLI.
|
||||
|
||||
From a clone of the repository, `python scripts/build_port.py` builds the
|
||||
same archive into `dist/RetroBIOS.zip`.
|
||||
|
||||
**What it shows.** RetroBIOS appears among the ports. Started, it runs
|
||||
`install.py --check --events` and opens on the result: the system's name,
|
||||
how many BIOS files are present, missing, or present with a hash that does
|
||||
not match. The menu
|
||||
offers to update everything missing, to choose systems (each listed with what
|
||||
it lacks, selected or cleared with A), to check again, or to quit. An update
|
||||
shows a progress bar, one step per file, then what was installed, what failed
|
||||
and why, and what the collection cannot supply. Every list has its own Back or
|
||||
Quit entry, since older PortMaster releases do not read B there.
|
||||
|
||||
**What it needs.**
|
||||
|
||||
- A network connection: the list of files to check comes from the internet,
|
||||
as do the files. Without one the port says so and offers to check again.
|
||||
- Python 3.8 or newer. The port uses the system's `python3` when it is recent
|
||||
enough, else PortMaster's `python_3.11` runtime on aarch64 devices, which it
|
||||
downloads through PortMaster on first use. An armhf device with an older
|
||||
Python is told so, with the computer route below.
|
||||
- A BIOS folder the installer detects (see [Platform detection](#platform-detection)).
|
||||
On a system it does not know yet, the port says it found no BIOS folder.
|
||||
|
||||
**The computer route.** With the SD card mounted on a computer, the one-line
|
||||
command fills it, naming the platform and the card's BIOS folder:
|
||||
`--platform <platform> --dest <BIOS folder on the card>`. The port's screens
|
||||
for an unknown system or an old Python name this route.
|
||||
|
||||
**Which installer runs.** The archive carries `install.py`, the file the
|
||||
one-line command of the same commit downloads: the build refuses any other
|
||||
(`scripts/build_port.py` compares it with the SHA-256 `install.sh` pins). A
|
||||
port that downloaded the installer at each start against a pinned SHA-256
|
||||
would stop working at the next change to `install.py`, since the pin is fixed
|
||||
when the port is built. The file lists come from `main`, as for the one-line
|
||||
command. A newer port version carries a newer installer. When a list is in a
|
||||
format the carried installer does not read, the port says to update it.
|
||||
|
||||
The launcher (`ports/RetroBIOS/RetroBIOS.sh`) and its screens
|
||||
(`retrobios/screens.txt`) read the installer through
|
||||
[Machine-readable events](#machine-readable-events) only.
|
||||
|
||||
## Where the files come from
|
||||
|
||||
Each manifest entry names either `repo_path`, served from the repository at the
|
||||
|
||||
@@ -202,6 +202,11 @@ python scripts/pipeline.py --include-archived
|
||||
# and the set compared with the pack before the pack is removed.
|
||||
python scripts/split_pack.py dist/
|
||||
|
||||
# 2b. The handheld port, RetroBIOS.zip, built from ports/RetroBIOS with the
|
||||
# installer the bootstraps pin. It is a release file, listed in the
|
||||
# checksums below, and check_release.py holds it to them.
|
||||
python scripts/build_port.py
|
||||
|
||||
# 3. Checksums of the files as published, then sign the list. The checksums
|
||||
# answer corruption, and the signature answers a rewritten release, which is
|
||||
# the one thing a checksum published beside its own artifacts cannot
|
||||
|
||||
@@ -44,6 +44,7 @@ Every module in `tests/`, by the area it guards.
|
||||
| Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
|
||||
| Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
|
||||
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` |
|
||||
| Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
|
||||
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
|
||||
| Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
|
||||
| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
|
||||
@@ -251,6 +252,23 @@ python scripts/generate_pack.py --all --verify-packs --output-dir dist/
|
||||
Integrated as pipeline step 6/8 (runs after consistency check, before
|
||||
README generation). Requires packs in `dist/`, skip with `--skip-packs`.
|
||||
|
||||
### test_port.py
|
||||
|
||||
The PortMaster port in `ports/RetroBIOS/`. `bash -n` on its scripts, the
|
||||
archive `scripts/build_port.py` writes (layout, reproducibility, the bundled
|
||||
`install.py` equal to the one `install.sh` pins), and the event reader. The
|
||||
screens run in bash with PortMasterDialog's functions replaced by stubs that
|
||||
record each command and answer from a script, against a fake `install.py`
|
||||
whose event streams follow `EVENT_KEYS`: all files present, some missing
|
||||
(update everything, choose systems), no network, no BIOS folder detected,
|
||||
Python too old, and an answer that arrives after the 30-second wait of the
|
||||
older PortMaster releases. Drawing, the pad, `mount` and the network belong
|
||||
to a device and are not exercised.
|
||||
|
||||
```bash
|
||||
python -m unittest tests.test_port -v
|
||||
```
|
||||
|
||||
## Verification discipline
|
||||
|
||||
The test suite is one layer of verification. The full quality gate is:
|
||||
|
||||
Reference in new issue
Block a user