feat: add install.py universal installer engine

This commit is contained in:
Abdessamad Derraz
2026-03-28 18:07:28 +01:00
parent 42f2cc5617
commit 46426aa1e4
2 changed files with 708 additions and 0 deletions

626
install.py Normal file
View File

@@ -0,0 +1,626 @@
#!/usr/bin/env python3
"""Universal BIOS installer for retrogaming platforms.
Self-contained script using only Python stdlib. Downloads missing BIOS files
from the retrobios repository and places them in the correct location for
the detected emulator platform.
Usage:
python install.py
python install.py --platform retroarch --dest ~/custom/bios
python install.py --check
python install.py --list-platforms
"""
from __future__ import annotations
import argparse
import concurrent.futures
import hashlib
import json
import os
import platform
import shutil
import sys
import tempfile
import urllib.error
import urllib.request
from pathlib import Path
BASE_URL = "https://raw.githubusercontent.com/Abdess/retrobios/main"
MANIFEST_URL = f"{BASE_URL}/install/{{platform}}.json"
TARGETS_URL = f"{BASE_URL}/install/targets/{{platform}}.json"
RAW_FILE_URL = f"{BASE_URL}/{{path}}"
RELEASE_URL = (
"https://github.com/Abdess/retrobios/releases/download/large-files/{asset}"
)
MAX_RETRIES = 3
def detect_os() -> str:
"""Return normalized OS identifier."""
system = platform.system().lower()
if system == "linux":
proc_version = Path("/proc/version")
if proc_version.exists():
try:
content = proc_version.read_text(encoding="utf-8", errors="replace")
if "microsoft" in content.lower():
return "wsl"
except OSError:
pass
return "linux"
if system == "darwin":
return "darwin"
if system == "windows":
return "windows"
return system
def _parse_os_release() -> dict[str, str]:
"""Parse /etc/os-release KEY=value format."""
result: dict[str, str] = {}
path = Path("/etc/os-release")
if not path.exists():
return result
try:
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
line = line.strip()
if "=" not in line or line.startswith("#"):
continue
key, _, value = line.partition("=")
value = value.strip('"').strip("'")
result[key] = value
except OSError:
pass
return result
def _parse_retroarch_system_dir(cfg_path: Path) -> Path | None:
"""Parse system_directory from retroarch.cfg."""
if not cfg_path.exists():
return None
try:
for line in cfg_path.read_text(encoding="utf-8", errors="replace").splitlines():
line = line.strip()
if line.startswith("system_directory"):
_, _, value = line.partition("=")
value = value.strip().strip('"').strip("'")
if not value or value == "default":
return cfg_path.parent / "system"
value = os.path.expandvars(os.path.expanduser(value))
return Path(value)
except OSError:
pass
return None
def _parse_bash_var(path: Path, key: str) -> str | None:
"""Extract value of key= from a bash/shell file."""
if not path.exists():
return None
try:
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
line = line.strip()
if line.startswith(f"{key}="):
_, _, value = line.partition("=")
return value.strip('"').strip("'")
except OSError:
pass
return None
def _parse_ps1_var(path: Path, key: str) -> str | None:
"""Extract value of $key= or key= from a PowerShell file."""
if not path.exists():
return None
normalized = key.lstrip("$")
try:
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
line = line.strip()
check = line.lstrip("$")
if check.startswith(f"{normalized}="):
_, _, value = check.partition("=")
return value.strip('"').strip("'")
except OSError:
pass
return None
def _detect_embedded() -> list[tuple[str, Path]]:
"""Check for embedded Linux retrogaming OSes."""
found: list[tuple[str, Path]] = []
os_release = _parse_os_release()
os_id = os_release.get("ID", "").lower()
if os_id == "rocknix":
found.append(("retroarch", Path("/storage/roms/bios")))
return found
if Path("/etc/knulli-release").exists():
found.append(("batocera", Path("/userdata/bios")))
return found
if os_id == "lakka":
found.append(("lakka", Path("/storage/system")))
return found
if Path("/etc/batocera-version").exists():
found.append(("batocera", Path("/userdata/bios")))
return found
if Path("/usr/bin/recalbox-settings").exists():
found.append(("recalbox", Path("/recalbox/share/bios")))
return found
if Path("/opt/muos").exists() or Path("/mnt/mmc/MUOS/").exists():
found.append(("retroarch", Path("/mnt/mmc/MUOS/bios")))
return found
if Path("/home/ark").exists() and Path("/opt/system").exists():
found.append(("retroarch", Path("/roms/bios")))
return found
if Path("/mnt/vendor/bin/dmenu.bin").exists():
found.append(("retroarch", Path("/mnt/mmc/bios")))
return found
return found
def detect_platforms(os_type: str) -> list[tuple[str, Path]]:
"""Detect installed emulator platforms and their BIOS directories."""
found: list[tuple[str, Path]] = []
if os_type in ("linux", "wsl"):
found.extend(_detect_embedded())
# EmuDeck (Linux/SteamOS)
home = Path.home()
emudeck_settings = home / "emudeck" / "settings.sh"
if emudeck_settings.exists():
emu_path = _parse_bash_var(emudeck_settings, "emulationPath")
if emu_path:
bios_dir = Path(emu_path) / "bios"
found.append(("emudeck", bios_dir))
# RetroDECK
retrodeck_cfg = home / ".var" / "app" / "net.retrodeck.retrodeck" / "config" / "retrodeck" / "retrodeck.cfg"
if retrodeck_cfg.exists():
bios_path = _parse_bash_var(retrodeck_cfg, "rdhome")
if bios_path:
found.append(("retrodeck", Path(bios_path) / "bios"))
else:
found.append(("retrodeck", home / "retrodeck" / "bios"))
# RetroArch Flatpak
flatpak_cfg = home / ".var" / "app" / "org.libretro.RetroArch" / "config" / "retroarch" / "retroarch.cfg"
ra_dir = _parse_retroarch_system_dir(flatpak_cfg)
if ra_dir:
found.append(("retroarch", ra_dir))
# RetroArch Snap
snap_cfg = home / "snap" / "retroarch" / "current" / ".config" / "retroarch" / "retroarch.cfg"
ra_dir = _parse_retroarch_system_dir(snap_cfg)
if ra_dir:
found.append(("retroarch", ra_dir))
# RetroArch native
native_cfg = home / ".config" / "retroarch" / "retroarch.cfg"
ra_dir = _parse_retroarch_system_dir(native_cfg)
if ra_dir:
found.append(("retroarch", ra_dir))
if os_type == "darwin":
home = Path.home()
mac_cfg = home / "Library" / "Application Support" / "RetroArch" / "retroarch.cfg"
ra_dir = _parse_retroarch_system_dir(mac_cfg)
if ra_dir:
found.append(("retroarch", ra_dir))
if os_type in ("windows", "wsl"):
# EmuDeck Windows
home = Path.home()
emudeck_ps1 = home / "emudeck" / "settings.ps1"
if emudeck_ps1.exists():
emu_path = _parse_ps1_var(emudeck_ps1, "$emulationPath")
if emu_path:
found.append(("emudeck", Path(emu_path) / "bios"))
# RetroArch Windows
appdata = os.environ.get("APPDATA", "")
if appdata:
win_cfg = Path(appdata) / "RetroArch" / "retroarch.cfg"
ra_dir = _parse_retroarch_system_dir(win_cfg)
if ra_dir:
found.append(("retroarch", ra_dir))
return found
def fetch_manifest(plat: str) -> dict:
"""Download platform manifest JSON."""
url = MANIFEST_URL.format(platform=plat)
try:
with urllib.request.urlopen(url, timeout=30) as resp:
return json.loads(resp.read().decode("utf-8"))
except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
print(f" Failed to fetch manifest for {plat}: {exc}", file=sys.stderr)
sys.exit(1)
def fetch_targets(plat: str) -> dict:
"""Download target core list. Returns empty dict on 404."""
url = TARGETS_URL.format(platform=plat)
try:
with urllib.request.urlopen(url, timeout=30) as resp:
return json.loads(resp.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
if exc.code == 404:
return {}
print(f" Warning: failed to fetch targets for {plat}: {exc}", file=sys.stderr)
return {}
except (urllib.error.URLError, OSError):
return {}
def _filter_by_target(
files: list[dict], target_cores: list[str]
) -> list[dict]:
"""Keep files where cores is None or overlaps with target_cores."""
result: list[dict] = []
target_set = set(target_cores)
for f in files:
cores = f.get("cores")
if cores is None or any(c in target_set for c in cores):
result.append(f)
return result
def _sha1_file(path: Path) -> str:
"""Compute SHA1 of a file."""
h = hashlib.sha1()
with open(path, "rb") as fh:
while True:
chunk = fh.read(65536)
if not chunk:
break
h.update(chunk)
return h.hexdigest()
def check_local(
files: list[dict], bios_path: Path
) -> tuple[list[dict], list[dict], list[dict]]:
"""Check which files exist locally and have correct hashes.
Returns (to_download, up_to_date, mismatched).
"""
to_download: list[dict] = []
up_to_date: list[dict] = []
mismatched: list[dict] = []
for f in files:
dest = bios_path / f["dest"]
if not dest.exists():
to_download.append(f)
continue
expected_sha1 = f.get("sha1", "")
if not expected_sha1:
up_to_date.append(f)
continue
actual = _sha1_file(dest)
if actual == expected_sha1:
up_to_date.append(f)
else:
mismatched.append(f)
return to_download, up_to_date, mismatched
def _download_one(
f: dict, bios_path: Path, verbose: bool = False
) -> tuple[str, bool]:
"""Download a single file. Returns (dest, success)."""
dest = bios_path / f["dest"]
dest.parent.mkdir(parents=True, exist_ok=True)
if f.get("release_asset"):
url = RELEASE_URL.format(asset=f["release_asset"])
else:
url = RAW_FILE_URL.format(path=f["repo_path"])
tmp_path = dest.with_suffix(dest.suffix + ".tmp")
for attempt in range(1, MAX_RETRIES + 1):
try:
with urllib.request.urlopen(url, timeout=60) as resp:
with open(tmp_path, "wb") as out:
shutil.copyfileobj(resp, out)
expected_sha1 = f.get("sha1", "")
if expected_sha1:
actual = _sha1_file(tmp_path)
if actual != expected_sha1:
if verbose:
print(f" SHA1 mismatch on attempt {attempt}", file=sys.stderr)
tmp_path.unlink(missing_ok=True)
continue
tmp_path.rename(dest)
return f["dest"], True
except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
if verbose:
print(f" Attempt {attempt} failed: {exc}", file=sys.stderr)
tmp_path.unlink(missing_ok=True)
return f["dest"], False
def download_files(
files: list[dict], bios_path: Path, jobs: int = 8, verbose: bool = False
) -> list[str]:
"""Download files in parallel. Returns list of failed file names."""
failed: list[str] = []
total = len(files)
with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
future_map = {
pool.submit(_download_one, f, bios_path, verbose): f
for f in files
}
done_count = 0
for future in concurrent.futures.as_completed(future_map):
done_count += 1
dest, success = future.result()
status = "ok" if success else "FAILED"
print(f" [{done_count}/{total}] {dest} {status}")
if not success:
failed.append(dest)
return failed
def do_standalone_copies(
manifest: dict, bios_path: Path, os_type: str
) -> tuple[int, int]:
"""Copy BIOS files to standalone emulator directories.
Returns (copied_count, skipped_count).
"""
copies = manifest.get("standalone_copies", [])
if not copies:
return 0, 0
copied = 0
skipped = 0
for entry in copies:
src = bios_path / entry["file"]
if not src.exists():
continue
targets = entry.get("targets", {}).get(os_type, [])
for target_dir_str in targets:
target_dir = Path(os.path.expandvars(os.path.expanduser(target_dir_str)))
if target_dir.is_dir():
dest = target_dir / src.name
try:
shutil.copy2(src, dest)
copied += 1
except OSError:
skipped += 1
else:
skipped += 1
return copied, skipped
def format_size(n: int) -> str:
"""Human-readable file size."""
if n < 1024:
return f"{n} B"
if n < 1024 * 1024:
return f"{n / 1024:.1f} KB"
if n < 1024 * 1024 * 1024:
return f"{n / (1024 * 1024):.1f} MB"
return f"{n / (1024 * 1024 * 1024):.1f} GB"
def _prompt_platform_choice(
platforms: list[tuple[str, Path]],
) -> list[tuple[str, Path]]:
"""Prompt user to choose among detected platforms."""
print("\nInstall for:")
for i, (name, path) in enumerate(platforms, 1):
print(f" {i}) {name.capitalize()} ({path})")
if len(platforms) > 1:
print(f" {len(platforms) + 1}) All")
print()
while True:
try:
choice = input("> ").strip()
except (EOFError, KeyboardInterrupt):
print()
sys.exit(0)
if not choice:
continue
try:
idx = int(choice)
except ValueError:
continue
if 1 <= idx <= len(platforms):
return [platforms[idx - 1]]
if idx == len(platforms) + 1 and len(platforms) > 1:
return platforms
return platforms
def main() -> None:
"""Entry point."""
parser = argparse.ArgumentParser(
description="Download missing BIOS files for retrogaming emulators.",
)
parser.add_argument(
"--platform",
help="target platform (retroarch, batocera, emudeck, ...)",
)
parser.add_argument(
"--dest",
type=Path,
help="override BIOS destination directory",
)
parser.add_argument(
"--target",
help="hardware target for core filtering (switch, rpi4, ...)",
)
parser.add_argument(
"--check",
action="store_true",
help="check existing files without downloading",
)
parser.add_argument(
"--list-platforms",
action="store_true",
help="list detected platforms and exit",
)
parser.add_argument(
"--list-targets",
action="store_true",
help="list available targets for a platform and exit",
)
parser.add_argument(
"--jobs", "-j",
type=int,
default=8,
help="parallel download threads (default: 8)",
)
parser.add_argument(
"--verbose", "-v",
action="store_true",
help="verbose output",
)
args = parser.parse_args()
print("RetroBIOS\n")
os_type = detect_os()
# Platform detection or override
if args.platform and args.dest:
platforms = [(args.platform, args.dest)]
elif args.platform:
print("Detecting platform...")
detected = detect_platforms(os_type)
matched = [(n, p) for n, p in detected if n == args.platform]
if matched:
platforms = matched
else:
print(f" Platform '{args.platform}' not detected, using default path.")
platforms = [(args.platform, Path.home() / "bios")]
elif args.dest:
print(f" Using destination: {args.dest}")
platforms = [("retroarch", args.dest)]
else:
print("Detecting platform...")
platforms = detect_platforms(os_type)
if not platforms:
print(" No supported platform detected.")
print(" Use --platform and --dest to specify manually.")
sys.exit(1)
for name, path in platforms:
print(f" Found {name.capitalize()} at {path}")
if args.list_platforms:
if not platforms:
platforms = detect_platforms(os_type)
for name, path in platforms:
print(f" {name}: {path}")
return
if len(platforms) > 1 and not args.list_targets and sys.stdin.isatty():
platforms = _prompt_platform_choice(platforms)
total_downloaded = 0
total_up_to_date = 0
total_errors = 0
for plat_name, bios_path in platforms:
print(f"\nFetching file index for {plat_name}...")
manifest = fetch_manifest(plat_name)
files = manifest.get("files", [])
if args.list_targets:
targets = fetch_targets(plat_name)
if not targets:
print(f" No targets available for {plat_name}")
else:
for t in sorted(targets.keys()):
cores = targets[t].get("cores", [])
print(f" {t} ({len(cores)} cores)")
continue
# Target filtering
if args.target:
targets = fetch_targets(plat_name)
target_info = targets.get(args.target)
if not target_info:
print(f" Warning: target '{args.target}' not found for {plat_name}")
else:
target_cores = target_info.get("cores", [])
before = len(files)
files = _filter_by_target(files, target_cores)
print(f" Filtered {before} -> {len(files)} files for target {args.target}")
total_size = sum(f.get("size", 0) for f in files)
print(f" {len(files)} files ({format_size(total_size)})")
print("\nChecking existing files...")
to_download, up_to_date, mismatched = check_local(files, bios_path)
present = len(up_to_date) + len(mismatched)
print(
f" {present}/{len(files)} present "
f"({len(up_to_date)} verified, {len(mismatched)} wrong hash)"
)
# Mismatched files need re-download
to_download.extend(mismatched)
if args.check:
if to_download:
print(f"\n {len(to_download)} files need downloading.")
else:
print("\n All files up to date.")
continue
if to_download:
dl_size = sum(f.get("size", 0) for f in to_download)
print(f"\nDownloading {len(to_download)} files ({format_size(dl_size)})...")
bios_path.mkdir(parents=True, exist_ok=True)
failed = download_files(
to_download, bios_path, jobs=args.jobs, verbose=args.verbose
)
total_downloaded += len(to_download) - len(failed)
total_errors += len(failed)
else:
print("\n All files up to date.")
total_up_to_date += len(up_to_date)
# Standalone copies
if manifest.get("standalone_copies") and not args.check:
print("\nStandalone emulators:")
copied, skipped = do_standalone_copies(manifest, bios_path, os_type)
if copied or skipped:
print(f" {copied} copied, {skipped} skipped (dir not found)")
if not args.check and not args.list_targets:
print(
f"\nDone. {total_downloaded} downloaded, "
f"{total_up_to_date} up to date, {total_errors} errors."
)
if __name__ == "__main__":
main()

View File

@@ -24,6 +24,7 @@ import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
@@ -2524,5 +2525,86 @@ class TestE2E(unittest.TestCase):
self.assertEqual(manifest_dests, zip_names)
# ---------------------------------------------------------------
# install.py tests
# ---------------------------------------------------------------
def test_93_parse_retroarch_cfg(self):
"""Parse system_directory from retroarch.cfg."""
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from install import _parse_retroarch_system_dir
cfg = os.path.join(self.root, "retroarch.cfg")
# Quoted absolute path
with open(cfg, "w") as f:
f.write('system_directory = "/home/user/ra/system"\n')
result = _parse_retroarch_system_dir(Path(cfg))
self.assertEqual(result, Path("/home/user/ra/system"))
# Default value
with open(cfg, "w") as f:
f.write('system_directory = "default"\n')
result = _parse_retroarch_system_dir(Path(cfg))
self.assertEqual(result, Path(self.root) / "system")
# Unquoted
with open(cfg, "w") as f:
f.write('system_directory = /tmp/ra_system\n')
result = _parse_retroarch_system_dir(Path(cfg))
self.assertEqual(result, Path("/tmp/ra_system"))
def test_94_parse_emudeck_settings(self):
"""Parse emulationPath from EmuDeck settings.sh."""
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from install import _parse_bash_var
settings = os.path.join(self.root, "settings.sh")
with open(settings, "w") as f:
f.write('emulationPath="/home/deck/Emulation"\nromsPath="/home/deck/Emulation/roms"\n')
result = _parse_bash_var(Path(settings), "emulationPath")
self.assertEqual(result, "/home/deck/Emulation")
def test_95_parse_ps1_var(self):
"""Parse $emulationPath from EmuDeck settings.ps1."""
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from install import _parse_ps1_var
settings = os.path.join(self.root, "settings.ps1")
with open(settings, "w") as f:
f.write('$emulationPath="C:\\Emulation"\n')
result = _parse_ps1_var(Path(settings), "$emulationPath")
self.assertEqual(result, "C:\\Emulation")
def test_96_target_filtering(self):
"""--target filters files by cores field."""
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from install import _filter_by_target
files = [
{"dest": "a.bin", "cores": None},
{"dest": "b.bin", "cores": ["flycast", "redream"]},
{"dest": "c.bin", "cores": ["dolphin"]},
]
filtered = _filter_by_target(files, ["flycast", "snes9x"])
dests = [f["dest"] for f in filtered]
self.assertIn("a.bin", dests)
self.assertIn("b.bin", dests)
self.assertNotIn("c.bin", dests)
def test_97_standalone_copies(self):
"""Standalone keys copied to existing emulator dirs."""
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from install import do_standalone_copies
bios_dir = Path(self.root) / "bios"
bios_dir.mkdir(exist_ok=True)
(bios_dir / "prod.keys").write_bytes(b"KEYS")
yuzu_dir = Path(self.root) / "yuzu_keys"
yuzu_dir.mkdir()
missing_dir = Path(self.root) / "nonexistent"
manifest = {
"base_destination": "bios",
"standalone_copies": [{"file": "prod.keys", "targets": {"linux": [str(yuzu_dir), str(missing_dir)]}}]
}
copied, skipped = do_standalone_copies(manifest, bios_dir, "linux")
self.assertEqual(copied, 1)
self.assertEqual(skipped, 1)
self.assertTrue((yuzu_dir / "prod.keys").exists())
self.assertFalse((missing_dir / "prod.keys").exists())
if __name__ == "__main__":
unittest.main()