kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-10 13:33:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: harden the installer boundary

Browse Source 
The manifest is untrusted input. It is now read under a size limit and
validated before use: destinations are contained below the BIOS root,
repo_path stays inside bios/, release_asset is a basename, hashes match
their shape, declared totals match their lists, and every entry has a
download source.

Downloads stream against the declared size, are checked by SHA-256 then
SHA-1, land in a per-process temporary file and are installed with
os.replace. Copies into standalone-emulator directories are opt-in with
--standalone-copies so a detection never writes outside the selected
tree.

Both bootstraps verify install.py against an embedded SHA-256 before
running it, and require the Python version install.py actually needs.

A target that publishes no core list is a target with no filter, not a
broken manifest: rejecting it disabled --target for the whole platform.
This commit is contained in:
Abdessamad Derraz committed 2026-08-10 13:36:52 +02:00
1 parent bb915ed6aa
commit 9001ebb992
4 files changed
+586 -382
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

install.ps1
+58 -330
View File
@@ -1,335 +1,63 @@
# RetroBIOS installer for Windows (PowerShell 5+, no Python required)
# One-line bootstrap and local wrapper. The downloaded installer is accepted
# only when it matches the SHA-256 embedded in this wrapper.
[CmdletBinding()]
param(
[Parameter(ValueFromRemainingArguments = $true)]
[string[]]$InstallerArguments
)
$ErrorActionPreference = "Stop"
$baseUrl = if ($env:RETROBIOS_BASE_URL) { $env:RETROBIOS_BASE_URL } else { "https://raw.githubusercontent.com/Abdess/retrobios/main" }
$releaseUrl = "https://github.com/Abdess/retrobios/releases/download/large-files"
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
$defaultInstallSha256 = "79630030c1b7445e2df02bcf0272c4530d24827b2589780b214489ab036d2e8c"
$maximumInstallerBytes = 2MB
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
$temporary = $null
$platform = $null
$biosPath = $null
try {
if (-not $installer -or -not (Test-Path -LiteralPath $installer -PathType Leaf)) {
$url = if ($env:RETROBIOS_INSTALL_URL) { $env:RETROBIOS_INSTALL_URL } else { $defaultInstallUrl }
$expected = if ($env:RETROBIOS_INSTALL_SHA256) { $env:RETROBIOS_INSTALL_SHA256 } else { $defaultInstallSha256 }
$uri = [Uri]$url
if ($uri.Scheme -ne "https") {
throw "RETROBIOS_INSTALL_URL must use HTTPS."
}
if ($expected -notmatch '^[0-9a-fA-F]{64}$') {
throw "Installer SHA-256 must contain exactly 64 hexadecimal characters."
}
$temporary = Join-Path ([IO.Path]::GetTempPath()) ("retrobios-install-{0}.py" -f [Guid]::NewGuid())
Invoke-WebRequest -Uri $uri -OutFile $temporary -UseBasicParsing
if ((Get-Item -LiteralPath $temporary).Length -gt $maximumInstallerBytes) {
throw "Downloaded installer exceeds the size limit."
}
$actual = (Get-FileHash -LiteralPath $temporary -Algorithm SHA256).Hash.ToLowerInvariant()
if ($actual -ne $expected.ToLowerInvariant()) {
throw "install.py SHA-256 mismatch."
}
$installer = $temporary
}
# Detect EmuDeck
$emudeckSettings = Join-Path $env:APPDATA "EmuDeck\settings.ps1"
if (Test-Path $emudeckSettings) {
$content = Get-Content $emudeckSettings -Raw
if ($content -match '\$emulationPath\s*=\s*"([^"]+)"') {
$platform = "emudeck"
$biosPath = Join-Path $Matches[1] "bios"
Write-Host "Found EmuDeck at $biosPath"
$python = Get-Command py -ErrorAction SilentlyContinue
if ($python) {
& $python.Source -3 -c "import sys; raise SystemExit(sys.version_info < (3, 8))"
if ($LASTEXITCODE -ne 0) { throw "Python 3.8 or newer is required." }
& $python.Source -3 $installer @InstallerArguments
if ($LASTEXITCODE -ne 0) { throw "RetroBIOS installer failed with exit code $LASTEXITCODE." }
return
}
$python = Get-Command python3 -ErrorAction SilentlyContinue
if (-not $python) {
$python = Get-Command python -ErrorAction SilentlyContinue
}
if (-not $python) {
throw "Python 3.8 or newer is required."
}
& $python.Source -c "import sys; raise SystemExit(sys.version_info < (3, 8))"
if ($LASTEXITCODE -ne 0) { throw "Python 3.8 or newer is required." }
& $python.Source $installer @InstallerArguments
if ($LASTEXITCODE -ne 0) { throw "RetroBIOS installer failed with exit code $LASTEXITCODE." }
}
finally {
if ($temporary -and (Test-Path -LiteralPath $temporary)) {
Remove-Item -LiteralPath $temporary -Force
}
}
# Expand the notations RetroArch writes into its config values:
# '~' is the home directory and ':' the application directory, both dropping
# two leading characters (libretro-common/file/file_path.c).
function Expand-RetroArchPath {
param([string]$Value, [string]$AppDir)
if ($Value.StartsWith('~')) { return Join-Path $env:USERPROFILE $Value.Substring(2) }
if ($Value.StartsWith(':')) { return Join-Path $AppDir $Value.Substring(2) }
return [Environment]::ExpandEnvironmentVariables($Value)
}
function Get-RetroArchSystemDir {
param([string]$CfgPath, [string]$AppDir)
if (-not (Test-Path $CfgPath)) { return $null }
foreach ($line in Get-Content $CfgPath) {
if ($line -match '^\s*system_directory\s*=\s*"?([^"]*)"?') {
$val = $Matches[1].Trim()
if (-not $val -or $val -eq "default") { return Join-Path $AppDir "system" }
return Expand-RetroArchPath -Value $val -AppDir $AppDir
}
}
return $null
}
# Detect RetroArch
if (-not $platform) {
$raCfg = Join-Path $env:APPDATA "RetroArch\retroarch.cfg"
if (Test-Path $raCfg) {
$platform = "retroarch"
$raRoot = Join-Path $env:APPDATA "RetroArch"
$found = Get-RetroArchSystemDir -CfgPath $raCfg -AppDir $raRoot
$biosPath = if ($found) { $found } else { Join-Path $raRoot "system" }
Write-Host "Found RetroArch at $biosPath"
}
}
# Locate LaunchBox. It installs wherever the user points its installer and
# writes no uninstall registry key, so the Start menu shortcut is the only
# record of that choice.
function Get-LaunchBoxRoot {
$candidates = @()
$lnk = Join-Path $env:APPDATA "Microsoft\Windows\Start Menu\Programs\LaunchBox\LaunchBox.lnk"
if (Test-Path $lnk) {
$bytes = [System.IO.File]::ReadAllBytes($lnk)
$text = [System.Text.Encoding]::ASCII.GetString($bytes)
if ($text -match '([A-Za-z]:\\[ -~]{0,260}?LaunchBox\.exe)') {
# The shortcut points at Core\LaunchBox.exe
$exeDir = Split-Path $Matches[1] -Parent
$candidates += (Split-Path $exeDir -Parent)
$candidates += $exeDir
}
}
$candidates += (Join-Path $env:USERPROFILE "LaunchBox")
foreach ($root in $candidates) {
if (Test-Path (Join-Path $root "Data\Emulators.xml")) { return $root }
}
return $null
}
# Map each emulator LaunchBox knows about to its installation directory.
# ApplicationPath is either absolute or relative to the LaunchBox root.
function Get-LaunchBoxEmulators {
param([string]$Root)
$map = @{}
if (-not $Root) { return $map }
$xmlPath = Join-Path $Root "Data\Emulators.xml"
if (-not (Test-Path $xmlPath)) { return $map }
$doc = $null
try { $doc = [xml](Get-Content $xmlPath -Raw) } catch { return $map }
foreach ($emu in $doc.LaunchBox.Emulator) {
$app = "$($emu.ApplicationPath)"
if (-not $app) { continue }
$exe = if ($app -match '^[A-Za-z]:' -or $app.StartsWith('\')) { $app } else { Join-Path $Root $app }
$dir = Split-Path $exe -Parent
$name = (Split-Path $exe -Leaf).ToLower()
if ((Test-Path $dir) -and -not $map.ContainsKey($name)) { $map[$name] = $dir }
}
return $map
}
# BIOS directories LaunchBox itself computes for the emulators it manages.
function Get-LaunchBoxBiosDirs {
$dirs = @{}
$root = Get-LaunchBoxRoot
if (-not $root) { return $dirs }
$emulators = Get-LaunchBoxEmulators -Root $root
$documents = Join-Path $env:USERPROFILE "Documents"
if ($emulators.ContainsKey("pcsx2.exe")) {
# portable.ini or portable.txt next to the executable moves the data
# root there, portable.txt naming a subfolder when it holds one
# (EmuFolders in pcsx2/Pcsx2Config.cpp). Otherwise it is Documents.
$emuDir = $emulators["pcsx2.exe"]
$portableTxt = Join-Path $emuDir "portable.txt"
$portable = (Test-Path (Join-Path $emuDir "portable.ini")) -or (Test-Path $portableTxt)
if ($portable) {
$subpath = if (Test-Path $portableTxt) { (Get-Content $portableTxt -Raw).Trim() } else { "" }
$dataRoot = if ($subpath) { Join-Path $emuDir $subpath } else { $emuDir }
} else {
$dataRoot = Join-Path $documents "PCSX2"
}
$ini = Join-Path $dataRoot "inis\PCSX2.ini"
$bios = Join-Path $dataRoot "bios"
if (Test-Path $ini) {
foreach ($line in Get-Content $ini) {
if ($line.StartsWith("Bios = ")) {
$val = $line.Substring(7).Trim()
$bios = if ($val -match '^[A-Za-z]:' -or $val.StartsWith('\')) { $val } else { Join-Path $dataRoot $val }
break
}
}
}
$dirs["pcsx2"] = $bios
}
if ($emulators.ContainsKey("xemu.exe")) {
# bootrom_path and flashrom_path name a file whose directory holds
# the images; both default to bios/ under the executable.
$emuDir = $emulators["xemu.exe"]
$toml = Join-Path $emuDir "xemu.toml"
if (-not (Test-Path $toml)) { $toml = Join-Path $env:APPDATA "xemu\xemu\xemu.toml" }
$bios = Join-Path $emuDir "bios"
if (Test-Path $toml) {
foreach ($line in Get-Content $toml) {
if ($line.StartsWith("bootrom_path") -or $line.StartsWith("flashrom_path")) {
$parts = $line.Split("'")
if ($parts.Count -gt 1 -and (Test-Path $parts[1])) {
$bios = Split-Path $parts[1] -Parent
break
}
}
}
}
$dirs["xemu"] = $bios
}
if ($emulators.ContainsKey("dolphin.exe")) {
# portable.txt beside the executable moves the user directory to User
# (SetUserDirectory in Source/Core/UICommon/UICommon.cpp)
$emuDir = $emulators["dolphin.exe"]
if (Test-Path (Join-Path $emuDir "portable.txt")) {
$dirs["dolphin"] = Join-Path $emuDir "User"
}
}
return $dirs
}
# Detect LaunchBox (portable RetroArch referenced in Data\Emulators.xml)
if (-not $platform) {
$lbRoot = Get-LaunchBoxRoot
$lbXml = if ($lbRoot) { Join-Path $lbRoot "Data\Emulators.xml" } else { $null }
if ($lbXml -and (Test-Path $lbXml)) {
$lb = $null
try { $lb = [xml](Get-Content $lbXml -Raw) } catch { Write-Host "Warning: could not parse $lbXml" }
if ($lb) {
foreach ($emu in $lb.LaunchBox.Emulator) {
$app = "$($emu.ApplicationPath)".Replace('\', '/')
if ($app -notmatch 'retroarch\.exe$') { continue }
$lbRoot = Split-Path (Split-Path $lbXml -Parent) -Parent
if ($app -match '^[A-Za-z]:' -or $app.StartsWith('/')) {
$exe = $app
} else {
$exe = Join-Path $lbRoot $app
}
$raDir = Split-Path $exe -Parent
if (Test-Path $raDir) {
$platform = "retroarch"
$found = Get-RetroArchSystemDir -CfgPath (Join-Path $raDir "retroarch.cfg") -AppDir $raDir
$biosPath = if ($found) { $found } else { Join-Path $raDir "system" }
Write-Host "Found LaunchBox with RetroArch at $biosPath"
break
}
}
}
}
}
# Fallback
if (-not $platform) {
$available = @("retroarch", "batocera", "recalbox", "retrobat", "emudeck", "lakka", "retrodeck", "rocknix", "romm", "bizhawk", "misterfpga")
$platform = (Read-Host "Platform ($($available -join ', '))").Trim().ToLower()
$biosPath = (Read-Host "BIOS directory path").Trim()
if (-not $platform -or -not $biosPath) {
Write-Host "Aborted." -ForegroundColor Red; exit 1
}
if ($available -notcontains $platform) {
Write-Host "Unknown platform '$platform'. Available: $($available -join ', ')" -ForegroundColor Red
exit 1
}
}
Write-Host "`nFetching file index for $platform..."
$manifest = Invoke-RestMethod "$baseUrl/install/$platform.json"
$files = $manifest.files
Write-Host " $($files.Count) files"
Write-Host "`nChecking existing files..."
$toDownload = @()
$upToDate = 0
foreach ($f in $files) {
$dest = Join-Path $biosPath $f.dest
if (Test-Path $dest) {
if ($f.sha1) {
$actual = (Get-FileHash $dest -Algorithm SHA1).Hash.ToLower()
if ($actual -eq $f.sha1) { $upToDate++; continue }
} else {
$upToDate++; continue
}
}
$toDownload += $f
}
Write-Host " $upToDate/$($files.Count) up to date, $($toDownload.Count) to download"
$downloaded = 0
$errors = 0
$total = $toDownload.Count
foreach ($f in $toDownload) {
$dest = Join-Path $biosPath $f.dest
$dir = Split-Path $dest -Parent
if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null }
if ($f.release_asset) {
$url = "$releaseUrl/$($f.release_asset)"
} else {
$url = "$baseUrl/$($f.repo_path)"
}
$tmp = "$dest.tmp"
$ok = $false
foreach ($attempt in 1..3) {
try {
Invoke-WebRequest -Uri $url -OutFile $tmp -UseBasicParsing
} catch {
continue
}
if ($f.sha1) {
$actual = (Get-FileHash $tmp -Algorithm SHA1).Hash.ToLower()
if ($actual -ne $f.sha1) {
Remove-Item $tmp -Force -ErrorAction SilentlyContinue
continue
}
}
Move-Item $tmp $dest -Force
$ok = $true
break
}
if ($ok) {
$downloaded++
$i = $downloaded + $errors
Write-Host " [$i/$total] $($f.dest) ok"
} else {
Remove-Item $tmp -Force -ErrorAction SilentlyContinue
$errors++
$i = $downloaded + $errors
Write-Host " [$i/$total] $($f.dest) FAILED" -ForegroundColor Red
}
}
# Standalone emulator copies
if ($manifest.standalone_copies) {
Write-Host "`nStandalone emulators:"
$extraDirs = Get-LaunchBoxBiosDirs
foreach ($entry in $manifest.standalone_copies) {
if ($entry.note) {
$detectPaths = @()
if ($entry.detect -and $entry.detect.windows) {
$detectPaths = $entry.detect.windows
}
foreach ($dp in $detectPaths) {
$expanded = [Environment]::ExpandEnvironmentVariables($dp)
if (Test-Path $expanded) {
Write-Host " $($entry.note)"
break
}
}
continue
}
$sources = @()
if ($entry.pattern) {
$sources = Get-ChildItem -Path $biosPath -Filter $entry.pattern -File -ErrorAction SilentlyContinue
} elseif ($entry.file) {
$src = Join-Path $biosPath $entry.file
if (Test-Path $src) { $sources = @(Get-Item $src) }
}
if ($sources.Count -eq 0) { continue }
$targetDirs = @()
if ($entry.targets -and $entry.targets.windows) {
$targetDirs = $entry.targets.windows
}
if ($entry.emulator -and $extraDirs.ContainsKey($entry.emulator)) {
$extra = $extraDirs[$entry.emulator]
$subdir = if ($entry.file) { Split-Path $entry.file -Parent } else { "" }
if ($subdir) { $extra = Join-Path $extra $subdir }
$targetDirs += $extra
}
foreach ($td in $targetDirs) {
$expanded = [Environment]::ExpandEnvironmentVariables($td)
if (-not (Test-Path $expanded)) { continue }
foreach ($s in $sources) {
$dest = Join-Path $expanded $s.Name
try {
Copy-Item $s.FullName $dest -Force
Write-Host " $($s.Name) -> $expanded"
} catch {
Write-Host " $($s.Name) -> $expanded FAILED" -ForegroundColor Red
}
}
}
}
}
Write-Host "`nDone. $downloaded downloaded, $upToDate already up to date."
install.py
+358 -25
View File
@@ -29,9 +29,15 @@ import urllib.request
import xml.etree.ElementTree as ET
from pathlib import Path, PurePosixPath
# Manifests are read from the same ref the bootstrap verified this installer
# against, so a file list and the code reading it always come from one commit.
# RETROBIOS_REF pins an installation to a tag when reproducibility matters.
DEFAULT_RELEASE_REF = "main"
RELEASE_REF = os.environ.get("RETROBIOS_REF", DEFAULT_RELEASE_REF)
BASE_URL = os.environ.get(
"RETROBIOS_BASE_URL",
"https://raw.githubusercontent.com/Abdess/retrobios/main",
"https://raw.githubusercontent.com/Abdess/retrobios/"
+ urllib.parse.quote(RELEASE_REF, safe=""),
)
MANIFEST_URL = f"{BASE_URL}/install/{{platform}}.json"
TARGETS_URL = f"{BASE_URL}/install/targets/{{platform}}.json"
@@ -40,6 +46,13 @@ RELEASE_URL = (
"https://github.com/Abdess/retrobios/releases/download/large-files/{asset}"
)
MAX_RETRIES = 3
MAX_MANIFEST_BYTES = 16 * 1024 * 1024
MAX_TARGETS_BYTES = 4 * 1024 * 1024
MAX_MANIFEST_FILES = 100_000
MAX_DOWNLOAD_SIZE = 1024 * 1024 * 1024
MAX_TOTAL_DOWNLOAD_SIZE = 64 * 1024 * 1024 * 1024
_SHA1_RE = re.compile(r"^[0-9a-fA-F]{40}$")
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
# Platforms with a manifest in install/. Manifest URLs are case sensitive,
# so user input is normalized against this list before any fetch.
@@ -567,13 +580,235 @@ def normalize_platform(name: str) -> str:
return plat
def _read_limited_json(response, limit: int, label: str) -> object:
"""Read a bounded UTF-8 JSON response."""
raw_length = response.headers.get("Content-Length") if response.headers else None
if raw_length:
try:
if int(raw_length) > limit:
raise ValueError(f"{label} exceeds {limit} bytes")
except ValueError as exc:
if "exceeds" in str(exc):
raise
payload = response.read(limit + 1)
if len(payload) > limit:
raise ValueError(f"{label} exceeds {limit} bytes")
try:
return json.loads(payload.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ValueError(f"invalid {label}: {exc}") from exc
def _safe_relative_path(value: object, field: str) -> PurePosixPath:
"""Validate a manifest-controlled relative POSIX path."""
if not isinstance(value, str) or not value or len(value) > 1024:
raise ValueError(f"invalid {field}")
if (
"\\" in value
or "\x00" in value
or "//" in value
or value.endswith("/")
or re.match(r"^[A-Za-z]:", value)
):
raise ValueError(f"unsafe {field}: {value!r}")
path = PurePosixPath(value)
if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
raise ValueError(f"unsafe {field}: {value!r}")
return path
def _destination_path(root: Path, value: object) -> Path:
"""Resolve a manifest destination and prove it remains below *root*."""
relative = _safe_relative_path(value, "dest")
resolved_root = root.resolve()
candidate = (resolved_root / Path(*relative.parts)).resolve()
try:
candidate.relative_to(resolved_root)
except ValueError as exc:
raise ValueError(f"destination escapes BIOS directory: {value!r}") from exc
return candidate
def _validate_manifest(data: object, plat: str) -> dict:
"""Validate the untrusted install-manifest boundary using stdlib only."""
if not isinstance(data, dict):
raise ValueError("manifest root must be an object")
if data.get("manifest_version") not in (1, 2):
raise ValueError("unsupported manifest_version")
if data.get("platform") != plat:
raise ValueError("manifest platform does not match request")
files = data.get("files")
if not isinstance(files, list) or len(files) > MAX_MANIFEST_FILES:
raise ValueError("invalid manifest files list")
seen_destinations: set[str] = set()
total_size = 0
for index, entry in enumerate(files):
if not isinstance(entry, dict):
raise ValueError(f"files[{index}] must be an object")
dest = str(_safe_relative_path(entry.get("dest"), f"files[{index}].dest"))
if dest in seen_destinations:
raise ValueError(f"duplicate manifest destination: {dest}")
seen_destinations.add(dest)
size = entry.get("size")
if isinstance(size, bool) or not isinstance(size, int) or not (0 <= size <= MAX_DOWNLOAD_SIZE):
raise ValueError(f"invalid size for {dest}")
total_size += size
if total_size > MAX_TOTAL_DOWNLOAD_SIZE:
raise ValueError("manifest total size exceeds safety limit")
sha1 = entry.get("sha1", "")
sha256 = entry.get("sha256", "")
if sha1 and (not isinstance(sha1, str) or not _SHA1_RE.fullmatch(sha1)):
raise ValueError(f"invalid SHA1 for {dest}")
if sha256 and (
not isinstance(sha256, str) or not _SHA256_RE.fullmatch(sha256)
):
raise ValueError(f"invalid SHA256 for {dest}")
if not sha1 and not sha256:
raise ValueError(f"missing content hash for {dest}")
release_asset = entry.get("release_asset")
repo_path = entry.get("repo_path")
if release_asset:
asset = _safe_relative_path(release_asset, f"files[{index}].release_asset")
if len(asset.parts) != 1:
raise ValueError(f"release asset must be a basename: {release_asset}")
elif repo_path:
source = _safe_relative_path(repo_path, f"files[{index}].repo_path")
if source.parts[0] != "bios":
raise ValueError(f"repo_path outside bios/: {repo_path}")
else:
raise ValueError(f"no download source for {dest}")
cores = entry.get("cores")
if cores is not None and (
not isinstance(cores, list) or not all(isinstance(core, str) for core in cores)
):
raise ValueError(f"invalid cores list for {dest}")
declared_total_files = data.get("total_files")
if declared_total_files is not None and declared_total_files != len(files):
raise ValueError("manifest total_files does not match files list")
declared_total_size = data.get("total_size")
if declared_total_size is not None and declared_total_size != total_size:
raise ValueError("manifest total_size does not match file sizes")
omitted = data.get("omitted_files", [])
if not isinstance(omitted, list) or len(omitted) > MAX_MANIFEST_FILES:
raise ValueError("invalid omitted_files list")
seen_omitted: set[str] = set()
allowed_omission_reasons = {
"hash_mismatch", "not_found", "external", "user_provided"
}
for index, entry in enumerate(omitted):
if not isinstance(entry, dict):
raise ValueError(f"omitted_files[{index}] must be an object")
dest = str(
_safe_relative_path(
entry.get("dest"), f"omitted_files[{index}].dest"
)
)
if dest in seen_destinations or dest in seen_omitted:
raise ValueError(f"duplicate or conflicting omitted destination: {dest}")
seen_omitted.add(dest)
if not isinstance(entry.get("name"), str) or not entry["name"]:
raise ValueError(f"invalid omitted file name for {dest}")
if not isinstance(entry.get("system", ""), str):
raise ValueError(f"invalid omitted system for {dest}")
if not isinstance(entry.get("required"), bool):
raise ValueError(f"invalid omitted required flag for {dest}")
if entry.get("reason") not in allowed_omission_reasons:
raise ValueError(f"invalid omission reason for {dest}")
cores = entry.get("cores")
if cores is not None and (
not isinstance(cores, list)
or not all(isinstance(core, str) for core in cores)
):
raise ValueError(f"invalid omitted cores list for {dest}")
declared_total_omitted = data.get("total_omitted")
if (
declared_total_omitted is not None
and declared_total_omitted != len(omitted)
):
raise ValueError("manifest total_omitted does not match omitted_files")
copies = data.get("standalone_copies", [])
if not isinstance(copies, list) or len(copies) > 10_000:
raise ValueError("invalid standalone_copies")
for index, entry in enumerate(copies):
if not isinstance(entry, dict):
raise ValueError(f"standalone_copies[{index}] must be an object")
if "file" in entry:
_safe_relative_path(
entry["file"], f"standalone_copies[{index}].file"
)
if "pattern" in entry:
pattern = entry["pattern"]
if (
not isinstance(pattern, str)
or not pattern
or len(pattern) > 256
or "/" in pattern
or "\\" in pattern
or ".." in pattern
):
raise ValueError(f"invalid standalone copy pattern: {pattern!r}")
targets = entry.get("targets", {})
if targets and (
not isinstance(targets, dict)
or any(
not isinstance(values, list)
or len(values) > 100
or not all(
isinstance(value, str) and len(value) <= 2048
for value in values
)
for values in targets.values()
)
):
raise ValueError(f"invalid standalone copy targets at index {index}")
return data
def _validate_targets(data: object) -> dict[str, dict]:
"""Validate and normalize legacy list-valued target manifests.
A null core list means the target publishes no core inventory. That is a
known target with no filter, not a broken manifest: rejecting it would
discard every other target on the platform.
"""
if not isinstance(data, dict) or len(data) > 10_000:
raise ValueError("invalid targets manifest")
normalized: dict[str, dict] = {}
for target, value in data.items():
if not isinstance(target, str) or not target or len(target) > 128:
raise ValueError("invalid target name")
if isinstance(value, dict):
cores = value.get("cores")
else:
cores = value
if cores is None:
normalized[target] = {"cores": None}
continue
if not isinstance(cores, list) or len(cores) > 10_000 or not all(
isinstance(core, str) and 0 < len(core) <= 256 for core in cores
):
raise ValueError(f"invalid core list for target {target}")
normalized[target] = {"cores": cores}
return normalized
def fetch_manifest(plat: str) -> dict:
"""Download platform manifest JSON."""
url = MANIFEST_URL.format(platform=plat)
try:
with urllib.request.urlopen(url, timeout=30) as resp:
return json.loads(resp.read().decode("utf-8"))
except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
return _validate_manifest(
_read_limited_json(resp, MAX_MANIFEST_BYTES, "manifest"), plat
)
except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError) as exc:
print(f" Failed to fetch manifest for {plat}: {exc}", file=sys.stderr)
sys.exit(1)
@@ -583,13 +818,15 @@ def fetch_targets(plat: str) -> dict:
url = TARGETS_URL.format(platform=plat)
try:
with urllib.request.urlopen(url, timeout=30) as resp:
return json.loads(resp.read().decode("utf-8"))
return _validate_targets(
_read_limited_json(resp, MAX_TARGETS_BYTES, "targets manifest")
)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return {}
print(f" Warning: failed to fetch targets for {plat}: {exc}", file=sys.stderr)
return {}
except (urllib.error.URLError, OSError):
except (urllib.error.URLError, OSError, ValueError):
return {}
@@ -618,6 +855,15 @@ def _sha1_file(path: Path) -> str:
return h.hexdigest()
def _sha256_file(path: Path) -> str:
"""Compute SHA256 of a file."""
h = hashlib.sha256()
with open(path, "rb") as fh:
for chunk in iter(lambda: fh.read(65536), b""):
h.update(chunk)
return h.hexdigest()
def check_local(
files: list[dict], bios_path: Path
) -> tuple[list[dict], list[dict], list[dict]]:
@@ -630,16 +876,21 @@ def check_local(
mismatched: list[dict] = []
for f in files:
dest = bios_path / f["dest"]
dest = _destination_path(bios_path, f["dest"])
if not dest.exists():
to_download.append(f)
continue
expected_sha256 = f.get("sha256", "")
expected_sha1 = f.get("sha1", "")
if not expected_sha1:
if not expected_sha256 and not expected_sha1:
up_to_date.append(f)
continue
actual = _sha1_file(dest)
if actual == expected_sha1:
verified = True
if expected_sha256:
verified = _sha256_file(dest) == expected_sha256.lower()
if verified and expected_sha1:
verified = _sha1_file(dest) == expected_sha1.lower()
if verified:
up_to_date.append(f)
else:
mismatched.append(f)
@@ -651,38 +902,77 @@ def _download_one(
f: dict, bios_path: Path, verbose: bool = False
) -> tuple[str, bool]:
"""Download a single file. Returns (dest, success)."""
dest = bios_path / f["dest"]
try:
dest = _destination_path(bios_path, f["dest"])
except ValueError:
return str(f.get("dest", "?")), False
dest.parent.mkdir(parents=True, exist_ok=True)
if f.get("release_asset"):
url = RELEASE_URL.format(asset=urllib.parse.quote(f["release_asset"], safe="/"))
url = RELEASE_URL.format(asset=urllib.parse.quote(f["release_asset"], safe=""))
else:
url = RAW_FILE_URL.format(path=urllib.parse.quote(f["repo_path"], safe="/"))
tmp_path = dest.with_suffix(dest.suffix + ".tmp")
for attempt in range(1, MAX_RETRIES + 1):
tmp_path: Path | None = None
try:
with urllib.request.urlopen(url, timeout=60) as resp:
with open(tmp_path, "wb") as out:
shutil.copyfileobj(resp, out)
expected_size = f["size"]
raw_length = resp.headers.get("Content-Length") if resp.headers else None
if raw_length and int(raw_length) != expected_size:
raise ValueError(
f"Content-Length {raw_length} != expected {expected_size}"
)
with tempfile.NamedTemporaryFile(
mode="wb",
dir=dest.parent,
prefix=f".{dest.name}.",
suffix=".part",
delete=False,
) as out:
tmp_path = Path(out.name)
downloaded = 0
while True:
chunk = resp.read(1024 * 1024)
if not chunk:
break
downloaded += len(chunk)
if downloaded > expected_size or downloaded > MAX_DOWNLOAD_SIZE:
raise ValueError("download exceeded declared size")
out.write(chunk)
if downloaded != expected_size:
raise ValueError(
f"downloaded {downloaded} bytes; expected {expected_size}"
)
expected_sha256 = f.get("sha256", "")
expected_sha1 = f.get("sha1", "")
if expected_sha256 and _sha256_file(tmp_path) != expected_sha256.lower():
if verbose:
print(f" SHA256 mismatch on attempt {attempt}", file=sys.stderr)
tmp_path.unlink(missing_ok=True)
continue
if expected_sha1:
actual = _sha1_file(tmp_path)
if actual != expected_sha1:
if actual != expected_sha1.lower():
if verbose:
print(f" SHA1 mismatch on attempt {attempt}", file=sys.stderr)
tmp_path.unlink(missing_ok=True)
continue
tmp_path.rename(dest)
os.replace(tmp_path, dest)
return f["dest"], True
except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
except (
urllib.error.URLError,
urllib.error.HTTPError,
OSError,
ValueError,
) as exc:
if verbose:
print(f" Attempt {attempt} failed: {exc}", file=sys.stderr)
tmp_path.unlink(missing_ok=True)
if tmp_path is not None:
tmp_path.unlink(missing_ok=True)
return f["dest"], False
@@ -880,8 +1170,15 @@ def main() -> None:
action="store_true",
help="verbose output",
)
parser.add_argument(
"--standalone-copies",
action="store_true",
help="opt in to copies into detected standalone-emulator directories",
)
args = parser.parse_args()
if not 1 <= args.jobs <= 32:
parser.error("--jobs must be between 1 and 32")
print("RetroBIOS\n")
os_type = detect_os()
@@ -940,11 +1237,13 @@ def main() -> None:
total_downloaded = 0
total_up_to_date = 0
total_errors = 0
total_omitted = 0
for plat_name, bios_path in platforms:
print(f"\nFetching file index for {plat_name}...")
manifest = fetch_manifest(plat_name)
files = manifest.get("files", [])
omitted_files = manifest.get("omitted_files", [])
if args.list_targets:
targets = fetch_targets(plat_name)
@@ -952,24 +1251,48 @@ def main() -> None:
print(f" No targets available for {plat_name}")
else:
for t in sorted(targets.keys()):
cores = targets[t].get("cores", [])
print(f" {t} ({len(cores)} cores)")
cores = targets[t].get("cores")
label = "no core list" if cores is None else f"{len(cores)} cores"
print(f" {t} ({label})")
continue
# Target filtering
if args.target:
targets = fetch_targets(plat_name)
target_info = targets.get(args.target)
if not target_info:
if target_info is None:
print(f" Warning: target '{args.target}' not found for {plat_name}")
elif target_info.get("cores") is None:
print(
f" Target '{args.target}' publishes no core list; "
"installing every file"
)
else:
target_cores = target_info.get("cores", [])
target_cores = target_info["cores"]
before = len(files)
files = _filter_by_target(files, target_cores)
omitted_files = _filter_by_target(omitted_files, target_cores)
print(f" Filtered {before} -> {len(files)} files for target {args.target}")
total_size = sum(f.get("size", 0) for f in files)
print(f" {len(files)} files ({format_size(total_size)})")
if omitted_files:
required_omitted = sum(
1 for entry in omitted_files if entry.get("required", True)
)
reasons: dict[str, int] = {}
for entry in omitted_files:
reason = entry.get("reason", "unknown")
reasons[reason] = reasons.get(reason, 0) + 1
reason_summary = ", ".join(
f"{reason.replace('_', ' ')}: {count}"
for reason, count in sorted(reasons.items())
)
print(
f" Safety notice: {len(omitted_files)} unavailable or unsafe "
f"entries omitted ({required_omitted} required; {reason_summary})."
)
total_omitted += len(omitted_files)
print("\nChecking existing files...")
to_download, up_to_date, mismatched = check_local(files, bios_path)
@@ -1004,7 +1327,11 @@ def main() -> None:
total_up_to_date += len(up_to_date)
# Standalone copies
if manifest.get("standalone_copies") and not args.check:
if (
manifest.get("standalone_copies")
and not args.check
and args.standalone_copies
):
print("\nStandalone emulators:")
lb_root = launchbox_root(os_type)
extra_dirs = launchbox_bios_dirs(lb_root) if lb_root else None
@@ -1013,11 +1340,17 @@ def main() -> None:
)
if copied or skipped:
print(f" {copied} copied, {skipped} skipped (dir not found)")
elif manifest.get("standalone_copies") and not args.check:
print(
"\nStandalone copies skipped "
"(use --standalone-copies to opt in)."
)
if not args.check and not args.list_targets:
print(
f"\nDone. {total_downloaded} downloaded, "
f"{total_up_to_date} up to date, {total_errors} errors."
f"{total_up_to_date} up to date, {total_errors} errors, "
f"{total_omitted} safely omitted."
)
install.sh
+90 -14
View File
@@ -1,20 +1,96 @@
#!/bin/sh
set -e
REPO="https://raw.githubusercontent.com/Abdess/retrobios/main"
SCRIPT=$(mktemp)
trap 'rm -f "$SCRIPT"' EXIT
if command -v curl >/dev/null 2>&1; then
curl -fsSL "$REPO/install.py" -o "$SCRIPT"
elif command -v wget >/dev/null 2>&1; then
wget -qO "$SCRIPT" "$REPO/install.py"
else
echo "Error: curl or wget required" >&2; exit 1
set -eu
# One-line bootstrap and local wrapper. The downloaded installer is accepted
# only when it matches the SHA-256 embedded in this wrapper.
INSTALLER=""
# When sourced from stdin, $0 is the shell name and the working directory is
# not a trusted location for install.py. Reuse an adjacent installer only for
# an actual local install.sh invocation.
case "$0" in
install.sh|*/install.sh)
if [ -f "$0" ]; then
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
INSTALLER="$SCRIPT_DIR/install.py"
fi
;;
esac
TEMP_INSTALLER=""
TEMP_DIRECTORY=""
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
DEFAULT_INSTALL_SHA256="79630030c1b7445e2df02bcf0272c4530d24827b2589780b214489ab036d2e8c"
MAX_INSTALLER_BYTES=2097152
cleanup() {
if [ -n "$TEMP_INSTALLER" ] && [ -f "$TEMP_INSTALLER" ]; then
rm -f -- "$TEMP_INSTALLER"
fi
if [ -n "$TEMP_DIRECTORY" ] && [ -d "$TEMP_DIRECTORY" ]; then
rmdir -- "$TEMP_DIRECTORY" 2>/dev/null || true
fi
}
trap cleanup EXIT HUP INT TERM
if [ -z "$INSTALLER" ] || [ ! -f "$INSTALLER" ]; then
install_url=${RETROBIOS_INSTALL_URL:-$DEFAULT_INSTALL_URL}
expected=${RETROBIOS_INSTALL_SHA256:-$DEFAULT_INSTALL_SHA256}
case "$install_url" in
https://*) ;;
*) echo "Error: installer URL must use HTTPS." >&2; exit 1 ;;
esac
case "$expected" in
*[!0-9A-Fa-f]*)
echo "Error: installer SHA-256 must contain exactly 64 hexadecimal characters." >&2
exit 1
;;
esac
if [ "${#expected}" -ne 64 ]; then
echo "Error: installer SHA-256 must contain exactly 64 hexadecimal characters." >&2
exit 1
fi
TEMP_DIRECTORY=$(mktemp -d)
TEMP_INSTALLER="$TEMP_DIRECTORY/install.py"
if command -v curl >/dev/null 2>&1; then
curl --fail --location --proto '=https' --tlsv1.2 \
"$install_url" --output "$TEMP_INSTALLER"
elif command -v wget >/dev/null 2>&1; then
wget --https-only --output-document="$TEMP_INSTALLER" "$install_url"
else
echo "Error: curl or wget is required." >&2
exit 1
fi
actual_size=$(wc -c < "$TEMP_INSTALLER" | tr -d ' ')
if [ "$actual_size" -gt "$MAX_INSTALLER_BYTES" ]; then
echo "Error: downloaded installer exceeds the size limit." >&2
exit 1
fi
if command -v sha256sum >/dev/null 2>&1; then
actual=$(sha256sum "$TEMP_INSTALLER" | awk '{print $1}')
elif command -v shasum >/dev/null 2>&1; then
actual=$(shasum -a 256 "$TEMP_INSTALLER" | awk '{print $1}')
else
echo "Error: sha256sum or shasum is required." >&2
exit 1
fi
expected=$(printf '%s' "$expected" | tr '[:upper:]' '[:lower:]')
if [ "$actual" != "$expected" ]; then
echo "Error: install.py SHA-256 mismatch." >&2
exit 1
fi
INSTALLER="$TEMP_INSTALLER"
fi
PYTHON=""
for cmd in python3 python; do
if command -v "$cmd" >/dev/null 2>&1; then PYTHON="$cmd"; break; fi
for command_name in python3 python; do
if command -v "$command_name" >/dev/null 2>&1 \
&& "$command_name" -c 'import sys; raise SystemExit(sys.version_info < (3, 8))' 2>/dev/null; then
PYTHON=$command_name
break
fi
done
if [ -z "$PYTHON" ]; then
echo "Error: Python 3 required" >&2; exit 1
echo "Error: Python 3 is required." >&2
exit 1
fi
"$PYTHON" "$SCRIPT" "$@"
"$PYTHON" "$INSTALLER" "$@"
tests/test_install.py
+80 -13
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import functools
import hashlib
import http.server
import importlib.util
import json
@@ -575,9 +576,11 @@ class TestLaunchboxDetectionPowershell(unittest.TestCase):
seed(userprofile)
serve_root = Path(tempfile.mkdtemp())
(serve_root / "install").mkdir()
(serve_root / "install" / "retroarch.json").write_text(
json.dumps(manifest if manifest is not None else {"files": []})
)
payload = dict(manifest if manifest is not None else {"files": []})
payload.setdefault("manifest_version", 2)
payload.setdefault("platform", "retroarch")
payload.setdefault("files", [])
(serve_root / "install" / "retroarch.json").write_text(json.dumps(payload))
handler = functools.partial(
http.server.SimpleHTTPRequestHandler, directory=str(serve_root)
)
@@ -591,7 +594,10 @@ class TestLaunchboxDetectionPowershell(unittest.TestCase):
RETROBIOS_BASE_URL=f"http://127.0.0.1:{httpd.server_address[1]}",
)
proc = subprocess.run(
["pwsh", "-NoProfile", "-File", str(REPO_ROOT / "install.ps1")],
[
"pwsh", "-NoProfile", "-File",
str(REPO_ROOT / "install.ps1"), "--standalone-copies",
],
env=env,
capture_output=True,
text=True,
@@ -599,10 +605,12 @@ class TestLaunchboxDetectionPowershell(unittest.TestCase):
)
finally:
httpd.shutdown()
httpd.server_close()
return proc, ra_dir
def _assert_resolved(self, proc, ra_dir):
self.assertIn("Found LaunchBox with RetroArch at", proc.stdout)
self.assertIn("Found LaunchBox", proc.stdout)
self.assertIn("Found Retroarch at", proc.stdout)
self.assertIn(str(ra_dir), proc.stdout)
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertIn("Done.", proc.stdout)
@@ -642,7 +650,8 @@ class TestLaunchboxDetectionPowershell(unittest.TestCase):
" </Emulator>\n",
seed=seed,
)
self.assertIn("Found LaunchBox with RetroArch at", proc.stdout)
self.assertIn("Found LaunchBox", proc.stdout)
self.assertIn("Found Retroarch at", proc.stdout)
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertIn(str(ra_dir / "bios"), proc.stdout)
@@ -741,16 +750,74 @@ class TestAvailablePlatforms(unittest.TestCase):
manifests = {p.stem for p in (REPO_ROOT / "install").glob("*.json")}
self.assertEqual(set(install.AVAILABLE_PLATFORMS), manifests)
def test_powershell_installer_same_list(self):
def test_powershell_wrapper_pins_installer_hash(self):
content = (REPO_ROOT / "install.ps1").read_text()
match = re.search(r"\$available = @\(([^)]*)\)", content)
self.assertIsNotNone(match, "install.ps1 must define $available")
ps_list = set(re.findall(r'"([^"]+)"', match.group(1)))
self.assertEqual(ps_list, set(install.AVAILABLE_PLATFORMS))
match = re.search(r'\$defaultInstallSha256 = "([0-9a-f]{64})"', content)
self.assertIsNotNone(match, "install.ps1 must embed the installer SHA256")
expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
self.assertEqual(match.group(1), expected)
def test_powershell_normalizes_input(self):
def test_powershell_wrapper_rejects_non_https_bootstrap(self):
content = (REPO_ROOT / "install.ps1").read_text()
self.assertIn(".Trim().ToLower()", content)
self.assertIn('$uri.Scheme -ne "https"', content)
def test_shell_wrapper_pins_installer_hash(self):
content = (REPO_ROOT / "install.sh").read_text()
match = re.search(r'DEFAULT_INSTALL_SHA256="([0-9a-f]{64})"', content)
self.assertIsNotNone(match, "install.sh must embed the installer SHA256")
expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
self.assertEqual(match.group(1), expected)
def test_shell_one_liner_downloads_verifies_and_forwards_arguments(self):
scratch = REPO_ROOT / "tmp" / "tests"
scratch.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=scratch) as directory:
root = Path(directory)
# A piped script must not trust a same-named file in the caller's
# working directory; only a real local install.sh may use its peer.
(root / "install.py").write_text(
"raise SystemExit('untrusted cwd installer ran')\n",
encoding="utf-8",
)
fake_bin = root / "bin"
fake_bin.mkdir()
marker = root / "curl-called"
fake_curl = fake_bin / "curl"
fake_curl.write_text(
"#!/bin/sh\n"
"output=\n"
"while [ \"$#\" -gt 0 ]; do\n"
" if [ \"$1\" = --output ]; then output=$2; shift 2; else shift; fi\n"
"done\n"
"cp -- \"$FAKE_INSTALLER_SOURCE\" \"$output\"\n"
": > \"$FAKE_CURL_MARKER\"\n",
encoding="utf-8",
)
fake_curl.chmod(0o755)
env = os.environ.copy()
env.update(
PATH=f"{fake_bin}{os.pathsep}{env['PATH']}",
TMPDIR=str(root),
RETROBIOS_INSTALL_URL="https://example.invalid/install.py",
RETROBIOS_INSTALL_SHA256=hashlib.sha256(
(REPO_ROOT / "install.py").read_bytes()
).hexdigest(),
FAKE_INSTALLER_SOURCE=str(REPO_ROOT / "install.py"),
FAKE_CURL_MARKER=str(marker),
)
proc = subprocess.run(
["sh", "-s", "--", "--list-platforms"],
cwd=root,
env=env,
input=(REPO_ROOT / "install.sh").read_text(encoding="utf-8"),
capture_output=True,
text=True,
timeout=30,
)
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertTrue(marker.is_file(), "piped bootstrap did not download install.py")
self.assertIn("retroarch", proc.stdout)
if __name__ == "__main__":