mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-10 13:33:24 -05:00
Compare commits
13
Commits
e2863b13ca
...
022888e9ea
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
022888e9ea | ||
|
|
ee6e7558f9 | ||
|
|
626c012a6b | ||
|
|
361b958f41 | ||
|
|
d368b37165 | ||
|
|
11c4a6a945 | ||
|
|
b1d1eeab06 | ||
|
|
f2346d6001 | ||
|
|
9669ae06d2 | ||
|
|
7ff6ea20d1 | ||
|
|
26df60db75 | ||
|
|
3d7852cdb7 | ||
|
|
3adb34d322 |
No files matched your search
@@ -1,156 +0,0 @@
|
||||
name: Build & Release
|
||||
|
||||
# Releasing is a deliberate act, not a consequence of pushing. Cutting one is
|
||||
# a manual dispatch: someone decides the collection is in a state worth
|
||||
# publishing, and the rate limit below still guards against doing it twice by
|
||||
# accident.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_release:
|
||||
description: "Force release even if rate limited"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: build
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
# The release notes are built from `git log -15`, which returns a single
|
||||
# commit on the default shallow clone.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- run: pip install pyyaml
|
||||
|
||||
- name: Run tests
|
||||
run: python -m unittest discover tests
|
||||
|
||||
- name: Rate limit
|
||||
if: github.event.inputs.force_release != 'true'
|
||||
id: rate
|
||||
run: |
|
||||
LAST=$(gh release list --repo "${{ github.repository }}" --json createdAt -q '.[0].createdAt' 2>/dev/null || echo "")
|
||||
if [ -n "$LAST" ] && [ "$LAST" != "null" ]; then
|
||||
LAST_TS=$(date -d "$LAST" +%s 2>/dev/null || echo 0)
|
||||
DIFF=$(( ($(date +%s) - LAST_TS) / 86400 ))
|
||||
if [ "$DIFF" -lt 7 ]; then
|
||||
echo "Skipping: last release ${DIFF} days ago"
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Restore large files from release
|
||||
if: steps.rate.outputs.skip != 'true'
|
||||
run: |
|
||||
mkdir -p .cache/large
|
||||
gh release download large-files -D .cache/large/ 2>/dev/null || true
|
||||
python scripts/restore_large_files.py
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Refresh data directories
|
||||
if: steps.rate.outputs.skip != 'true'
|
||||
run: python scripts/refresh_data_dirs.py
|
||||
continue-on-error: true
|
||||
|
||||
- name: Build packs
|
||||
if: steps.rate.outputs.skip != 'true'
|
||||
run: python scripts/generate_pack.py --all --output-dir dist/
|
||||
|
||||
- name: Split oversized packs
|
||||
if: steps.rate.outputs.skip != 'true'
|
||||
run: |
|
||||
# GitHub releases cap assets at 2 GB. Use the .zip.001 volume
|
||||
# convention: 7-Zip and PeaZip open the .001 part directly,
|
||||
# unlike .partNN names which they reject as corrupt archives.
|
||||
for f in dist/*.zip; do
|
||||
size=$(stat -c%s "$f")
|
||||
if [ "$size" -gt 2000000000 ]; then
|
||||
split --bytes=1900M --numeric-suffixes=1 --suffix-length=3 "$f" "$f."
|
||||
rm "$f"
|
||||
echo "Split $(basename "$f") into $(ls "$f".* | wc -l) parts"
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Release
|
||||
if: steps.rate.outputs.skip != 'true'
|
||||
run: |
|
||||
DATE=$(date +%Y.%m.%d)
|
||||
EXISTING=$(gh release list --repo "${{ github.repository }}" \
|
||||
--json tagName -q ".[].tagName" | grep -c "^v${DATE}" || true)
|
||||
TAG="v${DATE}"
|
||||
[ "$EXISTING" -gt 0 ] && TAG="v${DATE}.$((EXISTING+1))"
|
||||
|
||||
CHANGES=$(git log --oneline -15 --no-merges \
|
||||
-- bios/ platforms/ emulators/ | sed 's/^/- /')
|
||||
TOTAL=$(python3 -c "import json; print(json.load(open('database.json'))['total_files'])")
|
||||
SIZE=$(python3 -c "import json; print(f'{json.load(open(\"database.json\"))[\"total_size\"]/1024/1024:.0f}')")
|
||||
PACKS=$(ls dist/*.zip dist/*.zip.001 2>/dev/null | while read f; do echo "- **$(basename $f)** ($(du -m "$f" | cut -f1) MB)"; done)
|
||||
|
||||
gh release create "$TAG" dist/*.zip* \
|
||||
--repo "${{ github.repository }}" \
|
||||
--title "BIOS Pack $TAG" \
|
||||
--notes "${TOTAL} files, ${SIZE} MB, verified checksums.
|
||||
|
||||
### Packs
|
||||
${PACKS}
|
||||
|
||||
### Install
|
||||
Download the pack matching your frontend, extract to the BIOS directory.
|
||||
|
||||
Packs over 2 GB are split into numbered volumes (.zip.001, .zip.002).
|
||||
Download every part, then either open the .001 file directly with
|
||||
7-Zip / PeaZip, or join the parts first:
|
||||
- Linux/macOS: \`cat PackName.zip.0* > PackName.zip\`
|
||||
- Windows (cmd): \`copy /b PackName.zip.001+PackName.zip.002 PackName.zip\`
|
||||
|
||||
| Platform | Extract to |
|
||||
|----------|------------|
|
||||
| RetroArch | system/ |
|
||||
| Lakka | /storage/system/ |
|
||||
| RetroPie | ~/RetroPie/BIOS/ |
|
||||
| Batocera | /userdata/bios/ |
|
||||
| Recalbox | /recalbox/share/bios/ |
|
||||
| RetroBat | bios/ |
|
||||
| RetroDECK | ~/retrodeck/ (the pack carries its own bios/) |
|
||||
| EmuDeck | ~/Emulation/bios/ |
|
||||
| ROCKNIX | /storage/roms/bios/ |
|
||||
| MiSTer FPGA | /media/fat/games/ |
|
||||
| BizHawk | Firmware/ |
|
||||
| RomM | bios/{platform_slug}/ |
|
||||
|
||||
### Changes
|
||||
${CHANGES}
|
||||
" \
|
||||
--latest
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Cleanup old releases
|
||||
if: steps.rate.outputs.skip != 'true'
|
||||
run: |
|
||||
gh release list --repo "${{ github.repository }}" --json tagName,createdAt \
|
||||
--jq 'sort_by(.createdAt) | reverse | .[].tagName' | \
|
||||
grep -v "^large-files$" | tail -n +4 | while read tag; do
|
||||
gh release delete "$tag" --repo "${{ github.repository }}" --yes --cleanup-tag
|
||||
done
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -3,7 +3,6 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/Abdess/retrobios/actions/workflows/build.yml"><img src="https://github.com/Abdess/retrobios/actions/workflows/build.yml/badge.svg" alt="Build"></a>
|
||||
<a href="https://github.com/Abdess/retrobios/actions/workflows/deploy-site.yml"><img src="https://github.com/Abdess/retrobios/actions/workflows/deploy-site.yml/badge.svg" alt="Site"></a>
|
||||
</p>
|
||||
|
||||
@@ -30,7 +29,7 @@ The script auto-detects your platform, downloads only missing files, and verifie
|
||||
|
||||
## Download BIOS packs
|
||||
|
||||
Pick your platform, download the ZIP, extract to the BIOS path.
|
||||
One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file, and `--target` narrows it to one machine; for a region or a bare minimum, build your own pack below.
|
||||
The size is what the files occupy once extracted; the ZIP itself downloads smaller, and anything over 2 GB arrives split into `.zip.001`, `.zip.002` volumes. Open the `.001` with 7-Zip or PeaZip, or join them first (`cat Pack.zip.0* > Pack.zip`, or `copy /b Pack.zip.001+Pack.zip.002 Pack.zip` on Windows).
|
||||
|
||||
| Platform | Extracted size | Extract to | Download |
|
||||
@@ -99,7 +98,7 @@ Checked by is the test your platform runs on its own, replicated here from its s
|
||||
Clone the repo and generate packs for any platform, emulator, or system:
|
||||
|
||||
```bash
|
||||
# Full platform pack
|
||||
# The pack of a platform, as released
|
||||
python scripts/generate_pack.py --platform retroarch --output-dir dist/
|
||||
python scripts/generate_pack.py --platform batocera --output-dir dist/
|
||||
|
||||
@@ -107,6 +106,10 @@ python scripts/generate_pack.py --platform batocera --output-dir dist/
|
||||
python scripts/generate_pack.py --emulator dolphin
|
||||
python scripts/generate_pack.py --system sony-playstation-2
|
||||
|
||||
# One region, best first, one file per system and region
|
||||
python scripts/generate_pack.py --platform retroarch --region us,eu,jp
|
||||
python scripts/generate_pack.py --platform retroarch --region us --one-per-slot
|
||||
|
||||
# List available emulators and systems
|
||||
python scripts/generate_pack.py --list-emulators
|
||||
python scripts/generate_pack.py --list-systems
|
||||
@@ -166,4 +169,4 @@ The scripts and tooling are released under the [MIT License](LICENSE).
|
||||
The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed.
|
||||
The reasoning, and where it is weakest, is in the [FAQ](https://abdess.github.io/retrobios/wiki/faq/#is-this-legal).
|
||||
|
||||
*Auto-generated on 2026-09-03T21:10:11Z*
|
||||
*Auto-generated on 2026-09-04T05:36:10Z*
|
||||
+1
-1
@@ -8,7 +8,7 @@ param(
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
$defaultInstallSha256 = "1cb76eb63b57e1390bfb4b6c2b24a5991819541414150aa88edc76d1d1188cd6"
|
||||
$defaultInstallSha256 = "7dbb5c48a834d31a470b1952a9b8659b4c1f49f8d37c709a4032a56abe495f27"
|
||||
$maximumInstallerBytes = 2MB
|
||||
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
|
||||
$temporary = $null
|
||||
|
||||
+9
-1
@@ -106,7 +106,15 @@ DEFAULT_DESTS = {
|
||||
|
||||
|
||||
def detect_os() -> str:
|
||||
"""Return normalized OS identifier."""
|
||||
"""Return normalized OS identifier.
|
||||
|
||||
RETROBIOS_OS names the platform outright (linux, wsl, windows, darwin),
|
||||
for a run that has to behave like another host: the PowerShell wrapper
|
||||
tests drive a Windows layout on a Linux runner.
|
||||
"""
|
||||
forced = os.environ.get("RETROBIOS_OS", "").strip().lower()
|
||||
if forced in ("linux", "wsl", "windows", "darwin"):
|
||||
return forced
|
||||
system = platform.system().lower()
|
||||
if system == "linux":
|
||||
proc_version = Path("/proc/version")
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ esac
|
||||
TEMP_INSTALLER=""
|
||||
TEMP_DIRECTORY=""
|
||||
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
DEFAULT_INSTALL_SHA256="1cb76eb63b57e1390bfb4b6c2b24a5991819541414150aa88edc76d1d1188cd6"
|
||||
DEFAULT_INSTALL_SHA256="7dbb5c48a834d31a470b1952a9b8659b4c1f49f8d37c709a4032a56abe495f27"
|
||||
MAX_INSTALLER_BYTES=2097152
|
||||
|
||||
cleanup() {
|
||||
|
||||
@@ -665,6 +665,7 @@ nav:
|
||||
- Wiki:
|
||||
- Overview: wiki/index.md
|
||||
- Getting started: wiki/getting-started.md
|
||||
- Installer: wiki/installer.md
|
||||
- FAQ: wiki/faq.md
|
||||
- Troubleshooting: wiki/troubleshooting.md
|
||||
- Architecture: wiki/architecture.md
|
||||
|
||||
+179
-118
@@ -3,11 +3,13 @@
|
||||
|
||||
Cross-platform tool (Linux/macOS/Windows) using only Python stdlib.
|
||||
|
||||
A pack over 2 GB is published as numbered volumes (`.zip.001`, `.zip.002`),
|
||||
so a platform is a group of assets rather than a single one.
|
||||
|
||||
Usage:
|
||||
python scripts/download.py --list # List platforms
|
||||
python scripts/download.py retroarch ~/path/ # Download pack
|
||||
python scripts/download.py --verify retroarch ~/path # Verify local files
|
||||
python scripts/download.py --info retroarch # Show coverage info
|
||||
python scripts/download.py --info retroarch # Show pack info
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -15,22 +17,63 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import compute_hashes, safe_extract_zip
|
||||
|
||||
GITHUB_API = "https://api.github.com"
|
||||
DEFAULT_API = "https://api.github.com"
|
||||
REPO = "Abdess/retrobios"
|
||||
PACK_SUFFIX = "_BIOS_Pack.zip"
|
||||
CHECKSUMS_ASSET = "SHA256SUMS.txt"
|
||||
STAGING_DIR = ".retrobios-download"
|
||||
MAX_CHECKSUMS_BYTES = 1 << 20
|
||||
CHUNK = 1 << 20
|
||||
|
||||
_VOLUME_RE = re.compile(rf"^(?P<base>.+{re.escape(PACK_SUFFIX)})\.(?P<index>\d+)$")
|
||||
_LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"}
|
||||
|
||||
|
||||
def _checked_url(value: str, label: str) -> str:
|
||||
"""Refuse a URL that is neither HTTPS nor loopback.
|
||||
|
||||
This endpoint names the assets and, through them, the bytes that land in
|
||||
the BIOS directory, so it is the trust anchor of the whole download.
|
||||
Plain HTTP to loopback stays allowed: it is how this is tested end to end.
|
||||
"""
|
||||
parsed = urllib.parse.urlparse(value)
|
||||
if parsed.scheme == "https":
|
||||
return value.rstrip("/")
|
||||
host = (parsed.hostname or "").lower()
|
||||
if parsed.scheme == "http" and host in _LOOPBACK_HOSTS:
|
||||
return value.rstrip("/")
|
||||
print(f"Error: {label} must use HTTPS, got {value!r}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
API = _checked_url(os.environ.get("RETROBIOS_API", DEFAULT_API), "RETROBIOS_API")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pack:
|
||||
"""A platform pack: one asset, or the volumes it was split into."""
|
||||
|
||||
name: str
|
||||
platform: str
|
||||
parts: tuple[dict, ...]
|
||||
size: int
|
||||
|
||||
|
||||
def get_latest_release() -> dict:
|
||||
"""Fetch latest release info from GitHub API."""
|
||||
url = f"{GITHUB_API}/repos/{REPO}/releases/latest"
|
||||
url = f"{API}/repos/{REPO}/releases/latest"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
@@ -49,31 +92,75 @@ def get_latest_release() -> dict:
|
||||
raise
|
||||
|
||||
|
||||
def list_platforms(release: dict) -> list[str]:
|
||||
"""List available platform packs from release assets."""
|
||||
platforms = []
|
||||
def group_packs(release: dict) -> list[Pack]:
|
||||
"""Group release assets into packs, volumes folded into their archive."""
|
||||
groups: dict[str, list[tuple[int, dict]]] = {}
|
||||
for asset in release.get("assets", []):
|
||||
name = asset["name"]
|
||||
if name.endswith("_BIOS_Pack.zip"):
|
||||
platform = name.replace("_BIOS_Pack.zip", "").replace("_", " ")
|
||||
platforms.append(platform)
|
||||
return sorted(platforms)
|
||||
volume = _VOLUME_RE.match(name)
|
||||
if volume:
|
||||
groups.setdefault(volume["base"], []).append((int(volume["index"]), asset))
|
||||
elif name.endswith(PACK_SUFFIX):
|
||||
groups.setdefault(name, []).append((0, asset))
|
||||
|
||||
packs = []
|
||||
for base, volumes in sorted(groups.items()):
|
||||
parts = tuple(asset for _, asset in sorted(volumes, key=lambda v: v[0]))
|
||||
packs.append(
|
||||
Pack(
|
||||
name=base,
|
||||
platform=base[: -len(PACK_SUFFIX)].replace("_", " "),
|
||||
parts=parts,
|
||||
size=sum(part.get("size", 0) for part in parts),
|
||||
)
|
||||
)
|
||||
return packs
|
||||
|
||||
|
||||
def find_asset(release: dict, platform: str) -> dict | None:
|
||||
"""Find the release asset for a specific platform."""
|
||||
normalized = platform.lower().replace(" ", "_").replace("-", "_")
|
||||
def list_platforms(release: dict) -> list[str]:
|
||||
"""List available platform packs from release assets."""
|
||||
return sorted(pack.platform for pack in group_packs(release))
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
asset_name = asset["name"].lower().replace(" ", "_").replace("-", "_")
|
||||
if normalized in asset_name and asset_name.endswith("_bios_pack.zip"):
|
||||
return asset
|
||||
|
||||
def _match_key(value: str) -> str:
|
||||
"""Letters and digits only: the platform is `misterfpga`, the asset MiSTer_FPGA."""
|
||||
return re.sub(r"[^a-z0-9]", "", value.lower())
|
||||
|
||||
|
||||
def find_pack(release: dict, platform: str) -> Pack | None:
|
||||
"""Find the pack for a platform name."""
|
||||
needle = _match_key(platform)
|
||||
if not needle:
|
||||
return None
|
||||
for pack in group_packs(release):
|
||||
if needle in _match_key(pack.name):
|
||||
return pack
|
||||
return None
|
||||
|
||||
|
||||
def download_file(url: str, dest: str, expected_size: int = 0):
|
||||
def fetch_checksums(release: dict) -> dict[str, str]:
|
||||
"""Read the published SHA-256 of each pack, keyed by archive name."""
|
||||
for asset in release.get("assets", []):
|
||||
if asset["name"] != CHECKSUMS_ASSET:
|
||||
continue
|
||||
url = _checked_url(asset["browser_download_url"], "asset URL")
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "retrobios-downloader/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
body = resp.read(MAX_CHECKSUMS_BYTES).decode("utf-8", "replace")
|
||||
sums = {}
|
||||
for line in body.splitlines():
|
||||
digest, _, name = line.strip().partition(" ")
|
||||
if digest and name:
|
||||
sums[name] = digest.lower()
|
||||
return sums
|
||||
return {}
|
||||
|
||||
|
||||
def download_file(url: str, dest: Path, expected_size: int = 0, label: str = ""):
|
||||
"""Download a file with progress indication."""
|
||||
url = _checked_url(url, "asset URL")
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "retrobios-downloader/1.0"}
|
||||
)
|
||||
@@ -94,95 +181,76 @@ def download_file(url: str, dest: str, expected_size: int = 0):
|
||||
pct = downloaded * 100 // total
|
||||
bar = "=" * (pct // 2) + " " * (50 - pct // 2)
|
||||
print(
|
||||
f"\r [{bar}] {pct}% ({downloaded:,}/{total:,})",
|
||||
f"\r {label}[{bar}] {pct}% ({downloaded:,}/{total:,})",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print()
|
||||
if expected_size and downloaded != expected_size:
|
||||
raise ValueError(f"downloaded {downloaded} bytes; expected {expected_size}")
|
||||
|
||||
|
||||
def extract_pack(zip_path: str, dest_dir: str):
|
||||
"""Extract a BIOS pack ZIP to destination."""
|
||||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||||
members = zf.namelist()
|
||||
print(f" Extracting {len(members)} files to {dest_dir}/")
|
||||
safe_extract_zip(zip_path, dest_dir)
|
||||
def join_volumes(parts: list[Path], dest: Path) -> None:
|
||||
"""Concatenate split volumes into one archive, streaming."""
|
||||
with open(dest, "wb") as out:
|
||||
for part in parts:
|
||||
with open(part, "rb") as src:
|
||||
shutil.copyfileobj(src, out, CHUNK)
|
||||
|
||||
|
||||
def verify_files(platform: str, dest_dir: str, release: dict):
|
||||
"""Verify local files against database.json from release."""
|
||||
db_asset = None
|
||||
for asset in release.get("assets", []):
|
||||
if asset["name"] == "database.json":
|
||||
db_asset = asset
|
||||
break
|
||||
def fetch_pack(pack: Pack, staging: Path, checksums: dict[str, str]) -> Path:
|
||||
"""Download every volume of a pack and return the assembled archive."""
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
volumes = []
|
||||
for index, part in enumerate(pack.parts, start=1):
|
||||
label = f"part {index}/{len(pack.parts)} " if len(pack.parts) > 1 else ""
|
||||
target = staging / part["name"]
|
||||
print(f"Downloading {part['name']} ({part.get('size', 0):,} bytes)...")
|
||||
download_file(part["browser_download_url"], target, part.get("size", 0), label)
|
||||
volumes.append(target)
|
||||
|
||||
if not db_asset:
|
||||
print("No database.json found in release assets. Cannot verify.")
|
||||
return
|
||||
archive = staging / pack.name
|
||||
if volumes != [archive]:
|
||||
if len(volumes) > 1:
|
||||
print(f"Joining {len(volumes)} parts into {pack.name}...")
|
||||
join_volumes(volumes, archive)
|
||||
for volume in volumes:
|
||||
volume.unlink()
|
||||
else:
|
||||
volumes[0].replace(archive)
|
||||
|
||||
import tempfile
|
||||
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".json", delete=False)
|
||||
tmp.close()
|
||||
|
||||
try:
|
||||
download_file(
|
||||
db_asset["browser_download_url"], tmp.name, db_asset.get("size", 0)
|
||||
)
|
||||
with open(tmp.name) as f:
|
||||
db = json.load(f)
|
||||
finally:
|
||||
os.unlink(tmp.name)
|
||||
|
||||
dest = Path(dest_dir)
|
||||
verified = 0
|
||||
missing = 0
|
||||
mismatched = 0
|
||||
|
||||
for sha1, entry in db.get("files", {}).items():
|
||||
name = entry["name"]
|
||||
found = False
|
||||
for local_file in dest.rglob(name):
|
||||
if local_file.is_file():
|
||||
local_sha1 = compute_hashes(local_file)["sha1"]
|
||||
|
||||
if local_sha1 == sha1:
|
||||
verified += 1
|
||||
found = True
|
||||
break
|
||||
else:
|
||||
mismatched += 1
|
||||
print(
|
||||
f" MISMATCH: {name} (expected {sha1[:12]}..., got {local_sha1[:12]}...)"
|
||||
)
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
missing += 1
|
||||
|
||||
total = verified + missing + mismatched
|
||||
print(f"\n Verified: {verified}/{total}")
|
||||
if missing:
|
||||
print(f" Missing: {missing}")
|
||||
if mismatched:
|
||||
print(f" Mismatched: {mismatched}")
|
||||
expected = checksums.get(pack.name)
|
||||
if expected:
|
||||
print("Checking the archive...")
|
||||
actual = compute_hashes(str(archive))["sha256"].lower()
|
||||
if actual != expected:
|
||||
archive.unlink()
|
||||
print(
|
||||
f"Error: checksum mismatch for {pack.name}\n"
|
||||
f" expected {expected}\n got {actual}\n"
|
||||
"Download the parts again; a truncated part gives this.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"No {CHECKSUMS_ASSET} in the release, skipping the checksum.")
|
||||
return archive
|
||||
|
||||
|
||||
def show_info(platform: str, release: dict):
|
||||
"""Show coverage information for a platform."""
|
||||
asset = find_asset(release, platform)
|
||||
if not asset:
|
||||
"""Show pack information for a platform."""
|
||||
pack = find_pack(release, platform)
|
||||
if not pack:
|
||||
print(f"Platform '{platform}' not found in release")
|
||||
return
|
||||
|
||||
print(f" Platform: {platform}")
|
||||
print(f" File: {asset['name']}")
|
||||
print(f" Size: {asset['size']:,} bytes ({asset['size'] / (1024 * 1024):.1f} MB)")
|
||||
print(f" Downloads: {asset.get('download_count', 'N/A')}")
|
||||
print(f" Updated: {asset.get('updated_at', 'N/A')}")
|
||||
print(f" Platform: {pack.platform}")
|
||||
print(f" File: {pack.name}")
|
||||
print(f" Parts: {len(pack.parts)} part{'s' if len(pack.parts) > 1 else ''}")
|
||||
print(f" Size: {pack.size:,} bytes ({pack.size / (1024 * 1024):.1f} MB)")
|
||||
for part in pack.parts:
|
||||
print(f" {part['name']} ({part.get('size', 0):,} bytes)")
|
||||
|
||||
|
||||
def main():
|
||||
@@ -193,14 +261,14 @@ def main():
|
||||
Examples:
|
||||
%(prog)s --list List available platforms
|
||||
%(prog)s retroarch ~/RetroArch/system Download RetroArch pack
|
||||
%(prog)s --verify retroarch ~/path Verify local files
|
||||
%(prog)s --info retroarch Show pack info
|
||||
|
||||
To check files already in place, use: python install.py --check
|
||||
""",
|
||||
)
|
||||
parser.add_argument("platform", nargs="?", help="Platform name")
|
||||
parser.add_argument("dest", nargs="?", help="Destination directory")
|
||||
parser.add_argument("--list", action="store_true", help="List available platforms")
|
||||
parser.add_argument("--verify", action="store_true", help="Verify existing files")
|
||||
parser.add_argument("--info", action="store_true", help="Show platform info")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -220,7 +288,8 @@ Examples:
|
||||
OSError,
|
||||
json.JSONDecodeError,
|
||||
) as e:
|
||||
print(f"Error: {e}")
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
if not args.platform:
|
||||
@@ -229,43 +298,35 @@ Examples:
|
||||
try:
|
||||
release = get_latest_release()
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e:
|
||||
print(f"Error fetching release info: {e}")
|
||||
print(f"Error fetching release info: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.info:
|
||||
show_info(args.platform, release)
|
||||
return
|
||||
|
||||
if args.verify:
|
||||
if not args.dest:
|
||||
parser.error("Destination directory required for --verify")
|
||||
verify_files(args.platform, args.dest, release)
|
||||
return
|
||||
|
||||
if not args.dest:
|
||||
parser.error("Destination directory required")
|
||||
|
||||
asset = find_asset(release, args.platform)
|
||||
if not asset:
|
||||
pack = find_pack(release, args.platform)
|
||||
if not pack:
|
||||
print(f"Platform '{args.platform}' not found in release.")
|
||||
print("Available:", ", ".join(list_platforms(release)))
|
||||
sys.exit(1)
|
||||
|
||||
import tempfile
|
||||
dest = Path(os.path.expanduser(args.dest))
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fd, zip_path = tempfile.mkstemp(suffix=".zip")
|
||||
os.close(fd)
|
||||
|
||||
print(f"Downloading {asset['name']} ({asset['size']:,} bytes)...")
|
||||
download_file(asset["browser_download_url"], zip_path, asset["size"])
|
||||
|
||||
dest = os.path.expanduser(args.dest)
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
|
||||
print(f"Extracting to {dest}/...")
|
||||
extract_pack(zip_path, dest)
|
||||
|
||||
os.unlink(zip_path)
|
||||
# Staged inside the destination: a pack is gigabytes, and that is the
|
||||
# filesystem the user picked for them. The system temp directory is a RAM
|
||||
# disk on the appliances these packs target.
|
||||
staging = dest / STAGING_DIR
|
||||
try:
|
||||
archive = fetch_pack(pack, staging, fetch_checksums(release))
|
||||
print(f"Extracting to {dest}/...")
|
||||
safe_extract_zip(str(archive), str(dest))
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
print("Done!")
|
||||
|
||||
|
||||
|
||||
+145
-23
@@ -1,16 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download BIOS pack from GitHub Releases (Linux/macOS one-liner compatible)
|
||||
#
|
||||
# A pack over 2 GB is published as numbered volumes (.zip.001, .zip.002),
|
||||
# which are downloaded, joined and checked here.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/download.sh retroarch ~/RetroArch/system/
|
||||
# bash scripts/download.sh --list
|
||||
#
|
||||
# Requires: curl, unzip, jq (optional, for --list)
|
||||
# Requires: curl, unzip
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO="Abdess/retrobios"
|
||||
API="https://api.github.com/repos/${REPO}/releases/latest"
|
||||
API_BASE="${RETROBIOS_API:-https://api.github.com}"
|
||||
|
||||
# This endpoint names the assets and so the bytes that land in the BIOS
|
||||
# directory. Plain HTTP stays allowed to loopback, which is how the script
|
||||
# is exercised end to end.
|
||||
case "$API_BASE" in
|
||||
https://*) ;;
|
||||
http://127.0.0.1[:/]*|http://localhost[:/]*|"http://[::1]"[:/]*) ;;
|
||||
*)
|
||||
echo "Error: RETROBIOS_API must use HTTPS, got '$API_BASE'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
API="${API_BASE%/}/repos/${REPO}/releases/latest"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <platform> <destination>"
|
||||
@@ -25,53 +41,159 @@ usage() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# One asset URL per line, in the order the release lists them.
|
||||
asset_urls() {
|
||||
printf '%s' "$1" | tr ',' '\n' |
|
||||
sed -n 's/.*"browser_download_url"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p'
|
||||
}
|
||||
|
||||
# The archive name behind each asset, volumes folded into their archive.
|
||||
pack_names() {
|
||||
asset_urls "$1" | sed 's#.*/##' |
|
||||
sed -n 's/\(.*_BIOS_Pack\.zip\)\(\.[0-9][0-9]*\)\{0,1\}$/\1/p' |
|
||||
LC_ALL=C sort -u
|
||||
}
|
||||
|
||||
platform_names() {
|
||||
pack_names "$1" | sed 's/_BIOS_Pack\.zip$//' | tr '_' ' '
|
||||
}
|
||||
|
||||
# Letters and digits only: the platform is `misterfpga`, the asset MiSTer_FPGA.
|
||||
normalize() {
|
||||
printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9'
|
||||
}
|
||||
|
||||
fetch_release() {
|
||||
curl -fsSL "$API"
|
||||
}
|
||||
|
||||
list_platforms() {
|
||||
echo "Fetching available platforms..."
|
||||
if command -v jq &>/dev/null; then
|
||||
curl -sL "$API" | jq -r '.assets[].name' | grep '_BIOS_Pack.zip' | sed 's/_BIOS_Pack.zip//' | tr '_' ' '
|
||||
else
|
||||
curl -sL "$API" | grep -oP '"name":\s*"\K[^"]*_BIOS_Pack\.zip' | sed 's/_BIOS_Pack.zip//' | tr '_' ' '
|
||||
fi
|
||||
platform_names "$(fetch_release)"
|
||||
}
|
||||
|
||||
download_pack() {
|
||||
local platform="$1"
|
||||
local dest="$2"
|
||||
local normalized
|
||||
normalized=$(echo "$platform" | tr ' ' '_' | tr '[:upper:]' '[:lower:]')
|
||||
normalized=$(normalize "$platform")
|
||||
|
||||
echo "Fetching release info..."
|
||||
local release_json
|
||||
release_json=$(curl -sL "$API")
|
||||
release_json=$(fetch_release)
|
||||
|
||||
# Find matching asset URL
|
||||
local download_url
|
||||
download_url=$(echo "$release_json" | grep -oP "\"browser_download_url\":\s*\"[^\"]*${normalized}[^\"]*_BIOS_Pack\.zip\"" | head -1 | grep -oP 'https://[^"]+')
|
||||
local archive=""
|
||||
local candidate
|
||||
while read -r candidate; do
|
||||
[ -n "$candidate" ] || continue
|
||||
case "$(normalize "$candidate")" in
|
||||
*"$normalized"*)
|
||||
archive="$candidate"
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done <<EOF
|
||||
$(pack_names "$release_json")
|
||||
EOF
|
||||
|
||||
if [[ -z "$download_url" ]]; then
|
||||
if [ -z "$archive" ]; then
|
||||
echo "Error: Platform '$platform' not found in latest release."
|
||||
echo "Available platforms:"
|
||||
list_platforms
|
||||
platform_names "$release_json"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local filename
|
||||
filename=$(basename "$download_url")
|
||||
local volumes
|
||||
volumes=$(asset_urls "$release_json" |
|
||||
grep -E "/${archive//./\\.}(\.[0-9]+)?$" | LC_ALL=C sort)
|
||||
if [ -z "$volumes" ]; then
|
||||
echo "Error: no asset found for ${archive}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local tmpfile
|
||||
tmpfile=$(mktemp "/tmp/${filename}.XXXXXX")
|
||||
# Staged inside the destination: a pack is gigabytes, and /tmp is a RAM
|
||||
# disk on the appliances these packs target.
|
||||
mkdir -p "$dest"
|
||||
local staging="${dest%/}/.retrobios-download"
|
||||
rm -rf "$staging"
|
||||
mkdir -p "$staging"
|
||||
trap 'rm -rf "$staging"' EXIT
|
||||
|
||||
echo "Downloading ${filename}..."
|
||||
curl -L --progress-bar -o "$tmpfile" "$download_url"
|
||||
local count
|
||||
count=$(printf '%s\n' "$volumes" | wc -l | tr -d ' ')
|
||||
|
||||
local index=0
|
||||
local url name
|
||||
while read -r url; do
|
||||
[ -n "$url" ] || continue
|
||||
index=$((index + 1))
|
||||
name=$(basename "$url")
|
||||
if [ "$count" -gt 1 ]; then
|
||||
echo "Downloading ${name} (part ${index}/${count})..."
|
||||
else
|
||||
echo "Downloading ${name}..."
|
||||
fi
|
||||
curl -fL --progress-bar -o "${staging}/${name}" "$url"
|
||||
done <<EOF
|
||||
$volumes
|
||||
EOF
|
||||
|
||||
if [ "$count" -gt 1 ]; then
|
||||
echo "Joining ${count} parts into ${archive}..."
|
||||
# split(1) writes plain byte ranges, so concatenation rebuilds the ZIP.
|
||||
cat "${staging}/${archive}".[0-9][0-9][0-9] > "${staging}/${archive}"
|
||||
rm -f "${staging}/${archive}".[0-9][0-9][0-9]
|
||||
fi
|
||||
|
||||
verify_checksum "$release_json" "$staging" "$archive"
|
||||
|
||||
echo "Extracting to ${dest}/..."
|
||||
mkdir -p "$dest"
|
||||
unzip -o -q "$tmpfile" -d "$dest"
|
||||
unzip -o -q "${staging}/${archive}" -d "$dest"
|
||||
|
||||
rm -f "$tmpfile"
|
||||
rm -rf "$staging"
|
||||
trap - EXIT
|
||||
echo "Done! BIOS files extracted to ${dest}/"
|
||||
}
|
||||
|
||||
verify_checksum() {
|
||||
local release_json="$1" staging="$2" archive="$3"
|
||||
|
||||
local sums_url
|
||||
sums_url=$(asset_urls "$release_json" | grep -E '/SHA256SUMS\.txt$' | head -1)
|
||||
if [ -z "$sums_url" ]; then
|
||||
echo "No SHA256SUMS.txt in the release, skipping the checksum."
|
||||
return 0
|
||||
fi
|
||||
|
||||
local hasher
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
hasher="sha256sum"
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
hasher="shasum -a 256"
|
||||
else
|
||||
echo "No sha256sum available, skipping the checksum."
|
||||
return 0
|
||||
fi
|
||||
|
||||
local expected
|
||||
expected=$(curl -fsSL "$sums_url" | awk -v name="$archive" '$2 == name {print $1}')
|
||||
if [ -z "$expected" ]; then
|
||||
echo "No checksum published for ${archive}, skipping."
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Checking the archive..."
|
||||
local actual
|
||||
actual=$($hasher "${staging}/${archive}" | cut -d' ' -f1)
|
||||
if [ "$actual" != "$expected" ]; then
|
||||
echo "Error: checksum mismatch for ${archive}" >&2
|
||||
echo " expected ${expected}" >&2
|
||||
echo " got ${actual}" >&2
|
||||
echo "Download the parts again; a truncated part gives this." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main
|
||||
case "${1:-}" in
|
||||
--list|-l)
|
||||
|
||||
@@ -261,8 +261,6 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"</p>",
|
||||
"",
|
||||
'<p align="center">',
|
||||
' <a href="https://github.com/Abdess/retrobios/actions/workflows/build.yml">'
|
||||
'<img src="https://github.com/Abdess/retrobios/actions/workflows/build.yml/badge.svg" alt="Build"></a>',
|
||||
' <a href="https://github.com/Abdess/retrobios/actions/workflows/deploy-site.yml">'
|
||||
'<img src="https://github.com/Abdess/retrobios/actions/workflows/deploy-site.yml/badge.svg" alt="Site"></a>',
|
||||
"</p>",
|
||||
@@ -298,7 +296,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"",
|
||||
"## Download BIOS packs",
|
||||
"",
|
||||
"Pick your platform, download the ZIP, extract to the BIOS path.",
|
||||
"One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file, and `--target` narrows it to one machine; for a region or a bare minimum, build your own pack below.",
|
||||
"The size is what the files occupy once extracted; the ZIP itself"
|
||||
" downloads smaller, and anything over 2 GB arrives split into"
|
||||
" `.zip.001`, `.zip.002` volumes. Open the `.001` with 7-Zip or"
|
||||
@@ -492,7 +490,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"Clone the repo and generate packs for any platform, emulator, or system:",
|
||||
"",
|
||||
"```bash",
|
||||
"# Full platform pack",
|
||||
"# The pack of a platform, as released",
|
||||
"python scripts/generate_pack.py --platform retroarch --output-dir dist/",
|
||||
"python scripts/generate_pack.py --platform batocera --output-dir dist/",
|
||||
"",
|
||||
@@ -500,6 +498,10 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"python scripts/generate_pack.py --emulator dolphin",
|
||||
"python scripts/generate_pack.py --system sony-playstation-2",
|
||||
"",
|
||||
"# One region, best first, one file per system and region",
|
||||
"python scripts/generate_pack.py --platform retroarch --region us,eu,jp",
|
||||
"python scripts/generate_pack.py --platform retroarch --region us --one-per-slot",
|
||||
"",
|
||||
"# List available emulators and systems",
|
||||
"python scripts/generate_pack.py --list-emulators",
|
||||
"python scripts/generate_pack.py --list-systems",
|
||||
|
||||
+19
-19
@@ -3389,7 +3389,7 @@ curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh |
|
||||
**Windows (PowerShell):**
|
||||
|
||||
```powershell
|
||||
iwr -useb https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1 | iex
|
||||
irm https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1 | iex
|
||||
```
|
||||
|
||||
That is the complete default flow. Extra copies into detected standalone
|
||||
@@ -3397,6 +3397,9 @@ emulator directories are deliberately opt-in with `--standalone-copies`, so
|
||||
automatic setup never writes outside the selected platform tree unexpectedly.
|
||||
Use `python install.py --check` from a checkout for a read-only verification.
|
||||
|
||||
Options, environment overrides, how each platform is detected and what the
|
||||
installer is allowed to write: [Installer](wiki/installer.md).
|
||||
|
||||
---
|
||||
|
||||
## Manual download
|
||||
@@ -3476,26 +3479,22 @@ extract the whole archive directly. To join the parts manually instead:
|
||||
|
||||
---
|
||||
|
||||
## Full pack or Platform pack?
|
||||
## One pack per platform
|
||||
|
||||
Each platform has two pack types on the [latest release]({rel}).
|
||||
The [latest release]({rel}) carries one pack for each platform, and that
|
||||
pack holds everything the platform runs: the BIOS list it publishes, plus
|
||||
every file its emulator cores load, read from their source code. There is no
|
||||
lighter variant to choose, because a pack that leaves out a file a core needs
|
||||
means a game that does not boot, with no message saying why. It is not a
|
||||
guarantee either: source profiles can document files nobody has dumped or
|
||||
that only the user can provide, all of them visible in the
|
||||
[gap analysis](gaps.md).
|
||||
|
||||
**Full pack** (recommended)
|
||||
|
||||
Contains the platform's own BIOS list plus all files needed by each
|
||||
emulator core available on that platform. This covers alternate cores,
|
||||
optional firmware that improves accuracy, and edge cases. Larger download,
|
||||
and the best default when storage is not constrained. It is not a guarantee:
|
||||
source profiles can document missing, user-provided or unsourceable files, all
|
||||
of which remain visible in the [gap analysis](gaps.md).
|
||||
|
||||
**Platform pack**
|
||||
|
||||
Contains only the files the platform officially checks for. Much smaller
|
||||
download. Good for limited storage (SD cards, handhelds) or setups that
|
||||
only use default cores.
|
||||
|
||||
When in doubt, take the full pack and verify it against the platform page.
|
||||
Want less than everything? The installer takes `--target switch` to install
|
||||
only what one machine's cores need. From a clone of the repository,
|
||||
`python scripts/generate_pack.py --platform retroarch --region us` keeps one
|
||||
BIOS per region and `--required-only` the bare minimum each core needs to
|
||||
start; `--help` lists every way to build your own.
|
||||
|
||||
---
|
||||
|
||||
@@ -3574,6 +3573,7 @@ def generate_mkdocs_nav(
|
||||
wiki_nav = [
|
||||
{"Overview": "wiki/index.md"},
|
||||
{"Getting started": "wiki/getting-started.md"},
|
||||
{"Installer": "wiki/installer.md"},
|
||||
{"FAQ": "wiki/faq.md"},
|
||||
{"Troubleshooting": "wiki/troubleshooting.md"},
|
||||
{"Architecture": "wiki/architecture.md"},
|
||||
|
||||
@@ -14,13 +14,10 @@ from hashing import compute_hashes
|
||||
|
||||
|
||||
LARGE_FILES_RELEASE = "large-files"
|
||||
|
||||
LARGE_FILES_RELEASE = "large-files"
|
||||
LARGE_FILES_REPO = "Abdess/retrobios"
|
||||
|
||||
LARGE_FILES_REPO = "Abdess/retrobios"
|
||||
LARGE_FILES_CACHE = ".cache/large"
|
||||
|
||||
|
||||
def fetch_large_file(
|
||||
name: str,
|
||||
dest_dir: str = LARGE_FILES_CACHE,
|
||||
|
||||
@@ -235,6 +235,17 @@ def main():
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# A second run on the same output directory is refused before any work:
|
||||
# the database rebuild alone takes minutes, and the reader holding the
|
||||
# directory would otherwise see the answer only after all of it.
|
||||
if not args.skip_packs and Path(args.output_dir).is_dir():
|
||||
try:
|
||||
with artifact_lock(args.output_dir):
|
||||
pass
|
||||
except ArtifactLockBusy as exc:
|
||||
print(f"ERROR: {exc}")
|
||||
sys.exit(1)
|
||||
|
||||
results = {}
|
||||
all_ok = True
|
||||
total_start = time.monotonic()
|
||||
|
||||
@@ -9,8 +9,6 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import urllib.request
|
||||
WIKI_SRC_DIR = "wiki" # manually maintained wiki sources
|
||||
SYSTEM_ICON_BASE = "https://raw.githubusercontent.com/libretro/retroarch-assets/master/xmb/systematic/png"
|
||||
|
||||
SYSTEM_ICON_BASE = "https://raw.githubusercontent.com/libretro/retroarch-assets/master/xmb/systematic/png"
|
||||
ICON_CACHE_PATH = Path(".cache") / "system_icons.json"
|
||||
|
||||
|
||||
+2
-4
@@ -75,24 +75,22 @@ def build_zip_contents_index(db: dict, max_entry_size: int = 512 * 1024 * 1024)
|
||||
_zip_contents_cache = (fingerprint, index)
|
||||
return index
|
||||
|
||||
|
||||
MAX_ZIP_MEMBERS = 100_000
|
||||
|
||||
MAX_ZIP_MEMBER_SIZE = 8 * 1024 * 1024 * 1024
|
||||
|
||||
MAX_ZIP_MEMBER_SIZE = 8 * 1024 * 1024 * 1024
|
||||
# The largest generated pack is already ~5 GB uncompressed and the collection
|
||||
# only grows; this bounds a malicious archive without capping a real one.
|
||||
MAX_ZIP_TOTAL_SIZE = 64 * 1024 * 1024 * 1024
|
||||
|
||||
MAX_ZIP_TOTAL_SIZE = 64 * 1024 * 1024 * 1024
|
||||
# DEFLATE cannot exceed roughly 1,032:1, so this rejects a declared ratio no
|
||||
# real DEFLATE member can reach. Methods with a higher ceiling (bzip2, LZMA)
|
||||
# are exempt and bounded by the per-member and per-archive size limits alone.
|
||||
MAX_ZIP_COMPRESSION_RATIO = 1_100
|
||||
|
||||
MAX_ZIP_COMPRESSION_RATIO = 1_100
|
||||
_BOUNDED_RATIO_METHODS = (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED)
|
||||
|
||||
|
||||
def safe_extract_zip(
|
||||
zip_path: str,
|
||||
dest_dir: str,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
@@ -724,12 +725,6 @@ class CheckoutCompletenessRegressions(unittest.TestCase):
|
||||
self.assertLess(self._index(steps, "restore_large_files.py"), generate)
|
||||
self.assertLess(self._index(steps, "refresh_data_dirs.py"), generate)
|
||||
|
||||
def test_pack_build_completes_the_checkout_before_building(self):
|
||||
steps = self._steps("build.yml", "release")
|
||||
build = self._index(steps, "Build packs")
|
||||
self.assertLess(self._index(steps, "restore_large_files.py"), build)
|
||||
self.assertLess(self._index(steps, "refresh_data_dirs.py"), build)
|
||||
|
||||
def test_restore_matches_assets_by_content_not_by_name(self):
|
||||
from scripts.restore_large_files import restore
|
||||
|
||||
@@ -1015,3 +1010,36 @@ class ContributorsSurviveAFailedRequest(unittest.TestCase):
|
||||
text.count('<a href="https://github.com/'), 0,
|
||||
"the section is present but empty",
|
||||
)
|
||||
|
||||
|
||||
class ModuleConstantsDeclaredOnce(unittest.TestCase):
|
||||
"""Splitting common.py into modules emitted some constants twice.
|
||||
|
||||
The values matched, so nothing broke, but the second assignment orphans
|
||||
the comment written above the first and leaves two lines to keep in step
|
||||
the day a value changes.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _redeclared(path: Path) -> list[str]:
|
||||
seen: dict[str, str] = {}
|
||||
again = []
|
||||
for node in ast.parse(path.read_text()).body:
|
||||
if not isinstance(node, ast.Assign) or len(node.targets) != 1:
|
||||
continue
|
||||
target = node.targets[0]
|
||||
if not isinstance(target, ast.Name):
|
||||
continue
|
||||
value = ast.unparse(node.value)
|
||||
if seen.get(target.id) == value:
|
||||
again.append(target.id)
|
||||
seen[target.id] = value
|
||||
return again
|
||||
|
||||
def test_no_module_assigns_the_same_constant_twice(self):
|
||||
offenders = {}
|
||||
for path in sorted(ROOT.glob("scripts/**/*.py")) + [ROOT / "install.py"]:
|
||||
again = self._redeclared(path)
|
||||
if again:
|
||||
offenders[path.name] = again
|
||||
self.assertEqual(offenders, {})
|
||||
@@ -0,0 +1,425 @@
|
||||
"""Tests for the release pack downloaders.
|
||||
|
||||
Packs over 2 GB are published as numbered volumes (`.zip.001`, `.zip.002`),
|
||||
so a downloader that expects one asset per platform finds nothing for most
|
||||
of them. These tests pin the grouping, the join and the staging location.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import hashlib
|
||||
import http.server
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"download", REPO_ROOT / "scripts" / "download.py"
|
||||
)
|
||||
download = importlib.util.module_from_spec(_spec)
|
||||
# Registered before execution: a dataclass resolves its annotations through
|
||||
# sys.modules and cannot be built from a module that is not there yet.
|
||||
sys.modules["download"] = download
|
||||
_spec.loader.exec_module(download)
|
||||
|
||||
SHELL = REPO_ROOT / "scripts" / "download.sh"
|
||||
|
||||
# A slice of a real release: two whole packs, three split, and the checksums.
|
||||
RELEASE_ASSETS = [
|
||||
"Batocera_43.1_BIOS_Pack.zip.001",
|
||||
"Batocera_43.1_BIOS_Pack.zip.002",
|
||||
"BizHawk_2.11.1_BIOS_Pack.zip",
|
||||
"EmuDeck_2.3.8_BIOS_Pack.zip",
|
||||
"RetroArch_Lakka_v1.22.2_BIOS_Pack.zip.001",
|
||||
"RetroArch_Lakka_v1.22.2_BIOS_Pack.zip.002",
|
||||
"RetroDECK_0.10.9b_BIOS_Pack.zip.001",
|
||||
"RetroDECK_0.10.9b_BIOS_Pack.zip.002",
|
||||
"RetroDECK_0.10.9b_BIOS_Pack.zip.003",
|
||||
"SHA256SUMS.txt",
|
||||
]
|
||||
|
||||
|
||||
def _closed_port() -> int:
|
||||
"""A loopback port nothing listens on, so connecting is refused at once."""
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
|
||||
class _QuietHandler(http.server.SimpleHTTPRequestHandler):
|
||||
def log_message(self, *args): # keep the test output pristine
|
||||
pass
|
||||
|
||||
|
||||
def _release(names, base="https://example.invalid", size=10):
|
||||
return {
|
||||
"assets": [
|
||||
{
|
||||
"name": name,
|
||||
"size": size,
|
||||
"browser_download_url": f"{base}/assets/{name}",
|
||||
}
|
||||
for name in names
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
class TestPackGrouping(unittest.TestCase):
|
||||
def test_whole_pack_has_one_part(self):
|
||||
packs = download.group_packs(_release(["EmuDeck_2.3.8_BIOS_Pack.zip"]))
|
||||
self.assertEqual([p.name for p in packs], ["EmuDeck_2.3.8_BIOS_Pack.zip"])
|
||||
self.assertEqual(len(packs[0].parts), 1)
|
||||
|
||||
def test_volumes_group_into_one_pack(self):
|
||||
packs = download.group_packs(
|
||||
_release(
|
||||
[
|
||||
"Batocera_43.1_BIOS_Pack.zip.001",
|
||||
"Batocera_43.1_BIOS_Pack.zip.002",
|
||||
]
|
||||
)
|
||||
)
|
||||
self.assertEqual([p.name for p in packs], ["Batocera_43.1_BIOS_Pack.zip"])
|
||||
self.assertEqual(
|
||||
[part["name"] for part in packs[0].parts],
|
||||
[
|
||||
"Batocera_43.1_BIOS_Pack.zip.001",
|
||||
"Batocera_43.1_BIOS_Pack.zip.002",
|
||||
],
|
||||
)
|
||||
|
||||
def test_pack_size_is_the_sum_of_its_volumes(self):
|
||||
packs = download.group_packs(
|
||||
_release(
|
||||
[
|
||||
"Batocera_43.1_BIOS_Pack.zip.001",
|
||||
"Batocera_43.1_BIOS_Pack.zip.002",
|
||||
],
|
||||
size=7,
|
||||
)
|
||||
)
|
||||
self.assertEqual(packs[0].size, 14)
|
||||
|
||||
def test_volumes_order_numerically_not_lexically(self):
|
||||
packs = download.group_packs(
|
||||
_release(
|
||||
[
|
||||
"Big_BIOS_Pack.zip.010",
|
||||
"Big_BIOS_Pack.zip.009",
|
||||
"Big_BIOS_Pack.zip.001",
|
||||
]
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
[part["name"] for part in packs[0].parts],
|
||||
["Big_BIOS_Pack.zip.001", "Big_BIOS_Pack.zip.009", "Big_BIOS_Pack.zip.010"],
|
||||
)
|
||||
|
||||
def test_other_assets_are_not_packs(self):
|
||||
packs = download.group_packs(_release(["SHA256SUMS.txt", "database.json"]))
|
||||
self.assertEqual(packs, [])
|
||||
|
||||
def test_list_platforms_names_split_packs(self):
|
||||
names = download.list_platforms(_release(RELEASE_ASSETS))
|
||||
self.assertIn("Batocera 43.1", names)
|
||||
self.assertIn("RetroDECK 0.10.9b", names)
|
||||
self.assertEqual(len(names), 5)
|
||||
|
||||
def test_find_pack_resolves_a_split_platform(self):
|
||||
pack = download.find_pack(_release(RELEASE_ASSETS), "batocera")
|
||||
self.assertIsNotNone(pack)
|
||||
self.assertEqual(pack.name, "Batocera_43.1_BIOS_Pack.zip")
|
||||
self.assertEqual(len(pack.parts), 2)
|
||||
|
||||
def test_find_pack_maps_retroarch_to_the_lakka_asset(self):
|
||||
pack = download.find_pack(_release(RELEASE_ASSETS), "retroarch")
|
||||
self.assertEqual(pack.name, "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip")
|
||||
|
||||
def test_find_pack_returns_none_for_an_unknown_platform(self):
|
||||
self.assertIsNone(download.find_pack(_release(RELEASE_ASSETS), "nintendo"))
|
||||
|
||||
def test_find_pack_resolves_a_platform_id_that_drops_separators(self):
|
||||
# The platform is `misterfpga` everywhere else; the asset is MiSTer_FPGA.
|
||||
pack = download.find_pack(
|
||||
_release(RELEASE_ASSETS + ["MiSTer_FPGA_2026-08-29_BIOS_Pack.zip"]),
|
||||
"misterfpga",
|
||||
)
|
||||
self.assertIsNotNone(pack)
|
||||
self.assertEqual(pack.name, "MiSTer_FPGA_2026-08-29_BIOS_Pack.zip")
|
||||
|
||||
|
||||
class TestJoinVolumes(unittest.TestCase):
|
||||
def test_joined_volumes_reproduce_the_archive(self):
|
||||
tmp = Path(tempfile.mkdtemp())
|
||||
archive = tmp / "pack.zip"
|
||||
with zipfile.ZipFile(archive, "w") as zf:
|
||||
zf.writestr("bios/scph5501.bin", b"\x01\x02" * 5000)
|
||||
zf.writestr("bios/dc_boot.bin", b"\x03\x04" * 5000)
|
||||
raw = archive.read_bytes()
|
||||
cut = len(raw) // 3
|
||||
parts = []
|
||||
for index, start in enumerate(range(0, len(raw), cut), start=1):
|
||||
part = tmp / f"pack.zip.{index:03d}"
|
||||
part.write_bytes(raw[start : start + cut])
|
||||
parts.append(part)
|
||||
self.assertGreater(len(parts), 1)
|
||||
|
||||
joined = tmp / "joined.zip"
|
||||
download.join_volumes(parts, joined)
|
||||
|
||||
self.assertEqual(joined.read_bytes(), raw)
|
||||
with zipfile.ZipFile(joined) as zf:
|
||||
self.assertEqual(zf.testzip(), None)
|
||||
|
||||
|
||||
class ReleaseServer:
|
||||
"""Serves a release index and its assets over loopback."""
|
||||
|
||||
def __init__(self, pack_name: str, payload: dict[str, bytes], volumes: int):
|
||||
self.root = Path(tempfile.mkdtemp())
|
||||
(self.root / "assets").mkdir()
|
||||
archive = self.root / "assets" / pack_name
|
||||
with zipfile.ZipFile(archive, "w") as zf:
|
||||
for name, data in payload.items():
|
||||
zf.writestr(name, data)
|
||||
raw = archive.read_bytes()
|
||||
self.digest = hashlib.sha256(raw).hexdigest()
|
||||
archive.unlink()
|
||||
|
||||
self.names: list[str] = []
|
||||
if volumes == 1:
|
||||
(self.root / "assets" / pack_name).write_bytes(raw)
|
||||
self.names.append(pack_name)
|
||||
else:
|
||||
cut = len(raw) // volumes + 1
|
||||
for index, start in enumerate(range(0, len(raw), cut), start=1):
|
||||
name = f"{pack_name}.{index:03d}"
|
||||
(self.root / "assets" / name).write_bytes(raw[start : start + cut])
|
||||
self.names.append(name)
|
||||
|
||||
(self.root / "assets" / "SHA256SUMS.txt").write_text(
|
||||
f"{self.digest} {pack_name}\n"
|
||||
)
|
||||
|
||||
handler = functools.partial(_QuietHandler, directory=str(self.root))
|
||||
self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}"
|
||||
|
||||
index = self.root / "repos" / "Abdess" / "retrobios" / "releases"
|
||||
index.mkdir(parents=True)
|
||||
assets = [
|
||||
{
|
||||
"name": name,
|
||||
"size": (self.root / "assets" / name).stat().st_size,
|
||||
"browser_download_url": f"{self.base}/assets/{name}",
|
||||
}
|
||||
for name in self.names + ["SHA256SUMS.txt"]
|
||||
]
|
||||
(index / "latest").write_text(json.dumps({"assets": assets}))
|
||||
threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
|
||||
|
||||
def corrupt_last_volume(self) -> None:
|
||||
target = self.root / "assets" / self.names[-1]
|
||||
target.write_bytes(target.read_bytes()[:-16] + b"0" * 16)
|
||||
|
||||
def close(self) -> None:
|
||||
self.httpd.shutdown()
|
||||
self.httpd.server_close()
|
||||
|
||||
|
||||
class DownloaderCase(unittest.TestCase):
|
||||
"""Common fixture: a two-volume pack served over loopback."""
|
||||
|
||||
volumes = 2
|
||||
payload = {
|
||||
"bios/scph5501.bin": b"\x10\x20" * 4096,
|
||||
"bios/dc/dc_boot.bin": b"\x30\x40" * 4096,
|
||||
}
|
||||
|
||||
pack_name = "Batocera_43.1_BIOS_Pack.zip"
|
||||
platform = "batocera"
|
||||
platform_label = "Batocera 43.1"
|
||||
|
||||
def setUp(self):
|
||||
self.server = ReleaseServer(self.pack_name, self.payload, self.volumes)
|
||||
self.addCleanup(self.server.close)
|
||||
self.dest = Path(tempfile.mkdtemp()) / "bios"
|
||||
# A pack is gigabytes: staging it in the system temp directory fills
|
||||
# the RAM disk that /tmp is on the appliances these packs target.
|
||||
self.tmpdir = Path(tempfile.mkdtemp())
|
||||
|
||||
def env(self) -> dict[str, str]:
|
||||
env = dict(os.environ)
|
||||
env.update(
|
||||
RETROBIOS_API=self.base_api(),
|
||||
TMPDIR=str(self.tmpdir),
|
||||
TMP=str(self.tmpdir),
|
||||
TEMP=str(self.tmpdir),
|
||||
)
|
||||
return env
|
||||
|
||||
def base_api(self) -> str:
|
||||
return self.server.base
|
||||
|
||||
def assert_extracted(self):
|
||||
for name, data in self.payload.items():
|
||||
extracted = self.dest / name
|
||||
self.assertTrue(extracted.is_file(), f"{name} not extracted")
|
||||
self.assertEqual(extracted.read_bytes(), data)
|
||||
|
||||
def assert_listed(self, out: str):
|
||||
names = [
|
||||
line.strip().lstrip("- ").strip()
|
||||
for line in out.splitlines()
|
||||
if "BIOS" not in line and line.strip()
|
||||
]
|
||||
self.assertIn(self.platform_label, names)
|
||||
self.assertNotIn(".001", out)
|
||||
|
||||
def assert_refused(self, proc):
|
||||
self.assertNotEqual(proc.returncode, 0)
|
||||
self.assertIn("checksum", (proc.stdout + proc.stderr).lower())
|
||||
self.assertFalse((self.dest / "bios/scph5501.bin").exists())
|
||||
|
||||
def assert_no_leftovers(self):
|
||||
self.assertEqual(
|
||||
sorted(p.name for p in self.tmpdir.iterdir()),
|
||||
[],
|
||||
"pack staged in the system temp directory",
|
||||
)
|
||||
searched = list(self.dest.parent.iterdir()) + list(self.dest.rglob("*"))
|
||||
leftovers = sorted(
|
||||
p.name for p in searched if "_BIOS_Pack" in p.name or p.name.startswith(".")
|
||||
)
|
||||
self.assertEqual(leftovers, [], "download staging left behind")
|
||||
|
||||
|
||||
class WholePackCase(DownloaderCase):
|
||||
"""A pack published whole, under a name the platform id does not spell."""
|
||||
|
||||
volumes = 1
|
||||
pack_name = "MiSTer_FPGA_2026-08-29_BIOS_Pack.zip"
|
||||
platform = "misterfpga"
|
||||
platform_label = "MiSTer FPGA 2026-08-29"
|
||||
|
||||
|
||||
class TestDownloadPython(DownloaderCase):
|
||||
def run_cli(self, *args, expect_success=True):
|
||||
proc = subprocess.run(
|
||||
["python3", str(REPO_ROOT / "scripts" / "download.py"), *args],
|
||||
env=self.env(),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if expect_success:
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
||||
return proc
|
||||
|
||||
def test_list_names_the_pack(self):
|
||||
proc = self.run_cli("--list")
|
||||
self.assert_listed(proc.stdout)
|
||||
|
||||
def test_pack_downloads_and_extracts(self):
|
||||
self.run_cli(self.platform, str(self.dest))
|
||||
self.assert_extracted()
|
||||
|
||||
def test_volumes_are_not_staged_in_the_system_temp_directory(self):
|
||||
self.run_cli(self.platform, str(self.dest))
|
||||
self.assert_no_leftovers()
|
||||
|
||||
def test_a_corrupt_volume_is_refused(self):
|
||||
self.server.corrupt_last_volume()
|
||||
proc = self.run_cli(self.platform, str(self.dest), expect_success=False)
|
||||
self.assert_refused(proc)
|
||||
|
||||
def test_info_reports_every_volume(self):
|
||||
proc = self.run_cli("--info", self.platform)
|
||||
self.assertIn("2 parts", proc.stdout)
|
||||
|
||||
def test_an_unreachable_release_endpoint_fails(self):
|
||||
env = self.env()
|
||||
env["RETROBIOS_API"] = f"http://127.0.0.1:{_closed_port()}"
|
||||
proc = subprocess.run(
|
||||
["python3", str(REPO_ROOT / "scripts" / "download.py"), "--list"],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
self.assertNotEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
||||
|
||||
def test_a_non_loopback_http_api_is_refused(self):
|
||||
env = self.env()
|
||||
env["RETROBIOS_API"] = "http://api.example.com"
|
||||
proc = subprocess.run(
|
||||
["python3", str(REPO_ROOT / "scripts" / "download.py"), "--list"],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
self.assertNotEqual(proc.returncode, 0)
|
||||
self.assertIn("RETROBIOS_API", proc.stderr)
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
shutil.which("curl") and shutil.which("unzip"), "curl and unzip required"
|
||||
)
|
||||
class TestDownloadShell(DownloaderCase):
|
||||
def run_cli(self, *args, expect_success=True):
|
||||
proc = subprocess.run(
|
||||
["bash", str(SHELL), *args],
|
||||
env=self.env(),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if expect_success:
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
||||
return proc
|
||||
|
||||
def test_list_names_the_pack(self):
|
||||
proc = self.run_cli("--list")
|
||||
self.assert_listed(proc.stdout)
|
||||
|
||||
def test_pack_downloads_and_extracts(self):
|
||||
self.run_cli(self.platform, str(self.dest))
|
||||
self.assert_extracted()
|
||||
|
||||
def test_volumes_are_not_staged_in_the_system_temp_directory(self):
|
||||
self.run_cli(self.platform, str(self.dest))
|
||||
self.assert_no_leftovers()
|
||||
|
||||
def test_a_corrupt_volume_is_refused(self):
|
||||
self.server.corrupt_last_volume()
|
||||
proc = self.run_cli(self.platform, str(self.dest), expect_success=False)
|
||||
self.assert_refused(proc)
|
||||
|
||||
def test_an_unknown_platform_lists_what_exists(self):
|
||||
proc = self.run_cli("nintendo", str(self.dest), expect_success=False)
|
||||
self.assertIn(self.platform_label, proc.stdout + proc.stderr)
|
||||
|
||||
|
||||
class TestWholePackPython(WholePackCase, TestDownloadPython):
|
||||
def test_info_reports_every_volume(self):
|
||||
proc = self.run_cli("--info", self.platform)
|
||||
self.assertIn("1 part", proc.stdout)
|
||||
|
||||
|
||||
class TestWholePackShell(WholePackCase, TestDownloadShell):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -852,6 +852,8 @@ class TestLaunchboxDetectionPowershell(unittest.TestCase):
|
||||
USERPROFILE=str(userprofile),
|
||||
APPDATA=str(tempfile.mkdtemp()),
|
||||
RETROBIOS_BASE_URL=f"http://127.0.0.1:{httpd.server_address[1]}",
|
||||
# A LaunchBox layout is a Windows layout: the runner is Linux.
|
||||
RETROBIOS_OS="wsl",
|
||||
)
|
||||
proc = subprocess.run(
|
||||
[
|
||||
@@ -953,6 +955,18 @@ class TestLaunchboxDetectionPowershell(unittest.TestCase):
|
||||
self.assertTrue(landed.exists(), proc.stdout)
|
||||
|
||||
|
||||
class TestForcedOs(unittest.TestCase):
|
||||
"""RETROBIOS_OS overrides detection with a known name only."""
|
||||
|
||||
def test_known_name_wins(self):
|
||||
with unittest.mock.patch.dict(os.environ, {"RETROBIOS_OS": "Windows"}):
|
||||
self.assertEqual(install.detect_os(), "windows")
|
||||
|
||||
def test_unknown_name_is_ignored(self):
|
||||
with unittest.mock.patch.dict(os.environ, {"RETROBIOS_OS": "amiga"}):
|
||||
self.assertIn(install.detect_os(), ("linux", "wsl", "windows", "darwin"))
|
||||
|
||||
|
||||
class TestDetectFrontends(unittest.TestCase):
|
||||
"""Frontends have no BIOS directory of their own but hint at the setup."""
|
||||
|
||||
|
||||
@@ -347,12 +347,11 @@ pattern and how to add a test.
|
||||
|
||||
| Workflow | File | Trigger | Role |
|
||||
|----------|------|---------|------|
|
||||
| Build & Release | `build.yml` | manual dispatch only | restore large files, build packs, create GitHub release |
|
||||
| Deploy Site | `deploy-site.yml` | push to main (platforms, emulators, wiki, scripts) + manual | validate contracts, generate site, build with MkDocs, validate rendered HTML, deploy to Pages |
|
||||
| PR Validation | `validate.yml` | pull request on bios/, platforms/, emulators/, schemas/, scripts/, tests/ | validate BIOS hashes, schema check, run the full test suite, auto-label PR |
|
||||
|
||||
The build workflow has no push trigger: a release is dispatched by hand. It
|
||||
keeps a 7-day rate limit between releases and the 3 most recent tags. See the
|
||||
Releases are not built in CI: the packs are generated and checked on the
|
||||
maintainer's machine and uploaded with `gh`, see the
|
||||
[release process](release-process.md).
|
||||
|
||||
## License
|
||||
|
||||
+42
@@ -92,6 +92,48 @@ python scripts/verify.py --emulator beetle_psx --verbose
|
||||
|
||||
The `--verbose` flag shows source references and expected values from the emulator's source code.
|
||||
|
||||
## Is piping the installer into a shell safe?
|
||||
|
||||
The one-liner runs a small bootstrap, not the installer. That bootstrap fetches
|
||||
`install.py` over HTTPS only, refuses anything over 2 MB, and requires its
|
||||
SHA-256 to equal the value written inside the bootstrap itself. A substituted
|
||||
installer aborts with `install.py SHA-256 mismatch` before a single line of it
|
||||
runs.
|
||||
|
||||
The installer then verifies every file it downloads against the size, SHA-256
|
||||
and SHA-1 the manifest declares, and moves it into place only once those match.
|
||||
The manifest is treated as untrusted: a destination cannot be absolute, carry a
|
||||
drive letter or climb out of the BIOS directory.
|
||||
|
||||
Reading the bootstrap before running it is two lines:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh -o install.sh
|
||||
less install.sh && sh install.sh
|
||||
```
|
||||
|
||||
See [Installer](installer.md) for the full behaviour.
|
||||
|
||||
## Can I run the installer again?
|
||||
|
||||
Yes, and re-running is the normal way to update. A file already present with
|
||||
the expected hash is left untouched, one whose contents do not match is
|
||||
replaced by the verified copy, and files the manifest does not name are never
|
||||
read, moved or deleted. `--check` does the same inspection and exits without
|
||||
writing.
|
||||
|
||||
## Can I install onto an SD card or another machine's drive?
|
||||
|
||||
Yes, `--dest` takes any path and the one-liner forwards arguments:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh \
|
||||
| sh -s -- --platform retroarch --dest /path/to/sdcard
|
||||
```
|
||||
|
||||
On Windows the arguments need a script block, since `iex` would read them as
|
||||
its own; the form is on the [Installer](installer.md#passing-options) page.
|
||||
|
||||
## Is this legal?
|
||||
|
||||
Redistributing firmware is not settled law, and this page does not pretend otherwise. What follows is the reasoning the project acts on, with the strength of each argument stated plainly so anyone can weigh it. None of it is legal advice, and none of it has been tested in court.
|
||||
|
||||
+22
-8
@@ -24,7 +24,7 @@ irm https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1 | iex
|
||||
```
|
||||
|
||||
Running `install.py` directly works the same way and needs nothing beyond
|
||||
Python 3.10+:
|
||||
Python 3.8+, the floor both bootstraps enforce before they hand over:
|
||||
|
||||
```bash
|
||||
python install.py
|
||||
@@ -45,10 +45,10 @@ python install.py --standalone-copies # opt in to extra standalone-emulator pat
|
||||
|
||||
The default flow writes inside the detected platform tree only. Copies into
|
||||
separate standalone-emulator directories are opt-in, so discovery cannot cause
|
||||
unexpected writes elsewhere on the machine.
|
||||
Entries the collection cannot satisfy are reported as safely omitted and the
|
||||
run continues; the installer never substitutes a same-named file for one a
|
||||
hash-verifying platform would reject.
|
||||
unexpected writes elsewhere on the machine. Entries the collection cannot
|
||||
satisfy are reported as safely omitted and the run continues; the installer
|
||||
never substitutes a same-named file for one a hash-verifying platform would
|
||||
reject.
|
||||
|
||||
Arguments pass through the one-liner too, which is how you target an SD card
|
||||
mounted on another machine:
|
||||
@@ -58,6 +58,10 @@ curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh \
|
||||
| sh -s -- --platform retroarch --dest /path/to/sdcard
|
||||
```
|
||||
|
||||
PowerShell needs another form, the environment overrides and the detection
|
||||
rules are listed per platform, and the trust boundary is described in full on
|
||||
the [Installer](installer.md) page.
|
||||
|
||||
### Option 2: download.sh (Linux/macOS, from a clone)
|
||||
|
||||
Downloads a whole pack rather than the missing files. Needs `curl` and `unzip`:
|
||||
@@ -67,6 +71,10 @@ bash scripts/download.sh retroarch ~/RetroArch/system/
|
||||
bash scripts/download.sh --list # show available packs
|
||||
```
|
||||
|
||||
A pack published in several volumes is downloaded part by part, joined, and
|
||||
checked against the SHA-256 the release publishes before anything is
|
||||
extracted. `python scripts/download.py` does the same on Windows.
|
||||
|
||||
### Option 3: manual download
|
||||
|
||||
1. Go to the [releases page](https://github.com/Abdess/retrobios/releases)
|
||||
@@ -74,9 +82,15 @@ bash scripts/download.sh --list # show available packs
|
||||
3. Extract to the BIOS directory listed below
|
||||
|
||||
Packs over 2 GB are split into numbered volumes (`.zip.001`, `.zip.002`).
|
||||
Download every part and open the `.001` file with 7-Zip or PeaZip, which
|
||||
extracts the whole set. See [Download](../which-pack.md) for the pack
|
||||
variants and per-setup instructions.
|
||||
Download every part into the same folder, then open the `.001` with 7-Zip or
|
||||
PeaZip, which read the whole set. To join them into one ZIP first:
|
||||
|
||||
- Linux/macOS: `cat Pack.zip.0* > Pack.zip`
|
||||
- Windows (cmd): `copy /b Pack.zip.001+Pack.zip.002 Pack.zip`
|
||||
|
||||
A frontend's own extractor may refuse a volume: Batocera answers `Archive
|
||||
type: '001' is not yet supported`. Join the parts from a shell there. See
|
||||
[Download](../which-pack.md) for the per-setup instructions.
|
||||
|
||||
## BIOS directory by platform
|
||||
|
||||
|
||||
+3
-1
@@ -10,6 +10,7 @@ each page takes one job from start to finish.
|
||||
## For users
|
||||
|
||||
- **[Getting started](getting-started.md)** - installation, BIOS directory paths per platform, verification
|
||||
- **[Installer](installer.md)** - the one-liner in full: bootstrap, options, environment overrides, platform detection, trust boundary
|
||||
- **[FAQ](faq.md)** - common questions, troubleshooting, hash explanations
|
||||
|
||||
If you just want to download BIOS packs, see the [home page](../index.md).
|
||||
@@ -18,7 +19,7 @@ If you just want to download BIOS packs, see the [home page](../index.md).
|
||||
|
||||
- **[Architecture](architecture.md)** - directory structure, data flow, platform inheritance, pack grouping, security, edge cases, CI workflows
|
||||
- **[Tools](tools.md)** - CLI reference for every script, pipeline usage, scrapers
|
||||
- **[Advanced usage](advanced-usage.md)** - custom packs, target filtering, truth generation, emulator verification, offline workflow
|
||||
- **[Advanced usage](advanced-usage.md)** - custom packs, region filtering, one file per slot, target filtering, truth generation, emulator verification, offline workflow
|
||||
- **[Verification modes](verification-modes.md)** - how each platform verifies BIOS files, severity matrix, resolution chain
|
||||
- **[Data model](data-model.md)** - database.json structure, indexes, file resolution order, YAML formats
|
||||
- **[Troubleshooting](troubleshooting.md)** - diagnosis by symptom: missing BIOS, hash mismatch, pack issues, verify errors
|
||||
@@ -56,6 +57,7 @@ See [contributing](../contributing.md) for submission guidelines.
|
||||
- **scraper** - a script that fetches BIOS requirement data from an upstream source (System.dat, es_bios.xml, etc.)
|
||||
- **exporter** - a script that converts ground truth data back into a platform's native format
|
||||
- **target** - a hardware architecture that a platform runs on (e.g. switch, rpi4, x86_64, steamos)
|
||||
- **region** - the territory an emulator's code branches on to select a file; `--region` builds a pack from an ordered priority list, `--one-per-slot` keeps one file per system and region
|
||||
- **variant** - an alternative version of a BIOS file (different revision, region, or dump), stored in `.variants/`
|
||||
- **required** - a file the core needs to function; determined by source code behavior
|
||||
- **optional** - a file the core functions without, possibly with reduced accuracy or missing features
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
# Installer - RetroBIOS
|
||||
|
||||
Reference for `install.sh`, `install.ps1` and `install.py`: what the one-liner
|
||||
runs, what it detects, where the bytes come from and what it is allowed to
|
||||
write. For BIOS directory paths per platform, see
|
||||
[Getting started](getting-started.md).
|
||||
|
||||
## One line
|
||||
|
||||
```bash
|
||||
# Linux / macOS / Steam Deck
|
||||
curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh
|
||||
```
|
||||
|
||||
```powershell
|
||||
# Windows
|
||||
irm https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1 | iex
|
||||
```
|
||||
|
||||
From a clone, run the installer itself. Nothing beyond the standard library is
|
||||
needed, and Python 3.8 is the floor both bootstraps enforce before handing over:
|
||||
|
||||
```bash
|
||||
python install.py
|
||||
```
|
||||
|
||||
## Passing options
|
||||
|
||||
The shell bootstrap forwards everything after `-s --`:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh \
|
||||
| sh -s -- --platform retroarch --dest /path/to/sdcard
|
||||
```
|
||||
|
||||
PowerShell needs another form. `iex` takes the script text as its own argument,
|
||||
so anything appended to `irm ... | iex` is read as an argument to `iex` and
|
||||
rejected with `A positional parameter cannot be found`. Building a script block
|
||||
from the text and calling it passes the arguments through:
|
||||
|
||||
```powershell
|
||||
&([scriptblock]::Create((irm https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1))) `
|
||||
--platform retroarch --dest D:\bios
|
||||
```
|
||||
|
||||
## What the bootstrap does
|
||||
|
||||
Both wrappers do the same four things before any BIOS file is considered.
|
||||
|
||||
They fetch `install.py` over HTTPS only, from `RETROBIOS_INSTALL_URL` or the
|
||||
pinned raw URL. A URL with any other scheme is refused outright.
|
||||
|
||||
They cap the download at 2 MB and require the SHA-256 to equal the 64 hex
|
||||
characters embedded in the wrapper, `RETROBIOS_INSTALL_SHA256` when set. A
|
||||
mismatch aborts with `install.py SHA-256 mismatch` and nothing runs. Any change
|
||||
to `install.py` therefore means recomputing that pin in both wrappers.
|
||||
|
||||
They locate a Python 3.8 or newer interpreter (`python3` then `python` on
|
||||
POSIX, `py -3` then `python3` then `python` on Windows) and refuse to continue
|
||||
without one. `install.sh` needs `curl` or `wget` and `sha256sum` or `shasum`.
|
||||
Windows PowerShell 5.1 still negotiates SSL 3.0 and TLS 1.0 by default and
|
||||
GitHub refuses both, so `install.ps1` raises the floor to TLS 1.2 before
|
||||
downloading; `install.sh` pins the same floor with `curl --tlsv1.2`.
|
||||
|
||||
They delete the temporary installer on the way out, including on interrupt.
|
||||
Run from a clone, `install.sh` and `install.ps1` reuse the `install.py` sitting
|
||||
next to them and skip the download entirely; piped from stdin, `install.sh`
|
||||
never treats the working directory as a trusted location for it.
|
||||
|
||||
## What the installer does
|
||||
|
||||
1. Detects the host OS and the platforms installed on it.
|
||||
2. Fetches `install/<platform>.json` from the same revision the bootstrap
|
||||
verified the installer against, so the file list and the code reading it
|
||||
always come from one commit.
|
||||
3. Prints the file count and total size, then a safety notice counting the
|
||||
entries the collection cannot serve, how many of those the platform marks
|
||||
required, and why.
|
||||
4. Hashes what is already in the destination and reports how many entries are
|
||||
present, verified, or present with the wrong contents.
|
||||
5. Downloads what is missing or wrong, up to 8 files at a time.
|
||||
6. Verifies each download against the declared size, SHA-256 and SHA-1 before
|
||||
it touches the destination, retrying up to three times, then moves it into
|
||||
place with `os.replace` so a partial file is never visible.
|
||||
7. Copies into standalone emulator directories, only with `--standalone-copies`.
|
||||
|
||||
A run looks like this:
|
||||
|
||||
```
|
||||
Fetching file index for retroarch...
|
||||
1872 files (5.5 GB)
|
||||
Safety notice: 9 unavailable or unsafe entries omitted (1 required; not found: 9).
|
||||
|
||||
Checking existing files...
|
||||
0/1872 present (0 verified, 0 wrong hash)
|
||||
|
||||
1872 files need downloading.
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
| Option | Effect |
|
||||
|--------|--------|
|
||||
| `--platform NAME` | Install for this platform instead of the detected one. Unknown names are refused with the available list |
|
||||
| `--dest PATH` | Destination directory, overriding detection. With `--dest` alone the file list is RetroArch's |
|
||||
| `--target NAME` | Keep only the files the cores of that hardware target need. An unknown target is refused rather than ignored, since carrying on would install everything |
|
||||
| `--check` | Report and exit without writing |
|
||||
| `--list-platforms` | Print the supported platforms and what was detected here |
|
||||
| `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries |
|
||||
| `--jobs N`, `-j N` | Parallel downloads, 1 to 32, default 8 |
|
||||
| `--verbose`, `-v` | Print per-attempt failures and hash mismatches |
|
||||
| `--standalone-copies` | Also copy into detected standalone emulator directories |
|
||||
| `--help`, `-h` | Print the option list and exit |
|
||||
|
||||
`--check` reads the same file list, hashes the destination and exits. Without
|
||||
it, a file already present with the expected hash is left untouched, and one
|
||||
whose contents do not match is re-downloaded and replaced by the verified copy.
|
||||
Files the list does not name are never read, moved or deleted.
|
||||
|
||||
## Environment overrides
|
||||
|
||||
| Variable | Effect |
|
||||
|----------|--------|
|
||||
| `RETROBIOS_REF` | Branch or tag the file list and the payloads are read from. Defaults to `main`; a release tag pins a reproducible file set |
|
||||
| `RETROBIOS_BASE_URL` | Base URL for the manifest and the files it declares. HTTPS only, plain HTTP allowed to loopback for end-to-end tests. Whoever controls this base controls the expected hashes, hence the check |
|
||||
| `RETROBIOS_OS` | Names the host outright (`linux`, `wsl`, `windows`, `darwin`) instead of detecting it |
|
||||
| `RETROBIOS_INSTALL_URL` | Where the bootstrap fetches `install.py`. HTTPS only |
|
||||
| `RETROBIOS_INSTALL_SHA256` | The SHA-256 the bootstrap requires of that installer, 64 hex characters |
|
||||
| `HTTPS_PROXY`, `NO_PROXY` | Honoured for every download through the standard library's default proxy handling: one names the proxy, the other the hosts to reach directly |
|
||||
|
||||
## Platform detection
|
||||
|
||||
Detection reads what the platform itself writes, never a guess from a directory
|
||||
name. The embedded systems are tested in the order below and the first match
|
||||
wins, since a machine is only one of them.
|
||||
|
||||
| Evidence | Platform | BIOS directory |
|
||||
|----------|----------|----------------|
|
||||
| `/etc/os-release` `ID=rocknix` | ROCKNIX | `/storage/roms/bios` |
|
||||
| `/media/fat/MiSTer` | MiSTer FPGA | `/media/fat/games` |
|
||||
| `/etc/knulli-release` | Batocera | `/userdata/bios` |
|
||||
| `/etc/os-release` `ID=lakka` | Lakka | `/storage/system` |
|
||||
| `/etc/batocera-version` | Batocera | `/userdata/bios` |
|
||||
| `/recalbox/recalbox.version` or `/usr/bin/recalbox-settings` | Recalbox | `/recalbox/share/bios` |
|
||||
| `/opt/muos` or `/mnt/mmc/MUOS/` | RetroArch | `/mnt/mmc/MUOS/bios` |
|
||||
| `/home/ark` and `/opt/system` | RetroArch | `/roms/bios` |
|
||||
| `/mnt/vendor/bin/dmenu.bin` | RetroArch | `/mnt/mmc/bios` |
|
||||
|
||||
Desktop installs are read from their own configuration, and several can be
|
||||
found on one machine.
|
||||
|
||||
| Evidence | Platform | BIOS directory |
|
||||
|----------|----------|----------------|
|
||||
| `~/.config/EmuDeck/settings.sh` | EmuDeck | `emulationPath` + `/bios` |
|
||||
| `%APPDATA%\EmuDeck\settings.ps1` | EmuDeck | `$emulationPath` + `\bios` |
|
||||
| `~/.var/app/net.retrodeck.retrodeck/config/retrodeck/retrodeck.json` | RetroDECK | `paths.rd_home_path`, falling back to the pre-migration `retrodeck.cfg` and its `rdhome` |
|
||||
| `~/.var/app/org.libretro.RetroArch/config/retroarch/retroarch.cfg` | RetroArch (Flatpak) | `system_directory` |
|
||||
| `~/snap/retroarch/current/.config/retroarch/retroarch.cfg` | RetroArch (Snap) | `system_directory` |
|
||||
| `~/.config/retroarch/retroarch.cfg` | RetroArch (native) | `system_directory` |
|
||||
| `~/Library/Application Support/RetroArch/retroarch.cfg` | RetroArch (macOS) | `system_directory` |
|
||||
| `%APPDATA%\RetroArch\retroarch.cfg` | RetroArch (Windows) | `system_directory` |
|
||||
| `%ProgramFiles(x86)%\Steam\steamapps\common\RetroArch\retroarch.cfg` | RetroArch (Steam) | `system_directory` |
|
||||
| LaunchBox `Data\Emulators.xml` | RetroArch (portable) | the system directory that entry points at |
|
||||
|
||||
`system_directory` is read the way RetroArch expands it: a leading `~` is the
|
||||
home directory, a leading `:` is the application directory, and `default` means
|
||||
`system` next to the application. The LaunchBox installation is located through
|
||||
its Start menu shortcut, the only record of where its installer was pointed.
|
||||
|
||||
ES-DE and LaunchBox are reported when present and nothing is written to them:
|
||||
they reference emulators rather than owning a BIOS directory. ES-DE is looked
|
||||
up at `$ESDE_APPDATA_DIR` or `~/ES-DE`, the two locations its own
|
||||
`getAppDataDirectory` resolves.
|
||||
|
||||
BizHawk, RetroBat, RomM and RetroPie have no detection and are selected with
|
||||
`--platform`. When a forced platform is not found on the machine, the
|
||||
destination falls back to `/userdata/bios` for Batocera, `/recalbox/share/bios`
|
||||
for Recalbox, `/storage/system` for Lakka, `~/retrodeck` for RetroDECK,
|
||||
`~/Emulation/bios` for EmuDeck, `/storage/roms/bios` for ROCKNIX,
|
||||
`/media/fat/games` for MiSTer, `~/RetroPie/BIOS` for RetroPie, and `~/bios`
|
||||
for anything else. Pass `--dest` to say where instead of relying on that.
|
||||
|
||||
With nothing detected, the installer lists the platforms and asks. With several
|
||||
detected, it asks which one. Both prompts need a terminal: piped into a script
|
||||
with no platform to install for, it prints the manual invocation and exits 1.
|
||||
|
||||
## Standalone copies
|
||||
|
||||
Some files are read by a standalone emulator from its own data directory
|
||||
rather than from the platform's BIOS tree. `--standalone-copies` copies them
|
||||
there after the install, and only then: without the flag nothing is written
|
||||
outside the selected tree.
|
||||
|
||||
The manifest carries the list, fifteen entries on the six platforms that
|
||||
declare them (Batocera, EmuDeck, Recalbox, RetroArch, RetroBat, RetroDECK):
|
||||
|
||||
| Files | Copied to |
|
||||
|-------|-----------|
|
||||
| `prod.keys`, `title.keys` | yuzu, eden, citron, suyu and Ryujinx key directories |
|
||||
| `Citra/sysdata/aes_keys.txt`, `Citra/sysdata/boot9.bin` | Azahar `sysdata` |
|
||||
| `scph*.bin` | DuckStation `bios` |
|
||||
| `ps2-*.bin` | PCSX2 `bios`, native and Flatpak |
|
||||
| `GC/USA/IPL.bin`, `GC/EUR/IPL.bin`, `GC/JAP/IPL.bin`, `dsp_rom.bin`, `dsp_coef.bin` | Dolphin, per region for the IPL |
|
||||
| `PPSSPP/ppge_atlas.zim` | PPSSPP `PSP/SYSTEM` |
|
||||
| `dc/dc_boot.bin`, `dc/dc_nvmem.bin` | Flycast `data`, native and Flatpak |
|
||||
|
||||
An entry names one file or a glob, and one entry carries no file at all: when
|
||||
an RPCS3 configuration directory is present it prints that `PS3UPDAT.PUP` is
|
||||
installed through RPCS3's own File menu, since that firmware is an installer to
|
||||
run rather than a file to copy.
|
||||
|
||||
Targets are per OS, WSL falling back to the Linux ones. A directory that does
|
||||
not exist is skipped rather than created, so nothing is copied for an emulator
|
||||
that is not installed. A destination that is already a symbolic link is skipped
|
||||
too: the copy leaves the tree the user opted into, and a link there would
|
||||
redirect the write somewhere else again.
|
||||
|
||||
## Android
|
||||
|
||||
There is no Android detection. Termux reports itself as Linux, so nothing is
|
||||
found and the platform has to be named along with where it writes:
|
||||
|
||||
```bash
|
||||
python install.py --platform retroarch --dest /storage/emulated/0/RetroArch/system
|
||||
```
|
||||
|
||||
## Where the files come from
|
||||
|
||||
Each manifest entry names either `repo_path`, served from the repository at the
|
||||
pinned revision, or `release_asset`, served from the `large-files` release for
|
||||
the files above the size GitHub carries in a tree. Everything else in the entry
|
||||
(`dest`, `size`, `sha1`, `sha256`, `cores`) describes what must land where and
|
||||
what it must hash to.
|
||||
|
||||
Entries the collection cannot satisfy are not silently dropped: they travel in
|
||||
`omitted_files` with a reason, and the run prints the count before downloading
|
||||
anything. `not_found` means no file in the collection matches the entry, which
|
||||
is what the [gap analysis](../gaps.md) tracks.
|
||||
|
||||
## Trust boundary
|
||||
|
||||
The manifest is fetched over the network and treated as untrusted input. Every
|
||||
path it carries goes through one validator that refuses an absolute path, a
|
||||
drive letter, a backslash, a null byte, a doubled separator, a trailing slash
|
||||
and any `..` component, so a destination cannot climb out of the BIOS
|
||||
directory. Manifest, target list and payloads each have a size cap, the file
|
||||
list a count cap, and the sum of the declared sizes is checked against the
|
||||
total the manifest claims.
|
||||
|
||||
Symbolic links met under the BIOS root are followed on purpose: they are the
|
||||
user's own layout, and EmuDeck links `bios/shadps4/sys_modules` into shadPS4's
|
||||
data directory where the emulator actually reads it. The boundary is lexical,
|
||||
not a resolved-path comparison, which is what keeps that case working.
|
||||
Standalone copies are stricter, since they write outside the tree the user
|
||||
selected: a symlink already sitting at the destination is skipped rather than
|
||||
followed.
|
||||
|
||||
## Exit status
|
||||
|
||||
`0` on success, `1` on failure: an unusable base URL, an unknown platform or
|
||||
target, a manifest that fails validation, a fetch that never succeeded, or no
|
||||
platform to install for in a non-interactive run.
|
||||
|
||||
Failures that are per-file, such as a download exhausting its three attempts,
|
||||
are counted and reported at the end.
|
||||
|
||||
## When something goes wrong
|
||||
|
||||
See [Troubleshooting](troubleshooting.md#installation-script-fails) for network,
|
||||
permission and detection failures.
|
||||
+66
-74
@@ -5,7 +5,7 @@ are built, and how to run the process manually.
|
||||
|
||||
## CI workflows overview
|
||||
|
||||
The project uses 3 GitHub Actions workflows. All use only official GitHub
|
||||
The project uses 2 GitHub Actions workflows. All use only official GitHub
|
||||
actions (`actions/checkout`, `actions/setup-python`, `actions/upload-pages-artifact`,
|
||||
`actions/deploy-pages`). No third-party actions.
|
||||
|
||||
@@ -13,49 +13,15 @@ Budget target: ~175 minutes/month on the GitHub free tier.
|
||||
|
||||
| Workflow | File | Trigger |
|
||||
|----------|------|---------|
|
||||
| Build & Release | `build.yml` | Manual dispatch only |
|
||||
| Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, mkdocs.yml), manual |
|
||||
| PR Validation | `validate.yml` | PR touching `bios/**`, `platforms/**` or `emulators/**` |
|
||||
|
||||
Upstream BIOS lists are not scraped on a schedule. A maintainer runs the
|
||||
scrapers by hand (see [adding a scraper](adding-a-scraper.md)), reviews the
|
||||
diff, and commits the refreshed platform YAML.
|
||||
|
||||
## build.yml - Build & Release
|
||||
|
||||
Releasing is deliberate. Pushing never cuts one: somebody decides the
|
||||
collection is worth publishing and dispatches the workflow.
|
||||
|
||||
**Trigger.** `workflow_dispatch` only, with an optional `force_release` flag to
|
||||
bypass the rate limit.
|
||||
|
||||
**Concurrency.** Group `build`, queued rather than cancelled: a run that is
|
||||
already uploading assets must finish.
|
||||
|
||||
**Steps:**
|
||||
|
||||
1. Checkout, Python 3.12, install `pyyaml`
|
||||
2. Run `test_e2e`
|
||||
3. Rate limit check: skip if last release was less than 7 days ago (unless
|
||||
`force_release` is set)
|
||||
4. Restore large files from the `large-files` release into `.cache/large/`
|
||||
5. Refresh data directories (`refresh_data_dirs.py`)
|
||||
6. Build packs (`generate_pack.py --all --output-dir dist/`)
|
||||
7. Split any pack over 2 GB into `.zip.001`, `.zip.002`, ... volumes. GitHub
|
||||
caps release assets at 2 GB, and the `.001` convention is what 7-Zip and
|
||||
PeaZip open directly (they reject `.partNN` names as corrupt)
|
||||
8. Create GitHub release with tag `v{YYYY.MM.DD}` (appends `.N` suffix if
|
||||
a same-day release already exists)
|
||||
9. Clean up old releases, keeping the 3 most recent plus `large-files`
|
||||
|
||||
**Release notes** include file count, total size, per-pack sizes, the extract
|
||||
path per platform, and the last 15 non-merge commits touching `bios/` or
|
||||
`platforms/`.
|
||||
|
||||
**Pack variants.** The workflow builds full packs only. Releases that also
|
||||
carry `*_Platform_BIOS_Pack.zip` had a second `--source platform` run added by
|
||||
hand, as in the manual process below. See
|
||||
[advanced usage](advanced-usage.md#pack-source-variants).
|
||||
diff, and commits the refreshed platform YAML. Releases are built on the
|
||||
maintainer's machine and uploaded with `gh`, see
|
||||
[cutting a release](#cutting-a-release): the packs weigh 25 GB, more than a
|
||||
hosted runner should rebuild and re-upload.
|
||||
|
||||
## deploy-site.yml - Deploy Documentation Site
|
||||
|
||||
@@ -163,52 +129,78 @@ gh release upload large-files "bios/Sony/PS3/PS3UPDAT.PUP#PS3UPDAT.PUP"
|
||||
**Local cache.** `generate_pack.py` calls `fetch_large_file()` which downloads
|
||||
from the release and caches in `.cache/large/` for subsequent runs.
|
||||
|
||||
## Manual release process
|
||||
## Cutting a release
|
||||
|
||||
When `build.yml` is disabled, build and release manually:
|
||||
Releasing is deliberate and local. Nothing on GitHub builds a pack: the
|
||||
pipeline runs here, the archives are checked here, and `gh` uploads them.
|
||||
|
||||
```bash
|
||||
# Run the full pipeline (DB + verify + packs + manifests + integrity + docs)
|
||||
# 1. Full pipeline, online, so data directories and MAME/FBNeo hashes are fresh
|
||||
python scripts/pipeline.py
|
||||
|
||||
# Or step by step:
|
||||
python scripts/generate_db.py --force --bios-dir bios --output database.json
|
||||
python scripts/verify.py --all
|
||||
python scripts/generate_pack.py --all --output-dir dist/ # full packs
|
||||
python scripts/generate_pack.py --all --source platform --output-dir dist/ # platform packs
|
||||
python scripts/generate_pack.py --all --verify-packs --output-dir dist/
|
||||
# 2. RetroPie, which is archived but still served
|
||||
python scripts/generate_pack.py --platform retropie --output-dir dist/
|
||||
python scripts/generate_pack.py --platform retropie --verify-packs --output-dir dist/
|
||||
|
||||
# Split anything over 2 GB (GitHub asset cap)
|
||||
# 3. Checksums of the full ZIPs, before splitting
|
||||
(cd dist && sha256sum *.zip > SHA256SUMS.txt)
|
||||
|
||||
# 4. Split anything over 2 GB (GitHub asset cap); 7-Zip and PeaZip open .001 directly
|
||||
for f in dist/*.zip; do
|
||||
[ "$(stat -c%s "$f")" -gt 2000000000 ] || continue
|
||||
split --bytes=1900M --numeric-suffixes=1 --suffix-length=3 "$f" "$f." && rm "$f"
|
||||
done
|
||||
|
||||
# Create the release
|
||||
# 5. The two sizes a pack has. Extracted runs well above downloaded, so the
|
||||
# notes table names the one it carries.
|
||||
python3 - <<'PY'
|
||||
import json, pathlib, sys
|
||||
sys.path.insert(0, "scripts")
|
||||
from download import _match_key
|
||||
|
||||
parts = {}
|
||||
for path in sorted(pathlib.Path("dist").glob("*_BIOS_Pack.zip*")):
|
||||
parts.setdefault(path.name.split(".zip")[0] + ".zip", []).append(path)
|
||||
|
||||
def size(n):
|
||||
return f"{n / 1024 ** 3:.1f} GB" if n >= 1024 ** 3 else f"{n / 1024 ** 2:.0f} MB"
|
||||
|
||||
for manifest in sorted(pathlib.Path("install").glob("*.json")):
|
||||
base = next((b for b in parts if _match_key(manifest.stem) in _match_key(b)), None)
|
||||
if not base:
|
||||
continue
|
||||
data = json.loads(manifest.read_text())
|
||||
download = sum(p.stat().st_size for p in parts[base])
|
||||
print(f"{base:<42} download {size(download):>8}"
|
||||
f" extracted {size(data['total_size']):>8} {data['total_files']} files")
|
||||
PY
|
||||
|
||||
# 6. Create the release as a DRAFT, upload every asset, and only then publish it.
|
||||
# A public release with half its assets is a broken download for everyone
|
||||
# during the whole upload.
|
||||
DATE=$(date +%Y.%m.%d)
|
||||
gh release create "v${DATE}" dist/*.zip* \
|
||||
--title "BIOS Pack v${DATE}" \
|
||||
--notes "Release notes here" \
|
||||
--latest
|
||||
gh release create "v${DATE}" --draft --title "BIOS Pack v${DATE}" --notes-file notes.md
|
||||
for f in dist/SHA256SUMS.txt dist/*.zip dist/*.zip.0*; do
|
||||
[ -f "$f" ] && gh release upload "v${DATE}" "$f#$(basename "$f")" --clobber
|
||||
done
|
||||
gh release view "v${DATE}" --json assets --jq '.assets | length' # expect every file
|
||||
gh release edit "v${DATE}" --draft=false --latest
|
||||
|
||||
# 7. Keep only the new release plus large-files: an older pack carries hashes
|
||||
# the platforms no longer check, so it misleads more than it helps
|
||||
gh release list --json tagName,createdAt \
|
||||
--jq 'sort_by(.createdAt) | reverse | .[].tagName' | grep -v '^large-files$' \
|
||||
| tail -n +2 | while read tag; do gh release delete "$tag" --yes --cleanup-tag; done
|
||||
```
|
||||
|
||||
The two `generate_pack.py` runs are what puts both `*_BIOS_Pack.zip` and
|
||||
`*_Platform_BIOS_Pack.zip` on the release. `--all-variants` builds all six
|
||||
combinations instead, which is more than a release needs.
|
||||
|
||||
Run the pipeline online for a release: `--offline` skips the data directory
|
||||
refresh and the MAME/FBNeo hash refresh, so the packs would ship stale data
|
||||
directories.
|
||||
|
||||
The workflow carries no push trigger, so there is nothing to disable and no
|
||||
guard to remove. Cutting a release means dispatching `build.yml` from the
|
||||
Actions tab, or:
|
||||
|
||||
```bash
|
||||
gh workflow run build.yml
|
||||
gh workflow run build.yml -f force_release=true # within 7 days of the last
|
||||
```
|
||||
|
||||
The rate limit refuses a second release inside seven days unless
|
||||
`force_release` is set, which is what keeps a stray dispatch from publishing
|
||||
twice in a day.
|
||||
One pack per platform, the full one: the platform's list plus everything its
|
||||
cores load. Platform-only and per-emulator packs are build options, not
|
||||
release assets, since a lighter pack means a core that fails with no message.
|
||||
The release notes follow the previous release: the quick install commands,
|
||||
the pack table, what changed since the previous tag, and the contributors of
|
||||
the closed issues. A pack has two sizes and they are far apart: Batocera
|
||||
downloads as 2.4 GB and extracts to 4.0 GB. Step 5 prints both, and whichever
|
||||
one the table carries, the header names it. Someone sizing a USB drive is
|
||||
reading that column. The README table is the extracted size, from the install
|
||||
manifests. `SHA256SUMS.txt` lists the checksums of the full ZIPs before
|
||||
splitting.
|
||||
+36
-2
@@ -82,6 +82,7 @@ python scripts/verify.py --emulator dolphin # single emulator
|
||||
python scripts/verify.py --emulator dolphin --standalone # standalone mode only
|
||||
python scripts/verify.py --system atari-lynx # single system
|
||||
python scripts/verify.py --platform retroarch --target switch # filter by hardware
|
||||
python scripts/verify.py --platform recalbox --region us # regional priority list
|
||||
python scripts/verify.py --list-emulators # list all emulators
|
||||
python scripts/verify.py --list-systems # list all systems
|
||||
python scripts/verify.py --platform retroarch --list-targets # list available targets
|
||||
@@ -102,6 +103,12 @@ Verification modes per platform:
|
||||
| BizHawk | sha1 | SHA1 per firmware from `FirmwareDatabase.cs` |
|
||||
|
||||
Full details and severity mapping: [verification modes](verification-modes.md).
|
||||
`--region` narrows the report to the file set a regional pack would carry, and a
|
||||
mode that cannot apply it refuses it rather than ignoring it: see
|
||||
[region filtering](advanced-usage.md#region-filtering).
|
||||
|
||||
`--db`, `--platforms-dir` and `--emulators-dir` point the run at another
|
||||
database or source tree, which is how the tests drive it against fixtures.
|
||||
|
||||
### generate_pack.py
|
||||
|
||||
@@ -129,6 +136,11 @@ python scripts/generate_pack.py --platform retroarch --list-systems
|
||||
python scripts/generate_pack.py --all --target x86_64
|
||||
python scripts/generate_pack.py --platform retroarch --target switch
|
||||
|
||||
# Regional filtering
|
||||
python scripts/generate_pack.py --platform retroarch --region us
|
||||
python scripts/generate_pack.py --platform retroarch --region us,eu,jp
|
||||
python scripts/generate_pack.py --platform retroarch --region us --one-per-slot
|
||||
|
||||
# Source variants
|
||||
python scripts/generate_pack.py --platform retroarch --source platform # YAML baseline only
|
||||
python scripts/generate_pack.py --platform retroarch --source truth # emulator profiles only
|
||||
@@ -172,10 +184,22 @@ would have loaded.
|
||||
- `--from-md5`: look up a hash in the database, or build a custom pack with `--platform`/`--emulator`
|
||||
- `--from-md5-file`: same, reading hashes from a file (one per line, comments with #)
|
||||
- `--target`: filter by hardware target (e.g. `switch`, `rpi4`, `x86_64`)
|
||||
- `--region`: ordered priority list, best first. Regions nest, a system with no
|
||||
regional split keeps everything, and a group with no candidate in any named
|
||||
region is kept whole rather than emptied
|
||||
- `--one-per-slot`: keep one file per system and declared region, ranked by the
|
||||
`priority:` the emulator source states, lowest first
|
||||
- `--include-extras`: with `--emulator` or `--system`, add the files the cores
|
||||
pull in beyond the selection
|
||||
- `--db`, `--platforms-dir`, `--emulators-dir`: read another database or source
|
||||
tree instead of the repository's
|
||||
- `--source {platform,truth,full}`: select file source (platform YAML only, emulator profiles only, or both)
|
||||
- `--all-variants`: generate all 6 combinations of source x required_only
|
||||
- `--refresh-data`: force re-download all data directories before packing
|
||||
|
||||
How regions are ordered, what a slot is and what each narrowing adds to the
|
||||
pack name: [region filtering](advanced-usage.md#region-filtering).
|
||||
|
||||
### cross_reference.py
|
||||
|
||||
Compare emulator profiles against platform configs.
|
||||
@@ -327,9 +351,10 @@ python scripts/refresh_data_dirs.py --registry path/to/_data_dirs.yml
|
||||
| `common.py` | Shared library: hash computation, file resolution, platform config loading, emulator profiles, target filtering |
|
||||
| `dedup.py` | Deduplicate `bios/` (`--dry-run`, `--bios-dir`), move duplicates to `.variants/`. RPG Maker and ScummVM excluded (NODEDUP) |
|
||||
| `validate_pr.py` | Validate BIOS files in pull requests, post markdown report |
|
||||
| `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs |
|
||||
| `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline) |
|
||||
| `list_platforms.py` | List active platforms (`--all` includes archived, used by CI) |
|
||||
| `download.py` | Download packs from GitHub releases (Python, multi-threaded) |
|
||||
| `download.py` | Download a pack from GitHub releases, split volumes joined and checked (Python, stdlib only) |
|
||||
| `download.sh` | Same, as a shell one-liner (`curl` + `unzip`) |
|
||||
| `provenance_report.py` | Dump-catalog coverage and acquisition targets (see above) |
|
||||
| `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database |
|
||||
@@ -370,7 +395,16 @@ same-named file.
|
||||
|
||||
`scripts/download.sh` remains available for downloading a prebuilt platform
|
||||
ZIP when a manually reviewed pack release contains it; it is separate from the
|
||||
per-file automatic installer above.
|
||||
per-file automatic installer above. A pack over 2 GB is published as numbered
|
||||
volumes: both downloaders group them under one platform name, download each
|
||||
one, join them and check the result against the release's `SHA256SUMS.txt`.
|
||||
Volumes are staged inside the destination directory rather than the system
|
||||
temp directory, which is a RAM disk on Batocera, ROCKNIX and RetroDECK.
|
||||
`RETROBIOS_API` points them at another releases endpoint, HTTPS only, plain
|
||||
HTTP allowed to loopback for end-to-end tests.
|
||||
|
||||
Options, environment overrides, platform detection and the trust boundary are
|
||||
documented on the [Installer](installer.md) page.
|
||||
|
||||
## Romset recipes
|
||||
|
||||
|
||||
@@ -261,3 +261,26 @@ bash scripts/download.sh --list
|
||||
Some platforms share packs (Lakka uses the RetroArch pack). The installer handles
|
||||
this mapping automatically, but if you're downloading manually, check which pack
|
||||
name corresponds to your platform.
|
||||
|
||||
**A split pack will not extract:**
|
||||
|
||||
A pack over 2 GB is published as `.zip.001`, `.zip.002`. Put every part in one
|
||||
folder; 7-Zip and PeaZip open the `.001` directly. A frontend's own extractor
|
||||
may refuse it, Batocera among them:
|
||||
|
||||
```
|
||||
Archive type: '001' is not yet supported
|
||||
```
|
||||
|
||||
The volumes are plain byte ranges, so joining them from a shell rebuilds the
|
||||
ZIP:
|
||||
|
||||
```bash
|
||||
cat Pack.zip.0* > Pack.zip
|
||||
unzip Pack.zip -d /userdata/bios/
|
||||
rm Pack.zip
|
||||
```
|
||||
|
||||
`scripts/download.sh` and `scripts/download.py` do this on their own, checksum
|
||||
included. Both stage inside the destination directory, never in `/tmp`, which
|
||||
is a RAM disk on most of these systems.
|
||||
Reference in new issue
Block a user