From 16ec5fa2d237365c8beb8e2b53a38b408633145f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:28:32 +0100 Subject: [PATCH] docs: turn the readme into a front page The README ran 211 lines and 2,000 words: five commands with options, an 18-row download table, coverage figures and the build commands, all before the help. It now says what the project does, gives the line that installs for Windows and for the rest, shows it, lists the six ZIPs and the setups that install by command, and leads to the site, where the figures and every option live. The build commands sit in a closed block. No personal pronoun is left in its text. --- CONTRIBUTING.md | 2 +- README.md | 177 ++++------------ scripts/generate_readme.py | 360 +++++++++----------------------- scripts/site_markdown.py | 6 +- tests/test_audit_regressions.py | 9 +- tests/test_doc_pages.py | 23 +- tests/test_doc_surfaces.py | 13 +- tests/test_release_packs.py | 6 +- tests/test_site_bootstrap.py | 2 +- tests/test_site_demo.py | 4 +- tests/test_site_integration.py | 2 +- 11 files changed, 176 insertions(+), 428 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e85902a47..7acb100ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,7 @@ handheld launchers, `bwrap` (bubblewrap) for the probe sandbox, and a Python 3.8 with pyte (`python3.8 -m pip install "pyte==0.8.2"`) for the full screen. CI has all three. `profile_sync.py` compares the branch with this repository's `main`, and -fetches it in a fork's clone. A profile you only edit is not held to +fetches it in a fork's clone. An edited profile is not held to `--strict`: its older refs are reviewed by hand. The full pipeline is not needed for a contribution: it rebuilds every pack diff --git a/README.md b/README.md index d927af444..4814dbab5 100644 --- a/README.md +++ b/README.md @@ -5,183 +5,80 @@

-

- Site -

+

The BIOS files emulators need, put in the right folder and checked.

-BIOS and firmware files for AmberELEC, Batocera, BizHawk, EmuDeck, EmuELEC, KNULLI, Lakka, MiSTer FPGA, MinUI, MyMinUI, NextUI, ROCKNIX, Recalbox, RetroArch, RetroBat, RetroDECK, RetroPie, and RomM. +

Website · Get the files · A game does not start · Questions

-A pack holds the files its emulators read, as their source code names them, and leaves out what the emulators install themselves. +Some consoles need a small program, the **BIOS**, to start their games: PlayStation 1 and 2, Saturn, Neo Geo, Dreamcast and more. RetroBIOS installs the BIOS and firmware files for AmberELEC, Batocera, BizHawk, EmuDeck, EmuELEC, KNULLI, Lakka, MiSTer FPGA, MinUI, MyMinUI, NextUI, ROCKNIX, Recalbox, RetroArch, RetroBat, RetroDECK, RetroPie, and RomM, checked the way each one checks them. -## Quick install +## Install in one line -Copy one command into your terminal: +**Windows**: open the Start menu, type **PowerShell** and open it. Paste this line (right-click in the window) and press Enter. + +```powershell +irm https://abdess.github.io/retrobios/i.ps1 | iex +``` + +**Linux, macOS, Steam Deck, Android (Termux)**: open a terminal, paste this line and press Enter. ```bash -# Linux / macOS / Steam Deck curl -fsSL https://abdess.github.io/retrobios/i | sh - -# Windows (PowerShell) -irm https://abdess.github.io/retrobios/i.ps1 | iex - -# Android (Termux: it installs Python and asks for storage access) -curl -fsSL https://abdess.github.io/retrobios/i | sh - -# SD card mounted on a PC: a muOS, KNULLI, ROCKNIX, Lakka, MiSTer, Onion, spruceOS or MinUI card -# is found by itself. For another card, --dest is its BIOS folder -# (RetroArch on Switch: retroarch/cores/system, on Android: RetroArch/system) -curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --platform retroarch --dest /path/to/sdcard/retroarch/cores/system - -# Options in PowerShell: anything after | iex goes to iex, so the script runs as a block -&([scriptblock]::Create((irm https://abdess.github.io/retrobios/i.ps1))) --check ``` +**A handheld, an SD card, or not sure?** [Pick the device](https://abdess.github.io/retrobios/which-pack/): its page gives the folder and the command. A muOS, Onion, spruceOS, MinUI, KNULLI, ROCKNIX, Lakka or MiSTer SD card plugged into the computer is found by itself. + ![Recording of the one-line command in a terminal. The installer finds an ArkOS SD card with 6 games, plans 43 BIOS files (15.1 MB) for them, installs them and shows the summary.](docs_assets/demo/install.gif) -The script opens a full-screen installer, driven with the arrow keys and Enter, or with a controller on a handheld. It finds your setup and its BIOS folder, downloads only the missing files and checks each one. It can also copy the BIOS files from your old SD card. -Every option, and what the installer detects and writes, is on the [installer page](https://abdess.github.io/retrobios/wiki/installer/#passing-options). +The installer finds the setup and its BIOS folder, installs the files it needs (only what the games need when it finds them on the card) and checks each one. Its screen installs everything, or fewer under **Choose systems**. Arrow keys and Enter drive it, or a controller on a handheld. It sends nothing back ([what is counted](https://abdess.github.io/retrobios/wiki/privacy/)). Options go after the line in the form the [installer page](https://abdess.github.io/retrobios/wiki/installer/#passing-options) gives. -First time? Follow the [first install, step by step](https://abdess.github.io/retrobios/wiki/first-install/). A game does not start: [what to check](https://abdess.github.io/retrobios/wiki/troubleshooting/). Other questions: the [FAQ](https://abdess.github.io/retrobios/wiki/faq/). +First time? [The first install, step by step](https://abdess.github.io/retrobios/wiki/first-install/). A game does not start: [what to check](https://abdess.github.io/retrobios/wiki/troubleshooting/). Other questions: [FAQ](https://abdess.github.io/retrobios/wiki/faq/). -## Download BIOS packs +## Or download a ZIP -A pack holds its platform's own BIOS list plus the files its emulator cores load. What no pack carries yet is in the [gap analysis](https://abdess.github.io/retrobios/gaps/). Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **Choose systems**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below. +The Files column counts what a pack holds once unzipped. -A pack over 2 GB comes in several parts (`Pack.part1of2.zip`, `Pack.part2of2.zip`), and every part is needed. Each part is an ordinary ZIP: extract them all into the same folder. - -Every release ships `SHA256SUMS.txt` and a detached signature of it, checkable against `allowed_signers` in this repository: [verifying a release](https://abdess.github.io/retrobios/wiki/release-process/#verifying-a-release). - -The Files column counts what a pack holds once extracted, the figure a file manager shows for the folder. It differs from one platform to the next because a pack carries only what that platform's emulators load. The size is what the files occupy once extracted, and the ZIP itself downloads smaller. - -| Platform | Files | Extracted size | Extract to | Download | -|----------|------:|---------------:|-----------|----------| -| AmberELEC | - | - | `/storage/roms/` | [Use the command](https://abdess.github.io/retrobios/get/amberelec/) | +| Setup | Files | Size | Unzip into | Download | +|-------|------:|-----:|------------|----------| | Batocera | 12,810 | 7.5 GB | `/userdata/bios/` | [Download](https://abdess.github.io/retrobios/get/batocera/) | -| BizHawk | - | - | `Firmware/` | [Use the command](https://abdess.github.io/retrobios/get/bizhawk/) | | EmuDeck | 620 | 3.2 GB | `~/Emulation/bios/` | [Download](https://abdess.github.io/retrobios/get/emudeck/) | -| EmuELEC | - | - | `/storage/roms/` | [Use the command](https://abdess.github.io/retrobios/get/emuelec/) | -| KNULLI | - | - | `/userdata/bios/` | [Use the command](https://abdess.github.io/retrobios/get/knulli/) | -| Lakka | - | - | `/storage/system/` | [Use the command](https://abdess.github.io/retrobios/get/lakka/) | | MiSTer FPGA | 83 | 29 MB | `/media/fat/games/` | [Download](https://abdess.github.io/retrobios/get/misterfpga/) | -| MinUI | - | - | `/` | [Use the command](https://abdess.github.io/retrobios/get/minui/) | -| MyMinUI | - | - | `/` | [Use the command](https://abdess.github.io/retrobios/get/myminui/) | -| NextUI | - | - | `/` | [Use the command](https://abdess.github.io/retrobios/get/nextui/) | -| ROCKNIX | - | - | `/storage/roms/bios/` | [Use the command](https://abdess.github.io/retrobios/get/rocknix/) | -| Recalbox | - | - | `/recalbox/share/bios/` | [Use the command](https://abdess.github.io/retrobios/get/recalbox/) | | RetroArch | 8,504 | 6.0 GB | `system/` | [Download](https://abdess.github.io/retrobios/get/retroarch/) | | RetroBat | 12,399 | 7.9 GB | `bios/` | [Download](https://abdess.github.io/retrobios/get/retrobat/) | -| RetroDECK | - | - | `~/retrodeck/` | [Use the command](https://abdess.github.io/retrobios/get/retrodeck/) | -| RetroPie | - | - | `~/RetroPie/BIOS/` | [Use the command](https://abdess.github.io/retrobios/get/retropie/) | | RomM | 1,087 | 1.5 GB | `library/bios/` | [Download](https://abdess.github.io/retrobios/get/romm/) | -ZIPs are made for the setups most people download. The others read "Use the command": the installer above installs every file for them and checks each one, and `--dry-run` shows what it would install ([why no ZIP](https://abdess.github.io/retrobios/wiki/faq/#why-is-there-no-zip-for-my-setup)). +ZIPs are made for the setups most people download. The others install with the line above, and each page gives the command: [AmberELEC](https://abdess.github.io/retrobios/get/amberelec/), [BizHawk](https://abdess.github.io/retrobios/get/bizhawk/), [EmuELEC](https://abdess.github.io/retrobios/get/emuelec/), [KNULLI](https://abdess.github.io/retrobios/get/knulli/), [Lakka](https://abdess.github.io/retrobios/get/lakka/), [MinUI](https://abdess.github.io/retrobios/get/minui/), [MyMinUI](https://abdess.github.io/retrobios/get/myminui/), [NextUI](https://abdess.github.io/retrobios/get/nextui/), [ROCKNIX](https://abdess.github.io/retrobios/get/rocknix/), [Recalbox](https://abdess.github.io/retrobios/get/recalbox/), [RetroDECK](https://abdess.github.io/retrobios/get/retrodeck/), [RetroPie](https://abdess.github.io/retrobios/get/retropie/). -## What's included +Unzip into the BIOS folder, files straight in it. A big pack comes in parts (`Pack.part1of2.zip`): download every part ([why some setups have no ZIP](https://abdess.github.io/retrobios/wiki/faq/#why-is-there-no-zip-for-my-setup)). Each release lists its files' SHA-256 with a signature ([verifying a release](https://abdess.github.io/retrobios/wiki/release-process/#verifying-a-release)). -BIOS, firmware, and system files for consoles from Atari to PlayStation 3. -These are the files an emulator loads from disk instead of carrying inside itself. Some are required to boot a system, others improve accuracy or unlock a feature, and the packs carry both. +## Help -Each file is checked the way your platform checks it. Most compare a checksum, the fingerprint of a file's contents, which catches a corrupt or unexpected copy. RetroArch, Lakka and RetroPie check that the file is present under its name without reading its contents (`path_is_valid` in RetroArch's `core_info.c`), and MinUI, NextUI and MyMinUI hand each core its folder without opening a file in it: the Coverage table says which applies to you. Independently of that, the collection records five fingerprints per file, and wherever an emulator's code states an expected size or hash, that value is read from its source and rechecked here. +- "BIOS missing" or a game does not start: [what to check](https://abdess.github.io/retrobios/wiki/troubleshooting/) +- Have a file? [Check it by its hash](https://abdess.github.io/retrobios/files/) +- Every console and the files it needs: [systems](https://abdess.github.io/retrobios/systems/) +- Asking an AI assistant: it reads the site best from [llms.txt](https://abdess.github.io/retrobios/llms.txt) -- **1,087 files** the platforms' emulators load are still to be found, and 138 more the collection does not hold and does not serve by name (retail data, per-user keys, user-filled slots), each with its reason. Both are named in the [gap analysis](https://abdess.github.io/retrobios/gaps/) -- **18 platforms** supported with platform-specific verification -- **623 emulators** profiled from source (RetroArch cores + standalone) -- **532 systems** handled by those emulators (NES, SNES, PlayStation, Saturn, Dreamcast, ...) -- **19,947 files in the collection**, each with its SHA1, MD5, SHA256, CRC32 and Adler-32 fingerprints: 5,744 system files, 2,975 arcade ROM sets, 11,228 game and engine data files. That is every platform and emulator together, so no pack holds them all: each carries what its own emulators load, counted in the download table -- **566 of 5,744 system files** matched to dump-preservation catalogs (No-Intro, Redump, TOSEC). Arcade sets and engine data fall outside what those catalogs index -- **15.8 GB** for the whole collection +## What makes it different -## Supported systems +- **623 emulators profiled from their source code**: each file an emulator loads, its hash and the line of code that reads it +- **18 setups**, each checked its own way: MD5 for Batocera and Recalbox, the file name for RetroArch, as their own code does +- **532 systems**, 19,947 files in the collection, [what is still missing](https://abdess.github.io/retrobios/gaps/) named file by file +- Public data: [JSON, CSV and SQLite](https://abdess.github.io/retrobios/data/), MIT licensed -NES, SNES, Nintendo 64, GameCube, Wii, Game Boy, Game Boy Advance, Nintendo DS, Nintendo 3DS, Switch, PlayStation, PlayStation 2, PlayStation 3, PSP, PS Vita, Mega Drive, Saturn, Dreamcast, Game Gear, Master System, Neo Geo, Atari 2600, Atari 7800, Atari Lynx, Atari ST, MSX, PC Engine, TurboGrafx-16, ColecoVision, Intellivision, Commodore 64, Amiga, ZX Spectrum, Arcade (MAME), and 498+ more. - -Full list with per-file details: **[https://abdess.github.io/retrobios/](https://abdess.github.io/retrobios/)** - -## Coverage - -| Platform | On its BIOS list | Files its emulators load | Checked by | -|----------|-----------------:|-------------------------:|------------| -| AmberELEC | 414/434 | 4,754/4,937 | MD5 | -| Batocera | 353/353 | 14,171/15,291 | MD5 | -| BizHawk | 118/118 | - | SHA-1 | -| EmuDeck | 168/168 | 8,426/8,528 | MD5 | -| EmuELEC | 543/572 | 4,788/10,217 | MD5 | -| KNULLI | 290/290 | 9,126/15,350 | MD5 | -| Lakka | 532/532 | 7,909/7,994 | name only | -| MiSTer FPGA | 81/81 | - | MD5 | -| MinUI | 0/0 | 78/79 | name only | -| MyMinUI | 0/0 | 356/367 | name only | -| NextUI | 0/0 | 343/350 | name only | -| ROCKNIX | 38/38 | 7,427/9,139 | MD5 | -| Recalbox | 351/351 | 3,125/3,344 | MD5 | -| RetroArch | 526/526 | 7,918/7,985 | name only | -| RetroBat | 348/348 | 5,135/9,659 | MD5 | -| RetroDECK | 2,007/2,008 | 6,705/6,992 | MD5 | -| RetroPie | 532/532 | 8,117/12,771 | name only | -| RomM | 383/383 | 83/874 | size, then any hash | - -Each fraction is what the pack has over what is needed, counting required and optional files alike since both ship. The first column is the BIOS list the platform publishes. The second counts files its emulators load that this list never mentions, found by reading their source code, and it is routinely several times larger. A short fraction means files are still missing, and they are named in the [gap analysis](https://abdess.github.io/retrobios/gaps/). -That second number is a floor, not a ceiling: an emulator that accepts any file handed to it names none in its code, so nothing there can be counted. -Checked by is the test your platform runs on its own, replicated here from its source code, so the result matches what you would see in the frontend ([how each one works](https://abdess.github.io/retrobios/wiki/verification-modes/)). - -## Build your own pack +
+For developers: build a pack Packs are built from a clone, and the clone carries the collection: its checkout alone is over 7.7 GB. `git clone --depth 1 https://github.com/Abdess/retrobios.git` leaves the history out. Files kept as release assets are downloaded when a pack needs them. Then generate packs for any platform, emulator, or system: ```bash -# The pack of a platform, as released python scripts/generate_pack.py --platform retroarch --output-dir dist/ -python scripts/generate_pack.py --platform batocera --output-dir dist/ - -# Single emulator or system python scripts/generate_pack.py --emulator dolphin -python scripts/generate_pack.py --system sony-playstation-2 - -# One region, best first, one file per system and region -python scripts/generate_pack.py --platform retroarch --region us,eu,jp -python scripts/generate_pack.py --platform retroarch --region us --one-per-slot - -# Firmware only: the game data a list or a core names is left out, each file named -python scripts/generate_pack.py --platform emuelec --no-game-data - -# The bare minimum: only the files each core needs to start python scripts/generate_pack.py --platform batocera --required-only - -# List available emulators and systems -python scripts/generate_pack.py --list-emulators -python scripts/generate_pack.py --list-systems - -# Coverage of this repository's collection per platform, emulator or system -# (it reads database.json and bios/: for your own BIOS folder, run the installer with --check) -python scripts/verify.py --all python scripts/verify.py --platform batocera -python scripts/verify.py --emulator flycast -python scripts/verify.py --platform retroarch --verbose # emulator ground truth ``` -Requires Python 3.9 or newer and `pyyaml`. The one-line installer alone runs on Python 3.8. +Requires Python 3.9 or newer and `pyyaml`. The one-line installer alone runs on Python 3.8. Every tool and option: [tools](https://abdess.github.io/retrobios/wiki/tools/). How the lists are built: [architecture](https://abdess.github.io/retrobios/wiki/architecture/). -## Documentation site - -The [documentation site](https://abdess.github.io/retrobios/) provides: - -- **Per-platform pages** with file-by-file verification status and hashes -- **Per-emulator profiles** with source code references for every file -- **Per-system pages** showing which emulators and platforms cover each console -- **Gap analysis** identifying missing files and undeclared core requirements -- **Cross-reference** mapping files across 18 platforms and 623 emulators -- **Versioned data access** through JSON, CSV and SQLite exports with published SHA-256 checksums - -## How it works - -A platform's BIOS list and an emulator's code are maintained separately. Each list is compared with the emulator source code, file by file where a profile exists, and where the two differ the code decides. - -Hashes document what emulator code loads and accepts, not dump provenance. That boundary, and how it relates to preservation catalogs such as No-Intro, is drawn in the [technical questions](https://abdess.github.io/retrobios/wiki/technical-faq/#are-these-files-verified-against-original-hardware-dumps). - -1. **Read emulator source code**: trace every file the code loads, its expected hash and size -2. **Cross-reference with platforms**: match against what each platform declares -3. **Build packs**: include baseline files plus what each platform's cores need -4. **Verify**: run platform-native checks and emulator-level validation +
## Contributors @@ -196,7 +93,7 @@ Hashes document what emulator code loads and accepts, not dump provenance. That ## Community tools -- [BIOS Preservation Tool](https://github.com/monster-penguin/BIOS-Preservation-Tool) by [monster-penguin](https://github.com/monster-penguin) - scan, verify, and stage your own BIOS collection using RetroBIOS hash metadata +- [BIOS Preservation Tool](https://github.com/monster-penguin/BIOS-Preservation-Tool) by [monster-penguin](https://github.com/monster-penguin) - scans, verifies and stages a BIOS collection using RetroBIOS hash metadata ## Contributing @@ -208,4 +105,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-11-06T23:09:06Z* +*Auto-generated on 2026-11-09T06:27:53Z* diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index f7b8aa383..3c6ed58b1 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -594,6 +594,15 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" system_ids = system_index(profiles) + # The README is the repository's front page and its most read page: a + # visitor from a search lands here, often new to terminals. It says what + # the project does, gives the one line that installs, shows it, and leads + # on to the site, where every detail lives. The figures and the lists it + # once carried (coverage, collection, build commands) are the site's. + record = release_record.load_advertised(platforms_dir=platforms_dir) + extract_paths = _extract_cells(platforms_dir) + released = [name for name in sorted(coverages, key=lambda n: coverages[n]["platform"]) + if release_totals(name, record)[0]] lines = [ '

', " ", @@ -602,281 +611,118 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" " ", "

", "", - '

', - ' ' - 'Site', + '

The BIOS files emulators need, put in the right folder and checked.

', + "", + '

' + f'Website · ' + f'Get the files · ' + f'A game does not start · ' + f'Questions' "

", "", - f"BIOS and firmware files for {_names_list(coverages)}.", + "Some consoles need a small program, the **BIOS**, to start their games: PlayStation 1" + " and 2, Saturn, Neo Geo, Dreamcast and more. RetroBIOS installs the BIOS and firmware" + f" files for {_names_list(coverages)}, checked the way each one checks them.", "", - "A pack holds the files its emulators read, as their source code names" - " them, and leaves out what the emulators install themselves.", + "## Install in one line", "", - "## Quick install", + "**Windows**: open the Start menu, type **PowerShell** and open it. Paste this line" + " (right-click in the window) and press Enter.", "", - "Copy one command into your terminal:", + "```powershell", + install_command("powershell"), + "```", + "", + "**Linux, macOS, Steam Deck, Android (Termux)**: open a terminal, paste this line" + " and press Enter.", "", "```bash", - "# Linux / macOS / Steam Deck", install_command("sh"), - "", - "# Windows (PowerShell)", - install_command("powershell"), - "", - "# Android (Termux: it installs Python and asks for storage access)", - install_command("sh"), - "", - "# SD card mounted on a PC: a muOS, KNULLI, ROCKNIX, Lakka, MiSTer, Onion, spruceOS or MinUI card", - "# is found by itself. For another card, --dest is its BIOS folder", - "# (RetroArch on Switch: retroarch/cores/system, on Android: RetroArch/system)", - install_command("sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system"), - "", - "# Options in PowerShell: anything after | iex goes to iex, so the script runs as a block", - install_command("powershell", "--check"), "```", "", + f"**A handheld, an SD card, or not sure?** [Pick the device]({SITE_URL}which-pack/):" + " its page gives the folder and the command. A muOS, Onion, spruceOS, MinUI, KNULLI," + " ROCKNIX, Lakka or MiSTer SD card plugged into the computer is found by itself.", + "", f"![{demo_description()}]({DEMO_GIF})", "", - "The script opens a full-screen installer, driven with the arrow keys and Enter, or with a" - " controller on a handheld. It finds your setup and its BIOS folder, downloads only the missing" - " files and checks each one. It can also copy the BIOS files from your old SD card.", - f"Every option, and what the installer detects and writes, is on the [installer page]({SITE_URL}wiki/installer/#passing-options).", + "The installer finds the setup and its BIOS folder, installs the files it needs (only" + " what the games need when it finds them on the card) and checks each one. Its screen" + " installs everything, or fewer under **" + _installer_words().TUI_SYSTEMS_ROW + "**." + " Arrow keys and Enter drive it, or a controller on a handheld. It sends nothing back" + f" ([what is counted]({SITE_URL}wiki/privacy/)). Options go after the line in the" + f" form the [installer page]({SITE_URL}wiki/installer/#passing-options) gives.", "", - f"First time? Follow the [first install, step by step]({SITE_URL}wiki/first-install/)." + f"First time? [The first install, step by step]({SITE_URL}wiki/first-install/)." f" A game does not start: [what to check]({SITE_URL}wiki/troubleshooting/)." - f" Other questions: the [FAQ]({SITE_URL}wiki/faq/).", + f" Other questions: [FAQ]({SITE_URL}wiki/faq/).", "", - "## Download BIOS packs", + "## Or download a ZIP", "", - f"A pack holds its platform's own BIOS list plus the files its emulator cores load. What no pack carries yet is in the [gap analysis]({SITE_URL}gaps/). Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **" + _installer_words().TUI_SYSTEMS_ROW + "**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.", - "", - "A pack over 2 GB comes in several parts (`Pack.part1of2.zip`," - " `Pack.part2of2.zip`), and every part is needed. Each part is an" - " ordinary ZIP: extract them all into the same folder.", - "", - "Every release ships `SHA256SUMS.txt` and a detached signature of it," - " checkable against `allowed_signers` in this repository:" - " [verifying a release]" - "(https://abdess.github.io/retrobios/wiki/release-process/" - "#verifying-a-release).", - "", - "The Files column counts what a pack holds once extracted, the figure a" - " file manager shows for the folder. It differs from one platform to" - " the next because a pack carries only what that platform's emulators" - " load. The size is what the files occupy once extracted, and the ZIP" - " itself downloads smaller.", + "The Files column counts what a pack holds once unzipped.", "", + "| Setup | Files | Size | Unzip into | Download |", + "|-------|------:|-----:|------------|----------|", ] - - # Where the pack itself is extracted, which is not always the BIOS folder: - # a pack whose entries already carry their own root (RetroDECK) extracts - # one level above it. - extract_paths = _extract_cells(platforms_dir) - - lines.extend( - download_table( - coverages, extract_paths, - release_record.load_advertised(platforms_dir=platforms_dir), - ) - ) - lines.extend(["", no_pack_line()]) - - lines.extend(_paragraphs(extract_notes(platforms_dir, released_only=True))) - - lines.extend( - [ - "", - "## What's included", - "", - "BIOS, firmware, and system files for consoles from Atari to PlayStation 3.", - "These are the files an emulator loads from disk instead of" - " carrying inside itself. Some are required to boot a system," - " others improve accuracy or unlock a feature, and the packs carry" - " both.", - "", - "Each file is checked the way your platform checks it. Most compare" - " a checksum, the fingerprint of a file's contents, which catches a" - " corrupt or unexpected copy. RetroArch, Lakka and RetroPie check" - " that the file is present under its name without reading its" - " contents (`path_is_valid` in RetroArch's `core_info.c`), and" - " MinUI, NextUI and MyMinUI hand each core its folder without" - " opening a file in it: the" - " Coverage table says which applies to you. Independently of that," - " the collection records five fingerprints per file, and wherever" - " an emulator's code states an expected size or hash, that value is" - " read from its source and rechecked here.", - "", - ( - f"- **{acquirable_total:,} files** the platforms' emulators load" - f" are still to be found, and {unsourceable_total:,} more the" - f" collection does not hold and does not serve by name (retail" - f" data, per-user keys, user-filled slots), each with its reason." - f" Both are named in the" - f" [gap analysis]({SITE_URL}gaps/)" - if missing_total - else "- **Nothing missing**: every file the platforms'" - " emulators load is in the collection" - ), - f"- **{len(coverages)} platforms** supported with platform-specific verification", - f"- **{emulator_count} emulators** profiled from source (RetroArch cores + standalone)", - f"- **{len(system_ids)} systems** handled by those emulators (NES, SNES, PlayStation, Saturn, Dreamcast, ...)", - collection_line(total_files, comp), - *_catalog_matched_line(db), - f"- **{format_size(total_size)}** for the whole collection", - "", - "## Supported systems", - "", - ] - ) - - # Show well-known systems for SEO, link to full list - well_known = [ - "NES", - "SNES", - "Nintendo 64", - "GameCube", - "Wii", - "Game Boy", - "Game Boy Advance", - "Nintendo DS", - "Nintendo 3DS", - "Switch", - "PlayStation", - "PlayStation 2", - "PlayStation 3", - "PSP", - "PS Vita", - "Mega Drive", - "Saturn", - "Dreamcast", - "Game Gear", - "Master System", - "Neo Geo", - "Atari 2600", - "Atari 7800", - "Atari Lynx", - "Atari ST", - "MSX", - "PC Engine", - "TurboGrafx-16", - "ColecoVision", - "Intellivision", - "Commodore 64", - "Amiga", - "ZX Spectrum", - "Arcade (MAME)", - ] - lines.extend( - [ - ", ".join(well_known) + f", and {len(system_ids) - len(well_known)}+ more.", - "", - f"Full list with per-file details: **[{SITE_URL}]({SITE_URL})**", - "", - "## Coverage", - "", - "| Platform | On its BIOS list | Files its emulators load | Checked by |", - "|----------|-----------------:|-------------------------:|------------|", - ] - ) - - for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): - display = cov["platform"] - checked = check_label(cov["mode"], cov["config"].get("hash_rule")) - core_total = ( - cov["core_present"] + cov["core_missing"] + cov["core_unsourceable"] - ) - core_cell = f"{cov['core_present']:,}/{core_total:,}" if core_total else "-" + for name in released: + files, size = release_totals(name, record) + display = coverages[name]["platform"] lines.append( - f"| {display} | {cov['present']:,}/{cov['total']:,} |" - f" {core_cell} | {checked} |" + f"| {display} | {files:,} | {format_size(size) if size else '-'} |" + f" {extract_paths.get(display, '')} |" + f" {download_links(name, record, 'Download', site=SITE_URL)} |" ) - - lines.extend( - [ - "", - "Each fraction is what the pack has over what is needed, counting" - " required and optional files alike since both ship. The first" - " column is the BIOS list the platform publishes. The second counts" - " files its emulators load that this list never mentions, found by" - " reading their source code, and it is routinely several times" - " larger. A short fraction means files are still missing, and they" - " are named in the" - f" [gap analysis]({SITE_URL}gaps/).", - "That second number is a floor, not a ceiling: an emulator that" - " accepts any file handed to it names none in its code, so nothing" - " there can be counted.", - "Checked by is the test your platform runs on its own, replicated" - " here from its source code, so the result matches what you would" - f" see in the frontend ([how each one works]({SITE_URL}wiki/verification-modes/)).", - "", - "## Build your own pack", - "", - clone_line(db), - "", - "```bash", - "# The pack of a platform, as released", - "python scripts/generate_pack.py --platform retroarch --output-dir dist/", - "python scripts/generate_pack.py --platform batocera --output-dir dist/", - "", - "# Single emulator or system", - "python scripts/generate_pack.py --emulator dolphin", - "python scripts/generate_pack.py --system sony-playstation-2", - "", - "# One region, best first, one file per system and region", - "python scripts/generate_pack.py --platform retroarch --region us,eu,jp", - "python scripts/generate_pack.py --platform retroarch --region us --one-per-slot", - "", - "# Firmware only: the game data a list or a core names is left out, each file named", - "python scripts/generate_pack.py --platform emuelec --no-game-data", - "", - "# The bare minimum: only the files each core needs to start", - "python scripts/generate_pack.py --platform batocera --required-only", - "", - "# List available emulators and systems", - "python scripts/generate_pack.py --list-emulators", - "python scripts/generate_pack.py --list-systems", - "", - "# Coverage of this repository's collection per platform, emulator or system", - "# (it reads database.json and bios/: for your own BIOS folder, run the installer with --check)", - "python scripts/verify.py --all", - "python scripts/verify.py --platform batocera", - "python scripts/verify.py --emulator flycast", - "python scripts/verify.py --platform retroarch --verbose # emulator ground truth", - "```", - "", - "Requires Python 3.9 or newer and `pyyaml`. The one-line installer alone runs on Python 3.8.", - "", - "## Documentation site", - "", - f"The [documentation site]({SITE_URL}) provides:", - "", - "- **Per-platform pages** with file-by-file verification status and hashes", - "- **Per-emulator profiles** with source code references for every file", - "- **Per-system pages** showing which emulators and platforms cover each console", - "- **Gap analysis** identifying missing files and undeclared core requirements", - f"- **Cross-reference** mapping files across {len(coverages)} platforms and {emulator_count} emulators", - "- **Versioned data access** through JSON, CSV and SQLite exports with published SHA-256 checksums", - "", - "## How it works", - "", - "A platform's BIOS list and an emulator's code are maintained" - " separately. Each list is compared with the emulator source code," - " file by file where a profile exists, and where the two differ the" - " code decides.", - "", - "Hashes document what emulator code loads and accepts, not dump" - " provenance. That boundary, and how it relates to preservation" - " catalogs such as No-Intro, is drawn in the" - f" [technical questions]({SITE_URL}wiki/technical-faq/" - "#are-these-files-verified-against-original-hardware-dumps).", - "", - "1. **Read emulator source code**: trace every file the code loads, its expected hash and size", - "2. **Cross-reference with platforms**: match against what each platform declares", - "3. **Build packs**: include baseline files plus what each platform's cores need", - "4. **Verify**: run platform-native checks and emulator-level validation", - "", - ] - ) + others = [name for name in sorted(coverages, key=lambda n: coverages[n]["platform"]) + if name not in released] + lines.extend([ + "", + "ZIPs are made for the setups most people download. The others install with the line" + " above, and each page gives the command: " + + ", ".join(f"[{coverages[name]['platform']}]({SITE_URL}get/{name}/)" for name in others) + + ".", + "", + "Unzip into the BIOS folder, files straight in it. A big pack comes in parts" + " (`Pack.part1of2.zip`): download every part" + f" ([why some setups have no ZIP]({SITE_URL}wiki/faq/#why-is-there-no-zip-for-my-setup))." + " Each release lists its files' SHA-256 with a signature" + f" ([verifying a release]({SITE_URL}wiki/release-process/#verifying-a-release)).", + "", + "## Help", + "", + f"- \"BIOS missing\" or a game does not start: [what to check]({SITE_URL}wiki/troubleshooting/)", + f"- Have a file? [Check it by its hash]({SITE_URL}files/)", + f"- Every console and the files it needs: [systems]({SITE_URL}systems/)", + f"- Asking an AI assistant: it reads the site best from [llms.txt]({SITE_URL}llms.txt)", + "", + "## What makes it different", + "", + f"- **{emulator_count} emulators profiled from their source code**: each file an emulator" + " loads, its hash and the line of code that reads it", + f"- **{len(coverages)} setups**, each checked its own way: MD5 for Batocera and Recalbox," + " the file name for RetroArch, as their own code does", + f"- **{len(system_ids)} systems**, {total_files:,} files in the collection," + f" [what is still missing]({SITE_URL}gaps/) named file by file", + f"- Public data: [JSON, CSV and SQLite]({SITE_URL}data/), MIT licensed", + "", + "
", + "For developers: build a pack", + "", + clone_line(db), + "", + "```bash", + "python scripts/generate_pack.py --platform retroarch --output-dir dist/", + "python scripts/generate_pack.py --emulator dolphin", + "python scripts/generate_pack.py --platform batocera --required-only", + "python scripts/verify.py --platform batocera", + "```", + "", + "Requires Python 3.9 or newer and `pyyaml`. The one-line installer alone runs on Python 3.8." + f" Every tool and option: [tools]({SITE_URL}wiki/tools/). How the lists are built:" + f" [architecture]({SITE_URL}wiki/architecture/).", + "", + "
", + "", + ]) contributors = fetch_contributors() if not contributors: @@ -910,7 +756,7 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" "", "- [BIOS Preservation Tool](https://github.com/monster-penguin/BIOS-Preservation-Tool)" " by [monster-penguin](https://github.com/monster-penguin)" - " - scan, verify, and stage your own BIOS collection" + " - scans, verifies and stages a BIOS collection" " using RetroBIOS hash metadata", "", "## Contributing", @@ -1053,7 +899,7 @@ handheld launchers, `bwrap` (bubblewrap) for the probe sandbox, and a Python 3.8 with pyte (`{floor_line}`) for the full screen. CI has all three. `profile_sync.py` compares the branch with this repository's `main`, and -fetches it in a fork's clone. A profile you only edit is not held to +fetches it in a fork's clone. An edited profile is not held to `--strict`: its older refs are reviewed by hand. The full pipeline is not needed for a contribution: it rebuilds every pack diff --git a/scripts/site_markdown.py b/scripts/site_markdown.py index c72b972e3..db9e5b143 100644 --- a/scripts/site_markdown.py +++ b/scripts/site_markdown.py @@ -20,11 +20,15 @@ on the built site: from __future__ import annotations import argparse +import os import re import sys from html.parser import HTMLParser from pathlib import Path +sys.path.insert(0, os.path.dirname(__file__)) +from common import run_cli # noqa: E402 + SITE = "https://abdess.github.io" BASE = "/retrobios/" # Elements whose content is never words a reader needs. @@ -342,4 +346,4 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + run_cli(main) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 38832433e..ea06e86c9 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -56,8 +56,8 @@ class ReadmeRegressions(unittest.TestCase): def _quick_install(self) -> str: readme = (ROOT / "README.md").read_text(encoding="utf-8") - return readme.split("## Quick install", 1)[1].split( - "## Download BIOS packs", 1 + return readme.split("## Install in one line", 1)[1].split( + "## Or download a ZIP", 1 )[0] def test_advertised_bootstrap_urls_are_the_shipped_ones(self): @@ -1121,7 +1121,10 @@ class CatalogRatioRegressions(unittest.TestCase): match = re.search( r"\*\*([\d,]+) of ([\d,]+) system files\*\* matched to", readme ) - self.assertIsNotNone(match, "README no longer states the catalog ratio") + if match is None: + # The README states no ratio since it leads to the site: a ratio + # it states must still be the database's. + return matched, total = (int(g.replace(",", "")) for g in match.groups()) self.assertEqual(matched, count_catalog_matched(db)) self.assertEqual(total, compute_composition(db)["systems"]["files"]) diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 6ca5886f1..0b025c755 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -340,7 +340,7 @@ class FilesColumnParagraph(unittest.TestCase): paragraph = next(p for p in readme.split("\n\n") if "The Files column" in p) self.assertNotIn("\n", paragraph.strip()) after = readme.split(paragraph, 1)[1].lstrip("\n") - self.assertTrue(after.startswith("| Platform | Files |")) + self.assertTrue(after.startswith("| Setup | Files |")) class PageOpenings(unittest.TestCase): @@ -769,14 +769,12 @@ class ReadmeInstallerOptions(unittest.TestCase): import generate_readme source = inspect.getsource(generate_readme.generate_readme) - self.assertIn('install_command("powershell", "--check")', source) self.assertIn("wiki/installer/#passing-options", source) self.assertIn("## Passing options", _page("installer.md")) readme = (ROOT / "README.md").read_text(encoding="utf-8") - quick = _section(readme, "Quick install") - from siteguides import INSTALL_PS1 - - self.assertIn(f"&([scriptblock]::Create((irm {INSTALL_PS1}))) --check", quick) + quick = _section(readme, "Install in one line") + # The README gives the bare line and sends options to the page that + # writes their PowerShell form: an option after `| iex` reaches iex. self.assertIn("wiki/installer/#passing-options", quick) self.assertNotRegex(readme, r"(?m)^irm \S+ \| iex \S") @@ -791,7 +789,7 @@ class PublishedNumbersShareOneFormat(unittest.TestCase): readme_source = inspect.getsource(generate_readme.generate_readme) self.assertNotIn("size_mb", readme_source) - self.assertIn("format_size(total_size)", readme_source) + self.assertIn("format_size(size)", readme_source) self.assertIn("{total_files:,}", inspect.getsource(generate_site)) self.assertEqual(generate_readme.format_size(16011 * 1024 * 1024), "15.6 GB") @@ -1713,9 +1711,7 @@ class ReadmeKeepsItsPointers(unittest.TestCase): """The README sent a reader to 'build your own pack below' for a bare minimum, and the block below never named --required-only.""" readme = (ROOT / "README.md").read_text(encoding="utf-8") - self.assertIn("For a bare minimum, build your own pack below.", readme) - block = readme[readme.index("## Build your own pack"):] - block = block[:block.index("\n## ", 1)] + block = readme[readme.index("For developers"):readme.index("")] self.assertIn("--required-only", block) def test_unsourceable_files_are_not_said_to_exist_nowhere(self): @@ -1725,7 +1721,8 @@ class ReadmeKeepsItsPointers(unittest.TestCase): readme = (ROOT / "README.md").read_text(encoding="utf-8") self.assertNotIn("sourced at all", readme) self.assertNotIn("nobody has made", readme) - self.assertRegex(readme, r"more the collection does not hold and does not serve by name") + # The README names no count: the gap analysis it links does, by reason. + self.assertIn("gaps/", readme) class ArchitectureStatesTheArbitration(unittest.TestCase): @@ -2041,8 +2038,8 @@ class BuildYourOwnPackSaysWhatTheCloneCosts(unittest.TestCase): line = generate_readme.clone_line(db) self.assertIn("git clone --depth 1", line) self.assertRegex(line, r"over \d+(\.\d)? GB") - section = _section((ROOT / "README.md").read_text(encoding="utf-8"), - "Build your own pack") + readme = (ROOT / "README.md").read_text(encoding="utf-8") + section = readme[readme.index("For developers"):readme.index("")] self.assertIn("git clone --depth 1", section) def test_advanced_usage_names_the_shallow_clone(self): diff --git a/tests/test_doc_surfaces.py b/tests/test_doc_surfaces.py index c87d74052..df04632bf 100644 --- a/tests/test_doc_surfaces.py +++ b/tests/test_doc_surfaces.py @@ -191,16 +191,17 @@ class DownloadLinksKeepTheirAddress(unittest.TestCase): from common import list_registered_platforms, list_release_pack_platforms readme = _read("README.md") - table = _section(readme, "## Download BIOS packs") + table = _section(readme, "## Or download a ZIP") self.assertNotIn("/releases/download/", table) released = set(list_release_pack_platforms("platforms")) + rows = [line for line in table.splitlines() if line.startswith("| ")] for key in list_registered_platforms("platforms"): with self.subTest(platform=key): + # Every setup's lasting page is linked: a ZIP's from its row, + # the others from the line that says they install by command. self.assertIn(f"]({SITE_URL}get/{key}/)", table) - row = next(line for line in table.splitlines() - if f"get/{key}/)" in line) - if key not in released: - self.assertIn("[Use the command]", row) + in_rows = any(f"get/{key}/)" in row for row in rows) + self.assertEqual(in_rows, key in released) def test_the_download_page_rows_link_get_pages(self): page = _which_pack() @@ -258,7 +259,7 @@ class TheReadmeLeadsBeginnersToTheirPages(unittest.TestCase): def test_quick_install_links_the_start_pages(self): # The code block holds `#` comment lines, so the section runs to # the next level-two heading. - section = _read("README.md").split("## Quick install", 1)[1].split("\n## ", 1)[0] + section = _read("README.md").split("## Install in one line", 1)[1].split("\n## ", 1)[0] for page in ("first-install", "troubleshooting", "faq"): with self.subTest(page=page): self.assertIn(f"wiki/{page}/)", section) diff --git a/tests/test_release_packs.py b/tests/test_release_packs.py index e3d54147a..65a859c26 100644 --- a/tests/test_release_packs.py +++ b/tests/test_release_packs.py @@ -284,9 +284,9 @@ class EverySurfaceSaysWhy(unittest.TestCase): def test_the_readme_rows_read_use_the_command(self): readme = (REPO_ROOT / "README.md").read_text(encoding="utf-8") - row = next(line for line in readme.splitlines() if line.startswith("| KNULLI |")) - self.assertIn(f"[Use the command]({SITE}get/knulli/)", row) - self.assertIn("ZIPs are made for the setups most people download.", readme) + line = next(text for text in readme.split("\n\n") if "ZIPs are made for the setups most people download." in text) + self.assertIn(f"[KNULLI]({SITE}get/knulli/)", line) + self.assertIn("install with the line above", line) def test_the_faq_answers_why(self): faq = (REPO_ROOT / "wiki" / "faq.md").read_text(encoding="utf-8") diff --git a/tests/test_site_bootstrap.py b/tests/test_site_bootstrap.py index 3b736cad1..b4ed3814d 100644 --- a/tests/test_site_bootstrap.py +++ b/tests/test_site_bootstrap.py @@ -271,7 +271,7 @@ class GeneratedSurfaces(unittest.TestCase): def test_the_readme_shows_the_short_form(self): readme = (REPO_ROOT / "README.md").read_text(encoding="utf-8") - quick = readme.split("## Quick install", 1)[1].split("\n## ", 1)[0] + quick = readme.split("## Install in one line", 1)[1].split("\n## ", 1)[0] self.assertIn(f"\n{install_command('sh')}\n", quick) self.assertIn(f"\n{install_command('powershell')}\n", quick) self.assertNotIn("raw.githubusercontent.com", quick) diff --git a/tests/test_site_demo.py b/tests/test_site_demo.py index 3d653e213..f8d79333d 100644 --- a/tests/test_site_demo.py +++ b/tests/test_site_demo.py @@ -84,8 +84,8 @@ class TheReadmeShowsTheGif(unittest.TestCase): readme = (ROOT / "README.md").read_text(encoding="utf-8") line = f"![{generate_readme.demo_description()}]({generate_readme.DEMO_GIF})" self.assertIn(line, readme) - self.assertLess(readme.index("## Quick install"), readme.index(line)) - self.assertLess(readme.index(line), readme.index("## Download BIOS packs")) + self.assertLess(readme.index("## Install in one line"), readme.index(line)) + self.assertLess(readme.index(line), readme.index("## Or download a ZIP")) def test_the_description_names_what_the_recording_measured(self): facts = _facts() diff --git a/tests/test_site_integration.py b/tests/test_site_integration.py index d2f58137c..c1cfd155a 100644 --- a/tests/test_site_integration.py +++ b/tests/test_site_integration.py @@ -237,7 +237,7 @@ class FaqInstallsFromWhatYouHold(unittest.TestCase): word = rule.name.split()[0] with self.subTest(share=rule.name): self.assertIn(word, section) - self.assertRegex(readme, rf"# SD card mounted on a PC: .*\b{word}\b") + self.assertRegex(readme, rf"\b{word}\b[^.]*SD card plugged into the computer is found by itself") # The label the first screen gives a card found: "Onion SD card". self.assertEqual(install._card_word(("retroarch", Path("/x/BIOS"), "Onion")), "SD card") self.assertEqual(install._card_word(("batocera", Path("/x/bios"), "Batocera")),