From 32c648b67d3116c39e4501e617a77d36d0611f4d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:06:46 +0200 Subject: [PATCH 1/3] feat: release packs for marked platforms only release_pack: true in the registry is the list a release builds: RetroArch, EmuDeck, RetroBat, Batocera, RomM and MiSTer, 83% of the downloads of v2026.10.10. generate_pack --all, --verify-packs, the pipeline consistency step, release_record and check_release read it. Manifests are still written for every platform. --- platforms/_registry.yml | 10 ++++++++ schemas/registry.schema.json | 32 ++++++++++++++++++++++++ scripts/check_release.py | 28 +++++++++++++++++++-- scripts/common.py | 19 ++++++++++++++ scripts/generate_pack.py | 43 +++++++++++++++++++++----------- scripts/pipeline.py | 30 +++++++++++++++++++--- scripts/release_record.py | 40 ++++++++++++++++++++++++++--- scripts/validate_schemas.py | 26 +++++++++++++++++++ tests/test_tool_messages.py | 3 ++- tests/test_verify_pack_choice.py | 9 ++++++- 10 files changed, 216 insertions(+), 24 deletions(-) create mode 100644 schemas/registry.schema.json diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 90d660fd..cb9d2d9a 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -177,10 +177,15 @@ _standalone_copies: &standalone_copies android: - $EXTERNAL_STORAGE/Android/data/com.flycast.emulator/files/data +# release_pack: true marks the platforms a release publishes a ZIP for, the +# ones most people download (download_count of v2026.10.10, part 1 of each +# pack). Every other platform is served by the installer, whose manifest is +# written for all of them. wiki/release-process.md says how to revisit it. platforms: retroarch: config: retroarch.yml status: active + release_pack: true device_group: computer extract_to: system/ logo: https://raw.githubusercontent.com/libretro/RetroArch/master/media/retroarch-vector_invader-only.svg @@ -486,6 +491,7 @@ platforms: batocera: config: batocera.yml status: active + release_pack: true device_group: gaming-os extract_to: /userdata/bios/ logo: https://raw.githubusercontent.com/batocera-linux/batocera-emulationstation/master/resources/splash_batocera.svg @@ -561,6 +567,7 @@ platforms: retrobat: config: retrobat.yml status: active + release_pack: true device_group: computer extract_to: bios/ logo: https://raw.githubusercontent.com/RetroBat-Official/retrobat/main/system/resources/retrobat_logo_notext.png @@ -608,6 +615,7 @@ platforms: emudeck: config: emudeck.yml status: active + release_pack: true device_group: computer extract_to: ~/Emulation/bios/ logo: https://raw.githubusercontent.com/dragoonDorise/EmuDeck/main/icons/EmuDeck.png @@ -749,6 +757,7 @@ platforms: romm: config: romm.yml status: active + release_pack: true device_group: server extract_to: library/bios/ logo: https://avatars.githubusercontent.com/u/164058018 @@ -1024,6 +1033,7 @@ platforms: misterfpga: config: misterfpga.yml status: active + release_pack: true device_group: fpga extract_to: /media/fat/games/ logo: https://avatars.githubusercontent.com/u/29345272 diff --git a/schemas/registry.schema.json b/schemas/registry.schema.json new file mode 100644 index 00000000..111facc6 --- /dev/null +++ b/schemas/registry.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/registry.schema.json", + "title": "RetroBIOS platform registry", + "description": "platforms/_registry.yml: each registered platform, its status, and whether a release publishes a ZIP pack for it. Install detection and the other per-platform fields are read by their own tools.", + "type": "object", + "required": ["platforms"], + "properties": { + "platforms": { + "type": "object", + "minProperties": 1, + "propertyNames": { "pattern": "^[a-z0-9_]+$" }, + "additionalProperties": { + "type": "object", + "required": ["config", "status"], + "properties": { + "config": { "type": "string", "pattern": "\\.yml$" }, + "status": { + "description": "active: scraped and released. archived: the configuration is kept, no scheduled scrape.", + "enum": ["active", "archived"] + }, + "release_pack": { + "description": "true when a release publishes the platform's full ZIP pack. Absent or false: the installer serves the platform, its manifest is written all the same.", + "type": "boolean" + }, + "device_group": { "type": "string" }, + "extract_to": { "type": ["string", "null"] } + } + } + } + } +} diff --git a/scripts/check_release.py b/scripts/check_release.py index 018dbc16..409ee555 100644 --- a/scripts/check_release.py +++ b/scripts/check_release.py @@ -5,7 +5,9 @@ Run on the draft before publishing it, and on the published release at any time. Every pack asset release.json names must be on the release at the sha256 SHA256SUMS.txt lists for it, as must the handheld port (RetroBIOS.zip) when the list names it, the parts of a pack must add up to the download -size the README prints, and nothing else may be attached. An upload that +size the README prints, and nothing else may be attached. The record holds +one pack for each platform the registry marks release_pack, and no other +pack. An upload that failed in the loop, or a part re-uploaded from an older build, otherwise publishes a release that `sha256sum --check` rejects for every user. @@ -29,6 +31,7 @@ from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from build_port import PORT_ZIP from check_release_assets import digests_of +from common import list_release_pack_platforms from largefiles import LARGE_FILES_REPO from release_record import RECORD, load_record @@ -69,15 +72,31 @@ def compare_release( assets: dict[str, Asset], tag: str, local_files: dict[str, str] | None = None, + released: set[str] | None = None, ) -> list[Finding]: """Every way the release can differ from what was built. *local_files* maps a file kept beside the packs (the checksum list, its signature) to its local sha256: the copy on the release must be that one. + *released* names the platforms the registry marks release_pack: each one + has a recorded pack, and no recorded pack serves only other platforms. """ findings: list[Finding] = [] if record.get("tag") != tag: findings.append(Finding("tag", tag, f"release.json records {record.get('tag')}")) + if released is not None: + packs = record.get("packs", {}) + served = {p for pack in packs.values() for p in pack.get("platforms", ())} + for platform in sorted(released - served): + findings.append(Finding( + "no-pack", platform, "release_pack: true, no pack in release.json" + )) + for pack_name, pack in sorted(packs.items()): + if not set(pack.get("platforms", ())) & released: + findings.append(Finding( + "not-released", pack_name, + "serves no platform marked release_pack in platforms/_registry.yml", + )) recorded = { name: pack_name @@ -167,6 +186,10 @@ def main() -> int: help=f"local {SUMS} the packs were listed in. Its copy and signature on " f"the release must be these files (default: the list attached to the release)", ) + parser.add_argument( + "--platforms-dir", default="platforms", + help="registry whose release_pack platforms each need a pack (default: %(default)s)", + ) parser.add_argument("--json", action="store_true", help="JSON output") args = parser.parse_args() @@ -199,7 +222,8 @@ def main() -> int: print(f"ERROR: {SUMS}: {exc}", file=sys.stderr) return 2 - findings = compare_release(record, sums, assets, tag, local_files) + released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True)) + findings = compare_release(record, sums, assets, tag, local_files, released) if args.json: print(json.dumps( {"tag": tag, "draft": draft, "assets": len(assets), diff --git a/scripts/common.py b/scripts/common.py index 4b603ac4..eed59b06 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -437,6 +437,25 @@ def list_registered_platforms( return platforms +def list_release_pack_platforms( + platforms_dir: str = "platforms", + include_archived: bool = False, +) -> list[str]: + """Registered platforms a release publishes a ZIP pack for. + + `release_pack: true` in _registry.yml is the whole list: the build of + `--all`, the pack checks, the release record and every download table + read it. The other platforms keep their install manifest and are served + by the installer. + """ + registry = load_platform_registry(platforms_dir) + return [ + name + for name in list_registered_platforms(platforms_dir, include_archived) + if registry.get(name, {}).get("release_pack") is True + ] + + # A list this short is printed whole rather than pointed at. SHORT_LIST = 12 diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 319cb8f8..5f22e6bd 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -57,6 +57,7 @@ from common import ( canonical_target_name, platform_files_target, list_registered_platforms, + list_release_pack_platforms, list_system_ids, load_data_dir_registry, load_database, @@ -2623,7 +2624,9 @@ def _run_verify_packs(args: argparse.Namespace) -> None: if args.platform: platforms = [args.platform] elif args.all: - platforms = list_registered_platforms( + # The packs --all builds: a platform without release_pack has no pack + # to check, and its absence from the directory is no failure. + platforms = list_release_pack_platforms( args.platforms_dir, include_archived=args.include_archived ) else: @@ -3112,7 +3115,9 @@ def _main(held: contextlib.ExitStack) -> None: help="Platform name (e.g., retroarch)", ) parser.add_argument( - "--all", action="store_true", help="Generate packs for all active platforms" + "--all", action="store_true", + help="Generate the release packs (release_pack in the registry), or with" + " --manifest the install manifests of every platform", ) parser.add_argument( "--emulator", "-e", help="Emulator profile name(s), comma-separated" @@ -3264,10 +3269,14 @@ def _main(held: contextlib.ExitStack) -> None: ) return if args.list: - # Archived platforms build like the others but have no release pack. + # Every platform builds with --platform. --all builds the release + # packs only, the others are served by the installer. active = set(list_registered_platforms(args.platforms_dir)) + released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True)) for p in list_platforms(args.platforms_dir): - print(p if p in active else f"{p} (archived)") + marks = [m for m, on in (("archived", p not in active), + ("installer only", p not in released)) if on] + print(f"{p} ({', '.join(marks)})" if marks else p) return if args.list_emulators: list_emulator_profiles(args.emulators_dir) @@ -3393,15 +3402,20 @@ def _main(held: contextlib.ExitStack) -> None: system_filter = getattr(args, "system_ids", None) # Platform mode - if args.all: - # Install manifests cover every registered platform. "Archived" means + if args.all and args.manifest: + # Install manifests cover every registered platform: the installer + # serves each one, with or without a release pack. "Archived" means # upstream is no longer scraped on a schedule, not that the platform - # lost its users: its packs still ship, so the installer still needs - # its file list. - platforms = list_registered_platforms( - args.platforms_dir, - include_archived=args.include_archived or args.manifest, + # lost its users. + platforms = list_registered_platforms(args.platforms_dir, include_archived=True) + elif args.all: + # A release publishes the packs the registry marks release_pack and + # nothing else: a pack built for another platform is one nobody uploads. + platforms = list_release_pack_platforms( + args.platforms_dir, include_archived=args.include_archived ) + if not platforms: + parser.error("no registered platform has release_pack: true") elif args.platform: platforms = [args.platform] else: @@ -4789,10 +4803,11 @@ def _expected_pack_names( } for p in platforms } - # Groups depend on which platforms a run built: --all alone or with - # --include-archived. Both compositions are names a pack can carry. + # Groups depend on which platforms a run built: the release packs of + # --all, alone or with --include-archived. Both compositions are names a + # pack can carry. for include_archived in (False, True): - pool = list_registered_platforms( + pool = list_release_pack_platforms( platforms_dir, include_archived=include_archived ) cache = None diff --git a/scripts/pipeline.py b/scripts/pipeline.py index f1daf953..738a9a03 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -317,8 +317,27 @@ def _match_key(label: str) -> set[str]: } - {""} -def check_consistency(verify_output: str, pack_output: str) -> bool: - """Verify that check counts match between verify and pack for each platform.""" +def release_pack_keys(platforms_dir: str = "platforms", include_archived: bool = False) -> set[str]: + """Comparable identities of the platforms --all builds a pack for: their + registry ids and display names, normalized as _match_key does.""" + from common import list_release_pack_platforms, load_platform_config # noqa: PLC0415 + + keys: set[str] = set() + for name in list_release_pack_platforms(platforms_dir, include_archived): + keys |= _match_key(name) + keys |= _match_key(str(load_platform_config(name, platforms_dir).get("platform", name))) + return keys + + +def check_consistency( + verify_output: str, pack_output: str, released: set[str] | None = None, +) -> bool: + """Verify that check counts match between verify and pack for each platform. + + *released* holds the identities of the platforms with a release pack + (release_pack_keys). verify judges every platform, the build only those: + a platform outside it has no pack to compare, which is no failure. + """ v = parse_verify_counts(verify_output) p = parse_pack_counts(pack_output) excluded = parse_pack_exclusions(pack_output) @@ -377,6 +396,8 @@ def check_consistency(verify_output: str, pack_output: str) -> bool: f"({p_excluded} unsafe excluded) != verify {v_ok}/{v_total}" ) all_ok = False + elif released is not None and not (v_names & released): + print(f" {v_label}: {v_ok}/{v_total} (no release pack, served by the installer)") else: print(f" {v_label}: {v_ok}/{v_total} (no separate pack)") all_ok = False @@ -1023,7 +1044,10 @@ def main() -> None: skip_notice("consistency", "no full pack built") results["consistency"] = SKIPPED elif pack_output and verify_output: - ok = check_consistency(verify_output, pack_output) + ok = check_consistency( + verify_output, pack_output, + release_pack_keys(include_archived=args.include_archived), + ) # Counts can agree while the paths differ, both ways at once. ok = check_pack_destinations(destinations_file, pack_output) and ok ok = check_emulator_packs() and ok diff --git a/scripts/release_record.py b/scripts/release_record.py index 1926398f..6737b509 100644 --- a/scripts/release_record.py +++ b/scripts/release_record.py @@ -26,11 +26,15 @@ import split_pack from common import ( ArtifactLockBusy, artifact_lock, - list_registered_platforms, + list_release_pack_platforms, write_text_atomic, ) RECORD = "release.json" +# Each registered platform has a page at get// on the site: the +# current pack's files, or the command when no release publishes one. The +# address outlives every release, so a link written elsewhere keeps working. +GET_DIR = "get" class UnfinishedSplitError(Exception): @@ -46,11 +50,12 @@ def release_owners(platforms_dir: str = "platforms") -> dict[str, list[str]]: Rebuilt from the name builder, never matched by substring: matched by substring, RetroArch_v1.22.2_AndroidArm64V8A_BIOS_Pack.zip sorts before - the full pack and was taken for the RetroArch pack. + the full pack and was taken for the RetroArch pack. Only the platforms + marked release_pack own a release pack. """ import generate_pack - platforms = list_registered_platforms(platforms_dir, include_archived=True) + platforms = list_release_pack_platforms(platforms_dir, include_archived=True) owners: dict[str, list[str]] = {} for platform, names in generate_pack._expected_pack_names( platforms, platforms_dir, None, None @@ -127,6 +132,35 @@ def load_record(path: str | Path = RECORD) -> dict: return {} +def get_page(platform: str, site: str = "") -> str: + """The permanent download address of a platform, under *site*.""" + return f"{site}{GET_DIR}/{platform}/" + + +def advertised(record: dict, platforms_dir: str = "platforms") -> dict: + """The record as the download tables show it: release_pack platforms only. + + A release cut before a platform lost its release_pack still holds that + platform's pack. The tables follow the registry, so a platform left out + is offered the command, and a pack two platforms shared names the one + still released. + """ + if not record: + return {} + released = set(list_release_pack_platforms(platforms_dir, include_archived=True)) + packs = {} + for name, pack in record.get("packs", {}).items(): + serves = [p for p in pack.get("platforms", ()) if p in released] + if serves: + packs[name] = {**pack, "platforms": serves} + return {**record, "packs": packs} + + +def load_advertised(path: str | Path = RECORD, platforms_dir: str = "platforms") -> dict: + """The committed record, reduced to the platforms a release publishes.""" + return advertised(load_record(path), platforms_dir) + + def pack_name_for(platform: str, record: dict) -> str | None: """Name of the release pack that serves a platform, if it has one.""" for name, pack in sorted(record.get("packs", {}).items()): diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 165f71d1..5e002a78 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -391,6 +391,31 @@ def _save_folder_once_paths(path: Path, document: dict) -> list[str]: ] +def _registry_contract() -> list[str]: + """The registry against its schema, and a release that still ships a pack. + + `release_pack` is the only list of the ZIPs a release publishes: a + registry where no active platform carries it would cut a release with + no pack at all, and every download link would lead to the command. + """ + registry_path = ROOT / "platforms" / "_registry.yml" + if not registry_path.is_file(): + return [] + with registry_path.open(encoding="utf-8") as handle: + registry = yaml_load(handle) or {} + label = str(registry_path.relative_to(ROOT)) + out = _errors(_validator("registry.schema.json"), registry, label) + platforms = registry.get("platforms") or {} + if isinstance(platforms, dict) and not any( + isinstance(entry, dict) + and entry.get("release_pack") is True + and entry.get("status", "active") == "active" + for entry in platforms.values() + ): + out.append(f"{label}: no active platform has release_pack: true") + return out + + def _save_directory_blocks() -> list[str]: """Each registry save_directory is null or a folder and a by_core flag. @@ -572,6 +597,7 @@ def main() -> int: errors.extend(_unknown_data_dir_refs()) errors.extend(_repository_layout_in_profiles()) errors.extend(_save_directory_blocks()) + errors.extend(_registry_contract()) if not args.source_only: database_path = ROOT / "database.json" diff --git a/tests/test_tool_messages.py b/tests/test_tool_messages.py index 9a5731df..adfcdd59 100644 --- a/tests/test_tool_messages.py +++ b/tests/test_tool_messages.py @@ -346,7 +346,8 @@ class ListPlatformsTakesTheSharedFlag(unittest.TestCase): ) self.assertEqual(proc.returncode, 0, proc.stderr) lines = proc.stdout.splitlines() - self.assertIn("retropie (archived)", lines) + self.assertIn("retropie (archived, installer only)", lines) + self.assertIn("knulli (installer only)", lines) self.assertIn("retroarch", lines) diff --git a/tests/test_verify_pack_choice.py b/tests/test_verify_pack_choice.py index 0f243da4..4aac344d 100644 --- a/tests/test_verify_pack_choice.py +++ b/tests/test_verify_pack_choice.py @@ -140,12 +140,19 @@ class VerifyPackChoice(unittest.TestCase): self._run(self._args()) self.assertEqual(out.getvalue().strip().splitlines()[-1], "Verified 1 pack: all OK") - def test_include_archived_reaches_the_archived_pack(self): + def test_the_sweep_checks_release_packs_and_a_named_platform_its_own(self): + """--all checks the packs a release publishes (release_pack): the + RetroPie pack, archived and served by the installer, is left to a + check that names it.""" stem = self.gp._platform_pack_stem(["retropie"], "retropie", "platforms") self._touch(f"{stem}_BIOS_Pack.zip") + self._touch(f"{self.stem}_BIOS_Pack.zip") checked = self._run( self._args(platform=None, all=True, include_archived=True) ) + self.assertIn(("retroarch", f"{self.stem}_BIOS_Pack.zip"), checked) + self.assertNotIn(("retropie", f"{stem}_BIOS_Pack.zip"), checked) + checked = self._run(self._args(platform="retropie")) self.assertIn(("retropie", f"{stem}_BIOS_Pack.zip"), checked) From a2fc09a89e9ca13fb4ad63542378eea0bc99f0a7 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:06:51 +0200 Subject: [PATCH 2/3] feat: permanent download page per platform get// lists the files of the current release pack, read from release.json, or gives the command for a setup without one. The README, setup pages, Get your files and the release notes link it instead of asset URLs, which die with each deleted tag. Setups without a ZIP say so with the command beside it, and validate_site checks every get/ page. --- README.md | 56 +++-- platforms/_guides.yml | 14 +- scripts/download.py | 11 + scripts/download.sh | 2 + scripts/generate_readme.py | 91 +++++--- scripts/generate_site.py | 169 ++++++++++++--- scripts/release_notes.py | 82 +++++--- scripts/validate_site.py | 65 +++++- tests/test_doc_surfaces.py | 115 +++++++---- tests/test_minui_platform.py | 4 +- tests/test_pack_counts.py | 4 +- tests/test_release_packs.py | 390 +++++++++++++++++++++++++++++++++++ tests/test_site_r2_entry.py | 11 +- tests/test_site_r3_entry.py | 13 +- tests/test_which_pack.py | 19 +- 15 files changed, 872 insertions(+), 174 deletions(-) create mode 100644 tests/test_release_packs.py diff --git a/README.md b/README.md index 46fa5918..54ee929a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Site

-BIOS and firmware files for AmberELEC, Batocera, BizHawk, EmuDeck, EmuELEC, KNULLI, Lakka, MiSTer FPGA, MinUI, MyMinUI, NextUI, ROCKNIX, Recalbox, RetroArch, RetroBat, RetroDECK, RetroPie, and RomM, one pack per platform. +BIOS and firmware files for AmberELEC, Batocera, BizHawk, EmuDeck, EmuELEC, KNULLI, Lakka, MiSTer FPGA, MinUI, MyMinUI, NextUI, ROCKNIX, Recalbox, RetroArch, RetroBat, RetroDECK, RetroPie, and RomM. A pack holds the files its emulators read, as their source code names them, and leaves out what the emulators install themselves. @@ -37,7 +37,7 @@ Every option, and what the installer detects and writes, is on the [installer pa ## Download BIOS packs -One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file. At a terminal it asks whether to install everything or to choose systems, cores and regions, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below. +A pack holds everything its platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file. At a terminal it asks whether to install everything or to choose systems, cores and regions, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below. A pack over 2 GB comes in several parts (`Pack.part1of2.zip`, `Pack.part2of2.zip`), and every part is needed. Each part is an ordinary ZIP: extract them all into the same folder. @@ -47,38 +47,28 @@ The Files column counts what a pack holds once extracted, the figure a file mana | Platform | Files | Extracted size | Extract to | Download | |----------|------:|---------------:|-----------|----------| -| AmberELEC | - | - | `/storage/roms/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| Batocera | 12,810 | 7.5 GB | `/userdata/bios/` | Download, 3 files, all needed: [part 1 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part3of3.zip) | -| BizHawk | 950 | 2.4 GB | `Firmware/` | [Download](../../releases/download/v2026.10.10/BizHawk_2.11.1_BIOS_Pack.zip) | -| EmuDeck | 620 | 3.2 GB | `~/Emulation/bios/` | [Download](../../releases/download/v2026.10.10/EmuDeck_2.7EA_BIOS_Pack.zip) | -| EmuELEC | - | - | `/storage/roms/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| KNULLI | - | - | `/userdata/bios/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| Lakka | 8,504 | 6.0 GB | `/storage/system/` | Download, 2 files, all needed: [part 1 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part2of2.zip) | -| MiSTer FPGA | 83 | 29 MB | `/media/fat/games/` | [Download](../../releases/download/v2026.10.10/MiSTer_FPGA_2026-09-29_BIOS_Pack.zip) | -| MinUI | - | - | `/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| MyMinUI | - | - | `/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| NextUI | - | - | `/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| ROCKNIX | 5,804 | 6.1 GB | `/storage/roms/bios/` | Download, 2 files, all needed: [part 1 of 2](../../releases/download/v2026.10.10/ROCKNIX_20261001_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/ROCKNIX_20261001_BIOS_Pack.part2of2.zip) | -| Recalbox | 3,222 | 3.9 GB | `/recalbox/share/bios/` | Download, 2 files, all needed: [part 1 of 2](../../releases/download/v2026.10.10/Recalbox_10.1.1_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/Recalbox_10.1.1_BIOS_Pack.part2of2.zip) | -| RetroArch | 8,504 | 6.0 GB | `system/` | Download, 2 files, all needed: [part 1 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part2of2.zip) | -| RetroBat | 12,399 | 7.9 GB | `bios/` | Download, 3 files, all needed: [part 1 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part3of3.zip) | -| RetroDECK | 7,793 | 8.0 GB | `~/retrodeck/` | Download, 3 files, all needed: [part 1 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part3of3.zip) | -| RetroPie * | 12,533 | 8.1 GB | `~/RetroPie/BIOS/` | Download, 3 files, all needed: [part 1 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part3of3.zip) | -| RomM | 1,087 | 1.5 GB | `library/bios/` | [Download](../../releases/download/v2026.10.10/RomM_5.3.1_BIOS_Pack.zip) | +| AmberELEC | - | - | `/storage/roms/` | [Use the command](https://abdess.github.io/retrobios/get/amberelec/) | +| Batocera | 12,810 | 7.5 GB | `/userdata/bios/` | [Download](https://abdess.github.io/retrobios/get/batocera/) | +| BizHawk | - | - | `Firmware/` | [Use the command](https://abdess.github.io/retrobios/get/bizhawk/) | +| EmuDeck | 620 | 3.2 GB | `~/Emulation/bios/` | [Download](https://abdess.github.io/retrobios/get/emudeck/) | +| EmuELEC | - | - | `/storage/roms/` | [Use the command](https://abdess.github.io/retrobios/get/emuelec/) | +| KNULLI | - | - | `/userdata/bios/` | [Use the command](https://abdess.github.io/retrobios/get/knulli/) | +| Lakka | - | - | `/storage/system/` | [Use the command](https://abdess.github.io/retrobios/get/lakka/) | +| MiSTer FPGA | 83 | 29 MB | `/media/fat/games/` | [Download](https://abdess.github.io/retrobios/get/misterfpga/) | +| MinUI | - | - | `/` | [Use the command](https://abdess.github.io/retrobios/get/minui/) | +| MyMinUI | - | - | `/` | [Use the command](https://abdess.github.io/retrobios/get/myminui/) | +| NextUI | - | - | `/` | [Use the command](https://abdess.github.io/retrobios/get/nextui/) | +| ROCKNIX | - | - | `/storage/roms/bios/` | [Use the command](https://abdess.github.io/retrobios/get/rocknix/) | +| Recalbox | - | - | `/recalbox/share/bios/` | [Use the command](https://abdess.github.io/retrobios/get/recalbox/) | +| RetroArch | 8,504 | 6.0 GB | `system/` | [Download](https://abdess.github.io/retrobios/get/retroarch/) | +| RetroBat | 12,399 | 7.9 GB | `bios/` | [Download](https://abdess.github.io/retrobios/get/retrobat/) | +| RetroDECK | - | - | `~/retrodeck/` | [Use the command](https://abdess.github.io/retrobios/get/retrodeck/) | +| RetroPie * | - | - | `~/RetroPie/BIOS/` | [Use the command](https://abdess.github.io/retrobios/get/retropie/) | +| RomM | 1,087 | 1.5 GB | `library/bios/` | [Download](https://abdess.github.io/retrobios/get/romm/) | -The AmberELEC pack holds its own `bios/` folder. Unzip it into `/storage/roms/`. +ZIPs are made for the setups most people download. The others read "Use the command": the installer above installs every file for them and checks each one, and `--dry-run` shows what it would install ([why no ZIP](https://abdess.github.io/retrobios/wiki/faq/#why-is-there-no-zip-for-my-setup)). -The EmuELEC pack holds its own `bios/` folder. Unzip it into `/storage/roms/`. - -The MinUI pack holds its own `Bios/` folder. Unzip it into `/`. - -The MyMinUI pack holds its own `Bios/` folder. Unzip it into `/`. - -The NextUI pack holds its own `Bios/` folder. Unzip it into `/`. - -The RetroDECK pack holds its own `bios/` folder. Unzip it into `~/retrodeck/`. - -\* Archived: the configuration is kept and packs are still built, but upstream is no longer scraped on a schedule. +\* Archived: the configuration is kept and the installer serves it, but upstream is no longer scraped on a schedule. ## What's included @@ -209,4 +199,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-10-23T05:39:12Z* +*Auto-generated on 2026-10-23T13:26:34Z* diff --git a/platforms/_guides.yml b/platforms/_guides.yml index 7bd61954..c9eec77c 100644 --- a/platforms/_guides.yml +++ b/platforms/_guides.yml @@ -530,7 +530,7 @@ guides: - >- AmberELEC runs on the RG351P, RG351V, RG351MP, RG353, RG503 and RG552. Its list names files under `/storage/roms`: `bios/` for most, `neogeo/` for the rest. The - pack holds those folders. + command fills those folders. logo_owner: AmberELEC logo_licence: GitHub organisation avatar of AmberELEC emuelec: @@ -553,7 +553,7 @@ guides: - >- EmuELEC runs on Amlogic TV boxes, the Odroid Go Advance, GameForce and RK356x devices. Its list names files under `/storage/roms`: `bios/` for most, `neogeo/` and `ports/` - for the rest. The pack holds those folders. + for the rest. The command fills those folders. logo_owner: EmuELEC logo_licence: GitHub organisation avatar of EmuELEC knulli: @@ -598,15 +598,15 @@ guides: Press **F4** in the game menu for a terminal. Paste the line. Press Enter. notes: - >- - Use the RetroPie pack, not RetroArch's. The command does not run on RetroPie 4.8: use - the pack there. + Install with `--platform retropie`, not RetroArch's files. The command does not run on + RetroPie 4.8: run it on a computer there, with the network folder above after `--dest`. reference_notes: - >- RetroPie also installs standalone emulators (openMSX, PCSX2, XRoar, Amiberry and - others) whose files only the RetroPie pack carries. + others) whose files only the RetroPie file list carries. - >- - RetroPie is archived in this project: its configuration is kept and its pack is - still built, but its upstream list is no longer read again on a schedule. + RetroPie is archived in this project: its configuration is kept and the command + serves it, but its upstream list is no longer read again on a schedule. pack_manual: | Option B: Via network share -------------------------- diff --git a/scripts/download.py b/scripts/download.py index f1f9c853..5402965f 100644 --- a/scripts/download.py +++ b/scripts/download.py @@ -85,6 +85,15 @@ class Pack: expected_parts: int = 1 +# A release publishes packs for the setups most people download. The others +# are served file by file by the installer. +NO_PACK_HINT = ( + "A setup without a pack is served by the installer:\n" + " curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh" + " | sh -s -- --platform " +) + + def get_latest_release() -> dict: """Fetch latest release info from GitHub API.""" url = f"{API}/repos/{REPO}/releases/latest" @@ -319,6 +328,7 @@ def show_info(platform: str, release: dict) -> None: if not pack: print(f"Platform '{platform}' not found in release.") print("Available:", ", ".join(list_platforms(release))) + print(NO_PACK_HINT) sys.exit(1) print(f" Platform: {pack.platform}") @@ -389,6 +399,7 @@ To check files already in place, use: python install.py --check if not pack: print(f"Platform '{args.platform}' not found in release.") print("Available:", ", ".join(list_platforms(release))) + print(NO_PACK_HINT) sys.exit(1) dest = Path(os.path.expanduser(args.dest)) diff --git a/scripts/download.sh b/scripts/download.sh index e2416779..1140a9aa 100755 --- a/scripts/download.sh +++ b/scripts/download.sh @@ -115,6 +115,8 @@ EOF echo "Error: Platform '$platform' not found in latest release." echo "Available platforms:" platform_names "$release_json" + echo "A setup without a pack is served by the installer:" + echo " curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --platform " exit 1 fi diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index dcfee052..ad10095c 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -229,22 +229,23 @@ def release_totals(platform_name: str, record: dict) -> tuple[int | None, int | return pack["files"], pack["extracted_size"] -def download_links( +NO_PACK = "Use the command" + + +def asset_links( platform_name: str, record: dict, repo: str, label: str, attrs: str = "", button: Callable[[str, str], str] | None = None, ) -> str: """Markdown links to the files of a platform's pack in the recorded release. - A pack published whole gets one link labelled *label*. A pack in parts - gets one link per part, since every part is needed, after the number of - files to download: a reader who took one link for the pack extracted - half of it. *repo* is the - repository URL, absolute or relative, and *attrs* an attribute list - put after each link, or *button* the function that draws each link as a - button on the site. Without a recorded release the link is the latest - release page. A release recorded without this platform's pack says so and - names the installer, which reads the file lists on main: a button to the - release served six platforms a page with no pack for them. + Written on the platform's get/ page only: every other surface links that + page, whose address does not change with the release. A pack published + whole gets one link labelled *label*. A pack in parts gets one link per + part, since every part is needed, after the number of files to download: + a reader who took one link for the pack extracted half of it. *repo* is + the repository URL, *attrs* an attribute list put after each link, or + *button* the function that draws each link as a button. Without a + recorded release the link is the latest release page. """ pack = release_record.pack_for(platform_name, record) tag = record.get("tag") @@ -252,10 +253,8 @@ def download_links( def link(text: str, url: str) -> str: return button(text, url) if button else f"[{text}]({url}){attrs}" - if not tag: + if not tag or not assets: return link(label, f"{repo}/releases/latest") - if not assets: - return f"Not in {tag}: [use the installer]({SITE_URL}wiki/installer/)" urls = [f"{repo}/releases/download/{tag}/{asset}" for asset in assets] if len(urls) == 1: return link(label, urls[0]) @@ -269,6 +268,34 @@ def download_links( return f"{label}, {len(urls)} files, all needed: " + ", ".join(parts) +def download_links( + platform_name: str, record: dict, label: str, attrs: str = "", + button: Callable[[str, str], str] | None = None, + site: str = "", platforms_dir: str = "platforms", page: str | None = None, +) -> str: + """The download cell of a platform: its permanent get/ page. + + *record* is the release record reduced to the release_pack platforms + (release_record.advertised). A platform with a pack in it gets *label* + linked to its page, which lists the files of the current release. A + platform marked release_pack but missing from the release, added since, + gets "No ZIP yet". Any other platform is served by the installer, and + the cell says so instead of staying empty. *site* is the site root the + get/ address hangs from, or *page* the address itself as the site links + it, and *attrs* and *button* style the link. + """ + page = page or release_record.get_page(platform_name, site) + def link(text: str) -> str: + return button(text, page) if button else f"[{text}]({page}){attrs}" + + if (load_platform_registry(platforms_dir).get(platform_name) or {}).get("release_pack") is not True: + return link(NO_PACK) + pack = release_record.pack_for(platform_name, record) + if record.get("tag") and not (pack and pack.get("assets")): + return f"No ZIP yet: {link(NO_PACK.lower())}" + return link(label) + + def extract_targets(platforms_dir: str) -> list[tuple[str, str]]: """(display name, folder) per registered platform, where its pack extracts. @@ -296,11 +323,17 @@ def own_root(config: dict) -> str: return tops.pop() if len(tops) == 1 else "bios" -def extract_notes(platforms_dir: str) -> list[str]: - """A pack whose entries carry their own root extracts above the BIOS folder.""" +def extract_notes(platforms_dir: str, released_only: bool = False) -> list[str]: + """A pack whose entries carry their own root extracts above the BIOS folder. + + *released_only* keeps the platforms a release publishes a pack for. + """ folders = dict(extract_targets(platforms_dir)) + registry = load_platform_registry(platforms_dir) notes = [] - for key in sorted(load_platform_registry(platforms_dir)): + for key in sorted(registry): + if released_only and registry[key].get("release_pack") is not True: + continue config = load_platform_config(key, platforms_dir) display = config.get("platform", key) if config.get("base_destination") == "" and display in folders: @@ -340,11 +373,21 @@ def download_table( f"| {display} | {f'{files:,}' if files else '-'} |" f" {format_size(size) if size else '-'} |" f" {extract_paths.get(cov['platform'], '')} |" - f" {download_links(name, record, REPO_PATH, 'Download')} |" + f" {download_links(name, record, 'Download', site=SITE_URL)} |" ) return lines +def no_pack_line() -> str: + """Why some rows read "Use the command", in one sentence.""" + return ( + "ZIPs are made for the setups most people download. The others read" + f" \"{NO_PACK}\": the installer above installs every file for them and" + " checks each one, and `--dry-run` shows what it would install" + f" ([why no ZIP]({SITE_URL}wiki/faq/#why-is-there-no-zip-for-my-setup))." + ) + + def _names_list(coverages: dict) -> str: """The platforms' names in reading order, the last after "and".""" names = sorted(c["platform"] for c in coverages.values()) @@ -500,7 +543,7 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" 'Site', "

", "", - f"BIOS and firmware files for {_names_list(coverages)}, one pack per platform.", + f"BIOS and firmware files for {_names_list(coverages)}.", "", "A pack holds the files its emulators read, as their source code names" " them, and leaves out what the emulators install themselves.", @@ -532,7 +575,7 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" "", "## Download BIOS packs", "", - "One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file. At a terminal it asks whether to install everything or to choose systems, cores and regions, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.", + "A pack holds everything its platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file. At a terminal it asks whether to install everything or to choose systems, cores and regions, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.", "", "A pack over 2 GB comes in several parts (`Pack.part1of2.zip`," " `Pack.part2of2.zip`), and every part is needed. Each part is an" @@ -564,16 +607,18 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" lines.extend( download_table( - coverages, archived, extract_paths, release_record.load_record() + coverages, archived, extract_paths, + release_record.load_advertised(platforms_dir=platforms_dir), ) ) + lines.extend(["", no_pack_line()]) - lines.extend(_paragraphs(extract_notes(platforms_dir))) + lines.extend(_paragraphs(extract_notes(platforms_dir, released_only=True))) if archived: lines.extend( [ "", - "\\* Archived: the configuration is kept and packs are still built," + "\\* Archived: the configuration is kept and the installer serves it," " but upstream is no longer scraped on a schedule.", ] ) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index c2e33910..46a4a248 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -70,6 +70,8 @@ yaml = require_yaml() from generate_readme import ( compute_coverage, platform_coverages, + NO_PACK, + asset_links, contributing_text, download_links, extract_notes, @@ -115,7 +117,8 @@ RELEASE_URL = f"{REPO_URL}/releases/latest" # metadata. The BIOS files themselves are under NOTICE, never this. DATA_LICENSE = "MIT (LICENSE: scripts, tooling, schemas, documentation and metadata)" SITE_URL = "https://abdess.github.io/retrobios/" -GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads"] +GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads", + release_record.GET_DIR] # Generated directories of files served as they are, not pages. PUBLIC_DIRS = ("api", "downloads") @@ -696,24 +699,115 @@ def version_line(platform_name: str, record: dict, named: bool = False) -> str: return f"Made for {made_from}. {OTHER_VERSION}" -def _pack_buttons(platform_name: str, label: str = "Pack") -> str: - """Buttons to the files of the platform's pack in the recorded release, - the release they come from, and the setup version its list was read at. - "Does this pack fit my version?" is the first question a frontend user - asks.""" - record = release_record.load_record() +# Why a setup has no ZIP, answered once on the FAQ. +NO_ZIP_FAQ = "wiki/faq.md#why-is-there-no-zip-for-my-setup" + + +def has_release_pack(platform_name: str) -> bool: + """Whether a release publishes a ZIP for the platform (`release_pack`).""" + entry = load_platform_registry("platforms").get(platform_name) or {} + return entry.get("release_pack") is True + + +def no_zip_line(display: str) -> str: + """The one line a setup without a release pack shows where the ZIP would be.""" + return f"No ZIP for {display}. The command installs every file and checks it." + + +def get_page_link(platform_name: str, prefix: str = "") -> str: + """The platform's permanent download page, as a site link.""" + return f"{prefix}{release_record.GET_DIR}/{platform_name}.md" + + +def _pack_buttons(platform_name: str, label: str = "Pack", prefix: str = "../", + display: str | None = None) -> str: + """The way to the platform's ZIP from its setup page: a button to its + get/ page, the release it comes from, and the setup version its list + was read at. "Does this pack fit my version?" is the first question a + frontend user asks. A setup with no release pack gets the line that + says so, and why.""" + if not has_release_pack(platform_name): + display = display or load_platform_config(platform_name, "platforms").get( + "platform", platform_name) + return f"{no_zip_line(display)} [Why no ZIP?]({prefix}{NO_ZIP_FAQ})" + record = release_record.load_advertised() buttons = download_links( platform_name, record, - REPO_URL, label, button=lambda text, url: markup.button(text, url, primary=True), + page=get_page_link(platform_name, prefix), ) released = release_line(platform_name, record) return "\n\n".join([buttons, *released, version_line(platform_name, record, named=len(released) > 1)]) +def generate_get_page(platform_name: str) -> str: + """get//: the address a link outside the site can keep. + + Every release deletes the previous tag, and with it each asset URL: six + of the eight links a wiki had to our packs went dead. This page lists + the files of the pack the current release holds, read from release.json, + or the command when no release publishes one. The release process + never touches it: committing release.json rebuilds it.""" + config = load_platform_config(platform_name, "platforms") + display = config.get("platform", platform_name) + record = release_record.load_advertised() + pack = release_record.pack_for(platform_name, record) + released = has_release_pack(platform_name) + lines = ["---", "search: exclude", "---", "", f"# Download {display}", ""] + if released and pack and pack.get("assets"): + parts = len(pack["assets"]) + release = release_line(platform_name, record) + extract_to = (load_platform_registry("platforms").get(platform_name) or {}).get("extract_to") + lines += [ + "This address stays the same. It always gives the latest ZIP.", + "", + asset_links(platform_name, record, REPO_URL, f"Download {display} Pack", + button=lambda text, url: markup.button(text, url, primary=True)), + "", + f"- {_fmt_size(pack['download_size'])} to download, " + f"{_fmt_size(pack['extracted_size'])} once unzipped.", + *(["- You need both parts." if parts == 2 else f"- You need all {parts} parts."] + if parts > 1 else []), + f"- Unzip {'them all' if parts > 1 else 'it'} into " + + (f"`{extract_to}`." if extract_to else "your BIOS folder."), + "", + *(line for text in release for line in (text, "")), + version_line(platform_name, record, named=len(release) > 1), + "", + ] + elif released and record.get("tag"): + lines += [ + f"No {display} ZIP yet. The next release has one. " + "The command installs every file and checks it.", + "", + *install_commands(f"--platform {platform_name}"), + ] + elif released: + lines += [ + "This address stays the same. It always gives the latest ZIP.", + "", + markup.button(f"Download {display} Pack", RELEASE_URL, primary=True), + "", + ] + else: + lines += [ + no_zip_line(display), + "", + *install_commands(f"--platform {platform_name}"), + f"[Why no ZIP?](../{NO_ZIP_FAQ})", + "", + ] + lines += _next_block([ + f"[{display}](../platforms/{platform_name}.md#install): the folder, " + "and how to run the command on the device", + "[All installer options](../wiki/installer.md)", + ]) + return "\n".join(lines) + + # The home tile counts the whole collection. Labelled "Files" above a table # of pack buttons, it read as what one pack contains. COLLECTION_FILES_LABEL = "Files collected" @@ -2066,6 +2160,9 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: if key in PART_PAGES: return (f"{title}: every file with its hashes, size, source references " "and who reads it.") + if key.startswith(f"{release_record.GET_DIR}/"): + return (f"{title}: the BIOS files of this setup, as the ZIP of the latest " + "release or one command, at an address that stays the same.") if key.startswith("platforms/") and relative.stem != "index": return (f"{title}: where the BIOS files go, the command that installs them, the pack, " "and every file by system with its hashes.") @@ -2487,7 +2584,7 @@ def page_level(relative: Path) -> str: path = relative.as_posix() if path in ("index.md", "which-pack.md"): return "start" - if relative.parts[0] == "platforms": + if relative.parts[0] in ("platforms", release_record.GET_DIR): return "setup" if relative.parts[0] in ("systems", "emulators") or path == "explore.md": return "explore" @@ -2647,8 +2744,9 @@ def setup_table(coverages: dict, registry: dict | None = None) -> list[str]: status = registry.get(name, {}).get("status") archived |= status == "archived" listed = f"{cov['total']:,}" if cov["total"] else "No list of its own" - files, _size = release_totals(name, release_record.load_record()) - zipped = f"{files:,}" if files else "No ZIP yet" + files, _size = release_totals(name, release_record.load_advertised()) + zipped = (f"{files:,}" if files + else "No ZIP yet" if has_release_pack(name) else "No ZIP") lines.append( f"| [{cov['platform']}](../platforms/{name}.md)" f"{' (archived)' if status == 'archived' else ''} | {listed} | {zipped} | " @@ -2659,8 +2757,10 @@ def setup_table(coverages: dict, registry: dict | None = None) -> list[str]: "The ZIP adds the files its emulators load to the files on the setup's own list.", "", ] + lines += ["No ZIP: the setup is served by the command alone. " + f"[Why?](../{NO_ZIP_FAQ})", ""] if archived: - lines += ["An archived setup keeps its page and its ZIP. Its list is no longer " + lines += ["An archived setup keeps its page and the command. Its list is no longer " "read again on a schedule.", ""] return lines @@ -3217,7 +3317,7 @@ def _computer_first(name: str, guide: Guide) -> bool: """A system that runs only on its own device, with a ZIP to download: a beginner fills its folder from a computer more easily than from a terminal on the TV. Without a ZIP the command is the only route.""" - pack = release_record.pack_for(name, release_record.load_record()) + pack = release_record.pack_for(name, release_record.load_advertised()) return guide.run == ("device",) and bool(pack and pack.get("assets")) @@ -3235,7 +3335,7 @@ def _guide_download(name: str, display: str, guide: Guide, extract_to: str | Non heading: str) -> list[str]: """The pack: its buttons, its size, and where it is unzipped.""" lines = [heading, ""] - record = release_record.load_record() + record = release_record.load_advertised() pack = release_record.pack_for(name, record) network = any(label.startswith("From ") for label, _path in guide.folders) if pack and pack.get("assets"): @@ -3260,6 +3360,8 @@ def _guide_download(name: str, display: str, guide: Guide, extract_to: str | Non *(f"- {b}" for b in bullets), "", ] + elif not has_release_pack(name): + lines += [_pack_buttons(name, display=display), ""] elif record.get("tag"): made_from = setup_version(name) lines += [ @@ -3383,12 +3485,13 @@ def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list: if guide.lead: lines += [guide.lead, ""] lines += _folder_rows(guide.folders) - if extract_to: + if extract_to and has_release_pack(name): lines.append(f"| Unzip the pack into | `{extract_to}` |") lines.append("") for note in guide.notes: lines += [note, ""] - lines += [line for note in extract_notes("platforms") if f"The {display} pack " in note + lines += [line for note in extract_notes("platforms", released_only=True) + if f"The {display} pack " in note for line in (note, "")] if guide.more_folders: detected = _detected_systems(registry, name) if guide.more_folders_detected else [] @@ -3570,7 +3673,7 @@ def generate_platform_page( detect = (registry.get(name, {}).get("install") or {}).get("detect") or [] on_card = any(rule.get("method") == "card_marker" for rule in detect) lines.extend(["", *_platform_install_block(name, display, on_card)]) - lines.extend(["", _pack_buttons(name, f"Download {display} Pack")]) + lines.extend(["", _pack_buttons(name, f"Download {display} Pack", display=display)]) lines.append("") # The declaration behind each detail, by system and name in declaration @@ -8374,15 +8477,17 @@ def _folder_cell(name: str, guide: Guide, prefix: str = "") -> str: def _pack_cell(name: str, record: dict, prefix: str = "") -> str: """The pack's parts and size, or the command where the release has none.""" + if not has_release_pack(name): + return f"No ZIP. [{NO_PACK}]({prefix}platforms/{name}.md#install)." pack = release_record.pack_for(name, record) if record.get("tag") and not (pack and pack.get("assets")): - return f"No ZIP yet. [Use the command]({prefix}platforms/{name}.md#install)." - # Small buttons, one per part, the size after the last: links wrapped - # mid-label ("part 1 of / 3") in a narrow cell. - links = download_links(name, record, REPO_URL, "ZIP", - button=lambda text, url: markup.button(text, url)) + return f"No ZIP yet. [{NO_PACK}]({prefix}platforms/{name}.md#install)." + # One small button to the setup's get/ page, which lists every part, and + # the size after it. + link = download_links(name, record, "ZIP", page=get_page_link(name, prefix), + button=lambda text, url: markup.button(text, url)) size = pack.get("download_size") if pack else None - return f"{links} ({_fmt_size(size)})" if size else links + return f"{link} ({_fmt_size(size)})" if size else link def _group_keys(registry: dict, group: str, display: dict[str, str]) -> list[str]: @@ -8459,7 +8564,7 @@ def generate_which_pack() -> str: The tiles are the choice. Open tables under them listed every setup a second time and made the page 11,800 px tall on a phone.""" registry = load_platform_registry("platforms") - record = release_record.load_record() + record = release_record.load_advertised() display = {key: load_platform_config(key, "platforms").get("platform", key) for key in registry} all_tabs = ("windows", "macos", "linux", "steamdeck", "android") lines = [ @@ -8482,10 +8587,12 @@ def generate_which_pack() -> str: *markup.details("note", "Every folder and ZIP in one table", _folder_tables(registry, record, display)), ] - no_zip = any(not (pack and pack.get("assets")) - for pack in (release_record.pack_for(key, record) for key in registry)) - if record.get("tag") and no_zip: + pending = any(has_release_pack(key) and not (pack and pack.get("assets")) + for key, pack in ((key, release_record.pack_for(key, record)) for key in registry)) + if record.get("tag") and pending: lines += ["Setups added after the last release have no ZIP until the next one.", ""] + lines += ["No ZIP on a row: the command serves that setup. ZIPs are made for the " + f"setups most people download. [Why?]({NO_ZIP_FAQ})", ""] lines += [ f"The ZIPs come from the [latest release]({RELEASE_URL}). The command may " "install newer files.", @@ -9112,6 +9219,14 @@ def _generate(args: argparse.Namespace) -> None: print("Generating which-pack page...") write_if_changed(str(docs / "which-pack.md"), generate_which_pack()) + # One permanent download address per registered platform, archived ones + # included: the README's table links each of them. + print("Generating download pages...") + get_dir = docs / release_record.GET_DIR + get_dir.mkdir(parents=True, exist_ok=True) + for name in platform_names: + write_if_changed(str(get_dir / f"{name}.md"), generate_get_page(name)) + print("Generating contributing page...") write_if_changed(str(docs / "contributing.md"), generate_contributing()) diff --git a/scripts/release_notes.py b/scripts/release_notes.py index 9a40d660..3661568d 100644 --- a/scripts/release_notes.py +++ b/scripts/release_notes.py @@ -4,8 +4,10 @@ The notes carried the pack table, typed by hand from figures release.json already held: a copied cell or an old collection total at the top is how issue 81 happened. Install commands, the parts sentence and the table come -from the record and the registry. The maintainer writes what changed, and -that text is appended as given. +from the record and the registry. Each row links the platform's page under +get/ on the site, which outlives the release, and the setups the registry +does not mark release_pack are named with the command. The maintainer writes +what changed, and that text is appended as given. Usage: python scripts/release_notes.py --whats-new tmp/whats-new.md --output notes.md @@ -19,10 +21,17 @@ import sys from pathlib import Path import release_record -from common import load_platform_registry, write_text_atomic +from common import ( + list_registered_platforms, + list_release_pack_platforms, + load_platform_config, + load_platform_registry, + write_text_atomic, +) REPO = "Abdess/retrobios" RAW = f"https://raw.githubusercontent.com/{REPO}/main" +SITE = "https://abdess.github.io/retrobios/" def _size(count: int) -> str: @@ -36,39 +45,63 @@ def _display(pack: str) -> str: return pack.split("_BIOS_Pack", 1)[0].replace("_", " ") +def _owner(name: str, pack: dict) -> str: + """The platform a pack is named after: RetroArch_Lakka is RetroArch's.""" + key = _match_key(name) + return sorted( + pack.get("platforms", ()), key=lambda p: key.find(_match_key(p)) % (len(key) + 1) + )[0] + + def _extract_folders(record: dict, platforms_dir: str) -> dict[str, str]: """Pack name to the folder the registry says it extracts into.""" registry = load_platform_registry(platforms_dir) folders: dict[str, str] = {} for name, pack in sorted(record.get("packs", {}).items()): - # A shared pack extracts where the platform it is named after does: - # RetroArch_Lakka is RetroArch's. - key = _match_key(name) - for platform in sorted( - pack.get("platforms", ()), key=lambda p: key.find(_match_key(p)) % (len(key) + 1) - ): - folder = registry.get(platform, {}).get("extract_to") - if folder: - folders[name] = folder - break + folder = registry.get(_owner(name, pack), {}).get("extract_to") + if folder: + folders[name] = folder return folders +def without_pack(platforms_dir: str) -> list[str]: + """Display names of the registered setups no release publishes a ZIP for.""" + released = set(list_release_pack_platforms(platforms_dir, include_archived=True)) + names = [ + str(load_platform_config(key, platforms_dir).get("platform", key)) + for key in list_registered_platforms(platforms_dir, include_archived=True) + if key not in released + ] + return sorted(names, key=str.casefold) + + +def no_pack_paragraph(platforms_dir: str) -> list[str]: + """One paragraph naming the setups served by the command alone.""" + names = without_pack(platforms_dir) + if not names: + return [] + listed = ", ".join(names[:-1]) + f" and {names[-1]}" if len(names) > 1 else names[0] + return [ + (f"No ZIP for {listed}. ZIPs are made for the setups most people download." + " The command above installs every file for these setups and checks each one." + " Add `--dry-run` to see what it would install."), + "", + ] + + def _match_key(value: str) -> str: """Letters and digits only: the platform is `misterfpga`, the pack MiSTer_FPGA.""" return re.sub(r"[^a-z0-9]", "", value.lower()) -def _links(tag: str, assets: list[str]) -> str: - base = f"https://github.com/{REPO}/releases/download/{tag}" - if len(assets) == 1: - return f"[{assets[0]}]({base}/{assets[0]})" - parts = " + ".join(f"[part {n}]({base}/{asset})" for n, asset in enumerate(assets, 1)) - return f"{len(assets)} files, all needed: {parts}" +def _link(name: str, pack: dict) -> str: + """The pack's permanent page, which lists the files of the current release.""" + parts = len(pack["assets"]) + label = "Download" if parts == 1 else f"Download, {parts} parts" + return f"[{label}]({release_record.get_page(_owner(name, pack), SITE)})" def render(record: dict, platforms_dir: str, whats_new: str) -> str: - tag = record["tag"] packs = record.get("packs", {}) folders = _extract_folders(record, platforms_dir) lines = [ @@ -89,8 +122,9 @@ def render(record: dict, platforms_dir: str, whats_new: str) -> str: "", "### Download a pack", "", - ("One pack per platform, the complete one: the BIOS list the platform" - " publishes plus every file its emulators open. Files is what the pack" + ("One complete pack for each platform below: the BIOS list the platform" + " publishes plus every file its emulators open. Each link is a page" + " that keeps its address from one release to the next. Files is what the pack" " holds once extracted, the count a file manager shows. Extracted size" " is the space those files take, and the download is smaller."), "", @@ -109,11 +143,11 @@ def render(record: dict, platforms_dir: str, whats_new: str) -> str: for name, pack in sorted(packs.items(), key=lambda item: -item[1]["extracted_size"]): folder = folders.get(name) lines.append( - f"| {_display(name)} | {_links(tag, pack['assets'])} |" + f"| {_display(name)} | {_link(name, pack)} |" f" {_size(pack['extracted_size'])} | {pack['files']:,} |" f" {f'`{folder}`' if folder else '-'} |" ) - lines += ["", whats_new.strip(), ""] + lines += ["", *no_pack_paragraph(platforms_dir), whats_new.strip(), ""] return "\n".join(lines) diff --git a/scripts/validate_site.py b/scripts/validate_site.py index c5205df0..e6e547de 100644 --- a/scripts/validate_site.py +++ b/scripts/validate_site.py @@ -4,7 +4,8 @@ This complements Starlight's own link check (starlight-links-validator, which fails the build) with checks on the rendered HTML: metadata, heading structure, image alternatives, duplicate IDs, structured data, local -resources, links and fragments, and the budgets of what a visitor downloads. +resources, links and fragments, the budgets of what a visitor downloads, and +a get/ download page for every registered platform. """ from __future__ import annotations @@ -352,7 +353,54 @@ def _check_links(pages: dict, site: Path, base_path: str) -> list[str]: return issues -def validate_site(site: Path, base_path: str = "/retrobios/") -> list[str]: +RELEASE_DOWNLOAD = "https://github.com/Abdess/retrobios/releases/download/" + + +def _check_get_pages( + pages: dict, site: Path, platforms_dir: Path, record_path: Path, +) -> list[str]: + """get// resolves for every registered platform, as a page. + + Links outside the site keep these addresses across releases, so a + missing one is a dead download link for whoever followed it. A platform + with a pack in the release record links every one of its assets at the + recorded tag, and one without links none and gives the command. + """ + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import release_record # noqa: PLC0415 + from common import list_registered_platforms, list_release_pack_platforms # noqa: PLC0415 + + if not (platforms_dir / "_registry.yml").is_file(): + return [f"no platform registry in {platforms_dir}"] + record = release_record.load_advertised(record_path, str(platforms_dir)) + released = set(list_release_pack_platforms(str(platforms_dir), include_archived=True)) + issues: list[str] = [] + for platform in list_registered_platforms(str(platforms_dir), include_archived=True): + rel = f"{release_record.GET_DIR}/{platform}/index.html" + page = pages.get((site / rel).resolve()) + if page is None or page.redirect: + issues.append(f"{rel}: no download page for {platform}") + continue + assets = [link for link in page.links if link.startswith(RELEASE_DOWNLOAD)] + pack = release_record.pack_for(platform, record) + if platform in released and pack: + expected = {f"{RELEASE_DOWNLOAD}{record['tag']}/{a}" for a in pack["assets"]} + missing = sorted(expected - set(assets)) + if missing: + issues.append(f"{rel}: does not link {', '.join(missing)}") + elif platform not in released: + packs = [a for a in assets if "_BIOS_Pack" in a] + if packs: + issues.append(f"{rel}: links a pack for a setup without one: {packs[0]}") + if "install.sh" not in (site / rel).read_text(encoding="utf-8"): + issues.append(f"{rel}: gives no install command") + return issues + + +def validate_site( + site: Path, base_path: str = "/retrobios/", platforms_dir: Path | None = None, + record_path: Path | None = None, +) -> list[str]: site = site.resolve() if not site.is_dir(): return [f"site directory does not exist: {site}"] @@ -379,6 +427,9 @@ def validate_site(site: Path, base_path: str = "/retrobios/") -> list[str]: issues.extend(_check_links(pages, site, base_path)) issues.extend(_check_pagefind(site)) + if platforms_dir is not None: + issues.extend(_check_get_pages( + pages, site, platforms_dir, record_path or platforms_dir.parent / "release.json")) return issues @@ -397,6 +448,14 @@ def main() -> None: "--base", default="/retrobios/", help="the path the site is served under (default: %(default)s)", ) + parser.add_argument( + "--platforms-dir", default="platforms", type=Path, + help="registry whose platforms each need a get/ page (default: %(default)s)", + ) + parser.add_argument( + "--record", default="release.json", type=Path, + help="release record the get/ pages link (default: %(default)s)", + ) parser.add_argument( "--max-errors", type=int, default=100, help="stop listing after this many errors (default: %(default)s)", @@ -404,7 +463,7 @@ def main() -> None: args = parser.parse_args() try: - issues = validate_site(Path(args.site_dir), args.base) + issues = validate_site(Path(args.site_dir), args.base, args.platforms_dir, args.record) except (OSError, RuntimeError, ValueError) as exc: print(f"Site validation failed: {exc}", file=sys.stderr) raise SystemExit(1) from exc diff --git a/tests/test_doc_surfaces.py b/tests/test_doc_surfaces.py index 8c7ec4c3..89055d9c 100644 --- a/tests/test_doc_surfaces.py +++ b/tests/test_doc_surfaces.py @@ -20,6 +20,8 @@ sys.path.insert(0, str(REPO_ROOT / "scripts")) import release_record # noqa: E402 from generate_readme import ( # noqa: E402 + SITE_URL, + asset_links, contributing_text, download_links, download_table, @@ -96,15 +98,17 @@ class TheDownloadTablesFollowTheRegistry(unittest.TestCase): self.assertIn(label, row.split(" | ")[1]) -class DownloadLinksNameTheFile(unittest.TestCase): +class AssetLinksNameTheFile(unittest.TestCase): + """The get/ page of a platform links the files of its pack.""" + def test_a_whole_pack_links_its_asset(self): self.assertEqual( - download_links("romm", SPLIT_RECORD, "../..", "Download"), + asset_links("romm", SPLIT_RECORD, "../..", "Download"), "[Download](../../releases/download/v2000.01.01/RomM_1_BIOS_Pack.zip)", ) def test_a_split_pack_links_every_part(self): - cell = download_links("batocera", SPLIT_RECORD, "../..", "Download") + cell = asset_links("batocera", SPLIT_RECORD, "../..", "Download") for part in ("part1of2", "part2of2"): self.assertIn( f"releases/download/v2000.01.01/Batocera_1_BIOS_Pack.{part}.zip", cell @@ -115,81 +119,110 @@ class DownloadLinksNameTheFile(unittest.TestCase): """One link taken for the pack left half of it out of the folder.""" self.assertIn( "Download, 2 files, all needed: ", - download_links("batocera", SPLIT_RECORD, "../..", "Download"), + asset_links("batocera", SPLIT_RECORD, "../..", "Download"), ) - buttons = download_links("batocera", SPLIT_RECORD, REPO, "Pack", " { .x }") + buttons = asset_links("batocera", SPLIT_RECORD, REPO, "Pack", " { .x }") self.assertTrue(buttons.startswith("2 files to download, all needed: "), buttons) def test_buttons_carry_their_attributes_on_each_part(self): - cell = download_links("batocera", SPLIT_RECORD, REPO, "Pack", " { .x }") + cell = asset_links("batocera", SPLIT_RECORD, REPO, "Pack", " { .x }") self.assertEqual(cell.count("{ .x }"), 2) - def test_a_release_without_the_pack_names_the_installer(self): - self.assertEqual( - download_links("bizhawk", SPLIT_RECORD, "../..", "Download"), - "Not in v2000.01.01: " - "[use the installer](https://abdess.github.io/retrobios/wiki/installer/)", - ) - def test_without_a_recorded_release_the_link_is_the_release_page(self): self.assertEqual( - download_links("bizhawk", {}, "../..", "Download"), + asset_links("romm", {}, "../..", "Download"), "[Download](../../releases/latest)", ) - def test_the_readme_table_links_each_recorded_pack(self): + +class DownloadLinksKeepTheirAddress(unittest.TestCase): + """Every release deletes the previous tag and its asset URLs: six of the + eight links a community wiki had to our packs answered 404. A download + cell links the platform's get/ page, which outlives the release.""" + + def test_a_released_pack_links_its_get_page(self): + self.assertEqual( + download_links("romm", SPLIT_RECORD, "Download", site=SITE_URL), + f"[Download]({SITE_URL}get/romm/)", + ) + self.assertEqual( + download_links("batocera", SPLIT_RECORD, "Download", site=SITE_URL), + f"[Download]({SITE_URL}get/batocera/)", + ) + + def test_a_setup_without_a_release_pack_reads_use_the_command(self): + self.assertEqual( + download_links("bizhawk", SPLIT_RECORD, "Download", site=SITE_URL), + f"[Use the command]({SITE_URL}get/bizhawk/)", + ) + + def test_a_marked_setup_the_release_lacks_has_no_zip_yet(self): + self.assertEqual( + download_links("retroarch", SPLIT_RECORD, "Download", site=SITE_URL), + f"No ZIP yet: [use the command]({SITE_URL}get/retroarch/)", + ) + + def test_without_a_recorded_release_the_get_page_answers(self): + self.assertEqual( + download_links("romm", {}, "Download", site=SITE_URL), + f"[Download]({SITE_URL}get/romm/)", + ) + + def test_the_readme_table_links_get_pages(self): coverages = { "batocera": {"platform": "Batocera"}, + "bizhawk": {"platform": "BizHawk"}, "romm": {"platform": "RomM"}, } rows = download_table(coverages, set(), {}, SPLIT_RECORD)[2:] - self.assertEqual(len(rows), 2) + self.assertEqual(len(rows), 3) for row in rows: - self.assertNotIn("releases/latest", row) - self.assertIn("releases/download/v2000.01.01/", row) + self.assertNotIn("/releases/", row) + self.assertIn(f"{SITE_URL}get/", row) + self.assertIn("[Use the command]", next(r for r in rows if "BizHawk" in r)) - def test_the_generated_readme_links_every_released_asset(self): + def test_the_generated_readme_links_every_get_page(self): if not RECORD.get("packs"): self.skipTest("release.json is not written") + from common import list_registered_platforms, list_release_pack_platforms + readme = _read("README.md") - for pack in RECORD["packs"].values(): - for asset in pack["assets"]: - with self.subTest(asset=asset): - self.assertIn( - f"../../releases/download/{RECORD['tag']}/{asset})", readme - ) + table = _section(readme, "## Download BIOS packs") + self.assertNotIn("/releases/download/", table) + released = set(list_release_pack_platforms("platforms", include_archived=True)) + for key in list_registered_platforms("platforms", include_archived=True): + with self.subTest(platform=key): + self.assertIn(f"]({SITE_URL}get/{key}/)", table) + row = next(line for line in table.splitlines() + if f"get/{key}/)" in line) + if key not in released: + self.assertIn("[Use the command]", row) - def test_the_download_page_rows_link_assets(self): - if not RECORD.get("packs"): - self.skipTest("release.json is not written") + def test_the_download_page_rows_link_get_pages(self): page = _which_pack() rows = [line for line in page.splitlines() if line.startswith("| ")] self.assertTrue(rows) for row in rows: - self.assertNotIn("releases/latest", row) - for pack in RECORD["packs"].values(): - for asset in pack["assets"]: - self.assertRegex(page, re.escape(f"/releases/download/{RECORD['tag']}/{asset}") - + r"[)\"]") + self.assertNotIn("/releases/", row) - def test_a_platform_the_release_lacks_links_no_release(self): + def test_a_platform_without_a_zip_offers_the_command(self): """KNULLI, AmberELEC and EmuELEC were registered after the recorded - release: a row for them linked a release page holding no such pack. - Every platform is on the picker now, so the row offers the command.""" - if not RECORD.get("packs"): - self.skipTest("release.json is not written") + release, and the platforms without release_pack never get a ZIP: a + row for them offers the command.""" from common import list_registered_platforms, load_platform_config - from release_record import pack_for + import generate_site + record = release_record.load_advertised() page = _which_pack() for key in list_registered_platforms("platforms", include_archived=True): - if pack_for(key, RECORD) is not None: + if release_record.pack_for(key, record) is not None: continue label = load_platform_config(key, "platforms").get("platform", key) row = next(line for line in page.splitlines() if line.startswith(f"| [{label}](")) + state = "No ZIP yet." if generate_site.has_release_pack(key) else "No ZIP." with self.subTest(platform=key): self.assertNotIn("/releases/", row) - self.assertIn(f"No ZIP yet. [Use the command](platforms/{key}.md#install).", row) + self.assertIn(f"{state} [Use the command](platforms/{key}.md#install).", row) class SlicesLeaveWithTheirRelease(unittest.TestCase): diff --git a/tests/test_minui_platform.py b/tests/test_minui_platform.py index a167ae6e..baa34512 100644 --- a/tests/test_minui_platform.py +++ b/tests/test_minui_platform.py @@ -429,7 +429,9 @@ class TheSitePageOfACardPlatform(unittest.TestCase): self.assertNotIn("0/0", page) self.assertNotIn("## Systems overview", page) self.assertIn("| `Bios/PS/` | [pcsx_rearmed](../emulators/pcsx_rearmed.md) |", page) - self.assertIn("| Unzip the pack into | `/` |", page) + # MinUI has no release pack: no unzip row, the command instead. + self.assertNotIn("| Unzip the pack into |", page) + self.assertIn("No ZIP for MinUI. The command installs every file and checks it.", page) if __name__ == "__main__": diff --git a/tests/test_pack_counts.py b/tests/test_pack_counts.py index cc0d4541..85fbead0 100644 --- a/tests/test_pack_counts.py +++ b/tests/test_pack_counts.py @@ -536,8 +536,8 @@ class TheReleaseNotesReadTheRecord(unittest.TestCase): text = self.render() row = next(line for line in text.splitlines() if line.startswith("| RetroArch")) self.assertIn("8,504", row) - self.assertIn("part1of2.zip", row) - self.assertIn("part2of2.zip", row) + self.assertIn("[Download, 2 parts](https://abdess.github.io/retrobios/get/retroarch/)", row) + self.assertNotIn("/releases/download/", row) self.assertIn("`system/`", row) self.assertIn("| Platform | Download | Extracted size | Files |", text) diff --git a/tests/test_release_packs.py b/tests/test_release_packs.py new file mode 100644 index 00000000..95618067 --- /dev/null +++ b/tests/test_release_packs.py @@ -0,0 +1,390 @@ +"""Which platforms a release publishes a ZIP for, and how every surface says it. + +The release carried a pack for every platform, 35 GB, while five of them +drew 83% of the downloads. `release_pack: true` in the registry is now the +list: the build, the pack checks, the release record and every download +table read it. The other platforms keep their install manifest, and each +page that would show a ZIP says there is none and gives the command. + +Every release also deleted the previous tag, and with it each asset URL: +six of the eight links a community wiki had to our packs answered 404. Each +platform has a permanent page at get// instead. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import common # noqa: E402 +import release_record # noqa: E402 + +PLATFORMS = str(REPO_ROOT / "platforms") +TMP_ROOT = REPO_ROOT / "tmp" +SITE = "https://abdess.github.io/retrobios/" +# The owner's choice of 2026-10-23, from the download counts of v2026.10.10. +RELEASED = {"retroarch", "emudeck", "retrobat", "batocera", "romm", "misterfpga"} + + +def _write_registry(root: Path, platforms: dict) -> str: + platforms_dir = root / "platforms" + platforms_dir.mkdir() + for name in platforms: + (platforms_dir / f"{name}.yml").write_text( + yaml.dump({"platform": name.title(), "systems": {}}), encoding="utf-8" + ) + registry = {name: {"config": f"{name}.yml", **entry} for name, entry in platforms.items()} + (platforms_dir / "_registry.yml").write_text( + yaml.dump({"platforms": registry}), encoding="utf-8" + ) + return str(platforms_dir) + + +def _which_pack() -> str: + import generate_site + + with contextlib.chdir(REPO_ROOT): + return generate_site.generate_which_pack() + + +class TheFieldIsTheList(unittest.TestCase): + def test_the_registry_marks_the_most_downloaded_setups(self): + self.assertEqual( + set(common.list_release_pack_platforms(PLATFORMS, include_archived=True)), RELEASED + ) + + def test_only_marked_platforms_are_listed_and_archived_ones_on_request(self): + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp: + platforms_dir = _write_registry(Path(tmp), { + "big": {"status": "active", "release_pack": True}, + "small": {"status": "active"}, + "off": {"status": "active", "release_pack": False}, + "old": {"status": "archived", "release_pack": True}, + }) + self.assertEqual(common.list_release_pack_platforms(platforms_dir), ["big"]) + self.assertEqual( + common.list_release_pack_platforms(platforms_dir, include_archived=True), + ["big", "old"], + ) + self.assertEqual( + common.list_registered_platforms(platforms_dir), ["big", "off", "small"] + ) + + def test_the_schema_holds_the_field(self): + try: + from jsonschema import Draft202012Validator + except ImportError: + self.skipTest("jsonschema is a tooling dependency") + + schema = json.loads((REPO_ROOT / "schemas" / "registry.schema.json").read_text()) + validator = Draft202012Validator(schema) + registry = yaml.safe_load((REPO_ROOT / "platforms" / "_registry.yml").read_text()) + self.assertEqual([e.message for e in validator.iter_errors(registry)], []) + registry["platforms"]["knulli"]["release_pack"] = "yes" + self.assertTrue(list(validator.iter_errors(registry))) + + +class TheBuildReadsTheField(unittest.TestCase): + def _groups(self, *argv: str) -> list[str]: + import generate_pack + + built: list[str] = [] + + def capture(args, groups, *rest, **kw): + built.extend(name for group, _rep in groups for name in group) + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as out, \ + contextlib.chdir(REPO_ROOT), \ + mock.patch.object(generate_pack, "_run_platform_packs", capture), \ + mock.patch.object(generate_pack, "_run_manifest_mode", capture), \ + mock.patch.object(sys, "argv", ["generate_pack.py", *argv, "--output-dir", out]), \ + contextlib.redirect_stdout(io.StringIO()): + generate_pack.main() + return built + + def test_all_builds_the_release_packs_only(self): + self.assertEqual(set(self._groups("--all", "--offline")), RELEASED) + + def test_archived_platforms_without_the_field_stay_out(self): + self.assertEqual( + set(self._groups("--all", "--include-archived", "--offline")), RELEASED + ) + + def test_manifests_cover_every_platform(self): + self.assertEqual( + set(self._groups("--all", "--manifest", "--offline")), + set(common.list_registered_platforms(PLATFORMS, include_archived=True)), + ) + + def test_the_release_owners_are_the_marked_platforms(self): + owners = release_record.release_owners(PLATFORMS) + served = {p for platforms in owners.values() for p in platforms} + self.assertEqual(served, RELEASED) + + def test_the_list_marks_installer_only_platforms(self): + import subprocess + + proc = subprocess.run( + [sys.executable, "scripts/generate_pack.py", "--list"], + capture_output=True, text=True, cwd=REPO_ROOT, check=True, timeout=120, + ) + lines = proc.stdout.splitlines() + for name in RELEASED: + self.assertIn(name, lines) + self.assertIn("recalbox (installer only)", lines) + + +class APlatformWithoutAPackFailsNoCheck(unittest.TestCase): + VERIFY = ( + "RetroArch: 10/10 OK [existence]\n" + "KNULLI: 5/5 OK [md5]\n" + ) + PACK = ( + "Generating pack for RetroArch...\n" + " dist/RetroArch_BIOS_Pack.zip: 10 files packed, 10/10 files OK [existence]\n" + ) + + def test_the_consistency_step_skips_a_setup_served_by_the_installer(self): + import pipeline + + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertTrue(pipeline.check_consistency( + self.VERIFY, self.PACK, pipeline.release_pack_keys(PLATFORMS))) + self.assertIn("KNULLI: 5/5 (no release pack, served by the installer)", out.getvalue()) + + def test_a_released_platform_without_its_pack_still_fails(self): + import pipeline + + verify = self.VERIFY + "Batocera: 3/3 OK [md5]\n" + with contextlib.redirect_stdout(io.StringIO()): + self.assertFalse(pipeline.check_consistency( + verify, self.PACK, pipeline.release_pack_keys(PLATFORMS))) + + def test_the_pipeline_passes_the_release_platforms(self): + source = (REPO_ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8") + self.assertIn("release_pack_keys(include_archived=args.include_archived)", source) + + def test_verify_packs_sweeps_the_release_packs_only(self): + import subprocess + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory: + proc = subprocess.run( + [sys.executable, "scripts/generate_pack.py", "--all", "--verify-packs", + "--output-dir", directory], + capture_output=True, text=True, cwd=REPO_ROOT, timeout=600, + ) + swept = {line.split(":")[0].strip() for line in proc.stdout.splitlines() + if "SKIP (no pack in" in line} + self.assertEqual(swept, RELEASED) + + +class TheReleaseCheckReadsTheField(unittest.TestCase): + RECORD = {"tag": "v1", "packs": { + "A_BIOS_Pack.zip": {"download_size": 1, "assets": ["A_BIOS_Pack.zip"], + "platforms": ["retroarch"]}, + "K_BIOS_Pack.zip": {"download_size": 1, "assets": ["K_BIOS_Pack.zip"], + "platforms": ["knulli"]}, + }} + + def test_a_missing_and_an_unlisted_pack_are_reported(self): + from check_release import Asset, compare_release + + sums = {"A_BIOS_Pack.zip": "a" * 64, "K_BIOS_Pack.zip": "b" * 64} + assets = {name: Asset(1, digest) for name, digest in sums.items()} + assets["SHA256SUMS.txt"] = Asset(1, None) + findings = compare_release(self.RECORD, sums, assets, "v1", None, + {"retroarch", "batocera"}) + kinds = {(f.kind, f.name) for f in findings} + self.assertIn(("no-pack", "batocera"), kinds) + self.assertIn(("not-released", "K_BIOS_Pack.zip"), kinds) + self.assertNotIn(("no-pack", "retroarch"), kinds) + + +class TheTablesFollowTheField(unittest.TestCase): + def test_a_pack_of_a_platform_without_the_field_is_not_advertised(self): + record = {"tag": "v1", "packs": { + "RetroArch_Lakka_BIOS_Pack.zip": {"platforms": ["lakka", "retroarch"], + "assets": ["x.zip"]}, + "Recalbox_BIOS_Pack.zip": {"platforms": ["recalbox"], "assets": ["y.zip"]}, + }} + shown = release_record.advertised(record, PLATFORMS) + self.assertEqual(list(shown["packs"]), ["RetroArch_Lakka_BIOS_Pack.zip"]) + self.assertEqual(shown["packs"]["RetroArch_Lakka_BIOS_Pack.zip"]["platforms"], + ["retroarch"]) + self.assertIsNone(release_record.pack_for("lakka", shown)) + self.assertEqual(release_record.advertised({}, PLATFORMS), {}) + + +class EverySurfaceSaysWhy(unittest.TestCase): + def test_a_setup_page_says_no_zip_with_the_command_beside_it(self): + import generate_site + + line = generate_site.no_zip_line("KNULLI") + self.assertEqual(line, "No ZIP for KNULLI. The command installs every file and checks it.") + with contextlib.chdir(REPO_ROOT): + block = generate_site._pack_buttons("knulli") + guide = generate_site._guide_download( + "knulli", "KNULLI", generate_site.GUIDES["knulli"], None, "## Download") + self.assertIn(line, block) + self.assertIn(generate_site.NO_ZIP_FAQ, block) + self.assertIn(line, "\n".join(guide)) + self.assertNotIn("yet", block) + + def test_a_built_setup_page_has_the_line(self): + page = REPO_ROOT / "website" / "src" / "content" / "docs" / "platforms" / "knulli.md" + if not page.is_file(): + self.skipTest("the site is not generated") + text = page.read_text(encoding="utf-8") + self.assertIn("No ZIP for KNULLI. The command installs every file and checks it.", text) + install = text.index("## Install") + self.assertLess(install, text.index("No ZIP for KNULLI")) + + def test_get_your_files_explains_the_rows(self): + page = _which_pack() + self.assertIn("No ZIP. [Use the command](platforms/knulli.md#install).", page) + self.assertIn("ZIPs are made for the setups most people download.", page) + self.assertIn("wiki/faq.md#why-is-there-no-zip-for-my-setup", page) + + def test_the_readme_rows_read_use_the_command(self): + readme = (REPO_ROOT / "README.md").read_text(encoding="utf-8") + row = next(line for line in readme.splitlines() if line.startswith("| KNULLI |")) + self.assertIn(f"[Use the command]({SITE}get/knulli/)", row) + self.assertIn("ZIPs are made for the setups most people download.", readme) + + def test_the_faq_answers_why(self): + faq = (REPO_ROOT / "wiki" / "faq.md").read_text(encoding="utf-8") + section = faq.split("## Why is there no ZIP for my setup?", 1)[1].split("\n## ", 1)[0] + self.assertIn("most people download", section) + self.assertIn("checks each one", section) + self.assertIn("--dry-run", section) + + def test_the_release_process_documents_the_field_and_the_counts(self): + page = (REPO_ROOT / "wiki" / "release-process.md").read_text(encoding="utf-8") + self.assertIn("release_pack: true", page) + self.assertIn("download_count", page) + self.assertIn("part1of", page) + + +class TheReleaseNotesNameTheSetupsWithoutAZip(unittest.TestCase): + RECORD = {"tag": "v2026.10.24", "packs": { + "RomM_5_BIOS_Pack.zip": { + "files": 1, "extracted_size": 1, "download_size": 1, + "assets": ["RomM_5_BIOS_Pack.zip"], "platforms": ["romm"]}, + }} + + def test_the_paragraph_names_them_and_points_to_the_command(self): + import release_notes + + text = release_notes.render(self.RECORD, PLATFORMS, "### What changed\n\n- x\n") + paragraph = next(line for line in text.splitlines() if line.startswith("No ZIP for ")) + for name in ("KNULLI", "Recalbox", "RetroPie", "Lakka"): + self.assertIn(name, paragraph) + for name in ("RetroArch", "Batocera", "RomM"): + self.assertNotIn(f" {name},", paragraph) + self.assertIn("The command above", paragraph) + self.assertIn("--dry-run", paragraph) + self.assertLess(text.index(paragraph), text.index("### What changed")) + + def test_the_rows_link_get_pages(self): + import release_notes + + text = release_notes.render(self.RECORD, PLATFORMS, "x") + self.assertIn(f"[Download]({SITE}get/romm/)", text) + self.assertNotIn("/releases/download/", text) + + +class EveryPlatformHasAPermanentPage(unittest.TestCase): + def test_a_page_for_every_registered_platform(self): + import generate_site + + record = release_record.load_advertised() + with contextlib.chdir(REPO_ROOT): + for name in common.list_registered_platforms(PLATFORMS, include_archived=True): + page = generate_site.generate_get_page(name) + pack = release_record.pack_for(name, record) + with self.subTest(platform=name): + self.assertIn("# Download ", page) + if name in RELEASED and pack: + for asset in pack["assets"]: + self.assertIn( + f"/releases/download/{record['tag']}/{asset}", page) + elif name not in RELEASED: + self.assertNotIn("_BIOS_Pack", page) + self.assertIn(f"install.sh | sh -s -- --platform {name}", page) + self.assertIn("No ZIP for ", page) + + def test_the_generator_writes_and_sweeps_them(self): + import generate_site + + self.assertIn(release_record.GET_DIR, generate_site.GENERATED_DIRS) + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('write_if_changed(str(get_dir / f"{name}.md"), generate_get_page(name))', + source) + + def test_built_pages_exist(self): + docs = REPO_ROOT / "website" / "src" / "content" / "docs" / release_record.GET_DIR + if not docs.is_dir(): + self.skipTest("the site is not generated") + for name in common.list_registered_platforms(PLATFORMS, include_archived=True): + with self.subTest(platform=name): + self.assertTrue(any(docs.glob(f"{name}.md*"))) + + +class TheSiteCheckResolvesEveryGetPage(unittest.TestCase): + def _site(self, root: Path, pages: dict[str, str]) -> Path: + site = root / "dist" + for rel, body in pages.items(): + path = site / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{body}", + encoding="utf-8") + return site + + def test_missing_and_wrong_pages_are_reported(self): + import validate_site + + record = {"tag": "v1", "packs": {"RomM_BIOS_Pack.zip": { + "platforms": ["romm"], "assets": ["RomM_BIOS_Pack.zip"], + "files": 1, "extracted_size": 1, "download_size": 1}}} + download = validate_site.RELEASE_DOWNLOAD + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp: + root = Path(tmp) + record_path = root / "release.json" + record_path.write_text(json.dumps(record), encoding="utf-8") + pages = { + f"get/{name}/index.html": "install.sh" + for name in common.list_registered_platforms(PLATFORMS, include_archived=True) + } + pages["get/romm/index.html"] = f'x' + pages["get/knulli/index.html"] = f'x' + del pages["get/lakka/index.html"] + site = self._site(root, pages) + parsed = {p.resolve(): validate_site._parse_page(p) for p in site.rglob("*.html")} + issues = validate_site._check_get_pages(parsed, site.resolve(), Path(PLATFORMS), + record_path) + joined = "\n".join(issues) + self.assertIn("get/lakka/index.html: no download page", joined) + self.assertIn("get/knulli/index.html: links a pack", joined) + self.assertIn("get/knulli/index.html: gives no install command", joined) + self.assertNotIn("get/romm/", joined) + self.assertNotIn("get/retroarch/", joined) + + def test_the_cli_runs_the_check(self): + source = (REPO_ROOT / "scripts" / "validate_site.py").read_text(encoding="utf-8") + self.assertIn("validate_site(Path(args.site_dir), args.base, args.platforms_dir, args.record)", + source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_r2_entry.py b/tests/test_site_r2_entry.py index 9a01f98f..da795d69 100644 --- a/tests/test_site_r2_entry.py +++ b/tests/test_site_r2_entry.py @@ -175,12 +175,17 @@ class GetYourFilesOrder(unittest.TestCase): def test_a_setup_after_the_release_says_no_zip_yet(self): text = _page("which-pack.md") self.assertNotIn("Not in v", text) - record = release_record.load_record() - missing = [k for k in REGISTRY if not release_record.pack_for(k, record)] - if record.get("tag") and missing: + record = release_record.load_advertised() + pending = [k for k in REGISTRY if REGISTRY[k].get("release_pack") + and not release_record.pack_for(k, record)] + if record.get("tag") and pending: self.assertIn("No ZIP yet. [Use the command]", text) self.assertIn("Setups added after the last release have no ZIP until the next one.", text) + else: + self.assertNotIn("No ZIP yet.", text) + if any(not entry.get("release_pack") for entry in REGISTRY.values()): + self.assertIn("No ZIP. [Use the command]", text) def test_zip_advice_answers_the_usual_mistakes(self): text = _page("which-pack.md") diff --git a/tests/test_site_r3_entry.py b/tests/test_site_r3_entry.py index cdc8a7e6..4c3fee39 100644 --- a/tests/test_site_r3_entry.py +++ b/tests/test_site_r3_entry.py @@ -140,10 +140,15 @@ class PackNamesItsVersion(unittest.TestCase): f"RetroArch {load_platform_config('retroarch', 'platforms')['version'].lstrip('v')}") def test_every_setup_page_names_a_version(self): + """A setup with a release ZIP names the version it was made for. One + without says there is no ZIP: the command installs current files.""" for key in REGISTRY: with self.subTest(platform=key): text = _guide_part(_page(f"platforms/{key}.md")) - self.assertRegex(text, r"(?:[Mm]ade for|This pack was built for) \S") + if generate_site.has_release_pack(key): + self.assertRegex(text, r"(?:[Mm]ade for|This pack was built for) \S") + else: + self.assertIn(generate_site.no_zip_line(_display(key)), text) class SetupPagesOfferPartOfThePack(unittest.TestCase): @@ -214,10 +219,12 @@ class GetYourFilesListsEachSetupOnce(unittest.TestCase): self.assertTrue(release_record.pack_for(key, record)) def test_parts_are_buttons_with_the_size_after(self): - rows = [line for line in self.text.splitlines() if "Part 1 of" in line] + """A row links the setup's get/ page, which lists every part, with the + download size after the button.""" + rows = [line for line in self.text.splitlines() if ">ZIP" in line] self.assertTrue(rows) for row in rows: - self.assertRegex(row, r'Part 1 of \d') + self.assertRegex(row, r'ZIP') self.assertRegex(row, r" \(\d[\d.]* [KMG]B\) \|$") def test_the_zip_note(self): diff --git a/tests/test_which_pack.py b/tests/test_which_pack.py index fef42ee7..0323b4dc 100644 --- a/tests/test_which_pack.py +++ b/tests/test_which_pack.py @@ -26,20 +26,25 @@ class WhichPackRows(unittest.TestCase): """ def test_platform_rows_name_their_own_pack(self): + """A row links its own get/ page, never another setup's and never a + release asset: the asset URL dies with its tag.""" import generate_site # noqa: PLC0415 import release_record # noqa: PLC0415 - record = release_record.load_record() + record = release_record.load_advertised() page = generate_site.generate_which_pack() row = re.compile(r"^\| \[[^\]]+\]\(platforms/([a-z0-9]+)\.md\) \|(.*)$", re.M) - asset = re.compile(r"/releases/download/[^/]+/([^)\s\"]+)[)\"]") + get = re.compile(r'href="get/([a-z0-9]+)\.md"') checked = 0 for key, rest in row.findall(page): - linked = set(asset.findall(rest)) - pack = release_record.pack_for(key, record) or {} + linked = set(get.findall(rest)) + pack = release_record.pack_for(key, record) with self.subTest(platform=key): - self.assertLessEqual(linked, set(pack.get("assets") or [])) + self.assertNotIn("/releases/download/", rest) + self.assertLessEqual(linked, {key}) if pack: checked += 1 - self.assertTrue(linked) - self.assertGreater(checked, 5) + self.assertEqual(linked, {key}) + elif not generate_site.has_release_pack(key): + self.assertIn(f"No ZIP. [Use the command](platforms/{key}.md#install).", rest) + self.assertGreater(checked, 3) From ba3d5191890acd7674995de0969a943803591a7b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:06:54 +0200 Subject: [PATCH 3/3] docs: explain setups without a ZIP FAQ entry, the release_pack field and how to revisit it from download counts, the get/ pages, and RetroPie 4.8 now installs from a computer with the Pi's network folder as --dest. The start pages are back under their word ceilings. --- wiki/adding-a-platform.md | 7 ++++++ wiki/faq.md | 46 ++++++++++++++++++++------------------- wiki/first-install.md | 18 +++++++-------- wiki/glossary.md | 3 ++- wiki/installer.md | 12 +++++----- wiki/reference-terms.md | 3 ++- wiki/release-process.md | 46 +++++++++++++++++++++++++++++++++++---- wiki/testing-guide.md | 2 +- wiki/troubleshooting.md | 21 ++++++++---------- 9 files changed, 101 insertions(+), 57 deletions(-) diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index 607787b2..bed3b080 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -160,6 +160,7 @@ platforms: myplatform: config: myplatform.yml # platform YAML filename in platforms/ status: active # active or archived + release_pack: true # optional: a release publishes its ZIP device_group: handheld # computer, gaming-os, handheld, fpga or server scraper: myplatform # matches PLATFORM_NAME, null if hand-maintained schedule: weekly # re-scrape cadence, run by hand: weekly, monthly, or null @@ -682,6 +683,12 @@ A new platform is archived-by-default nowhere: `status: active` puts it in `status: archived` to keep the configuration without the automation. Archived platforms are excluded unless `--all` or `--include-archived` is passed. +A new platform has no release ZIP: `release_pack: true` is set for the setups +most people download, and the installer serves every other one from its +manifest. `generate_pack.py --platform myplatform` still builds its pack. +[Which platforms get a pack](release-process.md#which-platforms-get-a-pack) +says how the list is read from download counts. + Check the output for: - The new platform appears in verify results diff --git a/wiki/faq.md b/wiki/faq.md index f9109a35..bd5c0b6f 100644 --- a/wiki/faq.md +++ b/wiki/faq.md @@ -9,13 +9,13 @@ order: 50 - **Yes** for PlayStation 1 and 2, Saturn and Neo Geo. - **Better with it** for Dreamcast: some games need it. -- **Usually not** for NES or Super Nintendo cartridges. +- **Usually not** for NES or Super Nintendo. [What is a BIOS?](what-are-bios-files.md) ## How do I know which BIOS I need? -Run the command with `--dry-run`. It lists the files, changing nothing. +Add `--dry-run`. It lists the files, changing nothing. ```sh @@ -24,23 +24,21 @@ curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | ## Do I need every file of a console? -No. A console often has one BIOS per region. To keep only yours, add your -region, like `--region us`. +No. To keep only your region's BIOS, add it, like `--region us`. ## Does the BIOS set the game's language? -Rarely. It comes from the game. Some games follow the console's setting, like -Flycast's **Language** option. +Rarely. It comes from the game. Some games follow the console's setting, like Flycast's **Language** +option. ## Zip or unzip? { #zip-or-unzip } -Unzip the ZIP you downloaded into the folder your setup's page names. Keep +Unzip it into the folder your setup's page names. Leave the ZIPs inside zipped, like `neogeo.zip`. ## Does a BIOS work on every setup? -Usually. The command installs the version each setup expects, like its -`neogeo.zip`. +Usually. The command installs each setup's own version, like its `neogeo.zip`. ## My arcade game does not start @@ -50,35 +48,39 @@ Reference: [Which version each emulator reads](technical-faq.md#which-mame-versi ## Keys, firmware or BIOS? - **BIOS** or **firmware**: the console's own program, like `scph5501.bin`. -- Some newer consoles need a system update, like `PS3UPDAT.PUP`, installed from the +- Newer consoles need a system update, like `PS3UPDAT.PUP`, installed from the emulator's menu. - **Keys** unlock encrypted games, like `prod.keys` for Switch. ## Can I play without the BIOS? -Sometimes. Some emulators carry a stand-in, and some games then fail. +Sometimes, with an emulator's stand-in, but some games fail. ## Is the command safe? { #is-piping-the-installer-into-a-shell-safe } -- It stops if the installer does not match its expected fingerprint, then - checks each file before saving it. -- It writes in your BIOS folder, and for some emulators in your save folder. +- It stops if the installer does not match its expected fingerprint, and checks + each file. +- It writes in your BIOS folder, and for some emulators your save folder. - It never touches a game or a save. -- It keeps any file it replaces in `retrobios-backup`, next to your BIOS - folder. `--restore` puts it back. +- It keeps each file it replaces in `retrobios-backup`. `--restore` puts it + back. Read them first: [install.sh](https://github.com/Abdess/retrobios/blob/main/install.sh), [install.ps1](https://github.com/Abdess/retrobios/blob/main/install.ps1), [install.py](https://github.com/Abdess/retrobios/blob/main/install.py). -[How the installer checks itself](installer.md#what-the-bootstrap-does). ## Windows or my antivirus blocks the command -Download your setup's ZIP from [Get your files](../which-pack.md) instead. +Use your setup's ZIP, if it has one: [Get your files](../which-pack.md). + +## Why is there no ZIP for my setup? + +ZIPs are made for the setups most people download. The command installs every +setup's files and checks each one. `--dry-run` shows the list first. ## Fill an SD card from a computer? -Yes. Name your setup and the BIOS folder on the card. For RetroArch on a Switch +Yes. Name your setup and the card's BIOS folder. For RetroArch on a Switch card in drive `E:`: @@ -89,7 +91,7 @@ curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh \ ## No internet on the device? -Unzip your setup's ZIP into the folder its page names. To check each file, use +Fill its folder from a computer, as above, or unzip your ZIP and check it with `--from-pack`: [Without a network](installer.md#without-a-network-offline-install). ## My setup is not in the list @@ -100,11 +102,11 @@ Unzip your setup's ZIP into the folder its page names. To check each file, use ## Why is a file missing from the ZIP? -Some come with the emulator, so the ZIP leaves them out. +Some come with the emulator, so the ZIP skips them. ## My game shows a black screen -A BIOS file is often missing: [When a game does not start](troubleshooting.md). +Often a missing BIOS: [When a game does not start](troubleshooting.md). ## Is this legal? diff --git a/wiki/first-install.md b/wiki/first-install.md index f6a62a28..f99d6485 100644 --- a/wiki/first-install.md +++ b/wiki/first-install.md @@ -97,7 +97,7 @@ curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | ## 4. Wait -You can stop with Ctrl+C at any time. Run the command again to carry on. +Ctrl+C stops it. Run the command again to carry on. ## 5. Read the last line @@ -120,7 +120,7 @@ Add `--check` to the same command. It changes nothing. curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --check ``` -All good when every file is present and none is wrong: +All good: ``` 109/109 present (109 verified, 0 wrong hash) @@ -130,8 +130,8 @@ An empty folder also shows `0 wrong hash`, with `0/109 present`. ## Undo -It keeps every file it replaces in a `retrobios-backup` folder, next to your -BIOS folder. The end of the run prints the line that puts them back. +Every file it replaces goes to a `retrobios-backup` folder next to your BIOS +folder. The last lines say how to put them back. ## If it stops @@ -150,8 +150,8 @@ The message shows the command that installs Python. Run it, then run the installer again. A RetroPie 4.8 image has Python 3.7.3, and its update command finds nothing -newer. Use the [RetroPie ZIP](../platforms/retropie.md#download) there, -unzipped into `~/RetroPie/BIOS/`. +newer. Run the command on a computer, with +`--platform retropie --dest \\RETROPIE\bios\`. ### Error: the downloaded installer does not match its expected fingerprint. @@ -159,13 +159,11 @@ The download was cut short. Nothing was changed. Run the command again. ### Cannot create ... -You do not have the right to write there. Do what the message says. It names -what to run. +You cannot write there. Do what the message says. ### Not enough space on the drive holding ... -It stopped before writing. Free some space, or use `--dest` to pick another -drive. +Nothing was written. Free some space, or pick another drive with `--dest`. ### The game still does not start diff --git a/wiki/glossary.md b/wiki/glossary.md index 75c966e7..6acb9644 100644 --- a/wiki/glossary.md +++ b/wiki/glossary.md @@ -22,7 +22,8 @@ order: 60 : The folder where your setup looks for BIOS files. **Pack** -: The ZIP file with every BIOS file for one setup. +: The ZIP file with every BIOS file for one setup. Made for the setups most + people download. **Platform** : A setup, as the project's data calls it. diff --git a/wiki/installer.md b/wiki/installer.md index daf32cc5..0f42777f 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -992,8 +992,8 @@ file rather than followed. `install.py` runs on the user's machine, often an older handheld, so it stays on Python 3.8 and the standard library alone. The RetroPie 4.8 images (`retropie-buster-4.8-*.img.gz`) are Debian Buster, whose `python3` is 3.7.3: -there the bootstrap stops before running anything, and the RetroPie pack from -the releases page is the way to install +there the bootstrap stops before running anything, and the way to install is +the command run on a computer with the Pi's network folder as `--dest` ([first install](first-install.md#error-python-38-or-newer-is-required)). Tests refuse any 3.9 syntax or API, and one runs the installer end to end under a real 3.8 interpreter, which CI installs beside 3.12 so the check cannot skip. @@ -1167,10 +1167,10 @@ left out. `scripts/download.py --list` and `scripts/download.sh --list` print the packs of a release. A pack's name lists every platform it serves. Releases up to -v2026.10.10 ship RetroArch and Lakka as one `RetroArch_Lakka` pack, and the -next ones give Lakka its own, since Lakka runs on Linux where two names that -differ by case are two files. The installer and the download scripts find the -right pack either way. +v2026.10.10 ship RetroArch and Lakka as one `RetroArch_Lakka` pack. Later +releases publish packs only for the setups most people download, so Lakka, +which runs on Linux where two names that differ by case are two files, is +served by the installer. The download scripts name the installer for a setup without a pack. Parts named `Pack.part1of2.zip` are ordinary ZIPs: unzip each one into the same folder, and check that all of them were downloaded. The name says how many diff --git a/wiki/reference-terms.md b/wiki/reference-terms.md index 7f892478..a42aeee4 100644 --- a/wiki/reference-terms.md +++ b/wiki/reference-terms.md @@ -14,7 +14,8 @@ everyday words (BIOS, emulator, core, hash) are in the [glossary](glossary.md). **Pack** : One ZIP file with every BIOS file a setup uses. A pack over 2 GB comes in - several parts, and every part is needed. + several parts, and every part is needed. A release publishes packs for the + setups the registry marks `release_pack`, and the installer serves the rest. **Profile and source_ref** : A profile records which files one emulator opens, read from its code. diff --git a/wiki/release-process.md b/wiki/release-process.md index 126467c1..0c519813 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -206,8 +206,9 @@ pipeline runs here, the archives are checked here, and `gh` uploads them. ```bash # 1. Full pipeline, online, so data directories and MAME/FBNeo hashes are -# fresh. --include-archived builds RetroPie too, archived but still served, -# and holds its pack to the same consistency and integrity checks. +# fresh. It builds the packs of the platforms marked release_pack (see +# "Which platforms get a pack" below) and the install manifests of every +# platform, RetroPie included with --include-archived. python scripts/pipeline.py --include-archived # 2. A release file must be under 2 GiB. A larger pack becomes parts that are @@ -276,6 +277,43 @@ gh release list --json tagName,createdAt \ | tail -n +2 | while read tag; do gh release delete "$tag" --yes --cleanup-tag; done ``` +## Which platforms get a pack { #which-platforms-get-a-pack } + +`release_pack: true` in `platforms/_registry.yml` is the list. `generate_pack.py +--all` builds those packs and no other, `--verify-packs` and the consistency +step of the pipeline check those, `release_record.py` refuses any other pack +in `dist/`, and `check_release.py` reports a marked platform without a pack. +The README, the download page, each setup page and the release notes read the +same field: a platform without it shows "No ZIP" and the command. Its install +manifest is written all the same, and `generate_pack.py --platform NAME` still +builds its pack for anyone who wants one. + +The list follows downloads. In v2026.10.10, part 1 of each pack was fetched +7,251 times for RetroArch, 2,909 for EmuDeck, 1,988 for RetroBat, 1,933 for +Batocera and 1,003 for RomM, then 751 for RetroPie, 642 for Recalbox, 516 for +RetroDECK, 450 for MiSTer, 386 for ROCKNIX and 361 for BizHawk. The first five +are 83% of downloads for 17 GB of the 35 GB all packs weighed. MiSTer stays +for its 17 MB. To revisit the list, read the counts of the latest release, +counting a whole pack or the first part of a split one, so each download +counts once: + +```bash +gh api repos/Abdess/retrobios/releases/latest --jq '.assets[] + | select(.name | test("_BIOS_Pack(\\.part1of[0-9]+)?\\.zip$")) + | "\(.download_count)\t\(.name)"' | sort -rn +``` + +Then add or remove the field and regenerate the README and the site. + +Every platform has a page at `get//` on the site, its permanent +download address. It lists the files of the pack the latest release holds, +read from `release.json`, or gives the command when there is no pack. The +README, the setup pages and the release notes link it, never an asset URL: +step 6 deletes the previous tag, and with it every link to its assets. Six of +the eight links one community wiki had to our packs answered 404 that way. +Committing `release.json` in step 5 rebuilds these pages, so the release +never touches them. + ## Verifying a release `SHA256SUMS.txt` is signed with a key used for nothing else. Its public half @@ -320,8 +358,8 @@ fingerprint, `SHA256:jUcTBhDS5DhmheXuVhAzh3pb04uI0caOAaiZzWXvrk4`. Rotating the key means committing the new public half to `allowed_signers` and keeping the retired line, so signatures on past releases keep verifying. -One pack per platform, the full one: the platform's list plus everything its -cores load. Platform-only and per-emulator packs are build options, not +One pack per platform marked `release_pack`, the full one: the platform's list +plus everything its cores load. Platform-only and per-emulator packs are build options, not release assets, since a lighter pack means a core that fails with no message. The release notes are written by `release_notes.py`: the install commands, the pack table and its parts sentence come from `release.json`, then what diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 73858829..b70673a9 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -48,7 +48,7 @@ Every module in `tests/`, by the area it guards. | Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` | | File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_archive_recipes`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` | | Verification, cross-reference and profile checks | synthetic + profiles | `test_bios_layout`, `test_cache_independent_counts`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | -| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | +| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_offline`, `test_install_selection` | diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md index fdbfaeb7..6ea973d2 100644 --- a/wiki/troubleshooting.md +++ b/wiki/troubleshooting.md @@ -24,25 +24,23 @@ Read the count line: 1/109 present (0 verified, 1 wrong hash) ``` -- **`wrong hash`**: another region, model, or damaged. Batocera says - `UNTESTED`. +- **`wrong hash`**: another region or model, or damaged. - **Fewer present than the total**: files are missing. - **A line under a file**, like `still zipped`, says what to fix. - **`0 wrong hash`, all present**: go to step 3. ## 2. Fix the files -Run it again without `--check`. It adds missing files and replaces wrong ones. +Run it without `--check`. It adds missing files and replaces wrong ones. ## 3. Check the folder and the names -- **Right folder?** [Get your files](../which-pack.md) names the one your setup - reads. +- **Right folder?** [Get your files](../which-pack.md) names it. - **Exact name?** `scph5501.bin` and `SCPH-5501.BIN` differ. On Linux, capitals count too. - **A `bios` folder inside your BIOS folder?** Move its files up one level. -- **Unzipped by hand on Linux?** Run the command once. It adds names that - differ by capitals only. +- **Unzipped by hand on Linux?** The command adds names that differ by + capitals. - **A ZIP in parts, like `Pack.part1of2.zip`?** Unzip every part into one folder. @@ -58,17 +56,16 @@ version. ## The BIOS is fine, the game still fails -- **Another emulator?** Each one reads its own files. Its page lists them. +- **Another emulator?** Each reads its own files, listed on its page. - **A BIOS setting?** Some emulators can skip the BIOS. In PCSX ReARMed, set **BIOS Selection** to **Auto**. -- **A disc game in several files?** Open its `.cue` or `.m3u` file, not a - `.bin`. +- **A disc in several files?** Open its `.cue` or `.m3u`, not a `.bin`. ## If the command fails { #installation-script-fails } - **Network error:** try again once github.com opens. -- **`The clock reads ...`:** the date is wrong. Let it sync over Wi-Fi, or set - it, then run again. +- **`The clock reads ...`:** the date is wrong. Sync it over Wi-Fi or set it, + then run again. - **Permission denied:** do what the message says. It names what to run. - **No setup found:** name it and its folder: