diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 7ddab539..f2b792d8 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -19,6 +19,8 @@ on: - "install/**" - "install.py" - "database.json" + - "release.json" + - "scripts/release_record.py" - "mkdocs.yml" - ".github/workflows/deploy-site.yml" workflow_dispatch: diff --git a/README.md b/README.md index 8ddb8417..61bb2914 100644 --- a/README.md +++ b/README.md @@ -44,18 +44,18 @@ Every release ships `SHA256SUMS.txt` and a detached signature of it, checkable a | Platform | Files | Extracted size | Extract to | Download | |----------|------:|---------------:|-----------|----------| -| Batocera | 3,570 | 6.9 GB | `/userdata/bios/` | [Download](../../releases/latest) | -| BizHawk | 811 | 2.4 GB | `Firmware/` | [Download](../../releases/latest) | -| EmuDeck | 584 | 3.2 GB | `~/Emulation/bios/` | [Download](../../releases/latest) | -| Lakka | 8,225 | 6.2 GB | `/storage/system/` | [Download](../../releases/latest) | -| MiSTer FPGA | 79 | 25 MB | `/media/fat/games/` | [Download](../../releases/latest) | -| ROCKNIX | 5,452 | 6.1 GB | `/storage/roms/bios/` | [Download](../../releases/latest) | -| Recalbox | 2,850 | 3.9 GB | `/recalbox/share/bios/` | [Download](../../releases/latest) | -| RetroArch | 8,225 | 6.2 GB | `system/` | [Download](../../releases/latest) | -| RetroBat | 3,040 | 7.3 GB | `bios/` | [Download](../../releases/latest) | -| RetroDECK | 7,194 | 7.6 GB | `~/retrodeck/` | [Download](../../releases/latest) | -| RetroPie * | 8,630 | 7.6 GB | `~/RetroPie/BIOS/` | [Download](../../releases/latest) | -| RomM | 960 | 1.5 GB | `bios/{platform_slug}/` | [Download](../../releases/latest) | +| Batocera | 1,839 | 4.0 GB | `/userdata/bios/` | [Download](../../releases/latest) | +| BizHawk | 542 | 2.2 GB | `Firmware/` | [Download](../../releases/latest) | +| EmuDeck | 624 | 3.2 GB | `~/Emulation/bios/` | [Download](../../releases/latest) | +| Lakka | 4,525 | 5.5 GB | `/storage/system/` | [Download](../../releases/latest) | +| MiSTer FPGA | 74 | 24 MB | `/media/fat/games/` | [Download](../../releases/latest) | +| ROCKNIX | 1,741 | 5.3 GB | `/storage/roms/bios/` | [Download](../../releases/latest) | +| Recalbox | 1,330 | 3.6 GB | `/recalbox/share/bios/` | [Download](../../releases/latest) | +| RetroArch | 4,525 | 5.5 GB | `system/` | [Download](../../releases/latest) | +| RetroBat | 1,466 | 4.4 GB | `bios/` | [Download](../../releases/latest) | +| RetroDECK | 3,519 | 6.2 GB | `~/retrodeck/` | [Download](../../releases/latest) | +| RetroPie * | 4,525 | 5.5 GB | `~/RetroPie/BIOS/` | [Download](../../releases/latest) | +| RomM | 617 | 1.3 GB | `bios/{platform_slug}/` | [Download](../../releases/latest) | The RetroDECK pack already contains its own `bios/` folder, so it extracts into `~/retrodeck/` rather than into the BIOS folder. @@ -180,4 +180,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in the [FAQ](https://abdess.github.io/retrobios/wiki/faq/#is-this-legal). -*Auto-generated on 2026-10-04T16:12:24Z* +*Auto-generated on 2026-10-04T18:24:32Z* diff --git a/release.json b/release.json new file mode 100644 index 00000000..15590bc6 --- /dev/null +++ b/release.json @@ -0,0 +1,101 @@ +{ + "tag": "v2026.09.04", + "packs": { + "Batocera_43.1_BIOS_Pack.zip": { + "files": 1839, + "extracted_size": 4320230269, + "download_size": 2557851727, + "assets": [ + "Batocera_43.1_BIOS_Pack.zip.001", + "Batocera_43.1_BIOS_Pack.zip.002" + ] + }, + "BizHawk_2.11.1_BIOS_Pack.zip": { + "files": 542, + "extracted_size": 2332771486, + "download_size": 1223296442, + "assets": [ + "BizHawk_2.11.1_BIOS_Pack.zip" + ] + }, + "EmuDeck_2.3.8_BIOS_Pack.zip": { + "files": 624, + "extracted_size": 3398423896, + "download_size": 1889509442, + "assets": [ + "EmuDeck_2.3.8_BIOS_Pack.zip" + ] + }, + "MiSTer_FPGA_2026-08-29_BIOS_Pack.zip": { + "files": 74, + "extracted_size": 25209623, + "download_size": 16769418, + "assets": [ + "MiSTer_FPGA_2026-08-29_BIOS_Pack.zip" + ] + }, + "ROCKNIX_20260901_BIOS_Pack.zip": { + "files": 1741, + "extracted_size": 5715015067, + "download_size": 3337595108, + "assets": [ + "ROCKNIX_20260901_BIOS_Pack.zip.001", + "ROCKNIX_20260901_BIOS_Pack.zip.002" + ] + }, + "Recalbox_10.0.8_BIOS_Pack.zip": { + "files": 1330, + "extracted_size": 3883703707, + "download_size": 2096847659, + "assets": [ + "Recalbox_10.0.8_BIOS_Pack.zip.001", + "Recalbox_10.0.8_BIOS_Pack.zip.002" + ] + }, + "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip": { + "files": 4525, + "extracted_size": 5882727352, + "download_size": 3349430771, + "assets": [ + "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip.001", + "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip.002" + ] + }, + "RetroBat_8.2.1_BIOS_Pack.zip": { + "files": 1466, + "extracted_size": 4724194892, + "download_size": 3105818817, + "assets": [ + "RetroBat_8.2.1_BIOS_Pack.zip.001", + "RetroBat_8.2.1_BIOS_Pack.zip.002" + ] + }, + "RetroDECK_0.10.9b_BIOS_Pack.zip": { + "files": 3519, + "extracted_size": 6627934502, + "download_size": 4132852872, + "assets": [ + "RetroDECK_0.10.9b_BIOS_Pack.zip.001", + "RetroDECK_0.10.9b_BIOS_Pack.zip.002", + "RetroDECK_0.10.9b_BIOS_Pack.zip.003" + ] + }, + "RetroPie_v1.22.2_BIOS_Pack.zip": { + "files": 4525, + "extracted_size": 5882738533, + "download_size": 3349391169, + "assets": [ + "RetroPie_v1.22.2_BIOS_Pack.zip.001", + "RetroPie_v1.22.2_BIOS_Pack.zip.002" + ] + }, + "RomM_5.2.0_BIOS_Pack.zip": { + "files": 617, + "extracted_size": 1370456481, + "download_size": 765186250, + "assets": [ + "RomM_5.2.0_BIOS_Pack.zip" + ] + } + } +} diff --git a/schemas/release.schema.json b/schemas/release.schema.json new file mode 100644 index 00000000..a84a6ae3 --- /dev/null +++ b/schemas/release.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Abdess/retrobios/schemas/release.schema.json", + "title": "RetroBIOS release record", + "description": "What the packs of the latest release hold, read from the archives when the release is cut.", + "type": "object", + "additionalProperties": false, + "required": ["tag", "packs"], + "properties": { + "tag": { "type": "string", "pattern": "^v[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}$" }, + "packs": { + "type": "object", + "minProperties": 1, + "propertyNames": { "pattern": "_BIOS_Pack\\.zip$" }, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["files", "extracted_size", "download_size", "assets"], + "properties": { + "files": { "type": "integer", "minimum": 1 }, + "extracted_size": { "type": "integer", "minimum": 0 }, + "download_size": { "type": "integer", "minimum": 1 }, + "assets": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string" } + } + } + } + } + } +} diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 2c1399c0..4cf67189 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -29,6 +29,7 @@ from common import ( unique_emulator_profiles, write_if_changed, ) +import release_record from verify import verify_platform @@ -94,37 +95,28 @@ def compute_coverage( } -def manifest_totals( - platform_name: str, install_dir: str = "install" -) -> tuple[int | None, int | None]: - """Files and bytes a platform's pack holds once extracted. +def release_totals(platform_name: str, record: dict) -> tuple[int | None, int | None]: + """Files and bytes of the pack the latest release serves for a platform. - The install manifest lists what the installer fetches, and states apart - what the pack adds to that: the data directories and its own documents. - The pack figures are the ones a reader can check against an extraction. - Returns (None, None) when no manifest exists yet. + The Download link gives that release, so these are the figures a reader + can check against an extraction. The install manifests describe what + main would build today, which is a different pack as soon as a commit + lands after the release. Returns (None, None) for a platform the release + carries no pack for. """ - path = os.path.join(install_dir, f"{platform_name}.json") - if not os.path.exists(path): + pack = release_record.pack_for(platform_name, record) + if not pack: return None, None - try: - with open(path) as f: - manifest = json.load(f) - except (json.JSONDecodeError, OSError): - return None, None - return ( - manifest.get("pack_files", manifest.get("total_files")), - manifest.get("pack_size", manifest.get("total_size")), - ) + return pack["files"], pack["extracted_size"] def download_table( coverages: dict, archived: set[str], extract_paths: dict[str, str], - install_dir: str = "install", + record: dict, ) -> list[str]: - """One row per platform: what its pack holds and where it extracts.""" + """One row per platform: what its released pack holds and where it extracts.""" lines = [ "| Platform | Files | Extracted size | Extract to | Download |", "|----------|------:|---------------:|-----------|----------|", @@ -133,7 +125,7 @@ def download_table( display = cov["platform"] if name in archived: display = f"{display} *" - files, size = manifest_totals(name, install_dir) + files, size = release_totals(name, record) lines.append( f"| {display} | {f'{files:,}' if files else '-'} |" f" {format_size(size) if size else '-'} |" @@ -384,7 +376,11 @@ def generate_readme(db: dict, platforms_dir: str) -> str: if entry.get("status") == "archived" } - lines.extend(download_table(coverages, archived, extract_paths)) + lines.extend( + download_table( + coverages, archived, extract_paths, release_record.load_record() + ) + ) if archived: lines.extend( diff --git a/scripts/generate_site.py b/scripts/generate_site.py index fd410690..f63d918b 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -51,9 +51,10 @@ from nativemode import reads_file_contents yaml = require_yaml() -from generate_readme import compute_coverage, manifest_totals +from generate_readme import compute_coverage, release_totals from profile_sync import source_ref_values, split_source_ref from provenance_report import build_report +import release_record import upstream DOCS_DIR = "docs" @@ -281,7 +282,7 @@ def generate_home( lines.append( f"| {logo_md} | [{display}](platforms/{name}.md) | " - f"{manifest_totals(name)[0] or cov['present']:,} | {mode_label} | " + f"{_released_files(name)} | {mode_label} | " f"[Pack]({RELEASE_URL}){{ .md-button .md-button--primary }} |" ) @@ -410,6 +411,12 @@ def compute_stats(db: dict, coverages: dict, profiles: dict) -> dict: } +def _released_files(platform_name: str) -> str: + """File count of the pack the latest release serves, as a table cell.""" + files, _size = release_totals(platform_name, release_record.load_record()) + return f"{files:,}" if files else "-" + + # The home tile counts the whole collection. Labelled "Files" above a table # of pack buttons, it read as what one pack contains. COLLECTION_FILES_LABEL = "Files collected" @@ -1181,7 +1188,7 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st lines.append( f"| [{display}]({name}.md) | " - f"{cov['present']:,} | {manifest_totals(name)[0] or cov['present']:,} | " + f"{cov['present']:,} | {_released_files(name)} | " f"{mode_html} | {status_html} | " f"[Pack]({RELEASE_URL}){{ .md-button .md-button--primary }} |" ) diff --git a/scripts/release_record.py b/scripts/release_record.py new file mode 100644 index 00000000..6984f426 --- /dev/null +++ b/scripts/release_record.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Record what the packs of a release hold. + +The README and the site print, beside each download link, how many files a +pack holds and what they weigh once extracted. Those figures have to be the +ones of the release the link serves. Read from the install manifests they +described what main would build today, and drifted from the published pack +with every commit made since the release. + +The record is written from the archives themselves when a release is cut, +committed with it, and read by the generators. + +Usage: + python scripts/release_record.py dist/ --tag v2026.10.04 +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import zipfile +from pathlib import Path + +import split_pack + +RECORD = "release.json" + + +def _match_key(value: str) -> str: + """Letters and digits only: the platform is `misterfpga`, the pack MiSTer_FPGA.""" + return re.sub(r"[^a-z0-9]", "", value.lower()) + + +def build_record(dist: Path, tag: str) -> dict: + """Read every pack of a directory, parts counted once across them.""" + assets: dict[str, list[Path]] = {} + for path in sorted(Path(dist).glob("*_BIOS_Pack*.zip")): + assets.setdefault(split_pack.pack_of(path.name), []).append(path) + + packs = {} + for name, paths in sorted(assets.items()): + members: dict[str, int] = {} + for path in paths: + with zipfile.ZipFile(path) as archive: + for info in archive.infolist(): + members[info.filename] = info.file_size + packs[name] = { + "files": len(members), + "extracted_size": sum(members.values()), + "download_size": sum(path.stat().st_size for path in paths), + "assets": [path.name for path in paths], + } + return {"tag": tag, "packs": packs} + + +def load_record(path: str | Path = RECORD) -> dict: + """The committed record, empty when no release has written one.""" + try: + with open(path, encoding="utf-8") as handle: + return json.load(handle) + except (OSError, json.JSONDecodeError): + return {} + + +def pack_for(platform: str, record: dict) -> dict | None: + """The pack of the release that serves a platform, if it has one.""" + needle = _match_key(platform) + if not needle: + return None + for name, pack in sorted(record.get("packs", {}).items()): + if needle in _match_key(name): + return pack + return None + + +def manifest_mismatches(record: dict, install_dir: str | Path) -> list[str]: + """Packs whose archive does not hold what their install manifest expects.""" + found = [] + for manifest_path in sorted(Path(install_dir).glob("*.json")): + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + expected = manifest.get("pack_files") + if expected is None: + continue + for name, pack in sorted(record.get("packs", {}).items()): + if _match_key(manifest_path.stem) not in _match_key(name): + continue + if pack["files"] != expected: + found.append( + f"{name} holds {pack['files']} files, " + f"install/{manifest_path.name} expects {expected}" + ) + break + return found + + +def _size(count: int) -> str: + if count >= 1024**3: + return f"{count / 1024**3:.1f} GB" + return f"{count / 1024**2:.0f} MB" + + +def main() -> int: + parser = argparse.ArgumentParser(description="Record what a release's packs hold") + parser.add_argument("dist", type=Path, help="directory of the packs to publish") + parser.add_argument("--tag", required=True, help="release tag, e.g. v2026.10.04") + parser.add_argument("--install-dir", default="install", type=Path) + parser.add_argument("--output", default=RECORD, type=Path) + args = parser.parse_args() + + record = build_record(args.dist, args.tag) + if not record["packs"]: + print(f"Error: no pack in {args.dist}", file=sys.stderr) + return 1 + for name, pack in record["packs"].items(): + print( + f"{name:<44} download {_size(pack['download_size']):>8}" + f" extracted {_size(pack['extracted_size']):>8}" + f" {pack['files']:,} files {len(pack['assets'])} asset(s)" + ) + mismatches = manifest_mismatches(record, args.install_dir) + for line in mismatches: + print(f"Error: {line}", file=sys.stderr) + if mismatches: + return 1 + args.output.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {args.output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/split_pack.py b/scripts/split_pack.py index dbc0f67b..8ed3bb6c 100644 --- a/scripts/split_pack.py +++ b/scripts/split_pack.py @@ -45,6 +45,12 @@ def is_part(name: str) -> bool: return _PART_NAME.match(name) is not None +def pack_of(name: str) -> str: + """The pack a published file belongs to, a part folded into its archive.""" + part = _PART_NAME.match(name) + return f"{part['stem']}.zip" if part else name + + def part_name(pack_name: str, index: int, count: int) -> str: return f"{pack_name[: -len('.zip')]}.part{index}of{count}.zip" diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index a28e133a..952884da 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -259,6 +259,10 @@ def main() -> int: _validate_json_files(target_paths, "target-manifest.schema.json") ) + release_path = ROOT / "release.json" + if release_path.exists(): + errors.extend(_validate_json_files([release_path], "release.schema.json")) + stats_path = ROOT / "docs" / "stats.json" if stats_path.exists(): errors.extend(_validate_json_files([stats_path], "stats.schema.json")) diff --git a/tests/test_pack_counts.py b/tests/test_pack_counts.py index a0dc3897..096fc0ee 100644 --- a/tests/test_pack_counts.py +++ b/tests/test_pack_counts.py @@ -28,6 +28,8 @@ import generate_db # noqa: E402 import generate_pack as builder # noqa: E402 import generate_readme # noqa: E402 import generate_site # noqa: E402 +import release_record # noqa: E402 +import split_pack # noqa: E402 PROFILE = """\ emulator: Demo @@ -156,39 +158,120 @@ class ManifestStatesWhatThePackHolds(PackCountFixture): self.assertEqual(errors, []) -class ReadmeTotals(unittest.TestCase): +RECORD = { + "tag": "v2026.09.04", + "packs": { + "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip": { + "files": 4525, + "extracted_size": 5882727352, + "download_size": 3349430771, + "assets": [ + "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip.001", + "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip.002", + ], + }, + "MiSTer_FPGA_2026-08-29_BIOS_Pack.zip": { + "files": 74, + "extracted_size": 17000000, + "download_size": 16769418, + "assets": ["MiSTer_FPGA_2026-08-29_BIOS_Pack.zip"], + }, + }, +} + + +class TheTableDescribesTheRelease(unittest.TestCase): + """The Download link gives the last release, not what main would build. + + The install manifest of main already counted 8 225 files for RetroArch + while the published pack held 4 525: printed beside the link, the newer + figure would have sent the next reader looking for 3 700 missing files. + """ + + def test_a_platform_reads_the_pack_that_serves_it(self): + for platform in ("retroarch", "lakka"): + self.assertEqual( + generate_readme.release_totals(platform, RECORD), + (4525, 5882727352), + ) + self.assertEqual( + generate_readme.release_totals("misterfpga", RECORD), (74, 17000000) + ) + + def test_a_platform_the_release_does_not_carry_has_no_figure(self): + self.assertEqual( + generate_readme.release_totals("vita3k", RECORD), (None, None) + ) + self.assertEqual(generate_readme.release_totals("retroarch", {}), (None, None)) + + def test_the_download_table_gives_the_file_count_of_each_pack(self): + rows = generate_readme.download_table( + {"retroarch": {"platform": "RetroArch"}, "vita3k": {"platform": "Vita3K"}}, + set(), + {"RetroArch": "`system/`"}, + RECORD, + ) + self.assertIn("| Platform | Files | Extracted size |", rows[0]) + self.assertIn("| RetroArch | 4,525 | 5.5 GB | `system/` |", rows[2]) + self.assertIn("| Vita3K | - | - |", rows[3]) + + def test_the_committed_record_is_the_one_the_readme_prints(self): + record_path = REPO_ROOT / "release.json" + if not record_path.is_file(): + self.skipTest("no release.json") + record = json.loads(record_path.read_text(encoding="utf-8")) + readme = (REPO_ROOT / "README.md").read_text(encoding="utf-8") + files, _size = generate_readme.release_totals("retroarch", record) + self.assertTrue(f"| RetroArch | {files:,} |" in readme) + + +class ReleaseRecord(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory() - self.install = Path(self._tmp.name) + self.dist = Path(self._tmp.name) / "dist" + self.dist.mkdir() + self.install = Path(self._tmp.name) / "install" + self.install.mkdir() + self._pack("Whole_1.0_BIOS_Pack.zip", {"a.bin": b"a" * 10, "README.txt": b"r"}) + big = self._pack( + "Demo_2.0_BIOS_Pack.zip", + {f"dir/file{n}.bin": bytes([n]) * 700 + bytes(range(256)) for n in range(6)}, + ) + self.parts = split_pack.split_pack(big, limit=big.stat().st_size - 1) def tearDown(self): self._tmp.cleanup() - def _write(self, **fields) -> None: - (self.install / "demo.json").write_text(json.dumps(fields)) + def _pack(self, name: str, members: dict[str, bytes]) -> Path: + path = self.dist / name + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for member, data in members.items(): + archive.writestr(member, data) + return path - def test_the_pack_figures_win(self): - self._write(total_files=1872, total_size=10, pack_files=4525, pack_size=20) + def test_a_pack_in_parts_is_counted_once_across_them(self): + record = release_record.build_record(self.dist, "v1") + self.assertEqual(record["tag"], "v1") + demo = record["packs"]["Demo_2.0_BIOS_Pack.zip"] + self.assertEqual(demo["files"], 6) + self.assertEqual(demo["extracted_size"], 6 * (700 + 256)) + self.assertEqual(demo["assets"], [part.name for part in self.parts]) self.assertEqual( - generate_readme.manifest_totals("demo", str(self.install)), (4525, 20) + demo["download_size"], sum(part.stat().st_size for part in self.parts) ) + self.assertEqual(record["packs"]["Whole_1.0_BIOS_Pack.zip"]["files"], 2) - def test_a_manifest_written_before_them_still_answers(self): - self._write(total_files=1872, total_size=10) + def test_a_pack_that_disagrees_with_its_manifest_is_named(self): + """The count main predicts and the count the archive holds are two + computations of one thing. Apart, one of them is wrong.""" + record = release_record.build_record(self.dist, "v1") + (self.install / "demo.json").write_text(json.dumps({"pack_files": 6})) + (self.install / "whole.json").write_text(json.dumps({"pack_files": 5})) self.assertEqual( - generate_readme.manifest_totals("demo", str(self.install)), (1872, 10) + release_record.manifest_mismatches(record, self.install), + ["Whole_1.0_BIOS_Pack.zip holds 2 files, install/whole.json expects 5"], ) - def test_the_download_table_gives_the_file_count_of_each_pack(self): - self._write(total_files=1872, total_size=10, pack_files=4525, - pack_size=5882727352) - rows = generate_readme.download_table( - {"demo": {"platform": "Demo"}}, set(), {"Demo": "`system/`"}, - str(self.install), - ) - self.assertIn("| Platform | Files | Extracted size |", rows[0]) - self.assertIn("| Demo | 4,525 | 5.5 GB | `system/` |", rows[2]) - class TheCollectionTotalNamesTheCollection(unittest.TestCase): COMPOSITION = { diff --git a/wiki/release-process.md b/wiki/release-process.md index 8b6a159e..c99f0f5d 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -28,7 +28,7 @@ hosted runner should rebuild and re-upload. **Trigger.** Push to `main` when any of these paths change: `platforms/`, `emulators/`, `provenance/`, `wiki/`, `scripts/generate_site.py`, `scripts/generate_readme.py`, `scripts/verify.py`, `scripts/common.py`, -`database.json`, `mkdocs.yml`. Also manual dispatch. +`database.json`, `release.json`, `mkdocs.yml`. Also manual dispatch. The list is the set of inputs the site is generated from. Adding a new input to `generate_site.py` means adding its path here, or the site silently goes stale. @@ -174,35 +174,19 @@ python scripts/split_pack.py dist/ (cd dist && sha256sum *.zip > SHA256SUMS.txt) ssh-keygen -Y sign -f ~/.ssh/retrobios_signing -n file dist/SHA256SUMS.txt -# 5. The two sizes a pack has, and its file count. Extracted runs well above -# downloaded, so the notes table names the one it carries. The count is -# what a file manager shows once the pack is extracted. -python3 - <<'PY' -import json, pathlib, sys -sys.path.insert(0, "scripts") -from download import _match_key - -parts = {} -for path in sorted(pathlib.Path("dist").glob("*_BIOS_Pack*.zip")): - parts.setdefault(path.name.split("_BIOS_Pack")[0], []).append(path) - -def size(n): - return f"{n / 1024 ** 3:.1f} GB" if n >= 1024 ** 3 else f"{n / 1024 ** 2:.0f} MB" - -for manifest in sorted(pathlib.Path("install").glob("*.json")): - base = next((b for b in parts if _match_key(manifest.stem) in _match_key(b)), None) - if not base: - continue - data = json.loads(manifest.read_text()) - download = sum(p.stat().st_size for p in parts[base]) - print(f"{base:<42} download {size(download):>8}" - f" extracted {size(data['pack_size']):>8} {data['pack_files']:,} files") -PY +# 5. Record what each pack holds, read from the archives just checked: +# file count, extracted size, download size, published files. The README +# and the site print these beside the download links, and the notes table +# takes its Size and Files columns from the same lines. The command stops +# if an archive does not hold the count its install manifest expects. +DATE=$(date +%Y.%m.%d) +python scripts/release_record.py dist/ --tag "v${DATE}" +python scripts/generate_readme.py --db database.json --platforms-dir platforms +python scripts/generate_site.py # 6. Create the release as a DRAFT, upload every asset, and only then publish it. # A public release with half its assets is a broken download for everyone # during the whole upload. -DATE=$(date +%Y.%m.%d) gh release create "v${DATE}" --draft --title "BIOS Pack v${DATE}" --notes-file notes.md for f in dist/SHA256SUMS.txt dist/SHA256SUMS.txt.sig dist/*.zip; do [ -f "$f" ] && gh release upload "v${DATE}" "$f#$(basename "$f")" --clobber @@ -210,6 +194,11 @@ done gh release view "v${DATE}" --json assets --jq '.assets | length' # expect every file gh release edit "v${DATE}" --draft=false --latest +# The record and the pages it feeds go to main with the release, so the +# table never describes a pack other than the one the link serves. +git add release.json README.md mkdocs.yml +git commit -m "chore: record release v${DATE}" && git push + # 7. Keep only the new release plus large-files: an older pack carries hashes # the platforms no longer check, so it misleads more than it helps gh release list --json tagName,createdAt \ @@ -263,9 +252,16 @@ the pack table, what changed since the previous tag, and the contributors of the closed issues. A pack has two sizes and they are far apart: Batocera downloads as 2.4 GB and extracts to 4.0 GB. Step 5 prints both, and whichever one the table carries, the header names it. Someone sizing a USB drive is -reading that column. The README table is the extracted size, from the install -manifests. `SHA256SUMS.txt` lists the checksums of the files as published, -parts included. +reading that column. The README table is the extracted size of the released +pack, from `release.json`. `SHA256SUMS.txt` lists the checksums of the files +as published, parts included. + +`release.json` is the record step 5 writes: per pack, the files it holds, its +two sizes and the files it is published in. It exists because the pages are +regenerated on every push while a release is cut every few weeks. Read from +the install manifests, the table gave the count main would build that day: +a commit adding 3 700 files to the RetroArch pack moved it to 8 225 while +the download still held 4 525. A pack in parts is said so above the table, where the links are: every part is needed, each is an ordinary ZIP, and they extract into the same folder. @@ -277,7 +273,7 @@ and 79). The README, the download page and the troubleshooting page describe both layouts for as long as a release cut that way is still published; once step 7 has deleted it, the `.zip.001` paragraph leaves those three pages. -The table carries a Files column, the `pack_files` step 5 prints, and the +The table carries a Files column, the count step 5 prints, and the notes never open on the size of the collection. That total covers every platform and emulator together and no pack holds it: v2026.09.04 led with "10,330 files" right after "one pack per platform", and someone who extracted