diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 87f8522a..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,156 +0,0 @@ -name: Build & Release - -# Releasing is a deliberate act, not a consequence of pushing. Cutting one is -# a manual dispatch: someone decides the collection is in a state worth -# publishing, and the rate limit below still guards against doing it twice by -# accident. -on: - workflow_dispatch: - inputs: - force_release: - description: "Force release even if rate limited" - type: boolean - default: false - -permissions: {} - -concurrency: - group: build - cancel-in-progress: false - -jobs: - release: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - # The release notes are built from `git log -15`, which returns a single - # commit on the default shallow clone. - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.12" - - - run: pip install pyyaml jsonschema==4.23.0 - - - name: Run tests - run: python -m unittest discover tests - - - name: Rate limit - if: github.event.inputs.force_release != 'true' - id: rate - run: | - LAST=$(gh release list --repo "${{ github.repository }}" --json createdAt -q '.[0].createdAt' 2>/dev/null || echo "") - if [ -n "$LAST" ] && [ "$LAST" != "null" ]; then - LAST_TS=$(date -d "$LAST" +%s 2>/dev/null || echo 0) - DIFF=$(( ($(date +%s) - LAST_TS) / 86400 )) - if [ "$DIFF" -lt 7 ]; then - echo "Skipping: last release ${DIFF} days ago" - echo "skip=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - fi - echo "skip=false" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Restore large files from release - if: steps.rate.outputs.skip != 'true' - run: | - mkdir -p .cache/large - gh release download large-files -D .cache/large/ 2>/dev/null || true - python scripts/restore_large_files.py - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Refresh data directories - if: steps.rate.outputs.skip != 'true' - run: python scripts/refresh_data_dirs.py - continue-on-error: true - - - name: Build packs - if: steps.rate.outputs.skip != 'true' - run: python scripts/generate_pack.py --all --output-dir dist/ - - - name: Split oversized packs - if: steps.rate.outputs.skip != 'true' - run: | - # GitHub releases cap assets at 2 GB. Use the .zip.001 volume - # convention: 7-Zip and PeaZip open the .001 part directly, - # unlike .partNN names which they reject as corrupt archives. - for f in dist/*.zip; do - size=$(stat -c%s "$f") - if [ "$size" -gt 2000000000 ]; then - split --bytes=1900M --numeric-suffixes=1 --suffix-length=3 "$f" "$f." - rm "$f" - echo "Split $(basename "$f") into $(ls "$f".* | wc -l) parts" - fi - done - - - name: Release - if: steps.rate.outputs.skip != 'true' - run: | - DATE=$(date +%Y.%m.%d) - EXISTING=$(gh release list --repo "${{ github.repository }}" \ - --json tagName -q ".[].tagName" | grep -c "^v${DATE}" || true) - TAG="v${DATE}" - [ "$EXISTING" -gt 0 ] && TAG="v${DATE}.$((EXISTING+1))" - - CHANGES=$(git log --oneline -15 --no-merges \ - -- bios/ platforms/ emulators/ | sed 's/^/- /') - TOTAL=$(python3 -c "import json; print(json.load(open('database.json'))['total_files'])") - SIZE=$(python3 -c "import json; print(f'{json.load(open(\"database.json\"))[\"total_size\"]/1024/1024:.0f}')") - PACKS=$(ls dist/*.zip dist/*.zip.001 2>/dev/null | while read f; do echo "- **$(basename $f)** ($(du -m "$f" | cut -f1) MB)"; done) - - gh release create "$TAG" dist/*.zip* \ - --repo "${{ github.repository }}" \ - --title "BIOS Pack $TAG" \ - --notes "${TOTAL} files, ${SIZE} MB, verified checksums. - - ### Packs - ${PACKS} - - ### Install - Download the pack matching your frontend, extract to the BIOS directory. - - Packs over 2 GB are split into numbered volumes (.zip.001, .zip.002). - Download every part, then either open the .001 file directly with - 7-Zip / PeaZip, or join the parts first: - - Linux/macOS: \`cat PackName.zip.0* > PackName.zip\` - - Windows (cmd): \`copy /b PackName.zip.001+PackName.zip.002 PackName.zip\` - - | Platform | Extract to | - |----------|------------| - | RetroArch | system/ | - | Lakka | /storage/system/ | - | RetroPie | ~/RetroPie/BIOS/ | - | Batocera | /userdata/bios/ | - | Recalbox | /recalbox/share/bios/ | - | RetroBat | bios/ | - | RetroDECK | ~/retrodeck/ (the pack carries its own bios/) | - | EmuDeck | ~/Emulation/bios/ | - | ROCKNIX | /storage/roms/bios/ | - | MiSTer FPGA | /media/fat/games/ | - | BizHawk | Firmware/ | - | RomM | bios/{platform_slug}/ | - - ### Changes - ${CHANGES} - " \ - --latest - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Cleanup old releases - if: steps.rate.outputs.skip != 'true' - run: | - gh release list --repo "${{ github.repository }}" --json tagName,createdAt \ - --jq 'sort_by(.createdAt) | reverse | .[].tagName' | \ - grep -v "^large-files$" | tail -n +4 | while read tag; do - gh release delete "$tag" --repo "${{ github.repository }}" --yes --cleanup-tag - done - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 882c5055..795c4b2e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@

- Build Site

@@ -166,4 +165,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in the [FAQ](https://abdess.github.io/retrobios/wiki/faq/#is-this-legal). -*Auto-generated on 2026-09-03T21:10:11Z* +*Auto-generated on 2026-09-04T01:36:37Z* diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 90f2aa34..f1e10420 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -261,8 +261,6 @@ def generate_readme(db: dict, platforms_dir: str) -> str: "

", "", '

', - ' ' - 'Build', ' ' 'Site', "

", diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 28529226..2689ef8e 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -724,12 +724,6 @@ class CheckoutCompletenessRegressions(unittest.TestCase): self.assertLess(self._index(steps, "restore_large_files.py"), generate) self.assertLess(self._index(steps, "refresh_data_dirs.py"), generate) - def test_pack_build_completes_the_checkout_before_building(self): - steps = self._steps("build.yml", "release") - build = self._index(steps, "Build packs") - self.assertLess(self._index(steps, "restore_large_files.py"), build) - self.assertLess(self._index(steps, "refresh_data_dirs.py"), build) - def test_restore_matches_assets_by_content_not_by_name(self): from scripts.restore_large_files import restore diff --git a/wiki/architecture.md b/wiki/architecture.md index 22038c05..c8626309 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -347,12 +347,11 @@ pattern and how to add a test. | Workflow | File | Trigger | Role | |----------|------|---------|------| -| Build & Release | `build.yml` | manual dispatch only | restore large files, build packs, create GitHub release | | Deploy Site | `deploy-site.yml` | push to main (platforms, emulators, wiki, scripts) + manual | validate contracts, generate site, build with MkDocs, validate rendered HTML, deploy to Pages | | PR Validation | `validate.yml` | pull request on bios/, platforms/, emulators/, schemas/, scripts/, tests/ | validate BIOS hashes, schema check, run the full test suite, auto-label PR | -The build workflow has no push trigger: a release is dispatched by hand. It -keeps a 7-day rate limit between releases and the 3 most recent tags. See the +Releases are not built in CI: the packs are generated and checked on the +maintainer's machine and uploaded with `gh`, see the [release process](release-process.md). ## License diff --git a/wiki/release-process.md b/wiki/release-process.md index 211f980e..4be80fe7 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -5,7 +5,7 @@ are built, and how to run the process manually. ## CI workflows overview -The project uses 3 GitHub Actions workflows. All use only official GitHub +The project uses 2 GitHub Actions workflows. All use only official GitHub actions (`actions/checkout`, `actions/setup-python`, `actions/upload-pages-artifact`, `actions/deploy-pages`). No third-party actions. @@ -13,49 +13,15 @@ Budget target: ~175 minutes/month on the GitHub free tier. | Workflow | File | Trigger | |----------|------|---------| -| Build & Release | `build.yml` | Manual dispatch only | | Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, mkdocs.yml), manual | | PR Validation | `validate.yml` | PR touching `bios/**`, `platforms/**` or `emulators/**` | Upstream BIOS lists are not scraped on a schedule. A maintainer runs the scrapers by hand (see [adding a scraper](adding-a-scraper.md)), reviews the -diff, and commits the refreshed platform YAML. - -## build.yml - Build & Release - -Releasing is deliberate. Pushing never cuts one: somebody decides the -collection is worth publishing and dispatches the workflow. - -**Trigger.** `workflow_dispatch` only, with an optional `force_release` flag to -bypass the rate limit. - -**Concurrency.** Group `build`, queued rather than cancelled: a run that is -already uploading assets must finish. - -**Steps:** - -1. Checkout, Python 3.12, install `pyyaml` -2. Run `test_e2e` -3. Rate limit check: skip if last release was less than 7 days ago (unless - `force_release` is set) -4. Restore large files from the `large-files` release into `.cache/large/` -5. Refresh data directories (`refresh_data_dirs.py`) -6. Build packs (`generate_pack.py --all --output-dir dist/`) -7. Split any pack over 2 GB into `.zip.001`, `.zip.002`, ... volumes. GitHub - caps release assets at 2 GB, and the `.001` convention is what 7-Zip and - PeaZip open directly (they reject `.partNN` names as corrupt) -8. Create GitHub release with tag `v{YYYY.MM.DD}` (appends `.N` suffix if - a same-day release already exists) -9. Clean up old releases, keeping the 3 most recent plus `large-files` - -**Release notes** include file count, total size, per-pack sizes, the extract -path per platform, and the last 15 non-merge commits touching `bios/` or -`platforms/`. - -**Pack variants.** The workflow builds full packs only. Releases that also -carry `*_Platform_BIOS_Pack.zip` had a second `--source platform` run added by -hand, as in the manual process below. See -[advanced usage](advanced-usage.md#pack-source-variants). +diff, and commits the refreshed platform YAML. Releases are built on the +maintainer's machine and uploaded with `gh`, see +[cutting a release](#cutting-a-release): the packs weigh 25 GB, more than a +hosted runner should rebuild and re-upload. ## deploy-site.yml - Deploy Documentation Site @@ -163,52 +129,44 @@ gh release upload large-files "bios/Sony/PS3/PS3UPDAT.PUP#PS3UPDAT.PUP" **Local cache.** `generate_pack.py` calls `fetch_large_file()` which downloads from the release and caches in `.cache/large/` for subsequent runs. -## Manual release process +## Cutting a release -When `build.yml` is disabled, build and release manually: +Releasing is deliberate and local. Nothing on GitHub builds a pack: the +pipeline runs here, the archives are checked here, and `gh` uploads them. ```bash -# Run the full pipeline (DB + verify + packs + manifests + integrity + docs) +# 1. Full pipeline, online, so data directories and MAME/FBNeo hashes are fresh python scripts/pipeline.py -# Or step by step: -python scripts/generate_db.py --force --bios-dir bios --output database.json -python scripts/verify.py --all -python scripts/generate_pack.py --all --output-dir dist/ # full packs -python scripts/generate_pack.py --all --source platform --output-dir dist/ # platform packs +# 2. The platform-only packs, and RetroPie, which is archived but still served +python scripts/generate_pack.py --all --source platform --output-dir dist/ +python scripts/generate_pack.py --platform retropie --output-dir dist/ +python scripts/generate_pack.py --platform retropie --source platform --output-dir dist/ + +# 3. The standalone emulators that no frontend bundles +python scripts/generate_pack.py --emulator mesence --output-dir dist/ +python scripts/generate_pack.py --emulator lexaloffle --output-dir dist/ + +# 4. Every archive in dist/ extracted and hashed the way its platform checks it python scripts/generate_pack.py --all --verify-packs --output-dir dist/ -# Split anything over 2 GB (GitHub asset cap) +# 5. Split anything over 2 GB (GitHub asset cap); 7-Zip and PeaZip open .001 directly for f in dist/*.zip; do [ "$(stat -c%s "$f")" -gt 2000000000 ] || continue split --bytes=1900M --numeric-suffixes=1 --suffix-length=3 "$f" "$f." && rm "$f" done -# Create the release +# 6. Create the release, then keep the three most recent plus large-files DATE=$(date +%Y.%m.%d) -gh release create "v${DATE}" dist/*.zip* \ - --title "BIOS Pack v${DATE}" \ - --notes "Release notes here" \ - --latest +gh release create "v${DATE}" dist/*.zip* dist/SHA256SUMS.txt \ + --title "BIOS Pack v${DATE}" --notes-file notes.md --latest +gh release list --json tagName,createdAt \ + --jq 'sort_by(.createdAt) | reverse | .[].tagName' | grep -v '^large-files$' \ + | tail -n +4 | while read tag; do gh release delete "$tag" --yes --cleanup-tag; done ``` -The two `generate_pack.py` runs are what puts both `*_BIOS_Pack.zip` and -`*_Platform_BIOS_Pack.zip` on the release. `--all-variants` builds all six -combinations instead, which is more than a release needs. - -Run the pipeline online for a release: `--offline` skips the data directory -refresh and the MAME/FBNeo hash refresh, so the packs would ship stale data -directories. - -The workflow carries no push trigger, so there is nothing to disable and no -guard to remove. Cutting a release means dispatching `build.yml` from the -Actions tab, or: - -```bash -gh workflow run build.yml -gh workflow run build.yml -f force_release=true # within 7 days of the last -``` - -The rate limit refuses a second release inside seven days unless -`force_release` is set, which is what keeps a stray dispatch from publishing -twice in a day. +The release notes follow the previous release: the quick install commands, the +full and platform pack tables with file counts and sizes, the standalone +packs, what changed since the previous tag, and the contributors of the +closed issues. `SHA256SUMS.txt` lists the checksums of the full ZIPs before +splitting.