kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-10 13:33:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: build releases locally, retire the ci build

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-09-04 03:37:01 +02:00
1 parent 3d7852cdb7
commit 26df60db75
6 files changed
+34 -242
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/build.yml
-156
View File
@@ -1,156 +0,0 @@
name: Build & Release
# Releasing is a deliberate act, not a consequence of pushing. Cutting one is
# a manual dispatch: someone decides the collection is in a state worth
# publishing, and the rate limit below still guards against doing it twice by
# accident.
on:
workflow_dispatch:
inputs:
force_release:
description: "Force release even if rate limited"
type: boolean
default: false
permissions: {}
concurrency:
group: build
cancel-in-progress: false
jobs:
release:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
# The release notes are built from `git log -15`, which returns a single
# commit on the default shallow clone.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
with:
python-version: "3.12"
- run: pip install pyyaml jsonschema==4.23.0
- name: Run tests
run: python -m unittest discover tests
- name: Rate limit
if: github.event.inputs.force_release != 'true'
id: rate
run: |
LAST=$(gh release list --repo "${{ github.repository }}" --json createdAt -q '.[0].createdAt' 2>/dev/null || echo "")
if [ -n "$LAST" ] && [ "$LAST" != "null" ]; then
LAST_TS=$(date -d "$LAST" +%s 2>/dev/null || echo 0)
DIFF=$(( ($(date +%s) - LAST_TS) / 86400 ))
if [ "$DIFF" -lt 7 ]; then
echo "Skipping: last release ${DIFF} days ago"
echo "skip=true" >> "$GITHUB_OUTPUT"
exit 0
fi
fi
echo "skip=false" >> "$GITHUB_OUTPUT"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Restore large files from release
if: steps.rate.outputs.skip != 'true'
run: |
mkdir -p .cache/large
gh release download large-files -D .cache/large/ 2>/dev/null || true
python scripts/restore_large_files.py
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Refresh data directories
if: steps.rate.outputs.skip != 'true'
run: python scripts/refresh_data_dirs.py
continue-on-error: true
- name: Build packs
if: steps.rate.outputs.skip != 'true'
run: python scripts/generate_pack.py --all --output-dir dist/
- name: Split oversized packs
if: steps.rate.outputs.skip != 'true'
run: |
# GitHub releases cap assets at 2 GB. Use the .zip.001 volume
# convention: 7-Zip and PeaZip open the .001 part directly,
# unlike .partNN names which they reject as corrupt archives.
for f in dist/*.zip; do
size=$(stat -c%s "$f")
if [ "$size" -gt 2000000000 ]; then
split --bytes=1900M --numeric-suffixes=1 --suffix-length=3 "$f" "$f."
rm "$f"
echo "Split $(basename "$f") into $(ls "$f".* | wc -l) parts"
fi
done
- name: Release
if: steps.rate.outputs.skip != 'true'
run: |
DATE=$(date +%Y.%m.%d)
EXISTING=$(gh release list --repo "${{ github.repository }}" \
--json tagName -q ".[].tagName" | grep -c "^v${DATE}" || true)
TAG="v${DATE}"
[ "$EXISTING" -gt 0 ] && TAG="v${DATE}.$((EXISTING+1))"
CHANGES=$(git log --oneline -15 --no-merges \
-- bios/ platforms/ emulators/ | sed 's/^/- /')
TOTAL=$(python3 -c "import json; print(json.load(open('database.json'))['total_files'])")
SIZE=$(python3 -c "import json; print(f'{json.load(open(\"database.json\"))[\"total_size\"]/1024/1024:.0f}')")
PACKS=$(ls dist/*.zip dist/*.zip.001 2>/dev/null | while read f; do echo "- **$(basename $f)** ($(du -m "$f" | cut -f1) MB)"; done)
gh release create "$TAG" dist/*.zip* \
--repo "${{ github.repository }}" \
--title "BIOS Pack $TAG" \
--notes "${TOTAL} files, ${SIZE} MB, verified checksums.
### Packs
${PACKS}
### Install
Download the pack matching your frontend, extract to the BIOS directory.
Packs over 2 GB are split into numbered volumes (.zip.001, .zip.002).
Download every part, then either open the .001 file directly with
7-Zip / PeaZip, or join the parts first:
- Linux/macOS: \`cat PackName.zip.0* > PackName.zip\`
- Windows (cmd): \`copy /b PackName.zip.001+PackName.zip.002 PackName.zip\`
| Platform | Extract to |
|----------|------------|
| RetroArch | system/ |
| Lakka | /storage/system/ |
| RetroPie | ~/RetroPie/BIOS/ |
| Batocera | /userdata/bios/ |
| Recalbox | /recalbox/share/bios/ |
| RetroBat | bios/ |
| RetroDECK | ~/retrodeck/ (the pack carries its own bios/) |
| EmuDeck | ~/Emulation/bios/ |
| ROCKNIX | /storage/roms/bios/ |
| MiSTer FPGA | /media/fat/games/ |
| BizHawk | Firmware/ |
| RomM | bios/{platform_slug}/ |
### Changes
${CHANGES}
" \
--latest
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Cleanup old releases
if: steps.rate.outputs.skip != 'true'
run: |
gh release list --repo "${{ github.repository }}" --json tagName,createdAt \
--jq 'sort_by(.createdAt) | reverse | .[].tagName' | \
grep -v "^large-files$" | tail -n +4 | while read tag; do
gh release delete "$tag" --repo "${{ github.repository }}" --yes --cleanup-tag
done
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
README.md
+1 -2
View File
@@ -3,7 +3,6 @@
</p>
<p align="center">
<a href="https://github.com/Abdess/retrobios/actions/workflows/build.yml"><img src="https://github.com/Abdess/retrobios/actions/workflows/build.yml/badge.svg" alt="Build"></a>
<a href="https://github.com/Abdess/retrobios/actions/workflows/deploy-site.yml"><img src="https://github.com/Abdess/retrobios/actions/workflows/deploy-site.yml/badge.svg" alt="Site"></a>
</p>
@@ -166,4 +165,4 @@ The scripts and tooling are released under the [MIT License](LICENSE).
The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed.
The reasoning, and where it is weakest, is in the [FAQ](https://abdess.github.io/retrobios/wiki/faq/#is-this-legal).
*Auto-generated on 2026-09-03T21:10:11Z*
*Auto-generated on 2026-09-04T01:36:37Z*
scripts/generate_readme.py
-2
View File
@@ -261,8 +261,6 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
"</p>",
"",
'<p align="center">',
' <a href="https://github.com/Abdess/retrobios/actions/workflows/build.yml">'
'<img src="https://github.com/Abdess/retrobios/actions/workflows/build.yml/badge.svg" alt="Build"></a>',
' <a href="https://github.com/Abdess/retrobios/actions/workflows/deploy-site.yml">'
'<img src="https://github.com/Abdess/retrobios/actions/workflows/deploy-site.yml/badge.svg" alt="Site"></a>',
"</p>",
tests/test_audit_regressions.py
-6
View File
@@ -724,12 +724,6 @@ class CheckoutCompletenessRegressions(unittest.TestCase):
self.assertLess(self._index(steps, "restore_large_files.py"), generate)
self.assertLess(self._index(steps, "refresh_data_dirs.py"), generate)
def test_pack_build_completes_the_checkout_before_building(self):
steps = self._steps("build.yml", "release")
build = self._index(steps, "Build packs")
self.assertLess(self._index(steps, "restore_large_files.py"), build)
self.assertLess(self._index(steps, "refresh_data_dirs.py"), build)
def test_restore_matches_assets_by_content_not_by_name(self):
from scripts.restore_large_files import restore
wiki/architecture.md
+2 -3
View File
@@ -347,12 +347,11 @@ pattern and how to add a test.
| Workflow | File | Trigger | Role |
|----------|------|---------|------|
| Build & Release | `build.yml` | manual dispatch only | restore large files, build packs, create GitHub release |
| Deploy Site | `deploy-site.yml` | push to main (platforms, emulators, wiki, scripts) + manual | validate contracts, generate site, build with MkDocs, validate rendered HTML, deploy to Pages |
| PR Validation | `validate.yml` | pull request on bios/, platforms/, emulators/, schemas/, scripts/, tests/ | validate BIOS hashes, schema check, run the full test suite, auto-label PR |
The build workflow has no push trigger: a release is dispatched by hand. It
keeps a 7-day rate limit between releases and the 3 most recent tags. See the
Releases are not built in CI: the packs are generated and checked on the
maintainer's machine and uploaded with `gh`, see the
[release process](release-process.md).
## License
wiki/release-process.md
+31 -73
View File
@@ -5,7 +5,7 @@ are built, and how to run the process manually.
## CI workflows overview
The project uses 3 GitHub Actions workflows. All use only official GitHub
The project uses 2 GitHub Actions workflows. All use only official GitHub
actions (`actions/checkout`, `actions/setup-python`, `actions/upload-pages-artifact`,
`actions/deploy-pages`). No third-party actions.
@@ -13,49 +13,15 @@ Budget target: ~175 minutes/month on the GitHub free tier.
| Workflow | File | Trigger |
|----------|------|---------|
| Build & Release | `build.yml` | Manual dispatch only |
| Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, mkdocs.yml), manual |
| PR Validation | `validate.yml` | PR touching `bios/**`, `platforms/**` or `emulators/**` |
Upstream BIOS lists are not scraped on a schedule. A maintainer runs the
scrapers by hand (see [adding a scraper](adding-a-scraper.md)), reviews the
diff, and commits the refreshed platform YAML.
## build.yml - Build & Release
Releasing is deliberate. Pushing never cuts one: somebody decides the
collection is worth publishing and dispatches the workflow.
**Trigger.** `workflow_dispatch` only, with an optional `force_release` flag to
bypass the rate limit.
**Concurrency.** Group `build`, queued rather than cancelled: a run that is
already uploading assets must finish.
**Steps:**
1. Checkout, Python 3.12, install `pyyaml`
2. Run `test_e2e`
3. Rate limit check: skip if last release was less than 7 days ago (unless
`force_release` is set)
4. Restore large files from the `large-files` release into `.cache/large/`
5. Refresh data directories (`refresh_data_dirs.py`)
6. Build packs (`generate_pack.py --all --output-dir dist/`)
7. Split any pack over 2 GB into `.zip.001`, `.zip.002`, ... volumes. GitHub
caps release assets at 2 GB, and the `.001` convention is what 7-Zip and
PeaZip open directly (they reject `.partNN` names as corrupt)
8. Create GitHub release with tag `v{YYYY.MM.DD}` (appends `.N` suffix if
a same-day release already exists)
9. Clean up old releases, keeping the 3 most recent plus `large-files`
**Release notes** include file count, total size, per-pack sizes, the extract
path per platform, and the last 15 non-merge commits touching `bios/` or
`platforms/`.
**Pack variants.** The workflow builds full packs only. Releases that also
carry `*_Platform_BIOS_Pack.zip` had a second `--source platform` run added by
hand, as in the manual process below. See
[advanced usage](advanced-usage.md#pack-source-variants).
diff, and commits the refreshed platform YAML. Releases are built on the
maintainer's machine and uploaded with `gh`, see
[cutting a release](#cutting-a-release): the packs weigh 25 GB, more than a
hosted runner should rebuild and re-upload.
## deploy-site.yml - Deploy Documentation Site
@@ -163,52 +129,44 @@ gh release upload large-files "bios/Sony/PS3/PS3UPDAT.PUP#PS3UPDAT.PUP"
**Local cache.** `generate_pack.py` calls `fetch_large_file()` which downloads
from the release and caches in `.cache/large/` for subsequent runs.
## Manual release process
## Cutting a release
When `build.yml` is disabled, build and release manually:
Releasing is deliberate and local. Nothing on GitHub builds a pack: the
pipeline runs here, the archives are checked here, and `gh` uploads them.
```bash
# Run the full pipeline (DB + verify + packs + manifests + integrity + docs)
# 1. Full pipeline, online, so data directories and MAME/FBNeo hashes are fresh
python scripts/pipeline.py
# Or step by step:
python scripts/generate_db.py --force --bios-dir bios --output database.json
python scripts/verify.py --all
python scripts/generate_pack.py --all --output-dir dist/ # full packs
python scripts/generate_pack.py --all --source platform --output-dir dist/ # platform packs
# 2. The platform-only packs, and RetroPie, which is archived but still served
python scripts/generate_pack.py --all --source platform --output-dir dist/
python scripts/generate_pack.py --platform retropie --output-dir dist/
python scripts/generate_pack.py --platform retropie --source platform --output-dir dist/
# 3. The standalone emulators that no frontend bundles
python scripts/generate_pack.py --emulator mesence --output-dir dist/
python scripts/generate_pack.py --emulator lexaloffle --output-dir dist/
# 4. Every archive in dist/ extracted and hashed the way its platform checks it
python scripts/generate_pack.py --all --verify-packs --output-dir dist/
# Split anything over 2 GB (GitHub asset cap)
# 5. Split anything over 2 GB (GitHub asset cap); 7-Zip and PeaZip open .001 directly
for f in dist/*.zip; do
[ "$(stat -c%s "$f")" -gt 2000000000 ] || continue
split --bytes=1900M --numeric-suffixes=1 --suffix-length=3 "$f" "$f." && rm "$f"
done
# Create the release
# 6. Create the release, then keep the three most recent plus large-files
DATE=$(date +%Y.%m.%d)
gh release create "v${DATE}" dist/*.zip* \
--title "BIOS Pack v${DATE}" \
--notes "Release notes here" \
--latest
gh release create "v${DATE}" dist/*.zip* dist/SHA256SUMS.txt \
--title "BIOS Pack v${DATE}" --notes-file notes.md --latest
gh release list --json tagName,createdAt \
--jq 'sort_by(.createdAt) | reverse | .[].tagName' | grep -v '^large-files$' \
| tail -n +4 | while read tag; do gh release delete "$tag" --yes --cleanup-tag; done
```
The two `generate_pack.py` runs are what puts both `*_BIOS_Pack.zip` and
`*_Platform_BIOS_Pack.zip` on the release. `--all-variants` builds all six
combinations instead, which is more than a release needs.
Run the pipeline online for a release: `--offline` skips the data directory
refresh and the MAME/FBNeo hash refresh, so the packs would ship stale data
directories.
The workflow carries no push trigger, so there is nothing to disable and no
guard to remove. Cutting a release means dispatching `build.yml` from the
Actions tab, or:
```bash
gh workflow run build.yml
gh workflow run build.yml -f force_release=true # within 7 days of the last
```
The rate limit refuses a second release inside seven days unless
`force_release` is set, which is what keeps a stray dispatch from publishing
twice in a day.
The release notes follow the previous release: the quick install commands, the
full and platform pack tables with file counts and sizes, the standalone
packs, what changed since the previous tag, and the contributors of the
closed issues. `SHA256SUMS.txt` lists the checksums of the full ZIPs before
splitting.