From d66e8910514cb64dac4cb7e3bf38a4f75ad72151 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:34:22 +0200 Subject: [PATCH] chore: retire the weekly platform sync --- .github/workflows/watch.yml | 137 ------------------------------------ wiki/adding-a-platform.md | 4 +- wiki/architecture.md | 1 - wiki/faq.md | 2 +- wiki/release-process.md | 24 ++----- 5 files changed, 8 insertions(+), 160 deletions(-) delete mode 100644 .github/workflows/watch.yml diff --git a/.github/workflows/watch.yml b/.github/workflows/watch.yml deleted file mode 100644 index 70d7033e..00000000 --- a/.github/workflows/watch.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: Weekly Platform Sync - -on: - schedule: - - cron: "0 6 * * 1" # Every Monday at 06:00 UTC - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -concurrency: - group: watch-updates - cancel-in-progress: true - -jobs: - scrape-and-update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.12" - - - name: Install dependencies - run: pip install pyyaml - - - name: Regenerate platform configs from stable upstream sources - run: | - python3 -c " - import sys, yaml - sys.path.insert(0, 'scripts') - - def str_representer(dumper, data): - if any(c in data for c in '()[]{}:#'): - return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='\"') - return dumper.represent_scalar('tag:yaml.org,2002:str', data) - yaml.add_representer(str, str_representer) - - targets = { - 'libretro': 'retroarch', 'batocera': 'batocera', - 'recalbox': 'recalbox', 'retrobat': 'retrobat', - 'romm': 'romm', 'bizhawk': 'bizhawk', - 'retrodeck': 'retrodeck', 'emudeck': 'emudeck', - 'rocknix': 'rocknix', 'misterfpga': 'misterfpga', - } - for scraper_name, out in targets.items(): - mod = __import__(f'scraper.{scraper_name}_scraper', fromlist=['Scraper']) - config = mod.Scraper().generate_platform_yaml() - with open(f'platforms/{out}.yml', 'w') as f: - yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False) - print(f'{out}: {len(config[\"systems\"])} systems, version={config[\"version\"]}') - " - - - name: Refresh hardware target configs - run: | - python -m scripts.scraper.targets.batocera_targets_scraper -o platforms/targets/batocera.yml - python -m scripts.scraper.targets.retroarch_targets_scraper -o platforms/targets/retroarch.yml - python -m scripts.scraper.targets.emudeck_targets_scraper -o platforms/targets/emudeck.yml - python -m scripts.scraper.targets.retropie_targets_scraper -o platforms/targets/retropie.yml - continue-on-error: true - - - name: Auto-fetch missing BIOS files - run: | - python scripts/generate_db.py --bios-dir bios --output database.json - python scripts/auto_fetch.py --all 2>&1 | tee /tmp/fetch_report.txt || true - - - name: Refresh data directories - run: python scripts/refresh_data_dirs.py --force - - - name: Deduplicate BIOS files - run: python scripts/dedup.py - - - name: Regenerate database and documentation - run: | - python scripts/generate_db.py --force --bios-dir bios --output database.json - python scripts/generate_readme.py --db database.json --platforms-dir platforms - - - name: Check for changes - id: check - run: | - if git diff --quiet && [ -z "$(git ls-files --others --exclude-standard)" ]; then - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "has_changes=true" >> "$GITHUB_OUTPUT" - echo "Changes detected:" - git diff --stat - git ls-files --others --exclude-standard | head -20 - fi - - - name: Create or update PR - if: steps.check.outputs.has_changes == 'true' - run: | - DATE=$(date +%Y-%m-%d) - BRANCH="auto/update-platforms-${DATE}" - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - git checkout -b "$BRANCH" - git add bios/ platforms/ database.json README.md CONTRIBUTING.md - git commit -m "chore: auto-update platform data ($DATE)" - git push origin "$BRANCH" --force-with-lease - - existing_pr=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || true) - - if [ -n "$existing_pr" ] && [ "$existing_pr" != "null" ]; then - echo "Updating existing PR #${existing_pr}" - else - gh pr create \ - --title "Auto-update: Platform data ($DATE)" \ - --body "$(cat <<'EOF' - ## Automated Platform Update - - This PR was automatically generated by the weekly platform watch. - - ### What changed - - Scraped latest BIOS requirements from libretro System.dat and batocera-systems - - Auto-fetched new BIOS files from public sources - - Deduplicated and regenerated database + documentation - - ### Review Checklist - - [ ] New BIOS files are valid - - [ ] Platform configs are correct - - [ ] database.json and README look good - - --- - *Auto-generated by [watch-updates](../../actions/workflows/watch-updates.yml)* - EOF - )" \ - --label "automated,platform-update" - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index 43f60146..c108ed91 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -130,7 +130,7 @@ platforms: config: myplatform.yml # platform YAML filename in platforms/ status: active # active or archived scraper: myplatform # matches PLATFORM_NAME, null if hand-maintained - schedule: weekly # weekly, monthly, or null for no auto-scrape + schedule: weekly # re-scrape cadence, run by hand: weekly, monthly, or null source_url: https://... # upstream data URL source_format: json # json, xml, clrmamepro_dat, python_dict, # bash_script+csv, csharp_firmware_database, @@ -371,7 +371,7 @@ This executes in sequence: 8. `generate_site.py` - regenerate documentation site and `mkdocs.yml` A new platform is archived-by-default nowhere: `status: active` puts it in -`list_platforms.py`, the CI matrix, and the weekly scrape. Set +`list_platforms.py`, the CI matrix, and the scrapers. Set `status: archived` to keep the configuration without the automation; archived platforms are excluded unless `--all` or `--include-archived` is passed. diff --git a/wiki/architecture.md b/wiki/architecture.md index 0d78dc3c..22038c05 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -350,7 +350,6 @@ pattern and how to add a test. | Build & Release | `build.yml` | manual dispatch only | restore large files, build packs, create GitHub release | | Deploy Site | `deploy-site.yml` | push to main (platforms, emulators, wiki, scripts) + manual | validate contracts, generate site, build with MkDocs, validate rendered HTML, deploy to Pages | | PR Validation | `validate.yml` | pull request on bios/, platforms/, emulators/, schemas/, scripts/, tests/ | validate BIOS hashes, schema check, run the full test suite, auto-label PR | -| Weekly Sync | `watch.yml` | cron (Monday 6 AM UTC) + manual | scrape upstream sources, detect changes, create update PR | The build workflow has no push trigger: a release is dispatched by hand. It keeps a 7-day rate limit between releases and the 3 most recent tags. See the diff --git a/wiki/faq.md b/wiki/faq.md index ebb6417b..c95576af 100644 --- a/wiki/faq.md +++ b/wiki/faq.md @@ -174,4 +174,4 @@ The packs differ per platform because each platform declares its own set of supp ## How often are packs updated? -A weekly automated sync checks upstream sources (libretro System.dat, batocera-systems, etc.) for changes. If differences are found, a pull request is created automatically. Manual releases happen as needed when new BIOS files are added or profiles are updated. +Upstream sources (libretro System.dat, batocera-systems, etc.) are re-scraped by hand and the diff is reviewed before it lands. Releases happen as needed when new BIOS files are added or profiles are updated. diff --git a/wiki/release-process.md b/wiki/release-process.md index fc00a189..211f980e 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -5,7 +5,7 @@ are built, and how to run the process manually. ## CI workflows overview -The project uses 4 GitHub Actions workflows. All use only official GitHub +The project uses 3 GitHub Actions workflows. All use only official GitHub actions (`actions/checkout`, `actions/setup-python`, `actions/upload-pages-artifact`, `actions/deploy-pages`). No third-party actions. @@ -16,7 +16,10 @@ Budget target: ~175 minutes/month on the GitHub free tier. | Build & Release | `build.yml` | Manual dispatch only | | Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, mkdocs.yml), manual | | PR Validation | `validate.yml` | PR touching `bios/**`, `platforms/**` or `emulators/**` | -| Weekly Sync | `watch.yml` | Cron Monday 06:00 UTC, manual dispatch | + +Upstream BIOS lists are not scraped on a schedule. A maintainer runs the +scrapers by hand (see [adding a scraper](adding-a-scraper.md)), reviews the +diff, and commits the refreshed platform YAML. ## build.yml - Build & Release @@ -132,23 +135,6 @@ merge. | `platforms/` | `platform-config` | | `scripts/` | `automation` | -## watch.yml - Weekly Platform Sync - -**Trigger.** Cron schedule every Monday at 06:00 UTC, or manual dispatch. - -**Flow:** - -1. Scrape live upstream sources (System.dat, batocera-systems, es_bios.xml, - etc.) and regenerate platform YAML configs -2. Auto-fetch missing BIOS files -3. Refresh data directories -4. Run dedup -5. Regenerate `database.json` -6. Create or update a PR with labels `automated` and `platform-update` - -The PR contains all changes from the scrape cycle. A maintainer reviews and -merges. - ## Large files management Files larger than 50 MB are stored as assets on a permanent GitHub release