libretro/automation/jobs.yml

# Jobs scripts/autorun.py runs. A change here is a reviewed commit: the
# schedule, the gate and the paths a job may commit are all decided here.
#
# Commands run in the automation worktree (.cache/autorun/worktree) unless
# `worktree: false`, which runs them in the main checkout and forbids
# commits. `python` names the interpreter running autorun.py. A step is a
# command list, or a mapping with `run`, `ok_codes`, `stdout` (a file the
# command's output is written to) and `env` (variables passed through
# beside PATH, HOME, LANG, LC_*, TZ and TMPDIR). A step never sees a password.
# GITHUB_TOKEN goes to the push, to gh, and to a step marked `github_token:
# true`, which autorun accepts only on check_freshness.py, refresh_stale.py and
# `-m scripts.scraper.*`, tracked here, never in the campaign job. `twice`
# recomputes an artifact the gate wrote into `{out}` and compares both,
# timestamps aside. Each commit
# takes only the changed files its `paths` globs match, and a job that
# leaves any other change behind fails. `report` names a JSON file a step
# writes, kept in the job's status.
#
# Schedules are systemd OnCalendar expressions, written into user timers by
# `python scripts/autorun.py install-timers`.

defaults:
  min_free_gb: 30
  max_load_wait: 3600
  timeout: 21600

jobs:
  cleanup:
    schedule: "*-*-* 05:30"
    worktree: false
    min_free_gb: 0
    steps:
      - [python, scripts/cleanup_disk.py, --apply, --hours, "72"]

  disk-check:
    schedule: hourly
    worktree: false
    lock: false
    min_free_gb: 0
    max_load_wait: 0
    steps:
      - [python, scripts/cleanup_disk.py, --check]

  nightly:
    schedule: "*-*-* 02:00"
    network: true
    min_free_gb: 120
    steps:
      - run: [python, scripts/check_freshness.py, --json]
        ok_codes: [0, 1]
        stdout: tmp/autorun/freshness.json
        github_token: true
      - run: [python, scripts/refresh_stale.py, --findings, tmp/autorun/freshness.json,
              --only, "data,catalogs"]
        github_token: true
      - run: [python, -m, scripts.scraper.mame_hash_scraper]
        github_token: true
      - run: [python, -m, scripts.scraper.fbneo_hash_scraper]
        github_token: true
    gate:
      - [python, scripts/pipeline.py, --offline]
    twice:
      - artifact: database.json
        run: [python, scripts/generate_db.py, --force, --bios-dir, bios, --output, "{out}"]
      - artifact: install
        glob: "*.json"
        run: [python, scripts/generate_pack.py, --all, --manifest, --offline,
              --output-dir, "{out}"]
    commits:
      - subject: "chore: refresh mame and fbneo hashes"
        body: >-
          Written by mame_hash_scraper and fbneo_hash_scraper from the
          upstream sources, behind a green offline pipeline.
        paths: [emulators/mame*.yml, emulators/fbneo*.yml]
      - subject: "chore: refresh romset recipes"
        body: >-
          Snapshots refreshed by refresh_stale from the MAME and FBNeo DATs
          check_freshness reported stale, behind a green offline pipeline.
        paths: [recipes/*.json]
      - subject: "chore: regenerate collection records"
        body: >-
          Regenerated by the offline pipeline, database.json and the
          manifests computed twice with the same result.
        paths: [database.json, install/*.json, install/targets/*.json, README.md,
                CONTRIBUTING.md, website/src/data/*.json]
    scratch: [dist, tmp/autorun]

  bios-integrity:
    schedule: "Sun *-*-* 08:30"
    worktree: false
    max_load_wait: 14400
    steps:
      - run: [python, scripts/bios_integrity.py, --apply, --json]
        stdout: tmp/autorun/bios-integrity.json
    report: tmp/autorun/bios-integrity.json

  campaign:
    schedule: "*-*-* 06:00"
    network: true
    max_load_wait: 7200
    timeout: 12600
    steps:
      - run: [python, scripts/campaign.py, run, --budget, 150m]
        ok_codes: [0, 75]
        stdout: tmp/autorun/campaign.json
        env: [RETROBIOS_ROMFETCH, RETROBIOS_AUTO_LOCK, ROMFETCH_CONTENT_DIR, ROMFETCH_CACHE_DIR,
              ROMFETCH_DATS_DIR, ROMFETCH_RETOOL_DIR, ROMFETCH_MIN_INTERVAL, ROMFETCH_MAX_SIZE,
              ROMFETCH_DISABLED_SOURCES, XDG_CONFIG_HOME]
    gate:
      - [python, scripts/campaign.py, gate]
    commits:
      - subject: "chore: add campaign measurements"
        body: >-
          Core probe records, findings and the pass summary written by
          campaign.py, behind the measurement tests and the schemas.
        paths: [measurements/*.jsonl]
      - subject: "chore: apply fields the campaign proved"
        body: >-
          Fields the campaign's rule table settled from real runs, each with
          the record ids in evidence:, behind verify and an emulator pack of
          every changed profile.
        paths: [emulators/*.yml]
    report: tmp/autorun/campaign.json
    scratch: [tmp/autorun, tmp/core_probe]

  assets:
    schedule: "*-*-* 04:30"
    network: true
    steps:
      - [python, scripts/sync_release_assets.py]