diff --git a/scripts/dedup.py b/scripts/dedup.py index 89dd4d3f..9e8e776b 100644 --- a/scripts/dedup.py +++ b/scripts/dedup.py @@ -1,4 +1,4 @@ -"""Deduplicate bios/ directory -keep one canonical file per unique content. +"""Deduplicate bios/: keep one canonical file per unique content. Usage: python scripts/dedup.py [--dry-run] [--bios-dir bios] @@ -11,7 +11,7 @@ Two types of deduplication: 2. MAME DEVICE CLONES: Different filenames with identical content in the same MAME directory (e.g., bbc_m87.zip and bbc_24bbc.zip are identical ZIPs). - These are NOT aliases -MAME loads each by its unique name. Instead of + These are NOT aliases: MAME loads each by its unique name. Instead of deleting, we create a _mame_clones.json mapping so generate_pack.py can pack all names from a single canonical file. diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 0eab84b9..7a315fd7 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -3465,5 +3465,35 @@ class AReaderThatLeavesEarlyIsNoCrash(unittest.TestCase): self.assertEqual(found, []) + +class TheToolsPageSaysWhatDedupDoes(unittest.TestCase): + """tools.md said duplicates move to .variants/ (they are deleted) and + named 2 of the 11 directories dedup leaves alone.""" + + def test_every_protected_tree_is_named(self): + from scripts import dedup + + line = next( + l for l in (ROOT / "wiki" / "tools.md").read_text().splitlines() + if l.startswith("| `dedup.py`") + ) + self.assertNotIn(".variants", line) + for name in dedup.NODEDUP_DIRS: + self.assertIn(name, line) + + + +class ThePipelineTableNamesEveryStep(unittest.TestCase): + """tools.md listed the steps without 1c, 2b2, 3a and 3b.""" + + def test_every_labelled_step_is_documented(self): + labels = set(re.findall( + r'"(\d[a-z]?\d?)(?:/8)? [a-z]', (ROOT / "scripts" / "pipeline.py").read_text() + )) + doc = (ROOT / "wiki" / "tools.md").read_text() + documented = set(re.findall(r"^\| (\d[a-z]?\d?)(?:/8)? \|", doc, re.MULTILINE)) + self.assertEqual(labels - documented, set()) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/release-process.md b/wiki/release-process.md index e7c90298..06143210 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -14,7 +14,7 @@ Budget target: ~175 minutes/month on the GitHub free tier. | Workflow | File | Trigger | |----------|------|---------| | Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, mkdocs.yml), manual | -| Validation | `validate.yml` | PR and push to main touching `bios/**`, `platforms/**` or `emulators/**` | +| Validation | `validate.yml` | PR and push to main touching `bios/**`, `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh` or `install.ps1` | Upstream BIOS lists are not scraped on a schedule. A maintainer runs the scrapers by hand (see [adding a scraper](adding-a-scraper.md)), reviews the @@ -82,9 +82,11 @@ so a major theme release cannot change the site without a deliberate bump. ## validate.yml - Validation **Trigger.** Pull requests and direct pushes to main that modify `bios/**`, -`platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**` or -`install.py`. The path lists are spelled out once per event because the -workflow parser reads no YAML anchor. +`platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, +`install.py`, `install.sh` or `install.ps1` (the bootstraps pin the +installer's SHA-256, so editing one without the other fails the run). The +path lists are spelled out once per event because the workflow parser reads +no YAML anchor. **Concurrency.** Per-PR group on a pull request, per-ref on a push, cancel in-progress either way: a push series collapses to the tip. @@ -124,13 +126,16 @@ data bundle, the EasyRPG soundfont, the Dolphin/Ishiiruka SD card images, and the arcade sets over 100 MB. `.gitignore` is the authoritative list: every `bios/` path listed there is a release asset. -**Storage.** Listed in `.gitignore` so they stay out of git history. The -`large-files` release is excluded from cleanup (the build workflow only -deletes version-tagged releases). +**Storage.** Listed in `.gitignore` so they stay out of git history. Old +version-tagged releases are deleted by hand when a new one is published +(see [cutting a release](#cutting-a-release)); `large-files` is never one of +them. -**Build-time restore.** The build workflow downloads all assets from -`large-files` into `.cache/large/` and copies them to their expected paths -before pack generation. +**Restore.** No workflow builds a pack. `generate_pack.py` fetches an asset +into `.cache/large/` when a pack needs it (`fetch_large_file()`), and +`scripts/restore_large_files.py` puts every registered asset back at its +`bios/` path by the SHA-1 of its content; `deploy-site.yml` runs it before +generating anything, since the site's coverage reads the disk. **Asset name.** A gitignored path whose file name no other gitignored path shares is published under that file name. When two paths share one, each is diff --git a/wiki/tools.md b/wiki/tools.md index beddd449..0a556e69 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -25,14 +25,18 @@ Pipeline steps: |------|-------------|------------| | 1/8 | Generate database | - | | 1b | Dump-catalog coverage report | - (offline, reads `provenance/`) | +| 1c | Romset recipes: identify archives, list rebuild targets | - (offline, reads `recipes/`) | | 2/8 | Refresh data directories | `--offline` | | 2a | Refresh MAME BIOS hashes | `--offline` | | 2a2 | Refresh FBNeo BIOS hashes | `--offline` | | 2b | Check buildbot staleness | only with `--check-buildbot`, skipped by `--offline` | +| 2b2 | Release assets against the gitignored files (size and sha256) | `--offline` | | 2c | Generate truth YAMLs | only with `--with-truth` / `--with-export` | | 2d | Diff truth vs scraped | only with `--with-truth` / `--with-export` | | 2e | Export native formats | only with `--with-export` | -| 3/8 | Verify all platforms | - | +| 3/8 | Verify all platforms (`--strict`) | - | +| 3a | File identity: one content under two identities (`identity.py --strict`, gates the run) | - | +| 3b | Slot arbitration: destinations both layers claim (`slots.py`, reports only) | - | | 4/8 | Generate packs | `--skip-packs` | | 4b | Generate install manifests | `--skip-packs` | | 4c | Generate target manifests | `--skip-packs` | @@ -45,13 +49,13 @@ Pipeline flags: | Flag | Effect | |------|--------| -| `--offline` | skip every step that needs the network (2, 2a, 2a2, 2b) | +| `--offline` | skip every step that needs the network (2, 2a, 2a2, 2b, 2b2) | | `--skip-packs` | skip steps 4, 4b, 4c, 6 and the consistency check | | `--skip-docs` | skip steps 7 and 8 | | `--include-archived` | include archived platforms in verify, packs, truth and export | | `--target TARGET` | filter verify, packs and truth by hardware target | -| `--source {platform,truth,full}` | pack file source, passed through to `generate_pack.py` | -| `--all-variants` | build the 6 source x required combinations | +| `--source {platform,truth,full}` | pack file source, passed through to `generate_pack.py`; refused with `--skip-packs` | +| `--all-variants` | build the 6 source x required combinations; refused with `--skip-packs` and with `--source` | | `--check-buildbot` | run step 2b; additive, and ignored under `--offline` | | `--with-truth` | run steps 2c and 2d | | `--with-export` | run steps 2c, 2d and 2e | @@ -536,7 +540,9 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6 | Script | Purpose | |--------|---------| | `common.py` | Shared library: hash computation, file resolution, platform config loading, emulator profiles, target filtering | -| `dedup.py` | Deduplicate `bios/` (`--dry-run`, `--bios-dir`), move duplicates to `.variants/`. RPG Maker and ScummVM excluded (NODEDUP) | +| `dedup.py` | Deduplicate `bios/` (`--dry-run`, `--bios-dir`): byte-identical copies are deleted and one kept; identical MAME sets under other names stay one file, named in `_mame_clones.json`. Trees addressed by path are left alone (`NODEDUP_DIRS`): BBK, C-Dogs SDL, Ikemen GO, PinMAME, RPG Maker, ScummVM, Softdisk, TheXTech, Theme Hospital, c64-emu, tzdata | +| `identity.py` | One content filed under two identities: groups `bios/` by content and reports a group that crosses manufacturers outside `KNOWN_SHARED` (`--strict` exits 1, pipeline step 3a) | +| `slots.py` | Destinations the platform and a core both claim, and how the pack settles each (`--platform`, `--json`, `--strict`; pipeline step 3b reports without gating) | | `validate_pr.py` | Validate BIOS files in pull requests, post markdown report | | `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs | | `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline) |