diff --git a/tests/test_doc_surfaces.py b/tests/test_doc_surfaces.py index 6431b3afd..4acf3b158 100644 --- a/tests/test_doc_surfaces.py +++ b/tests/test_doc_surfaces.py @@ -299,7 +299,19 @@ class ShownOutputIsPrintedOutput(unittest.TestCase): (r"^\S.*: \d+/\d+ (OK|present)\b.*\[(md5|sha1|existence)\]$", "}: {', '.join(parts)} [{mode}]"), (r"^ (OK|UNTESTED|MISSING) \((required|optional)[^)]*\): \S+/\S+", '" UNTESTED ({req}{hle}): {key}: {reason}"'), (r"^ DISCREPANCY: \S+/\S+: ", '" DISCREPANCY: {key}: {disc}"'), - (r"^ Core files: \d+ in pack, \d+ required missing, \d+ optional missing$", '" Core files: {core_in_pack} in pack'), + # A count of optional files missing is followed by where to list them: + # the sample once showed "1 optional missing" with no such suffix. + (r"^ Core files: \d+ in pack, \d+ required missing, " + r"(0 optional missing|\d+ optional missing \(--verbose lists them\))$", + '" Core files: {core_in_pack} in pack'), + (r"^ Core files held but not packed: \d+ \(.+\)$", '" Core files held but not packed: '), + (r"^ Game data: \d+ in pack, (0 missing|\d+ missing \(--verbose lists them\))$", + '" Game data: {gd_packed} in pack'), + (r"^ No external files: \d+ cores \(--verbose lists them\)$", + '" No external files: {len(exclusions)} cores (--verbose lists them)"'), + (r"^ Ground truth: \d+/\d+ files have emulator validation \(\d+%\)$", + '" Ground truth: {gt_cov[\'with_validation\']}/{gt_cov[\'total\']} files have emulator validation'), + (r"^ \d+ platform-only \(no emulator profile\)$", '" {no_profile} platform-only (no emulator profile)"'), (r"^ \S", None), ) @@ -321,6 +333,10 @@ class ShownOutputIsPrintedOutput(unittest.TestCase): any(re.match(pattern, line) for pattern, _ in self.SHAPES), line ) + def test_an_optional_count_without_its_suffix_is_refused(self): + line = " Core files: 1495 in pack, 0 required missing, 1 optional missing" + self.assertFalse(any(re.match(pattern, line) for pattern, _ in self.SHAPES)) + def test_no_severity_column(self): self.assertNotIn("[CRITICAL]", _read("wiki/troubleshooting.md")) diff --git a/wiki/measurements.md b/wiki/measurements.md index 4c48a41cc..9ed594b98 100644 --- a/wiki/measurements.md +++ b/wiki/measurements.md @@ -16,7 +16,8 @@ A page that relies on a run says so in a **Proof** line. Every run is on the ## What a test does - A script runs the emulator with no window and no sound. -- It starts a game and runs it for a set time, at least twice. +- It starts a game and runs it for a set time. +- A test that compares files runs each case at least twice. - It compares the screens with ones we checked by eye. - It reads the emulator's log to see which BIOS it ran. diff --git a/wiki/reading-reports.md b/wiki/reading-reports.md index 8f2020023..159fef7b7 100644 --- a/wiki/reading-reports.md +++ b/wiki/reading-reports.md @@ -107,9 +107,18 @@ RetroDECK: 2007/2008 OK, 1 untested [md5] UNTESTED (required): nintendo-ds/dsifirmware.bin: declared hash contradicted by the local file DISCREPANCY: nintendo-snes/sgb2.boot.rom: RetroDECK says OK but supersnes9x says size mismatch: got 524288, accepted [256] supersnes9x [size] - Core files: 1495 in pack, 0 required missing, 1 optional missing + Core files: 1795 in pack, 0 required missing, 27 optional missing (--verbose lists them) + Core files held but not packed: 154 (installed with its emulator) + Game data: 5099 in pack, 38 missing (--verbose lists them) + No external files: 78 cores (--verbose lists them) + Ground truth: 96/2008 files have emulator validation (5%) + 439 platform-only (no emulator profile) ``` +The run prints more `DISCREPANCY` lines than this excerpt shows. The core +and game data lines count what the emulator cores add beyond the platform's +own list. A file held but not packed is left out for the reason in parentheses. + `--verbose` also lists the passing entries that an emulator validates, with the check and the source lines behind it: @@ -121,10 +130,10 @@ Recalbox: 351/351 OK [md5] ### Cross-reference section -After per-file results, `verify.py` prints a cross-reference report. This lists -files that emulator cores need but that the platform YAML doesn't declare. These -files are still included in packs automatically, but the report helps identify -gaps in platform coverage data. +The `Core files` and `Game data` lines are the cross-reference: files that +emulator cores need but that the platform YAML does not declare. These files +are still included in packs automatically. A required one the collection lacks +is listed under the counts, and `--verbose` lists the optional ones too. The cross-reference uses `resolve_platform_cores()` to determine which emulator profiles are relevant for each platform, then checks whether each profile's files diff --git a/wiki/release-process.md b/wiki/release-process.md index 7ccf74e0c..9e7746532 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -10,16 +10,20 @@ built, checked and uploaded from the maintainer's machine. ## CI workflows overview -Both workflows use only official GitHub -actions (`actions/checkout`, `actions/setup-python`, `actions/upload-pages-artifact`, -`actions/deploy-pages`). No third-party actions. +Both workflows use only official GitHub actions (`actions/checkout`, +`actions/setup-python`, `actions/setup-node`, `actions/configure-pages`, +`actions/upload-pages-artifact`, `actions/deploy-pages`), each pinned to a +commit. No third-party actions. Budget target: ~175 minutes/month on the GitHub free tier. | Workflow | File | Trigger | |----------|------|---------| -| Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, website), manual | -| Validation | `validate.yml` | PR and push to main touching `bios/**`, `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh`, `install.ps1`, or any other input of the site build (`wiki/**`, `docs_assets/**`, `provenance/**`, `exports/**`, `install/**`, `database.json`, `release.json`, `derivable.json`, `website/**`) | +| Deploy Site | `deploy-site.yml` | Push to main touching an input of the site, manual | +| Validation | `validate.yml` | PR and push to main touching the collection, the code, the tests, the installer, `ports/` or an input of the site | + +Each workflow's `paths:` lists are the reference, and +`tests/test_workflow_paths.py` holds them to what the site build reads. Upstream BIOS lists are not scraped on a schedule. A maintainer runs the scrapers by hand (see [adding a scraper](adding-a-scraper.md)), reviews the @@ -30,12 +34,13 @@ hosted runner should rebuild and re-upload. ## deploy-site.yml - Deploy documentation site -**Trigger.** Push to `main` when any of these paths change: `platforms/`, -`emulators/`, `provenance/`, `wiki/`, `schemas/`, `tests/`, `docs_assets/`, -`install/`, `install.py`, `database.json`, `release.json`, `website/` (the -Starlight project: configuration, components, styles, lockfile), the -workflow itself, and every script the build runs or imports. Also manual -dispatch. +**Trigger.** Push to `main` when an input of the site changes: the data +(`platforms/`, `emulators/`, `provenance/`, `exports/`, `measurements/`, +`database.json`, `release.json`, `derivable.json`, `install/`), the pages and +their assets (`wiki/`, `docs_assets/`, `website/`), the installer and its two +bootstraps the site serves, `tests/campaign/observations.yml` (the Proof lines +of the emulator pages), the workflow itself, and every script the build runs +or imports. Also manual dispatch. The list is the set of inputs the site is generated from. A script that `generate_site.py`, `generate_readme.py` or another build step starts to import @@ -61,11 +66,17 @@ is missing. 8. Run `validate_site.py --site-dir website/dist` on the rendered HTML (metadata, headings, image alternatives, duplicate ids, every local link and fragment again, page and search index budgets) -9. Require the committed README and CONTRIBUTING to match what the generator +9. Run `tests.test_site_bootstrap`: the one-line installers the site serves + pin their own `install.py`, and a copy that refuses every install stops + the deploy, the last good pair staying served +10. Run `website/tools/search_gate.py`: the queries a visitor types must reach + the page that answers them through the site's own search box, and the home + page must stay under its weight budget +11. Require the committed README and CONTRIBUTING to match what the generator just produced. `write_if_changed()` compares content with the timestamp line stripped, so a run that only moves the clock leaves the files untouched and the check stays meaningful -10. Upload artifact, deploy to GitHub Pages +12. Upload artifact, deploy to GitHub Pages Data contracts are validated with `scripts/validate_schemas.py` at step 3, before the site is generated. It refuses to run without the checkers for the `date-time` @@ -98,18 +109,24 @@ scripts stay Python. **Trigger.** Pull requests and direct pushes to main that modify `bios/**`, `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh` or `install.ps1` (the bootstraps pin the -installer's SHA-256, so editing one without the other fails the run). The -path lists are spelled out once per event because the workflow parser reads -no YAML anchor. +installer's SHA-256, so editing one without the other fails the run), +`ports/**` (the handheld port bundles the installer), or any input of the site +build. The path lists are spelled out once per event because the workflow +parser reads no YAML anchor, and a test holds the two equal. -**Concurrency.** Per-PR group on a pull request, per-ref on a push, cancel -in-progress either way: a push series collapses to the tip. +**Concurrency.** Per-PR group on a pull request, per-ref on a push. A pull +request cancels its run in progress. A push to main does not: GitHub keeps one +pending run per group and replaces it with each newer push, so main always gets +a finished run and then its tip. Five jobs, two of which read pull request context and carry an event guard: -**validate-bios** (pull requests only). Diffs the PR to find changed BIOS -files, runs `validate_pr.py --markdown` on each, and posts the validation -report as a PR comment (hash verification, database match status). +**validate-bios** (pull requests only). Diffs the PR to find added, +modified or renamed BIOS files, runs `validate_pr.py --markdown` on them and +writes the report (hash verification, database match status) to the job +summary. The report is also posted as a PR comment when the branch lives in +this repository, since a fork's token cannot write one. The job fails after +the report when a file did not validate. **validate-configs.** Runs `python scripts/validate_schemas.py --source-only`, which validates every platform YAML against `schemas/platform.schema.json` and @@ -125,12 +142,19 @@ A profile written from a clone can cite lines that do not hold the value it declares while schemas and tests pass it. This job confronts it with its upstream. An edited profile is not held to it. -**run-tests.** Runs `python -m unittest discover -s tests -t . -v`, then builds the -site as Deploy Site does (`generate_site.py`, `npm run build --prefix website`, -`validate_site.py`) whatever the suite returned. Must pass before a merge, and -again on the commit a direct push puts at the head of main. +**run-tests.** On Python 3.8, 3.9 and 3.12: holds `install.py` to 3.8 with +`vermin`, sets up bubblewrap for the sandbox tests, fetches the platforms' own +files (allowed to fail, the fidelity tests then skip) and the data +directories, generates the site, then runs +`python -m unittest discover -s tests -t . -v` with `RETROBIOS_OFFLINE=1`, the +full-screen installer tests again under 3.8, and the site build +(`npm run build --prefix website`, `validate_site.py`). The site generation, +the suite and the build each run whatever the step before returned, so one +red step does not hide another. Must pass before a merge, and again on the +commit a direct push puts at the head of main. -**label-pr** (pull requests only). Auto-labels the PR based on changed paths: +**label-pr** (pull requests from a branch of this repository). Auto-labels the +PR based on changed paths: | Path pattern | Label | |-------------|-------| @@ -287,8 +311,9 @@ gh release list --json tagName,createdAt \ | tail -n +2 | while read tag; do gh release delete "$tag" --yes --cleanup-tag; done ``` -Before a release, since 2026-10-28, a maintainer runs `python scripts/pack_probe.py run ---platform ` on the platforms that get a pack, and reads its findings before cutting. +Before a release, a maintainer runs `python scripts/pack_probe.py run --platform +` on the released platforms it supports (RetroArch, Batocera and RetroBat), +and reads its findings before cutting. ## Which platforms get a pack { #which-platforms-get-a-pack } diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 9259c8ed8..48889d430 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -256,9 +256,10 @@ in the platform YAML: Handles inner ZIP verification for MAME/FBNeo ROM sets (checkInsideZip, md5_composite, inner ROM MD5) and path collision deduplication. -One test per distinct pack contract: RetroArch, Batocera, BizHawk, EmuDeck, -Recalbox, RetroBat, RetroDECK, RomM. Inherited aliases such as Lakka reuse the -same artifact and are not counted twice. +One test walks every platform the registry knows, each a subtest, rather than +a list written by hand: eight names were once written out, and ROCKNIX and +MiSTer FPGA, both released, were never checked. A platform with no pack in +`dist/` is skipped, and so is a pack older than a profile it depends on. ```bash python -m unittest tests.test_pack_integrity -v @@ -266,8 +267,8 @@ python -m unittest tests.test_pack_integrity -v python scripts/generate_pack.py --all --verify-packs --output-dir dist/ ``` -Integrated as pipeline step 6/8 (runs after consistency check, before -README generation). Requires packs in `dist/`, skip with `--skip-packs`. +The pipeline runs the same check as step 6, after the consistency check and +before the README. It needs packs in `dist/`, and `--skip-packs` skips it. ### test_handheld_launchers.py @@ -384,7 +385,7 @@ the door open. The Start here pages and the guide part of each setup page are read by someone who knows nothing about BIOS files, often on a phone. Each word costs that reader attention, so `tests/test_site_copy.py` holds them to numbers measured -on the generated pages in `docs/`, code blocks left out: +on the generated pages in `website/src/content/docs/`, code blocks left out: | Budget | Start here | Setup guide | |--------|-----------:|------------:| @@ -411,7 +412,8 @@ a pack is a ZIP and a platform is a setup. The reasons: a reader holds about four things at once, scans more than reads, and reads less on a phone. Facts a newcomer does not act on (legal analysis, source lines, annotated runs) are kept, on the page one level down. To see -the numbers of every page, run `python -m tests.test_site_copy --report docs`. +the numbers of every page, run +`python -m tests.test_site_copy --report website/src/content/docs`. ### Adding a wiki page @@ -468,8 +470,11 @@ files it points at gives the installer a 404. The `validate.yml` workflow runs `python -m unittest discover -s tests -t . -v` on both roads into main: every pull request, and every direct push, that touches `bios/`, `platforms/`, `emulators/`, `schemas/`, `scripts/`, `tests/`, -`install.py` or an input of the site build such as `wiki/`. The same job then -builds the site with `npm run build --prefix website`, so a page that would fail the +`ports/`, the installer and its bootstraps, or an input of the site build such +as `wiki/` or `measurements/`. The `paths:` lists of `validate.yml` are the +reference, and `tests/test_workflow_paths.py` holds them to the site's inputs. The same job then +generates the site before the suite, since the tests read its pages, and builds +it with `npm run build --prefix website` after, so a page that would fail the deploy fails the pull request. Work reaches main by push as often as by pull request, so a suite wired to pull requests alone would guard the road nobody takes. The test job (`run-tests`) runs in parallel with schema validation, and on a pull request