diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 94b4c64f..2db97692 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -17,6 +17,17 @@ on: - "install.py" - "install.sh" - "install.ps1" + # Every input of the site build, so a page that would fail the deploy + # fails here first (tests/test_workflow_paths.py). + - "wiki/**" + - "docs_assets/**" + - "provenance/**" + - "exports/**" + - "install/**" + - "database.json" + - "release.json" + - "derivable.json" + - "mkdocs.yml" # Spelled out twice because the workflow parser reads no YAML anchor. A test # holds the two lists equal. push: @@ -31,6 +42,15 @@ on: - "install.py" - "install.sh" - "install.ps1" + - "wiki/**" + - "docs_assets/**" + - "provenance/**" + - "exports/**" + - "install/**" + - "database.json" + - "release.json" + - "derivable.json" + - "mkdocs.yml" permissions: contents: read @@ -174,9 +194,10 @@ jobs: 3.12 # markdown: the site validation test renders a wiki line to prove the - # heading guard's premise, and skipped on every run without it. + # heading guard's premise, and skipped on every run without it. The docs + # group builds the site as deploy-site.yml does, with the same bounds. - name: Install dependencies - run: pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" + run: pip install pyyaml "jsonschema[format-nongpl]==4.26.0" "markdown==3.10.2" "mkdocs-material>=9.7.5,<10" "pymdown-extensions>=10.14" # The exporter fidelity cases compare what we hand back against the # file each platform publishes, so they need those files. They skip @@ -202,6 +223,17 @@ jobs: RETROBIOS_OFFLINE: "1" run: python -m unittest discover -s tests -t . -v + # deploy-site.yml runs this after the merge, and a dead link or a stray + # heading failed it there with nothing in front of it. Run whatever the + # suite returned, so one red step does not hide the other. The release + # assets are not restored: they change counts, not links or headings. + - name: Build the site + if: ${{ !cancelled() }} + run: | + python scripts/generate_site.py + mkdocs build --strict + python scripts/validate_site.py + label-pr: runs-on: ubuntu-latest if: github.event_name == 'pull_request' diff --git a/tests/test_workflow_paths.py b/tests/test_workflow_paths.py index c831e372..a2aba810 100644 --- a/tests/test_workflow_paths.py +++ b/tests/test_workflow_paths.py @@ -108,6 +108,45 @@ class DeploySiteTriggers(unittest.TestCase): self.assertIn("scripts/sourceref.py", closure(["generate_site.py"])) +class ThePullRequestBuildsTheSite(unittest.TestCase): + """mkdocs was installed by deploy-site.yml alone, which runs after the + merge, and validate.yml did not watch wiki/: a dead link or a line read as + a second heading passed every check and failed the deploy on main.""" + + @staticmethod + def _workflow(name: str) -> dict: + return yaml.safe_load( + (REPO_ROOT / ".github/workflows" / name).read_text(encoding="utf-8") + ) + + def test_the_test_job_runs_the_deploy_build(self): + steps = self._workflow("validate.yml")["jobs"]["run-tests"]["steps"] + runs = "\n".join(str(step.get("run", "")) for step in steps) + for command in ( + "python scripts/generate_site.py", + "mkdocs build --strict", + "python scripts/validate_site.py", + ): + with self.subTest(command=command): + self.assertIn(command, runs) + for package in ("mkdocs-material", "pymdown-extensions"): + with self.subTest(package=package): + self.assertIn(package, runs) + + def test_every_site_input_triggers_validation(self): + deploy = self._workflow("deploy-site.yml") + validate = self._workflow("validate.yml") + watched = set(validate.get("on", validate.get(True))["push"]["paths"]) + inputs = { + path + for path in deploy.get("on", deploy.get(True))["push"]["paths"] + if not path.startswith(("scripts/", ".github/")) + } + self.assertIn("wiki/**", inputs) + self.assertEqual(sorted(inputs - watched), []) + self.assertIn("scripts/**", watched) + + class DocumentedSteps(unittest.TestCase): def test_the_profiling_reference_names_every_profile_field(self): """built_by, source_branch, bios_mode, standalone_path and diff --git a/wiki/release-process.md b/wiki/release-process.md index 043b9f3e..1eefd515 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -14,7 +14,7 @@ Budget target: ~175 minutes/month on the GitHub free tier. | Workflow | File | Trigger | |----------|------|---------| | Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, mkdocs.yml), manual | -| Validation | `validate.yml` | PR and push to main touching `bios/**`, `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh` or `install.ps1` | +| Validation | `validate.yml` | PR and push to main touching `bios/**`, `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh`, `install.ps1`, or any other input of the site build (`wiki/**`, `docs_assets/**`, `provenance/**`, `exports/**`, `install/**`, `database.json`, `release.json`, `derivable.json`, `mkdocs.yml`) | Upstream BIOS lists are not scraped on a schedule. A maintainer runs the scrapers by hand (see [adding a scraper](adding-a-scraper.md)), reviews the @@ -110,8 +110,10 @@ A profile written from a clone can cite lines that do not hold the value it declares while schemas and tests pass it. This job confronts it with its upstream. An edited profile is not held to it. -**run-tests.** Runs `python -m unittest discover -s tests -t . -v`. Must pass before a -merge, and again on the commit a direct push puts at the head of main. +**run-tests.** Runs `python -m unittest discover -s tests -t . -v`, then builds the +site as Deploy Site does (`generate_site.py`, `mkdocs build --strict`, +`validate_site.py`) whatever the suite returned. Must pass before a merge, and +again on the commit a direct push puts at the head of main. **label-pr** (pull requests only). Auto-labels the PR based on changed paths: diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index bf4c3c1d..19792761 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -299,8 +299,10 @@ files it points at gives the installer a 404. The `validate.yml` workflow runs `python -m unittest discover -s tests -t . -v` on both roads into main: every pull request, and every direct push, that touches -`bios/`, `platforms/`, `emulators/`, `schemas/`, `scripts/`, `tests/` or -`install.py`. Work reaches main by push as often as by pull request, so a suite +`bios/`, `platforms/`, `emulators/`, `schemas/`, `scripts/`, `tests/`, +`install.py` or an input of the site build such as `wiki/`. The same job then +builds the site with `mkdocs build --strict`, so a page that would fail the +deploy fails the pull request. Work reaches main by push as often as by pull request, so a suite wired to pull requests alone would guard the road nobody takes. The test job (`run-tests`) runs in parallel with schema validation, and on a pull request with BIOS validation and auto-labeling too. Those two read pull request context