fix: generate the site before CI runs the suite

docs/ is gitignored and was built after the tests, so the three guards
that read rendered pages skipped on every CI run. The site is generated
first, the suite runs whatever that step returns, and the guards fail
in CI when the pages are absent.

.github/workflows/validate.yml

@@ -231,7 +231,14 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      # docs/ is gitignored, and the tests that read the rendered pages fail
+      # in CI without it. The suite runs whatever this step returned, so a
+      # generator that stops still leaves every other test reporting.
+      - name: Generate the site
+        run: python scripts/generate_site.py
+
       - name: Run test suite
+        if: ${{ !cancelled() }}
         # No test downloads a release asset: 252 MB files fetched into the
         # shared cache filled the runner's disk and changed what later tests
         # resolved. The download tests mock the network and clear it.
@@ -246,7 +253,6 @@ jobs:
       - name: Build the site
         if: ${{ !cancelled() }}
         run: |
-          python scripts/generate_site.py
           mkdocs build --strict
           python scripts/validate_site.py
 

tests/test_audit_regressions.py

@@ -4866,6 +4866,8 @@ class NullReadsAsNone(unittest.TestCase):
     def test_the_rendered_site_has_no_python_none(self):
         docs = ROOT / "docs"
         if not docs.is_dir():
+            if os.environ.get("CI"):
+                self.fail("CI generates docs/ before the suite")
             self.skipTest("docs/ not generated")
         pattern = re.compile(r"\| None \||: None$|: None \|", re.M)
         from scripts.generate_site import LOCAL_NOTE_DIRS

tests/test_prose_style.py

@@ -10,6 +10,7 @@ place one comes back.
 from __future__ import annotations
 
 import io
+import os
 import re
 import subprocess
 import tokenize
@@ -317,6 +318,8 @@ class GeneratedSiteProse(unittest.TestCase):
             and "specs" not in p.parts
         ] if docs.is_dir() else []
         if not pages:
+            if os.environ.get("CI"):
+                self.fail("CI generates docs/ before the suite")
             self.skipTest("docs/ not generated (python scripts/generate_site.py)")
         offenders = []
         for page in pages:

tests/test_site_validation.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import os
 import re
 import sys
 import tempfile
@@ -216,8 +217,10 @@ class EveryTestDependencyIsInstalledInCi(unittest.TestCase):
         }
         workflow = (root / ".github" / "workflows" / "validate.yml").read_text()
         job = workflow[workflow.index("run-tests:"):]
+        # The suite runs on the default python: a line that installs into
+        # another interpreter (python3.9 -m pip) does not count.
         install_line = next(
-            line for line in job.splitlines() if "pip install" in line
+            line for line in job.splitlines() if "run: pip install" in line
         )
         missing = sorted(
             m for m in third_party if self.PACKAGES.get(m, m) not in install_line
@@ -289,6 +292,8 @@ class PlatformPagesPrintTheDeclaredCrc32(unittest.TestCase):
     def test_the_rendered_page(self):
         page = ROOT / "docs" / "platforms" / "retroarch.md"
         if not page.is_file():
+            if os.environ.get("CI"):
+                self.fail("CI generates docs/ before the suite")
             self.skipTest("site not generated")
         self.assertIn("CRC32: `91e93107`", page.read_text(encoding="utf-8"))
 

tests/test_workflow_paths.py

@@ -133,6 +133,18 @@ class ThePullRequestBuildsTheSite(unittest.TestCase):
             with self.subTest(package=package):
                 self.assertIn(package, runs)
 
+    def test_the_pages_exist_before_the_suite_reads_them(self):
+        """docs/ is gitignored. Built after the suite, the tests that read
+        the rendered pages found none and skipped on every CI run."""
+        steps = self._workflow("validate.yml")["jobs"]["run-tests"]["steps"]
+        runs = [str(step.get("run", "")) for step in steps]
+        generate = next(
+            i for i, run in enumerate(runs) if "scripts/generate_site.py" in run
+        )
+        suite = next(i for i, run in enumerate(runs) if "unittest discover" in run)
+        self.assertLess(generate, suite)
+        self.assertEqual(steps[suite].get("if"), "${{ !cancelled() }}")
+
     def test_every_site_input_triggers_validation(self):
         deploy = self._workflow("deploy-site.yml")
         validate = self._workflow("validate.yml")