fix: one id per card, and render the site locally

A copy filed under two consoles of one maker was listed twice with the same id, and a data directory listed the same bytes twice, so the site deploy refused 20 pages. The first card keeps the id. The pipeline only generated the pages, so it now renders them with mkdocs build --strict and runs validate_site as the deploy does. The resumed-speed test read the clock and failed on a slow runner, it now fixes the elapsed time.

scripts/generate_site.py

@@ -2415,7 +2415,9 @@ def _installed_links(sha1: str, consumers: FileConsumers, declared: set[str]) ->
     return links
 
 
-def _system_file_card(f: dict, consumers: FileConsumers) -> list[str]:
+def _system_file_card(
+    f: dict, consumers: FileConsumers, anchored: set[str] | None = None
+) -> list[str]:
     """One file of a system page: hashes, who reads it, its provenance."""
     name = f["name"]
     sha1_full = f.get("sha1", "unknown")
@@ -2426,6 +2428,13 @@ def _system_file_card(f: dict, consumers: FileConsumers) -> list[str]:
     emus = sorted(consumers.emulators.get(sha1_full, ()))
 
     anchor = f"sha1-{sha1_full[:12]}" if sha1_full != "unknown" else ""
+    # A copy filed under two consoles of one maker is listed twice on its
+    # page: the first card holds the anchor, as an id names one element.
+    if anchored is not None:
+        if anchor in anchored:
+            anchor = ""
+        else:
+            anchored.add(anchor)
     id_attr = f' id="{anchor}"' if anchor else ""
     lines = [f'<div class="rb-sys-file"{id_attr} markdown>', ""]
     # The anchor is this page's, the second link is the file's own address,
@@ -2479,6 +2488,7 @@ def generate_system_page(
         f"# {manufacturer}",
         "",
     ]
+    anchored: set[str] = set()
 
     for console_name in sorted(consoles.keys()):
         files = consoles[console_name]
@@ -2490,7 +2500,7 @@ def generate_system_page(
         variant_files = [f for f in files if "/.variants/" in f["path"]]
 
         for f in sorted(main_files, key=lambda x: x["name"]):
-            lines.extend(_system_file_card(f, consumers))
+            lines.extend(_system_file_card(f, consumers, anchored))
 
         if variant_files:
             # A variant gets the same card as a primary: the revision a
@@ -2501,7 +2511,7 @@ def generate_system_page(
             for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])):
                 lines.extend(
                     f"    {line}" if line else ""
-                    for line in _system_file_card(v, consumers)
+                    for line in _system_file_card(v, consumers, anchored)
                 )
             lines.append("")
 
@@ -3309,6 +3319,7 @@ def generate_emulator_page(
 
     if data_dirs:
         lines.extend(["## Data directories", ""])
+        member_anchors: set[str] = set()
         # Each tree with what it holds, where it comes from, which build reads
         # it and the code that does: these trees put thousands of files in
         # the packs, and the page said only `dolphin-sys` > `Sys`.
@@ -3346,8 +3357,13 @@ def generate_emulator_page(
                 ])
                 for member, info in sorted(members.items()):
                     sha1 = str(info.get("sha1", ""))
+                    # One id per page: two members can hold the same bytes
+                    # (every empty file), and Dolphin's trees share Sys.
+                    anchor = f"sha1-{sha1[:12]}"
+                    span = "" if anchor in member_anchors else f'<span id="{anchor}"></span>'
+                    member_anchors.add(anchor)
                     lines.append(
-                        f'        | <span id="sha1-{sha1[:12]}"></span>{_code(member)} '
+                        f"        | {span}{_code(member)} "
                         f"| {_fmt_size(int(info.get('size', 0)))} | {_code(sha1)} |"
                     )
                 lines.append("")

scripts/pipeline.py

@@ -24,6 +24,7 @@ Steps, numbered as the run prints them:
   6    --verify-packs               pack integrity, extraction and hashes
   7    generate_readme.py           README.md and CONTRIBUTING.md
   8    generate_site.py             MkDocs pages
+  8b   mkdocs build --strict        rendered as the site deploy renders it, then validate_site.py
 
 --offline skips 2 to 2b2 and the network side of 2e and 4: release assets
 come from .cache/large/ only.
@@ -43,6 +44,7 @@ import contextlib
 import importlib.util
 import os
 import re
+import shutil
 import subprocess
 import sys
 import time
@@ -801,6 +803,21 @@ def main() -> None:
         print("\n--- 8/8 generate site: SKIPPED (--skip-docs) ---")
         results["generate_site"] = SKIPPED
 
+    # Step 8b: the render the site deploy runs. Generating the pages without
+    # rendering them let 20 pages with duplicate ids through to CI.
+    if args.skip_docs:
+        print("\n--- 8b render site: SKIPPED (--skip-docs) ---")
+        results["render_site"] = SKIPPED
+    elif shutil.which("mkdocs") is None:
+        print("\n--- 8b render site: SKIPPED (mkdocs not installed) ---")
+        results["render_site"] = SKIPPED
+    else:
+        ok, _ = run(["mkdocs", "build", "--strict"], "8b render site")
+        if ok:
+            ok, _ = run([sys.executable, "scripts/validate_site.py"], "8b validate site")
+        results["render_site"] = ok
+        all_ok = all_ok and ok
+
     # Summary
     total_elapsed = time.monotonic() - total_start
     print(f"\n{'=' * 60}")

tests/test_audit_regressions.py

@@ -5683,6 +5683,16 @@ class ThePipelineRunsTheSchemaCheckCIRuns(unittest.TestCase):
         for flag in flags:
             self.assertIn(f'"{flag}"', source)
 
+    def test_the_site_is_rendered_as_the_deploy_renders_it(self):
+        """The pipeline generated pages without rendering them: 20 pages with
+        duplicate ids passed it and failed the deploy."""
+        workflow = (ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(encoding="utf-8")
+        source = (ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8")
+        self.assertIn("mkdocs build --strict", workflow)
+        self.assertIn('["mkdocs", "build", "--strict"]', source)
+        self.assertIn("scripts/validate_site.py", workflow)
+        self.assertIn('"scripts/validate_site.py"', source)
+
 
 class TheReleaseBuildsEveryServedPackInOneRun(unittest.TestCase):
     """The release ran the pipeline, then built and checked RetroPie by hand:

tests/test_install.py

@@ -2794,7 +2794,9 @@ class AResumedCopyIsNotSpeed(unittest.TestCase):
         term = install.Terminal(out)
         term.live = True
         progress = install.Progress(1, 4_000_000_000, term)
-        progress.start -= 10
+        # Ten seconds exactly: read from the clock, a slow runner added a few
+        # milliseconds and moved the last digit of the speed.
+        progress.elapsed = lambda: 10.0
         progress.bytes = 3_000_000_000 + 10_000_000
         progress.resumed = 3_000_000_000
         with unittest.mock.patch.object(

tests/test_site_system_links.py

@@ -123,5 +123,20 @@ class ACopyIsOnItsOwnFoldersPage(unittest.TestCase):
         self.assertRegex(gs.generate_systems_index(groups), r"2 consoles, 1 files? ")
 
 
+class AnIdNamesOneCardPerPage(unittest.TestCase):
+    """A copy filed under two consoles of one maker was listed twice with
+    the same id, and the deployed site refused 20 pages."""
+
+    def test_the_second_card_has_no_id(self):
+        from cross_reference import FileConsumers
+
+        entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3,
+                 "path": "bios/Nintendo/DS/a.bin"}
+        page = gs.generate_system_page(
+            "Nintendo", {"DS": [entry], "DSi": [entry]}, FileConsumers({"files": {}}))
+        self.assertEqual(page.count('id="sha1-aaaaaaaaaaaa"'), 1)
+        self.assertEqual(page.count("rb-sys-file"), 2)
+
+
 if __name__ == "__main__":
     unittest.main()