diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 8f97591c..19e7ff48 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2415,7 +2415,9 @@ def _installed_links(sha1: str, consumers: FileConsumers, declared: set[str]) -> return links -def _system_file_card(f: dict, consumers: FileConsumers) -> list[str]: +def _system_file_card( + f: dict, consumers: FileConsumers, anchored: set[str] | None = None +) -> list[str]: """One file of a system page: hashes, who reads it, its provenance.""" name = f["name"] sha1_full = f.get("sha1", "unknown") @@ -2426,6 +2428,13 @@ def _system_file_card(f: dict, consumers: FileConsumers) -> list[str]: emus = sorted(consumers.emulators.get(sha1_full, ())) anchor = f"sha1-{sha1_full[:12]}" if sha1_full != "unknown" else "" + # A copy filed under two consoles of one maker is listed twice on its + # page: the first card holds the anchor, as an id names one element. + if anchored is not None: + if anchor in anchored: + anchor = "" + else: + anchored.add(anchor) id_attr = f' id="{anchor}"' if anchor else "" lines = [f'
', ""] # The anchor is this page's, the second link is the file's own address, @@ -2479,6 +2488,7 @@ def generate_system_page( f"# {manufacturer}", "", ] + anchored: set[str] = set() for console_name in sorted(consoles.keys()): files = consoles[console_name] @@ -2490,7 +2500,7 @@ def generate_system_page( variant_files = [f for f in files if "/.variants/" in f["path"]] for f in sorted(main_files, key=lambda x: x["name"]): - lines.extend(_system_file_card(f, consumers)) + lines.extend(_system_file_card(f, consumers, anchored)) if variant_files: # A variant gets the same card as a primary: the revision a @@ -2501,7 +2511,7 @@ def generate_system_page( for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])): lines.extend( f" {line}" if line else "" - for line in _system_file_card(v, consumers) + for line in _system_file_card(v, consumers, anchored) ) lines.append("") @@ -3309,6 +3319,7 @@ def generate_emulator_page( if data_dirs: lines.extend(["## Data directories", ""]) + member_anchors: set[str] = set() # Each tree with what it holds, where it comes from, which build reads # it and the code that does: these trees put thousands of files in # the packs, and the page said only `dolphin-sys` > `Sys`. @@ -3346,8 +3357,13 @@ def generate_emulator_page( ]) for member, info in sorted(members.items()): sha1 = str(info.get("sha1", "")) + # One id per page: two members can hold the same bytes + # (every empty file), and Dolphin's trees share Sys. + anchor = f"sha1-{sha1[:12]}" + span = "" if anchor in member_anchors else f'' + member_anchors.add(anchor) lines.append( - f' | {_code(member)} ' + f" | {span}{_code(member)} " f"| {_fmt_size(int(info.get('size', 0)))} | {_code(sha1)} |" ) lines.append("") diff --git a/scripts/pipeline.py b/scripts/pipeline.py index e813e9bc..e1eee7bc 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -24,6 +24,7 @@ Steps, numbered as the run prints them: 6 --verify-packs pack integrity, extraction and hashes 7 generate_readme.py README.md and CONTRIBUTING.md 8 generate_site.py MkDocs pages + 8b mkdocs build --strict rendered as the site deploy renders it, then validate_site.py --offline skips 2 to 2b2 and the network side of 2e and 4: release assets come from .cache/large/ only. @@ -43,6 +44,7 @@ import contextlib import importlib.util import os import re +import shutil import subprocess import sys import time @@ -801,6 +803,21 @@ def main() -> None: print("\n--- 8/8 generate site: SKIPPED (--skip-docs) ---") results["generate_site"] = SKIPPED + # Step 8b: the render the site deploy runs. Generating the pages without + # rendering them let 20 pages with duplicate ids through to CI. + if args.skip_docs: + print("\n--- 8b render site: SKIPPED (--skip-docs) ---") + results["render_site"] = SKIPPED + elif shutil.which("mkdocs") is None: + print("\n--- 8b render site: SKIPPED (mkdocs not installed) ---") + results["render_site"] = SKIPPED + else: + ok, _ = run(["mkdocs", "build", "--strict"], "8b render site") + if ok: + ok, _ = run([sys.executable, "scripts/validate_site.py"], "8b validate site") + results["render_site"] = ok + all_ok = all_ok and ok + # Summary total_elapsed = time.monotonic() - total_start print(f"\n{'=' * 60}") diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 63694527..74a68199 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5683,6 +5683,16 @@ class ThePipelineRunsTheSchemaCheckCIRuns(unittest.TestCase): for flag in flags: self.assertIn(f'"{flag}"', source) + def test_the_site_is_rendered_as_the_deploy_renders_it(self): + """The pipeline generated pages without rendering them: 20 pages with + duplicate ids passed it and failed the deploy.""" + workflow = (ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(encoding="utf-8") + source = (ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8") + self.assertIn("mkdocs build --strict", workflow) + self.assertIn('["mkdocs", "build", "--strict"]', source) + self.assertIn("scripts/validate_site.py", workflow) + self.assertIn('"scripts/validate_site.py"', source) + class TheReleaseBuildsEveryServedPackInOneRun(unittest.TestCase): """The release ran the pipeline, then built and checked RetroPie by hand: diff --git a/tests/test_install.py b/tests/test_install.py index 4ab9f361..69718fdc 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -2794,7 +2794,9 @@ class AResumedCopyIsNotSpeed(unittest.TestCase): term = install.Terminal(out) term.live = True progress = install.Progress(1, 4_000_000_000, term) - progress.start -= 10 + # Ten seconds exactly: read from the clock, a slow runner added a few + # milliseconds and moved the last digit of the speed. + progress.elapsed = lambda: 10.0 progress.bytes = 3_000_000_000 + 10_000_000 progress.resumed = 3_000_000_000 with unittest.mock.patch.object( diff --git a/tests/test_site_system_links.py b/tests/test_site_system_links.py index ca568dbe..541912fc 100644 --- a/tests/test_site_system_links.py +++ b/tests/test_site_system_links.py @@ -123,5 +123,20 @@ class ACopyIsOnItsOwnFoldersPage(unittest.TestCase): self.assertRegex(gs.generate_systems_index(groups), r"2 consoles, 1 files? ") +class AnIdNamesOneCardPerPage(unittest.TestCase): + """A copy filed under two consoles of one maker was listed twice with + the same id, and the deployed site refused 20 pages.""" + + def test_the_second_card_has_no_id(self): + from cross_reference import FileConsumers + + entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3, + "path": "bios/Nintendo/DS/a.bin"} + page = gs.generate_system_page( + "Nintendo", {"DS": [entry], "DSi": [entry]}, FileConsumers({"files": {}})) + self.assertEqual(page.count('id="sha1-aaaaaaaaaaaa"'), 1) + self.assertEqual(page.count("rb-sys-file"), 2) + + if __name__ == "__main__": unittest.main()