From 954f213d42f21f1de87bb0e84979b59b4b3e3c58 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:04:51 +0200 Subject: [PATCH] fix: one id per card, and render the site locally A copy filed under two consoles of one maker was listed twice with the same id, and a data directory listed the same bytes twice, so the site deploy refused 20 pages. The first card keeps the id. The pipeline only generated the pages, so it now renders them with mkdocs build --strict and runs validate_site as the deploy does. The resumed-speed test read the clock and failed on a slow runner, it now fixes the elapsed time. --- scripts/generate_site.py | 24 ++++++++++++++++++++---- scripts/pipeline.py | 17 +++++++++++++++++ tests/test_audit_regressions.py | 10 ++++++++++ tests/test_install.py | 4 +++- tests/test_site_system_links.py | 15 +++++++++++++++ 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 8f97591c..19e7ff48 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2415,7 +2415,9 @@ def _installed_links(sha1: str, consumers: FileConsumers, declared: set[str]) -> return links -def _system_file_card(f: dict, consumers: FileConsumers) -> list[str]: +def _system_file_card( + f: dict, consumers: FileConsumers, anchored: set[str] | None = None +) -> list[str]: """One file of a system page: hashes, who reads it, its provenance.""" name = f["name"] sha1_full = f.get("sha1", "unknown") @@ -2426,6 +2428,13 @@ def _system_file_card(f: dict, consumers: FileConsumers) -> list[str]: emus = sorted(consumers.emulators.get(sha1_full, ())) anchor = f"sha1-{sha1_full[:12]}" if sha1_full != "unknown" else "" + # A copy filed under two consoles of one maker is listed twice on its + # page: the first card holds the anchor, as an id names one element. + if anchored is not None: + if anchor in anchored: + anchor = "" + else: + anchored.add(anchor) id_attr = f' id="{anchor}"' if anchor else "" lines = [f'
', ""] # The anchor is this page's, the second link is the file's own address, @@ -2479,6 +2488,7 @@ def generate_system_page( f"# {manufacturer}", "", ] + anchored: set[str] = set() for console_name in sorted(consoles.keys()): files = consoles[console_name] @@ -2490,7 +2500,7 @@ def generate_system_page( variant_files = [f for f in files if "/.variants/" in f["path"]] for f in sorted(main_files, key=lambda x: x["name"]): - lines.extend(_system_file_card(f, consumers)) + lines.extend(_system_file_card(f, consumers, anchored)) if variant_files: # A variant gets the same card as a primary: the revision a @@ -2501,7 +2511,7 @@ def generate_system_page( for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])): lines.extend( f" {line}" if line else "" - for line in _system_file_card(v, consumers) + for line in _system_file_card(v, consumers, anchored) ) lines.append("") @@ -3309,6 +3319,7 @@ def generate_emulator_page( if data_dirs: lines.extend(["## Data directories", ""]) + member_anchors: set[str] = set() # Each tree with what it holds, where it comes from, which build reads # it and the code that does: these trees put thousands of files in # the packs, and the page said only `dolphin-sys` > `Sys`. @@ -3346,8 +3357,13 @@ def generate_emulator_page( ]) for member, info in sorted(members.items()): sha1 = str(info.get("sha1", "")) + # One id per page: two members can hold the same bytes + # (every empty file), and Dolphin's trees share Sys. + anchor = f"sha1-{sha1[:12]}" + span = "" if anchor in member_anchors else f'' + member_anchors.add(anchor) lines.append( - f' | {_code(member)} ' + f" | {span}{_code(member)} " f"| {_fmt_size(int(info.get('size', 0)))} | {_code(sha1)} |" ) lines.append("") diff --git a/scripts/pipeline.py b/scripts/pipeline.py index e813e9bc..e1eee7bc 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -24,6 +24,7 @@ Steps, numbered as the run prints them: 6 --verify-packs pack integrity, extraction and hashes 7 generate_readme.py README.md and CONTRIBUTING.md 8 generate_site.py MkDocs pages + 8b mkdocs build --strict rendered as the site deploy renders it, then validate_site.py --offline skips 2 to 2b2 and the network side of 2e and 4: release assets come from .cache/large/ only. @@ -43,6 +44,7 @@ import contextlib import importlib.util import os import re +import shutil import subprocess import sys import time @@ -801,6 +803,21 @@ def main() -> None: print("\n--- 8/8 generate site: SKIPPED (--skip-docs) ---") results["generate_site"] = SKIPPED + # Step 8b: the render the site deploy runs. Generating the pages without + # rendering them let 20 pages with duplicate ids through to CI. + if args.skip_docs: + print("\n--- 8b render site: SKIPPED (--skip-docs) ---") + results["render_site"] = SKIPPED + elif shutil.which("mkdocs") is None: + print("\n--- 8b render site: SKIPPED (mkdocs not installed) ---") + results["render_site"] = SKIPPED + else: + ok, _ = run(["mkdocs", "build", "--strict"], "8b render site") + if ok: + ok, _ = run([sys.executable, "scripts/validate_site.py"], "8b validate site") + results["render_site"] = ok + all_ok = all_ok and ok + # Summary total_elapsed = time.monotonic() - total_start print(f"\n{'=' * 60}") diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 63694527..74a68199 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5683,6 +5683,16 @@ class ThePipelineRunsTheSchemaCheckCIRuns(unittest.TestCase): for flag in flags: self.assertIn(f'"{flag}"', source) + def test_the_site_is_rendered_as_the_deploy_renders_it(self): + """The pipeline generated pages without rendering them: 20 pages with + duplicate ids passed it and failed the deploy.""" + workflow = (ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(encoding="utf-8") + source = (ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8") + self.assertIn("mkdocs build --strict", workflow) + self.assertIn('["mkdocs", "build", "--strict"]', source) + self.assertIn("scripts/validate_site.py", workflow) + self.assertIn('"scripts/validate_site.py"', source) + class TheReleaseBuildsEveryServedPackInOneRun(unittest.TestCase): """The release ran the pipeline, then built and checked RetroPie by hand: diff --git a/tests/test_install.py b/tests/test_install.py index 4ab9f361..69718fdc 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -2794,7 +2794,9 @@ class AResumedCopyIsNotSpeed(unittest.TestCase): term = install.Terminal(out) term.live = True progress = install.Progress(1, 4_000_000_000, term) - progress.start -= 10 + # Ten seconds exactly: read from the clock, a slow runner added a few + # milliseconds and moved the last digit of the speed. + progress.elapsed = lambda: 10.0 progress.bytes = 3_000_000_000 + 10_000_000 progress.resumed = 3_000_000_000 with unittest.mock.patch.object( diff --git a/tests/test_site_system_links.py b/tests/test_site_system_links.py index ca568dbe..541912fc 100644 --- a/tests/test_site_system_links.py +++ b/tests/test_site_system_links.py @@ -123,5 +123,20 @@ class ACopyIsOnItsOwnFoldersPage(unittest.TestCase): self.assertRegex(gs.generate_systems_index(groups), r"2 consoles, 1 files? ") +class AnIdNamesOneCardPerPage(unittest.TestCase): + """A copy filed under two consoles of one maker was listed twice with + the same id, and the deployed site refused 20 pages.""" + + def test_the_second_card_has_no_id(self): + from cross_reference import FileConsumers + + entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3, + "path": "bios/Nintendo/DS/a.bin"} + page = gs.generate_system_page( + "Nintendo", {"DS": [entry], "DSi": [entry]}, FileConsumers({"files": {}})) + self.assertEqual(page.count('id="sha1-aaaaaaaaaaaa"'), 1) + self.assertEqual(page.count("rb-sys-file"), 2) + + if __name__ == "__main__": unittest.main()