From 294942afd08c3bef1e898060f177cf35f727feae Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:46:36 +0200 Subject: [PATCH] feat: port the site ports and picker checks Tables with a download link stack on phones, the browser check covers the lookup, a hash typed in the search, breadcrumbs, previous and next across tabs, glossary terms and the 404 hint. validate_site budgets the Pagefind index pieces and takes the base path without mkdocs.yml. A test holds the picker to install.narrow() on 180 selections. --- scripts/validate_site.py | 47 +++++++++++++++--- tests/test_site_picker.py | 91 ++++++++++++++++++++++++++++++++++ website/src/scripts/site.ts | 8 +-- website/tools/check_tools.mjs | 42 ++++++++++++++++ website/tools/shot_element.mjs | 13 +++++ wiki/testing-guide.md | 2 +- 6 files changed, 190 insertions(+), 13 deletions(-) create mode 100644 tests/test_site_picker.py create mode 100644 website/tools/shot_element.mjs diff --git a/scripts/validate_site.py b/scripts/validate_site.py index b39bf9ee..fe21d2d7 100644 --- a/scripts/validate_site.py +++ b/scripts/validate_site.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Validate the built RetroBIOS site as a navigable, accessible data product. -This complements ``mkdocs build --strict`` with checks on the rendered HTML: -metadata, heading structure, image alternatives, duplicate IDs, structured -data, local resources, links and fragments. +This complements the engine's own link check (``mkdocs build --strict`` or +Starlight's link validator) with checks on the rendered HTML: metadata, +heading structure, image alternatives, duplicate IDs, structured data, local +resources, links and fragments, and the budgets of what a visitor downloads. """ from __future__ import annotations @@ -53,6 +54,12 @@ MAX_PAGE_BYTES = 1_000_000 # 21.0 MB raw and 4.88 MB gzip with hashes in it, 7.3 MB and 1.44 MB without. MAX_SEARCH_INDEX_BYTES = 8_500_000 MAX_SEARCH_INDEX_GZIP_BYTES = 1_700_000 +# Pagefind loads its index in pieces: the metadata, then the chunks the words +# typed fall in (stored compressed). 59 KB for the largest chunk and 7.5 KB of +# metadata on 2026-10-22, 5.2 MB for the whole index on disk. +MAX_PAGEFIND_CHUNK_BYTES = 150_000 +MAX_PAGEFIND_META_BYTES = 50_000 +MAX_PAGEFIND_TOTAL_BYTES = 12_000_000 # Text inside these is shown as written, so markup there is not a leftover. @@ -167,10 +174,30 @@ def _heading_jumps(levels: list[int]) -> list[str]: if after > before + 1] +def _check_pagefind(site: Path) -> list[str]: + """Pagefind's index pieces, each against its budget.""" + root = site / "pagefind" + files = [path for path in root.rglob("*") if path.is_file()] + issues = [] + for path in files: + size = path.stat().st_size + rel = path.relative_to(site).as_posix() + if path.suffix == ".pf_index" and size > MAX_PAGEFIND_CHUNK_BYTES: + issues.append(f"{rel}: {size:,} bytes, over the {MAX_PAGEFIND_CHUNK_BYTES:,} budget of a chunk") + if path.suffix == ".pf_meta" and size > MAX_PAGEFIND_META_BYTES: + issues.append(f"{rel}: {size:,} bytes, over the {MAX_PAGEFIND_META_BYTES:,} budget") + total = sum(path.stat().st_size for path in files) + if total > MAX_PAGEFIND_TOTAL_BYTES: + issues.append(f"pagefind/: {total:,} bytes, over the {MAX_PAGEFIND_TOTAL_BYTES:,} budget") + return issues + + def _check_search_index(site: Path) -> list[str]: """The index every visitor who opens the search downloads, against its budget.""" import gzip + if (site / "pagefind").is_dir(): + return _check_pagefind(site) path = site / "search" / "search_index.json" if not path.is_file(): return [] @@ -343,12 +370,12 @@ def _check_links(pages: dict, site: Path, base_path: str) -> list[str]: return issues -def validate_site(site: Path, config_path: Path) -> list[str]: +def validate_site(site: Path, config_path: Path | None = None, base: str = "/retrobios/") -> list[str]: site = site.resolve() if not site.is_dir(): return [f"site directory does not exist: {site}"] - base_path = _base_path(config_path) + base_path = _base_path(config_path) if config_path and config_path.is_file() else base html_paths = sorted(site.rglob("*.html")) if not html_paths: return [f"no HTML pages found in {site}"] @@ -383,8 +410,12 @@ def main() -> None: help="the built site (default: %(default)s)", ) parser.add_argument( - "--config", default="mkdocs.yml", - help="the MkDocs configuration (default: %(default)s)", + "--config", default=None, + help="an MkDocs configuration to read the base path from", + ) + parser.add_argument( + "--base", default="/retrobios/", + help="the path the site is served under (default: %(default)s)", ) parser.add_argument( "--max-errors", type=int, default=100, @@ -393,7 +424,7 @@ def main() -> None: args = parser.parse_args() try: - issues = validate_site(Path(args.site_dir), Path(args.config)) + issues = validate_site(Path(args.site_dir), Path(args.config) if args.config else None, args.base) except (OSError, RuntimeError, ValueError) as exc: print(f"Site validation failed: {exc}", file=sys.stderr) raise SystemExit(1) from exc diff --git a/tests/test_site_picker.py b/tests/test_site_picker.py new file mode 100644 index 00000000..a4f2fc0e --- /dev/null +++ b/tests/test_site_picker.py @@ -0,0 +1,91 @@ +"""The file picker on Get your files lists what the installer installs. + +The picker reads a compacted copy of each install/.json and keeps +a file when one of its systems is chosen, or when it has none and "other" +is chosen, then prints `--system` with the choices. install.py --system +narrows with install.narrow(). The two must agree file for file, or the +list the page shows is not what the command it prints installs. +""" + +from __future__ import annotations + +import importlib.util +import json +import random +import re +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +_spec = importlib.util.spec_from_file_location("install", REPO_ROOT / "install.py") +install = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(install) + +PICKER = REPO_ROOT / "website" / "src" / "components" / "FilePicker.astro" + + +def picked(data: dict, chosen: set[str]) -> list[str]: + """The picker's rule, as FilePicker.astro applies it.""" + wanted = {index for index, system in enumerate(data["systems"]) if system["id"] in chosen} + other = install.OTHER_SYSTEMS in chosen + return [dest for dest, _size, systems in data["files"] + if (any(s in wanted for s in systems) if systems else other)] + + +class PickerMatchesInstaller(unittest.TestCase): + def setUp(self): + import sitestarlight # noqa: PLC0415 + + self.folder = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" if (REPO_ROOT / "tmp").is_dir() else None) + self.out = Path(self.folder.name) + self.count = sitestarlight.picker_data(REPO_ROOT / "install", self.out) + + def tearDown(self): + self.folder.cleanup() + + def test_every_manifest_has_its_list(self): + index = json.loads((self.out / "index.json").read_text(encoding="utf-8")) + manifests = [p for p in (REPO_ROOT / "install").glob("*.json") + if "files" in json.loads(p.read_text(encoding="utf-8"))] + self.assertEqual(len(index), len(manifests)) + self.assertEqual(self.count, len(manifests)) + + def test_random_selections_equal_install_narrow(self): + rng = random.Random(20261022) + checked = 0 + for entry in json.loads((self.out / "index.json").read_text(encoding="utf-8")): + platform = entry["platform"] + manifest = json.loads((REPO_ROOT / "install" / f"{platform}.json").read_text(encoding="utf-8")) + data = json.loads((self.out / f"{platform}.json").read_text(encoding="utf-8")) + choices = [s["id"] for s in data["systems"]] + ([install.OTHER_SYSTEMS] if data["other"] else []) + for _round in range(10): + chosen = set(rng.sample(choices, rng.randint(1, min(len(choices), 6)))) + files, _omitted = install.narrow(manifest["files"], [], sorted(chosen), [], []) + with self.subTest(platform=platform, systems=sorted(chosen)): + self.assertEqual(sorted(picked(data, chosen)), sorted(f["dest"] for f in files)) + checked += 1 + self.assertGreater(checked, 0) + + def test_choices_are_what_the_installer_offers(self): + for entry in json.loads((self.out / "index.json").read_text(encoding="utf-8")): + manifest = json.loads((REPO_ROOT / "install" / f"{entry['platform']}.json").read_text(encoding="utf-8")) + data = json.loads((self.out / f"{entry['platform']}.json").read_text(encoding="utf-8")) + offered = install.available_choices(manifest["files"], [])["systems"] + ours = [s["id"] for s in data["systems"]] + ([install.OTHER_SYSTEMS] if data["other"] else []) + with self.subTest(platform=entry["platform"]): + self.assertEqual(ours, offered) + + def test_component_applies_the_same_rule(self): + """The rule above is the component's: a change to one fails here.""" + source = PICKER.read_text(encoding="utf-8") + self.assertIn("systems.length ? systems.some((s) => wanted.has(s)) : other", source) + self.assertIn("picked.includes('other')", source) + self.assertRegex(source, re.escape("--system ${picked.join(',')}")) + + +if __name__ == "__main__": + unittest.main() diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 9d9bfe77..46a5bcb9 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -44,11 +44,11 @@ const markScrollable = (table: HTMLTableElement) => { } }; -// A table whose rows end in an action (a download button) is laid out one -// block per row on a phone, each cell under its column name, so the action -// never sits behind a horizontal scroll. +// A table whose rows end in an action (a button, a pack to download) is laid +// out one block per row on a phone, each cell under its column name, so the +// action never sits behind a horizontal scroll. const stackActionTable = (table: HTMLTableElement) => { - if (!table.querySelector('tbody .rb-button')) return; + if (!table.querySelector('tbody :is(.rb-button, a[href*="/releases/download/"], a[href$="/releases/latest"])')) return; const labels = [...table.querySelectorAll('thead th')].map((cell) => cell.textContent!.trim()); if (!labels.length) return; table.dataset.rbStack = 'true'; diff --git a/website/tools/check_tools.mjs b/website/tools/check_tools.mjs index 91c2e7cd..2e843d8b 100644 --- a/website/tools/check_tools.mjs +++ b/website/tools/check_tools.mjs @@ -39,6 +39,48 @@ await page.click('[data-none]'); assert.match(await page.textContent('.rb-pick-result'), /Choose at least one system/); await page.click('[data-all]'); assert.match(await page.textContent('.rb-pick-cmd code'), /--platform batocera$/); + +// A full hash typed in the search opens the lookup instead of text results. +await page.goto(base, { waitUntil: 'networkidle' }); +await page.click('site-search button[data-open-modal]'); +const search = page.locator('site-search dialog input').first(); +await search.waitFor(); +await search.fill('490f666e1afb15b7362b406ed1cea246'); +await Promise.all([page.waitForURL(/files\/\?hash=/), search.press('Enter')]); +console.log('hash in search ->', page.url()); + +// Breadcrumbs name the tab and the group of a deep page. +await page.goto(`${base}emulators/duckstation/`, { waitUntil: 'networkidle' }); +const crumbs = await page.locator('.rb-breadcrumbs li').allTextContents(); +console.log('breadcrumbs:', crumbs.join(' / ')); +assert.deepEqual(crumbs.slice(0, 2), ['Explore', 'Emulators']); + +// Previous and next run on from one tab into the next. +await page.goto(`${base}wiki/glossary/`, { waitUntil: 'networkidle' }); +const next = await page.locator('.pagination-links a[rel="next"]').getAttribute('href'); +console.log('after the last Start here page:', next); +assert.ok(next && next.endsWith('/platforms/'), next); + +// A glossary term opens its definition on a tap. +await page.goto(`${base}which-pack/`, { waitUntil: 'networkidle' }); +const term = page.locator('abbr.rb-term').first(); +await term.click(); +assert.equal(await term.getAttribute('data-open'), ''); +console.log('glossary term:', await term.textContent(), '=', await term.getAttribute('data-definition')); + +// On a phone, a table whose rows end in a download is one block per row. +await page.setViewportSize({ width: 390, height: 844 }); +await page.goto(`${base}which-pack/`, { waitUntil: 'networkidle' }); +const stacked = await page.locator('table[data-rb-stack]').count(); +const overflow = await page.evaluate(() => document.documentElement.scrollWidth > innerWidth + 1); +console.log('stacked tables:', stacked, 'page wider than the phone:', overflow); +assert.ok(stacked > 0); +assert.equal(overflow, false); + +// A stale address offers the list its page may be in. +await page.goto(`${base}emulators/no-such-core/`, { waitUntil: 'networkidle' }); +assert.match(await page.textContent('#rb-notfound-hint'), /list of emulators/); + assert.deepEqual(errors, []); console.log('ok'); await browser.close(); diff --git a/website/tools/shot_element.mjs b/website/tools/shot_element.mjs new file mode 100644 index 00000000..db35460e --- /dev/null +++ b/website/tools/shot_element.mjs @@ -0,0 +1,13 @@ +// Screenshot of one element of a page, to look at a component closely. +// node shot_element.mjs [width] [light|dark] +import { chromium } from 'playwright-core'; + +const [url, selector, out, width = '390', scheme = 'light'] = process.argv.slice(2); +const browser = await chromium.launch({ executablePath: '/usr/bin/google-chrome' }); +const page = await browser.newPage({ viewport: { width: Number(width), height: 900 }, colorScheme: scheme }); +await page.goto(url, { waitUntil: 'networkidle' }); +const element = page.locator(selector).first(); +await element.scrollIntoViewIfNeeded(); +await page.waitForTimeout(300); +await element.screenshot({ path: out }); +await browser.close(); diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 28022bda..5ad7e172 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -56,7 +56,7 @@ Every module in `tests/`, by the area it guards. | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_copy`, `test_site_content_data`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_copy`, `test_site_content_data`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture