feat: tune search weights and heavy page loads

Status badges and hash-listing blocks leave the search index, card titles
weigh as page titles, and Pagefind ranking is set inside a range where
the ten gate queries all answer (search_gate.py: 10/10). Tables are set
up and measured when they near the screen, which relaid out a 3,800-row
page after its first paint, and console icons hold their box.

scripts/generate_site.py

@@ -3295,7 +3295,7 @@ def _system_file_card(
     for label, value in (("SHA1", sha1_full), ("MD5", md5_full),
                          ("CRC32", f.get("crc32")), ("SHA256", f.get("sha256"))):
         if value and value != "unknown":
-            lines.append(f'- {label}: <span class="rb-hash">{_code(value)}</span>')
+            lines.append(f'- {label}: {markup.hash_value(value)}')
     if emus:
         lines.append(f"- Emulators: {', '.join(_emulator_links(sha1_full, consumers))}")
     provenance = f.get("provenance", {})
@@ -3420,8 +3420,11 @@ def _card_title(text: str, anchor: str, searchable: bool = True) -> str:
         # would add the engine's anchor link and icon to every card.
         title = re.sub(r"`([^`]+)`", lambda m: f"<code>{html.escape(m[1], quote=False)}</code>", text)
         ident = f' id="{anchor}"' if anchor else ""
-        ignore = "" if searchable else f" {markup.search_exclude()}"
-        return f'<h3{ident} class="rb-file-title"{ignore}>{title}</h3>'
+        # The card is the file's own page in all but name: its title weighs
+        # as a page title does, so a file name typed in the search finds
+        # the card before a guide that cites it as an example.
+        weight = ' data-pagefind-weight="7"' if searchable else f" {markup.search_exclude()}"
+        return f'<h3{ident} class="rb-file-title"{weight}>{title}</h3>'
     attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title"
     if not searchable:
         attributes += " data-search-exclude"
@@ -3873,7 +3876,7 @@ def _file_badges(f: dict, in_repo: bool) -> list[str]:
         )
     if archive:
         badges.append(
-            f'<span class="rb-badge rb-badge-muted">in {archive}</span>'
+            f'<span class="rb-badge rb-badge-muted rb-badge-archive">in {archive}</span>'
         )
     if ftype and ftype != "bios":
         badges.append(
@@ -4460,8 +4463,14 @@ def _render_structured_blocks(profile: dict) -> list[str]:
         body: list[str] = []
         _render_yaml_value(body, val, indent=0,
                            field_text=_structured_field_text(profile))
-        lines.extend(markup.details(
-            "abstract", label, [_escape_tags(line) for line in body], opened=True))
+        block = markup.details(
+            "abstract", label, [_escape_tags(line) for line in body], opened=True)
+        # A block listing hashes is the lookup's, as a card's hash lines are:
+        # izapple2's ROM table put "md5" 59 times in the search, above the
+        # page that explains MD5 checks.
+        if any(re.search(r"\b[0-9a-f]{32,64}\b", line) for line in body):
+            block = search_excluded(block)
+        lines.extend(block)
 
     return lines
 
@@ -4476,7 +4485,7 @@ def _hash_spans(value: str | list) -> str:
     from hashing import parse_hash_list  # noqa: PLC0415
 
     return ", ".join(
-        f'<span class="rb-hash">{_code(v)}</span>' for v in parse_hash_list(value)
+        markup.hash_value(v) for v in parse_hash_list(value)
     )
 
 

scripts/sitemarkup.py

@@ -153,6 +153,15 @@ def image(alt: str, src: str, **attributes: str) -> str:
     return f'<img src="{html.escape(src)}" alt="{html.escape(alt)}"{attrs}>'
 
 
+def hash_value(value: object) -> str:
+    """A hash printed whole, in the hash style. One element under Starlight:
+    a span around a code span was 1,400 extra elements on a page of cards."""
+    text = str(value).replace("`", "'")
+    if _engine == "mkdocs":
+        return f'<span class="rb-hash">`{text}`</span>'
+    return f'<code class="rb-hash">{html.escape(text, quote=False)}</code>'
+
+
 def block_open(tag: str, attributes: str = "") -> str:
     """The opening tag of an HTML block whose content is Markdown."""
     attributes = f" {attributes.strip()}" if attributes.strip() else ""

scripts/sitestarlight.py

@@ -369,6 +369,12 @@ def finalize_page(
         body = f'<div id="{html.escape(title_id)}" class="rb-anchor"></div>\n\n' + body
     if rel not in ("wiki/glossary.md", "wiki/reference-terms.md"):
         body = add_tooltips(body, terms)
+    # A badge is a status (required, missing, MD5), not a word a reader
+    # searches for: 141 "missing" badges put an emulator page above the
+    # troubleshooting page for "missing bios". The archive a file sits in
+    # is a name, and stays: it is how "neogeo.zip" finds the cores reading it.
+    body = re.sub(r'<span class="(rb-badge(?![^"]*rb-badge-archive)[^"]*)"',
+                  r'<span data-pagefind-ignore class="\1"', body)
     components = used_components(body)
     stem = rel.rsplit(".", 1)[0]
     if components:

website/astro.config.mjs

@@ -66,6 +66,13 @@ export default defineConfig({
 			expressiveCode: { defaultProps: { wrap: true } },
 			lastUpdated: false,
 			pagination: true,
+			// Measured on the ten queries of tools/search_gate.py, inside a range
+			// where all ten answer: Starlight's defaults (0.1, 0.1, 2, 9) let a guide
+			// that cites scph5501 once outrank the file's own card. A long page
+			// is penalised a little more, and how often a word appears counts.
+			pagefind: {
+				ranking: { pageLength: 0.3, termFrequency: 0.5, termSaturation: 1.4, termSimilarity: 9 },
+			},
 			plugins: [
 				starlightSidebarTopics(sidebar.topics, sidebar.options),
 				starlightLinksValidator(),

website/src/scripts/site.ts

@@ -30,10 +30,17 @@ const headingText = (element: Element): string => {
 };
 
 // A table that overflows is a named stop in the tab order, so a keyboard
-// can scroll it (WCAG 2.1.1). Checked again when a closed block opens.
-const markScrollable = (table: HTMLTableElement) => {
+// can scroll it (WCAG 2.1.1). Checked again when a closed block opens. All
+// widths are read first and the attributes written after, so a page of
+// twenty tables is laid out once, not twenty times.
+const markScrollable = (tables: HTMLTableElement[]) => {
+	const overflowing = tables.map((table) => table.scrollWidth > table.clientWidth + 1);
+	tables.forEach((table, index) => setScrollable(table, overflowing[index]));
+};
+
+const setScrollable = (table: HTMLTableElement, overflows: boolean) => {
 	const heading = table.dataset.rbHeading ?? 'Data table';
-	if (table.scrollWidth > table.clientWidth + 1) {
+	if (overflows) {
 		table.tabIndex = 0;
 		table.setAttribute('role', 'region');
 		table.setAttribute('aria-label', `${heading}, scrollable table`);
@@ -79,8 +86,10 @@ const filterTable = (table: HTMLTableElement, index: number, heading: string) =>
 	count.textContent = `${rows.length} rows`;
 	controls.append(label, input, count);
 	table.before(controls);
-	const searchable = rows.map((row) => ({ row, text: row.textContent!.normalize('NFKD').toLocaleLowerCase() }));
+	// Each row's text is read on the first keystroke, not on every page load.
+	let searchable: { row: HTMLTableRowElement; text: string }[] | null = null;
 	input.addEventListener('input', () => {
+		searchable ??= rows.map((row) => ({ row, text: row.textContent!.normalize('NFKD').toLocaleLowerCase() }));
 		const query = input.value.trim().normalize('NFKD').toLocaleLowerCase();
 		let visible = 0;
 		for (const entry of searchable) {
@@ -92,7 +101,8 @@ const filterTable = (table: HTMLTableElement, index: number, heading: string) =>
 };
 
 const enhanceTables = () => {
-	document.querySelectorAll<HTMLTableElement>('.sl-markdown-content table').forEach((table, index) => {
+	const tables = [...document.querySelectorAll<HTMLTableElement>('.sl-markdown-content table')];
+	const enhance = (table: HTMLTableElement) => {
 		for (const cell of table.querySelectorAll('thead th')) if (!cell.hasAttribute('scope')) cell.setAttribute('scope', 'col');
 		const heading = headingText(table);
 		table.dataset.rbHeading = heading;
@@ -103,18 +113,29 @@ const enhanceTables = () => {
 			table.prepend(caption);
 		}
 		stackActionTable(table);
-		markScrollable(table);
-		filterTable(table, index, heading);
-	});
-	const recheck = () => document.querySelectorAll<HTMLTableElement>('.sl-markdown-content table').forEach(markScrollable);
+		filterTable(table, tables.indexOf(table), heading);
+	};
+	// A table is set up and measured when it nears the screen. Doing all of
+	// them at load relaid out every row of a 3,800-row page after its first
+	// paint. One that appears later (a block opened, a tab chosen) is handled
+	// then.
+	const near = new Set<HTMLTableElement>();
+	const observer = new IntersectionObserver(
+		(entries) => {
+			const shown = entries.filter((entry) => entry.isIntersecting).map((entry) => entry.target as HTMLTableElement);
+			for (const table of shown) {
+				if (!near.has(table)) enhance(table);
+				near.add(table);
+			}
+			if (shown.length) requestAnimationFrame(() => markScrollable(shown));
+		},
+		{ rootMargin: '400px 0px' },
+	);
+	tables.forEach((table) => observer.observe(table));
 	let timer = 0;
 	addEventListener('resize', () => {
 		clearTimeout(timer);
-		timer = window.setTimeout(recheck, 150);
-	});
-	document.addEventListener('toggle', recheck, true);
-	document.addEventListener('click', (event) => {
-		if ((event.target as Element).closest?.('[role="tab"]')) setTimeout(recheck, 0);
+		timer = window.setTimeout(() => markScrollable([...near]), 150);
 	});
 };
 

website/src/styles/retrobios.css

@@ -216,7 +216,16 @@
 	background: #fff;
 }
 .sl-markdown-content img[src*='assets/logos/'] { padding: 2px; border-radius: 2px; background: #fff; }
-.sl-markdown-content :is(h2, h3) img { display: inline; vertical-align: middle; margin: 0 0.4rem 0 0; }
+/* A system icon beside a console heading holds its box before it loads:
+   it pushed every card below it down once it arrived. */
+.sl-markdown-content :is(h2, h3) img {
+	display: inline;
+	width: 24px;
+	height: 24px;
+	object-fit: contain;
+	vertical-align: middle;
+	margin: 0 0.4rem 0 0;
+}
 
 /* Card grids */
 .rb-cards {

website/tools/lighthouse.py

@@ -1,7 +1,11 @@
 #!/usr/bin/env python3
 """Lighthouse mobile (default form factor) on a list of pages, one summary line each.
 
-    python lighthouse.py <out dir> <name>=<base url> ... -- <page> ...
+    python lighthouse.py [--performance-only] <out dir> <name>=<base url> ... -- <page> ...
+
+With all four categories, Lighthouse runs its accessibility checks inside the
+page during the same load, and those tasks count toward Total Blocking Time
+on a page of 15,000 elements. --performance-only measures the page alone.
 
 Needs `npx lighthouse` and Chrome at /usr/bin/google-chrome.
 """
@@ -18,6 +22,10 @@ CATEGORIES = ("performance", "accessibility", "best-practices", "seo")
 
 def main() -> int:
     args = sys.argv[1:]
+    categories = CATEGORIES
+    if args[0] == "--performance-only":
+        args.pop(0)
+        categories = ("performance",)
     out = Path(args.pop(0))
     split = args.index("--")
     engines = dict(item.split("=", 1) for item in args[:split])
@@ -34,7 +42,7 @@ def main() -> int:
                 ["npx", "--prefix", str(Path(__file__).resolve().parent.parent), "lighthouse", base + page,
                  "--quiet", "--output=json", f"--output-path={report.resolve()}",
                  "--chrome-flags=--headless=new --no-sandbox",
-                 f"--only-categories={','.join(CATEGORIES)}"],
+                 f"--only-categories={','.join(categories)}"],
                 env=env, cwd=out, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
             data = json.loads(report.read_text())
             cats, audits = data["categories"], data["audits"]
@@ -42,12 +50,13 @@ def main() -> int:
                       if a.get("score") is not None and a["score"] < 1 and a.get("scoreDisplayMode") == "binary"]
             print(json.dumps({
                 "engine": name, "page": page or "/",
-                **{c: round(cats[c]["score"] * 100) for c in CATEGORIES},
+                **{c: round(cats[c]["score"] * 100) for c in categories},
                 "lcp": audits["largest-contentful-paint"]["displayValue"],
                 "tbt": audits["total-blocking-time"]["displayValue"],
                 "cls": audits["cumulative-layout-shift"]["displayValue"],
                 "bytes": audits["total-byte-weight"]["displayValue"],
-                "dom": audits["dom-size"]["displayValue"],
+                # Lighthouse 13 moved the DOM size audit into an insight.
+                "dom": (audits.get("dom-size") or audits.get("dom-size-insight") or {}).get("displayValue", ""),
                 "failed": failed,
             }))
     return 0

website/tools/search_gate.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""The search gate: ten queries a visitor types, and the pages that answer them.
+
+Runs measure.mjs against a served build and counts a query as answered when
+one of its expected pages is in the first three results of the site's own
+search box.
+
+    python website/tools/search_gate.py http://127.0.0.1:4321/retrobios/
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+TOOLS = Path(__file__).resolve().parent
+
+# Query, then the pages that answer it (path prefixes under the site).
+QUERIES: dict[str, tuple[str, ...]] = {
+    "playstation bios": ("systems/sony/",),
+    # The file itself: its card on the system page, or a core that reads it.
+    "scph5501": ("systems/sony/", "emulators/beetle_psx/", "emulators/duckstation/",
+                 "emulators/duckstation-legacy/", "emulators/pcsx1/", "emulators/swanstation/",
+                 "emulators/ares/", "emulators/pcsx_rearmed/", "files/"),
+    "batocera folder": ("platforms/batocera/",),
+    "neogeo.zip": ("emulators/fbneo", "emulators/mame", "systems/snk/", "systems/arcade"),
+    "duckstation": ("emulators/duckstation/", "emulators/duckstation-legacy/"),
+    "steam deck": ("platforms/emudeck/", "platforms/retrodeck/", "which-pack/"),
+    "missing bios": ("wiki/troubleshooting/", "wiki/what-are-bios-files/", "wiki/faq/"),
+    "md5": ("wiki/verification-modes/",),
+    "retroarch system folder": ("platforms/retroarch/",),
+    "knulli": ("platforms/knulli/",),
+}
+
+
+def main() -> int:
+    base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:4321/retrobios/"
+    root = "/" + base.split("/", 3)[3]
+    run = subprocess.run(
+        ["node", str(TOOLS / "measure.mjs"), "search", base, "starlight-ui", *QUERIES],
+        capture_output=True, text=True, check=True,
+    )
+    answered = 0
+    for line in run.stdout.splitlines():
+        result = json.loads(line)
+        if "query" not in result:
+            print(f"first search: {result['first_search_kb']} KB in {result['first_search_requests']} requests")
+            continue
+        top = [entry.split(" | ")[1][len(root):] for entry in result["top"][:3]]
+        hit = any(page.startswith(QUERIES[result["query"]]) for page in top)
+        answered += hit
+        print(f"{'ok  ' if hit else 'MISS'} {result['query']!r}: {', '.join(top)}")
+    print(f"{answered}/{len(QUERIES)} queries answered in the top 3")
+    return 0 if answered >= 9 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

website/tools/search_scores.mjs

@@ -0,0 +1,23 @@
+// Raw Pagefind results with their scores, to see why a page ranks where it does.
+//   node search_scores.mjs <base url> <query> [ranking json]
+import { chromium } from 'playwright-core';
+
+const [base, query, ranking] = process.argv.slice(2);
+const browser = await chromium.launch({ executablePath: '/usr/bin/google-chrome' });
+const page = await browser.newPage();
+await page.goto(base, { waitUntil: 'networkidle' });
+const rows = await page.evaluate(async ({ root, query, ranking }) => {
+	const pagefind = await import(`${root}pagefind/pagefind.js`);
+	if (ranking) await pagefind.options({ ranking: JSON.parse(ranking) });
+	const search = await pagefind.search(query);
+	const top = search.results.slice(0, 8);
+	const data = await Promise.all(top.map((result) => result.data()));
+	return top.map((result, index) => ({
+		score: result.score.toFixed(3),
+		url: data[index].url,
+		words: data[index].word_count,
+		hits: result.words.length,
+	}));
+}, { root: base, query, ranking });
+for (const row of rows) console.log(row.score, row.words, row.hits, row.url);
+await browser.close();