From 5b1120aeb57d009211c92f2ab1645b5080e0d93e Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:36:04 +0200 Subject: [PATCH 1/3] feat: rank console pages above setup lists A setup's list for one console is a reference page kept out of the search index, naming its setup as parent. System pages carry file names and the platforms' own ids as attribute search fields, never as text. Arcade gets its own sidebar group. --- scripts/generate_site.py | 28 ++++++++++++++++++--- scripts/sitestarlight.py | 53 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index df4b4a93..21588d04 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2205,6 +2205,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]: description=description, structured=structured, terms=terms, search=authored.get("search") != "exclude", logo=str(authored.get("logo") or ""), + parent=authored.get("parent"), ) write_decorated(docs / target, text) # One file per page: a page that gained or lost its components @@ -7509,12 +7510,19 @@ def _emulator_nav(profiles: dict) -> list: def _system_nav(manufacturers: dict) -> list: - """Manufacturer pages, the machines apart from the game and engine trees.""" - machines, games = [], [] + """Manufacturer pages, the machines apart from the arcade boards and the + game and engine trees: an arcade page's crumb read "Consoles and + computers".""" + machines, arcade, games = [], [], [] for mfr in sorted(manufacturers, key=str.casefold): entry = {mfr: f"systems/{_slug(mfr)}.md"} - (games if mfr in GAME_DATA_TOPS else machines).append(entry) + if mfr in GAME_DATA_TOPS: + games.append(entry) + else: + (arcade if mfr == "Arcade" else machines).append(entry) nav: list = ["systems/index.md", {"Consoles and computers": machines}] + if arcade: + nav.append({"Arcade": arcade}) if games: nav.append({"Game engines and PC games": games}) return nav @@ -7904,6 +7912,20 @@ def _generate(args: argparse.Namespace) -> None: if stem in coverages: raise RuntimeError(f"a system page takes the platform {stem}'s address") PART_PAGES.add(f"platforms/{stem}.md") + # One console of one setup's list, a reference: "playstation + # bios" found five setups' lists before the console's page. + # Weighed down, they still won on their title, which the page + # weight does not reach, and their file list is already kept out + # of the index: the title alone, found again on the setup's page + # and the console's, is all the search held of them. + SEARCH_DEMOTED.add(f"platforms/{stem}.md") + front = _front_matter(page) + if front: + page = page[page.find("\n---\n", 4) + 5:].lstrip("\n") + front.update(level="reference", search="exclude", + parent={"label": cov["platform"], "link": f"{name}.md"}) + page = "---\n" + yaml.safe_dump(front, sort_keys=False, allow_unicode=True) \ + + "---\n\n" + page write_if_changed(str(docs / "platforms" / f"{stem}.md"), page) PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov) diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py index 0b10e2f0..5623c57b 100644 --- a/scripts/sitestarlight.py +++ b/scripts/sitestarlight.py @@ -368,6 +368,17 @@ def split_title(markdown: str) -> tuple[str, str, str]: _CARD_NAME = re.compile(r']*class="rb-file-title"(?![^>]*data-pagefind-ignore)[^>]*>([^<]+)') +def search_field(key: str, words: list[str]) -> str: + """A search field of the page, held in an attribute of an empty + element: the text of an element is indexed and shown as the result's + excerpt (a run of 250 file names). Pagefind splits several fields at a + comma, so a value carries none.""" + value = " ".join(word.replace(",", " ") for word in words).strip() + if not value: + return "" + return f'\n\n' + + def file_names_meta(body: str) -> str: """The names of the files a system page holds cards for, as a search field of the page. A page of 250 cards is long, and the search weighs a @@ -375,9 +386,35 @@ def file_names_meta(body: str) -> str: after every short page citing the name. The field is weighted in the site's ranking (astro.config.mjs, metaWeights.files).""" names = sorted({unescape(name) for name in _CARD_NAME.findall(body)}) - if not names: - return "" - return f'\n\n' + return search_field("files", names) + + +# systems/ page -> the platforms' own ids for the consoles it holds, built +# on the first system page, once the generator has installed the sections. +_page_aliases: dict[str, list[str]] = {} +_page_aliases_built = [False] + + +def page_aliases(rel: str) -> list[str]: + """The names the platforms file the consoles of a system page under + (psx, dreamcast, gba), from siterender.system_search_aliases(): "psx + bios" finds the PlayStation page though the page never says psx.""" + if not _page_aliases_built[0]: + import siterender + + by_page: dict[str, set[str]] = {} + for sys_id in list(siterender._system_page_map): + section = siterender.system_section(sys_id) + names = siterender.system_search_aliases(sys_id) + if not section or not names: + continue + target = siterender.console_href(*section).partition("#")[0] + by_page.setdefault(urllib.parse.unquote(target), set()).update(names) + _page_aliases.clear() + _page_aliases.update({page: sorted(names, key=str.casefold) + for page, names in by_page.items()}) + _page_aliases_built[0] = bool(siterender._system_page_map) + return list(_page_aliases.get(rel, [])) _NEXT_STEPS = re.compile(r"^## Next steps\b[^\n]*\n+(?P(?:[ \t]*[-*] .*\n?)+)", re.M) @@ -407,9 +444,12 @@ def finalize_page( terms: dict[str, str], search: bool = True, logo: str = "", + parent: dict | None = None, ) -> tuple[str, str]: """The Starlight file for a page body: its path under the content - collection and its text.""" + collection and its text. *parent* ({label, link}, the link written + relative to the page) is the page a part page belongs to: the route + middleware makes it the last crumb and marks it in the sidebar.""" title, title_id, body = split_title(markdown) body = body.lstrip("\n") body = rewrite_links(body, rel) @@ -427,7 +467,7 @@ def finalize_page( body = re.sub(r' Date: Fri, 23 Oct 2026 10:36:09 +0200 Subject: [PATCH 2/3] feat: crumb to the parent and read forward A part page's parent is its last crumb and is marked in the sidebar. A Next steps link pointing back in the reading run gives way to the run order, so Start here reads every page once. --- website/src/content.config.ts | 3 +++ website/src/routeData.ts | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/website/src/content.config.ts b/website/src/content.config.ts index 581aeb95..5c215a88 100644 --- a/website/src/content.config.ts +++ b/website/src/content.config.ts @@ -19,6 +19,9 @@ export const collections = { // A mark drawn beside the title (a platform's logo), a path under // the site root such as assets/logos/batocera.svg. logo: z.string().optional(), + // The page a part page belongs to (a console of a setup's list), + // read by routeData.ts: the last crumb, marked in the sidebar. + parent: z.object({ label: z.string(), link: z.string() }).optional(), }), }), }), diff --git a/website/src/routeData.ts b/website/src/routeData.ts index de7626fb..8b56eca7 100644 --- a/website/src/routeData.ts +++ b/website/src/routeData.ts @@ -8,7 +8,12 @@ // - a pruned sidebar: in the tabs listing every system and emulator, a // group that does not hold the page becomes one link to its overview, so // an emulator page does not carry the links of every other emulator. The -// groups of the other tabs stay groups. +// groups of the other tabs stay groups; +// - a part page (one console of a setup's list) names the page it belongs +// to in its `parent:` front matter: that page is its last crumb, and its +// sidebar entry is marked when the part page has none of its own; +// - a page's "Next steps" link is its next only when it reads forward: one +// pointing back to an earlier page of the run made Start here loop. import { defineRouteMiddleware, type StarlightRouteData } from '@astrojs/starlight/route-data'; import sidebar from './generated/sidebar.json' with { type: 'json' }; @@ -69,6 +74,18 @@ const folderTrail = (entries: Entry[], labels: string[], folder: string): Crumb[ return []; }; +// The parent's entry marked current, for a page the sidebar does not list. +const markCurrent = (entries: Entry[], href: string): Entry[] => + entries.map((entry) => + entry.type === 'link' + ? entry.href === href + ? { ...entry, isCurrent: true } + : entry + : { ...entry, entries: markCurrent(entry.entries, href) }, + ); + +const bare = (link: string) => withBase(link).replace(/[?#].*$/, ''); + const asLink = (entry: { label: string; link: string }): Link => ({ type: 'link', label: entry.label, @@ -84,6 +101,8 @@ export const onRequest = defineRouteMiddleware(async (context, next) => { const path = context.url.pathname; const topic = context.locals.starlightSidebarTopics?.topics.find((t) => t.isCurrent); + const parent = route.entry.data.parent; + if (parent && !route.sidebar.some(holdsCurrent)) route.sidebar = markCurrent(route.sidebar, bare(parent.link)); const crumbs: Crumb[] = []; if (topic && withBase(topic.link) !== path && route.id !== '404') { crumbs.push({ label: topic.label, href: withBase(topic.link) }); @@ -98,6 +117,10 @@ export const onRequest = defineRouteMiddleware(async (context, next) => { if (crumb.href && crumb.href === crumbs.at(-1)?.href) continue; crumbs.push(crumb); } + const parentHref = parent ? withBase(parent.link) : undefined; + if (parent && parentHref !== path && parentHref !== crumbs.at(-1)?.href) { + crumbs.push({ label: parent.label, href: parentHref }); + } } context.locals.rbBreadcrumbs = crumbs; @@ -105,7 +128,12 @@ export const onRequest = defineRouteMiddleware(async (context, next) => { const run = RUNS.find((pages) => pages.some((entry) => withBase(entry.link) === path)); const position = run ? run.findIndex((entry) => withBase(entry.link) === path) : -1; if (prev === undefined) route.pagination.prev = run && position > 0 ? asLink(run[position - 1]) : undefined; - if (following === undefined) route.pagination.next = run && position < run.length - 1 ? asLink(run[position + 1]) : undefined; + const inOrder = run && position < run.length - 1 ? asLink(run[position + 1]) : undefined; + if (following === undefined) route.pagination.next = inOrder; + else if (run && typeof following === 'object' && following.link) { + const target = run.findIndex((entry) => bare(entry.link) === bare(following.link)); + if (target !== -1 && target <= position) route.pagination.next = inOrder; + } route.sidebar = prune(route.sidebar, [topic?.label ?? '']); }); From 9e1079f4a14c49c9d81c08f4150ab12b45c7b107 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:36:09 +0200 Subject: [PATCH 3/3] chore: gate round 3 navigation and search --- tests/test_site_funnel.py | 14 ++- tests/test_site_r3_nav.py | 217 +++++++++++++++++++++++++++++++++++ website/tools/search_gate.py | 21 +++- wiki/testing-guide.md | 2 +- 4 files changed, 246 insertions(+), 8 deletions(-) create mode 100644 tests/test_site_r3_nav.py diff --git a/tests/test_site_funnel.py b/tests/test_site_funnel.py index 2a58bfa4..ee4a1bb7 100644 --- a/tests/test_site_funnel.py +++ b/tests/test_site_funnel.py @@ -209,10 +209,15 @@ class DeepLinksCarryALabel(unittest.TestCase): if level not in ("start", "setup"): continue text = FENCED.sub("", page.read_text(encoding="utf-8")) - heading = "" + # The headings the line sits under, by depth: a system's h3 sits + # in the "## Reference: every file by system" section. + headings: dict[int, str] = {} for line in text.splitlines(): - if re.match(r"#{1,3} ", line): - heading = line.lstrip("#").strip() + level_match = re.match(r"(#{1,3}) ", line) + if level_match: + depth = len(level_match[1]) + headings = {d: h for d, h in headings.items() if d < depth} + headings[depth] = line.lstrip("#").strip() for match in LINK.finditer(line): target = by_url.get(match.group(1)) if target not in levels: @@ -221,7 +226,8 @@ class DeepLinksCarryALabel(unittest.TestCase): if ORDER.index(deeper) - ORDER.index(level) < 2: continue label = f"{generate_site.LEVELS[deeper]}:" - if label not in line[: match.start()] and not heading.startswith(label): + if label not in line[: match.start()] and not any( + heading.startswith(label) for heading in headings.values()): offenders.append(f"{page.relative_to(DOCS)}: {match.group(0)}") self.assertEqual(offenders, [], "\n".join(offenders[:30])) diff --git a/tests/test_site_r3_nav.py b/tests/test_site_r3_nav.py new file mode 100644 index 00000000..9b9890c3 --- /dev/null +++ b/tests/test_site_r3_nav.py @@ -0,0 +1,217 @@ +"""Navigation and search (round 3, package 3). + +"playstation bios" found five setups' lists for the console before its own +page: the pages a setup's list moves a console to weighed 1.5, three times a +reference page. The file names a system page holds were the text of a hidden +span, so the search showed them as the excerpt. Start here looped: First +install's "Next steps" pointed back to Get your files, Is it legal? to Common +questions, and both pages had the same page as previous and next. A part +page had no crumb to the page it belongs to and nothing marked in the +sidebar. "psx" or "dc" found nothing, though every platform files the +console under that name. These tests read the generator and, where they +are built, the pages and the site. +""" + +from __future__ import annotations + +import ast +import re +import sys +import unittest +from html import unescape +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site # noqa: E402 +import sitestarlight # noqa: E402 +from common import list_registered_platforms # noqa: E402 + +SEARCH_GATE = REPO_ROOT / "website" / "tools" / "search_gate.py" +DOCS = REPO_ROOT / "website" / "src" / "content" / "docs" +DIST = REPO_ROOT / "website" / "dist" +ROUTE = REPO_ROOT / "website" / "src" / "routeData.ts" +CONTENT_CONFIG = REPO_ROOT / "website" / "src" / "content.config.ts" +START_HERE = [ + "/retrobios/", "/retrobios/wiki/what-are-bios-files/", "/retrobios/which-pack/", + "/retrobios/wiki/first-install/", "/retrobios/wiki/troubleshooting/", + "/retrobios/wiki/faq/", "/retrobios/wiki/legal/", "/retrobios/wiki/glossary/", + "/retrobios/platforms/", +] + + +def _front(text: str) -> str: + end = text.find("\n---\n", 4) + return text[4:end] if text.startswith("---\n") and end != -1 else "" + + +def _moved_platform_pages() -> list[Path]: + platforms = set(list_registered_platforms(str(REPO_ROOT / "platforms"), include_archived=True)) + folder = DOCS / "platforms" + if not folder.is_dir(): + return [] + return [path for path in sorted(folder.glob("*.md*")) + if path.stem not in platforms and path.stem != "index"] + + +class MetaFields(unittest.TestCase): + def test_file_names_are_an_attribute_not_text(self) -> None: + body = ('

scph5501.bin

' + '

a,b.bin

') + meta = sitestarlight.file_names_meta(body) + self.assertIn('data-pagefind-meta="files:a b.bin scph5501.bin"', meta) + self.assertRegex(meta, r"]*hidden>") + + def test_a_field_escapes_its_value(self) -> None: + meta = sitestarlight.search_field("aliases", ['a"b', "c&d"]) + self.assertIn('data-pagefind-meta="aliases:a"b c&d"', meta) + + def test_an_empty_field_writes_nothing(self) -> None: + self.assertEqual(sitestarlight.search_field("aliases", []), "") + self.assertEqual(sitestarlight.file_names_meta("

no card

"), "") + + def test_system_pages_carry_the_platforms_names(self) -> None: + page = DOCS / "systems" / "sony-playstation.md" + if not page.is_file(): + self.skipTest("site pages not generated") + text = page.read_text(encoding="utf-8") + match = re.search(r'data-pagefind-meta="aliases:([^"]*)"', text) + self.assertIsNotNone(match, "no aliases field on the PlayStation page") + self.assertIn("psx", unescape(match[1]).split()) + dreamcast = (DOCS / "systems" / "sega-dreamcast.md").read_text(encoding="utf-8") + self.assertRegex(dreamcast, r'data-pagefind-meta="aliases:[^"]*\bdc\b') + + +class MovedPlatformPages(unittest.TestCase): + def test_a_setups_list_for_one_console_is_a_reference(self) -> None: + source = Path(generate_site.__file__).read_text(encoding="utf-8") + block = source[source.index("for stem, page in moved.items():"):] + block = block[: block.index("PAGE_DATA[f\"platforms/{name}.md\"]")] + self.assertIn('SEARCH_DEMOTED.add(f"platforms/{stem}.md")', block) + self.assertIn('level="reference"', block) + self.assertIn('search="exclude"', block) + self.assertIn("parent=", block) + + def test_generated_pages_weigh_as_references_and_name_their_setup(self) -> None: + pages = _moved_platform_pages() + if not pages: + self.skipTest("site pages not generated") + for path in pages: + front = _front(path.read_text(encoding="utf-8")) + with self.subTest(page=path.name): + self.assertIn("\nlevel: reference\n", f"\n{front}\n") + self.assertIn("\nsearchWeight: 0.5\n", f"\n{front}\n") + # The title alone was indexed, and it outranked the console. + self.assertIn("\npagefind: false\n", f"\n{front}\n") + self.assertRegex(front, r"parent:\n label: .+\n link: /retrobios/platforms/[a-z0-9_-]+/\n") + + def test_parent_link_resolves_against_the_page(self) -> None: + _target, text = sitestarlight.finalize_page( + "platforms/retroarch-sony-playstation.md", "# RetroArch: Sony PlayStation\n\nBody.\n", + level="reference", weight=0.5, browser_title="x", description="d", + structured={}, terms={}, parent={"label": "RetroArch", "link": "retroarch.md"}, + ) + self.assertIn("parent:\n label: RetroArch\n link: /retrobios/platforms/retroarch/\n", text) + + def test_the_schema_accepts_parent(self) -> None: + self.assertRegex(CONTENT_CONFIG.read_text(encoding="utf-8"), + r"parent: z\.object\(\{ label: z\.string\(\), link: z\.string\(\) \}\)") + + +class ArcadeGroup(unittest.TestCase): + def test_arcade_is_not_a_console_or_computer(self) -> None: + nav = generate_site._system_nav({"Sony": {}, "Arcade": {}, "Sega": {}}) + groups = {next(iter(entry)): entry[next(iter(entry))] for entry in nav if isinstance(entry, dict)} + self.assertEqual(groups["Arcade"], [{"Arcade": "systems/arcade.md"}]) + self.assertNotIn({"Arcade": "systems/arcade.md"}, groups["Consoles and computers"]) + + +class RouteMiddleware(unittest.TestCase): + def test_next_steps_link_is_kept_only_forward(self) -> None: + source = ROUTE.read_text(encoding="utf-8") + self.assertIn("target <= position", source) + + def test_parent_is_a_crumb_and_marked(self) -> None: + source = ROUTE.read_text(encoding="utf-8") + self.assertIn("route.entry.data.parent", source) + self.assertIn("markCurrent(route.sidebar", source) + + +def _gate_queries() -> dict: + """The search gate's QUERIES, read from its source: the tool is run + with node beside it, not imported as a package.""" + tree = ast.parse(SEARCH_GATE.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.AnnAssign) and getattr(node.target, "id", "") == "QUERIES": + return ast.literal_eval(node.value) + raise AssertionError("no QUERIES in search_gate.py") + + +class SearchGateQueries(unittest.TestCase): + def test_the_audits_queries_are_gated(self) -> None: + for query in ("playstation bios", "dreamcast bios", "neo geo bios", "bios not found", + "offline install", "r36s", "wonderswan color", "dc bios missing", + "which mame version", "psx bios"): + self.assertIn(query, _gate_queries()) + + def test_prefixes_match_a_consoles_own_page(self) -> None: + pages, _rank = _gate_queries()["playstation bios"] + self.assertTrue(any("systems/sony-playstation/".startswith(p) for p in pages)) + + +def _html(url: str) -> str: + rel = url[len("/retrobios/"):] + path = DIST / rel / "index.html" + return path.read_text(encoding="utf-8") if path.is_file() else "" + + +def _pagination(text: str) -> tuple[str | None, str | None]: + start = text.find('