diff --git a/scripts/generate_site.py b/scripts/generate_site.py index f60c3d43..f1b1368d 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -31,7 +31,7 @@ import sys import urllib.error import urllib.parse import urllib.request -from collections.abc import Callable +from collections.abc import Callable, Iterator from datetime import datetime, timezone from pathlib import Path @@ -6340,10 +6340,6 @@ def generate_which_pack() -> str: return "\n".join(lines) + "\n" -# Letters each alphabetical group of the emulators navigation covers. -EMULATOR_NAV_GROUPS = ("ABC", "DEF", "GHIJKL", "MNO", "PQR", "STU", "VWXYZ") - - def nav_emulators(profiles: dict) -> dict[str, dict]: """The emulators listed in the navigation, by key. @@ -6359,34 +6355,20 @@ def nav_emulators(profiles: dict) -> dict[str, dict]: def _emulator_nav(profiles: dict) -> list: - """Emulators in alphabetical groups ("A to C"), then by initial. - - Material writes the open section of the navigation into every page: a - group of 96 emulators put 65 KB of links before each emulator page's - content. One initial is at most 60. - """ - groups: dict[str, dict[str, list[tuple[str, str]]]] = {} + """Emulators by initial (A, B, C), one level deep. A name that does not + open on a letter goes under "0 to 9". The route middleware draws every + initial but the open one as a single link, so a page carries one + initial's links, at most 60.""" + groups: dict[str, list[tuple[str, str]]] = {} for key, profile in nav_emulators(profiles).items(): display = str(profile.get("emulator", key)) initial = display[:1].upper() - letters = next((g for g in EMULATOR_NAV_GROUPS if initial in g), "") - label = f"{letters[0]} to {letters[-1]}" if letters else "0 to 9" - sub = initial if letters else label - groups.setdefault(label, {}).setdefault(sub, []).append((display, f"emulators/{key}.md")) - - def entries(items: list[tuple[str, str]]) -> list[dict]: - return [{d: path} for d, path in sorted(items, key=lambda e: e[0].casefold())] - - order = ["0 to 9"] + [f"{g[0]} to {g[-1]}" for g in EMULATOR_NAV_GROUPS] + label = initial if "A" <= initial <= "Z" else "0 to 9" + groups.setdefault(label, []).append((display, f"emulators/{key}.md")) nav: list = ["emulators/index.md"] - for label in order: - if label not in groups: - continue - subs = groups[label] - if len(subs) == 1: - nav.append({label: entries(next(iter(subs.values())))}) - else: - nav.append({label: [{sub: entries(subs[sub])} for sub in sorted(subs)]}) + for label in sorted(groups, key=lambda g: (g != "0 to 9", g)): + items = sorted(groups[label], key=lambda e: e[0].casefold()) + nav.append({label: [{display: path} for display, path in items]}) return nav @@ -6435,16 +6417,20 @@ def _h1(markdown: str) -> str: return match.group(1).strip() if match else "" -def wiki_nav(wiki_dir: Path | None = None) -> dict[str, list[tuple[int, object]]]: +NavPlace = tuple[int, object, str] + + +def wiki_nav(wiki_dir: Path | None = None) -> dict[str, list[NavPlace]]: """Wiki pages by tab, each at the place its front matter gives. A page states `level:` (its tab), `order:` (its place in the tab, the smallest first) and, when the navigation names it otherwise than its - first heading, `nav_title:`. `nav: false` keeps a page out on purpose. - A page that says neither raises, naming what to add. + first heading, `nav_title:`. `group:` files it under a labelled group + of its tab. `nav: false` keeps a page out on purpose. A page that says + neither raises, naming what to add. """ wiki_dir = Path(wiki_dir) if wiki_dir else _REPO_ROOT / WIKI_SRC_DIR - tabs: dict[str, list[tuple[int, object]]] = {level: [] for level in TABS} + tabs: dict[str, list[NavPlace]] = {level: [] for level in TABS} for page in sorted(wiki_dir.glob("*.md")): where = f"{WIKI_SRC_DIR}/{page.name}" text = page.read_text(encoding="utf-8") @@ -6466,15 +6452,42 @@ def wiki_nav(wiki_dir: Path | None = None) -> dict[str, list[tuple[int, object]] ) path = f"{WIKI_SRC_DIR}/{page.name}" if page.name == "index.md": - tabs[level].append((order, path)) + tabs[level].append((order, path, "")) continue title = meta.get("nav_title") or _h1(text) if not title: raise NavError(f"{where}: no `# ` heading and no `nav_title:` to name it") - tabs[level].append((order, {str(title): path})) + group = meta.get("group") or "" + if not isinstance(group, str): + raise NavError(f"{where}: `group:` names a group of the {TABS[level]} tab (found {group!r})") + tabs[level].append((order, {str(title): path}, group)) return tabs +def _overview_anchors(nav: list, docs: Path) -> dict[str, set[str]]: + """The heading ids of each overview page the navigation names, where a + group of the sidebar finds its own section.""" + def overviews(value: object) -> Iterator[str]: + if isinstance(value, str): + yield value + elif isinstance(value, list): + for entry in value: + yield from overviews(entry) + elif isinstance(value, dict): + for child in value.values(): + if isinstance(child, list): + yield from overviews(child) + + anchors: dict[str, set[str]] = {} + for rel in overviews(nav): + stem = rel.rsplit(".", 1)[0] + page = next((docs / f"{stem}{suffix}" for suffix in (".md", ".mdx") + if (docs / f"{stem}{suffix}").is_file()), None) + if page: + anchors[rel] = sitestarlight.heading_ids(page.read_text(encoding="utf-8")) + return anchors + + def _nav_target(entry: object) -> str: return entry if isinstance(entry, str) else str(next(iter(entry.values()))) @@ -6495,8 +6508,10 @@ def generate_nav( The generated pages hold the places below. A wiki page takes its own place from its front matter (wiki_nav), so a new page needs no edit here. + A group of a tab sits at the place of its first page, its pages in their + own order. """ - generated: dict[str, list[tuple[int, object]]] = { + generated: dict[str, list[tuple]] = { "start": [(0, "index.md"), (20, {"Get your files": "which-pack.md"})], "setup": [(10 * i, item) for i, item in enumerate(_platform_nav(coverages, registry or {}))], @@ -6506,27 +6521,40 @@ def generate_nav( (20, {"Emulators": _emulator_nav(profiles)}), ], "reference": [ - (20, {"What the packs do not hold": "gaps.md"}), - (30, {"Cores by platform": "cross-reference.md"}), - (40, {"Dump provenance": "provenance.md"}), - (50, {"Updated platform files": "exports.md"}), - (60, {"Data and API": "data.md"}), - (70, {"Find a file": "files/index.md"}), - (80, {"Data model": "wiki/data-model.md"}), + # Find a file first: the one tool a reader comes to with a file in hand. + (5, {"Find a file": "files/index.md"}, "Checking files"), + (20, {"What the packs do not hold": "gaps.md"}, "Checking files"), + (30, {"Cores by platform": "cross-reference.md"}, "Checking files"), + (40, {"Dump provenance": "provenance.md"}, "Checking files"), + (60, {"Data and API": "data.md"}, "Data"), + (64, {"Data model": "wiki/data-model.md"}, "Data"), + (68, {"Updated platform files": "exports.md"}, "Data"), ], "contribute": [(0, "contributing.md")], } wiki = wiki_nav(wiki_dir) nav = [] for level, label in TABS.items(): - entries = sorted(generated[level] + wiki[level], key=lambda item: item[0]) - for (first, a), (second, b) in zip(entries, entries[1:]): + places = [(place[0], place[1], place[2] if len(place) > 2 else "") + for place in generated[level]] + entries = sorted(places + wiki[level], key=lambda item: item[0]) + for (first, a, _), (second, b, _) in zip(entries, entries[1:]): if first == second: raise NavError( f"{label} tab: {_nav_target(a)} and {_nav_target(b)} both take " f"order {first}; give one of them a free number" ) - nav.append({label: [entry for _order, entry in entries]}) + items: list = [] + groups: dict[str, list] = {} + for _order, entry, group in entries: + if not group: + items.append(entry) + elif group in groups: + groups[group].append(entry) + else: + groups[group] = [entry] + items.append({group: groups[group]}) + nav.append({label: items}) return nav @@ -6859,7 +6887,11 @@ def _generate(args: argparse.Namespace) -> None: generated = site / "src" / "generated" generated.mkdir(parents=True, exist_ok=True) write_if_changed(str(generated / "sidebar.json"), - json.dumps(sitestarlight.sidebar(nav, present), indent=1) + "\n") + json.dumps(sitestarlight.sidebar(nav, present, _overview_anchors(nav, docs)), + indent=1) + "\n") + write_if_changed(str(generated / "redirects.json"), + json.dumps(sitestarlight.redirects(profiles, present), indent=1, + sort_keys=True) + "\n") write_if_changed(str(generated / "lastmod.json"), json.dumps(dict(sorted(lastmod.items())), indent=1) + "\n") system_ids = set(_system_page_map) | { diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py index 3623299b..94bba427 100644 --- a/scripts/sitestarlight.py +++ b/scripts/sitestarlight.py @@ -311,14 +311,33 @@ def og_section(rel: str, level: str) -> str: return "reference" +# Sharing cards of their own, made by scripts/make_og_cards.py: one per +# platform page, titled with the platform's name. +OG_DIR = Path(__file__).resolve().parent.parent / "docs_assets" / "og" + + +def og_image(rel: str, level: str) -> str: + """The sharing card's address: the page's own when it has one, else + its section's.""" + own = re.fullmatch(r"platforms/([a-z0-9_-]+)\.md", rel) + if own and own[1] != "index" and (OG_DIR / "platforms" / f"{own[1]}.png").is_file(): + return f"{SITE_URL}assets/og/platforms/{own[1]}.png" + return f"{SITE_URL}assets/og/{og_section(rel, level)}.png" + + +NOT_FOUND = "404.md" + + def page_head(rel: str, level: str, title: str, browser_title: str, structured: dict) -> list[dict]: - """What a page adds to its head: its own window title, the sharing - card of its section, and its structured data.""" - section = og_section(rel, level) - image = f"{SITE_URL}assets/og/{section}.png" + """What a page adds to its head: its own window title, its sharing + card (og_image), and its structured data. The page a stale link + lands on asks not to be indexed and describes nothing.""" head: list[dict] = [] if browser_title != title: head.append({"tag": "title", "content": f"{browser_title} - RetroBIOS"}) + if rel == NOT_FOUND: + return head + [{"tag": "meta", "attrs": {"name": "robots", "content": "noindex"}}] + image = og_image(rel, level) head += [ {"tag": "meta", "attrs": {"property": "og:image", "content": image}}, {"tag": "meta", "attrs": {"property": "og:image:width", "content": "1200"}}, @@ -361,6 +380,21 @@ def file_names_meta(body: str) -> str: return f'\n\n' +_NEXT_STEPS = re.compile(r"^## Next steps\b[^\n]*\n+(?P(?:[ \t]*[-*] .*\n?)+)", re.M) + + +def next_step(body: str) -> dict | None: + """The first link of the page's "Next steps" list, as the theme's + `next:` front matter.""" + match = _NEXT_STEPS.search(body) + if not match: + return None + link = MD_LINK.search(match["list"]) + if not link or link["bang"]: + return None + return {"link": link["url"], "label": re.sub(r"[*_`]", "", link["text"]).strip()} + + def finalize_page( rel: str, markdown: str, @@ -414,7 +448,9 @@ def finalize_page( front["level"] = level if weight: front["searchWeight"] = weight - if rel.startswith(CARD_PAGES): + if rel == NOT_FOUND: + front["tableOfContents"] = False + elif rel.startswith(CARD_PAGES): sections = len(re.findall(r"^## ", body, re.M)) # A page of cards under one or two headings has nothing to list, and # the table of contents watches every block of the page to follow @@ -423,6 +459,13 @@ def finalize_page( {"minHeadingLevel": 2, "maxHeadingLevel": 2} if sections > 2 else False) if not search: front["pagefind"] = False + step = next_step(body) + if rel == NOT_FOUND or not reads_in_order(rel): + front["prev"] = False + front["next"] = False + elif step: + # The page's own "Next steps" list and the next link agree. + front["next"] = step front["head"] = page_head(rel, level, front["title"], browser_title, structured) if structured.get("dateModified"): front["lastmod"] = str(structured["dateModified"]) @@ -436,15 +479,62 @@ def _sidebar_link(rel: str) -> str: return page_url(rel)[len(BASE) - 1:] -def sidebar(nav: list, present: set[str]) -> dict: - """The navigation as Starlight topics, one per tab. +# Tabs whose groups hold pages by the hundred (every system, every +# emulator): a group that does not hold the current page is drawn as one +# link to its overview, so an emulator page does not carry the links of +# every other emulator. +PRUNED_TABS = ("explore",) + +# Reading runs from one tab into the next only here: the last Start here +# page leads to Your setup. Every other tab is read on its own. +CHAINED_TABS = (("start-here", "your-setup"),) + + +def topic_id(label: str) -> str: + return re.sub(r"[^a-z]+", "-", label.lower()).strip("-") + + +def heading_ids(text: str) -> set[str]: + """The anchors a page's headings carry, as written by pin_heading_ids.""" + ids = {match[1] for match in re.finditer(r"\{[^{}\n]*#([^\s}]+)[^{}\n]*\}[ \t]*$", text, re.M)} + return ids | {match["id"] for match in ID_ATTR.finditer(text)} + + +def reads_in_order(rel: str) -> bool: + """A page the previous and next links pass through. A system, emulator + or platform page is reached by its name, not read after its neighbour + in the alphabet: "SNK, Sony, Sord" is no reading order.""" + return not rel.startswith(CARD_PAGES) or rel.endswith("/index.md") + + +def sidebar(nav: list, present: set[str], anchors: dict[str, set[str]] | None = None) -> dict: + """The navigation as Starlight topics, one per tab, with what the route + middleware (website/src/routeData.ts) reads beside it. A page the navigation does not list (the parts of a split page) joins the topic that holds most pages of its directory. - """ - owner: dict[str, str] = {} - def item(entry: object, topic: str) -> dict | None: + `groups` gives each group, keyed by the labels from its tab down, the + page that introduces it: its own overview page, or the heading named + after it on the overview above it (`anchors` lists each page's heading + ids). A group with neither has no address, never the address of its + first page, which is a sibling of the page being read. `prune` marks the + groups drawn as one link when they do not hold the current page. + + `reading` is the order of the previous and next links, one list per run + of tabs read in sequence. + """ + anchors = anchors or {} + owner: dict[str, str] = {} + groups: dict[str, dict] = {} + + def label_of(entry: object) -> str: + return next(iter(entry)) if isinstance(entry, dict) else "" + + def overview_of(value: list) -> str | None: + return next((v for v in value if isinstance(v, str)), None) + + def item(entry: object, topic: str, trail: tuple[str, ...], parent: str | None) -> dict | None: if isinstance(entry, str): owner[entry] = topic return {"label": "Overview", "link": _sidebar_link(entry)} if entry in present else None @@ -454,14 +544,33 @@ def sidebar(nav: list, present: set[str]) -> dict: return {"label": label, "link": value} owner[value] = topic return {"label": label, "link": _sidebar_link(value)} if value in present else None - children = [child for child in (item(v, topic) for v in value) if child] - return {"label": label, "items": children, "collapsed": True} if children else None + if "/" in label: + raise ValueError(f"a navigation group label holds a slash: {label!r}") + own = overview_of(value) + here = (*trail, label) + href = None + if own and own in present: + href = page_url(own) + elif parent and parent in present: + anchor = toc_slug(label) + if anchor in anchors.get(parent, set()): + href = f"{page_url(parent)}#{anchor}" + elif topic in PRUNED_TABS: + # Drawn as a link, it needs an address: the overview it + # sits under, never one of its own pages. + href = page_url(parent) + children = [child for child in (item(v, topic, here, own or parent) for v in value) if child] + if not children: + return None + groups["/".join(here)] = {"href": href, "prune": topic in PRUNED_TABS and href is not None} + return {"label": label, "items": children, "collapsed": True} topics = [] for tab in nav: (label, value), = tab.items() - topic = re.sub(r"[^a-z]+", "-", label.lower()).strip("-") - children = [child for child in (item(v, topic) for v in value) if child] + topic = topic_id(label) + children = [child for child in (item(v, topic, (label,), overview_of(value)) for v in value) + if child] if children: topics.append({"id": topic, "label": label, "link": next(_links(children))["link"], "items": children}) @@ -470,9 +579,11 @@ def sidebar(nav: list, present: set[str]) -> dict: for rel, topic in owner.items(): tally = votes.setdefault(rel.rpartition("/")[0], {}) tally[topic] = tally.get(topic, 0) + 1 - extra: dict[str, list[str]] = {} - # The page a stale link lands on belongs to no tab. - exclude: list[str] = ["/404"] + # The page a stale link lands on opens on the first tab: its sidebar + # is the way back in. Excluded from every tab, the theme named the tabs + # by their position, 0 to 4. + extra: dict[str, list[str]] = {topics[0]["id"]: ["/404"]} if topics else {} + exclude: list[str] = [] for rel in sorted(present - owner.keys()): tally = votes.get(rel.rpartition("/")[0]) path = "/" + (_sidebar_link(rel).strip("/") or "index") @@ -480,7 +591,70 @@ def sidebar(nav: list, present: set[str]) -> dict: extra.setdefault(max(tally, key=tally.get), []).append(path) else: exclude.append(path) - return {"topics": topics, "options": {"topics": extra, "exclude": exclude}} + return {"topics": topics, "options": {"topics": extra, "exclude": exclude}, + "groups": groups, "reading": _reading(nav, present)} + + +def _reading(nav: list, present: set[str]) -> list[list[dict]]: + """The previous and next links, in runs. An overview is named after the + group or tab it opens ("Your setup", not "Overview").""" + def walk(value: list, name: str) -> Iterator[dict]: + for entry in value: + if isinstance(entry, str): + if entry in present and reads_in_order(entry): + yield {"label": name, "link": page_url(entry)} + continue + (label, target), = entry.items() + if isinstance(target, list): + yield from walk(target, label) + elif not target.startswith("http") and target in present and reads_in_order(target): + yield {"label": label, "link": page_url(target)} + + tabs = {topic_id(label): list(walk(value, label)) + for tab in nav for label, value in tab.items()} + runs: list[list[dict]] = [] + previous = "" + for tab, pages in tabs.items(): + if runs and (previous, tab) in CHAINED_TABS: + runs[-1].extend(pages) + else: + runs.append(list(pages)) + previous = tab + return [run for run in runs if run] + + +# Addresses that once held a page, and where its content went. +MOVED_PAGES = {"wiki/getting-started.md": "which-pack.md"} + +_CORE_NAME = re.compile(r"[a-z0-9_-]+") + + +def redirects(profiles: dict, present: set[str]) -> dict[str, str]: + """Old and alternative addresses, for src/generated/redirects.json. + + A page that moved, once its source is gone, and each name a profile's + `cores:` gives the emulator (the buildbot's `mednafen_psx` for + beetle_psx), to the profile's page. A name two profiles claim, or one + that is already a page's address, is left out. Addresses are written + under the site root, without its base.""" + def address(rel: str) -> str: + return "/" + page_url(rel)[len(BASE):] + + out = {address(old): address(new) for old, new in MOVED_PAGES.items() + if old not in present and new in present} + pages = {rel.casefold() for rel in present} + claims: dict[str, set[str]] = {} + for key, profile in profiles.items(): + if f"emulators/{key}.md" not in present: + continue + for name in profile.get("cores") or []: + name = str(name) + if _CORE_NAME.fullmatch(name) and name != key: + claims.setdefault(name, set()).add(key) + for name, keys in sorted(claims.items()): + if len(keys) == 1 and f"emulators/{name}.md".casefold() not in pages: + out[address(f"emulators/{name}.md")] = address(f"emulators/{next(iter(keys))}.md") + return out def _links(items: list[dict]) -> Iterator[dict]: diff --git a/tests/test_site_funnel.py b/tests/test_site_funnel.py index bf2234a1..491661f6 100644 --- a/tests/test_site_funnel.py +++ b/tests/test_site_funnel.py @@ -155,8 +155,8 @@ class SampleCoresLeaveTheNavigation(unittest.TestCase): self.assertEqual(nav[0], "emulators/index.md") groups = {next(iter(group)): next(iter(group.values())) for group in nav[1:]} self.assertEqual(groups, { - "D to F": [{"DuckStation": "emulators/duckstation.md"}], - "V to Z": [{"zx81": "emulators/zx.md"}], + "D": [{"DuckStation": "emulators/duckstation.md"}], + "Z": [{"zx81": "emulators/zx.md"}], }) diff --git a/tests/test_site_r2_nav.py b/tests/test_site_r2_nav.py new file mode 100644 index 00000000..e82f7bc3 --- /dev/null +++ b/tests/test_site_r2_nav.py @@ -0,0 +1,364 @@ +"""Navigation (round 2, package 3). + +The 404 page's sidebar read "0 1 2 3 4". A sidebar group label was a link +to its first page ("Handhelds and TV boxes" opened AmberELEC), and a crumb +linked a sibling page. Emulators sat three levels deep under "D to F / D". +Previous and next ran from Sony to Sord and from one tab into another, +and disagreed with the page's own "Next" list. The Reference tab was a flat +list of fifteen. These tests read the generator and, where it is built, +the site. +""" + +from __future__ import annotations + +import json +import re +import struct +import sys +import tempfile +import unittest +from html.parser import HTMLParser +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site # noqa: E402 +import sitestarlight # noqa: E402 +from common import list_registered_platforms, load_platform_config, load_platform_registry # noqa: E402 + +DIST = REPO_ROOT / "website" / "dist" +ROUTE = REPO_ROOT / "website" / "src" / "routeData.ts" +TAB_ROOTS = {"/retrobios/", "/retrobios/platforms/", "/retrobios/explore/", + "/retrobios/wiki/", "/retrobios/contributing/"} + + +def _nav() -> list: + platforms = REPO_ROOT / "platforms" + registry = load_platform_registry(str(platforms)) + coverages = { + key: {"platform": load_platform_config(key, str(platforms))["platform"]} + for key in list_registered_platforms(str(platforms), include_archived=True) + } + manufacturers = {"Sony": {}, "Sega": {}, "ScummVM": {}} + profiles = { + "duckstation": {"emulator": "DuckStation", "systems": ["sony-playstation"]}, + "dolphin": {"emulator": "Dolphin", "systems": ["nintendo-gamecube"]}, + "mame": {"emulator": "MAME", "systems": ["arcade"]}, + } + return generate_site.generate_nav(coverages, manufacturers, profiles, registry) + + +def _targets(value: object) -> set[str]: + if isinstance(value, str): + return {value} + if isinstance(value, list): + return set().union(*(_targets(v) for v in value)) if value else set() + return set().union(*(_targets(v) for v in value.values())) + + +def _present(nav: list) -> set[str]: + return {t for t in _targets(nav) if not t.startswith("http")} + + +ANCHORS = { + "platforms/index.md": {"computers-and-steam-deck", "handhelds-and-tv-boxes", + "game-systems-for-pcs-and-small-boards", "mister-fpga", + "self-hosted-game-library"}, + "systems/index.md": {"consoles-and-computers"}, +} + + +class NotFoundPage(unittest.TestCase): + def test_the_404_opens_on_the_first_tab(self): + side = sitestarlight.sidebar(_nav(), _present(_nav()), ANCHORS) + self.assertNotIn("/404", side["options"]["exclude"]) + self.assertIn("/404", side["options"]["topics"]["start-here"]) + + def test_the_404_has_no_contents_no_index_and_no_structured_data(self): + _target, text = sitestarlight.finalize_page( + "404.md", "# Page not found\n\nGone.\n", level="", weight=None, + browser_title="Page not found", description="d", + structured={"@type": "TechArticle"}, terms={}) + front = generate_site._front_matter(text) + self.assertIs(front["tableOfContents"], False) + self.assertIs(front["prev"], False) + self.assertIs(front["next"], False) + self.assertIn({"tag": "meta", "attrs": {"name": "robots", "content": "noindex"}}, front["head"]) + self.assertNotIn("ld+json", text) + + +class GroupsAreGroups(unittest.TestCase): + def setUp(self): + nav = _nav() + self.side = sitestarlight.sidebar(nav, _present(nav), ANCHORS) + self.groups = self.side["groups"] + + def test_setup_groups_stay_groups_and_point_at_their_section(self): + setup = {key: value for key, value in self.groups.items() if key.startswith("Your setup/")} + self.assertTrue(setup) + for key, group in setup.items(): + with self.subTest(group=key): + self.assertFalse(group["prune"]) + self.assertTrue(group["href"].startswith("/retrobios/platforms/#"), group["href"]) + + def test_explore_groups_collapse_to_their_overview(self): + explore = {key: value for key, value in self.groups.items() if key.startswith("Explore/")} + self.assertEqual(explore["Explore/Systems"], {"href": "/retrobios/systems/", "prune": True}) + self.assertEqual(explore["Explore/Systems/Consoles and computers"]["href"], + "/retrobios/systems/#consoles-and-computers") + self.assertEqual(explore["Explore/Emulators/D"], {"href": "/retrobios/emulators/", "prune": True}) + + def test_no_group_points_at_one_of_its_own_pages(self): + """The overview a group opens on is its own index page or a section of + the page above it, never the first page it lists.""" + overview_pages = {sitestarlight.page_url(rel) for rel in _present(_nav()) + if rel == "index.md" or rel.endswith("/index.md") or rel == "explore.md"} + for key, group in self.groups.items(): + if not group["href"]: + continue + with self.subTest(group=key): + self.assertTrue("#" in group["href"] or group["href"] in overview_pages, group["href"]) + + def test_the_middleware_reads_the_groups_not_the_first_link(self): + route = ROUTE.read_text(encoding="utf-8") + self.assertNotIn("firstLink", route) + self.assertIn("sidebar.groups", route) + self.assertIn("sidebar.reading", route) + + +class EmulatorsOneLevelDeep(unittest.TestCase): + def test_initials_hold_pages_not_groups(self): + nav = generate_site._emulator_nav({ + "duckstation": {"emulator": "DuckStation", "systems": ["s"]}, + "dolphin": {"emulator": "Dolphin", "systems": ["s"]}, + }) + self.assertEqual(nav, ["emulators/index.md", {"D": [ + {"Dolphin": "emulators/dolphin.md"}, {"DuckStation": "emulators/duckstation.md"}]}]) + + def test_no_initial_holds_more_than_sixty(self): + from common import load_emulator_profiles + + nav = generate_site._emulator_nav(load_emulator_profiles(str(REPO_ROOT / "emulators"))) + for group in nav[1:]: + (label, pages), = group.items() + with self.subTest(initial=label): + self.assertTrue(all(isinstance(next(iter(p.values())), str) for p in pages)) + self.assertLessEqual(len(pages), 60) + + +class OneNextPerPage(unittest.TestCase): + def setUp(self): + nav = _nav() + self.runs = sitestarlight.sidebar(nav, _present(nav), ANCHORS)["reading"] + self.links = [[entry["link"] for entry in run] for run in self.runs] + + def test_start_here_runs_into_your_setup_and_stops(self): + first = self.links[0] + self.assertEqual(first[0], "/retrobios/") + self.assertIn("/retrobios/platforms/", first) + self.assertNotIn("/retrobios/explore/", first) + + def test_each_other_tab_is_read_on_its_own(self): + starts = [run[0] for run in self.links] + self.assertEqual(starts, ["/retrobios/", "/retrobios/explore/", "/retrobios/wiki/", + "/retrobios/contributing/"]) + + def test_system_emulator_and_platform_pages_are_not_in_the_reading(self): + for link in (link for run in self.links for link in run): + with self.subTest(link=link): + self.assertNotRegex(link, r"^/retrobios/(systems|emulators|platforms)/[^/]+/$") + + def test_an_overview_is_named_after_what_it_opens(self): + labels = {entry["link"]: entry["label"] for run in self.runs for entry in run} + self.assertEqual(labels["/retrobios/platforms/"], "Your setup") + self.assertEqual(labels["/retrobios/systems/"], "Systems") + self.assertNotIn("Overview", labels.values()) + + def test_card_pages_turn_pagination_off(self): + for rel in ("systems/sony.md", "emulators/duckstation.md", "platforms/batocera.md"): + _target, text = sitestarlight.finalize_page( + rel, "# T\n\nBody.\n", level="explore", weight=None, browser_title="T", + description="d", structured={}, terms={}) + front = generate_site._front_matter(text) + with self.subTest(page=rel): + self.assertIs(front["prev"], False) + self.assertIs(front["next"], False) + + def test_the_next_link_is_the_first_of_next_steps(self): + body = ("# Troubleshooting\n\nText.\n\n## Next steps\n\n" + "- [Common questions](faq.md)\n- [Glossary](glossary.md)\n") + _target, text = sitestarlight.finalize_page( + "wiki/troubleshooting.md", body, level="start", weight=None, + browser_title="Troubleshooting", description="d", structured={}, terms={}) + front = generate_site._front_matter(text) + self.assertEqual(front["next"], {"link": "/retrobios/wiki/faq/", "label": "Common questions"}) + + def test_a_page_without_next_steps_keeps_the_reading_order(self): + _target, text = sitestarlight.finalize_page( + "wiki/faq.md", "# FAQ\n\nText.\n", level="start", weight=None, + browser_title="FAQ", description="d", structured={}, terms={}) + self.assertNotIn("next", generate_site._front_matter(text)) + + +class ReferenceGrouped(unittest.TestCase): + def _reference(self, wiki: Path | None = None) -> list: + nav = generate_site.generate_nav({}, {}, {}, {}, wiki) + return next(tab["Reference"] for tab in nav if "Reference" in tab) + + def test_checking_files_opens_on_find_a_file(self): + reference = self._reference() + groups = {next(iter(item)): next(iter(item.values())) for item in reference + if isinstance(item, dict) and isinstance(next(iter(item.values())), list)} + self.assertEqual(groups["Checking files"][0], {"Find a file": "files/index.md"}) + self.assertIn({"What the packs do not hold": "gaps.md"}, groups["Checking files"]) + self.assertEqual(groups["Data"][:3], [{"Data and API": "data.md"}, + {"Data model": "wiki/data-model.md"}, + {"Updated platform files": "exports.md"}]) + self.assertEqual(reference[0], "wiki/index.md") + + def test_a_wiki_page_joins_a_group_from_its_front_matter(self): + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + wiki = Path(tmp) + (wiki / "index.md").write_text("---\nlevel: reference\norder: 0\n---\n# Reference\n") + (wiki / "modes.md").write_text( + "---\nlevel: reference\norder: 10\ngroup: Checking files\n---\n# How each platform checks files\n") + reference = self._reference(wiki) + group = next(item["Checking files"] for item in reference + if isinstance(item, dict) and "Checking files" in item) + self.assertEqual(group[:2], [{"Find a file": "files/index.md"}, + {"How each platform checks files": "wiki/modes.md"}]) + + +class Redirects(unittest.TestCase): + def test_core_names_lead_to_their_profile(self): + profiles = { + "beetle_psx": {"cores": ["mednafen_psx", "beetle_psx", "Beetle PSX"]}, + "a": {"cores": ["shared"]}, + "b": {"cores": ["shared", "dolphin"]}, + "dolphin": {"cores": []}, + } + present = {"emulators/beetle_psx.md", "emulators/a.md", "emulators/b.md", + "emulators/dolphin.md", "which-pack.md"} + self.assertEqual(sitestarlight.redirects(profiles, present), { + "/emulators/mednafen_psx/": "/emulators/beetle_psx/", + "/wiki/getting-started/": "/which-pack/", + }) + + def test_a_moved_page_redirects_only_once_its_source_is_gone(self): + present = {"wiki/getting-started.md", "which-pack.md"} + self.assertEqual(sitestarlight.redirects({}, present), {}) + + def test_the_generator_writes_the_file(self): + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('generated / "redirects.json"', source) + + +class PlatformShareCards(unittest.TestCase): + def test_every_platform_page_shares_its_own_card(self): + for key in list_registered_platforms(str(REPO_ROOT / "platforms"), include_archived=True): + with self.subTest(platform=key): + card = REPO_ROOT / "docs_assets" / "og" / "platforms" / f"{key}.png" + data = card.read_bytes() + self.assertEqual(struct.unpack(">II", data[16:24]), (1200, 630)) + self.assertEqual(sitestarlight.og_image(f"platforms/{key}.md", "setup"), + f"{sitestarlight.SITE_URL}assets/og/platforms/{key}.png") + + def test_other_pages_keep_their_section_card(self): + self.assertEqual(sitestarlight.og_image("platforms/index.md", "setup"), + f"{sitestarlight.SITE_URL}assets/og/setup.png") + self.assertEqual(sitestarlight.og_image("systems/sony.md", "explore"), + f"{sitestarlight.SITE_URL}assets/og/explore.png") + + +# The built site + + +class _Page(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.where: list[str] = [] + self.sidebar: list[str] = [] + self.crumbs: list[str | None] = [] + self.pagination = False + self._sidebar = 0 + self._crumbs = 0 + self._text: list[str] | None = None + self._pagination = False + + def handle_starttag(self, tag: str, attrs: list) -> None: + a = dict(attrs) + if tag == "nav" and a.get("aria-label") == "Breadcrumb": + self._crumbs = 1 + elif self._crumbs and tag == "a": + self.crumbs.append(a.get("href")) + if "sidebar-content" in (a.get("class") or ""): + self._sidebar = 1 + elif self._sidebar and tag in ("a", "summary"): + self._text = [] + if "pagination-links" in (a.get("class") or ""): + self._pagination = True + elif self._pagination and tag == "a": + self.pagination = True + + def handle_endtag(self, tag: str) -> None: + if tag == "nav" and self._crumbs: + self._crumbs = 0 + if tag == "div" and self._pagination and not self.pagination: + self._pagination = False + if self._text is not None and tag in ("a", "summary"): + self.sidebar.append(" ".join("".join(self._text).split())) + self._text = None + + def handle_data(self, data: str) -> None: + if self._text is not None: + self._text.append(data) + + +def _read(path: Path) -> _Page: + page = _Page() + page.feed(path.read_text(encoding="utf-8")) + return page + + +@unittest.skipUnless((DIST / "index.html").is_file(), "site not built") +class BuiltSite(unittest.TestCase): + def test_the_404_sidebar_names_pages(self): + page = _read(DIST / "404.html") + self.assertTrue(page.sidebar) + self.assertFalse([label for label in page.sidebar if re.fullmatch(r"\d+", label)], + page.sidebar) + + def test_crumbs_link_a_tab_an_overview_or_a_section(self): + overviews = TAB_ROOTS | {"/retrobios/systems/", "/retrobios/emulators/"} + pages = [DIST / rel for rel in ( + "platforms/batocera/index.html", "platforms/retroarch/index.html", + "systems/sony/index.html", "emulators/duckstation/index.html", + "wiki/verification-modes/index.html", "files/index.html")] + for path in pages: + if not path.is_file(): + continue + for href in _read(path).crumbs: + with self.subTest(page=str(path.relative_to(DIST)), href=href): + self.assertTrue(href in overviews or "#" in (href or ""), href) + + def test_a_setup_page_names_its_group(self): + page = _read(DIST / "platforms" / "batocera" / "index.html") + self.assertEqual(len(page.crumbs), 2, page.crumbs) + + def test_card_pages_have_no_previous_or_next(self): + for rel in ("systems/sony/index.html", "emulators/duckstation/index.html", + "platforms/batocera/index.html"): + with self.subTest(page=rel): + self.assertFalse(_read(DIST / rel).pagination) + self.assertTrue(_read(DIST / "wiki" / "faq" / "index.html").pagination) + + def test_the_redirects_file_is_written(self): + data = json.loads((REPO_ROOT / "website" / "src" / "generated" / "redirects.json") + .read_text(encoding="utf-8")) + self.assertEqual(data.get("/emulators/mednafen_psx/"), "/emulators/beetle_psx/") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_weight.py b/tests/test_site_weight.py index 9cc55b7a..745f1e16 100644 --- a/tests/test_site_weight.py +++ b/tests/test_site_weight.py @@ -134,18 +134,19 @@ class APlatformSystemOverItsBudgetMoves(unittest.TestCase): class TheNavigationOpensOneInitial(unittest.TestCase): - def test_a_group_of_several_initials_is_nested(self): + def test_one_group_per_initial_one_level_deep(self): profiles = { key: {"emulator": name, "systems": ["s"]} - for key, name in (("a1", "Alpha"), ("b1", "Beta"), ("b2", "bravo"), ("z", "Zed")) + for key, name in (("a1", "Alpha"), ("b1", "Beta"), ("b2", "bravo"), ("z", "Zed"), + ("n", "2ship")) } nav = generate_site._emulator_nav(profiles) - groups = {next(iter(g)): next(iter(g.values())) for g in nav[1:]} - self.assertEqual(groups["A to C"], [ + self.assertEqual(nav[1:], [ + {"0 to 9": [{"2ship": "emulators/n.md"}]}, {"A": [{"Alpha": "emulators/a1.md"}]}, {"B": [{"Beta": "emulators/b1.md"}, {"bravo": "emulators/b2.md"}]}, + {"Z": [{"Zed": "emulators/z.md"}]}, ]) - self.assertEqual(groups["V to Z"], [{"Zed": "emulators/z.md"}]) class TheTableOfContentsOfACardPageStopsAtLevelTwo(unittest.TestCase): diff --git a/website/src/routeData.ts b/website/src/routeData.ts index b9f63a27..0c4cd1f1 100644 --- a/website/src/routeData.ts +++ b/website/src/routeData.ts @@ -1,63 +1,70 @@ -// What each page knows of its place in the site, computed once per page: -// - breadcrumbs: the tab and the groups above the page; -// - previous and next: the reading order runs on from one tab into the next, -// so the last Start here page leads to Your setup; -// - a pruned sidebar: a group that does not hold the page becomes one link -// to its first page, so an emulator page does not carry the 600 links of -// every other emulator (the equivalent of Material's navigation.prune). +// What each page knows of its place in the site, computed once per page +// from what scripts/sitestarlight.py writes beside the sidebar: +// - breadcrumbs: the tab and the groups above the page, each linked to the +// page or the section that introduces it, never to a sibling page; +// - previous and next: the reading order of `reading`. Start here runs on +// into Your setup, every other tab is read on its own, and a system, +// emulator or platform page has none; +// - a pruned sidebar: in the tabs listing every system and emulator, a +// group that does not hold the page becomes one link to its overview, so +// an emulator page does not carry the links of every other emulator. The +// groups of the other tabs stay groups. import { defineRouteMiddleware, type StarlightRouteData } from '@astrojs/starlight/route-data'; import sidebar from './generated/sidebar.json' with { type: 'json' }; type Entry = StarlightRouteData['sidebar'][number]; type Link = Extract; -type Item = { label: string; link?: string; items?: Item[] }; +type Group = { href: string | null; prune: boolean }; export type Crumb = { label: string; href?: string }; const BASE = import.meta.env.BASE_URL.replace(/\/?$/, '/'); const withBase = (link: string) => (link.startsWith(BASE) ? link : BASE + link.replace(/^\//, '')); -const flatten = (items: Item[]): { label: string; link: string }[] => - items.flatMap((item) => (item.items ? flatten(item.items) : item.link && !item.link.startsWith('http') ? [{ label: item.label, link: item.link }] : [])); +const GROUPS = sidebar.groups as Record; +const RUNS = sidebar.reading as { label: string; link: string }[][]; -// Every listed page in reading order, tab after tab. -const READING_ORDER = sidebar.topics.flatMap((topic) => - flatten(topic.items as Item[]).map((entry) => ({ ...entry, label: entry.label === 'Overview' ? topic.label : entry.label })), -); +const groupOf = (trail: string[]): Group | undefined => GROUPS[trail.join('/')]; const holdsCurrent = (entry: Entry): boolean => entry.type === 'link' ? entry.isCurrent : entry.entries.some(holdsCurrent); -const firstLink = (entry: Entry): string | undefined => - entry.type === 'link' ? entry.href : entry.entries.map(firstLink).find(Boolean); - -const prune = (entries: Entry[]): Entry[] => +const prune = (entries: Entry[], trail: string[]): Entry[] => entries.map((entry) => { if (entry.type === 'link') return entry; - if (holdsCurrent(entry)) return { ...entry, collapsed: false, entries: prune(entry.entries) }; - const href = firstLink(entry); - return href ? { type: 'link', label: entry.label, href, isCurrent: false, badge: entry.badge, attrs: {} } : entry; + const here = [...trail, entry.label]; + if (holdsCurrent(entry)) return { ...entry, collapsed: false, entries: prune(entry.entries, here) }; + const group = groupOf(here); + if (group?.prune && group.href) { + return { type: 'link', label: entry.label, href: withBase(group.href), isCurrent: false, badge: entry.badge, attrs: {} }; + } + return { ...entry, entries: prune(entry.entries, here) }; }); // The groups above the current page, outermost first. -const trail = (entries: Entry[], path: string): Crumb[] | undefined => { +const trail = (entries: Entry[], labels: string[]): Crumb[] | undefined => { for (const entry of entries) { if (entry.type === 'link') { if (entry.isCurrent) return []; continue; } - const below = trail(entry.entries, path); - if (below) return [{ label: entry.label, href: firstLink(entry) }, ...below]; + const here = [...labels, entry.label]; + const below = trail(entry.entries, here); + if (below) { + const href = groupOf(here)?.href; + return [{ label: entry.label, href: href ? withBase(href) : undefined }, ...below]; + } } return undefined; }; // A page the navigation does not list (part of a split page) sits under the // group holding the pages of its folder. -const folderTrail = (entries: Entry[], folder: string): Crumb[] => { +const folderTrail = (entries: Entry[], labels: string[], folder: string): Crumb[] => { + const links = (entry: Entry): string[] => (entry.type === 'link' ? [entry.href] : entry.entries.flatMap(links)); for (const entry of entries) { - if (entry.type !== 'group') continue; - const links = entry.entries.map(firstLink).filter(Boolean) as string[]; - if (links.some((href) => href.startsWith(folder))) return [{ label: entry.label, href: links[0] }]; + if (entry.type !== 'group' || !links(entry).some((href) => href.startsWith(folder))) continue; + const href = groupOf([...labels, entry.label])?.href; + return [{ label: entry.label, href: href ? withBase(href) : undefined }]; } return []; }; @@ -78,19 +85,28 @@ export const onRequest = defineRouteMiddleware(async (context, next) => { const topic = context.locals.starlightSidebarTopics?.topics.find((t) => t.isCurrent); const crumbs: Crumb[] = []; - if (topic && withBase(topic.link) !== path) { + if (topic && withBase(topic.link) !== path && route.id !== '404') { crumbs.push({ label: topic.label, href: withBase(topic.link) }); const segments = path.slice(BASE.length).split('/').filter(Boolean); - const groups = trail(route.sidebar, path) ?? (segments.length > 1 ? folderTrail(route.sidebar, `${BASE}${segments[0]}/`) : []); - crumbs.push(...groups.filter((crumb) => crumb.href !== path)); + const groups = + trail(route.sidebar, [topic.label]) ?? + (segments.length > 1 ? folderTrail(route.sidebar, [topic.label], `${BASE}${segments[0]}/`) : []); + for (const crumb of groups) { + // A group introduced by the page itself is the page; one with no + // section of its own (the initial D of the emulators) adds nothing + // to the crumb before it. + if (crumb.href === path) continue; + if (crumb.href && crumb.href === crumbs.at(-1)?.href) continue; + crumbs.push(crumb); + } } context.locals.rbBreadcrumbs = crumbs; - const position = READING_ORDER.findIndex((entry) => withBase(entry.link) === path); - if (position !== -1) { - if (!route.pagination.prev && position > 0) route.pagination.prev = asLink(READING_ORDER[position - 1]); - if (!route.pagination.next && position < READING_ORDER.length - 1) route.pagination.next = asLink(READING_ORDER[position + 1]); - } + const { prev, next: following } = route.entry.data; + const run = RUNS.find((pages) => pages.some((entry) => withBase(entry.link) === path)); + const position = run ? run.findIndex((entry) => withBase(entry.link) === path) : -1; + if (prev === undefined) route.pagination.prev = run && position > 0 ? asLink(run[position - 1]) : undefined; + if (following === undefined) route.pagination.next = run && position < run.length - 1 ? asLink(run[position + 1]) : undefined; - route.sidebar = prune(route.sidebar); + route.sidebar = prune(route.sidebar, [topic?.label ?? '']); }); diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 353e260c..193fbcfb 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -56,7 +56,7 @@ Every module in `tests/`, by the area it guards. | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_p0`, `test_site_r2_nav`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture