diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 2c7ff7c60..d8e7ff3d6 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2834,6 +2834,36 @@ def emulator_file_aliases(files: list[dict], anchors: list[str]) -> list[str]: return aliases +# A commit id: hexadecimal with a letter and a digit, 7 to 40 long. A version +# that holds one (0.8.139-2-g25e1ee2fd5) names no release. +_COMMIT_ID = re.compile( + r"(? dict[str, str]: + """The `core_version` of a profile by build mode ("" when one covers + every build), kept only where it names a release: a number and no + commit id. A commit id of digits only reads as a number, so a run that + starts one of the profile's own pins is one too (supermodel3's + "0.3a-1481545-mm").""" + pins = [str(value).casefold() + for field in ("source_commit", "upstream_commit") + for value in (profile[field].values() if isinstance(profile.get(field), dict) + else [profile.get(field)]) + if value] + + def names_a_commit(value: str) -> bool: + return bool(_COMMIT_ID.search(value)) or any( + pin.startswith(run) for run in re.findall(r"[0-9a-f]{7,40}", value.casefold()) + for pin in pins) + + stated = profile.get("core_version") + pairs = stated.items() if isinstance(stated, dict) else [("", stated)] + versions = {str(mode): str(value or "").strip() for mode, value in pairs} + return {mode: value for mode, value in versions.items() + if re.search(r"[0-9]", value) and not names_a_commit(value)} + + def emulator_structured_data(profile: dict) -> dict: """What an emulator page states, for a reader of its JSON-LD. @@ -2841,27 +2871,26 @@ def emulator_structured_data(profile: dict) -> dict: the core loads, their hashes, the source read or when. """ data: dict = {} - # Every repository the profile reads, each with its own pin: xemu's - # libretro build lives in another repository at another commit, and a - # port's upstream_commit was never published. + # Every repository the profile reads. The source repository carries the + # release the profile describes, never a commit id: a pin is no version. + # The upstream of a port is another code base and states none. + versions = release_versions(profile) about = [] seen: set[str] = set() for field in ("source", "upstream"): urls = profile.get(field) - pins = profile.get(f"{field}_commit") for mode, url in (urls.items() if isinstance(urls, dict) else [("", urls)]): if not url or not str(url).startswith("http") or str(url) in seen: continue seen.add(str(url)) - pin = pins.get(mode) if isinstance(pins, dict) else pins entry = { "@type": "SoftwareSourceCode", "name": str(profile.get("emulator", "")), "codeRepository": str(url), - "description": " ".join(filter(None, [mode, field])), } - if pin: - entry["version"] = str(pin) + version = (versions.get(mode) or versions.get("")) if field == "source" else None + if version: + entry["version"] = version about.append(entry) if about: data["about"] = about[0] if len(about) == 1 else about @@ -3036,6 +3065,14 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]: """ lastmod: dict[str, str] = {} terms = sitestarlight.glossary_terms(_REPO_ROOT / WIKI_SRC_DIR / "glossary.md") + # The console words of every system page, decorated or not: a console + # another page's words cover gets a search field of its own. + console_words: dict[str, set[str]] = {} + for path in sorted((docs / "systems").glob("*.md*")): + text = path.read_text(encoding="utf-8") + words = sitestarlight.page_console_words(_front_matter(text), text) + if words: + console_words[path.relative_to(docs).with_suffix("").as_posix()] = words for path in sorted(docs.rglob("*.md")): relative = path.relative_to(docs) if any(part in LOCAL_NOTE_DIRS for part in relative.parts[:-1]): @@ -3083,6 +3120,10 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]: }, **PAGE_DATA.get(relative.as_posix(), {}), } + if schema_type == "WebSite": + # The home page is the site: named as the site, part of nothing. + structured["name"] = SITE_NAME + del structured["isPartOf"] target, text = sitestarlight.finalize_page( relative.as_posix(), markdown, level=level, weight=search_weight(relative, level), @@ -3092,6 +3133,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]: logo=str(authored.get("logo") or ""), parent=authored.get("parent"), authored=authored, + words_by_page=console_words, ) write_decorated(docs / target, text) # One file per page: a page that gained or lost its components diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py index 5763f19b4..949f17744 100644 --- a/scripts/sitestarlight.py +++ b/scripts/sitestarlight.py @@ -16,7 +16,7 @@ import json import re import unicodedata import urllib.parse -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Mapping from html import unescape from pathlib import Path, PurePosixPath @@ -366,6 +366,8 @@ def split_title(markdown: str) -> tuple[str, str, str]: _CARD_NAME = re.compile(r']*class="rb-file-title"(?![^>]*data-pagefind-ignore)[^>]*>([^<]+)') +# What separates the parts of a file name: anything but a letter or a digit. +_NAME_PART = re.compile(r"[\W_]+") def search_field(key: str, words: list[str]) -> str: @@ -384,9 +386,20 @@ def file_names_meta(body: str) -> str: field of the page. A page of 250 cards is long, and the search weighs a long page down: the Sony page, where scph5501.bin is downloaded, came after every short page citing the name. The field is weighted in the - site's ranking (astro.config.mjs, metaWeights.files).""" + site's ranking (astro.config.mjs, metaWeights.files). + + A name is written as its parts (scph5501 bin), each once. The search + matches a word as the start of a longer one and scores every distinct + word it reaches: whole names gave "ds" twelve rare words on the DSi page + (dsi_bios7.bin, DSi_Nand_USA.bin...), which then ranked first for + "nintendo ds bios".""" names = sorted({unescape(name) for name in _CARD_NAME.findall(body)}) - return search_field("files", names) + parts: dict[str, str] = {} + for name in names: + for part in _NAME_PART.split(name): + if part: + parts.setdefault(part.casefold(), part) + return search_field("files", list(parts.values())) # systems/ page -> the platforms' own ids for the consoles it holds, built @@ -417,10 +430,6 @@ def page_aliases(rel: str) -> list[str]: return list(_page_aliases.get(rel, [])) -# Search fields a page's own front matter fills, never typed here. -SEARCH_FIELDS = ("consoles", "terms") - - def field_words(value: object) -> list[str]: """The words of a front matter search field: a list, or one string with its words or phrases split at commas.""" @@ -431,6 +440,41 @@ def field_words(value: object) -> list[str]: return [] +def console_words(names: list[str]) -> list[str]: + """What a visitor types for a console: its name with "BIOS". A page that + answers "No file needed" says BIOS only where the search does not read, + so "nes bios" found NES.emu and never the NES page. As a task word it + weighs like one: in the consoles field, NES.emu's title still won.""" + return [f"{name} BIOS" for name in names] + + +_CONSOLES_META = re.compile(r'data-pagefind-meta="consoles:([^"]*)"') + + +def page_console_words(authored: dict, text: str) -> set[str]: + """The words of the consoles a page answers for: its `consoles` front + matter (*authored*) before decoration, its `consoles` field after.""" + names = field_words(authored.get("consoles")) + if names: + return {word.casefold() for name in names for word in name.split()} + match = _CONSOLES_META.search(text) + return set(unescape(match[1]).casefold().split()) if match else set() + + +def covered_names(names: list[str], page: str, words_by_page: Mapping[str, set[str]]) -> list[str]: + """The names of *names* whose every word another page's consoles also + hold, as a word or the start of one (PlayStation on the PlayStation 2 + page, Nintendo DS on the DSi page, Nintendo Super System on the + Nintendo page). The search matches a word as the start of a longer one + and ignores the words a page has that the query lacks: it cannot tell + these pages apart by words, only by a weight the other page lacks.""" + held = [words for other, words in words_by_page.items() if other != page] + return [name for name in names + if any(all(any(word.startswith(part) for word in words) + for part in name.casefold().split()) + for words in held)] + + def _neighbour(rel: str, value: object) -> object: """A `prev:` or `next:` the page states, its link resolved like any other link of the page.""" @@ -472,6 +516,7 @@ def finalize_page( logo: str = "", parent: dict | None = None, authored: dict | None = None, + words_by_page: Mapping[str, set[str]] | None = None, ) -> tuple[str, str]: """The Starlight file for a page body: its path under the content collection and its text. *parent* ({label, link}, the link written @@ -483,7 +528,10 @@ def finalize_page( page states its neighbours and its weight), and its `consoles` and `terms` become search fields (astro.config.mjs, metaWeights): the consoles a manufacturer page holds sections for, and the words a page - answers that its text does not say, as an error message words them.""" + answers that its text does not say, as an error message words them. + *words_by_page* holds the console words of every system page, by path + without extension: a console another page's words cover is written + once more, in `exact`.""" authored = authored or {} title, title_id, body = split_title(markdown) body = body.lstrip("\n") @@ -503,8 +551,10 @@ def finalize_page( r' bool: return not rel.startswith(CARD_PAGES) or rel.endswith("/index.md") +_DIGIT_RANGE = re.compile(r"[0-9] to [0-9]") + + +def by_initial(label: str, children: list[dict]) -> bool: + """A group that files its pages by the first character of their names + (A, B, "0 to 9"): a way through a long list, not a place, so it is no + crumb.""" + if not children or any("items" in child for child in children): + return False + firsts = {str(child["label"])[:1].upper() for child in children} + if len(label) == 1: + return firsts == {label} + return bool(_DIGIT_RANGE.fullmatch(label)) and not any("A" <= first <= "Z" for first in firsts) + + def sidebar(nav: list, present: set[str], anchors: dict[str, set[str]] | None = None) -> dict: """The navigation as Starlight topics, one per tab, with what the route middleware (website/src/routeData.ts) reads beside it. A page the navigation does not list (the parts of a split page) joins - the topic that holds most pages of its directory. + the topic that holds most pages of its directory. In a directory the + navigation lists nothing of (get/), it joins the topic of the pages of + the same name, and a page with neither joins the first topic: no page + is left out of every tab. + + A group that files names by their first letter is marked `crumb: false`: + the breadcrumbs skip it. `groups` gives each group, keyed by the labels from its tab down, the page that introduces it: its own overview page, or the heading named @@ -649,7 +720,8 @@ def sidebar(nav: list, present: set[str], anchors: dict[str, set[str]] | None = children = [child for child in (item(v, topic, here, own or parent) for v in value) if child] if not children: return None - groups["/".join(here)] = {"href": href, "prune": topic in PRUNED_TABS and href is not None} + groups["/".join(here)] = {"href": href, "prune": topic in PRUNED_TABS and href is not None, + "crumb": not by_initial(label, children)} return {"label": label, "items": children, "collapsed": True} topics = [] @@ -670,15 +742,35 @@ def sidebar(nav: list, present: set[str], anchors: dict[str, set[str]] | None = # is the way back in. Excluded from every tab, the theme named the tabs # by their position, 0 to 4. extra: dict[str, list[str]] = {topics[0]["id"]: ["/404"]} if topics else {} - exclude: list[str] = [] - for rel in sorted(present - owner.keys()): - tally = votes.get(rel.rpartition("/")[0]) - path = "/" + (_sidebar_link(rel).strip("/") or "index") + # A folder the navigation lists nothing of (get/, one download page per + # setup) has no vote, and excluded from every tab its pages had the + # tabs numbered 0 to 4 too. Its pages take the tab of the pages of the + # same name (get/batocera.md, platforms/batocera.md), and its overview + # the tab its pages took. + by_name: dict[str, dict[str, int]] = {} + for rel, topic in owner.items(): + tally = by_name.setdefault(PurePosixPath(rel).name, {}) + tally[topic] = tally.get(topic, 0) + 1 + unlisted = sorted(present - owner.keys()) + adopted: dict[str, dict[str, int]] = {} + for rel in unlisted: + folder, name = rel.rpartition("/")[0], PurePosixPath(rel).name + if folder in votes or name == "index.md": + continue + tally = adopted.setdefault(folder, {}) + for topic, count in by_name.get(name, {}).items(): + tally[topic] = tally.get(topic, 0) + count + for rel in unlisted: + folder = rel.rpartition("/")[0] + tally = votes.get(folder) or adopted.get(folder) if tally: - extra.setdefault(max(tally, key=tally.get), []).append(path) + topic = max(tally, key=tally.get) + elif topics: + topic = topics[0]["id"] else: - exclude.append(path) - return {"topics": topics, "options": {"topics": extra, "exclude": exclude}, + continue + extra.setdefault(topic, []).append("/" + (_sidebar_link(rel).strip("/") or "index")) + return {"topics": topics, "options": {"topics": extra, "exclude": []}, "groups": groups, "reading": _reading(nav, present)} diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 28c95c2ae..9aad3b0e5 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -4360,31 +4360,44 @@ class EmulatorPagesStateTheirDataForMachines(unittest.TestCase): data = emulator_structured_data({ "emulator": "DuckStation", "source": "https://github.com/stenzek/duckstation", - "source_commit": "4122fed9", "profiled_date": "2026-03-23", + "source_commit": "4122fed9", "core_version": "v0.1-12070", + "profiled_date": "2026-03-23", "files": [{"name": "scph5501.bin", "md5": "490F666E", "sha1": ["a" * 40]}, {"name": "free.bin"}], }) self.assertEqual(data["about"]["codeRepository"], "https://github.com/stenzek/duckstation") - self.assertEqual(data["about"]["version"], "4122fed9") + # The release the profile describes, never the pin (round 5, 3.5). + self.assertEqual(data["about"]["version"], "v0.1-12070") + self.assertNotIn("description", data["about"]) self.assertEqual(data["dateModified"], "2026-03-23") self.assertEqual(data["hasPart"][0]["identifier"], ["sha1:" + "a" * 40, "md5:490f666e"]) self.assertNotIn("identifier", data["hasPart"][1]) - def test_every_repository_with_its_own_pin(self): - """xemu's libretro build lives in another repository at another - commit, and a port's upstream_commit was never published.""" + def test_every_repository_with_its_own_release(self): + """xemu's libretro build lives in another repository. Each source + repository states the release of its build, a version holding a + commit id states none, and a port's upstream is another code base.""" from scripts.generate_site import emulator_structured_data data = emulator_structured_data({ - "emulator": "xemu", - "source": {"standalone": "https://github.com/xemu-project/xemu", - "libretro": "https://github.com/LibretroAdmin/xemu-libretro"}, + "emulator": "MesenCE", + "source": {"standalone": "https://github.com/nesdev-org/MesenCE", + "libretro": "https://github.com/libretro/MesenCE"}, "source_commit": {"standalone": "f9b14039", "libretro": "d9804db7"}, - "upstream": "https://github.com/mborgerson/xemu", "upstream_commit": "aa11", + "core_version": {"standalone": "2.2.1", "libretro": "2.0.0"}, + "upstream": "https://github.com/SourMesen/Mesen2", "upstream_commit": "aa11", }) self.assertEqual( [(a["codeRepository"].rsplit("/", 1)[1], a.get("version")) for a in data["about"]], - [("xemu", "f9b14039"), ("xemu-libretro", "d9804db7"), ("xemu", "aa11")]) + [("MesenCE", "2.2.1"), ("MesenCE", "2.0.0"), ("Mesen2", None)]) + xemu = emulator_structured_data({ + "emulator": "xemu", + "source": {"standalone": "https://github.com/xemu-project/xemu", + "libretro": "https://github.com/LibretroAdmin/xemu-libretro"}, + "core_version": {"standalone": "0.8.139-2-g25e1ee2fd5", + "libretro": "libretro-experimental"}, + }) + self.assertEqual([a.get("version") for a in xemu["about"]], [None, None]) diff --git a/tests/test_site_r2_nav.py b/tests/test_site_r2_nav.py index c3f549ac2..6e3f134a7 100644 --- a/tests/test_site_r2_nav.py +++ b/tests/test_site_r2_nav.py @@ -117,10 +117,12 @@ class GroupsAreGroups(unittest.TestCase): def test_explore_groups_collapse_to_their_overview(self): explore = {key: value for key, value in self.groups.items() if key.startswith("Explore/")} - self.assertEqual(explore["Explore/Systems"], {"href": "/retrobios/systems/", "prune": True}) + self.assertEqual(explore["Explore/Systems"], + {"href": "/retrobios/systems/", "prune": True, "crumb": True}) self.assertEqual(explore["Explore/Systems/Consoles and computers"]["href"], "/retrobios/systems/#consoles-and-computers") - self.assertEqual(explore["Explore/Emulators/D"], {"href": "/retrobios/emulators/#d", "prune": True}) + self.assertEqual(explore["Explore/Emulators/D"], + {"href": "/retrobios/emulators/#d", "prune": True, "crumb": False}) def test_no_group_points_at_one_of_its_own_pages(self): """The overview a group opens on is its own index page or a section of diff --git a/tests/test_site_r2_systems.py b/tests/test_site_r2_systems.py index d6fcee001..6579e2d97 100644 --- a/tests/test_site_r2_systems.py +++ b/tests/test_site_r2_systems.py @@ -293,7 +293,12 @@ class TheBuiltSonyPage(unittest.TestCase): def test_playstation_opens_on_a_bios_it_needs(self): section = self.text.split('id="playstation"', 1)[1].split("]*>scph\d+\.bin') # PlayStation has a page of its own, and its files open there. own = (DIST / "systems" / "sony-playstation" / "index.html").read_text(encoding="utf-8") self.assertIn('href="/retrobios/systems/sony-playstation/"', section) diff --git a/tests/test_site_r3_components.py b/tests/test_site_r3_components.py index 2572e0b93..d2fdc1bc6 100644 --- a/tests/test_site_r3_components.py +++ b/tests/test_site_r3_components.py @@ -291,8 +291,8 @@ class SearchFoldsPagesUnderAHash(unittest.TestCase): def test_search_fields_are_weighted_and_never_shown(self): config = _read(CONFIG) - self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 5, devices: 5, consoles: 10, terms: 30 },", config) - self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'terms']);", _read(SITE_TS)) + self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 10, devices: 5, consoles: 100, exact: 20, terms: 30 },", config) + self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'exact', 'terms']);", _read(SITE_TS)) class OsTabsNeverScrollThePage(unittest.TestCase): diff --git a/tests/test_site_r3_nav.py b/tests/test_site_r3_nav.py index 04c75b1c4..2e68f69e5 100644 --- a/tests/test_site_r3_nav.py +++ b/tests/test_site_r3_nav.py @@ -65,7 +65,8 @@ class MetaFields(unittest.TestCase): body = ('

scph5501.bin

' '

a,b.bin

') meta = sitestarlight.file_names_meta(body) - self.assertIn('data-pagefind-meta="files:a b.bin scph5501.bin"', meta) + # Each name as its parts, each part once (round 5, 3.2). + self.assertIn('data-pagefind-meta="files:a b bin scph5501"', meta) self.assertRegex(meta, r"]*hidden>") def test_a_field_escapes_its_value(self) -> None: diff --git a/tests/test_site_r4_components.py b/tests/test_site_r4_components.py index 67bb9f689..d21f94fb5 100644 --- a/tests/test_site_r4_components.py +++ b/tests/test_site_r4_components.py @@ -214,8 +214,8 @@ class SearchFields(unittest.TestCase): def test_declared_and_never_shown(self): config = _read(CONFIG) - self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 5, devices: 5, consoles: 10, terms: 30 },", config) - self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'terms']);", _read(SITE_TS)) + self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 10, devices: 5, consoles: 100, exact: 20, terms: 30 },", config) + self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'exact', 'terms']);", _read(SITE_TS)) class SearchLoadsOnFirstUse(unittest.TestCase): diff --git a/tests/test_site_r4_search.py b/tests/test_site_r4_search.py index 188b2aa4e..8d244baaf 100644 --- a/tests/test_site_r4_search.py +++ b/tests/test_site_r4_search.py @@ -120,7 +120,8 @@ class FieldsFromFrontMatter(unittest.TestCase): "terms": "bios not found, bios missing", })["text"] self.assertIn('data-pagefind-meta="consoles:Neo Geo Neo Geo Pocket Color"', text) - self.assertIn('data-pagefind-meta="terms:bios not found bios missing"', text) + self.assertIn('data-pagefind-meta="terms:bios not found bios missing Neo Geo BIOS ' + 'Neo Geo Pocket Color BIOS"', text) # A field value is an attribute: none of it reaches an excerpt. self.assertNotRegex(text, r">[^<]*bios missing[^<]*<") diff --git a/tests/test_site_r5_search.py b/tests/test_site_r5_search.py new file mode 100644 index 000000000..ccf666095 --- /dev/null +++ b/tests/test_site_r5_search.py @@ -0,0 +1,375 @@ +"""Round 5, package 3: search, sidebar and head. + +The get/ download pages sat in no tab, and the theme then numbered the tabs +0 to 4. "snes bios" found NES.emu and never the SNES page, which answers +"No file needed" and so never says BIOS where the search reads. An emulator +crumb ran through its letter group ("Emulators / D"). The home page's +structured data named the page and called it part of itself, and an +emulator's source repository published a commit id as its version beside a +placeholder description. These tests read the generator and, where it is +built, the site. +""" + +from __future__ import annotations + +import ast +import json +import re +import sys +import unittest +from html.parser import HTMLParser +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site # noqa: E402 +import sitestarlight # noqa: E402 +from common import list_registered_platforms, load_emulator_profiles # noqa: E402 +from tests.test_site_r2_nav import ANCHORS, _nav, _present # noqa: E402 + + +def _gate_queries() -> dict[str, tuple[tuple[str, ...], int]]: + """The gate's QUERIES, read from website/tools/search_gate.py: a tool + script, not a module the suite installs.""" + tree = ast.parse((REPO_ROOT / "website" / "tools" / "search_gate.py").read_text(encoding="utf-8")) + node = next(node for node in tree.body if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) and node.target.id == "QUERIES") + return ast.literal_eval(node.value) + + +QUERIES = _gate_queries() +DIST = REPO_ROOT / "website" / "dist" +ROUTE = REPO_ROOT / "website" / "src" / "routeData.ts" +PLATFORMS = str(REPO_ROOT / "platforms") + + +def _sidebar_with_get_pages() -> tuple[dict, list[str]]: + nav = _nav() + names = list_registered_platforms(PLATFORMS, include_archived=True) + get_pages = ["get/index.md", *(f"get/{name}.md" for name in names)] + return sitestarlight.sidebar(nav, _present(nav) | set(get_pages), ANCHORS), get_pages + + +class GetPagesSitInYourSetup(unittest.TestCase): + """3.1: a folder the navigation lists nothing of takes the tab of the + pages of the same name, and nothing is excluded from every tab.""" + + def test_every_get_page_joins_your_setup(self): + side, get_pages = _sidebar_with_get_pages() + self.assertEqual(side["options"]["exclude"], []) + setup = set(side["options"]["topics"].get("your-setup", [])) + for rel in get_pages: + with self.subTest(page=rel): + self.assertIn("/" + sitestarlight._sidebar_link(rel).strip("/"), setup) + + def test_no_page_is_left_out_of_every_tab(self): + nav = _nav() + present = _present(nav) | {"orphans/alone.md"} + side = sitestarlight.sidebar(nav, present, ANCHORS) + self.assertEqual(side["options"]["exclude"], []) + placed = {path for paths in side["options"]["topics"].values() for path in paths} + self.assertIn("/orphans/alone", placed) + + +class ConsoleWordsReachTheConsole(unittest.TestCase): + """3.2: " BIOS" for each console the page names, from its own + `consoles` field, never a word typed in the generator.""" + + def test_each_console_with_bios(self): + self.assertEqual(sitestarlight.console_words(["SNES", "Super Famicom"]), + ["SNES BIOS", "Super Famicom BIOS"]) + + def test_the_terms_field_carries_them(self): + _target, text = sitestarlight.finalize_page( + "systems/nintendo-snes.md", "# SNES\n\nNo file needed.\n", level="0", + weight=None, browser_title="SNES", description="d", + structured={"@type": "TechArticle"}, terms={}, + authored={"consoles": ["SNES"]}) + self.assertIn('data-pagefind-meta="consoles:SNES"', text) + self.assertIn('data-pagefind-meta="terms:SNES BIOS"', text) + + def test_no_search_word_is_typed_in_the_generator(self): + tree = ast.parse((REPO_ROOT / "scripts" / "sitestarlight.py").read_text(encoding="utf-8")) + docstrings = {id(node.value) for node in ast.walk(tree) if isinstance(node, ast.Expr)} + strings = {node.value.casefold() for node in ast.walk(tree) + if isinstance(node, ast.Constant) and isinstance(node.value, str) + and id(node) not in docstrings} + typed = {text for text in strings for query in QUERIES + if re.search(rf"(? dict[str, set[str]]: + return {page: sitestarlight.page_console_words({"consoles": names}, "") + for page, names in self.PAGES.items()} + + def test_names_another_page_covers(self): + covered = {page: sitestarlight.covered_names(names, page, self.words()) + for page, names in self.PAGES.items()} + self.assertEqual(covered, { + "systems/sony-playstation": ["PlayStation"], + "systems/sony-playstation-2": [], + # Every word of PlayStation 3 is on the PlayStation 2 page, but + # not "3" itself: no page covers it. + "systems/sony": [], + "systems/nintendo-nintendo-ds": ["Nintendo DS"], + "systems/nintendo-nintendo-dsi": [], + "systems/nintendo": [], + # Nintendo, Super and System, each in another name. + "systems/arcade-nintendo-super-system": ["Nintendo Super System"], + "systems/sega-saturn": [], + }) + + def test_words_of_a_decorated_page(self): + text = '' + self.assertEqual(sitestarlight.page_console_words({}, text), {"nintendo", "ds"}) + self.assertEqual(sitestarlight.page_console_words({}, "No field."), set()) + + def test_the_exact_field_is_written_on_the_covered_page_only(self): + words = self.words() + + def text(page: str) -> str: + return sitestarlight.finalize_page( + f"{page}.md", "# T\n\nBody.\n", level="0", weight=None, + browser_title="T", description="d", structured={"@type": "TechArticle"}, + terms={}, authored={"consoles": self.PAGES[page]}, words_by_page=words)[1] + + self.assertIn('data-pagefind-meta="exact:PlayStation"', text("systems/sony-playstation")) + for page in ("systems/sony-playstation-2", "systems/sega-saturn"): + with self.subTest(page=page): + self.assertNotIn('data-pagefind-meta="exact:', text(page)) + + def test_the_field_is_weighted_and_never_shown(self): + config = (REPO_ROOT / "website" / "astro.config.mjs").read_text(encoding="utf-8") + weights = re.search(r"metaWeights: \{([^}]*)\}", config)[1] + fields = dict(re.findall(r"(\w+): ([\d.]+)", weights)) + self.assertIn("exact", fields) + # Below the console field, so the longer name keeps its own query. + self.assertLess(float(fields["exact"]), float(fields["consoles"])) + site = (REPO_ROOT / "website" / "src" / "scripts" / "site.ts").read_text(encoding="utf-8") + self.assertRegex(site, r"UNSHOWN_FIELDS = new Set\(\[[^\]]*'exact'") + + def test_file_names_as_parts(self): + body = "".join(f'

{name}

' + for name in ("dsi_bios7.bin", "DSi_Nand_USA.bin", "NDS_Bios7.bin")) + meta = sitestarlight.file_names_meta(body) + self.assertIn('data-pagefind-meta="files:DSi Nand USA bin NDS Bios7"', meta) + # A letter of any script is a letter. + self.assertIn("ファミコン", sitestarlight.file_names_meta( + '

ファミコン_bios.rom

')) + + +class GateQueries(unittest.TestCase): + """3.2, 3.3 and 3.6: the queries the gate runs, the pages that answer + them and the rank they must reach.""" + + EXPECTED = { + "snes bios": (("systems/nintendo-snes/",), 1), + "nes bios": (("systems/nintendo-nes/",), 1), + "gamecube bios": (("systems/nintendo-gamecube/",), 1), + "game boy bios": (("systems/nintendo-game-boy/",), 1), + "3ds bios": (("systems/nintendo-3ds/",), 1), + "playstation bios": (("systems/sony-playstation/",), 1), + "playstation 2 bios": (("systems/sony-playstation-2/",), 1), + "playstation 3 bios": (("systems/sony/",), 1), + "game boy advance bios": (("systems/nintendo/",), 1), + "game boy color bios": (("systems/nintendo/",), 1), + "nintendo ds bios": (("systems/nintendo-nintendo-ds/",), 1), + "nintendo dsi bios": (("systems/nintendo-nintendo-dsi/",), 1), + "wii bios": (("systems/nintendo/",), 1), + "wii u bios": (("systems/nintendo-wii-u/",), 1), + "saturn bios": (("systems/sega-saturn/",), 1), + "check my bios folder": (("wiki/troubleshooting/",), 3), + "is my bios correct": (("files/",), 3), + "schema": (("data/",), 3), + "json schema": (("data/",), 3), + "license": (("data/", "wiki/legal/"), 3), + "check bios folder script": (("data/",), 3), + "ios": (("which-pack/", "wiki/faq/"), 3), + } + + def test_the_gate_runs_each_query(self): + for query, expected in self.EXPECTED.items(): + with self.subTest(query=query): + self.assertEqual(QUERIES.get(query), expected) + + def test_each_console_query_names_a_generated_console_page(self): + docs = REPO_ROOT / "website" / "src" / "content" / "docs" + if not docs.is_dir(): + self.skipTest("site not generated") + for query, (pages, _rank) in self.EXPECTED.items(): + if not query.endswith(" bios"): + continue + stem = pages[0].strip("/") + with self.subTest(query=query): + self.assertTrue(any((docs / f"{stem}{ext}").is_file() for ext in (".md", ".mdx")), + stem) + + +class LetterGroupsAreNoCrumb(unittest.TestCase): + """3.4: a group of names by their first letter is a way through a + list, not a place.""" + + def test_by_initial(self): + names = [{"label": "Dolphin", "link": "/d"}, {"label": "DuckStation", "link": "/e"}] + self.assertTrue(sitestarlight.by_initial("D", names)) + self.assertFalse(sitestarlight.by_initial("E", names)) + digits = [{"label": "2600.emu", "link": "/a"}, {"label": "81", "link": "/b"}] + self.assertTrue(sitestarlight.by_initial("0 to 9", digits)) + self.assertFalse(sitestarlight.by_initial("Sony", names)) + self.assertFalse(sitestarlight.by_initial("D", [{"label": "D", "items": names}])) + + def test_emulator_letter_groups_are_marked(self): + side = sitestarlight.sidebar(_nav(), _present(_nav()), ANCHORS) + letters = {key: value for key, value in side["groups"].items() + if key.startswith("Explore/Emulators/")} + self.assertTrue(letters) + for key, group in letters.items(): + with self.subTest(group=key): + self.assertIs(group["crumb"], False) + self.assertIs(side["groups"]["Explore/Emulators"]["crumb"], True) + self.assertIs(side["groups"]["Explore/Systems"]["crumb"], True) + + def test_the_route_middleware_skips_them(self): + self.assertIn("group?.crumb === false", ROUTE.read_text(encoding="utf-8")) + + +class StructuredData(unittest.TestCase): + """3.5: the home page is the site, and a source repository states the + release, never a commit id, and no placeholder description.""" + + def test_versions_name_releases_only(self): + release = generate_site.release_versions + self.assertEqual(release({"core_version": "v0.1-12070"}), {"": "v0.1-12070"}) + self.assertEqual(release({"core_version": "1.5.85"}), {"": "1.5.85"}) + self.assertEqual(release({"core_version": "8dc77b0"}), {}) + self.assertEqual(release({"core_version": "0.8.139-2-g25e1ee2fd5"}), {}) + self.assertEqual(release({"core_version": "libretro branch"}), {}) + self.assertEqual(release({"core_version": "edge"}), {}) + self.assertEqual(release({}), {}) + self.assertEqual(release({"core_version": {"standalone": "v1.1", "libretro": "4122fed"}}), + {"standalone": "v1.1"}) + # A pin of digits only, in the version: supermodel3. + self.assertEqual(release({"core_version": "0.3a-1481545-mm DirtBagXon (Sinden)", + "upstream_commit": "14815455dc28fca2c8662fb1ed99d447ecf8f364"}), + {}) + # A dated release whose digits start no pin stays. + self.assertEqual(release({"core_version": "20200712", "source_commit": "a1626f4e"}), + {"": "20200712"}) + + def test_the_pin_is_never_a_version(self): + data = generate_site.emulator_structured_data({ + "emulator": "Redream", "source": "https://github.com/x/redream", + "source_commit": "0123abcd", + }) + self.assertEqual(data["about"], {"@type": "SoftwareSourceCode", "name": "Redream", + "codeRepository": "https://github.com/x/redream"}) + + +class _Head(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.ld: list[dict] = [] + self.sidebar: list[str] = [] + self.tabs_current = 0 + self._ld: list[str] | None = None + self._label: list[str] | None = None + self._tabs = False + + def handle_starttag(self, tag: str, attrs: list) -> None: + a = dict(attrs) + if tag == "script" and a.get("type") == "application/ld+json": + self._ld = [] + if tag == "summary": + self._label = [] + if tag == "nav" and a.get("aria-label") == "Sections": + self._tabs = True + elif self._tabs and tag == "a" and a.get("aria-current"): + self.tabs_current += 1 + + def handle_endtag(self, tag: str) -> None: + if tag == "nav": + self._tabs = False + if tag == "script" and self._ld is not None: + self.ld.append(json.loads("".join(self._ld))) + self._ld = None + if tag == "summary" and self._label is not None: + self.sidebar.append(" ".join("".join(self._label).split())) + self._label = None + + def handle_data(self, data: str) -> None: + for buffer in (self._ld, self._label): + if buffer is not None: + buffer.append(data) + + +def _head(path: Path) -> _Head: + page = _Head() + page.feed(path.read_text(encoding="utf-8")) + return page + + +@unittest.skipUnless((DIST / "index.html").is_file(), "site not built") +class BuiltSite(unittest.TestCase): + def test_no_sidebar_group_is_named_by_a_number(self): + for path in sorted(DIST.rglob("*.html")): + labels = _head(path).sidebar + with self.subTest(page=str(path.relative_to(DIST))): + self.assertFalse([label for label in labels if re.fullmatch(r"\d+", label)]) + + def test_every_get_page_has_one_current_tab(self): + pages = sorted((DIST / "get").rglob("index.html")) + self.assertTrue(pages) + for path in [*pages, DIST / "404.html", DIST / "index.html"]: + with self.subTest(page=str(path.relative_to(DIST))): + self.assertEqual(_head(path).tabs_current, 1) + + def test_the_home_page_is_the_site(self): + ld = _head(DIST / "index.html").ld + self.assertEqual(len(ld), 1) + self.assertEqual(ld[0]["@type"], "WebSite") + self.assertEqual(ld[0]["name"], "RetroBIOS") + self.assertNotIn("isPartOf", ld[0]) + + def test_no_source_repository_states_a_commit_or_a_description(self): + pins = {str(value).casefold()[:7] + for profile in load_emulator_profiles(str(REPO_ROOT / "emulators")).values() + for field in ("source_commit", "upstream_commit") + for value in (profile[field].values() if isinstance(profile.get(field), dict) + else [profile.get(field)]) + if value} + self.assertTrue(pins) + seen = 0 + for path in sorted((DIST / "emulators").rglob("index.html")): + for data in _head(path).ld: + about = data.get("about") or [] + for entry in about if isinstance(about, list) else [about]: + seen += 1 + with self.subTest(page=str(path.relative_to(DIST)), repo=entry.get("codeRepository")): + self.assertNotIn("description", entry) + runs = re.findall(r"[0-9a-f]{7,40}", str(entry.get("version", "")).casefold()) + self.assertFalse([run for run in runs if run[:7] in pins]) + self.assertFalse(generate_site._COMMIT_ID.search(str(entry.get("version", "")))) + self.assertTrue(seen) + + +if __name__ == "__main__": + unittest.main() diff --git a/website/astro.config.mjs b/website/astro.config.mjs index 157a23fbb..3d94fdcc3 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -108,26 +108,37 @@ export default defineConfig({ // ten times its text, so "dolphin" finds Dolphin, and the file names of // a system page (its `files` field, written by sitestarlight.py) count // half, so "scph5501" finds the Sony page without a file name winning - // every other query. Two fields name the page's subject in other words - // and weigh half a title: `aliases`, the ids setups file a system under - // (psx, dreamcast, gba), and `devices`, the handhelds and boxes a - // setup's own site names (R36S, Miyoo Mini). `consoles`, the consoles a - // manufacturer page holds (Neo Geo, WonderSwan Color), weighs as much - // as a title. `terms`, the words of a task or a message on screen (bios - // not found, which mame version, check my dump), weighs three times - // that: a field match scores below a title match, and the cards of a - // part page keep their own weight, so below 30 the MAME parts, whose - // card titles name MAME files, came before the version table for - // "which mame version". All of them are - // attributes on an empty element, so no text of theirs reaches an - // excerpt. + // every other query. `devices`, the handhelds and boxes a setup's own + // site names (R36S, Miyoo Mini), weighs half a title. `aliases`, the ids + // setups file a system under (psx, dreamcast, gba), weighs a title: with + // file names written as their parts, half a title put GBA.emu and two + // more emulators above the Nintendo page for "gba bios". `terms`, the + // words of a task or a message on screen (bios not found, which mame + // version, check my dump), weighs three titles: a field match scores + // below a title match, and the cards of a part page keep their own + // weight, so below 30 the MAME parts, whose card titles name MAME files, + // came before the version table for "which mame version". + // + // A field adds the same score to every page whose field holds the + // word, whatever else the field holds, and a query word also matches + // the start of a longer one. `consoles`, the consoles a system page + // answers for, weighs ten titles: below that an emulator named after + // its console (Saturn.emu, BK, PIECE) came before the console. The + // PlayStation 2 page holds every word of "playstation bios", so + // `exact` repeats, on its own page, a console whose every word another + // page's consoles hold (PlayStation, Nintendo DS, Nintendo Super + // System). It weighs two titles: enough over the other page, and less + // than the "3" of "playstation 3 bios" adds to the Sony page. Measured + // on the " bios" query of every system page and on + // tools/search_gate.py. All of them are attributes on an empty + // element, so no text of theirs reaches an excerpt. pagefind: { ranking: { pageLength: 0.1, termFrequency: 0.5, termSaturation: 1.4, termSimilarity: 9, - metaWeights: { title: 10, files: 0.5, aliases: 5, devices: 5, consoles: 10, terms: 30 }, + metaWeights: { title: 10, files: 0.5, aliases: 10, devices: 5, consoles: 100, exact: 20, terms: 30 }, }, }, plugins: [ diff --git a/website/src/routeData.ts b/website/src/routeData.ts index 104d20270..31c41b190 100644 --- a/website/src/routeData.ts +++ b/website/src/routeData.ts @@ -1,7 +1,8 @@ // What each page knows of its place in the site, computed once per page // from what scripts/sitestarlight.py writes beside the sidebar: // - breadcrumbs: the tab and the groups above the page, each linked to the -// page or the section that introduces it, never to a sibling page; +// page or the section that introduces it, never to a sibling page. A +// group of names by their first letter (A, B) is no crumb; // - previous and next: the reading order of `reading`. Start here runs on // into Your setup, every other tab is read on its own, and a system, // emulator or platform page has none; @@ -19,7 +20,7 @@ import sidebar from './generated/sidebar.json' with { type: 'json' }; type Entry = StarlightRouteData['sidebar'][number]; type Link = Extract; -type Group = { href: string | null; prune: boolean }; +type Group = { href: string | null; prune: boolean; crumb?: boolean }; export type Crumb = { label: string; href?: string }; const BASE = import.meta.env.BASE_URL.replace(/\/?$/, '/'); @@ -55,7 +56,11 @@ const trail = (entries: Entry[], labels: string[]): Crumb[] | undefined => { const here = [...labels, entry.label]; const below = trail(entry.entries, here); if (below) { - const href = groupOf(here)?.href; + const group = groupOf(here); + // A group of names by their first letter is a way through the + // list, not a place: the crumb stops at the list. + if (group?.crumb === false) return below; + const href = group?.href; return [{ label: entry.label, href: href ? withBase(href) : undefined }, ...below]; } } diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 81fa3a2a7..3c7315a1a 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -784,11 +784,11 @@ const hashLookupRow = (drawer: HTMLElement, value: string) => { // Pagefind lists a page's title twice: once for the page and once for the // anchor of its h1 (#_top). The second title is hidden, its excerpt kept. -// The `files`, `aliases`, `devices`, `consoles` and `terms` fields weigh in +// The `files`, `aliases`, `devices`, `consoles`, `exact` and `terms` fields weigh in // the ranking and are never shown: a result is not a list of 120 names. Pagefind reuses // its nodes from one query to the next, so both are worked out again on // every change. -const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'terms']); +const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'exact', 'terms']); const tidyResults = (root: Element) => { for (const title of root.querySelectorAll('.pagefind-ui__result-nested > .pagefind-ui__result-title')) { diff --git a/website/tools/search_gate.py b/website/tools/search_gate.py index d5ede06f0..72ece91a2 100644 --- a/website/tools/search_gate.py +++ b/website/tools/search_gate.py @@ -63,6 +63,26 @@ QUERIES: dict[str, tuple[tuple[str, ...], int]] = { "psx bios": (("systems/sony",), 3), "dc bios missing": (("systems/sega", "emulators/flycast/"), 3), "gba bios": (("systems/nintendo",), 3), + # A console whose page answers "No file needed" says BIOS only where the + # search does not read: its `terms` field carries " BIOS". + "snes bios": (("systems/nintendo-snes/",), 1), + "nes bios": (("systems/nintendo-nes/",), 1), + "gamecube bios": (("systems/nintendo-gamecube/",), 1), + "game boy bios": (("systems/nintendo-game-boy/",), 1), + "3ds bios": (("systems/nintendo-3ds/",), 1), + # A console another one's name begins with, both ways: the longer name's + # page holds every word of the shorter query. GBA, GBC and the + # PlayStation 3 are sections of their maker's page; Saturn.emu is an + # emulator named after its console. + "playstation 2 bios": (("systems/sony-playstation-2/",), 1), + "playstation 3 bios": (("systems/sony/",), 1), + "game boy advance bios": (("systems/nintendo/",), 1), + "game boy color bios": (("systems/nintendo/",), 1), + "nintendo ds bios": (("systems/nintendo-nintendo-ds/",), 1), + "nintendo dsi bios": (("systems/nintendo-nintendo-dsi/",), 1), + "wii bios": (("systems/nintendo/",), 1), + "wii u bios": (("systems/nintendo-wii-u/",), 1), + "saturn bios": (("systems/sega-saturn/",), 1), # The words an emulator prints, and the page that answers them. "bios not found": (("wiki/troubleshooting/",), 3), "please insert playstation cd-rom": (("wiki/troubleshooting/",), 3), @@ -74,12 +94,20 @@ QUERIES: dict[str, tuple[tuple[str, ...], int]] = { # Checking a dump is what the Find a file page does. "check my dump": (("files/",), 1), "md5 check": (("files/",), 1), + "is my bios correct": (("files/",), 3), + # A folder is checked by the setup's command, which troubleshooting runs. + "check my bios folder": (("wiki/troubleshooting/",), 3), # A tool author's words: the data reference or the installer's. "manifest": (("data/", "wiki/installer/"), 3), "events": (("data/", "wiki/installer/"), 3), "exit code": (("data/", "wiki/installer/"), 3), "rate limit": (("data/", "wiki/installer/"), 3), - "ios": (("wiki/faq/",), 3), + "schema": (("data/",), 3), + "json schema": (("data/",), 3), + "license": (("data/", "wiki/legal/"), 3), + "check bios folder script": (("data/",), 3), + # The iPhone and iPad answer: copy the files to the app, no terminal. + "ios": (("which-pack/", "wiki/faq/"), 3), # A device name: the guide of the handhelds that run it. "r36s": (("platforms/", "which-pack/"), 3), "steam deck": (("platforms/emudeck/", "platforms/retrodeck/", "which-pack/"), 1), diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 9091f6571..7ba0d734b 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -57,7 +57,7 @@ Every module in `tests/`, by the area it guards. | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_offline`, `test_install_selection`, `test_install_tui` | | Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture