feat: round 4 search

.github/workflows/deploy-site.yml

@@ -134,6 +134,16 @@ jobs:
           npm run build --prefix website
           python scripts/validate_site.py --site-dir website/dist
 
+      # The queries a visitor types must reach the page that answers them,
+      # read through the site's own search box, and the home page must stay
+      # under its weight budget. A miss blocks the deploy. The browser driver
+      # is not a dependency of the build: pinned here, not saved to the
+      # lockfile, and it drives the Chrome the runner image ships.
+      - name: Search gate
+        run: |
+          npm install --no-save --prefix website playwright-core@1.58.0
+          python website/tools/search_gate.py --site-dir website/dist
+
       # write_if_changed() compares content with its timestamp line stripped,
       # so a run that only moves the clock leaves the file untouched and this
       # stays a real staleness check rather than a guaranteed failure.

scripts/generate_site.py

@@ -2109,12 +2109,23 @@ def _browser_title(relative: Path, title: str) -> str:
     }
     if key in index_titles:
         return index_titles[key]
-    if key in PART_PAGES:
+    # A console page is filed with the parts of its manufacturer's page, and
+    # titled "Sega: Dreamcast". A part names its range of files.
+    part = key in PART_PAGES and (
+        not key.startswith("systems/") or ": " not in title
+        or re.search(r", part \d+ of \d+$", title))
+    if part:
         return title
+    # The words a web search for a console or an emulator types: "dreamcast
+    # bios", "duckstation bios". The h1 and the sidebar keep the name.
     if key.startswith("emulators/"):
-        return f"{title} emulator firmware"
+        has_files = PAGE_DATA.get(key, {}).get("hasPart")
+        # Lexaloffle, QEMU and ScummVM name a system page too.
+        own = " emulator" if f"systems/{relative.name}" in PAGE_DATA else ""
+        return f"{title}{own} BIOS files" if has_files else f"{title} emulator"
     if key.startswith("systems/"):
-        return f"{title} systems"
+        maker, _, console = title.partition(": ")
+        return f"{console} BIOS files ({maker})" if console else f"{title} BIOS files"
     return title
 
 
@@ -2157,45 +2168,129 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
     }
     if key in specific:
         return specific[key]
-    if key in PART_PAGES:
-        return (f"{title}: every file with its hashes, size, source references "
-                "and who reads it.")
+    # What a search engine shows is the page's own first sentence: 707 pages
+    # shared five templates of level 3 words ("source-pinned", "provenance").
+    browser = _browser_title(relative, title)
+    held = re.search(r"\bThis part holds (.+?)\.(?: |$)", markdown, re.M)
+    if key in PART_PAGES and held:
+        return _shorten(f"{browser}. This part holds {_plain_markdown(held[1])}.", 155)
+    lead = _lead(markdown)
+    if key in PART_PAGES and re.search(r"\bpart \d+ of \d+", title):
+        # The parts of one list open on the same sentence.
+        first = _first_paragraph(lead) or _first_paragraph(markdown)
+        return _shorten(f"{title}. {first}", 155)
     if key.startswith(f"{release_record.GET_DIR}/"):
-        return (f"{title}: the BIOS files of this setup, as the ZIP of the latest "
-                "release or one command, at an address that stays the same.")
-    if key.startswith("platforms/") and relative.stem != "index":
-        return (f"{title}: where the BIOS files go, the command that installs them, the pack, "
-                "and every file by system with its hashes.")
-    if key.startswith("emulators/") and relative.stem != "index":
-        # Pinned only when the profile names the revision it was read at:
-        # 48 pages without one called themselves source-pinned.
-        about = PAGE_DATA.get(key, {}).get("about", {})
-        pinned = any(a.get("version") for a in (about if isinstance(about, list) else [about]))
-        kind = "source-pinned emulator firmware profile" if pinned else "emulator firmware profile"
-        if not PAGE_DATA.get(key, {}).get("hasPart"):
-            # 152 cores that load no file announced paths and hashes.
-            kind = kind.replace(" firmware", "")
-            trees = (" The data directories it reads are listed."
-                     if "\n## Data directories\n" in markdown else "")
-            return f"{title}: {kind}. No BIOS or firmware file is required.{trees}"
-        return f"{title}: {kind} with paths, hashes, requirements and validation behavior."
+        size = re.search(r"^- (.+ to download\b.*?)\.?$", markdown, re.M)
+        if size:
+            return _shorten(f"{title}: the BIOS pack of this setup, {_plain_markdown(size[1])}. "
+                            "The address stays the same and always gives the latest ZIP.", 155)
+        return _shorten(_first_paragraph(lead) or title, 155)
     if key.startswith("systems/") and relative.stem != "index":
-        return f"{title}: indexed firmware files, hashes, variants, provenance and platform or emulator usage."
+        return _system_description(markdown, browser)
+    if key.startswith("emulators/") and relative.stem != "index":
+        return _emulator_description(markdown, key, title, _first_paragraph(lead))
+    first = _first_paragraph(lead, minimum=35)
+    questions = [_plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading))
+                 for heading in re.findall(r"^## (.+\?)(?:[ \t]*\{[^}]*\})?[ \t]*$", markdown, re.M)]
+    if not first and len(questions) >= 3:
+        # A page of questions with no intro: its first paragraph is the
+        # answer to one of them ("Run the command with --dry-run").
+        return _shorten(f"{title}: {' '.join(questions)}", 155)
+    first = first or _first_paragraph(markdown, minimum=35)
+    if first:
+        return _shorten(first, 155)
+    return f"{title}: RetroBIOS source-traced retrogaming firmware reference."
 
-    paragraphs = re.split(r"\n\s*\n", markdown)
-    for paragraph in paragraphs:
+
+def _lead(markdown: str) -> str:
+    """The text a page opens with, before its first section, details
+    block or ignored section (the install box, the profile table)."""
+    return re.split(r"^(?:## |<details|<section)", markdown, maxsplit=1, flags=re.M)[0]
+
+
+def _emulator_description(markdown: str, key: str, title: str, intro: str) -> str:
+    """An emulator page by its intro sentence ("DuckStation emulates Sony
+    PlayStation. It picks the BIOS for your game's region: ..."), or by
+    its profile table when the profile names no system."""
+    notes = re.search(r"<summary>Technical notes</summary>\s*\n\n(.+?)(?:\n\n|$)", markdown, re.S)
+    if not intro and notes:
+        sentence = re.split(r"(?<=\.)\s", _plain_markdown(notes[1]), maxsplit=1)[0]
+        intro = f"{title}: {sentence[:1].lower()}{sentence[1:]}"
+    if not intro:
+        row = {name: _plain_markdown(value) for name, value in re.findall(
+            r"^\| (Type|Classification|Version) \| (.+?) \|$", markdown, re.M)}
+        kind = row.get("Classification") or row.get("Type") or "emulator"
+        version = row.get("Version", "")
+        known = f", version {version}" if version and version[0].isdigit() else ""
+        intro = f"{title}: {kind.lower()}{known}."
+    elif title not in intro:
+        # "It reads 1 file": the profile names no system to open on.
+        intro = f"{title}: {intro[:1].lower()}{intro[1:]}"
+    # An alias page sends the reader to the core whose files it shares.
+    if PAGE_DATA.get(key, {}).get("hasPart") or "uses the same firmware as" in intro:
+        return _shorten(intro, 155)
+    needs = "It needs no BIOS or firmware file."
+    if "\n## Data directories" in markdown:
+        needs = "It needs no BIOS or firmware file, only the data folders listed."
+    return _shorten(f"{intro} {needs}", 155)
+
+
+# Lines a page opens with that say where it sits, not what it holds.
+_PLACE_LINES = ("Part of ", "This part holds ", "On a card, ")
+
+
+def _first_paragraph(markdown: str, minimum: int = 1) -> str:
+    """The first paragraph of prose of a page body, as plain text: no
+    heading, table, list, fence, admonition or HTML block, and none that
+    only says where the page sits."""
+    for paragraph in re.split(r"\n\s*\n", markdown):
         stripped = paragraph.strip()
         if (
             not stripped
-            or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* "))
+            or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* ", "import "))
         ):
             continue
         plain = _plain_markdown(stripped)
+        if plain.startswith(_PLACE_LINES):
+            continue
         # A paragraph that ends on a colon introduces what follows: alone,
         # it describes nothing ("One file needs no clone and no local setup:").
-        if len(plain) >= 35 and not plain.endswith(":"):
-            return _shorten(plain)
-    return f"{title}: RetroBIOS source-traced retrogaming firmware reference."
+        if len(plain) >= minimum and not plain.endswith(":"):
+            return plain
+    return ""
+
+
+# Section headings of a system page that are not a console.
+_NOT_A_CONSOLE = {"Next steps", "Files", "Other files"}
+_CARD_TITLE = re.compile(r'class="rb-file-title"[^>]*><code>([^<]+)</code>')
+
+
+def _system_description(markdown: str, browser: str) -> str:
+    """A system page by what it holds: a manufacturer page by its consoles,
+    a console page by its "Start with" line, else by its first files."""
+    consoles = [
+        _plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading))
+        for heading in re.findall(r"^## (.+?)[ \t]*$", markdown, re.M)
+    ]
+    consoles = [name for name in consoles if name and name not in _NOT_A_CONSOLE]
+    if len(consoles) > 1:
+        shown = consoles if len(consoles) <= 4 else consoles[:3] + [f"{len(consoles) - 3} more"]
+        return _shorten(f"{browser} for {_said_as_list(shown)}, one card per file with its "
+                        "hashes and the emulators that read it.", 155)
+    start = re.search(r"^\*\*Start with\*\*(?P<line>.*)$(?P<list>(?:\n+- .*)*)", markdown, re.M)
+    if start:
+        # "Start with the files of your machine:" introduces one line per machine.
+        machines = [_plain_markdown(item) for item in re.findall(r"^- (.+)$", start["list"], re.M)]
+        text = _plain_markdown(start["line"]).lstrip(": ")
+        said = "; ".join(machines) if machines and text.endswith(":") else text
+        return _shorten(f"{browser}. Start with {said}", 155)
+    names = list(dict.fromkeys(html.unescape(name) for name in _CARD_TITLE.findall(markdown)))
+    if names:
+        shown = names if len(names) <= 3 else names[:3] + [f"{len(names) - 3} more"]
+        return _shorten(f"{browser}: {_said_as_list(shown)}, with their hashes and the "
+                        "emulators that read them.", 155)
+    return _shorten(f"{browser}, one card per file with its hashes and the emulators "
+                    "that read it.", 155)
 
 
 # Structured data a page generator knows beyond its text, by page path
@@ -2555,15 +2650,25 @@ LEVELS = {
 # - A guide behind the first two levels (reference, options, contributing)
 #   names everything a reader types, and ranks after the page that answers.
 # - A launcher, an alias or a frozen copy answers a name after the emulator.
-SEARCH_WEIGHT = {"start": 10, "setup": 1.5, "reference": 0.5, "contribute": 0.5}
+# - A part of a split page (a console's files, 5 of 9) holds cards, each found
+#   by its own heading, and ranks after the page that answers the question.
+SEARCH_WEIGHT = {"start": 10, "setup": 1.5, "reference": 0.5, "contribute": 0.5, "part": 0.5}
 SYSTEM_PAGE_WEIGHT = 5
 DEEP_GUIDE_WEIGHT = 0.5
 SEARCH_DEMOTED: set[str] = set()
+# A tool page a reader reaches by a task, not its name: "check my dump",
+# "md5 check" found emulator pages and never the page that checks a file.
+SEARCH_WEIGHT_OF_PAGE = {"files/index.md": 2}
 DEMOTED_WEIGHT = 0.5
 
 
 def search_weight(relative: Path, level: str) -> float | None:
-    """The search weight of a page, None for the default."""
+    """The search weight of a page, None for the default. A part page
+    states its own in its front matter: search_weight(path, "part")."""
+    if level == "part":
+        return SEARCH_WEIGHT["part"]
+    if relative.as_posix() in SEARCH_WEIGHT_OF_PAGE:
+        return SEARCH_WEIGHT_OF_PAGE[relative.as_posix()]
     if relative.as_posix() in SEARCH_DEMOTED:
         return DEMOTED_WEIGHT
     if relative.parts[0] == "systems":
@@ -2643,7 +2748,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]:
 
         title = _page_title(markdown, relative)
         browser_title = _browser_title(relative, title)
-        description = _page_description(markdown, relative, title)
+        description = str(authored.get("description") or _page_description(markdown, relative, title))
         if relative.name == "index.md":
             page_path = "" if len(relative.parts) == 1 else "/".join(relative.parts[:-1]) + "/"
         else:
@@ -2675,6 +2780,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]:
             search=authored.get("search") != "exclude",
             logo=str(authored.get("logo") or ""),
             parent=authored.get("parent"),
+            authored=authored,
         )
         write_decorated(docs / target, text)
         # One file per page: a page that gained or lost its components
@@ -8855,7 +8961,7 @@ def generate_nav(
         ],
         "reference": [
             # Find a file first: the one tool a reader comes to with a file in hand.
-            (5, {"Find a file": "files/index.md"}, "Checking files"),
+            (5, {"Find or check a file": "files/index.md"}, "Checking files"),
             (20, {"What the packs do not hold": "gaps.md"}, "Checking files"),
             (30, {"Cores by platform": "cross-reference.md"}, "Checking files"),
             (40, {"Dump provenance": "provenance.md"}, "Checking files"),

scripts/sitestarlight.py

@@ -417,6 +417,32 @@ def page_aliases(rel: str) -> list[str]:
     return list(_page_aliases.get(rel, []))
 
 
+# Search fields a page's own front matter fills, never typed here.
+SEARCH_FIELDS = ("consoles", "terms")
+
+
+def field_words(value: object) -> list[str]:
+    """The words of a front matter search field: a list, or one string
+    with its words or phrases split at commas."""
+    if isinstance(value, str):
+        return [part.strip() for part in value.split(",") if part.strip()]
+    if isinstance(value, (list, tuple)):
+        return [str(part).strip() for part in value if str(part).strip()]
+    return []
+
+
+def _neighbour(rel: str, value: object) -> object:
+    """A `prev:` or `next:` the page states, its link resolved like any
+    other link of the page."""
+    if isinstance(value, dict) and value.get("link"):
+        # Starlight writes the link as given: a site path gets the base.
+        link = resolve(rel, str(value["link"]))
+        if link.startswith("/") and not link.startswith(BASE):
+            link = BASE + link.lstrip("/")
+        return {**value, "link": link}
+    return value
+
+
 _NEXT_STEPS = re.compile(r"^## Next steps\b[^\n]*\n+(?P<list>(?:[ \t]*[-*] .*\n?)+)", re.M)
 
 
@@ -445,11 +471,20 @@ def finalize_page(
     search: bool = True,
     logo: str = "",
     parent: dict | None = None,
+    authored: dict | None = None,
 ) -> tuple[str, str]:
     """The Starlight file for a page body: its path under the content
     collection and its text. *parent* ({label, link}, the link written
     relative to the page) is the page a part page belongs to: the route
-    middleware makes it the last crumb and marks it in the sidebar."""
+    middleware makes it the last crumb and marks it in the sidebar.
+
+    *authored* is the front matter the page was written with. Its
+    `searchWeight`, `prev` and `next` win over the computed ones (a part
+    page states its neighbours and its weight), and its `consoles` and
+    `terms` become search fields (astro.config.mjs, metaWeights): the
+    consoles a manufacturer page holds sections for, and the words a page
+    answers that its text does not say, as an error message words them."""
+    authored = authored or {}
     title, title_id, body = split_title(markdown)
     body = body.lstrip("\n")
     body = rewrite_links(body, rel)
@@ -468,6 +503,8 @@ def finalize_page(
                   r'<span data-pagefind-ignore class="\1"', body)
     if rel.startswith("systems/"):
         body += file_names_meta(body) + search_field("aliases", page_aliases(rel))
+    for key in SEARCH_FIELDS:
+        body += search_field(key, field_words(authored.get(key)))
     components = used_components(body)
     stem = rel.rsplit(".", 1)[0]
     if components:
@@ -492,6 +529,7 @@ def finalize_page(
     if parent:
         front["parent"] = {"label": str(parent["label"]),
                            "link": resolve(rel, str(parent["link"]))}
+    weight = authored.get("searchWeight", weight)
     if weight:
         front["searchWeight"] = weight
     if rel == NOT_FOUND:
@@ -512,6 +550,9 @@ def finalize_page(
     elif step:
         # The page's own "Next steps" list and the next link agree.
         front["next"] = step
+    for side in ("prev", "next"):
+        if rel != NOT_FOUND and side in authored:
+            front[side] = _neighbour(rel, authored[side])
     front["head"] = page_head(rel, level, front["title"], browser_title, structured)
     if structured.get("dateModified"):
         front["lastmod"] = str(structured["dateModified"])

scripts/validate_site.py

@@ -25,6 +25,7 @@ from urllib.parse import unquote, urlsplit
 class Page:
     path: Path
     title: str = ""
+    h1: str = ""
     lang: str = ""
     descriptions: list[str] = field(default_factory=list)
     canonical: list[str] = field(default_factory=list)
@@ -66,6 +67,11 @@ OVER_ELEMENT_BUDGET: dict[str, str] = {}
 MAX_PAGEFIND_CHUNK_BYTES = 150_000
 MAX_PAGEFIND_META_BYTES = 50_000
 MAX_PAGEFIND_TOTAL_BYTES = 12_000_000
+# Pages whose description is one text once the page's own name is taken
+# out. 707 pages shared five templates ("X: source-pinned emulator
+# firmware profile with paths, hashes..."), and a search engine showed the
+# same line under every result.
+MAX_SHARED_DESCRIPTION = 10
 
 
 # Text inside these is shown as written, so markup there is not a leftover.
@@ -78,6 +84,8 @@ class _PageParser(HTMLParser):
         self.page = Page(path)
         self._title_depth = 0
         self._title_parts: list[str] = []
+        self._h1_depth = 0
+        self._h1_parts: list[str] = []
         self._jsonld_depth = 0
         self._jsonld_parts: list[str] = []
         self._literal_depth = 0
@@ -108,6 +116,7 @@ class _PageParser(HTMLParser):
             self.page.descriptions.append(data.get("content", "").strip())
         elif tag == "h1":
             self.page.h1_count += 1
+            self._h1_depth += 1
         elif tag == "main":
             self.page.main_count += 1
         elif tag == "img" and "alt" not in data:
@@ -148,6 +157,8 @@ class _PageParser(HTMLParser):
             self._header_parts = None
         if tag == "title":
             self._title_depth -= 1
+        elif tag == "h1" and self._h1_depth:
+            self._h1_depth -= 1
         elif tag == "script" and self._jsonld_depth:
             self.page.jsonld.append("".join(self._jsonld_parts).strip())
             self._jsonld_depth -= 1
@@ -158,6 +169,8 @@ class _PageParser(HTMLParser):
             self._header_parts.append(data)
         if self._title_depth:
             self._title_parts.append(data)
+        if self._h1_depth:
+            self._h1_parts.append(data)
         if self._jsonld_depth:
             self._jsonld_parts.append(data)
         if not self._literal_depth and "**" in data:
@@ -166,6 +179,7 @@ class _PageParser(HTMLParser):
     def close(self) -> None:
         super().close()
         self.page.title = "".join(self._title_parts).strip()
+        self.page.h1 = re.sub(r"\s+", " ", "".join(self._h1_parts)).strip()
 
 
 def _parse_page(path: Path) -> Page:
@@ -322,6 +336,31 @@ def _check_pages(
     return issues
 
 
+def description_template(page: Page, site_name: str = "RetroBIOS") -> str:
+    """A page's description with its own names taken out: its h1 and its
+    window title. What is left is the text it shares with its template."""
+    text = page.descriptions[0] if page.descriptions else ""
+    names = {page.h1, page.title.removesuffix(f" - {site_name}").strip()}
+    for name in sorted(filter(None, names), key=len, reverse=True):
+        text = text.replace(name, "")
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def _check_description_templates(pages: dict, site: Path) -> list[str]:
+    """No more than MAX_SHARED_DESCRIPTION pages share one description
+    once their names are taken out."""
+    shared: dict[str, list[Path]] = defaultdict(list)
+    for path, page in pages.items():
+        if path.relative_to(site) != Path("404.html") and page.descriptions:
+            shared[description_template(page)].append(path)
+    return [
+        f"{len(paths)} pages share the description {text!r} after their name: "
+        + ", ".join(str(path.relative_to(site)) for path in paths[:3])
+        for text, paths in shared.items()
+        if len(paths) > MAX_SHARED_DESCRIPTION
+    ]
+
+
 def _check_links(pages: dict, site: Path, base_path: str) -> list[str]:
     """Every local link and fragment resolves to something on disk."""
     issues: list[str] = []
@@ -422,6 +461,7 @@ def validate_site(
                 rendered = ", ".join(str(path.relative_to(site)) for path in paths[:5])
                 issues.append(f"duplicate {label} {value!r}: {rendered}")
 
+    issues.extend(_check_description_templates(content, site))
     issues.extend(_check_links(pages, site, base_path))
     issues.extend(_check_pagefind(site))
     if platforms_dir is not None:

tests/test_site_emulator_page.py

@@ -103,9 +103,9 @@ class WhatASearchSnippetSays(unittest.TestCase):
         gs.PAGE_DATA[rel.as_posix()] = {}
         text = gs._page_description("# Lutro\n", rel, "Lutro")
         self.assertNotIn("hashes", text)
-        self.assertIn("No BIOS or firmware file is required.", text)
+        self.assertIn("It needs no BIOS or firmware file.", text)
         gs.PAGE_DATA[rel.as_posix()] = {"hasPart": [{"name": "a.bin"}]}
-        self.assertIn("paths, hashes", gs._page_description("# Lutro\n", rel, "Lutro"))
+        self.assertNotIn("needs no BIOS", gs._page_description("# Lutro\n", rel, "Lutro"))
 
     def test_a_long_paragraph_is_cut_at_a_word(self):
         plain = ("files/?sha1= is the address that does not move: it opens the card "

tests/test_site_exports.py

@@ -179,14 +179,15 @@ class SiteReferenceContracts(unittest.TestCase):
         self.assertNotIn("\n    #if FEATURE", rendered)
 
     def test_browser_titles_distinguish_emulators_from_system_pages(self):
-        self.assertEqual(
-            _browser_title(Path("emulators/qemu.md"), "QEMU"),
-            "QEMU emulator firmware",
-        )
-        self.assertEqual(
-            _browser_title(Path("systems/qemu.md"), "QEMU"),
-            "QEMU systems",
-        )
+        from unittest import mock  # noqa: PLC0415
+
+        import generate_site  # noqa: PLC0415
+
+        pages = {"systems/qemu.md": {}, "emulators/qemu.md": {"hasPart": [{"name": "a"}]}}
+        with mock.patch.dict(generate_site.PAGE_DATA, pages, clear=True):
+            self.assertEqual(_browser_title(Path("emulators/qemu.md"), "QEMU"),
+                             "QEMU emulator BIOS files")
+            self.assertEqual(_browser_title(Path("systems/qemu.md"), "QEMU"), "QEMU BIOS files")
 
     ACCEPTED = (
         ("md5", "c87a52c277f7952b41c639fc7bf0a43b"),
@@ -581,7 +582,8 @@ class OnlyAPinnedProfileClaimsAPin(unittest.TestCase):
         with mock.patch.dict(generate_site.PAGE_DATA, pages, clear=True):
             pinned = generate_site._page_description("", Path("emulators/pinned.md"), "P")
             loose = generate_site._page_description("", Path("emulators/redream.md"), "R")
-        self.assertIn("source-pinned", pinned)
+        # A search snippet is level 0: the pin is on the page, not in it.
+        self.assertNotIn("source-pinned", pinned)
         self.assertNotIn("source-pinned", loose)
 
 

tests/test_site_r2_nav.py

@@ -233,7 +233,7 @@ class ReferenceGrouped(unittest.TestCase):
         reference = self._reference()
         groups = {next(iter(item)): next(iter(item.values())) for item in reference
                   if isinstance(item, dict) and isinstance(next(iter(item.values())), list)}
-        self.assertEqual(groups["Checking files"][0], {"Find a file": "files/index.md"})
+        self.assertEqual(groups["Checking files"][0], {"Find or check a file": "files/index.md"})
         self.assertIn({"What the packs do not hold": "gaps.md"}, groups["Checking files"])
         self.assertEqual(groups["Data"][:3], [{"Data and API": "data.md"},
                                               {"Data model": "wiki/data-model.md"},
@@ -250,7 +250,7 @@ class ReferenceGrouped(unittest.TestCase):
             reference = self._reference(wiki)
         group = next(item["Checking files"] for item in reference
                      if isinstance(item, dict) and "Checking files" in item)
-        self.assertEqual(group[:2], [{"Find a file": "files/index.md"},
+        self.assertEqual(group[:2], [{"Find or check a file": "files/index.md"},
                                      {"How each platform checks files": "wiki/modes.md"}])
 
 

tests/test_site_r4_search.py

@@ -0,0 +1,259 @@
+"""Search (round 4, package 3).
+
+The search gate listed the queries that missed and no workflow ran it: its
+browser runner could not even start from the repository root. Console and
+error words had no field to reach their page ("wonderswan color", "bios not
+found"), parts of a split page carried the console page's weight and came
+first for "which mame version", and 707 pages gave search engines one of
+five template sentences of level 3 words as their description. These tests
+read the generator, the gate, the workflow and, where it is built, the site.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_site as gs  # noqa: E402
+import sitestarlight  # noqa: E402
+import validate_site  # noqa: E402
+
+TOOLS = REPO_ROOT / "website" / "tools"
+WORKFLOW = REPO_ROOT / ".github" / "workflows" / "deploy-site.yml"
+DOCS = REPO_ROOT / "website" / "src" / "content" / "docs"
+DIST = REPO_ROOT / "website" / "dist"
+
+
+def _front(text: str) -> dict:
+    return yaml.safe_load(text.split("---\n")[1])
+
+
+def _gate_queries() -> dict:
+    tree = ast.parse((TOOLS / "search_gate.py").read_text(encoding="utf-8"))
+    for node in tree.body:
+        if isinstance(node, ast.AnnAssign) and getattr(node.target, "id", "") == "QUERIES":
+            return ast.literal_eval(node.value)
+    raise AssertionError("no QUERIES in search_gate.py")
+
+
+def _finalize(rel: str, body: str, authored: dict) -> dict:
+    _target, text = sitestarlight.finalize_page(
+        rel, body, level="", weight=None, browser_title="T", description="d",
+        structured={}, terms={}, authored=authored)
+    return {"front": _front(text), "text": text}
+
+
+class TheGateRunsAndBlocks(unittest.TestCase):
+    def test_the_deploy_runs_the_gate_before_the_upload(self) -> None:
+        steps = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"]["build"]["steps"]
+        runs = [str(step.get("run", "")) for step in steps]
+        uses = [str(step.get("uses", "")) for step in steps]
+        gate = next(i for i, run in enumerate(runs)
+                    if "website/tools/search_gate.py --site-dir website/dist" in run)
+        build = next(i for i, run in enumerate(runs) if "npm run build --prefix website" in run)
+        upload = next(i for i, use in enumerate(uses) if "upload-pages-artifact" in use)
+        self.assertLess(build, gate)
+        self.assertLess(gate, upload)
+        # The driver is pinned to a version and kept out of the lockfile.
+        self.assertRegex(runs[gate], r"npm install --no-save --prefix website playwright-core@\d+\.\d+\.\d+")
+
+    def test_no_third_party_action_and_every_action_pinned(self) -> None:
+        text = WORKFLOW.read_text(encoding="utf-8")
+        for repo, ref in re.findall(r"uses:\s*([\w.-]+/[\w.-]+)@(\S+)", text):
+            with self.subTest(action=repo):
+                self.assertTrue(repo.startswith("actions/"))
+                self.assertRegex(ref, r"^[0-9a-f]{40}$")
+
+    def test_the_runner_resolves_from_website_and_says_what_is_missing(self) -> None:
+        source = (TOOLS / "measure.mjs").read_text(encoding="utf-8")
+        self.assertIn("await import('playwright-core')", source)
+        self.assertIn("npm install --no-save --prefix website playwright-core", source)
+        self.assertNotRegex(source, r"^import \{ chromium \}", "a static import fails before any message")
+
+    def test_the_gate_serves_a_build_dir_and_fails_on_a_miss(self) -> None:
+        source = (TOOLS / "search_gate.py").read_text(encoding="utf-8")
+        self.assertIn("--site-dir", source)
+        self.assertIn("serve.mjs", source)
+        self.assertIn("return 0 if answered == len(QUERIES)", source)
+
+    def test_the_plans_queries_are_gated(self) -> None:
+        queries = _gate_queries()
+        expected = {
+            "playstation bios": ("systems/sony-playstation/", 1),
+            "please insert playstation cd-rom": ("wiki/troubleshooting/", 3),
+            "bios not found": ("wiki/troubleshooting/", 3),
+            "check my dump": ("files/", 1),
+            "md5 check": ("files/", 1),
+            "manifest": ("data/", 3),
+            "events": ("wiki/installer/", 3),
+            "exit code": ("wiki/installer/", 3),
+            "rate limit": ("data/", 3),
+            "ios": ("wiki/faq/", 3),
+            "dc bios missing": ("emulators/flycast/", 3),
+            "neogeo.zip": ("systems/snk/", 1),
+            "scph5501": ("systems/sony-playstation/", 3),
+        }
+        for query, (page, rank) in expected.items():
+            with self.subTest(query=query):
+                pages, gated = queries[query]
+                self.assertTrue(any(page.startswith(prefix) for prefix in pages))
+                self.assertLessEqual(gated, rank)
+
+    def test_the_home_has_a_weight_budget(self) -> None:
+        source = (TOOLS / "search_gate.py").read_text(encoding="utf-8")
+        self.assertIn("HOME_BUDGET_KB = 80", source)
+
+
+class FieldsFromFrontMatter(unittest.TestCase):
+    def test_consoles_and_terms_become_search_fields(self) -> None:
+        text = _finalize("systems/snk.md", "# SNK\n\nBody.\n", {
+            "consoles": ["Neo Geo", "Neo Geo Pocket Color"],
+            "terms": "bios not found, bios missing",
+        })["text"]
+        self.assertIn('data-pagefind-meta="consoles:Neo Geo Neo Geo Pocket Color"', text)
+        self.assertIn('data-pagefind-meta="terms:bios not found bios missing"', text)
+        # A field value is an attribute: none of it reaches an excerpt.
+        self.assertNotRegex(text, r">[^<]*bios missing[^<]*<")
+
+    def test_no_search_word_is_typed_in_the_module(self) -> None:
+        source = (REPO_ROOT / "scripts" / "sitestarlight.py").read_text(encoding="utf-8")
+        for words in ("bios not found", "mame version", "WonderSwan"):
+            self.assertNotIn(words, source)
+
+    def test_a_page_without_fields_gets_none(self) -> None:
+        text = _finalize("wiki/legal.md", "# Legal\n\nBody.\n", {})["text"]
+        self.assertNotIn("consoles:", text)
+        self.assertNotIn("terms:", text)
+
+
+class FindOrCheckAFile(unittest.TestCase):
+    """"check my dump" and "md5 check" found emulator pages first."""
+
+    def test_the_page_weighs_two(self) -> None:
+        rel = Path("files/index.md")
+        self.assertEqual(gs.search_weight(rel, gs.page_level(rel)), 2)
+
+    def test_its_terms_become_a_field(self) -> None:
+        text = _finalize("files/index.md", "# Find or check a file\n\nBody.\n",
+                         {"terms": "check dump, md5 check"})["text"]
+        self.assertIn('data-pagefind-meta="terms:check dump md5 check"', text)
+
+    def test_the_sidebar_names_it_as_its_title(self) -> None:
+        source = (REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")
+        self.assertIn('{"Find or check a file": "files/index.md"}', source)
+        self.assertNotIn('{"Find a file": "files/index.md"}', source)
+
+
+class PartPages(unittest.TestCase):
+    def test_part_weight(self) -> None:
+        self.assertEqual(gs.SEARCH_WEIGHT["part"], 0.5)
+        self.assertEqual(gs.search_weight(Path("systems/arcade-mame-5.md"), "part"), 0.5)
+
+    def test_a_part_keeps_its_own_weight_and_neighbours(self) -> None:
+        """The MAME card parts write prev, next and searchWeight: the build
+        kept only parent and wrote prev: false, next: false."""
+        page = (
+            "---\n"
+            "parent:\n  label: MAME\n  link: mame.md\n"
+            "prev:\n  label: MAME files, part 2 of 8\n  link: mame-files-2.md\n"
+            "next:\n  label: MAME files, part 4 of 8\n  link: mame-files-4.md\n"
+            f"searchWeight: {gs.search_weight(Path('emulators/mame-files-3.md'), 'part')}\n"
+            "---\n\n# MAME files, part 3 of 8\n\nPart of [MAME](mame.md). "
+            "This part holds 196 entries, `a.zip` to `b.zip`.\n"
+        )
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+            path = Path(tmp) / "emulators" / "mame-files-3.md"
+            path.parent.mkdir()
+            path.write_text(page, encoding="utf-8")
+            gs.PART_PAGES.add("emulators/mame-files-3.md")
+            self.addCleanup(gs.PART_PAGES.discard, "emulators/mame-files-3.md")
+            gs.decorate_markdown_pages(Path(tmp))
+            front = _front(path.read_text(encoding="utf-8"))
+        self.assertEqual(front["searchWeight"], 0.5)
+        self.assertEqual(front["prev"], {"label": "MAME files, part 2 of 8",
+                                         "link": "/retrobios/emulators/mame-files-2/"})
+        self.assertEqual(front["next"]["link"], "/retrobios/emulators/mame-files-4/")
+        self.assertEqual(front["parent"]["link"], "/retrobios/emulators/mame/")
+        self.assertIn("This part holds 196 entries, a.zip to b.zip", front["description"])
+
+    def test_built_mame_parts_state_their_neighbours(self) -> None:
+        part = DOCS / "emulators" / "mame-files-3.md"
+        if not part.is_file():
+            self.skipTest("site pages not generated")
+        front = _front(part.read_text(encoding="utf-8"))
+        if "searchWeight" not in front and front.get("prev") is False:
+            self.skipTest("the emulator pages package is not merged")
+        self.assertEqual(front["searchWeight"], 0.5)
+        self.assertTrue(str(front["prev"]["link"]).startswith("/retrobios/emulators/"))
+
+
+class Descriptions(unittest.TestCase):
+    def test_titles_say_bios_files(self) -> None:
+        saved = dict(gs.PAGE_DATA)
+        self.addCleanup(lambda: (gs.PAGE_DATA.clear(), gs.PAGE_DATA.update(saved)))
+        gs.PAGE_DATA["emulators/duckstation.md"] = {"hasPart": [{"name": "scph5501.bin"}]}
+        self.assertEqual(gs._browser_title(Path("emulators/duckstation.md"), "DuckStation"),
+                         "DuckStation BIOS files")
+        self.assertEqual(gs._browser_title(Path("systems/sega.md"), "Sega"), "Sega BIOS files")
+        self.assertEqual(gs._browser_title(Path("systems/sega-dreamcast.md"), "Sega: Dreamcast"),
+                         "Dreamcast BIOS files (Sega)")
+
+    def test_an_emulator_page_is_described_by_its_intro(self) -> None:
+        saved = dict(gs.PAGE_DATA)
+        self.addCleanup(lambda: (gs.PAGE_DATA.clear(), gs.PAGE_DATA.update(saved)))
+        gs.PAGE_DATA["emulators/duckstation.md"] = {"hasPart": [{"name": "scph5501.bin"}]}
+        body = ("# DuckStation\n\nDuckStation emulates Sony PlayStation. It picks the BIOS for "
+                "your game's region.\n\n<details>\n\nThe installer finds your platform.\n")
+        text = gs._page_description(body, Path("emulators/duckstation.md"), "DuckStation")
+        self.assertTrue(text.startswith("DuckStation emulates Sony PlayStation."))
+        self.assertNotIn("source-pinned", text)
+
+    def test_a_console_page_is_described_by_its_start_line(self) -> None:
+        body = "# Sega: Dreamcast\n\nPart of Sega.\n\n**Start with**: `dc_boot.bin`, `dc_flash.bin`.\n"
+        text = gs._page_description(body, Path("systems/sega-dreamcast.md"), "Sega: Dreamcast")
+        self.assertEqual(text, "Dreamcast BIOS files (Sega). Start with dc_boot.bin, dc_flash.bin.")
+
+    def test_the_faq_is_described_by_its_questions(self) -> None:
+        body = ("# Common questions\n\n## Do I need BIOS files?\n\nYes.\n\n## Which one?\n\n"
+                "Run the command with --dry-run.\n\n## Zip or not?\n\nZip.\n")
+        text = gs._page_description(body, Path("wiki/faq.md"), "Common questions")
+        self.assertTrue(text.startswith("Common questions: Do I need BIOS files?"))
+
+    def test_a_written_description_wins(self) -> None:
+        source = (REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")
+        self.assertIn('authored.get("description") or _page_description(', source)
+
+    def test_the_check_counts_templates(self) -> None:
+        pages = {}
+        for n in range(12):
+            page = validate_site.Page(Path(f"/s/p{n}/index.html"))
+            page.h1 = f"Core {n}"
+            page.title = f"Core {n} BIOS files - RetroBIOS"
+            page.descriptions = [f"Core {n}: emulator firmware profile with paths."]
+            pages[page.path] = page
+        issues = validate_site._check_description_templates(pages, Path("/s"))
+        self.assertEqual(len(issues), 1)
+        self.assertIn("12 pages share", issues[0])
+        for page in list(pages.values())[:2]:
+            page.descriptions = [f"{page.h1} emulates a console of its own {page.h1[-1]}x."]
+        self.assertEqual(validate_site._check_description_templates(pages, Path("/s")), [])
+
+    def test_built_descriptions_are_not_templates(self) -> None:
+        if not (DIST / "index.html").is_file():
+            self.skipTest("site not built")
+        pages = {path.resolve(): validate_site._parse_page(path) for path in DIST.rglob("*.html")}
+        content = {path: page for path, page in pages.items() if not page.redirect}
+        self.assertEqual(validate_site._check_description_templates(content, DIST.resolve()), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

website/tools/measure.mjs

@@ -2,10 +2,17 @@
 //   node measure.mjs weight <base url> <page>...
 //   node measure.mjs search <base url> starlight|mkdocs <query>...
 // Transfer sizes are bytes on the wire (gzip from serve.mjs), by type.
-import { chromium } from 'playwright-core';
+// playwright-core is resolved from website/node_modules, above this file,
+// wherever the command is run from. It is not a dependency of the build:
+//   npm install --no-save --prefix website playwright-core
+const { chromium } = await import('playwright-core').catch((error) => {
+	console.error(`playwright-core is not in website/node_modules (${error.code}): `
+		+ 'npm install --no-save --prefix website playwright-core');
+	process.exit(2);
+});
 
 const [mode, base, ...rest] = process.argv.slice(2);
-const browser = await chromium.launch({ executablePath: '/usr/bin/google-chrome' });
+const browser = await chromium.launch({ executablePath: process.env.CHROME_PATH || '/usr/bin/google-chrome' });
 
 const track = (page) => {
 	const rows = [];

website/tools/search_gate.py

@@ -3,19 +3,36 @@
 
 Runs measure.mjs against a served build, through the site's own search box,
 and checks each query's expected pages against the rank it must reach: the
-first result, or the first three.
+first result, or the first three. Exits 1 when a query misses.
 
+    python website/tools/search_gate.py --site-dir website/dist
     python website/tools/search_gate.py http://127.0.0.1:4321/retrobios/
+
+With --site-dir the build is served by serve.mjs (as GitHub Pages serves it)
+on a free port for the run. The browser driver is playwright-core in
+website/node_modules (npm install --no-save --prefix website playwright-core)
+and Chrome at /usr/bin/google-chrome, or CHROME_PATH.
 """
 
 from __future__ import annotations
 
+import argparse
 import json
+import socket
 import subprocess
 import sys
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Iterator
+from contextlib import contextmanager
 from pathlib import Path
 
 TOOLS = Path(__file__).resolve().parent
+BASE = "/retrobios/"
+# What the home page transfers on a phone, everything it loads before a
+# visitor does anything (gzip, as GitHub Pages serves it).
+HOME_BUDGET_KB = 80
 
 # Query: (pages that answer it, as path prefixes under the site, and the rank one
 # of them must reach). An emulator name must find the emulator itself first,
@@ -29,23 +46,38 @@ QUERIES: dict[str, tuple[tuple[str, ...], int]] = {
     # The file's card on its system page, where it is downloaded. "systems/sony"
     # holds the manufacturer page and the console's own (systems/sony-playstation/).
     "scph5501": (("systems/sony",), 3),
-    "md5": (("wiki/verification-modes/",), 1),
+    # How a setup checks a hash, or the page that checks one.
+    "md5": (("wiki/verification-modes/", "files/"), 1),
     "where do bios files go": (("which-pack/", "wiki/first-install/"), 1),
     "dolphin": (("emulators/dolphin/",), 1),
     "duckstation": (("emulators/duckstation/",), 1),
-    # A console and "bios": the console's page, before any setup's list for it.
-    "playstation bios": (("systems/sony",), 3),
+    # A console and "bios": the console's own page first, before any setup's
+    # list for it or a part of its manufacturer's page.
+    "playstation bios": (("systems/sony-playstation/",), 1),
     "dreamcast bios": (("systems/sega",), 3),
     "neo geo bios": (("systems/snk",), 3),
     "wonderswan color": (("systems/bandai",), 3),
     # The platforms' own names for a console (psx, dc, gba), a search field
-    # of its page.
+    # of its page. Flycast's page names dc_boot.bin and where it goes.
     "psx bios": (("systems/sony",), 3),
-    "dc bios missing": (("systems/sega",), 3),
+    "dc bios missing": (("systems/sega", "emulators/flycast/"), 3),
     "gba bios": (("systems/nintendo",), 3),
+    # The words an emulator prints, and the page that answers them.
     "bios not found": (("wiki/troubleshooting/",), 3),
+    "please insert playstation cd-rom": (("wiki/troubleshooting/",), 3),
     "offline install": (("wiki/installer/",), 3),
-    "which mame version": (("wiki/technical-faq/",), 3),
+    # The table of versions, or the FAQ answer headed with the question,
+    # which links it.
+    "which mame version": (("wiki/technical-faq/", "wiki/faq/"), 3),
+    # Checking a dump is what the Find a file page does.
+    "check my dump": (("files/",), 1),
+    "md5 check": (("files/",), 1),
+    # A tool author's words: the data reference or the installer's.
+    "manifest": (("data/", "wiki/installer/"), 3),
+    "events": (("data/", "wiki/installer/"), 3),
+    "exit code": (("data/", "wiki/installer/"), 3),
+    "rate limit": (("data/", "wiki/installer/"), 3),
+    "ios": (("wiki/faq/",), 3),
     # A device name: the guide of the handhelds that run it.
     "r36s": (("platforms/", "which-pack/"), 3),
     "steam deck": (("platforms/emudeck/", "platforms/retrodeck/", "which-pack/"), 1),
@@ -53,13 +85,44 @@ QUERIES: dict[str, tuple[tuple[str, ...], int]] = {
 }
 
 
-def main() -> int:
-    base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:4321/retrobios/"
+def _free_port() -> int:
+    with socket.socket() as probe:
+        probe.bind(("127.0.0.1", 0))
+        return int(probe.getsockname()[1])
+
+
+@contextmanager
+def served(site_dir: Path) -> Iterator[str]:
+    """The build served by serve.mjs on a free port, for the run."""
+    port = _free_port()
+    server = subprocess.Popen(["node", str(TOOLS / "serve.mjs"), str(site_dir), str(port)])
+    url = f"http://127.0.0.1:{port}{BASE}"
+    try:
+        for _ in range(100):
+            try:
+                with urllib.request.urlopen(url, timeout=2):
+                    break
+            except (urllib.error.URLError, ConnectionError):
+                if server.poll() is not None:
+                    raise RuntimeError(f"serve.mjs exited with {server.returncode}") from None
+                time.sleep(0.1)
+        else:
+            raise RuntimeError(f"serve.mjs did not answer on {url}")
+        yield url
+    finally:
+        server.terminate()
+        server.wait(timeout=10)
+
+
+def run_gate(base: str) -> int:
     root = "/" + base.split("/", 3)[3]
     run = subprocess.run(
         ["node", str(TOOLS / "measure.mjs"), "search", base, "starlight-ui", *QUERIES],
-        capture_output=True, text=True, check=True,
+        capture_output=True, text=True, check=False,
     )
+    if run.returncode:
+        print(run.stderr.strip() or f"measure.mjs exited with {run.returncode}", file=sys.stderr)
+        return 1
     answered = 0
     for line in run.stdout.splitlines():
         result = json.loads(line)
@@ -70,10 +133,38 @@ def main() -> int:
         top = [entry.split(" | ") for entry in result["top"][:3]]
         hit = any(href[len(root):].startswith(pages) for _title, href in top[:rank])
         answered += hit
-        titles = "; ".join(title for title, _href in top)
-        print(f"{'ok  ' if hit else 'MISS'} {result['query']!r} (top {rank}): {titles}")
+        shown = "; ".join(f"{title} ({href[len(root):] or '/'})" for title, href in top)
+        print(f"{'ok  ' if hit else 'MISS'} {result['query']!r} (top {rank}): {shown}")
     print(f"{answered}/{len(QUERIES)} queries answered")
-    return 0 if answered == len(QUERIES) else 1
+    home = home_weight(base)
+    print(f"{'ok  ' if home <= HOME_BUDGET_KB else 'OVER'} home: {home:.1f} KB "
+          f"transferred at 390 px (budget {HOME_BUDGET_KB} KB)")
+    return 0 if answered == len(QUERIES) and home <= HOME_BUDGET_KB else 1
+
+
+def home_weight(base: str) -> float:
+    """The home page's transfer at 390 px, once the network is idle."""
+    run = subprocess.run(
+        ["node", str(TOOLS / "measure.mjs"), "weight", base, ""],
+        capture_output=True, text=True, check=True,
+    )
+    return float(json.loads(run.stdout.splitlines()[0])["total_kb"])
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("url", nargs="?", default="http://127.0.0.1:4321/retrobios/",
+                        help="a served build, under /retrobios/ (default: %(default)s)")
+    parser.add_argument("--site-dir", type=Path,
+                        help="serve this build for the run instead of reading URL")
+    args = parser.parse_args()
+    if args.site_dir is None:
+        return run_gate(args.url)
+    if not (args.site_dir / "pagefind").is_dir():
+        print(f"{args.site_dir}: no built site with a search index", file=sys.stderr)
+        return 1
+    with served(args.site_dir) as url:
+        return run_gate(url)
 
 
 if __name__ == "__main__":

wiki/testing-guide.md

@@ -57,7 +57,7 @@ Every module in `tests/`, by the area it guards.
 | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_from`, `test_install_offline`, `test_install_selection`, `test_install_tui` |
 | Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
-| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_p0`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_p0`, `test_site_r4_search`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
 
 ## Test architecture