feat: search fields and descriptions from pages

scripts/generate_site.py

@@ -2109,12 +2109,23 @@ def _browser_title(relative: Path, title: str) -> str:
     }
     if key in index_titles:
         return index_titles[key]
-    if key in PART_PAGES:
+    # A console page is filed with the parts of its manufacturer's page, and
+    # titled "Sega: Dreamcast". A part names its range of files.
+    part = key in PART_PAGES and (
+        not key.startswith("systems/") or ": " not in title
+        or re.search(r", part \d+ of \d+$", title))
+    if part:
         return title
+    # The words a web search for a console or an emulator types: "dreamcast
+    # bios", "duckstation bios". The h1 and the sidebar keep the name.
     if key.startswith("emulators/"):
-        return f"{title} emulator firmware"
+        has_files = PAGE_DATA.get(key, {}).get("hasPart")
+        # Lexaloffle, QEMU and ScummVM name a system page too.
+        own = " emulator" if f"systems/{relative.name}" in PAGE_DATA else ""
+        return f"{title}{own} BIOS files" if has_files else f"{title} emulator"
     if key.startswith("systems/"):
-        return f"{title} systems"
+        maker, _, console = title.partition(": ")
+        return f"{console} BIOS files ({maker})" if console else f"{title} BIOS files"
     return title
 
 
@@ -2157,45 +2168,129 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
     }
     if key in specific:
         return specific[key]
-    if key in PART_PAGES:
-        return (f"{title}: every file with its hashes, size, source references "
-                "and who reads it.")
+    # What a search engine shows is the page's own first sentence: 707 pages
+    # shared five templates of level 3 words ("source-pinned", "provenance").
+    browser = _browser_title(relative, title)
+    held = re.search(r"\bThis part holds (.+?)\.(?: |$)", markdown, re.M)
+    if key in PART_PAGES and held:
+        return _shorten(f"{browser}. This part holds {_plain_markdown(held[1])}.", 155)
+    lead = _lead(markdown)
+    if key in PART_PAGES and re.search(r"\bpart \d+ of \d+", title):
+        # The parts of one list open on the same sentence.
+        first = _first_paragraph(lead) or _first_paragraph(markdown)
+        return _shorten(f"{title}. {first}", 155)
     if key.startswith(f"{release_record.GET_DIR}/"):
-        return (f"{title}: the BIOS files of this setup, as the ZIP of the latest "
-                "release or one command, at an address that stays the same.")
-    if key.startswith("platforms/") and relative.stem != "index":
-        return (f"{title}: where the BIOS files go, the command that installs them, the pack, "
-                "and every file by system with its hashes.")
-    if key.startswith("emulators/") and relative.stem != "index":
-        # Pinned only when the profile names the revision it was read at:
-        # 48 pages without one called themselves source-pinned.
-        about = PAGE_DATA.get(key, {}).get("about", {})
-        pinned = any(a.get("version") for a in (about if isinstance(about, list) else [about]))
-        kind = "source-pinned emulator firmware profile" if pinned else "emulator firmware profile"
-        if not PAGE_DATA.get(key, {}).get("hasPart"):
-            # 152 cores that load no file announced paths and hashes.
-            kind = kind.replace(" firmware", "")
-            trees = (" The data directories it reads are listed."
-                     if "\n## Data directories\n" in markdown else "")
-            return f"{title}: {kind}. No BIOS or firmware file is required.{trees}"
-        return f"{title}: {kind} with paths, hashes, requirements and validation behavior."
+        size = re.search(r"^- (.+ to download\b.*?)\.?$", markdown, re.M)
+        if size:
+            return _shorten(f"{title}: the BIOS pack of this setup, {_plain_markdown(size[1])}. "
+                            "The address stays the same and always gives the latest ZIP.", 155)
+        return _shorten(_first_paragraph(lead) or title, 155)
     if key.startswith("systems/") and relative.stem != "index":
-        return f"{title}: indexed firmware files, hashes, variants, provenance and platform or emulator usage."
+        return _system_description(markdown, browser)
+    if key.startswith("emulators/") and relative.stem != "index":
+        return _emulator_description(markdown, key, title, _first_paragraph(lead))
+    first = _first_paragraph(lead, minimum=35)
+    questions = [_plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading))
+                 for heading in re.findall(r"^## (.+\?)(?:[ \t]*\{[^}]*\})?[ \t]*$", markdown, re.M)]
+    if not first and len(questions) >= 3:
+        # A page of questions with no intro: its first paragraph is the
+        # answer to one of them ("Run the command with --dry-run").
+        return _shorten(f"{title}: {' '.join(questions)}", 155)
+    first = first or _first_paragraph(markdown, minimum=35)
+    if first:
+        return _shorten(first, 155)
+    return f"{title}: RetroBIOS source-traced retrogaming firmware reference."
 
-    paragraphs = re.split(r"\n\s*\n", markdown)
-    for paragraph in paragraphs:
+
+def _lead(markdown: str) -> str:
+    """The text a page opens with, before its first section, details
+    block or ignored section (the install box, the profile table)."""
+    return re.split(r"^(?:## |<details|<section)", markdown, maxsplit=1, flags=re.M)[0]
+
+
+def _emulator_description(markdown: str, key: str, title: str, intro: str) -> str:
+    """An emulator page by its intro sentence ("DuckStation emulates Sony
+    PlayStation. It picks the BIOS for your game's region: ..."), or by
+    its profile table when the profile names no system."""
+    notes = re.search(r"<summary>Technical notes</summary>\s*\n\n(.+?)(?:\n\n|$)", markdown, re.S)
+    if not intro and notes:
+        sentence = re.split(r"(?<=\.)\s", _plain_markdown(notes[1]), maxsplit=1)[0]
+        intro = f"{title}: {sentence[:1].lower()}{sentence[1:]}"
+    if not intro:
+        row = {name: _plain_markdown(value) for name, value in re.findall(
+            r"^\| (Type|Classification|Version) \| (.+?) \|$", markdown, re.M)}
+        kind = row.get("Classification") or row.get("Type") or "emulator"
+        version = row.get("Version", "")
+        known = f", version {version}" if version and version[0].isdigit() else ""
+        intro = f"{title}: {kind.lower()}{known}."
+    elif title not in intro:
+        # "It reads 1 file": the profile names no system to open on.
+        intro = f"{title}: {intro[:1].lower()}{intro[1:]}"
+    # An alias page sends the reader to the core whose files it shares.
+    if PAGE_DATA.get(key, {}).get("hasPart") or "uses the same firmware as" in intro:
+        return _shorten(intro, 155)
+    needs = "It needs no BIOS or firmware file."
+    if "\n## Data directories" in markdown:
+        needs = "It needs no BIOS or firmware file, only the data folders listed."
+    return _shorten(f"{intro} {needs}", 155)
+
+
+# Lines a page opens with that say where it sits, not what it holds.
+_PLACE_LINES = ("Part of ", "This part holds ", "On a card, ")
+
+
+def _first_paragraph(markdown: str, minimum: int = 1) -> str:
+    """The first paragraph of prose of a page body, as plain text: no
+    heading, table, list, fence, admonition or HTML block, and none that
+    only says where the page sits."""
+    for paragraph in re.split(r"\n\s*\n", markdown):
         stripped = paragraph.strip()
         if (
             not stripped
-            or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* "))
+            or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* ", "import "))
         ):
             continue
         plain = _plain_markdown(stripped)
+        if plain.startswith(_PLACE_LINES):
+            continue
         # A paragraph that ends on a colon introduces what follows: alone,
         # it describes nothing ("One file needs no clone and no local setup:").
-        if len(plain) >= 35 and not plain.endswith(":"):
-            return _shorten(plain)
-    return f"{title}: RetroBIOS source-traced retrogaming firmware reference."
+        if len(plain) >= minimum and not plain.endswith(":"):
+            return plain
+    return ""
+
+
+# Section headings of a system page that are not a console.
+_NOT_A_CONSOLE = {"Next steps", "Files", "Other files"}
+_CARD_TITLE = re.compile(r'class="rb-file-title"[^>]*><code>([^<]+)</code>')
+
+
+def _system_description(markdown: str, browser: str) -> str:
+    """A system page by what it holds: a manufacturer page by its consoles,
+    a console page by its "Start with" line, else by its first files."""
+    consoles = [
+        _plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading))
+        for heading in re.findall(r"^## (.+?)[ \t]*$", markdown, re.M)
+    ]
+    consoles = [name for name in consoles if name and name not in _NOT_A_CONSOLE]
+    if len(consoles) > 1:
+        shown = consoles if len(consoles) <= 4 else consoles[:3] + [f"{len(consoles) - 3} more"]
+        return _shorten(f"{browser} for {_said_as_list(shown)}, one card per file with its "
+                        "hashes and the emulators that read it.", 155)
+    start = re.search(r"^\*\*Start with\*\*(?P<line>.*)$(?P<list>(?:\n+- .*)*)", markdown, re.M)
+    if start:
+        # "Start with the files of your machine:" introduces one line per machine.
+        machines = [_plain_markdown(item) for item in re.findall(r"^- (.+)$", start["list"], re.M)]
+        text = _plain_markdown(start["line"]).lstrip(": ")
+        said = "; ".join(machines) if machines and text.endswith(":") else text
+        return _shorten(f"{browser}. Start with {said}", 155)
+    names = list(dict.fromkeys(html.unescape(name) for name in _CARD_TITLE.findall(markdown)))
+    if names:
+        shown = names if len(names) <= 3 else names[:3] + [f"{len(names) - 3} more"]
+        return _shorten(f"{browser}: {_said_as_list(shown)}, with their hashes and the "
+                        "emulators that read them.", 155)
+    return _shorten(f"{browser}, one card per file with its hashes and the emulators "
+                    "that read it.", 155)
 
 
 # Structured data a page generator knows beyond its text, by page path
@@ -2555,15 +2650,25 @@ LEVELS = {
 # - A guide behind the first two levels (reference, options, contributing)
 #   names everything a reader types, and ranks after the page that answers.
 # - A launcher, an alias or a frozen copy answers a name after the emulator.
-SEARCH_WEIGHT = {"start": 10, "setup": 1.5, "reference": 0.5, "contribute": 0.5}
+# - A part of a split page (a console's files, 5 of 9) holds cards, each found
+#   by its own heading, and ranks after the page that answers the question.
+SEARCH_WEIGHT = {"start": 10, "setup": 1.5, "reference": 0.5, "contribute": 0.5, "part": 0.5}
 SYSTEM_PAGE_WEIGHT = 5
 DEEP_GUIDE_WEIGHT = 0.5
 SEARCH_DEMOTED: set[str] = set()
+# A tool page a reader reaches by a task, not its name: "check my dump",
+# "md5 check" found emulator pages and never the page that checks a file.
+SEARCH_WEIGHT_OF_PAGE = {"files/index.md": 2}
 DEMOTED_WEIGHT = 0.5
 
 
 def search_weight(relative: Path, level: str) -> float | None:
-    """The search weight of a page, None for the default."""
+    """The search weight of a page, None for the default. A part page
+    states its own in its front matter: search_weight(path, "part")."""
+    if level == "part":
+        return SEARCH_WEIGHT["part"]
+    if relative.as_posix() in SEARCH_WEIGHT_OF_PAGE:
+        return SEARCH_WEIGHT_OF_PAGE[relative.as_posix()]
     if relative.as_posix() in SEARCH_DEMOTED:
         return DEMOTED_WEIGHT
     if relative.parts[0] == "systems":
@@ -2643,7 +2748,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]:
 
         title = _page_title(markdown, relative)
         browser_title = _browser_title(relative, title)
-        description = _page_description(markdown, relative, title)
+        description = str(authored.get("description") or _page_description(markdown, relative, title))
         if relative.name == "index.md":
             page_path = "" if len(relative.parts) == 1 else "/".join(relative.parts[:-1]) + "/"
         else:
@@ -2675,6 +2780,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]:
             search=authored.get("search") != "exclude",
             logo=str(authored.get("logo") or ""),
             parent=authored.get("parent"),
+            authored=authored,
         )
         write_decorated(docs / target, text)
         # One file per page: a page that gained or lost its components
@@ -8855,7 +8961,7 @@ def generate_nav(
         ],
         "reference": [
             # Find a file first: the one tool a reader comes to with a file in hand.
-            (5, {"Find a file": "files/index.md"}, "Checking files"),
+            (5, {"Find or check a file": "files/index.md"}, "Checking files"),
             (20, {"What the packs do not hold": "gaps.md"}, "Checking files"),
             (30, {"Cores by platform": "cross-reference.md"}, "Checking files"),
             (40, {"Dump provenance": "provenance.md"}, "Checking files"),

scripts/sitestarlight.py

@@ -417,6 +417,32 @@ def page_aliases(rel: str) -> list[str]:
     return list(_page_aliases.get(rel, []))
 
 
+# Search fields a page's own front matter fills, never typed here.
+SEARCH_FIELDS = ("consoles", "terms")
+
+
+def field_words(value: object) -> list[str]:
+    """The words of a front matter search field: a list, or one string
+    with its words or phrases split at commas."""
+    if isinstance(value, str):
+        return [part.strip() for part in value.split(",") if part.strip()]
+    if isinstance(value, (list, tuple)):
+        return [str(part).strip() for part in value if str(part).strip()]
+    return []
+
+
+def _neighbour(rel: str, value: object) -> object:
+    """A `prev:` or `next:` the page states, its link resolved like any
+    other link of the page."""
+    if isinstance(value, dict) and value.get("link"):
+        # Starlight writes the link as given: a site path gets the base.
+        link = resolve(rel, str(value["link"]))
+        if link.startswith("/") and not link.startswith(BASE):
+            link = BASE + link.lstrip("/")
+        return {**value, "link": link}
+    return value
+
+
 _NEXT_STEPS = re.compile(r"^## Next steps\b[^\n]*\n+(?P<list>(?:[ \t]*[-*] .*\n?)+)", re.M)
 
 
@@ -445,11 +471,20 @@ def finalize_page(
     search: bool = True,
     logo: str = "",
     parent: dict | None = None,
+    authored: dict | None = None,
 ) -> tuple[str, str]:
     """The Starlight file for a page body: its path under the content
     collection and its text. *parent* ({label, link}, the link written
     relative to the page) is the page a part page belongs to: the route
-    middleware makes it the last crumb and marks it in the sidebar."""
+    middleware makes it the last crumb and marks it in the sidebar.
+
+    *authored* is the front matter the page was written with. Its
+    `searchWeight`, `prev` and `next` win over the computed ones (a part
+    page states its neighbours and its weight), and its `consoles` and
+    `terms` become search fields (astro.config.mjs, metaWeights): the
+    consoles a manufacturer page holds sections for, and the words a page
+    answers that its text does not say, as an error message words them."""
+    authored = authored or {}
     title, title_id, body = split_title(markdown)
     body = body.lstrip("\n")
     body = rewrite_links(body, rel)
@@ -468,6 +503,8 @@ def finalize_page(
                   r'<span data-pagefind-ignore class="\1"', body)
     if rel.startswith("systems/"):
         body += file_names_meta(body) + search_field("aliases", page_aliases(rel))
+    for key in SEARCH_FIELDS:
+        body += search_field(key, field_words(authored.get(key)))
     components = used_components(body)
     stem = rel.rsplit(".", 1)[0]
     if components:
@@ -492,6 +529,7 @@ def finalize_page(
     if parent:
         front["parent"] = {"label": str(parent["label"]),
                            "link": resolve(rel, str(parent["link"]))}
+    weight = authored.get("searchWeight", weight)
     if weight:
         front["searchWeight"] = weight
     if rel == NOT_FOUND:
@@ -512,6 +550,9 @@ def finalize_page(
     elif step:
         # The page's own "Next steps" list and the next link agree.
         front["next"] = step
+    for side in ("prev", "next"):
+        if rel != NOT_FOUND and side in authored:
+            front[side] = _neighbour(rel, authored[side])
     front["head"] = page_head(rel, level, front["title"], browser_title, structured)
     if structured.get("dateModified"):
         front["lastmod"] = str(structured["dateModified"])

scripts/validate_site.py

@@ -25,6 +25,7 @@ from urllib.parse import unquote, urlsplit
 class Page:
     path: Path
     title: str = ""
+    h1: str = ""
     lang: str = ""
     descriptions: list[str] = field(default_factory=list)
     canonical: list[str] = field(default_factory=list)
@@ -66,6 +67,11 @@ OVER_ELEMENT_BUDGET: dict[str, str] = {}
 MAX_PAGEFIND_CHUNK_BYTES = 150_000
 MAX_PAGEFIND_META_BYTES = 50_000
 MAX_PAGEFIND_TOTAL_BYTES = 12_000_000
+# Pages whose description is one text once the page's own name is taken
+# out. 707 pages shared five templates ("X: source-pinned emulator
+# firmware profile with paths, hashes..."), and a search engine showed the
+# same line under every result.
+MAX_SHARED_DESCRIPTION = 10
 
 
 # Text inside these is shown as written, so markup there is not a leftover.
@@ -78,6 +84,8 @@ class _PageParser(HTMLParser):
         self.page = Page(path)
         self._title_depth = 0
         self._title_parts: list[str] = []
+        self._h1_depth = 0
+        self._h1_parts: list[str] = []
         self._jsonld_depth = 0
         self._jsonld_parts: list[str] = []
         self._literal_depth = 0
@@ -108,6 +116,7 @@ class _PageParser(HTMLParser):
             self.page.descriptions.append(data.get("content", "").strip())
         elif tag == "h1":
             self.page.h1_count += 1
+            self._h1_depth += 1
         elif tag == "main":
             self.page.main_count += 1
         elif tag == "img" and "alt" not in data:
@@ -148,6 +157,8 @@ class _PageParser(HTMLParser):
             self._header_parts = None
         if tag == "title":
             self._title_depth -= 1
+        elif tag == "h1" and self._h1_depth:
+            self._h1_depth -= 1
         elif tag == "script" and self._jsonld_depth:
             self.page.jsonld.append("".join(self._jsonld_parts).strip())
             self._jsonld_depth -= 1
@@ -158,6 +169,8 @@ class _PageParser(HTMLParser):
             self._header_parts.append(data)
         if self._title_depth:
             self._title_parts.append(data)
+        if self._h1_depth:
+            self._h1_parts.append(data)
         if self._jsonld_depth:
             self._jsonld_parts.append(data)
         if not self._literal_depth and "**" in data:
@@ -166,6 +179,7 @@ class _PageParser(HTMLParser):
     def close(self) -> None:
         super().close()
         self.page.title = "".join(self._title_parts).strip()
+        self.page.h1 = re.sub(r"\s+", " ", "".join(self._h1_parts)).strip()
 
 
 def _parse_page(path: Path) -> Page:
@@ -322,6 +336,31 @@ def _check_pages(
     return issues
 
 
+def description_template(page: Page, site_name: str = "RetroBIOS") -> str:
+    """A page's description with its own names taken out: its h1 and its
+    window title. What is left is the text it shares with its template."""
+    text = page.descriptions[0] if page.descriptions else ""
+    names = {page.h1, page.title.removesuffix(f" - {site_name}").strip()}
+    for name in sorted(filter(None, names), key=len, reverse=True):
+        text = text.replace(name, "")
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def _check_description_templates(pages: dict, site: Path) -> list[str]:
+    """No more than MAX_SHARED_DESCRIPTION pages share one description
+    once their names are taken out."""
+    shared: dict[str, list[Path]] = defaultdict(list)
+    for path, page in pages.items():
+        if path.relative_to(site) != Path("404.html") and page.descriptions:
+            shared[description_template(page)].append(path)
+    return [
+        f"{len(paths)} pages share the description {text!r} after their name: "
+        + ", ".join(str(path.relative_to(site)) for path in paths[:3])
+        for text, paths in shared.items()
+        if len(paths) > MAX_SHARED_DESCRIPTION
+    ]
+
+
 def _check_links(pages: dict, site: Path, base_path: str) -> list[str]:
     """Every local link and fragment resolves to something on disk."""
     issues: list[str] = []
@@ -422,6 +461,7 @@ def validate_site(
                 rendered = ", ".join(str(path.relative_to(site)) for path in paths[:5])
                 issues.append(f"duplicate {label} {value!r}: {rendered}")
 
+    issues.extend(_check_description_templates(content, site))
     issues.extend(_check_links(pages, site, base_path))
     issues.extend(_check_pagefind(site))
     if platforms_dir is not None: