diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 446e91a3..e9c10f66 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2109,12 +2109,23 @@ def _browser_title(relative: Path, title: str) -> str: } if key in index_titles: return index_titles[key] - if key in PART_PAGES: + # A console page is filed with the parts of its manufacturer's page, and + # titled "Sega: Dreamcast". A part names its range of files. + part = key in PART_PAGES and ( + not key.startswith("systems/") or ": " not in title + or re.search(r", part \d+ of \d+$", title)) + if part: return title + # The words a web search for a console or an emulator types: "dreamcast + # bios", "duckstation bios". The h1 and the sidebar keep the name. if key.startswith("emulators/"): - return f"{title} emulator firmware" + has_files = PAGE_DATA.get(key, {}).get("hasPart") + # Lexaloffle, QEMU and ScummVM name a system page too. + own = " emulator" if f"systems/{relative.name}" in PAGE_DATA else "" + return f"{title}{own} BIOS files" if has_files else f"{title} emulator" if key.startswith("systems/"): - return f"{title} systems" + maker, _, console = title.partition(": ") + return f"{console} BIOS files ({maker})" if console else f"{title} BIOS files" return title @@ -2157,45 +2168,129 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: } if key in specific: return specific[key] - if key in PART_PAGES: - return (f"{title}: every file with its hashes, size, source references " - "and who reads it.") + # What a search engine shows is the page's own first sentence: 707 pages + # shared five templates of level 3 words ("source-pinned", "provenance"). + browser = _browser_title(relative, title) + held = re.search(r"\bThis part holds (.+?)\.(?: |$)", markdown, re.M) + if key in PART_PAGES and held: + return _shorten(f"{browser}. This part holds {_plain_markdown(held[1])}.", 155) + lead = _lead(markdown) + if key in PART_PAGES and re.search(r"\bpart \d+ of \d+", title): + # The parts of one list open on the same sentence. + first = _first_paragraph(lead) or _first_paragraph(markdown) + return _shorten(f"{title}. {first}", 155) if key.startswith(f"{release_record.GET_DIR}/"): - return (f"{title}: the BIOS files of this setup, as the ZIP of the latest " - "release or one command, at an address that stays the same.") - if key.startswith("platforms/") and relative.stem != "index": - return (f"{title}: where the BIOS files go, the command that installs them, the pack, " - "and every file by system with its hashes.") - if key.startswith("emulators/") and relative.stem != "index": - # Pinned only when the profile names the revision it was read at: - # 48 pages without one called themselves source-pinned. - about = PAGE_DATA.get(key, {}).get("about", {}) - pinned = any(a.get("version") for a in (about if isinstance(about, list) else [about])) - kind = "source-pinned emulator firmware profile" if pinned else "emulator firmware profile" - if not PAGE_DATA.get(key, {}).get("hasPart"): - # 152 cores that load no file announced paths and hashes. - kind = kind.replace(" firmware", "") - trees = (" The data directories it reads are listed." - if "\n## Data directories\n" in markdown else "") - return f"{title}: {kind}. No BIOS or firmware file is required.{trees}" - return f"{title}: {kind} with paths, hashes, requirements and validation behavior." + size = re.search(r"^- (.+ to download\b.*?)\.?$", markdown, re.M) + if size: + return _shorten(f"{title}: the BIOS pack of this setup, {_plain_markdown(size[1])}. " + "The address stays the same and always gives the latest ZIP.", 155) + return _shorten(_first_paragraph(lead) or title, 155) if key.startswith("systems/") and relative.stem != "index": - return f"{title}: indexed firmware files, hashes, variants, provenance and platform or emulator usage." + return _system_description(markdown, browser) + if key.startswith("emulators/") and relative.stem != "index": + return _emulator_description(markdown, key, title, _first_paragraph(lead)) + first = _first_paragraph(lead, minimum=35) + questions = [_plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading)) + for heading in re.findall(r"^## (.+\?)(?:[ \t]*\{[^}]*\})?[ \t]*$", markdown, re.M)] + if not first and len(questions) >= 3: + # A page of questions with no intro: its first paragraph is the + # answer to one of them ("Run the command with --dry-run"). + return _shorten(f"{title}: {' '.join(questions)}", 155) + first = first or _first_paragraph(markdown, minimum=35) + if first: + return _shorten(first, 155) + return f"{title}: RetroBIOS source-traced retrogaming firmware reference." - paragraphs = re.split(r"\n\s*\n", markdown) - for paragraph in paragraphs: + +def _lead(markdown: str) -> str: + """The text a page opens with, before its first section, details + block or ignored section (the install box, the profile table).""" + return re.split(r"^(?:## | str: + """An emulator page by its intro sentence ("DuckStation emulates Sony + PlayStation. It picks the BIOS for your game's region: ..."), or by + its profile table when the profile names no system.""" + notes = re.search(r"Technical notes\s*\n\n(.+?)(?:\n\n|$)", markdown, re.S) + if not intro and notes: + sentence = re.split(r"(?<=\.)\s", _plain_markdown(notes[1]), maxsplit=1)[0] + intro = f"{title}: {sentence[:1].lower()}{sentence[1:]}" + if not intro: + row = {name: _plain_markdown(value) for name, value in re.findall( + r"^\| (Type|Classification|Version) \| (.+?) \|$", markdown, re.M)} + kind = row.get("Classification") or row.get("Type") or "emulator" + version = row.get("Version", "") + known = f", version {version}" if version and version[0].isdigit() else "" + intro = f"{title}: {kind.lower()}{known}." + elif title not in intro: + # "It reads 1 file": the profile names no system to open on. + intro = f"{title}: {intro[:1].lower()}{intro[1:]}" + # An alias page sends the reader to the core whose files it shares. + if PAGE_DATA.get(key, {}).get("hasPart") or "uses the same firmware as" in intro: + return _shorten(intro, 155) + needs = "It needs no BIOS or firmware file." + if "\n## Data directories" in markdown: + needs = "It needs no BIOS or firmware file, only the data folders listed." + return _shorten(f"{intro} {needs}", 155) + + +# Lines a page opens with that say where it sits, not what it holds. +_PLACE_LINES = ("Part of ", "This part holds ", "On a card, ") + + +def _first_paragraph(markdown: str, minimum: int = 1) -> str: + """The first paragraph of prose of a page body, as plain text: no + heading, table, list, fence, admonition or HTML block, and none that + only says where the page sits.""" + for paragraph in re.split(r"\n\s*\n", markdown): stripped = paragraph.strip() if ( not stripped - or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* ")) + or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* ", "import ")) ): continue plain = _plain_markdown(stripped) + if plain.startswith(_PLACE_LINES): + continue # A paragraph that ends on a colon introduces what follows: alone, # it describes nothing ("One file needs no clone and no local setup:"). - if len(plain) >= 35 and not plain.endswith(":"): - return _shorten(plain) - return f"{title}: RetroBIOS source-traced retrogaming firmware reference." + if len(plain) >= minimum and not plain.endswith(":"): + return plain + return "" + + +# Section headings of a system page that are not a console. +_NOT_A_CONSOLE = {"Next steps", "Files", "Other files"} +_CARD_TITLE = re.compile(r'class="rb-file-title"[^>]*>([^<]+)') + + +def _system_description(markdown: str, browser: str) -> str: + """A system page by what it holds: a manufacturer page by its consoles, + a console page by its "Start with" line, else by its first files.""" + consoles = [ + _plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading)) + for heading in re.findall(r"^## (.+?)[ \t]*$", markdown, re.M) + ] + consoles = [name for name in consoles if name and name not in _NOT_A_CONSOLE] + if len(consoles) > 1: + shown = consoles if len(consoles) <= 4 else consoles[:3] + [f"{len(consoles) - 3} more"] + return _shorten(f"{browser} for {_said_as_list(shown)}, one card per file with its " + "hashes and the emulators that read it.", 155) + start = re.search(r"^\*\*Start with\*\*(?P.*)$(?P(?:\n+- .*)*)", markdown, re.M) + if start: + # "Start with the files of your machine:" introduces one line per machine. + machines = [_plain_markdown(item) for item in re.findall(r"^- (.+)$", start["list"], re.M)] + text = _plain_markdown(start["line"]).lstrip(": ") + said = "; ".join(machines) if machines and text.endswith(":") else text + return _shorten(f"{browser}. Start with {said}", 155) + names = list(dict.fromkeys(html.unescape(name) for name in _CARD_TITLE.findall(markdown))) + if names: + shown = names if len(names) <= 3 else names[:3] + [f"{len(names) - 3} more"] + return _shorten(f"{browser}: {_said_as_list(shown)}, with their hashes and the " + "emulators that read them.", 155) + return _shorten(f"{browser}, one card per file with its hashes and the emulators " + "that read it.", 155) # Structured data a page generator knows beyond its text, by page path @@ -2555,15 +2650,25 @@ LEVELS = { # - A guide behind the first two levels (reference, options, contributing) # names everything a reader types, and ranks after the page that answers. # - A launcher, an alias or a frozen copy answers a name after the emulator. -SEARCH_WEIGHT = {"start": 10, "setup": 1.5, "reference": 0.5, "contribute": 0.5} +# - A part of a split page (a console's files, 5 of 9) holds cards, each found +# by its own heading, and ranks after the page that answers the question. +SEARCH_WEIGHT = {"start": 10, "setup": 1.5, "reference": 0.5, "contribute": 0.5, "part": 0.5} SYSTEM_PAGE_WEIGHT = 5 DEEP_GUIDE_WEIGHT = 0.5 SEARCH_DEMOTED: set[str] = set() +# A tool page a reader reaches by a task, not its name: "check my dump", +# "md5 check" found emulator pages and never the page that checks a file. +SEARCH_WEIGHT_OF_PAGE = {"files/index.md": 2} DEMOTED_WEIGHT = 0.5 def search_weight(relative: Path, level: str) -> float | None: - """The search weight of a page, None for the default.""" + """The search weight of a page, None for the default. A part page + states its own in its front matter: search_weight(path, "part").""" + if level == "part": + return SEARCH_WEIGHT["part"] + if relative.as_posix() in SEARCH_WEIGHT_OF_PAGE: + return SEARCH_WEIGHT_OF_PAGE[relative.as_posix()] if relative.as_posix() in SEARCH_DEMOTED: return DEMOTED_WEIGHT if relative.parts[0] == "systems": @@ -2643,7 +2748,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]: title = _page_title(markdown, relative) browser_title = _browser_title(relative, title) - description = _page_description(markdown, relative, title) + description = str(authored.get("description") or _page_description(markdown, relative, title)) if relative.name == "index.md": page_path = "" if len(relative.parts) == 1 else "/".join(relative.parts[:-1]) + "/" else: @@ -2675,6 +2780,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]: search=authored.get("search") != "exclude", logo=str(authored.get("logo") or ""), parent=authored.get("parent"), + authored=authored, ) write_decorated(docs / target, text) # One file per page: a page that gained or lost its components @@ -8855,7 +8961,7 @@ def generate_nav( ], "reference": [ # Find a file first: the one tool a reader comes to with a file in hand. - (5, {"Find a file": "files/index.md"}, "Checking files"), + (5, {"Find or check a file": "files/index.md"}, "Checking files"), (20, {"What the packs do not hold": "gaps.md"}, "Checking files"), (30, {"Cores by platform": "cross-reference.md"}, "Checking files"), (40, {"Dump provenance": "provenance.md"}, "Checking files"), diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py index c6c3b6f3..5763f19b 100644 --- a/scripts/sitestarlight.py +++ b/scripts/sitestarlight.py @@ -417,6 +417,32 @@ def page_aliases(rel: str) -> list[str]: return list(_page_aliases.get(rel, [])) +# Search fields a page's own front matter fills, never typed here. +SEARCH_FIELDS = ("consoles", "terms") + + +def field_words(value: object) -> list[str]: + """The words of a front matter search field: a list, or one string + with its words or phrases split at commas.""" + if isinstance(value, str): + return [part.strip() for part in value.split(",") if part.strip()] + if isinstance(value, (list, tuple)): + return [str(part).strip() for part in value if str(part).strip()] + return [] + + +def _neighbour(rel: str, value: object) -> object: + """A `prev:` or `next:` the page states, its link resolved like any + other link of the page.""" + if isinstance(value, dict) and value.get("link"): + # Starlight writes the link as given: a site path gets the base. + link = resolve(rel, str(value["link"])) + if link.startswith("/") and not link.startswith(BASE): + link = BASE + link.lstrip("/") + return {**value, "link": link} + return value + + _NEXT_STEPS = re.compile(r"^## Next steps\b[^\n]*\n+(?P(?:[ \t]*[-*] .*\n?)+)", re.M) @@ -445,11 +471,20 @@ def finalize_page( search: bool = True, logo: str = "", parent: dict | None = None, + authored: dict | None = None, ) -> tuple[str, str]: """The Starlight file for a page body: its path under the content collection and its text. *parent* ({label, link}, the link written relative to the page) is the page a part page belongs to: the route - middleware makes it the last crumb and marks it in the sidebar.""" + middleware makes it the last crumb and marks it in the sidebar. + + *authored* is the front matter the page was written with. Its + `searchWeight`, `prev` and `next` win over the computed ones (a part + page states its neighbours and its weight), and its `consoles` and + `terms` become search fields (astro.config.mjs, metaWeights): the + consoles a manufacturer page holds sections for, and the words a page + answers that its text does not say, as an error message words them.""" + authored = authored or {} title, title_id, body = split_title(markdown) body = body.lstrip("\n") body = rewrite_links(body, rel) @@ -468,6 +503,8 @@ def finalize_page( r' None: super().close() self.page.title = "".join(self._title_parts).strip() + self.page.h1 = re.sub(r"\s+", " ", "".join(self._h1_parts)).strip() def _parse_page(path: Path) -> Page: @@ -322,6 +336,31 @@ def _check_pages( return issues +def description_template(page: Page, site_name: str = "RetroBIOS") -> str: + """A page's description with its own names taken out: its h1 and its + window title. What is left is the text it shares with its template.""" + text = page.descriptions[0] if page.descriptions else "" + names = {page.h1, page.title.removesuffix(f" - {site_name}").strip()} + for name in sorted(filter(None, names), key=len, reverse=True): + text = text.replace(name, "") + return re.sub(r"\s+", " ", text).strip() + + +def _check_description_templates(pages: dict, site: Path) -> list[str]: + """No more than MAX_SHARED_DESCRIPTION pages share one description + once their names are taken out.""" + shared: dict[str, list[Path]] = defaultdict(list) + for path, page in pages.items(): + if path.relative_to(site) != Path("404.html") and page.descriptions: + shared[description_template(page)].append(path) + return [ + f"{len(paths)} pages share the description {text!r} after their name: " + + ", ".join(str(path.relative_to(site)) for path in paths[:3]) + for text, paths in shared.items() + if len(paths) > MAX_SHARED_DESCRIPTION + ] + + def _check_links(pages: dict, site: Path, base_path: str) -> list[str]: """Every local link and fragment resolves to something on disk.""" issues: list[str] = [] @@ -422,6 +461,7 @@ def validate_site( rendered = ", ".join(str(path.relative_to(site)) for path in paths[:5]) issues.append(f"duplicate {label} {value!r}: {rendered}") + issues.extend(_check_description_templates(content, site)) issues.extend(_check_links(pages, site, base_path)) issues.extend(_check_pagefind(site)) if platforms_dir is not None: diff --git a/tests/test_site_emulator_page.py b/tests/test_site_emulator_page.py index 0342897b..ba592d3e 100644 --- a/tests/test_site_emulator_page.py +++ b/tests/test_site_emulator_page.py @@ -103,9 +103,9 @@ class WhatASearchSnippetSays(unittest.TestCase): gs.PAGE_DATA[rel.as_posix()] = {} text = gs._page_description("# Lutro\n", rel, "Lutro") self.assertNotIn("hashes", text) - self.assertIn("No BIOS or firmware file is required.", text) + self.assertIn("It needs no BIOS or firmware file.", text) gs.PAGE_DATA[rel.as_posix()] = {"hasPart": [{"name": "a.bin"}]} - self.assertIn("paths, hashes", gs._page_description("# Lutro\n", rel, "Lutro")) + self.assertNotIn("needs no BIOS", gs._page_description("# Lutro\n", rel, "Lutro")) def test_a_long_paragraph_is_cut_at_a_word(self): plain = ("files/?sha1= is the address that does not move: it opens the card " diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index a4a873c1..cedc664b 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -179,14 +179,15 @@ class SiteReferenceContracts(unittest.TestCase): self.assertNotIn("\n #if FEATURE", rendered) def test_browser_titles_distinguish_emulators_from_system_pages(self): - self.assertEqual( - _browser_title(Path("emulators/qemu.md"), "QEMU"), - "QEMU emulator firmware", - ) - self.assertEqual( - _browser_title(Path("systems/qemu.md"), "QEMU"), - "QEMU systems", - ) + from unittest import mock # noqa: PLC0415 + + import generate_site # noqa: PLC0415 + + pages = {"systems/qemu.md": {}, "emulators/qemu.md": {"hasPart": [{"name": "a"}]}} + with mock.patch.dict(generate_site.PAGE_DATA, pages, clear=True): + self.assertEqual(_browser_title(Path("emulators/qemu.md"), "QEMU"), + "QEMU emulator BIOS files") + self.assertEqual(_browser_title(Path("systems/qemu.md"), "QEMU"), "QEMU BIOS files") ACCEPTED = ( ("md5", "c87a52c277f7952b41c639fc7bf0a43b"), @@ -581,7 +582,8 @@ class OnlyAPinnedProfileClaimsAPin(unittest.TestCase): with mock.patch.dict(generate_site.PAGE_DATA, pages, clear=True): pinned = generate_site._page_description("", Path("emulators/pinned.md"), "P") loose = generate_site._page_description("", Path("emulators/redream.md"), "R") - self.assertIn("source-pinned", pinned) + # A search snippet is level 0: the pin is on the page, not in it. + self.assertNotIn("source-pinned", pinned) self.assertNotIn("source-pinned", loose) diff --git a/tests/test_site_r2_nav.py b/tests/test_site_r2_nav.py index 0510ced7..c3f549ac 100644 --- a/tests/test_site_r2_nav.py +++ b/tests/test_site_r2_nav.py @@ -233,7 +233,7 @@ class ReferenceGrouped(unittest.TestCase): reference = self._reference() groups = {next(iter(item)): next(iter(item.values())) for item in reference if isinstance(item, dict) and isinstance(next(iter(item.values())), list)} - self.assertEqual(groups["Checking files"][0], {"Find a file": "files/index.md"}) + self.assertEqual(groups["Checking files"][0], {"Find or check a file": "files/index.md"}) self.assertIn({"What the packs do not hold": "gaps.md"}, groups["Checking files"]) self.assertEqual(groups["Data"][:3], [{"Data and API": "data.md"}, {"Data model": "wiki/data-model.md"}, @@ -250,7 +250,7 @@ class ReferenceGrouped(unittest.TestCase): reference = self._reference(wiki) group = next(item["Checking files"] for item in reference if isinstance(item, dict) and "Checking files" in item) - self.assertEqual(group[:2], [{"Find a file": "files/index.md"}, + self.assertEqual(group[:2], [{"Find or check a file": "files/index.md"}, {"How each platform checks files": "wiki/modes.md"}])