diff --git a/scripts/generate_site.py b/scripts/generate_site.py index c1eff29ed..d8e7ff3d6 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3065,6 +3065,14 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]: """ lastmod: dict[str, str] = {} terms = sitestarlight.glossary_terms(_REPO_ROOT / WIKI_SRC_DIR / "glossary.md") + # The console words of every system page, decorated or not: a console + # another page's words cover gets a search field of its own. + console_words: dict[str, set[str]] = {} + for path in sorted((docs / "systems").glob("*.md*")): + text = path.read_text(encoding="utf-8") + words = sitestarlight.page_console_words(_front_matter(text), text) + if words: + console_words[path.relative_to(docs).with_suffix("").as_posix()] = words for path in sorted(docs.rglob("*.md")): relative = path.relative_to(docs) if any(part in LOCAL_NOTE_DIRS for part in relative.parts[:-1]): @@ -3125,6 +3133,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]: logo=str(authored.get("logo") or ""), parent=authored.get("parent"), authored=authored, + words_by_page=console_words, ) write_decorated(docs / target, text) # One file per page: a page that gained or lost its components diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py index 616c0da95..949f17744 100644 --- a/scripts/sitestarlight.py +++ b/scripts/sitestarlight.py @@ -16,7 +16,7 @@ import json import re import unicodedata import urllib.parse -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Mapping from html import unescape from pathlib import Path, PurePosixPath @@ -366,6 +366,8 @@ def split_title(markdown: str) -> tuple[str, str, str]: _CARD_NAME = re.compile(r']*class="rb-file-title"(?![^>]*data-pagefind-ignore)[^>]*>([^<]+)') +# What separates the parts of a file name: anything but a letter or a digit. +_NAME_PART = re.compile(r"[\W_]+") def search_field(key: str, words: list[str]) -> str: @@ -384,9 +386,20 @@ def file_names_meta(body: str) -> str: field of the page. A page of 250 cards is long, and the search weighs a long page down: the Sony page, where scph5501.bin is downloaded, came after every short page citing the name. The field is weighted in the - site's ranking (astro.config.mjs, metaWeights.files).""" + site's ranking (astro.config.mjs, metaWeights.files). + + A name is written as its parts (scph5501 bin), each once. The search + matches a word as the start of a longer one and scores every distinct + word it reaches: whole names gave "ds" twelve rare words on the DSi page + (dsi_bios7.bin, DSi_Nand_USA.bin...), which then ranked first for + "nintendo ds bios".""" names = sorted({unescape(name) for name in _CARD_NAME.findall(body)}) - return search_field("files", names) + parts: dict[str, str] = {} + for name in names: + for part in _NAME_PART.split(name): + if part: + parts.setdefault(part.casefold(), part) + return search_field("files", list(parts.values())) # systems/ page -> the platforms' own ids for the consoles it holds, built @@ -435,6 +448,33 @@ def console_words(names: list[str]) -> list[str]: return [f"{name} BIOS" for name in names] +_CONSOLES_META = re.compile(r'data-pagefind-meta="consoles:([^"]*)"') + + +def page_console_words(authored: dict, text: str) -> set[str]: + """The words of the consoles a page answers for: its `consoles` front + matter (*authored*) before decoration, its `consoles` field after.""" + names = field_words(authored.get("consoles")) + if names: + return {word.casefold() for name in names for word in name.split()} + match = _CONSOLES_META.search(text) + return set(unescape(match[1]).casefold().split()) if match else set() + + +def covered_names(names: list[str], page: str, words_by_page: Mapping[str, set[str]]) -> list[str]: + """The names of *names* whose every word another page's consoles also + hold, as a word or the start of one (PlayStation on the PlayStation 2 + page, Nintendo DS on the DSi page, Nintendo Super System on the + Nintendo page). The search matches a word as the start of a longer one + and ignores the words a page has that the query lacks: it cannot tell + these pages apart by words, only by a weight the other page lacks.""" + held = [words for other, words in words_by_page.items() if other != page] + return [name for name in names + if any(all(any(word.startswith(part) for word in words) + for part in name.casefold().split()) + for words in held)] + + def _neighbour(rel: str, value: object) -> object: """A `prev:` or `next:` the page states, its link resolved like any other link of the page.""" @@ -476,6 +516,7 @@ def finalize_page( logo: str = "", parent: dict | None = None, authored: dict | None = None, + words_by_page: Mapping[str, set[str]] | None = None, ) -> tuple[str, str]: """The Starlight file for a page body: its path under the content collection and its text. *parent* ({label, link}, the link written @@ -487,7 +528,10 @@ def finalize_page( page states its neighbours and its weight), and its `consoles` and `terms` become search fields (astro.config.mjs, metaWeights): the consoles a manufacturer page holds sections for, and the words a page - answers that its text does not say, as an error message words them.""" + answers that its text does not say, as an error message words them. + *words_by_page* holds the console words of every system page, by path + without extension: a console another page's words cover is written + once more, in `exact`.""" authored = authored or {} title, title_id, body = split_title(markdown) body = body.lstrip("\n") @@ -509,6 +553,7 @@ def finalize_page( body += file_names_meta(body) + search_field("aliases", page_aliases(rel)) consoles = field_words(authored.get("consoles")) body += search_field("consoles", consoles) + body += search_field("exact", covered_names(consoles, rel.rsplit(".", 1)[0], words_by_page or {})) body += search_field("terms", field_words(authored.get("terms")) + console_words(consoles)) components = used_components(body) stem = rel.rsplit(".", 1)[0] diff --git a/tests/test_site_r3_components.py b/tests/test_site_r3_components.py index 2572e0b93..d2fdc1bc6 100644 --- a/tests/test_site_r3_components.py +++ b/tests/test_site_r3_components.py @@ -291,8 +291,8 @@ class SearchFoldsPagesUnderAHash(unittest.TestCase): def test_search_fields_are_weighted_and_never_shown(self): config = _read(CONFIG) - self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 5, devices: 5, consoles: 10, terms: 30 },", config) - self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'terms']);", _read(SITE_TS)) + self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 10, devices: 5, consoles: 100, exact: 20, terms: 30 },", config) + self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'exact', 'terms']);", _read(SITE_TS)) class OsTabsNeverScrollThePage(unittest.TestCase): diff --git a/tests/test_site_r3_nav.py b/tests/test_site_r3_nav.py index 04c75b1c4..2e68f69e5 100644 --- a/tests/test_site_r3_nav.py +++ b/tests/test_site_r3_nav.py @@ -65,7 +65,8 @@ class MetaFields(unittest.TestCase): body = ('

scph5501.bin

' '

a,b.bin

') meta = sitestarlight.file_names_meta(body) - self.assertIn('data-pagefind-meta="files:a b.bin scph5501.bin"', meta) + # Each name as its parts, each part once (round 5, 3.2). + self.assertIn('data-pagefind-meta="files:a b bin scph5501"', meta) self.assertRegex(meta, r"]*hidden>") def test_a_field_escapes_its_value(self) -> None: diff --git a/tests/test_site_r4_components.py b/tests/test_site_r4_components.py index 67bb9f689..d21f94fb5 100644 --- a/tests/test_site_r4_components.py +++ b/tests/test_site_r4_components.py @@ -214,8 +214,8 @@ class SearchFields(unittest.TestCase): def test_declared_and_never_shown(self): config = _read(CONFIG) - self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 5, devices: 5, consoles: 10, terms: 30 },", config) - self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'terms']);", _read(SITE_TS)) + self.assertIn("metaWeights: { title: 10, files: 0.5, aliases: 10, devices: 5, consoles: 100, exact: 20, terms: 30 },", config) + self.assertIn("const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'exact', 'terms']);", _read(SITE_TS)) class SearchLoadsOnFirstUse(unittest.TestCase): diff --git a/tests/test_site_r5_search.py b/tests/test_site_r5_search.py index 07f2b1b1c..ccf666095 100644 --- a/tests/test_site_r5_search.py +++ b/tests/test_site_r5_search.py @@ -100,6 +100,83 @@ class ConsoleWordsReachTheConsole(unittest.TestCase): self.assertEqual(typed, set()) +class ANameAnotherPageCovers(unittest.TestCase): + """3.2: the PlayStation 2 page holds every word of "playstation bios". + A console whose words another page's consoles also hold is written once + more, in `exact`, on its own page, and file names reach the search as + their parts: "ds" found twelve dsi_* names on the DSi page.""" + + PAGES = { + "systems/sony-playstation": ["PlayStation"], + "systems/sony-playstation-2": ["PlayStation 2"], + "systems/sony": ["PlayStation 3", "PlayStation Portable"], + "systems/nintendo-nintendo-ds": ["Nintendo DS"], + "systems/nintendo-nintendo-dsi": ["Nintendo DSi"], + "systems/nintendo": ["Famicom Disk System", "Nintendo 64", "Super Famicom Box"], + "systems/arcade-nintendo-super-system": ["Nintendo Super System"], + "systems/sega-saturn": ["Saturn"], + } + + def words(self) -> dict[str, set[str]]: + return {page: sitestarlight.page_console_words({"consoles": names}, "") + for page, names in self.PAGES.items()} + + def test_names_another_page_covers(self): + covered = {page: sitestarlight.covered_names(names, page, self.words()) + for page, names in self.PAGES.items()} + self.assertEqual(covered, { + "systems/sony-playstation": ["PlayStation"], + "systems/sony-playstation-2": [], + # Every word of PlayStation 3 is on the PlayStation 2 page, but + # not "3" itself: no page covers it. + "systems/sony": [], + "systems/nintendo-nintendo-ds": ["Nintendo DS"], + "systems/nintendo-nintendo-dsi": [], + "systems/nintendo": [], + # Nintendo, Super and System, each in another name. + "systems/arcade-nintendo-super-system": ["Nintendo Super System"], + "systems/sega-saturn": [], + }) + + def test_words_of_a_decorated_page(self): + text = '' + self.assertEqual(sitestarlight.page_console_words({}, text), {"nintendo", "ds"}) + self.assertEqual(sitestarlight.page_console_words({}, "No field."), set()) + + def test_the_exact_field_is_written_on_the_covered_page_only(self): + words = self.words() + + def text(page: str) -> str: + return sitestarlight.finalize_page( + f"{page}.md", "# T\n\nBody.\n", level="0", weight=None, + browser_title="T", description="d", structured={"@type": "TechArticle"}, + terms={}, authored={"consoles": self.PAGES[page]}, words_by_page=words)[1] + + self.assertIn('data-pagefind-meta="exact:PlayStation"', text("systems/sony-playstation")) + for page in ("systems/sony-playstation-2", "systems/sega-saturn"): + with self.subTest(page=page): + self.assertNotIn('data-pagefind-meta="exact:', text(page)) + + def test_the_field_is_weighted_and_never_shown(self): + config = (REPO_ROOT / "website" / "astro.config.mjs").read_text(encoding="utf-8") + weights = re.search(r"metaWeights: \{([^}]*)\}", config)[1] + fields = dict(re.findall(r"(\w+): ([\d.]+)", weights)) + self.assertIn("exact", fields) + # Below the console field, so the longer name keeps its own query. + self.assertLess(float(fields["exact"]), float(fields["consoles"])) + site = (REPO_ROOT / "website" / "src" / "scripts" / "site.ts").read_text(encoding="utf-8") + self.assertRegex(site, r"UNSHOWN_FIELDS = new Set\(\[[^\]]*'exact'") + + def test_file_names_as_parts(self): + body = "".join(f'

{name}

' + for name in ("dsi_bios7.bin", "DSi_Nand_USA.bin", "NDS_Bios7.bin")) + meta = sitestarlight.file_names_meta(body) + self.assertIn('data-pagefind-meta="files:DSi Nand USA bin NDS Bios7"', meta) + # A letter of any script is a letter. + self.assertIn("ファミコン", sitestarlight.file_names_meta( + '

ファミコン_bios.rom

')) + + class GateQueries(unittest.TestCase): """3.2, 3.3 and 3.6: the queries the gate runs, the pages that answer them and the rank they must reach.""" @@ -110,6 +187,16 @@ class GateQueries(unittest.TestCase): "gamecube bios": (("systems/nintendo-gamecube/",), 1), "game boy bios": (("systems/nintendo-game-boy/",), 1), "3ds bios": (("systems/nintendo-3ds/",), 1), + "playstation bios": (("systems/sony-playstation/",), 1), + "playstation 2 bios": (("systems/sony-playstation-2/",), 1), + "playstation 3 bios": (("systems/sony/",), 1), + "game boy advance bios": (("systems/nintendo/",), 1), + "game boy color bios": (("systems/nintendo/",), 1), + "nintendo ds bios": (("systems/nintendo-nintendo-ds/",), 1), + "nintendo dsi bios": (("systems/nintendo-nintendo-dsi/",), 1), + "wii bios": (("systems/nintendo/",), 1), + "wii u bios": (("systems/nintendo-wii-u/",), 1), + "saturn bios": (("systems/sega-saturn/",), 1), "check my bios folder": (("wiki/troubleshooting/",), 3), "is my bios correct": (("files/",), 3), "schema": (("data/",), 3), diff --git a/website/astro.config.mjs b/website/astro.config.mjs index 157a23fbb..3d94fdcc3 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -108,26 +108,37 @@ export default defineConfig({ // ten times its text, so "dolphin" finds Dolphin, and the file names of // a system page (its `files` field, written by sitestarlight.py) count // half, so "scph5501" finds the Sony page without a file name winning - // every other query. Two fields name the page's subject in other words - // and weigh half a title: `aliases`, the ids setups file a system under - // (psx, dreamcast, gba), and `devices`, the handhelds and boxes a - // setup's own site names (R36S, Miyoo Mini). `consoles`, the consoles a - // manufacturer page holds (Neo Geo, WonderSwan Color), weighs as much - // as a title. `terms`, the words of a task or a message on screen (bios - // not found, which mame version, check my dump), weighs three times - // that: a field match scores below a title match, and the cards of a - // part page keep their own weight, so below 30 the MAME parts, whose - // card titles name MAME files, came before the version table for - // "which mame version". All of them are - // attributes on an empty element, so no text of theirs reaches an - // excerpt. + // every other query. `devices`, the handhelds and boxes a setup's own + // site names (R36S, Miyoo Mini), weighs half a title. `aliases`, the ids + // setups file a system under (psx, dreamcast, gba), weighs a title: with + // file names written as their parts, half a title put GBA.emu and two + // more emulators above the Nintendo page for "gba bios". `terms`, the + // words of a task or a message on screen (bios not found, which mame + // version, check my dump), weighs three titles: a field match scores + // below a title match, and the cards of a part page keep their own + // weight, so below 30 the MAME parts, whose card titles name MAME files, + // came before the version table for "which mame version". + // + // A field adds the same score to every page whose field holds the + // word, whatever else the field holds, and a query word also matches + // the start of a longer one. `consoles`, the consoles a system page + // answers for, weighs ten titles: below that an emulator named after + // its console (Saturn.emu, BK, PIECE) came before the console. The + // PlayStation 2 page holds every word of "playstation bios", so + // `exact` repeats, on its own page, a console whose every word another + // page's consoles hold (PlayStation, Nintendo DS, Nintendo Super + // System). It weighs two titles: enough over the other page, and less + // than the "3" of "playstation 3 bios" adds to the Sony page. Measured + // on the " bios" query of every system page and on + // tools/search_gate.py. All of them are attributes on an empty + // element, so no text of theirs reaches an excerpt. pagefind: { ranking: { pageLength: 0.1, termFrequency: 0.5, termSaturation: 1.4, termSimilarity: 9, - metaWeights: { title: 10, files: 0.5, aliases: 5, devices: 5, consoles: 10, terms: 30 }, + metaWeights: { title: 10, files: 0.5, aliases: 10, devices: 5, consoles: 100, exact: 20, terms: 30 }, }, }, plugins: [ diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 81fa3a2a7..3c7315a1a 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -784,11 +784,11 @@ const hashLookupRow = (drawer: HTMLElement, value: string) => { // Pagefind lists a page's title twice: once for the page and once for the // anchor of its h1 (#_top). The second title is hidden, its excerpt kept. -// The `files`, `aliases`, `devices`, `consoles` and `terms` fields weigh in +// The `files`, `aliases`, `devices`, `consoles`, `exact` and `terms` fields weigh in // the ranking and are never shown: a result is not a list of 120 names. Pagefind reuses // its nodes from one query to the next, so both are worked out again on // every change. -const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'terms']); +const UNSHOWN_FIELDS = new Set(['files', 'aliases', 'devices', 'consoles', 'exact', 'terms']); const tidyResults = (root: Element) => { for (const title of root.querySelectorAll('.pagefind-ui__result-nested > .pagefind-ui__result-title')) { diff --git a/website/tools/search_gate.py b/website/tools/search_gate.py index 8c7bdc3ac..72ece91a2 100644 --- a/website/tools/search_gate.py +++ b/website/tools/search_gate.py @@ -70,6 +70,19 @@ QUERIES: dict[str, tuple[tuple[str, ...], int]] = { "gamecube bios": (("systems/nintendo-gamecube/",), 1), "game boy bios": (("systems/nintendo-game-boy/",), 1), "3ds bios": (("systems/nintendo-3ds/",), 1), + # A console another one's name begins with, both ways: the longer name's + # page holds every word of the shorter query. GBA, GBC and the + # PlayStation 3 are sections of their maker's page; Saturn.emu is an + # emulator named after its console. + "playstation 2 bios": (("systems/sony-playstation-2/",), 1), + "playstation 3 bios": (("systems/sony/",), 1), + "game boy advance bios": (("systems/nintendo/",), 1), + "game boy color bios": (("systems/nintendo/",), 1), + "nintendo ds bios": (("systems/nintendo-nintendo-ds/",), 1), + "nintendo dsi bios": (("systems/nintendo-nintendo-dsi/",), 1), + "wii bios": (("systems/nintendo/",), 1), + "wii u bios": (("systems/nintendo-wii-u/",), 1), + "saturn bios": (("systems/sega-saturn/",), 1), # The words an emulator prints, and the page that answers them. "bios not found": (("wiki/troubleshooting/",), 3), "please insert playstation cd-rom": (("wiki/troubleshooting/",), 3),