feat: settle site integration items

scripts/artifacts.py

@@ -21,6 +21,7 @@ _TIMESTAMP_PATTERNS = [
     # The decorated pages carry the same stamp again as a rendered element,
     # and missing it rewrote every page on every run for the clock alone.
     re.compile(r'<div class="rb-timestamp">[^<]*</div>'),
+    re.compile(r'<time datetime="[^"]*">Data as of [^<]*</time>'),
 ]
 
 def write_if_changed(

scripts/generate_site.py

@@ -416,6 +416,11 @@ def data_as_of(moment: datetime | None = None) -> str:
             f"Data as of {moment.day} {moment.strftime('%B %Y')}</time>")
 
 
+def _date_line() -> str:
+    """The date a page closes on, in the words of data_as_of()."""
+    return f'<div class="rb-timestamp">{data_as_of()}.</div>'
+
+
 
 
 # Home page
@@ -2199,6 +2204,7 @@ def decorate_markdown_pages(docs: Path) -> dict[str, str]:
             browser_title=browser_title,
             description=description, structured=structured, terms=terms,
             search=authored.get("search") != "exclude",
+            logo=str(authored.get("logo") or ""),
         )
         write_decorated(docs / target, text)
         # One file per page: a page that gained or lost its components
@@ -2339,6 +2345,50 @@ def _file_runs(body: list[str], budget: int) -> list[list[str]]:
     return [head + run for run in runs]
 
 
+# A list moved off a reference page (gaps, provenance) is cut in parts of
+# at most this much Markdown: about 450 KB of HTML each.
+LIST_PART_BUDGET = 300_000
+
+
+def _blocks_in_runs(blocks: list[list[str]], budget: int) -> list[list[str]]:
+    """Blocks (table rows, folded lists) in runs of at most ``budget``
+    characters, a block never cut."""
+    runs: list[list[str]] = [[]]
+    size = 0
+    for block in blocks:
+        weight = sum(len(line) + 1 for line in block)
+        if runs[-1] and size + weight > budget:
+            runs.append([])
+            size = 0
+        runs[-1].extend(block)
+        size += weight
+    return runs
+
+
+def _list_pages(
+    stem: str, title: str, intro: str, head: list[str], blocks: list[list[str]],
+    moved: dict[str, str], budget: int = LIST_PART_BUDGET,
+) -> str:
+    """Pages of their own for a long list: ``head`` (a table header, say)
+    opens each part, ``blocks`` are cut between parts. Returns where the list
+    went, as the page it left says it."""
+    runs = _blocks_in_runs(blocks, budget)
+    stems = [stem + (f"-{n}" if n > 1 else "") for n in range(1, len(runs) + 1)]
+    for n, (here, run) in enumerate(zip(stems, runs), 1):
+        part = f" (part {n} of {len(runs)})" if len(runs) > 1 else ""
+        moved[here] = "\n".join([
+            "---", "level: reference", "---", "",
+            f"# {title}{part}", "", intro, "",
+            *([", ".join(f"[part {i}]({other}.md)" if other != here else f"part {i}"
+                         for i, other in enumerate(stems, 1)), ""] if len(runs) > 1 else []),
+            *head, *run, "", _date_line(),
+        ]) + "\n"
+    if len(stems) == 1:
+        return f"[a page of their own]({stems[0]}.md)"
+    return "pages of their own: " + ", ".join(
+        f"[part {i}]({other}.md)" for i, other in enumerate(stems, 1))
+
+
 @functools.lru_cache(maxsize=1)
 def _collection_sha1_by_path() -> dict[str, str]:
     """The sha1 of each file the collection holds, by its path."""
@@ -2354,13 +2404,13 @@ def _system_heading_link(sys_id: str, prefix: str = "") -> str:
     writes. A system with no section of its own is named, never sent to the
     page a storage folder chose.
     """
-    from siterender import _own_section
+    from siterender import system_section
 
     name = _escape_tags(system_name(sys_id))
-    section = _own_section(sys_id)
+    section = system_section(sys_id)
     if section is not None:
-        mfr, console = section
-        name = f"[{name}]({console_href(_slug(mfr), console, prefix)})"
+        slug, console = section
+        name = f"[{name}]({console_href(slug, console, prefix)})"
     return f"{name} <small>{_escape_tags(sys_id)}</small>"
 
 
@@ -2374,7 +2424,7 @@ def _served_line(f: dict) -> str:
     if f.get("status") == "missing":
         return "Not held"
     if not path:
-        # verify reports no path in existence mode: nothing to name.
+        # A slot the pack fills from a release asset or leaves to the user.
         return ""
     sha1 = _collection_sha1_by_path().get(path, "")
     label = "Served by" if f.get("status") == "ok" else "Held, not the declared hash"
@@ -2580,7 +2630,7 @@ def _render_platform_systems(
                         for i, other in enumerate(stems, 1))] if len(runs) > 1 else []),
                     "", "## Files", "",
                     *search_excluded(run),
-                    "", f"*Generated on {_timestamp()}*",
+                    "", _date_line(),
                 ]) + "\n"
             where = (f"[a page of their own]({stems[0]}.md)" if len(stems) == 1 else
                      "pages of their own: " + ", ".join(
@@ -2921,6 +2971,7 @@ def generate_platform_page(
     # The mark is drawn beside the title from the front matter: floated
     # beside the first line at 48 px it read as a broken image.
     logo_url = registry.get(name, {}).get("logo", "")
+    logo_url = local_images().get(logo_url, logo_url)
 
     homepage = config.get("homepage", "")
     version = config.get("version", "")
@@ -3086,7 +3137,7 @@ def generate_platform_page(
                                  (name, display), moved, config)
     )
 
-    lines.append(f"*Generated on {_timestamp()}*")
+    lines.extend(["", _date_line()])
     return "\n".join(lines) + "\n"
 
 
@@ -3222,12 +3273,15 @@ def file_location_items(
     for ref, files in sorted((members or {}).items()):
         if ref not in owners:
             continue
-        for _member, info in sorted(files.items()):
+        # A large tree is listed on pages of its own (data_tree_pages).
+        page_of = {member: stem for stem, names in data_tree_pages(ref, files)
+                   for member in names}
+        for member, info in sorted(files.items()):
             sha1 = str(info.get("sha1", ""))
             if sha1 and sha1 not in known:
                 known.add(sha1)
-                items.append({"sha1": sha1,
-                              "url": f"emulators/{owners[ref]}/#sha1-{sha1[:12]}"})
+                page = page_of.get(member, owners[ref])
+                items.append({"sha1": sha1, "url": f"emulators/{page}/#sha1-{sha1[:12]}"})
     return sorted(items, key=lambda item: item["sha1"])
 
 
@@ -3434,8 +3488,7 @@ def generate_systems_index(manufacturers: dict) -> str:
         ]
         for name, mfr, console, count in rows:
             cell = name.replace("|", "-")
-            anchor = _slugify_anchor(console)
-            href = f"{_slug(mfr)}.md" + (f"#{anchor}" if anchor else "")
+            href = console_href(_slug(mfr), console).removeprefix("systems/")
             lines.append(f"| [{cell}]({href}) | {count:,} | [{mfr}]({_slug(mfr)}.md) |")
         lines.append("")
 
@@ -3527,9 +3580,13 @@ def _date_in_words(stamp: str) -> str:
 
 
 def generate_provenance_page(
-    db: dict, report: dict, derivable: dict[str, dict] | None = None
+    db: dict, report: dict, derivable: dict[str, dict] | None = None,
+    moved: dict[str, str] | None = None,
 ) -> str:
-    """Page explaining the verified dump badges and listing catalog gaps."""
+    """Page explaining the verified dump badges and listing catalog gaps.
+
+    With ``moved``, each catalog's wanted list goes to a page of its own,
+    keyed by stem: the three lists made a 694 KB page."""
     matched_files = count_catalog_matched(db)
     total_files = compute_composition(db)["systems"]["files"]
     # When each catalog was read: a 76-day-old snapshot showed no age.
@@ -3664,6 +3721,7 @@ def generate_provenance_page(
             for dat in entry.get("dats") or [entry.get("dat", "")]:
                 by_dat.setdefault(dat, []).append(entry)
         lines.extend([f"### {label}", ""])
+        blocks: list[list[str]] = []
         for dat in sorted(by_dat):
             entries = by_dat[dat]
             table = ["| Name | Description | SHA1 |", "|------|-------------|------|"]
@@ -3679,7 +3737,20 @@ def generate_provenance_page(
                         f"`{hit['archive']}` as `{hit['member']}`."
                     )
                 table.append(f"| `{name}` | {desc} | `{sha1}` |")
-            lines.extend(markup.details("note", f"{dat} ({len(entries)})", table))
+            blocks.append(markup.details("note", f"{dat} ({len(entries)})", table))
+        if moved is None:
+            lines.extend(line for block in blocks for line in block)
+            continue
+        where = _list_pages(
+            f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}",
+            f"Dumps {label} catalogues and the collection lacks, by DAT. Part of "
+            f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).",
+            [], blocks, moved,
+        )
+        lines.extend([
+            f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: "
+            f"listed on {where}.", "",
+        ])
 
     if derived_entries:
         lines.extend(
@@ -3701,7 +3772,7 @@ def generate_provenance_page(
             lines.append(f"| `{name}` | {sources} | {how.get('transform', '')} | `{sha1}` |")
         lines.append("")
 
-    lines.append(f'<div class="rb-timestamp">Generated on {_timestamp()}.</div>')
+    lines.extend(["", _date_line()])
     return "\n".join(lines) + "\n"
 
 
@@ -4110,10 +4181,12 @@ def system_install_choices(coverages: dict) -> dict[str, dict[str, list[tuple[st
 
 
 # A manufacturer page above this much Markdown becomes a summary with one
-# page per console, and a console above it is cut into parts. 500 KB of
-# Markdown renders to about 750 KB of HTML: game-engines was one 10.5 MB
-# page that took 21 s of main-thread work on a phone.
-SYSTEM_PAGE_BUDGET = 500_000
+# page per console, and a console above it is cut into parts. Measured as
+# MkDocs Markdown, a page renders to up to 1.8 times its size in HTML with
+# its structured data: at 500 KB the Sony page was 849 KB, and every page
+# stays under 600 KB from 300 KB. game-engines was one 10.5 MB page that
+# took 21 s of main-thread work on a phone.
+SYSTEM_PAGE_BUDGET = 300_000
 
 # A system card: whether it is a variant, its lines, its sha1.
 Card = tuple[bool, list[str], str]
@@ -4289,11 +4362,12 @@ def _console_heading(manufacturer: str, console: str) -> str:
     """A console's heading: its icon hidden from screen readers, then the
     name with no space before it. The space read as "Section titled
     “ PlayStation”" on five of seven Sony headings."""
+    anchor = f" {{ #{console_anchor(manufacturer, console)} }}"
     icon = _icon_name(manufacturer, console)
     if not _icon_available.get(icon):
-        return f"## {console}"
+        return f"## {console}{anchor}"
     image = markup.image("", _icon_url(icon), width="24", **{"aria-hidden": "true"})
-    return f"## {image}{console}"
+    return f"## {image}{console}{anchor}"
 
 
 _NEXT_FROM_SYSTEMS = [
@@ -4355,7 +4429,7 @@ def system_pages(
             lines.extend(_cards_markdown(cards))
             lines.append("")
         lines.extend(_next_block(_NEXT_FROM_SYSTEMS))
-        lines.append(f'<div class="rb-timestamp">Generated on {_timestamp()}.</div>')
+        lines.extend(["", _date_line()])
         return "\n".join(lines) + "\n"
 
     def card_weight(card: list[str]) -> int:
@@ -4452,12 +4526,12 @@ def system_pages(
             lines.append("")
             lines.extend(_next_block([f"[{manufacturer}]({slug}.md), every section"]
                                      + _NEXT_FROM_SYSTEMS))
-            lines.append(f'<div class="rb-timestamp">Generated on {_timestamp()}.</div>')
+            lines.extend(["", _date_line()])
             held = {sha1 for _variant, _card, sha1 in part}
             pages[stem] = ("\n".join(lines) + "\n", {console: [
                 f for f in consoles[console] if str(f.get("sha1", "")) in held]})
     summary.extend(_next_block(_NEXT_FROM_SYSTEMS))
-    summary.append(f'<div class="rb-timestamp">Generated on {_timestamp()}.</div>')
+    summary.extend(["", _date_line()])
     kept = {console: files for console, files in consoles.items() if console not in moved}
     return {slug: ("\n".join(summary) + "\n", kept), **pages}
 
@@ -4498,13 +4572,13 @@ def _named_system_link(sys_id: str, prefix: str = "") -> str:
     The emulator pages printed the id ("sony-playstation") where every
     other page says "Sony PlayStation". The id stays in the address only.
     """
-    from siterender import _page_by_norm  # noqa: PLC0415
+    from siterender import system_section  # noqa: PLC0415
 
     name = system_name(sys_id).replace("[", r"\[").replace("]", r"\]")
-    page = _system_page_map.get(sys_id) or _page_by_norm(sys_id)
-    if not page:
+    section = system_section(sys_id)
+    if not section:
         return name
-    slug, console = page
+    slug, console = section
     return f"[{name}]({console_href(slug, console, prefix)})"
 
 
@@ -4543,6 +4617,16 @@ def _region_words(value: object, most: int = 0) -> str:
 _CATEGORY_WORDS = {"bios": "BIOS", "game_data": "game data", "bios_zip": "BIOS ZIP"}
 
 
+INITIAL_DIGITS = "0 to 9"
+
+
+def emulator_initial(display: str) -> str:
+    """The letter an emulator is filed under, in the navigation and on the
+    index alike. A name that does not open on a letter goes under "0 to 9"."""
+    initial = display[:1].upper()
+    return initial if "A" <= initial <= "Z" else INITIAL_DIGITS
+
+
 def generate_emulators_index(profiles: dict) -> str:
     unique = {
         k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test")
@@ -4562,13 +4646,13 @@ def generate_emulators_index(profiles: dict) -> str:
         "",
         "Each page is one emulator: the files it reads, whether it starts games "
         "without them, and the line of its source code that says so. The "
-        "classification below says how a core relates to the emulator it comes "
-        "from, which tells how far its own files can differ.",
+        "kind says how a core relates to the emulator it comes from, which "
+        "tells how far its own files can differ.",
         "",
         f"**{len(unique):,}** emulator profiles, **{total_files:,}** files total, **{len(aliases):,}** aliases.",
         "",
-        "| Classification | Count | Description |",
-        "|---------------|-------|-------------|",
+        "| Kind | Count | Description |",
+        "|------|-------|-------------|",
     ]
 
     cls_desc = {
@@ -4601,28 +4685,25 @@ def generate_emulators_index(profiles: dict) -> str:
             continue
         label = CLS_LABELS.get(cls, cls)
         desc = cls_desc.get(cls, "")
-        lines.append(f"| [{label}](#{cls}) | {len(entries)} | {desc} |")
+        lines.append(f"| {label} | {len(entries)} | {desc} |")
     lines.append("")
 
-    for cls in cls_order:
-        entries = by_class.get(cls, [])
-        if not entries:
-            continue
-        label = CLS_LABELS.get(cls, cls)
-        desc = cls_desc.get(cls, "")
-        lines.extend(
-            [
-                f"## {label} {{ #{cls} }}",
-                "",
-                f"{desc}. {_count(len(entries), 'profile')}.",
-                "",
-                "| Emulator | Systems | Files |",
-                "|--------|---------|-------|",
-            ]
-        )
-
+    # One section per initial, the groups the navigation draws: its letter
+    # links land on the section of the same name.
+    by_initial: dict[str, list[tuple[str, dict]]] = {}
+    for name, p in unique.items():
+        by_initial.setdefault(emulator_initial(str(p.get("emulator", name))), []).append((name, p))
+    for initial in sorted(by_initial, key=lambda g: (g != INITIAL_DIGITS, g)):
+        lines.extend([
+            f"## {initial}",
+            "",
+            "| Emulator | Kind | Systems | Files |",
+            "|--------|------|---------|-------|",
+        ])
+        entries = sorted(by_initial[initial], key=lambda e: str(e[1].get("emulator", e[0])).casefold())
         for name, p in entries:
             emu_name = p.get("emulator", name)
+            cls = p.get("core_classification", "other")
             systems = list(dict.fromkeys(
                 _named_system_link(s, "../") for s in p.get("systems") or []))
             files = p.get("files", [])
@@ -4631,7 +4712,7 @@ def generate_emulators_index(profiles: dict) -> str:
                 sys_str += f" and {len(systems) - 3} more"
             file_count = len(files)
             file_str = f"{file_count:,}" if file_count else "-"
-            lines.append(f"| [{emu_name}]({name}.md) | {sys_str} | {file_str} |")
+            lines.append(f"| [{emu_name}]({name}.md) | {CLS_LABEL_ONE.get(cls, cls)} | {sys_str} | {file_str} |")
         lines.append("")
 
     if aliases:
@@ -5440,14 +5521,73 @@ def _said_as_list(items: list[str]) -> str:
 # Data directory members and revisions, read once per build.
 _DATA_MEMBERS: dict[str, dict[str, dict]] = {}
 _DATA_REVISIONS: dict[str, dict] = {}
+# The emulator pages that list each data directory, by ref.
+_DATA_READERS: dict[str, list[tuple[str, str]]] = {}
+
+# A data directory listing more files than this goes to pages of its own,
+# this many files each: Dolphin's Sys tree put 2,164 rows and 16,900
+# elements on the Dolphin page, read on a phone.
+DATA_TREE_ROWS = 400
+DATA_TREE_ROWS_PER_PAGE = 1_000
+
+
+def data_tree_pages(ref: str, members: dict[str, dict]) -> list[tuple[str, list[str]]]:
+    """The pages under emulators/ listing a large data directory, with the
+    members each holds in name order, or none when the tree is listed on
+    the pages of the emulators reading it."""
+    if len(members) <= DATA_TREE_ROWS:
+        return []
+    names = sorted(members)
+    stem = f"data-{_slugify_anchor(ref)}"
+    return [(stem + (f"-{n}" if n > 1 else ""), names[start:start + DATA_TREE_ROWS_PER_PAGE])
+            for n, start in enumerate(range(0, len(names), DATA_TREE_ROWS_PER_PAGE), 1)]
+
+
+def _data_member_table(members: dict[str, dict], names: list[str], anchors: set[str]) -> list[str]:
+    """A data directory's files with their size and sha1, each sha1 an
+    anchor the first time the page lists it (``anchors``, the page's ids)."""
+    table = ["| File | Size | SHA1 |", "|------|-----:|------|"]
+    for member in names:
+        info = members[member]
+        sha1 = str(info.get("sha1", ""))
+        # One id per page: two members can hold the same bytes (every empty
+        # file), and Dolphin's trees share Sys.
+        anchor = f"sha1-{sha1[:12]}"
+        span = "" if anchor in anchors else f'<span id="{anchor}"></span>'
+        anchors.add(anchor)
+        table.append(f"| {span}{_code(member)} | {_fmt_size(int(info.get('size', 0)))} | {_code(sha1)} |")
+    return table
+
+
+def _data_tree_page(ref: str, entry: dict, pages: list[tuple[str, list[str]]],
+                    index: int) -> str:
+    """One page of a large data directory's file list."""
+    stem, names = pages[index]
+    members = _DATA_MEMBERS.get(ref) or {}
+    part = f" (part {index + 1} of {len(pages)})" if len(pages) > 1 else ""
+    readers = _said_as_list([f"[{_escape_tags(display)}]({key}.md#data-directories)"
+                             for key, display in _DATA_READERS.get(ref, [])])
+    lines = [
+        "---", "level: reference", "---", "",
+        f"# The {ref} data directory{part}", "",
+        f"{entry.get('description') or 'A data directory'}: "
+        f"{_count(len(members), 'file')}, as the packs ship them."
+        + (f" Read by {readers}." if readers else ""), "",
+    ]
+    if len(pages) > 1:
+        lines.extend([", ".join(f"[part {i}]({other}.md)" if other != stem else f"part {i}"
+                                for i, (other, _names) in enumerate(pages, 1)), ""])
+    lines.extend(search_excluded(_data_member_table(members, names, set())))
+    lines.extend(["", _date_line()])
+    return "\n".join(lines) + "\n"
 
 
 # An emulator page lists at most this many file cards: MAME's 1,564 made a
 # 3 MB page. Past it, the cards go to pages of their own and the emulator's
 # page keeps a table of every entry, each row holding the card's old
 # anchor. A count, not a size: system cards link to these cards and are
-# written before them.
-EMULATOR_CARDS_PER_PAGE = 400
+# written before them. At 400 a part of MAME's cards was 805 KB of HTML.
+EMULATOR_CARDS_PER_PAGE = 220
 
 
 # The headings an emulator's cards go under, in page order: what it needs
@@ -5818,19 +5958,16 @@ def emulator_pages(
             if dd.get("source_ref"):
                 lines.append(f"    - Source: {_source_ref_markdown(profile, dd['source_ref'])}")
             members = _DATA_MEMBERS.get(ref) or {}
-            if members:
-                table = ["| File | Size | SHA1 |", "|------|-----:|------|"]
-                for member, info in sorted(members.items()):
-                    sha1 = str(info.get("sha1", ""))
-                    # One id per page: two members can hold the same bytes
-                    # (every empty file), and Dolphin's trees share Sys.
-                    anchor = f"sha1-{sha1[:12]}"
-                    span = "" if anchor in member_anchors else f'<span id="{anchor}"></span>'
-                    member_anchors.add(anchor)
-                    table.append(
-                        f"| {span}{_code(member)} "
-                        f"| {_fmt_size(int(info.get('size', 0)))} | {_code(sha1)} |"
-                    )
+            tree_pages = data_tree_pages(ref, members)
+            if tree_pages:
+                where = (f"[a page of their own]({tree_pages[0][0]}.md)" if len(tree_pages) == 1
+                         else "pages of their own: " + ", ".join(
+                             f"[part {i}]({stem}.md)" for i, (stem, _names) in enumerate(tree_pages, 1)))
+                lines.append(f"    - {_count(len(members), 'file')}, listed on {where}")
+                for index, (stem, _names) in enumerate(tree_pages):
+                    part_pages[stem] = _data_tree_page(ref, entry, tree_pages, index)
+            elif members:
+                table = _data_member_table(members, sorted(members), member_anchors)
                 lines.append("")
                 lines.extend(markup.details(
                     "info", f"{_count(len(members), 'file')}, as the packs ship them",
@@ -5844,7 +5981,7 @@ def emulator_pages(
         "[Every system](../systems/index.md), and the files each one needs",
         "[When a game does not start](../wiki/troubleshooting.md)",
     ]))
-    lines.extend([f"*Generated on {_timestamp()}*"])
+    lines.extend(["", _date_line()])
     return {name: "\n".join(lines) + "\n", **part_pages}
 
 
@@ -5901,7 +6038,7 @@ def _emulator_part_page(
                     lambda group: [line for i in group for line in cards[i]]),
     ]
     lines.extend(_next_block([f"[{emu_name}]({name}.md), the profile and every entry"]))
-    lines.extend([f"*Generated on {_timestamp()}*"])
+    lines.extend(["", _date_line()])
     return "\n".join(lines) + "\n"
 
 
@@ -6194,9 +6331,13 @@ def generate_gap_analysis(
     data_names: set[str] | None = None,
     registry: dict | None = None,
     gap_report: dict | None = None,
+    moved: dict[str, str] | None = None,
 ) -> str:
     """Generate a unified gap analysis page.
 
+    With ``moved``, the list of files to source goes to pages of its own,
+    keyed by stem: on the page it made 662 KB of HTML.
+
     Combines verification results (from coverages/verify.py) with source
     provenance (from cross_reference) into a single truth dashboard.
 
@@ -6560,19 +6701,19 @@ def generate_gap_analysis(
 
         if src_missing_details:
             req_src = [m for m in src_missing_details if m["required"]]
-            lines.extend([
-                "### Files to source",
-                "",
+            counted = (
                 f"{_count(len(src_missing_details), 'file')}, {len(req_src)} required."
                 + (
                     f" They come from {_count(entries, 'entry', 'entries')}: a file "
                     "declared with the same hash at several places is listed once."
                     if entries != len(src_missing_details) else ""
-                ),
-                "",
+                )
+            )
+            head = [
                 "| File | Destination | Expected | Emulator | Required | Affects platforms | Source ref |",
                 "|------|-------------|----------|----------|----------|------------------|-----------|",
-            ])
+            ]
+            rows: list[str] = []
             for m in sorted(
                 src_missing_details,
                 key=lambda x: (not x["required"], x["name"]),
@@ -6596,12 +6737,23 @@ def generate_gap_analysis(
                     f"[{_escape_tags(display)}](emulators/{key}.md)"
                     for display, key in zip(m["emulators"], m["emu_keys"])
                 )
-                lines.append(
+                rows.append(
                     f"| `{m['name']}` | {destination} | {expected} | {emulators} | {req} | "
                     f"{plat_badges} | "
                     f"{_source_ref_markdown(profiles[m['emu_keys'][0]], m['source_ref'])} |"
                 )
-            lines.append("")
+            lines.extend(["### Files to source", "", counted, ""])
+            if moved is None:
+                lines.extend([*head, *rows, ""])
+            else:
+                where = _list_pages(
+                    "gaps-files-to-source", "Files to source",
+                    f"{counted} Each is a file an emulator loads that no platform lists and "
+                    "the collection does not hold. Part of "
+                    "[What the packs do not hold](gaps.md#files-to-source).",
+                    head, [[row] for row in rows], moved,
+                )
+                lines.extend([f"Listed on {where}.", ""])
         if bundled_rows:
             emulators = sorted({emulator for emulator, _name in bundled_rows})
             lines.extend([
@@ -6641,7 +6793,7 @@ def generate_gap_analysis(
             )
         lines.append("")
 
-    lines.extend(["", f'<div class="rb-timestamp">Generated on {_timestamp()}.</div>'])
+    lines.extend(["", _date_line()])
     return "\n".join(lines) + "\n"
 
 
@@ -6769,7 +6921,7 @@ def _xref_platform_rows(config: dict, unique: dict) -> tuple[int, list[str]]:
                             if upstream.startswith("http") else (_escape_tags(upstream) or "-"))
         rows.append(
             f"| [{_escape_tags(p.get('emulator', emu_name))}](emulators/{emu_name}.md) "
-            f"| {CLS_LABELS.get(cls_raw, cls_raw)} | {sys_str or '-'} | {file_str} "
+            f"| {CLS_LABEL_ONE.get(cls_raw, cls_raw)} | {sys_str or '-'} | {file_str} "
             f"| {upstream_display} |"
         )
     return len(matched), rows
@@ -6851,7 +7003,7 @@ def _render_xref_by_upstream(
         members = by_upstream[key]
         cores = sorted(c for c, _url, _rest in members)
         classifications = sorted({
-            CLS_LABELS.get(unique[c].get("core_classification", "-"),
+            CLS_LABEL_ONE.get(unique[c].get("core_classification", "-"),
                            unique[c].get("core_classification", "-"))
             for c in cores
         })
@@ -6912,7 +7064,7 @@ def generate_cross_reference(
     lines.extend(_render_xref_by_platform(coverages, unique, moved))
     lines.extend(_render_xref_by_upstream(coverages, unique))
 
-    lines.extend(["", f"*Generated on {_timestamp()}*"])
+    lines.extend(["", _date_line()])
     return "\n".join(lines) + "\n"
 
 
@@ -7062,7 +7214,7 @@ def generate_wiki_data_model(db: dict, profiles: dict) -> str:
         "## Static API and bulk exports",
         "",
         "The website publishes versioned JSON, CSV and SQLite metadata generated "
-        "from these same structures. Start with the [Data & API](../data.md) "
+        "from these same structures. Start with the [Data and API](../data.md) "
         "catalog. Each downloadable artifact carries a SHA256 in "
         "`api/v1/catalog.json`.",
         "",
@@ -7348,11 +7500,9 @@ def _emulator_nav(profiles: dict) -> list:
     groups: dict[str, list[tuple[str, str]]] = {}
     for key, profile in nav_emulators(profiles).items():
         display = str(profile.get("emulator", key))
-        initial = display[:1].upper()
-        label = initial if "A" <= initial <= "Z" else "0 to 9"
-        groups.setdefault(label, []).append((display, f"emulators/{key}.md"))
+        groups.setdefault(emulator_initial(display), []).append((display, f"emulators/{key}.md"))
     nav: list = ["emulators/index.md"]
-    for label in sorted(groups, key=lambda g: (g != "0 to 9", g)):
+    for label in sorted(groups, key=lambda g: (g != INITIAL_DIGITS, g)):
         items = sorted(groups[label], key=lambda e: e[0].casefold())
         nav.append({label: [{display: path} for display, path in items]})
     return nav
@@ -7755,6 +7905,17 @@ def _generate(args: argparse.Namespace) -> None:
     print("Generating emulator pages...")
     _DATA_MEMBERS.update(data_directory_members())
     _DATA_REVISIONS.update(data_directory_revisions())
+    for key in sorted(profiles):
+        if profiles[key].get("type") in ("alias", "test"):
+            continue
+        display = str(profiles[key].get("emulator", key))
+        for tree in page_data_directories(key, profiles):
+            readers = _DATA_READERS.setdefault(str(tree.get("ref", "")), [])
+            if (key, display) not in readers:
+                readers.append((key, display))
+    for ref, members in _DATA_MEMBERS.items():
+        for stem, _names in data_tree_pages(ref, members):
+            PART_PAGES.add(f"emulators/{stem}.md")
     write_if_changed(
         str(docs / "emulators" / "index.md"), generate_emulators_index(profiles)
     )
@@ -7810,12 +7971,15 @@ def _generate(args: argparse.Namespace) -> None:
         write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page))
 
     print("Generating gap analysis page...")
+    gap_pages: dict[str, str] = {}
     write_if_changed(
         str(docs / "gaps.md"),
         tables_out_of_search(generate_gap_analysis(
-            profiles, coverages, db, suppl_names, registry, gap_report
+            profiles, coverages, db, suppl_names, registry, gap_report, gap_pages,
         )) + reference_next(),
     )
+    for stem, page in gap_pages.items():
+        write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page))
 
     print("Generating updated platform files page...")
     write_if_changed(
@@ -7828,12 +7992,15 @@ def _generate(args: argparse.Namespace) -> None:
 
     print("Generating dump provenance page...")
     provenance_report = build_report(db, load_provenance_snapshots())
+    provenance_pages: dict[str, str] = {}
     write_if_changed(
         str(docs / "provenance.md"),
         tables_out_of_search(
-            generate_provenance_page(db, provenance_report, load_derivable())
+            generate_provenance_page(db, provenance_report, load_derivable(), provenance_pages)
         ) + reference_next(),
     )
+    for stem, page in provenance_pages.items():
+        write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page))
 
     # Wiki pages: copy manually maintained sources + generate dynamic ones
     print("Generating wiki pages...")
@@ -7895,7 +8062,7 @@ def _generate(args: argparse.Namespace) -> None:
         print(f"Removed {stale} page(s) nothing produced this run")
 
 
-from sitediagrams import prepare_page, unresolved_images  # noqa: E402,F401
+from sitediagrams import local_images, prepare_page, unresolved_images  # noqa: E402,F401
 from sitewrite import (  # noqa: E402,F401
     _produced,
     _record,
@@ -7927,6 +8094,7 @@ from siterender import (  # noqa: E402,F401
     system_name,
     card_page,
     console_href,
+    console_anchor,
     record_console_pages,
     record_emulator_cards,
     emulator_card_page,

scripts/sitediagrams.py

@@ -330,7 +330,9 @@ def setups() -> tuple[str, ...]:
 
 def setups_named(named: list[str]) -> str:
     """*named*, then how many other setups there are: "A, B and 15 more"."""
-    others = len(setups()) - len(named)
+    from siteguides import setup_count
+
+    others = setup_count() - len(named)
     return ", ".join(named) + (f" and {others} more" if others else "")
 
 

scripts/siterender.py

@@ -170,13 +170,31 @@ def split_system_stems() -> set[str]:
     return {stem for stems in _console_pages.values() for stem in stems}
 
 
+# (manufacturer slug, console) -> the id of the console's heading. Two
+# folders spelled alike (Sega's "Mega CD" and "Mega-CD") share a slug, and
+# the second heading took "mega-cd_1" while every link said "mega-cd".
+_console_anchors: dict[tuple[str, str], str] = {}
+
+
+def _manufacturer_slug(manufacturer: str) -> str:
+    return manufacturer.lower().replace(" ", "-")
+
+
+def console_anchor(manufacturer: str, console: str) -> str:
+    """The id of a console's heading on its manufacturer's page."""
+    from sitestarlight import toc_slug
+
+    key = (_manufacturer_slug(manufacturer), console)
+    return _console_anchors.get(key) or toc_slug(console)
+
+
 def console_href(slug: str, console: str, prefix: str = "") -> str:
     """Where a console's files are: its own page, or its section of the
     manufacturer's page (also where a console cut into parts lists them)."""
     stems = _console_pages.get((slug, console))
     if stems and len(stems) == 1:
         return f"{prefix}systems/{urllib.parse.quote(stems[0])}.md"
-    return f"{prefix}systems/{urllib.parse.quote(slug)}.md#{_slugify_anchor(console)}"
+    return f"{prefix}systems/{urllib.parse.quote(slug)}.md#{console_anchor(slug, console)}"
 
 
 def card_page(path: str, sha1: str) -> str:
@@ -204,11 +222,33 @@ def emulator_card_page(emulator: str, anchor: str) -> str:
     return _emulator_card_pages.get((emulator, anchor), emulator)
 
 
+def system_section(sys_id: str) -> tuple[str, str] | None:
+    """The console section a system id is named after, as (manufacturer
+    slug, console): its own section, else the one section whose heading
+    reads system_name(). The folder a system's files sit in is no answer:
+    Emerson's Arcadia 2001 led to the Saturn and Sega's NAOMI to the
+    Dreamcast."""
+    own = _own_section(sys_id)
+    if own is not None:
+        return _manufacturer_slug(own[0]), own[1]
+    try:
+        name = system_name(sys_id)
+    except KeyError:
+        return None
+    places = _sections_by_name.get(name.casefold(), set())
+    if len(places) != 1:
+        return None
+    mfr, console = next(iter(places))
+    if mfr in _PROGRAM_TOPS:
+        return None
+    return _manufacturer_slug(mfr), console
+
+
 def _system_link(sys_id: str, prefix: str = "") -> str:
-    """Generate a markdown link to a system page with anchor."""
-    page = _system_page_map.get(sys_id) or _page_by_norm(sys_id)
-    if page:
-        slug, console = page
+    """A system id linked to the section named after it, or the bare id."""
+    section = system_section(sys_id)
+    if section:
+        slug, console = section
         return f"[{sys_id}]({console_href(slug, console, prefix)})"
     return sys_id
 
@@ -322,6 +362,8 @@ _PROGRAM_TOPS = frozenset({"Other", "Arcade"})
 
 # Normalized id -> the console sections spelled that way.
 _console_sections: dict[str, set[tuple[str, str]]] = {}
+# A console section by the name its heading reads (Bandai WonderSwan).
+_sections_by_name: dict[str, set[tuple[str, str]]] = {}
 # Emulator and engine names. A heading spelled like one (Game Engines >
 # Solarus, ZC > ZC210) files that program's copies and names no system.
 _program_names: set[str] = set()
@@ -355,7 +397,21 @@ def set_console_sections(
         {(mfr, console) for mfr, consoles in sections.items() for console in consoles}
         if isinstance(sections, dict) else set(sections)
     )
+    from sitestarlight import _unique, toc_slug
+
     _console_sections.clear()
+    _sections_by_name.clear()
+    _console_anchors.clear()
+    by_maker: dict[str, list[str]] = {}
+    for mfr, console in pairs:
+        by_maker.setdefault(mfr, []).append(console)
+        for name in {_heading_name(mfr, console).casefold(), console.casefold()}:
+            _sections_by_name.setdefault(name, set()).add((mfr, console))
+    # In the order the page lists them, after the page title's own id.
+    for mfr, consoles in by_maker.items():
+        used = {toc_slug(mfr)}
+        for console in sorted(consoles):
+            _console_anchors[(_manufacturer_slug(mfr), console)] = _unique(toc_slug(console), used)
     for mfr, console in pairs:
         spelled = console.lower().replace(" ", "-")
         slug = mfr.lower().replace(" ", "-")

scripts/sitestarlight.py

@@ -406,6 +406,7 @@ def finalize_page(
     structured: dict,
     terms: dict[str, str],
     search: bool = True,
+    logo: str = "",
 ) -> tuple[str, str]:
     """The Starlight file for a page body: its path under the content
     collection and its text."""
@@ -446,6 +447,8 @@ def finalize_page(
         front["slug"] = stem
     if level:
         front["level"] = level
+    if logo:
+        front["logo"] = logo
     if weight:
         front["searchWeight"] = weight
     if rel == NOT_FOUND:

scripts/verify.py

@@ -189,12 +189,13 @@ def verify_entry_existence(
     resolve_status: str = "",
     platform_display: str = "",
 ) -> dict:
-    """RetroArch verification: path_is_valid(), a file that exists is OK."""
+    """RetroArch verification: path_is_valid(), a file that exists is OK.
+    The result names the collection file found, as the hash modes do."""
     name = file_entry.get("name", "")
     required = file_entry.get("required", True)
     if not local_path:
         return {"name": name, "status": Status.MISSING, "required": required}
-    result = {"name": name, "status": Status.OK, "required": required}
+    result = {"name": name, "status": Status.OK, "required": required, "path": local_path}
     if resolve_status == "hash_mismatch":
         # The frontend reads no bytes, so the file stays, and the declared hash
         # it contradicts is reported, as the pack does.

tests/test_site_r2_emulators.py

@@ -172,10 +172,10 @@ class TheIndexNamesEmulatorsAndSystems(unittest.TestCase):
             "duckstation": {"emulator": "DuckStation", "core_classification": "official_port",
                             "systems": ["sony-playstation"], "files": _files(3)},
         })
-        self.assertIn("| Emulator | Systems | Files |", page)
+        self.assertIn("| Emulator | Kind | Systems | Files |", page)
+        self.assertIn("\n## D\n", page)
         self.assertNotIn("| Engine |", page)
         self.assertNotIn(" -- ", page)
-        self.assertIn("1 profile.", page)
         row = next(line for line in page.splitlines() if line.startswith("| [DuckStation]"))
         self.assertIn(f"[{siterender.system_name('sony-playstation')}]", row)
         self.assertIn("(../systems/sony.md#playstation)", row)

tests/test_site_r2_gate.py

@@ -0,0 +1,273 @@
+"""Gate of the second site round, read from the built site (website/dist).
+
+Each package of the round was checked on its own. These checks hold the
+merged site to what the round set out to fix: no sidebar label a number, no
+"Next" tip, a sidebar label that is the page's own title, a card's link icon
+kept out of the heading a search indexes, crumbs that lead to a tab or a
+section, phone tables stacked where the reading level asks for it, and every
+page under 15,000 elements and 600 KB of HTML. The integration items of the
+round are held too: the platform logo beside the title, a section for every
+letter of the emulators index, system links to the section of the system's
+own name, dates in words and the moved getting-started page.
+
+Skips when the site is not built.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+import unittest
+from html import unescape
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+DIST = REPO_ROOT / "website" / "dist"
+DOCS = REPO_ROOT / "website" / "src" / "content" / "docs"
+SIDEBAR = REPO_ROOT / "website" / "src" / "generated" / "sidebar.json"
+BASE = "/retrobios/"
+
+PAGE_BYTES = 600_000
+PAGE_ELEMENTS = 15_000
+PHONE_COLUMN = 358
+READER_LEVELS = {"start", "setup"}
+# The size from which a table is a list scanned by column and keeps its
+# scroll (website/src/plugins/rehype-stack-tables.mjs, RECORD_ROWS).
+RECORD_ROWS = 20
+
+_SCRIPT = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.S | re.I)
+_ELEMENT = re.compile(r"<[a-zA-Z][a-zA-Z0-9-]*[\s/>]")
+_TAGS = re.compile(r"<[^>]+>")
+_TABLE = re.compile(r"<table\b(?P<attrs>[^>]*)>(?P<body>.*?)</table>", re.S)
+_CURRENT = re.compile(r'<a\b[^>]*aria-current="page"[^>]*>(?P<label>.*?)</a>', re.S)
+_H1 = re.compile(r"<h1\b[^>]*>(?P<title>.*?)</h1>", re.S)
+_HEADING = re.compile(r"<h(?P<level>[2-6])\b[^>]*>(?P<inner>.*?)</h(?P=level)>", re.S)
+_CRUMBS = re.compile(r'<nav\b[^>]*aria-label="Breadcrumb"[^>]*>(?P<inner>.*?)</nav>', re.S)
+_HREF = re.compile(r'href="(?P<href>[^"]*)"')
+_TIP = re.compile(r'<aside\b[^>]*starlight-aside--tip[^>]*>.*?</aside>', re.S)
+
+
+def _text(markup: str) -> str:
+    return " ".join(unescape(_TAGS.sub(" ", markup)).split())
+
+
+def _pages() -> list[Path]:
+    return sorted(DIST.rglob("*.html"))
+
+
+def _url(path: Path) -> str:
+    rel = path.relative_to(DIST).as_posix()
+    if rel.endswith("index.html"):
+        return BASE + rel[: -len("index.html")]
+    return BASE + rel
+
+
+def _levels() -> dict[str, str]:
+    """Page address -> the level its front matter declares."""
+    levels: dict[str, str] = {}
+    for source in DOCS.rglob("*.md*"):
+        text = source.read_text(encoding="utf-8")
+        if not text.startswith("---\n"):
+            continue
+        front = yaml.safe_load(text[4:text.find("\n---\n", 4)]) or {}
+        stem = str(front.get("slug") or source.relative_to(DOCS).with_suffix("").as_posix())
+        stem = re.sub(r"(^|/)index$", "", stem)
+        levels[BASE + (stem + "/" if stem else "")] = str(front.get("level") or "")
+    return levels
+
+
+@unittest.skipUnless((DIST / "index.html").is_file(), "site not built")
+class EveryBuiltPage(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.pages = {path: path.read_text(encoding="utf-8") for path in _pages()}
+        sidebar = json.loads(SIDEBAR.read_text(encoding="utf-8"))
+        cls.tab_roots = {BASE + topic["link"].strip("/") + ("/" if topic["link"].strip("/") else "")
+                         for topic in sidebar["topics"]}
+        cls.group_pages = {group["href"] for group in sidebar["groups"].values() if group["href"]}
+        cls.levels = _levels()
+
+    def test_under_the_page_budgets(self):
+        heavy, crowded = [], []
+        for path, html in self.pages.items():
+            size = len(html.encode("utf-8"))
+            if size >= PAGE_BYTES:
+                heavy.append((path.relative_to(DIST).as_posix(), size))
+            elements = len(_ELEMENT.findall(_SCRIPT.sub("", html)))
+            if elements >= PAGE_ELEMENTS:
+                crowded.append((path.relative_to(DIST).as_posix(), elements))
+        self.assertFalse(heavy, f"pages of {PAGE_BYTES:,} bytes or more")
+        self.assertFalse(crowded, f"pages of {PAGE_ELEMENTS:,} elements or more")
+
+    def test_no_sidebar_label_is_a_number(self):
+        for path, html in self.pages.items():
+            sidebar = html.split('id="starlight__sidebar"', 1)[-1].split("</nav>", 1)[0]
+            labels = [_text(m) for m in re.findall(r"<a\b[^>]*>(.*?)</a>", sidebar, re.S)]
+            numbers = [label for label in labels if re.fullmatch(r"\d+", label)]
+            with self.subTest(page=path.relative_to(DIST).as_posix()):
+                self.assertFalse(numbers)
+
+    def test_no_tip_titled_next(self):
+        for path, html in self.pages.items():
+            for aside in _TIP.findall(html):
+                title = re.search(r'class="starlight-aside__title"[^>]*>(.*?)</p>', aside, re.S)
+                with self.subTest(page=path.relative_to(DIST).as_posix()):
+                    self.assertNotEqual(_text(title[1]) if title else "", "Next")
+
+    def test_the_sidebar_names_a_page_by_its_title(self):
+        for path, html in self.pages.items():
+            url = _url(path)
+            current = _CURRENT.search(html.split('id="starlight__sidebar"', 1)[-1])
+            title = _H1.search(html)
+            # A tab, or a group of the sidebar, opens on its overview.
+            if url in self.tab_roots or url in self.group_pages or not current or not title:
+                continue
+            with self.subTest(page=url):
+                self.assertEqual(_text(current["label"]), _text(title["title"]))
+
+    def test_a_card_link_icon_says_nothing_in_its_heading(self):
+        for path, html in self.pages.items():
+            for heading in _HEADING.finditer(html):
+                for link in re.findall(r'<a\b[^>]*class="rb-permalink"[^>]*>(.*?)</a>',
+                                       heading["inner"], re.S):
+                    with self.subTest(page=path.relative_to(DIST).as_posix()):
+                        self.assertEqual(_text(link), "")
+
+    def test_every_crumb_leads_to_a_tab_or_a_section(self):
+        for path, html in self.pages.items():
+            crumbs = _CRUMBS.search(html)
+            if not crumbs:
+                continue
+            for href in _HREF.findall(crumbs["inner"]):
+                with self.subTest(page=_url(path), href=href):
+                    self.assertTrue(href in self.tab_roots or href in self.group_pages
+                                    or "#" in href, href)
+
+    def test_phone_tables_on_reader_pages_are_stacked(self):
+        """A table of three columns or more, under RECORD_ROWS rows, on a
+        level 0 or 1 page is read one row at a time on a phone: it carries
+        data-rb-stack. A two-column table is a list of label and value."""
+        unstacked = []
+        for path, html in self.pages.items():
+            if self.levels.get(_url(path)) not in READER_LEVELS:
+                continue
+            for table in _TABLE.finditer(html):
+                head = table["body"].split("</thead>", 1)[0]
+                columns = head.count("<th")
+                rows = table["body"].split("</thead>", 1)[-1].count("<tr")
+                # A table the page fills when it runs (the file picker) is
+                # not the build's.
+                if not rows:
+                    continue
+                if columns >= 2 and rows < RECORD_ROWS and "data-rb-stack" not in table["attrs"]:
+                    unstacked.append((_url(path), _text(head)[:60]))
+        self.assertFalse(unstacked)
+
+    def test_no_page_dates_itself_with_a_raw_stamp(self):
+        for path, html in self.pages.items():
+            with self.subTest(page=_url(path)):
+                self.assertNotIn("Generated on", html)
+
+
+@unittest.skipUnless((DIST / "index.html").is_file(), "site not built")
+class TheIntegrationItems(unittest.TestCase):
+
+    def test_a_platform_page_draws_its_logo_beside_the_title(self):
+        from common import load_platform_registry
+
+        registry = load_platform_registry(str(REPO_ROOT / "platforms"))
+        with_logo = [key for key, entry in registry.items() if entry.get("logo")]
+        self.assertTrue(with_logo)
+        for key in with_logo:
+            page = DIST / "platforms" / key / "index.html"
+            if not page.is_file():
+                continue
+            html = page.read_text(encoding="utf-8")
+            with self.subTest(platform=key):
+                block = re.search(r'<div class="rb-title-logo[^"]*">.*?</div>', html, re.S)
+                self.assertIsNotNone(block)
+                self.assertRegex(block[0], r'<img src="/retrobios/assets/logos/')
+                self.assertNotIn("align=", html.split("<main", 1)[-1])
+
+    def test_every_emulator_letter_lands_on_its_section(self):
+        sidebar = json.loads(SIDEBAR.read_text(encoding="utf-8"))
+        html = (DIST / "emulators" / "index.html").read_text(encoding="utf-8")
+        ids = set(re.findall(r'\bid="([^"]+)"', html))
+        letters = {key: group for key, group in sidebar["groups"].items()
+                   if key.startswith("Explore/Emulators/")}
+        self.assertGreater(len(letters), 20)
+        for key, group in letters.items():
+            with self.subTest(group=key):
+                self.assertRegex(group["href"] or "", r"^/retrobios/emulators/#")
+                self.assertIn(group["href"].split("#", 1)[1], ids)
+
+    def test_system_links_reach_the_section_of_the_name(self):
+        """Every link into a system page lands on an id the page holds."""
+        ids: dict[str, set[str]] = {}
+        for page in (DIST / "systems").glob("*/index.html"):
+            ids[page.parent.name] = set(re.findall(r'\bid="([^"]+)"', page.read_text(encoding="utf-8")))
+        target = re.compile(r'href="/retrobios/systems/(?P<page>[^/"#]+)/#(?P<anchor>[^"]+)"')
+        for page in ("emulators/index.html", "platforms/retroarch/index.html",
+                     "platforms/batocera/index.html", "systems/index.html"):
+            html = (DIST / page).read_text(encoding="utf-8")
+            for match in target.finditer(html):
+                with self.subTest(page=page, link=match[0]):
+                    self.assertIn(match["anchor"], ids.get(match["page"], set()))
+
+    def test_getting_started_sends_to_which_pack(self):
+        page = DIST / "wiki" / "getting-started" / "index.html"
+        self.assertTrue(page.is_file())
+        self.assertIn('url=/retrobios/which-pack/"', page.read_text(encoding="utf-8"))
+        self.assertFalse((REPO_ROOT / "wiki" / "getting-started.md").exists())
+
+    def test_existence_mode_pages_name_the_file_served(self):
+        for key in ("retroarch", "lakka", "retropie"):
+            page = DIST / "platforms" / key / "index.html"
+            if not page.is_file():
+                continue
+            with self.subTest(platform=key):
+                self.assertIn("Served by", page.read_text(encoding="utf-8"))
+
+
+class TheRules(unittest.TestCase):
+    """What the gate reads without a build."""
+
+    def test_the_logo_reaches_the_front_matter(self):
+        import sitestarlight
+
+        _target, text = sitestarlight.finalize_page(
+            "platforms/x.md", "# X\n\nBody.\n", level="setup", weight=None,
+            browser_title="X", description="d", structured={}, terms={},
+            logo="assets/logos/x.svg")
+        self.assertIn("logo: assets/logos/x.svg", text)
+
+    def test_a_large_data_directory_is_listed_on_pages_of_its_own(self):
+        """Dolphin's Sys tree put 16,900 elements on the Dolphin page. Its
+        files are found where they moved."""
+        import generate_site
+
+        members = {f"Sys/{i:05d}.ini": {"sha1": f"{i:040x}", "size": 1}
+                   for i in range(generate_site.DATA_TREE_ROWS_PER_PAGE + 1)}
+        pages = generate_site.data_tree_pages("dolphin-sys", members)
+        self.assertEqual([stem for stem, _names in pages], ["data-dolphin-sys", "data-dolphin-sys-2"])
+        self.assertEqual(generate_site.data_tree_pages("small", dict(list(members.items())[:3])), [])
+        items = generate_site.file_location_items(
+            {"files": {}}, {"dolphin": {"data_directories": [{"ref": "dolphin-sys"}]}},
+            {"dolphin-sys": members})
+        last = f"{generate_site.DATA_TREE_ROWS_PER_PAGE:040x}"
+        self.assertIn({"sha1": last, "url": f"emulators/data-dolphin-sys-2/#sha1-{last[:12]}"}, items)
+
+    def test_no_float_rule_for_right_aligned_images(self):
+        css = (REPO_ROOT / "website" / "src" / "styles" / "retrobios.css").read_text(encoding="utf-8")
+        self.assertNotIn("img[align='right']", css)
+        self.assertIn(":is(h2, h3) small", css)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_site_r2_nav.py

@@ -65,9 +65,20 @@ def _present(nav: list) -> set[str]:
 # follow the labels in platforms/_guides.yml instead of a copy of them.
 from siteguides import DEVICE_GROUPS  # noqa: E402
 
+def _index_anchors(markdown: str) -> set[str]:
+    """The heading ids a generated page carries once written."""
+    _title, title_id, body = sitestarlight.split_title(markdown)
+    return sitestarlight.heading_ids(sitestarlight.pin_heading_ids(body, title_id))
+
+
 ANCHORS = {
     "platforms/index.md": {sitestarlight.toc_slug(label) for label, _ in DEVICE_GROUPS.values()},
     "systems/index.md": {"consoles-and-computers"},
+    "emulators/index.md": _index_anchors(generate_site.generate_emulators_index({
+        "duckstation": {"emulator": "DuckStation", "systems": ["sony-playstation"]},
+        "dolphin": {"emulator": "Dolphin", "systems": ["nintendo-gamecube"]},
+        "mame": {"emulator": "MAME", "systems": ["arcade"]},
+    })),
 }
 
 
@@ -109,7 +120,7 @@ class GroupsAreGroups(unittest.TestCase):
         self.assertEqual(explore["Explore/Systems"], {"href": "/retrobios/systems/", "prune": True})
         self.assertEqual(explore["Explore/Systems/Consoles and computers"]["href"],
                          "/retrobios/systems/#consoles-and-computers")
-        self.assertEqual(explore["Explore/Emulators/D"], {"href": "/retrobios/emulators/", "prune": True})
+        self.assertEqual(explore["Explore/Emulators/D"], {"href": "/retrobios/emulators/#d", "prune": True})
 
     def test_no_group_points_at_one_of_its_own_pages(self):
         """The overview a group opens on is its own index page or a section of
@@ -138,6 +149,17 @@ class EmulatorsOneLevelDeep(unittest.TestCase):
         self.assertEqual(nav, ["emulators/index.md", {"D": [
             {"Dolphin": "emulators/dolphin.md"}, {"DuckStation": "emulators/duckstation.md"}]}])
 
+    def test_every_initial_has_its_section_on_the_index(self):
+        """A letter in the sidebar lands on the section of that letter."""
+        from common import load_emulator_profiles
+
+        profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
+        anchors = _index_anchors(generate_site.generate_emulators_index(profiles))
+        for group in generate_site._emulator_nav(profiles)[1:]:
+            (label, _pages), = group.items()
+            with self.subTest(initial=label):
+                self.assertIn(sitestarlight.toc_slug(label), anchors)
+
     def test_every_initial_holds_pages(self):
         from common import load_emulator_profiles
 

tests/test_site_r2_systems.py

@@ -219,13 +219,13 @@ class ConsoleHeadingsStartWithTheName(unittest.TestCase):
         icon = siterender._icon_name("Sony", "PlayStation")
         with mock.patch.dict(siterender._icon_available, {icon: True}):
             heading = gs._console_heading("Sony", "PlayStation")
-        self.assertRegex(heading, r'^## <img [^>]*aria-hidden="true"[^>]*>PlayStation$')
+        self.assertRegex(heading, r'^## <img [^>]*aria-hidden="true"[^>]*>PlayStation \{ #playstation \}$')
         self.assertIn('alt=""', heading)
 
     def test_no_icon_no_image(self):
         icon = siterender._icon_name("Nobody", "Nothing")
         with mock.patch.dict(siterender._icon_available, {icon: False}):
-            self.assertEqual(gs._console_heading("Nobody", "Nothing"), "## Nothing")
+            self.assertEqual(gs._console_heading("Nobody", "Nothing"), "## Nothing { #nothing }")
 
 
 class TheSystemsIndexListsConsoles(unittest.TestCase):
@@ -280,11 +280,15 @@ class TheBuiltSonyPage(unittest.TestCase):
             self.assertNotIn("permalink", re.sub(r"<[^>]+>", "", title))
 
     def test_playstation_opens_on_a_bios_it_needs(self):
-        section = self.text.split('id="playstation"', 1)[1]
+        section = self.text.split('id="playstation"', 1)[1].split("<h2", 1)[0]
+        self.assertIn("Start with", section.split('class="rb-sys-file"', 1)[0])
+        # A console with a page of its own opens there.
+        own = DIST / "systems" / "sony-playstation" / "index.html"
+        if 'class="rb-file-title"' not in section and own.is_file():
+            section = own.read_text(encoding="utf-8")
         first = re.search(r'class="rb-file-title"[^>]*><code>(.*?)</code>', section)[1]
         self.assertNotIn(first, {"LibPS.exe", "PSX_DTLH2000.bin"})
         self.assertRegex(first, r"^scph\d+")
-        self.assertIn("Start with", section.split('class="rb-sys-file"', 1)[0])
 
     def test_headings_have_no_leading_space(self):
         self.assertNotRegex(self.text, r'<h2 id="playstation[^"]*"><img[^>]*>\s')

tests/test_site_system_links.py

@@ -1,9 +1,12 @@
-"""A platform system links to the page of the folder its files live in.
+"""A system is placed by the folder its files live in, and a link to it
+goes to the section named after it.
 
 The site mapped each system by the bare name of its first file, the last
 console written winning: MiSTer's boot.rom sits under a dozen consoles, so
 wonderswan and tiger-game-com linked to VideoBrain, ngpc and tandy-coco to
-TI-99. Reads the repository: skips when the database is absent.
+TI-99. A link then followed that folder, not the name it printed:
+emerson-arcadia-2001 led to the Saturn and sega-naomi to the Dreamcast.
+Reads the repository: skips when the database is absent.
 """
 
 from __future__ import annotations
@@ -76,14 +79,14 @@ class OtherSpellingsOfOneSystem(unittest.TestCase):
         self.addCleanup(siterender.set_system_page_map, saved)
         siterender.set_system_page_map(mapping)
 
-    def test_a_profile_spelling_links_to_the_platform_page(self):
+    def test_a_profile_spelling_finds_the_platform_page(self):
         import siterender
 
         self._install({"sega-mega-drive": ("sega", "Mega Drive")})
-        self.assertIn("systems/sega.md#mega-drive", siterender._system_link("sega-megadrive"))
+        self.assertEqual(siterender._page_by_norm("sega-megadrive"), ("sega", "Mega Drive"))
         self._install({"sega-mega-drive": ("sega", "Mega Drive"),
                        "megadrive": ("other", "Elsewhere")})
-        self.assertEqual(siterender._system_link("sega-megadrive"), "sega-megadrive")
+        self.assertIsNone(siterender._page_by_norm("sega-megadrive"))
 
     def test_a_table_of_the_same_size_replaces_the_other_spellings(self):
         """The index of other spellings was kept while the table kept its
@@ -91,10 +94,9 @@ class OtherSpellingsOfOneSystem(unittest.TestCase):
         import siterender
 
         self._install({"sega-mega-drive": ("sega", "Mega Drive")})
-        self.assertIn("systems/sega.md#", siterender._system_link("sega-megadrive"))
+        self.assertEqual(siterender._page_by_norm("sega-megadrive"), ("sega", "Mega Drive"))
         self._install({"sega-mega-drive": ("other", "Elsewhere")})
-        self.assertEqual(siterender._system_link("sega-megadrive"),
-                         "[sega-megadrive](systems/other.md#elsewhere)")
+        self.assertEqual(siterender._page_by_norm("sega-megadrive"), ("other", "Elsewhere"))
 
     def test_a_system_only_profiles_name_is_placed_by_their_files(self):
         manufacturers = {"Commodore": {"PET": [{"name": "basic4.bin"}]}}
@@ -261,5 +263,72 @@ class SystemEmulatorsWhenNoIdMatches(unittest.TestCase):
         )
 
 
+DOCS = REPO_ROOT / "website" / "src" / "content" / "docs"
+
+
+@unittest.skipUnless((REPO_ROOT / "database.json").is_file()
+                     and (DOCS / "systems" / "sony.md").is_file(), "site not generated")
+class EverySystemLinkLandsOnItsSection(unittest.TestCase):
+    """Every system id of every platform and profile, linked as the pages
+    link it, against the generated pages: the anchor exists on its target
+    page, and the heading there is the one system_name() reads."""
+
+    @classmethod
+    def setUpClass(cls):
+        import re
+
+        import siterender
+        import sitestarlight
+        from common import (
+            list_registered_platforms, load_database, load_emulator_profiles, load_platform_config,
+        )
+
+        db = load_database(str(REPO_ROOT / "database.json"))
+        profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"), skip_aliases=False)
+        siterender.set_console_sections(gs._group_by_manufacturer(db), siterender.program_names(profiles))
+        platforms = REPO_ROOT / "platforms"
+        ids = {str(sid) for profile in profiles.values() for sid in profile.get("systems") or []}
+        for name in list_registered_platforms(str(platforms), include_archived=True):
+            ids |= {str(sid) for sid in load_platform_config(name, str(platforms)).get("systems") or {}}
+        cls.ids = sorted(ids)
+        cls.siterender = siterender
+        heading = re.compile(r"^##\s+(?:<img [^>]*>)?(?P<text>.+?)\s*\{\s*#(?P<id>[^\s}]+)[^}]*\}\s*$", re.M)
+        cls.headings: dict[str, dict[str, str]] = {}
+        for page in (DOCS / "systems").glob("*.md*"):
+            text = page.read_text(encoding="utf-8")
+            cls.headings[page.name.rsplit(".", 1)[0]] = {
+                m["id"]: m["text"] for m in heading.finditer(text)}
+            cls.headings[page.name.rsplit(".", 1)[0]].update(
+                dict.fromkeys(sitestarlight.heading_ids(text) - cls.headings[page.name.rsplit(".", 1)[0]].keys(), ""))
+
+    def test_every_anchor_exists_and_names_the_system(self):
+        import re
+        import urllib.parse
+
+        linked = 0
+        for sys_id in self.ids:
+            link = self.siterender._system_link(sys_id)
+            match = re.fullmatch(r"\[[^\]]*\]\(systems/(?P<page>[^)#]+)\.md(?:#(?P<anchor>[^)]+))?\)", link)
+            if not match:
+                self.assertEqual(link, sys_id)
+                continue
+            linked += 1
+            page = urllib.parse.unquote(match["page"])
+            with self.subTest(system=sys_id, link=link):
+                self.assertIn(page, self.headings)
+                if match["anchor"]:
+                    self.assertIn(match["anchor"], self.headings[page])
+                    _slug, console = self.siterender.system_section(sys_id)
+                    self.assertEqual(self.headings[page][match["anchor"]], console)
+        self.assertGreater(linked, 200)
+
+    def test_a_system_without_a_section_of_its_name_is_not_sent_elsewhere(self):
+        for sys_id in ("emerson-arcadia-2001", "sega-naomi"):
+            with self.subTest(system=sys_id):
+                link = self.siterender._system_link(sys_id)
+                self.assertNotIn("#saturn", link)
+                self.assertNotIn("#dreamcast", link)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_verify_listing.py

@@ -152,6 +152,16 @@ class ExistenceReportsWhatThePackReports(unittest.TestCase):
         self.assertEqual(result["status"], verify.Status.OK)
         self.assertIn("declared hash 9d2ff233", result["discrepancy"])
 
+    def test_the_file_found_is_named(self):
+        """RetroArch, Lakka and RetroPie pages showed no "Served by" line:
+        existence mode returned no path, where the hash modes return one."""
+        sys.path.insert(0, str(REPO_ROOT / "scripts"))
+        import verify  # noqa: PLC0415
+
+        found = verify.verify_entry_existence({"name": "a.bin"}, "bios/X/Y/a.bin")
+        self.assertEqual(found["path"], "bios/X/Y/a.bin")
+        self.assertNotIn("path", verify.verify_entry_existence({"name": "a.bin"}, None))
+
 
 class TheDestinationsFlagNeedsAPack(unittest.TestCase):
     """--destinations describes a platform's full pack: another mode refuses it."""

website/src/routeData.ts

@@ -92,9 +92,8 @@ export const onRequest = defineRouteMiddleware(async (context, next) => {
 			trail(route.sidebar, [topic.label]) ??
 			(segments.length > 1 ? folderTrail(route.sidebar, [topic.label], `${BASE}${segments[0]}/`) : []);
 		for (const crumb of groups) {
-			// A group introduced by the page itself is the page; one with no
-			// section of its own (the initial D of the emulators) adds nothing
-			// to the crumb before it.
+			// A group introduced by the page itself is the page, and one with
+			// no section of its own adds nothing to the crumb before it.
 			if (crumb.href === path) continue;
 			if (crumb.href && crumb.href === crumbs.at(-1)?.href) continue;
 			crumbs.push(crumb);

website/src/styles/retrobios.css

@@ -217,18 +217,6 @@
 .rb-stat-value { display: block; font-size: 1.25rem; font-weight: 700; font-variant-numeric: tabular-nums; }
 .rb-stat-label { display: block; font-size: 0.75rem; color: var(--rb-text-secondary); }
 
-/* Logos float right beside the opening paragraph */
-.sl-markdown-content img[align='right'] {
-	float: right;
-	width: 56px;
-	height: 56px;
-	object-fit: contain;
-	margin: 0 0 0.6rem 1rem;
-	padding: 4px;
-	border: 1px solid var(--rb-border);
-	border-radius: 4px;
-	background: #fff;
-}
 .sl-markdown-content img[src*='assets/logos/'] { padding: 2px; border-radius: 2px; background: #fff; }
 /* A system icon beside a console heading holds its box before it loads:
    it pushed every card below it down once it arrived. */
@@ -357,12 +345,40 @@
 .rb-meta-card td a { word-break: normal; }
 .rb-meta-card td:first-child { font-weight: 600; color: var(--rb-text-secondary); width: 11rem; }
 
-/* Cross-reference by upstream: the address column wraps instead of pushing
-   the cores and platforms off screen. */
-.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) + section td:has(a[href^='http']) {
+/* Cross-reference by upstream, the last table of its page: the address
+   column wraps instead of pushing the cores and platforms off screen. */
+.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) ~ section td:has(a[href^='http']) {
 	max-width: 22rem;
 	overflow-wrap: anywhere;
 }
+/* Beside the column, the five columns share its width: at 1440 px the
+   Platforms column sat 100 px past the edge. */
+@media (min-width: 50rem) {
+	.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) ~ section table {
+		display: table;
+		table-layout: fixed;
+		width: 100%;
+	}
+	.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) ~ section td {
+		overflow-wrap: anywhere;
+	}
+	.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) ~ section th:nth-child(1) { width: 25%; }
+	.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) ~ section th:nth-child(2) { width: 26%; }
+	.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) ~ section th:nth-child(3) { width: 15%; }
+	.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) ~ section th:nth-child(4) { width: 20%; }
+	.sl-markdown-content .sl-heading-wrapper:has(> #by-upstream-emulator) ~ section th:nth-child(5) { width: 14%; }
+}
+
+/* The id a platform writes for a system, beside the name in its heading:
+   smaller and quieter than the name, set as code. */
+.sl-markdown-content :is(h2, h3) small {
+	margin-left: 0.4em;
+	font-family: var(--sl-font-mono);
+	font-size: 0.65em;
+	font-weight: 400;
+	color: var(--rb-text-secondary);
+	overflow-wrap: anywhere;
+}
 
 .rb-timestamp {
 	font-size: 0.8rem;

wiki/testing-guide.md

@@ -54,8 +54,7 @@ Every module in `tests/`, by the area it guards.
 | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_offline`, `test_install_selection` |
 | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
-| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
-| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
 
 ## Test architecture