chore: merge the long list tables

.github/workflows/deploy-site.yml

@@ -45,6 +45,7 @@ on:
       - "scripts/safeparse.py"
       - "scripts/sect233r1.py"
       - "scripts/sitediagrams.py"
+      - "scripts/siteentries.py"
       - "scripts/siteguides.py"
       - "scripts/sitemarkup.py"
       - "scripts/sitemeasurements.py"

scripts/generate_site.py

@@ -83,6 +83,7 @@ from generate_readme import (
 )
 from provenance_report import build_report, distinct_missing
 import sitemarkup as markup
+import siteentries
 import sitestarlight
 from siteexports import generate_exports_page, load_export_reports
 from siteguides import (
@@ -5103,12 +5104,14 @@ def generate_provenance_page(
     # the public for 79 of them.
     derivable = derivable or {}
     derived_entries: dict[str, dict] = {}
-    for data in report.values():
+    derived_by_source: dict[str, int] = {}
+    for source, data in report.items():
         kept = []
         for entry in data["missing"]:
             sha1 = (entry.get("sha1") or "").lower()
             if sha1 in derivable:
                 derived_entries.setdefault(sha1, entry)
+                derived_by_source[source] = derived_by_source.get(source, 0) + 1
             else:
                 kept.append(entry)
         data["missing"] = kept
@@ -5136,7 +5139,19 @@ def generate_provenance_page(
             for dat in entry.get("dats") or [entry.get("dat", "")]:
                 by_dat.setdefault(dat, []).append(entry)
         lines.extend([f"### {label}", ""])
-        blocks: list[list[str]] = []
+        if moved is not None:
+            where = _list_pages(
+                f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}",
+                f"Dumps {label} catalogues and the collection lacks. Part of "
+                f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).",
+                [], [_wanted_list(source, label, data, by_dat, derived_by_source.get(source, 0))],
+                moved, parent=("Dump provenance", "/provenance/"),
+            )
+            lines.extend([
+                f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: "
+                f"listed on {where}.", "",
+            ])
+            continue
         for dat in sorted(by_dat):
             entries = by_dat[dat]
             table = ["| Name | Description | SHA1 |", "|------|-------------|------|"]
@@ -5152,20 +5167,7 @@ def generate_provenance_page(
                         f"`{hit['archive']}` as `{hit['member']}`."
                     )
                 table.append(f"| `{name}` | {desc} | `{sha1}` |")
-            blocks.append(markup.details("note", f"{dat} ({len(entries)})", table))
-        if moved is None:
-            lines.extend(line for block in blocks for line in block)
-            continue
-        where = _list_pages(
-            f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}",
-            f"Dumps {label} catalogues and the collection lacks, by DAT. Part of "
-            f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).",
-            [], blocks, moved, parent=("Dump provenance", "/provenance/"),
-        )
-        lines.extend([
-            f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: "
-            f"listed on {where}.", "",
-        ])
+            lines.extend(markup.details("note", f"{dat} ({len(entries)})", table))
 
     if derived_entries:
         lines.extend(
@@ -5191,6 +5193,69 @@ def generate_provenance_page(
     return "\n".join(lines) + "\n"
 
 
+def _wanted_list(source: str, label: str, data: dict, by_dat: dict[str, list],
+                 derived: int) -> list[str]:
+    """A catalog's wanted page: the counts, then every missing dump in one
+    table folded under a filter box.
+
+    The page was one closed block per DAT, 46 for TOSEC: a name was found
+    by opening each in turn. The box filters the whole list by name, DAT,
+    description or SHA1, and says how many rows it keeps.
+    """
+    missing = data["missing"]
+    held = data["matched"]
+    reached = held + len(missing) + derived
+    dats = _count(len(data["covered_dats"]), "DAT")
+    lines = [
+        f"Held: **{held:,}** of {reached:,}, in the {dats} the collection reaches. "
+        f"Wanted: **{len(missing):,}**, from {_count(len(by_dat), 'DAT')}.",
+        "",
+    ]
+    if derived:
+        verb = "is" if derived == 1 else "are"
+        lines.extend([
+            f"{derived:,} more {verb} [derivable]"
+            "(provenance.md#derivable-dumps-the-collection-already-produces).", ""])
+    if data.get("out_of_scope"):
+        lines.extend([f"{data['out_of_scope']:,} more sit in DATs the collection does not reach.", ""])
+    counts = ["| DAT | Wanted |", "|-----|--------|",
+              *(f"| {dat.replace('|', '-')} | {len(entries):,} |"
+                for dat, entries in sorted(by_dat.items()))]
+    lines.extend(markup.details("note", f"Wanted by DAT ({len(by_dat):,})", counts))
+    rows = []
+    for entry in sorted(missing, key=lambda e: (e["name"], e.get("sha1") or "")):
+        desc = markup.inline_html(entry.get("description") or "")
+        for hit in entry.get("romsets", []):
+            desc += markup.inline_html(
+                f" Ships in {_ROMSET_SOURCES.get(hit['source'], hit['source'])} "
+                f"`{hit['archive']}` as `{hit['member']}`.")
+        in_dats = ", ".join(html.escape(d) for d in (entry.get("dats") or [entry.get("dat", "")]))
+        sha1 = html.escape(entry.get("sha1") or "-")
+        # The search reads the name alone. The DAT names are searched once, in
+        # the count table above, and the site search holds no hashes. With the
+        # descriptions indexed too, the TOSEC list matched "2" 556 times
+        # (years, versions) and put the PlayStation page ahead of the
+        # PlayStation 2 one for "playstation 2 bios". The name and the hash
+        # sit in their cells as text: a catalog name is words, and in code
+        # each word wore its own box on a narrow column.
+        skip = markup.search_exclude()
+        rows.append("<tr>"
+                    f"<td>{html.escape(entry['name'], quote=False)}</td>"
+                    f"<td {skip}>{in_dats}</td><td {skip}>{desc or '-'}</td>"
+                    f'<td class="rb-hash" {skip}>{sha1}</td></tr>')
+    heads = ["Name", "DAT", "Description", "SHA1"]
+    lines.extend([
+        siteentries.filter_box(f"rb-wanted-filter-{_slugify_anchor(source)}",
+                               "dumps", heads, len(rows)),
+        '<details class="rb-details rb-details-note">',
+        f"<summary>All {_count(len(rows), 'wanted dump')}</summary>",
+        '<table class="rb-wanted" data-rb-stack="true">',
+        "<thead><tr>" + "".join(f'<th scope="col">{h}</th>' for h in heads) + "</tr></thead>",
+        "<tbody>", *rows, "</tbody>", "</table>", "</details>", "",
+    ])
+    return lines
+
+
 _RELEASE_ASSET_NAMES: dict[str, str] | None = None
 
 
@@ -7464,12 +7529,15 @@ def _render_emulator_file(
     owner: str = "",
     alias: str = "",
     data_href: str = "#data-folder",
+    badges_apart: bool = False,
 ) -> list[str]:
     """Render one row of an emulator profile's file table.
 
     `anchor` is the card's id on the page, so one file can be cited by URL,
     and `alias` a second id it answers to. `data_href` links the data
     folder an app reads the entry under, from the page the card lands on.
+    `badges_apart`: the title holds the name alone, for a table whose
+    Status column shows the badges (siteentries.py).
 
     Split out of generate_emulator_page, where it was a 228-line loop body
     carrying most of that function's branching: thirty-odd optional fields,
@@ -7518,7 +7586,8 @@ def _render_emulator_file(
         system = system_name((profile.get("systems") or [""])[0])
         label = (f"a {system} BIOS (example: {_code(fname)})" if _named_example(str(fname))
                  else f"a {system} BIOS, any file name")
-    lines.append(_card_title(f"{marker}{label} {badge_str}", anchor, weight=5))
+    title = f"{marker}{label}" if badges_apart else f"{marker}{label} {badge_str}"
+    lines.append(_card_title(title, anchor, weight=5))
     lines.append("")
     if desc:
         lines.extend([data_root_words(_escape_tags(str(desc)), data_href), ""])
@@ -8633,6 +8702,10 @@ def _data_tree_page(ref: str, entry: dict, pages: list[tuple[str, list[str]]],
 # anchor. A count, not a size: system cards link to these cards and are
 # written before them. At 400 a part of MAME's cards was 805 KB of HTML.
 EMULATOR_CARDS_PER_PAGE = 220
+# The most cards a part of a split page holds. Each card is a table row
+# since siteentries.py, about five elements and 60 bytes more: at 220 a
+# part, MAME's first was 612 KB and 9,777 elements.
+EMULATOR_CARDS_PER_PART = 180
 
 
 # The headings an emulator's cards go under, in page order: what it needs
@@ -8667,12 +8740,59 @@ def emulator_parts(name: str, files: list) -> list[tuple[str, list[int]]]:
     if len(files) <= EMULATOR_CARDS_PER_PAGE:
         return []
     order = _display_order(files)
-    count = -(-len(files) // EMULATOR_CARDS_PER_PAGE)
+    count = -(-len(files) // EMULATOR_CARDS_PER_PART)
     size = -(-len(files) // count)
     return [(f"{name}-files-{index + 1}", order[start:start + size])
             for index, start in enumerate(range(0, len(files), size))]
 
 
+# From this many entries an unsplit emulator page lists them as one table,
+# each row opening to its card (siteentries.py), in place of a summary
+# table above the cards: GroovyMAME's 171 cards read 14,400 words open.
+# A split emulator's card pages always do.
+EMULATOR_TABLE_MIN = 150
+ENTRY_HEADS = ["File", "Contents", "Status", "Hash"]
+# The end of the badge _file_badges writes for an entry of each section.
+_SECTION_BADGE = {"required": ">required</span>", "optional": ">optional</span>",
+                  "with_app": ">comes with the app</span>"}
+
+
+def _entry_rows(files: list[dict], cards: list[list[str]], profile: dict,
+                available: Callable[[dict], bool]) -> list[siteentries.Row]:
+    """Each entry as a table row: its card's title and body, the number of
+    members an archive holds, its badges and its hash."""
+    many = len(profile.get("systems") or []) > 1
+    rows = []
+    for f, card in zip(files, cards):
+        title, body = siteentries.card_title_and_body(card)
+        for_system = system_name(str(f["system"])) if f.get("system") and many else ""
+        contents = f.get("contents") or []
+        badges = _file_badges(f, available(f), for_system)
+        # The first badge says whether the entry is needed: under Required,
+        # Optional or Comes with the app it repeats the section heading.
+        # One element and two words fewer on each of MAME's 1,563 rows.
+        if badges and badges[0].endswith(_SECTION_BADGE.get(_card_section(f), "\0")):
+            badges = badges[1:]
+        rows.append(siteentries.Row(title, body, [
+            f"{len(contents):,}" if contents else "",
+            " ".join(badges),
+            _hash_short(f),
+        ]))
+    return rows
+
+
+def _entries_table(files: list[dict], group: list[int], rows: list[siteentries.Row]) -> list[str]:
+    """The rows of ``group``, one section's entries, as one table."""
+    key = _card_section(files[group[0]])
+    heading = dict(CARD_SECTIONS)[key]
+    return siteentries.entry_table([rows[i] for i in group], ENTRY_HEADS,
+                                   f"rb-entry-filter-{key.replace('_', '-')}", heading,
+                                   # The badges hide themselves from the search
+                                   # but for the archive's name ("in pgm.zip"),
+                                   # how a ZIP name finds the cores reading it.
+                                   searched=frozenset({"Status"}))
+
+
 def _sectioned(files: list, indexes: list[int], render: Callable[[list[int]], list[str]],
                fold: bool = False) -> list[str]:
     """Each section's heading, then ``render`` of its entries among ``indexes``.
@@ -9102,12 +9222,18 @@ def emulator_pages(
         aliases = emulator_file_aliases(files, anchors)
         parts = emulator_parts(name, files)
         data_href = f"{name}.md#data-folder" if parts else "#data-folder"
+        table_form = markup.starlight() and (bool(parts) or len(files) >= EMULATOR_TABLE_MIN)
         cards = [
             _render_emulator_file(f, profile, consumers, files, _file_available, anchor, name,
-                                  alias, data_href)
+                                  alias, data_href, badges_apart=table_form)
             for f, anchor, alias in zip(files, anchors, aliases)
         ]
-        if not parts:
+        rows = (_entry_rows(files, cards, profile, _file_available) if table_form else [])
+        if not parts and table_form:
+            lines.extend(held)
+            lines.extend(_sectioned(files, _display_order(files),
+                                    lambda group: _entries_table(files, group, rows)))
+        elif not parts:
             order = _display_order(files)
             indexed = len(files) >= EMULATOR_SUMMARY_MIN
             if indexed:
@@ -9129,7 +9255,8 @@ def emulator_pages(
             for number, (stem, entries) in enumerate(parts, 1):
                 part_pages[stem] = _emulator_part_page(
                     name, emu_name, number, parts, [cards[i] for i in entries],
-                    [files[i] for i in entries])
+                    [files[i] for i in entries],
+                    [rows[i] for i in entries] if table_form else None)
 
     lines.extend(sitemeasurements.tested_section(name, _MEASUREMENT_ITEMS.get(name, [])))
 
@@ -9259,9 +9386,16 @@ def _part_front_matter(parent: tuple[str, str], siblings: list[tuple[str, str]],
 
 def _emulator_part_page(
     name: str, emu_name: str, number: int, parts: list[tuple[str, range]],
-    cards: list[list[str]], files: list[dict],
+    cards: list[list[str]], files: list[dict], rows: list[siteentries.Row] | None = None,
 ) -> str:
-    """One page of a split emulator's cards."""
+    """One page of a split emulator's cards: with ``rows``, one table row
+    each, the card under its name."""
+    if rows is not None:
+        def render(group: list[int]) -> list[str]:
+            return _entries_table(files, group, rows)
+    else:
+        def render(group: list[int]) -> list[str]:
+            return [line for i in group for line in cards[i]]
     first = str(files[0].get("path") or files[0].get("name", ""))
     last = str(files[-1].get("path") or files[-1].get("name", ""))
     others = ", ".join(
@@ -9276,8 +9410,7 @@ def _emulator_part_page(
         f"This part holds {_count(len(files), 'entry', 'entries')}, "
         f"{_code(first)} to {_code(last)}. The other parts: {others}.",
         "",
-        *_sectioned(files, list(range(len(files))),
-                    lambda group: [line for i in group for line in cards[i]]),
+        *_sectioned(files, list(range(len(files))), render),
     ]
     lines.extend(_next_block([f"[{emu_name}]({name}.md), the profile and every entry"]))
     lines.extend(["", _date_line()])

scripts/siteentries.py

@@ -0,0 +1,112 @@
+"""Long lists as one filterable table, each row opening to its details.
+
+MAME's card pages read 13,000 to 16,000 words, one card after another,
+before a reader found the archive they came for. Here each entry is one
+row: its name, the facts that tell rows apart, and the full card in a
+closed block under the name. The table is written as HTML so the block can
+hold the card's Markdown. It carries what rehype-stack-tables.mjs gives a
+Markdown table: `data-rb-stack` (one block per row on a phone) and the
+filter box above, which site.ts wires. Without JavaScript the rows show and
+every block opens. A closed block stays in the page, so the browser's find
+and the site search still reach every name it holds.
+"""
+
+from __future__ import annotations
+
+import html
+from dataclasses import dataclass, field
+
+import sitemarkup as markup
+
+# rehype-stack-tables.mjs RECORD_ROWS: from this many rows a table has a
+# filter box, and its stacked labels are written by site.ts, not the build.
+RECORD_ROWS = 20
+# filterPlaceholder in rehype-stack-tables.mjs: the column names offered.
+PLACEHOLDER_COLUMNS = 4
+
+
+@dataclass
+class Row:
+    """One entry: ``title`` is the inline HTML of its name (a heading, so
+    the site search lands on it), ``body`` the Markdown lines shown when the
+    row opens, ``cells`` the HTML of the other columns."""
+
+    title: str
+    body: list[str]
+    cells: list[str] = field(default_factory=list)
+
+
+def filter_placeholder(labels: list[str]) -> str:
+    """"File, contents, status..." as rehype-stack-tables.mjs words it."""
+    words = [label for label in labels if label][:PLACEHOLDER_COLUMNS]
+    if not words:
+        return "Type to filter..."
+    lowered = [word if index == 0 or not (word[:1].isupper() and word[1:2].islower())
+               else word[0].lower() + word[1:] for index, word in enumerate(words)]
+    return f"{', '.join(lowered)}..."
+
+
+def filter_box(ident: str, heading: str, labels: list[str], rows: int, noun: str = "rows") -> str:
+    """The filter box rehype-stack-tables.mjs writes above a long table."""
+    return (
+        f'<div class="rb-table-filter"><label for="{ident}">Filter {html.escape(heading)}</label>'
+        f'<input id="{ident}" type="search" autocomplete="off" '
+        f'placeholder="{html.escape(filter_placeholder(labels))}">'
+        f'<span class="rb-table-filter-count" aria-live="polite">{rows:,} {noun}</span></div>'
+    )
+
+
+def card_title_and_body(card: list[str]) -> tuple[str, list[str]]:
+    """A card's heading line and the lines under it, its wrapping block
+    (the first line and the last closing tag) taken off."""
+    title_at = next(index for index, line in enumerate(card) if line.startswith("<h3"))
+    closing = max(index for index, line in enumerate(card) if line.strip() == "</div>")
+    body = card[title_at + 1:closing]
+    while body and not body[0].strip():
+        body.pop(0)
+    while body and not body[-1].strip():
+        body.pop()
+    return card[title_at], body
+
+
+def entry_table(rows: list[Row], heads: list[str], ident: str, heading: str,
+                stacked: list[str] | None = None,
+                searched: frozenset[str] = frozenset()) -> list[str]:
+    """``rows`` under ``heads`` (the first names the column of titles).
+
+    ``stacked`` gives a column's short label on a phone, as a header's
+    `title` does in a Markdown table. A column every row leaves empty
+    (``""``) is not written. The site search skips every column but the
+    titles and those ``searched`` names: a count or a short hash is no word
+    anyone types.
+    """
+    keep = [0] + [column for column in range(1, len(heads))
+                  if any(row.cells[column - 1] for row in rows)]
+    labels = [heads[c] for c in keep]
+    short = [(stacked or heads)[c] for c in keep]
+    skip = markup.search_exclude()
+    # The block holds the box and the table: an open card spans its width.
+    lines = ["", '<div class="rb-entries-wrap">']
+    if len(rows) >= RECORD_ROWS:
+        lines.append(filter_box(ident, heading, labels, len(rows)))
+    lines.append('<table class="rb-entries" data-rb-stack="true">')
+    head_cells = ""
+    for label, name in zip(labels, short):
+        title = f' title="{html.escape(name)}"' if name != label else ""
+        head_cells += f'<th scope="col"{title}>{html.escape(label)}</th>'
+    lines.extend(["<thead>", f"<tr>{head_cells}</tr>", "</thead>", "<tbody>"])
+    for row in rows:
+        lines.append("<tr>")
+        if row.body:
+            lines.extend(["<td>", '<details class="rb-entry">', f"<summary>{row.title}</summary>",
+                          "", *row.body, "", "</details>", "</td>"])
+        else:
+            lines.append(f"<td>{row.title}</td>")
+        for column in keep[1:]:
+            label = (f' data-label="{html.escape(short[keep.index(column)])}"'
+                     if len(rows) < RECORD_ROWS else "")
+            ignored = "" if heads[column] in searched else f" {skip}"
+            lines.append(f"<td{label}{ignored}>{row.cells[column - 1] or '-'}</td>")
+        lines.append("</tr>")
+    lines.extend(["</tbody>", "</table>", "</div>", ""])
+    return lines

tests/test_site_entry_tables.py

@@ -0,0 +1,311 @@
+"""Long lists read as a table first, details on demand (density round).
+
+MAME's card pages read 13,000 to 16,700 words, card after card, and
+GroovyMAME 14,400 before its last entry. The TOSEC wanted list was 46
+closed blocks, one per DAT, opened one by one to find a name. Each entry is
+now a table row that opens to its card, and each wanted list one filtered
+table under its counts. Nothing the pages said is lost: the tests below read
+every name and hash back.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+import unittest
+from html.parser import HTMLParser
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_site as gs  # noqa: E402
+import siteentries  # noqa: E402
+import validate_site  # noqa: E402
+
+EMPTY_DB = {"files": {}, "indexes": {"by_name": {}}}
+DIST = REPO_ROOT / "website" / "dist"
+
+
+def _archive(i: int, members: int = 3, **fields: object) -> dict:
+    return {
+        "name": f"set{i:04d}.zip", "required": True, "category": "bios_zip",
+        "contents": [{"name": f"m{i}_{k}.rom", "size": 1024, "crc32": f"{i:04x}{k:04x}",
+                      "sha1": f"{i:04x}{k:04x}" * 5} for k in range(members)],
+        **fields,
+    }
+
+
+class TheEntryTable(unittest.TestCase):
+    def _rows(self, count: int) -> list[siteentries.Row]:
+        return [siteentries.Row(f'<h3 id="file-{i}" class="rb-file-title">n{i}</h3>',
+                                [f"- Size: {i} bytes"], [str(i), "", f"MD5 <code>{i}</code>"])
+                for i in range(count)]
+
+    def test_a_long_table_has_its_filter_box_and_stacks(self):
+        lines = siteentries.entry_table(self._rows(25), gs.ENTRY_HEADS, "rb-entry-filter-x", "Required")
+        text = "\n".join(lines)
+        self.assertRegex(text, r'<div class="rb-table-filter"><label for="rb-entry-filter-x">'
+                               r"Filter Required</label><input id=\"rb-entry-filter-x\" type=\"search\"")
+        self.assertIn('aria-live="polite">25 rows</span></div>\n<table class="rb-entries" '
+                      'data-rb-stack="true">', text)
+        # The Status column is empty on every row: not written.
+        self.assertIn('<th scope="col">File</th><th scope="col">Contents</th>'
+                      '<th scope="col">Hash</th>', text)
+        self.assertNotIn("Status", text)
+        self.assertEqual(text.count('<details class="rb-entry">'), 25)
+        # Labels are written by site.ts on a table this long, as on a Markdown one.
+        self.assertNotIn("data-label", text)
+
+    def test_a_card_opens_under_its_name_as_markdown(self):
+        lines = siteentries.entry_table(self._rows(2), gs.ENTRY_HEADS, "x", "Required")
+        at = lines.index('<summary><h3 id="file-0" class="rb-file-title">n0</h3></summary>')
+        # A blank line after the summary: the body is read as Markdown.
+        self.assertEqual(lines[at + 1:at + 4], ["", "- Size: 0 bytes", ""])
+        self.assertNotIn("rb-table-filter", "\n".join(lines))
+        self.assertIn('<td data-label="Contents" data-pagefind-ignore>1</td>', lines)
+
+    def test_a_card_loses_only_its_wrapper(self):
+        card = ['<div class="rb-file-entry">', "", '<h3 id="a">x</h3>', "", "Desc.", "",
+                '<div class="rb-search-skip">', "", "- SHA1: s", "", "</div>", "", "</div>", ""]
+        title, body = siteentries.card_title_and_body(card)
+        self.assertEqual(title, '<h3 id="a">x</h3>')
+        self.assertEqual(body, ["Desc.", "", '<div class="rb-search-skip">', "", "- SHA1: s", "",
+                                "</div>"])
+
+    def test_the_placeholder_reads_like_the_plugin(self):
+        self.assertEqual(siteentries.filter_placeholder(["File", "Contents", "Status", "Hash"]),
+                         "File, contents, status, hash...")
+        self.assertEqual(siteentries.filter_placeholder(["Name", "DAT", "Description", "SHA1"]),
+                         "Name, DAT, description, SHA1...")
+
+
+class ASplitEmulatorListsItsCardsAsRows(unittest.TestCase):
+    def setUp(self):
+        self.files = [_archive(i, members=1 + i % 4) for i in range(gs.EMULATOR_CARDS_PER_PAGE * 2)]
+        self.pages = gs.emulator_pages(
+            "big", {"emulator": "Big", "type": "standalone", "files": self.files}, EMPTY_DB)
+
+    def test_parts_hold_at_most_a_part_of_cards(self):
+        parts = gs.emulator_parts("big", self.files)
+        self.assertEqual(len(parts), -(-len(self.files) // gs.EMULATOR_CARDS_PER_PART))
+        self.assertLessEqual(max(len(entries) for _, entries in parts), gs.EMULATOR_CARDS_PER_PART)
+
+    def test_every_name_and_member_hash_is_on_its_part(self):
+        parts = {stem: page for stem, page in self.pages.items() if stem != "big"}
+        anchors = gs.emulator_file_anchors(self.files)
+        for stem, entries in gs.emulator_parts("big", self.files):
+            page = parts[stem]
+            self.assertIn('<table class="rb-entries" data-rb-stack="true">', page)
+            self.assertNotIn('class="rb-file-entry"', page)
+            for i in entries:
+                f = self.files[i]
+                self.assertIn(f'<summary><h3 id="{anchors[i]}" class="rb-file-title"', page)
+                for member in f["contents"]:
+                    self.assertIn(f"`{member['name']}`", page)
+                    self.assertIn(f"CRC32 `{member['crc32']}`", page)
+                    self.assertIn(f"SHA1 `{member['sha1']}`", page)
+                # The row says how many members the archive holds.
+                self.assertIn(f"data-pagefind-ignore>{len(f['contents'])}</td>", page)
+
+    def test_the_section_badge_is_not_repeated_on_each_row(self):
+        page = self.pages["big-files-1"]
+        self.assertIn("## Required", page)
+        self.assertNotIn(">required</span>", page)
+        self.assertIn(">missing</span>", page)
+
+
+class ALongEmulatorPageIsOneTable(unittest.TestCase):
+    def test_rows_replace_the_index_and_the_cards(self):
+        files = ([_archive(i) for i in range(gs.EMULATOR_TABLE_MIN - 10)]
+                 + [{"name": f"opt{i}.bin", "md5": f"{i:032x}"} for i in range(10)])
+        page = gs.generate_emulator_page(
+            "long", {"emulator": "Long", "type": "standalone", "files": files}, EMPTY_DB, None, set())
+        self.assertNotIn("| File |", page)
+        self.assertNotIn('class="rb-file-entry"', page)
+        self.assertEqual(page.count('<details class="rb-entry">'), len(files))
+        optional = page[page.index("## Optional"):]
+        # Hash column: the first 12 characters, the whole value in the title
+        # and in the card.
+        md5 = f"{9:032x}"
+        self.assertIn(f'MD5 <code title="{md5}">{md5[:12]}</code>', optional)
+        self.assertIn(f'MD5: <code class="rb-hash">{md5}</code>', optional)
+
+    def test_a_shorter_page_keeps_its_cards(self):
+        files = [_archive(i) for i in range(gs.EMULATOR_TABLE_MIN - 1)]
+        page = gs.generate_emulator_page(
+            "short", {"emulator": "Short", "type": "standalone", "files": files}, EMPTY_DB, None, set())
+        self.assertIn('class="rb-file-entry"', page)
+        self.assertNotIn('class="rb-entries"', page)
+
+
+class TheWantedListIsOneFilteredTable(unittest.TestCase):
+    def setUp(self):
+        dats = ["Sony - PlayStation 2 - BIOS Images", "Nintendo - GameCube - BIOS Images"]
+        self.missing = [{"name": f"bios{i}.bin", "sha1": f"{i:040x}", "description": f"Dump {i}",
+                         "dat": dats[i % 2]} for i in range(30)]
+        self.missing.append({"name": "both.bin", "sha1": "f" * 40, "dats": dats})
+        report = {"redump": {
+            "imported_at": "2026-01-01", "dats": {}, "matched": 12, "out_of_scope": 7,
+            "covered_dats": dats + ["Sega - Dreamcast - BIOS Images"],
+            "missing": [dict(entry) for entry in self.missing] + [
+                {"name": "made.bin", "sha1": "a" * 40, "dat": dats[0]}],
+        }}
+        moved: dict[str, str] = {}
+        self.main = gs.generate_provenance_page(
+            {"files": {}}, report, {"a" * 40: {"from": ["bios/x.bin"], "transform": "half"}}, moved)
+        self.page = moved["provenance-redump"]
+
+    def test_the_counts_come_first(self):
+        head = self.page[:self.page.index("<details")]
+        self.assertIn("Held: **12** of 44, in the 3 DATs the collection reaches. "
+                      "Wanted: **31**, from 2 DATs.", head)
+        self.assertIn("1 more is [derivable]", head)
+        self.assertIn("7 more sit in DATs the collection does not reach.", head)
+        self.assertIn("| Sony - PlayStation 2 - BIOS Images | 16 |", self.page)
+        self.assertIn("| Nintendo - GameCube - BIOS Images | 16 |", self.page)
+
+    def test_every_dump_is_one_row_under_a_filter_box(self):
+        box = re.search(r'<div class="rb-table-filter">.*?</div>\n<details[^>]*>\n'
+                        r"<summary>All 31 wanted dumps</summary>\n<table class=\"rb-wanted\"",
+                        self.page, re.S)
+        self.assertIsNotNone(box)
+        self.assertIn('aria-live="polite">31 rows</span>', box.group(0))
+        self.assertEqual(self.page.count("<tr><td>"), 31)
+        for entry in self.missing:
+            self.assertIn(f"<tr><td>{entry['name']}</td>", self.page)
+            self.assertIn(f'<td class="rb-hash" data-pagefind-ignore>{entry["sha1"]}</td>',
+                          self.page)
+        self.assertIn("Sony - PlayStation 2 - BIOS Images, Nintendo - GameCube - BIOS Images",
+                      self.page)
+        self.assertNotIn("made.bin", self.page)
+
+
+class TheBrowserSide(unittest.TestCase):
+    CSS = (REPO_ROOT / "website" / "src" / "styles" / "retrobios.css").read_text(encoding="utf-8")
+    TS = (REPO_ROOT / "website" / "src" / "scripts" / "site.ts").read_text(encoding="utf-8")
+
+    def test_a_filtered_row_stays_hidden_on_a_phone(self):
+        # A stacked row is `display: block`, which outweighed `hidden`.
+        self.assertIn(".sl-markdown-content table[data-rb-stack] tr[hidden] { display: none; }",
+                      self.CSS)
+
+    def test_a_box_above_a_closed_block_opens_it(self):
+        self.assertIn("block?.previousElementSibling?.matches('.rb-table-filter')", self.TS)
+        self.assertIn("if (block && query) block.open = true;", self.TS)
+
+    def test_a_link_to_a_folded_card_opens_its_row(self):
+        start = self.TS.index("const openTargetBlock = () =>")
+        body = self.TS[start:self.TS.index("};", start)]
+        self.assertIn("closest('details')", body)
+        self.assertIn("block.open = true;", body)
+        self.assertIn("addEventListener('hashchange', openTargetBlock);", self.TS)
+
+    def test_an_open_card_takes_the_table_width(self):
+        self.assertIn(".sl-markdown-content .rb-entries-wrap { container-type: inline-size; }",
+                      self.CSS)
+        self.assertIn("details.rb-entry[open] > :not(summary) { width: 100cqi; }", self.CSS)
+
+
+class _Visible(HTMLParser):
+    """Words in <main> outside closed blocks (their summary shows)."""
+
+    VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link",
+                      "meta", "source", "track", "wbr"})
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.stack: list[tuple[str, str]] = []
+        self.hidden = self.main = self.skip = 0
+        self.words = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag in self.VOID:
+            return
+        effect = ""
+        if tag == "summary" and self.stack and self.stack[-1][1] == "hide":
+            self.hidden -= 1
+            effect = "show"
+        elif tag == "details" and "open" not in dict(attrs):
+            self.hidden += 1
+            effect = "hide"
+        if tag in ("script", "style", "template"):
+            self.skip += 1
+        if tag == "main":
+            self.main += 1
+        self.stack.append((tag, effect))
+
+    def handle_endtag(self, tag):
+        if tag in self.VOID:
+            return
+        while self.stack:
+            name, effect = self.stack.pop()
+            self.hidden += {"hide": -1, "show": 1}.get(effect, 0)
+            self.skip -= name in ("script", "style", "template")
+            self.main -= name == "main"
+            if name == tag:
+                break
+
+    def handle_data(self, data):
+        if self.main and not self.skip and not self.hidden:
+            self.words += len(data.split())
+
+
+def _visible_words(page: str) -> int:
+    parser = _Visible()
+    parser.feed((DIST / page / "index.html").read_text(encoding="utf-8"))
+    return parser.words
+
+
+# Words a visitor sees with every block closed, by page type: the largest
+# measured after the change plus 5 %, never above the largest before it.
+# Before: MAME parts 16,680, long emulator pages 14,396 (groovymame),
+# wanted lists 346 (tosec). After: 1,612, 2,000 (yaps2), 118.
+VISIBLE_CEILINGS = {
+    "emulator card parts": (r"emulators/[a-z0-9-]+-files-\d+$", 1_693),
+    "long emulator pages": (r"emulators/(groovymame|yaps2|bizhawk|bluemsx|fbneo)$", 2_100),
+    "wanted lists": (r"provenance-[a-z-]+$", 124),
+}
+
+
+@unittest.skipUnless((DIST / "emulators" / "mame-files-1" / "index.html").exists(),
+                     "website/dist is not built")
+class TheBuiltPagesStayShort(unittest.TestCase):
+    def _pages(self, pattern: str) -> list[str]:
+        found = [path.parent.relative_to(DIST).as_posix() for path in DIST.glob("**/index.html")]
+        return sorted(page for page in found if re.fullmatch(pattern, page))
+
+    def test_visible_words_under_each_ceiling(self):
+        for kind, (pattern, ceiling) in VISIBLE_CEILINGS.items():
+            pages = self._pages(pattern)
+            self.assertTrue(pages, kind)
+            for page in pages:
+                with self.subTest(page=page):
+                    self.assertLessEqual(_visible_words(page), ceiling, kind)
+
+    def test_under_the_element_budget(self):
+        for pattern, _ceiling in VISIBLE_CEILINGS.values():
+            for page in self._pages(pattern):
+                with self.subTest(page=page):
+                    parsed = validate_site._PageParser(DIST / page / "index.html")
+                    parsed.feed((DIST / page / "index.html").read_text(encoding="utf-8"))
+                    self.assertLessEqual(parsed.page.elements, validate_site.MAX_PAGE_ELEMENTS)
+
+    def test_every_mame_set_and_member_is_on_a_part(self):
+        profile = yaml.safe_load((REPO_ROOT / "emulators" / "mame.yml").read_text(encoding="utf-8"))
+        html = "".join((DIST / page / "index.html").read_text(encoding="utf-8")
+                       for page in self._pages(r"emulators/mame-files-\d+"))
+        for f in profile["files"]:
+            with self.subTest(name=f["name"]):
+                self.assertIn(f"{f['name']}</code></h3></summary>", html)
+                for member in f.get("contents") or []:
+                    if member.get("crc32"):
+                        self.assertIn(str(member["crc32"]), html)
+                    if member.get("sha1"):
+                        self.assertIn(str(member["sha1"]), html)
+
+
+if __name__ == "__main__":
+    unittest.main()

website/src/scripts/site.ts

@@ -87,9 +87,19 @@ const setScrollable = (table: HTMLTableElement, overflows: boolean) => {
 // The filter box above a long table is written when the page is built
 // (src/plugins/rehype-stack-tables.mjs), so the table never moves down once
 // drawn. Here it only filters.
-const filterTable = (table: HTMLTableElement) => {
+// A table folded in a closed block can have its box above the block
+// (the provenance lists): typing opens the block on the rows it keeps.
+const filterControls = (table: HTMLTableElement) => {
 	const controls = table.previousElementSibling;
-	if (!controls?.matches('.rb-table-filter')) return;
+	if (controls?.matches('.rb-table-filter')) return controls;
+	const block = table.closest('details');
+	return block?.previousElementSibling?.matches('.rb-table-filter') ? block.previousElementSibling : null;
+};
+
+const filterTable = (table: HTMLTableElement) => {
+	const controls = filterControls(table);
+	if (!controls) return;
+	const block = controls.nextElementSibling instanceof HTMLDetailsElement ? controls.nextElementSibling : null;
 	const input = controls.querySelector('input');
 	const count = controls.querySelector('.rb-table-filter-count');
 	if (!input || !count || input.dataset.rbWired) return;
@@ -100,6 +110,7 @@ const filterTable = (table: HTMLTableElement) => {
 	input.addEventListener('input', () => {
 		searchable ??= rows.map((row) => ({ row, text: row.textContent!.normalize('NFKD').toLocaleLowerCase() }));
 		const query = input.value.trim().normalize('NFKD').toLocaleLowerCase();
+		if (block && query) block.open = true;
 		let visible = 0;
 		for (const entry of searchable) {
 			entry.row.hidden = !!query && !entry.text.includes(query);
@@ -171,6 +182,9 @@ const enhanceTables = () => {
 		});
 	if (document.readyState === 'complete') start();
 	else addEventListener('load', start, { once: true });
+	// A box above a closed block is typed in before its table is ever near
+	// the screen: it is wired now.
+	for (const table of tables) if (table.closest('details:not([open])') && filterControls(table)) filterTable(table);
 	let timer = 0;
 	addEventListener('resize', () => {
 		clearTimeout(timer);
@@ -743,6 +757,18 @@ const followMovedCard = () => {
 	if (match && !target) location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot()));
 };
 
+// A link to a card folded in a table row (emulators with many entries)
+// opens its row. Chrome does it on its own. Where the browser left the
+// target hidden, the same address is followed again once the row is open,
+// so the browser brings the card into view as for any link.
+const openTargetBlock = () => {
+	const id = decodeURIComponent(location.hash.slice(1));
+	const block = id ? document.getElementById(id)?.closest('details') : null;
+	if (!block || block.open) return;
+	block.open = true;
+	location.replace(location.hash);
+};
+
 // The site search holds no hashes. A hash typed in it gets "Look up this
 // hash" as its first result, and Enter opens the lookup. The pages Pagefind
 // finds for the same characters (a file name that holds them, a word that
@@ -948,6 +974,8 @@ const notFound = () => {
 
 resolveFileAddress();
 followMovedCard();
+openTargetBlock();
+addEventListener('hashchange', openTargetBlock);
 enhanceTables();
 breakPaths();
 hashSearch();

website/src/styles/retrobios.css

@@ -530,6 +530,44 @@ body:has(.rb-sys-file, .rb-file-entry) { --sl-content-width: var(--rb-grid-width
 }
 :is(.rb-sys-file, .rb-file-entry):has(> .rb-file-title:target) { background: var(--rb-surface); }
 
+/* Entries as table rows (scripts/siteentries.py): the name takes the room
+   and its card opens under it. On a phone the row stacks like any table
+   with a hash column, the name on the full width. */
+body:has(table.rb-entries) { --sl-content-width: var(--rb-grid-width); }
+/* A row the filter leaves out stays out on a phone, where a stacked row is
+   a block: `display: block` outweighed the hidden attribute, and every
+   stacked table showed all its rows under "11 of 1,037 rows". */
+.sl-markdown-content table[data-rb-stack] tr[hidden] { display: none; }
+@media (min-width: 50rem) {
+	.sl-markdown-content table.rb-wanted td.rb-hash { min-width: 10rem; }
+}
+.sl-markdown-content .rb-entries-wrap { container-type: inline-size; }
+.sl-markdown-content table.rb-entries { width: 100%; }
+@media (min-width: 50rem) {
+	.sl-markdown-content table.rb-entries { display: table; }
+	.sl-markdown-content table.rb-entries td:first-child { width: 100%; }
+	/* An open card takes the table's width, not its column's: the other
+	   cells of its row keep to one line, so the room under them is free. */
+	.sl-markdown-content table.rb-entries tr:has(> td > details[open]) > td:not(:first-child) { white-space: nowrap; }
+	.sl-markdown-content details.rb-entry[open] { contain: inline-size; }
+	.sl-markdown-content details.rb-entry[open] > :not(summary) { width: 100cqi; }
+}
+.sl-markdown-content details.rb-entry { border: 0; padding: 0; }
+.sl-markdown-content details.rb-entry > summary { min-height: 24px; font-weight: 400; }
+.sl-markdown-content details.rb-entry > summary > .rb-file-title { display: inline; margin: 0; }
+.sl-markdown-content details.rb-entry[open] > summary { margin-bottom: 0.4rem; }
+.sl-markdown-content details.rb-entry ul { margin: 0.3rem 0 0; padding-left: 1.2rem; font-size: 0.85rem; }
+.sl-markdown-content details.rb-entry li { margin: 0.1rem 0; }
+.sl-markdown-content details.rb-entry:has(.rb-file-title:target) { background: var(--rb-surface); }
+@media (max-width: 50rem) {
+	.sl-markdown-content table.rb-entries td:first-child { padding-left: 0.6rem; font-size: 1rem; }
+	.sl-markdown-content table.rb-entries td:first-child::before { content: none; }
+	/* "required for Data East DECO Cassette System" ran 12 px past a phone
+	   beside its label: a long badge wraps in its cell. */
+	.sl-markdown-content table.rb-entries .rb-badge { white-space: normal; }
+	.sl-markdown-content details.rb-entry ul { font-size: 0.75rem; }
+}
+
 /* Collapsible blocks are quiet: a closed block must not outweigh the
    command it explains. Colour is kept for caution and danger. */
 .sl-markdown-content details.rb-details {

website/tools/search_gate.py

@@ -112,6 +112,13 @@ QUERIES: dict[str, tuple[tuple[str, ...], int]] = {
     "r36s": (("platforms/", "which-pack/"), 3),
     "steam deck": (("platforms/emudeck/", "platforms/retrodeck/", "which-pack/"), 1),
     "knulli": (("platforms/knulli/",), 1),
+    # A card folded in a table row, on the last part of MAME's cards.
+    "bbc_flop_torch.xml": (("emulators/mame-files-",), 3),
+    # FinalBurn Neo has no pgm.zip entry: the archive is named by the badge
+    # of each member it reads there, in the Status column of its table.
+    "pgm.zip": (("emulators/fbneo/",), 3),
+    # A dump name in a wanted list folded under its filter box.
+    "speeddos": (("provenance-tosec/",), 1),
 }
 
 

wiki/testing-guide.md

@@ -62,6 +62,7 @@ Every module in `tests/`, by the area it guards.
 | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` |
 | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
 | Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_version_basis`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
 
 ## Test architecture