diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index f73a52a08..9a84a5c55 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -45,6 +45,7 @@ on: - "scripts/safeparse.py" - "scripts/sect233r1.py" - "scripts/sitediagrams.py" + - "scripts/siteentries.py" - "scripts/siteguides.py" - "scripts/sitemarkup.py" - "scripts/sitemeasurements.py" diff --git a/scripts/generate_site.py b/scripts/generate_site.py index dd04f5697..4f4323fd5 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -83,6 +83,7 @@ from generate_readme import ( ) from provenance_report import build_report, distinct_missing import sitemarkup as markup +import siteentries import sitestarlight from siteexports import generate_exports_page, load_export_reports from siteguides import ( @@ -5101,12 +5102,14 @@ def generate_provenance_page( # the public for 79 of them. derivable = derivable or {} derived_entries: dict[str, dict] = {} - for data in report.values(): + derived_by_source: dict[str, int] = {} + for source, data in report.items(): kept = [] for entry in data["missing"]: sha1 = (entry.get("sha1") or "").lower() if sha1 in derivable: derived_entries.setdefault(sha1, entry) + derived_by_source[source] = derived_by_source.get(source, 0) + 1 else: kept.append(entry) data["missing"] = kept @@ -5134,7 +5137,19 @@ def generate_provenance_page( for dat in entry.get("dats") or [entry.get("dat", "")]: by_dat.setdefault(dat, []).append(entry) lines.extend([f"### {label}", ""]) - blocks: list[list[str]] = [] + if moved is not None: + where = _list_pages( + f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}", + f"Dumps {label} catalogues and the collection lacks. Part of " + f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).", + [], [_wanted_list(source, label, data, by_dat, derived_by_source.get(source, 0))], + moved, parent=("Dump provenance", "/provenance/"), + ) + lines.extend([ + f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: " + f"listed on {where}.", "", + ]) + continue for dat in sorted(by_dat): entries = by_dat[dat] table = ["| Name | Description | SHA1 |", "|------|-------------|------|"] @@ -5150,20 +5165,7 @@ def generate_provenance_page( f"`{hit['archive']}` as `{hit['member']}`." ) table.append(f"| `{name}` | {desc} | `{sha1}` |") - blocks.append(markup.details("note", f"{dat} ({len(entries)})", table)) - if moved is None: - lines.extend(line for block in blocks for line in block) - continue - where = _list_pages( - f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}", - f"Dumps {label} catalogues and the collection lacks, by DAT. Part of " - f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).", - [], blocks, moved, parent=("Dump provenance", "/provenance/"), - ) - lines.extend([ - f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: " - f"listed on {where}.", "", - ]) + lines.extend(markup.details("note", f"{dat} ({len(entries)})", table)) if derived_entries: lines.extend( @@ -5189,6 +5191,69 @@ def generate_provenance_page( return "\n".join(lines) + "\n" +def _wanted_list(source: str, label: str, data: dict, by_dat: dict[str, list], + derived: int) -> list[str]: + """A catalog's wanted page: the counts, then every missing dump in one + table folded under a filter box. + + The page was one closed block per DAT, 46 for TOSEC: a name was found + by opening each in turn. The box filters the whole list by name, DAT, + description or SHA1, and says how many rows it keeps. + """ + missing = data["missing"] + held = data["matched"] + reached = held + len(missing) + derived + dats = _count(len(data["covered_dats"]), "DAT") + lines = [ + f"Held: **{held:,}** of {reached:,}, in the {dats} the collection reaches. " + f"Wanted: **{len(missing):,}**, from {_count(len(by_dat), 'DAT')}.", + "", + ] + if derived: + verb = "is" if derived == 1 else "are" + lines.extend([ + f"{derived:,} more {verb} [derivable]" + "(provenance.md#derivable-dumps-the-collection-already-produces).", ""]) + if data.get("out_of_scope"): + lines.extend([f"{data['out_of_scope']:,} more sit in DATs the collection does not reach.", ""]) + counts = ["| DAT | Wanted |", "|-----|--------|", + *(f"| {dat.replace('|', '-')} | {len(entries):,} |" + for dat, entries in sorted(by_dat.items()))] + lines.extend(markup.details("note", f"Wanted by DAT ({len(by_dat):,})", counts)) + rows = [] + for entry in sorted(missing, key=lambda e: (e["name"], e.get("sha1") or "")): + desc = markup.inline_html(entry.get("description") or "") + for hit in entry.get("romsets", []): + desc += markup.inline_html( + f" Ships in {_ROMSET_SOURCES.get(hit['source'], hit['source'])} " + f"`{hit['archive']}` as `{hit['member']}`.") + in_dats = ", ".join(html.escape(d) for d in (entry.get("dats") or [entry.get("dat", "")])) + sha1 = html.escape(entry.get("sha1") or "-") + # The search reads the name alone. The DAT names are searched once, in + # the count table above, and the site search holds no hashes. With the + # descriptions indexed too, the TOSEC list matched "2" 556 times + # (years, versions) and put the PlayStation page ahead of the + # PlayStation 2 one for "playstation 2 bios". The name and the hash + # sit in their cells as text: a catalog name is words, and in code + # each word wore its own box on a narrow column. + skip = markup.search_exclude() + rows.append("" + f"{html.escape(entry['name'], quote=False)}" + f"{in_dats}{desc or '-'}" + f'{sha1}') + heads = ["Name", "DAT", "Description", "SHA1"] + lines.extend([ + siteentries.filter_box(f"rb-wanted-filter-{_slugify_anchor(source)}", + "dumps", heads, len(rows)), + '
', + f"All {_count(len(rows), 'wanted dump')}", + '', + "" + "".join(f'' for h in heads) + "", + "", *rows, "", "
{h}
", "
", "", + ]) + return lines + + _RELEASE_ASSET_NAMES: dict[str, str] | None = None @@ -7462,12 +7527,15 @@ def _render_emulator_file( owner: str = "", alias: str = "", data_href: str = "#data-folder", + badges_apart: bool = False, ) -> list[str]: """Render one row of an emulator profile's file table. `anchor` is the card's id on the page, so one file can be cited by URL, and `alias` a second id it answers to. `data_href` links the data folder an app reads the entry under, from the page the card lands on. + `badges_apart`: the title holds the name alone, for a table whose + Status column shows the badges (siteentries.py). Split out of generate_emulator_page, where it was a 228-line loop body carrying most of that function's branching: thirty-odd optional fields, @@ -7516,7 +7584,8 @@ def _render_emulator_file( system = system_name((profile.get("systems") or [""])[0]) label = (f"a {system} BIOS (example: {_code(fname)})" if _named_example(str(fname)) else f"a {system} BIOS, any file name") - lines.append(_card_title(f"{marker}{label} {badge_str}", anchor, weight=5)) + title = f"{marker}{label}" if badges_apart else f"{marker}{label} {badge_str}" + lines.append(_card_title(title, anchor, weight=5)) lines.append("") if desc: lines.extend([data_root_words(_escape_tags(str(desc)), data_href), ""]) @@ -8522,6 +8591,10 @@ def _data_tree_page(ref: str, entry: dict, pages: list[tuple[str, list[str]]], # anchor. A count, not a size: system cards link to these cards and are # written before them. At 400 a part of MAME's cards was 805 KB of HTML. EMULATOR_CARDS_PER_PAGE = 220 +# The most cards a part of a split page holds. Each card is a table row +# since siteentries.py, about five elements and 60 bytes more: at 220 a +# part, MAME's first was 612 KB and 9,777 elements. +EMULATOR_CARDS_PER_PART = 180 # The headings an emulator's cards go under, in page order: what it needs @@ -8556,12 +8629,59 @@ def emulator_parts(name: str, files: list) -> list[tuple[str, list[int]]]: if len(files) <= EMULATOR_CARDS_PER_PAGE: return [] order = _display_order(files) - count = -(-len(files) // EMULATOR_CARDS_PER_PAGE) + count = -(-len(files) // EMULATOR_CARDS_PER_PART) size = -(-len(files) // count) return [(f"{name}-files-{index + 1}", order[start:start + size]) for index, start in enumerate(range(0, len(files), size))] +# From this many entries an unsplit emulator page lists them as one table, +# each row opening to its card (siteentries.py), in place of a summary +# table above the cards: GroovyMAME's 171 cards read 14,400 words open. +# A split emulator's card pages always do. +EMULATOR_TABLE_MIN = 150 +ENTRY_HEADS = ["File", "Contents", "Status", "Hash"] +# The end of the badge _file_badges writes for an entry of each section. +_SECTION_BADGE = {"required": ">required", "optional": ">optional", + "with_app": ">comes with the app"} + + +def _entry_rows(files: list[dict], cards: list[list[str]], profile: dict, + available: Callable[[dict], bool]) -> list[siteentries.Row]: + """Each entry as a table row: its card's title and body, the number of + members an archive holds, its badges and its hash.""" + many = len(profile.get("systems") or []) > 1 + rows = [] + for f, card in zip(files, cards): + title, body = siteentries.card_title_and_body(card) + for_system = system_name(str(f["system"])) if f.get("system") and many else "" + contents = f.get("contents") or [] + badges = _file_badges(f, available(f), for_system) + # The first badge says whether the entry is needed: under Required, + # Optional or Comes with the app it repeats the section heading. + # One element and two words fewer on each of MAME's 1,563 rows. + if badges and badges[0].endswith(_SECTION_BADGE.get(_card_section(f), "\0")): + badges = badges[1:] + rows.append(siteentries.Row(title, body, [ + f"{len(contents):,}" if contents else "", + " ".join(badges), + _hash_short(f), + ])) + return rows + + +def _entries_table(files: list[dict], group: list[int], rows: list[siteentries.Row]) -> list[str]: + """The rows of ``group``, one section's entries, as one table.""" + key = _card_section(files[group[0]]) + heading = dict(CARD_SECTIONS)[key] + return siteentries.entry_table([rows[i] for i in group], ENTRY_HEADS, + f"rb-entry-filter-{key.replace('_', '-')}", heading, + # The badges hide themselves from the search + # but for the archive's name ("in pgm.zip"), + # how a ZIP name finds the cores reading it. + searched=frozenset({"Status"})) + + def _sectioned(files: list, indexes: list[int], render: Callable[[list[int]], list[str]], fold: bool = False) -> list[str]: """Each section's heading, then ``render`` of its entries among ``indexes``. @@ -8999,12 +9119,18 @@ def emulator_pages( aliases = emulator_file_aliases(files, anchors) parts = emulator_parts(name, files) data_href = f"{name}.md#data-folder" if parts else "#data-folder" + table_form = markup.starlight() and (bool(parts) or len(files) >= EMULATOR_TABLE_MIN) cards = [ _render_emulator_file(f, profile, consumers, files, _file_available, anchor, name, - alias, data_href) + alias, data_href, badges_apart=table_form) for f, anchor, alias in zip(files, anchors, aliases) ] - if not parts: + rows = (_entry_rows(files, cards, profile, _file_available) if table_form else []) + if not parts and table_form: + lines.extend(held) + lines.extend(_sectioned(files, _display_order(files), + lambda group: _entries_table(files, group, rows))) + elif not parts: order = _display_order(files) indexed = len(files) >= EMULATOR_SUMMARY_MIN if indexed: @@ -9026,7 +9152,8 @@ def emulator_pages( for number, (stem, entries) in enumerate(parts, 1): part_pages[stem] = _emulator_part_page( name, emu_name, number, parts, [cards[i] for i in entries], - [files[i] for i in entries]) + [files[i] for i in entries], + [rows[i] for i in entries] if table_form else None) lines.extend(sitemeasurements.tested_section(name, _MEASUREMENT_ITEMS.get(name, []))) @@ -9156,9 +9283,16 @@ def _part_front_matter(parent: tuple[str, str], siblings: list[tuple[str, str]], def _emulator_part_page( name: str, emu_name: str, number: int, parts: list[tuple[str, range]], - cards: list[list[str]], files: list[dict], + cards: list[list[str]], files: list[dict], rows: list[siteentries.Row] | None = None, ) -> str: - """One page of a split emulator's cards.""" + """One page of a split emulator's cards: with ``rows``, one table row + each, the card under its name.""" + if rows is not None: + def render(group: list[int]) -> list[str]: + return _entries_table(files, group, rows) + else: + def render(group: list[int]) -> list[str]: + return [line for i in group for line in cards[i]] first = str(files[0].get("path") or files[0].get("name", "")) last = str(files[-1].get("path") or files[-1].get("name", "")) others = ", ".join( @@ -9173,8 +9307,7 @@ def _emulator_part_page( f"This part holds {_count(len(files), 'entry', 'entries')}, " f"{_code(first)} to {_code(last)}. The other parts: {others}.", "", - *_sectioned(files, list(range(len(files))), - lambda group: [line for i in group for line in cards[i]]), + *_sectioned(files, list(range(len(files))), render), ] lines.extend(_next_block([f"[{emu_name}]({name}.md), the profile and every entry"])) lines.extend(["", _date_line()]) diff --git a/scripts/siteentries.py b/scripts/siteentries.py new file mode 100644 index 000000000..7fc5dc31c --- /dev/null +++ b/scripts/siteentries.py @@ -0,0 +1,112 @@ +"""Long lists as one filterable table, each row opening to its details. + +MAME's card pages read 13,000 to 16,000 words, one card after another, +before a reader found the archive they came for. Here each entry is one +row: its name, the facts that tell rows apart, and the full card in a +closed block under the name. The table is written as HTML so the block can +hold the card's Markdown. It carries what rehype-stack-tables.mjs gives a +Markdown table: `data-rb-stack` (one block per row on a phone) and the +filter box above, which site.ts wires. Without JavaScript the rows show and +every block opens. A closed block stays in the page, so the browser's find +and the site search still reach every name it holds. +""" + +from __future__ import annotations + +import html +from dataclasses import dataclass, field + +import sitemarkup as markup + +# rehype-stack-tables.mjs RECORD_ROWS: from this many rows a table has a +# filter box, and its stacked labels are written by site.ts, not the build. +RECORD_ROWS = 20 +# filterPlaceholder in rehype-stack-tables.mjs: the column names offered. +PLACEHOLDER_COLUMNS = 4 + + +@dataclass +class Row: + """One entry: ``title`` is the inline HTML of its name (a heading, so + the site search lands on it), ``body`` the Markdown lines shown when the + row opens, ``cells`` the HTML of the other columns.""" + + title: str + body: list[str] + cells: list[str] = field(default_factory=list) + + +def filter_placeholder(labels: list[str]) -> str: + """"File, contents, status..." as rehype-stack-tables.mjs words it.""" + words = [label for label in labels if label][:PLACEHOLDER_COLUMNS] + if not words: + return "Type to filter..." + lowered = [word if index == 0 or not (word[:1].isupper() and word[1:2].islower()) + else word[0].lower() + word[1:] for index, word in enumerate(words)] + return f"{', '.join(lowered)}..." + + +def filter_box(ident: str, heading: str, labels: list[str], rows: int, noun: str = "rows") -> str: + """The filter box rehype-stack-tables.mjs writes above a long table.""" + return ( + f'
' + f'' + f'{rows:,} {noun}
' + ) + + +def card_title_and_body(card: list[str]) -> tuple[str, list[str]]: + """A card's heading line and the lines under it, its wrapping block + (the first line and the last closing tag) taken off.""" + title_at = next(index for index, line in enumerate(card) if line.startswith("") + body = card[title_at + 1:closing] + while body and not body[0].strip(): + body.pop(0) + while body and not body[-1].strip(): + body.pop() + return card[title_at], body + + +def entry_table(rows: list[Row], heads: list[str], ident: str, heading: str, + stacked: list[str] | None = None, + searched: frozenset[str] = frozenset()) -> list[str]: + """``rows`` under ``heads`` (the first names the column of titles). + + ``stacked`` gives a column's short label on a phone, as a header's + `title` does in a Markdown table. A column every row leaves empty + (``""``) is not written. The site search skips every column but the + titles and those ``searched`` names: a count or a short hash is no word + anyone types. + """ + keep = [0] + [column for column in range(1, len(heads)) + if any(row.cells[column - 1] for row in rows)] + labels = [heads[c] for c in keep] + short = [(stacked or heads)[c] for c in keep] + skip = markup.search_exclude() + # The block holds the box and the table: an open card spans its width. + lines = ["", '
'] + if len(rows) >= RECORD_ROWS: + lines.append(filter_box(ident, heading, labels, len(rows))) + lines.append('') + head_cells = "" + for label, name in zip(labels, short): + title = f' title="{html.escape(name)}"' if name != label else "" + head_cells += f'' + lines.extend(["", f"{head_cells}", "", ""]) + for row in rows: + lines.append("") + if row.body: + lines.extend([""]) + else: + lines.append(f"") + for column in keep[1:]: + label = (f' data-label="{html.escape(short[keep.index(column)])}"' + if len(rows) < RECORD_ROWS else "") + ignored = "" if heads[column] in searched else f" {skip}" + lines.append(f"{row.cells[column - 1] or '-'}") + lines.append("") + lines.extend(["", "
{html.escape(label)}
", '
', f"{row.title}", + "", *row.body, "", "
", "		{row.title}
", "
", ""]) + return lines diff --git a/tests/test_site_entry_tables.py b/tests/test_site_entry_tables.py new file mode 100644 index 000000000..cd72b7c38 --- /dev/null +++ b/tests/test_site_entry_tables.py @@ -0,0 +1,311 @@ +"""Long lists read as a table first, details on demand (density round). + +MAME's card pages read 13,000 to 16,700 words, card after card, and +GroovyMAME 14,400 before its last entry. The TOSEC wanted list was 46 +closed blocks, one per DAT, opened one by one to find a name. Each entry is +now a table row that opens to its card, and each wanted list one filtered +table under its counts. Nothing the pages said is lost: the tests below read +every name and hash back. +""" + +from __future__ import annotations + +import re +import sys +import unittest +from html.parser import HTMLParser +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site as gs # noqa: E402 +import siteentries # noqa: E402 +import validate_site # noqa: E402 + +EMPTY_DB = {"files": {}, "indexes": {"by_name": {}}} +DIST = REPO_ROOT / "website" / "dist" + + +def _archive(i: int, members: int = 3, **fields: object) -> dict: + return { + "name": f"set{i:04d}.zip", "required": True, "category": "bios_zip", + "contents": [{"name": f"m{i}_{k}.rom", "size": 1024, "crc32": f"{i:04x}{k:04x}", + "sha1": f"{i:04x}{k:04x}" * 5} for k in range(members)], + **fields, + } + + +class TheEntryTable(unittest.TestCase): + def _rows(self, count: int) -> list[siteentries.Row]: + return [siteentries.Row(f'

n{i}

', + [f"- Size: {i} bytes"], [str(i), "", f"MD5 {i}"]) + for i in range(count)] + + def test_a_long_table_has_its_filter_box_and_stacks(self): + lines = siteentries.entry_table(self._rows(25), gs.ENTRY_HEADS, "rb-entry-filter-x", "Required") + text = "\n".join(lines) + self.assertRegex(text, r'
25 rows
\n', text) + # The Status column is empty on every row: not written. + self.assertIn('' + '', text) + self.assertNotIn("Status", text) + self.assertEqual(text.count('
'), 25) + # Labels are written by site.ts on a table this long, as on a Markdown one. + self.assertNotIn("data-label", text) + + def test_a_card_opens_under_its_name_as_markdown(self): + lines = siteentries.entry_table(self._rows(2), gs.ENTRY_HEADS, "x", "Required") + at = lines.index('

n0

') + # A blank line after the summary: the body is read as Markdown. + self.assertEqual(lines[at + 1:at + 4], ["", "- Size: 0 bytes", ""]) + self.assertNotIn("rb-table-filter", "\n".join(lines)) + self.assertIn('
', lines) + + def test_a_card_loses_only_its_wrapper(self): + card = ['
', "", '

x

', "", "Desc.", "", + '
', "", "- SHA1: s", "", "
", "", "
", ""] + title, body = siteentries.card_title_and_body(card) + self.assertEqual(title, '

x

') + self.assertEqual(body, ["Desc.", "", '
', "", "- SHA1: s", "", + "
"]) + + def test_the_placeholder_reads_like_the_plugin(self): + self.assertEqual(siteentries.filter_placeholder(["File", "Contents", "Status", "Hash"]), + "File, contents, status, hash...") + self.assertEqual(siteentries.filter_placeholder(["Name", "DAT", "Description", "SHA1"]), + "Name, DAT, description, SHA1...") + + +class ASplitEmulatorListsItsCardsAsRows(unittest.TestCase): + def setUp(self): + self.files = [_archive(i, members=1 + i % 4) for i in range(gs.EMULATOR_CARDS_PER_PAGE * 2)] + self.pages = gs.emulator_pages( + "big", {"emulator": "Big", "type": "standalone", "files": self.files}, EMPTY_DB) + + def test_parts_hold_at_most_a_part_of_cards(self): + parts = gs.emulator_parts("big", self.files) + self.assertEqual(len(parts), -(-len(self.files) // gs.EMULATOR_CARDS_PER_PART)) + self.assertLessEqual(max(len(entries) for _, entries in parts), gs.EMULATOR_CARDS_PER_PART) + + def test_every_name_and_member_hash_is_on_its_part(self): + parts = {stem: page for stem, page in self.pages.items() if stem != "big"} + anchors = gs.emulator_file_anchors(self.files) + for stem, entries in gs.emulator_parts("big", self.files): + page = parts[stem] + self.assertIn('
FileContentsHash1
', page) + self.assertNotIn('class="rb-file-entry"', page) + for i in entries: + f = self.files[i] + self.assertIn(f'

{len(f['contents'])}", page) + + def test_the_section_badge_is_not_repeated_on_each_row(self): + page = self.pages["big-files-1"] + self.assertIn("## Required", page) + self.assertNotIn(">required", page) + self.assertIn(">missing", page) + + +class ALongEmulatorPageIsOneTable(unittest.TestCase): + def test_rows_replace_the_index_and_the_cards(self): + files = ([_archive(i) for i in range(gs.EMULATOR_TABLE_MIN - 10)] + + [{"name": f"opt{i}.bin", "md5": f"{i:032x}"} for i in range(10)]) + page = gs.generate_emulator_page( + "long", {"emulator": "Long", "type": "standalone", "files": files}, EMPTY_DB, None, set()) + self.assertNotIn("| File |", page) + self.assertNotIn('class="rb-file-entry"', page) + self.assertEqual(page.count('
'), len(files)) + optional = page[page.index("## Optional"):] + # Hash column: the first 12 characters, the whole value in the title + # and in the card. + md5 = f"{9:032x}" + self.assertIn(f'MD5 {md5[:12]}', optional) + self.assertIn(f'MD5: {md5}', optional) + + def test_a_shorter_page_keeps_its_cards(self): + files = [_archive(i) for i in range(gs.EMULATOR_TABLE_MIN - 1)] + page = gs.generate_emulator_page( + "short", {"emulator": "Short", "type": "standalone", "files": files}, EMPTY_DB, None, set()) + self.assertIn('class="rb-file-entry"', page) + self.assertNotIn('class="rb-entries"', page) + + +class TheWantedListIsOneFilteredTable(unittest.TestCase): + def setUp(self): + dats = ["Sony - PlayStation 2 - BIOS Images", "Nintendo - GameCube - BIOS Images"] + self.missing = [{"name": f"bios{i}.bin", "sha1": f"{i:040x}", "description": f"Dump {i}", + "dat": dats[i % 2]} for i in range(30)] + self.missing.append({"name": "both.bin", "sha1": "f" * 40, "dats": dats}) + report = {"redump": { + "imported_at": "2026-01-01", "dats": {}, "matched": 12, "out_of_scope": 7, + "covered_dats": dats + ["Sega - Dreamcast - BIOS Images"], + "missing": [dict(entry) for entry in self.missing] + [ + {"name": "made.bin", "sha1": "a" * 40, "dat": dats[0]}], + }} + moved: dict[str, str] = {} + self.main = gs.generate_provenance_page( + {"files": {}}, report, {"a" * 40: {"from": ["bios/x.bin"], "transform": "half"}}, moved) + self.page = moved["provenance-redump"] + + def test_the_counts_come_first(self): + head = self.page[:self.page.index(".*?\n]*>\n' + r"All 31 wanted dumps\n

31 rows', box.group(0)) + self.assertEqual(self.page.count("", self.page) + self.assertIn(f'', + self.page) + self.assertIn("Sony - PlayStation 2 - BIOS Images, Nintendo - GameCube - BIOS Images", + self.page) + self.assertNotIn("made.bin", self.page) + + +class TheBrowserSide(unittest.TestCase): + CSS = (REPO_ROOT / "website" / "src" / "styles" / "retrobios.css").read_text(encoding="utf-8") + TS = (REPO_ROOT / "website" / "src" / "scripts" / "site.ts").read_text(encoding="utf-8") + + def test_a_filtered_row_stays_hidden_on_a_phone(self): + # A stacked row is `display: block`, which outweighed `hidden`. + self.assertIn(".sl-markdown-content table[data-rb-stack] tr[hidden] { display: none; }", + self.CSS) + + def test_a_box_above_a_closed_block_opens_it(self): + self.assertIn("block?.previousElementSibling?.matches('.rb-table-filter')", self.TS) + self.assertIn("if (block && query) block.open = true;", self.TS) + + def test_a_link_to_a_folded_card_opens_its_row(self): + start = self.TS.index("const openTargetBlock = () =>") + body = self.TS[start:self.TS.index("};", start)] + self.assertIn("closest('details')", body) + self.assertIn("block.open = true;", body) + self.assertIn("addEventListener('hashchange', openTargetBlock);", self.TS) + + def test_an_open_card_takes_the_table_width(self): + self.assertIn(".sl-markdown-content .rb-entries-wrap { container-type: inline-size; }", + self.CSS) + self.assertIn("details.rb-entry[open] > :not(summary) { width: 100cqi; }", self.CSS) + + +class _Visible(HTMLParser): + """Words in
outside closed blocks (their summary shows).""" + + VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link", + "meta", "source", "track", "wbr"}) + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.stack: list[tuple[str, str]] = [] + self.hidden = self.main = self.skip = 0 + self.words = 0 + + def handle_starttag(self, tag, attrs): + if tag in self.VOID: + return + effect = "" + if tag == "summary" and self.stack and self.stack[-1][1] == "hide": + self.hidden -= 1 + effect = "show" + elif tag == "details" and "open" not in dict(attrs): + self.hidden += 1 + effect = "hide" + if tag in ("script", "style", "template"): + self.skip += 1 + if tag == "main": + self.main += 1 + self.stack.append((tag, effect)) + + def handle_endtag(self, tag): + if tag in self.VOID: + return + while self.stack: + name, effect = self.stack.pop() + self.hidden += {"hide": -1, "show": 1}.get(effect, 0) + self.skip -= name in ("script", "style", "template") + self.main -= name == "main" + if name == tag: + break + + def handle_data(self, data): + if self.main and not self.skip and not self.hidden: + self.words += len(data.split()) + + +def _visible_words(page: str) -> int: + parser = _Visible() + parser.feed((DIST / page / "index.html").read_text(encoding="utf-8")) + return parser.words + + +# Words a visitor sees with every block closed, by page type: the largest +# measured after the change plus 5 %, never above the largest before it. +# Before: MAME parts 16,680, long emulator pages 14,396 (groovymame), +# wanted lists 346 (tosec). After: 1,612, 2,000 (yaps2), 118. +VISIBLE_CEILINGS = { + "emulator card parts": (r"emulators/[a-z0-9-]+-files-\d+$", 1_693), + "long emulator pages": (r"emulators/(groovymame|yaps2|bizhawk|bluemsx|fbneo)$", 2_100), + "wanted lists": (r"provenance-[a-z-]+$", 124), +} + + +@unittest.skipUnless((DIST / "emulators" / "mame-files-1" / "index.html").exists(), + "website/dist is not built") +class TheBuiltPagesStayShort(unittest.TestCase): + def _pages(self, pattern: str) -> list[str]: + found = [path.parent.relative_to(DIST).as_posix() for path in DIST.glob("**/index.html")] + return sorted(page for page in found if re.fullmatch(pattern, page)) + + def test_visible_words_under_each_ceiling(self): + for kind, (pattern, ceiling) in VISIBLE_CEILINGS.items(): + pages = self._pages(pattern) + self.assertTrue(pages, kind) + for page in pages: + with self.subTest(page=page): + self.assertLessEqual(_visible_words(page), ceiling, kind) + + def test_under_the_element_budget(self): + for pattern, _ceiling in VISIBLE_CEILINGS.values(): + for page in self._pages(pattern): + with self.subTest(page=page): + parsed = validate_site._PageParser(DIST / page / "index.html") + parsed.feed((DIST / page / "index.html").read_text(encoding="utf-8")) + self.assertLessEqual(parsed.page.elements, validate_site.MAX_PAGE_ELEMENTS) + + def test_every_mame_set_and_member_is_on_a_part(self): + profile = yaml.safe_load((REPO_ROOT / "emulators" / "mame.yml").read_text(encoding="utf-8")) + html = "".join((DIST / page / "index.html").read_text(encoding="utf-8") + for page in self._pages(r"emulators/mame-files-\d+")) + for f in profile["files"]: + with self.subTest(name=f["name"]): + self.assertIn(f"{f['name']}", html) + for member in f.get("contents") or []: + if member.get("crc32"): + self.assertIn(str(member["crc32"]), html) + if member.get("sha1"): + self.assertIn(str(member["sha1"]), html) + + +if __name__ == "__main__": + unittest.main() diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 3c7315a1a..3d9592b2a 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -87,9 +87,19 @@ const setScrollable = (table: HTMLTableElement, overflows: boolean) => { // The filter box above a long table is written when the page is built // (src/plugins/rehype-stack-tables.mjs), so the table never moves down once // drawn. Here it only filters. -const filterTable = (table: HTMLTableElement) => { +// A table folded in a closed block can have its box above the block +// (the provenance lists): typing opens the block on the rows it keeps. +const filterControls = (table: HTMLTableElement) => { const controls = table.previousElementSibling; - if (!controls?.matches('.rb-table-filter')) return; + if (controls?.matches('.rb-table-filter')) return controls; + const block = table.closest('details'); + return block?.previousElementSibling?.matches('.rb-table-filter') ? block.previousElementSibling : null; +}; + +const filterTable = (table: HTMLTableElement) => { + const controls = filterControls(table); + if (!controls) return; + const block = controls.nextElementSibling instanceof HTMLDetailsElement ? controls.nextElementSibling : null; const input = controls.querySelector('input'); const count = controls.querySelector('.rb-table-filter-count'); if (!input || !count || input.dataset.rbWired) return; @@ -100,6 +110,7 @@ const filterTable = (table: HTMLTableElement) => { input.addEventListener('input', () => { searchable ??= rows.map((row) => ({ row, text: row.textContent!.normalize('NFKD').toLocaleLowerCase() })); const query = input.value.trim().normalize('NFKD').toLocaleLowerCase(); + if (block && query) block.open = true; let visible = 0; for (const entry of searchable) { entry.row.hidden = !!query && !entry.text.includes(query); @@ -171,6 +182,9 @@ const enhanceTables = () => { }); if (document.readyState === 'complete') start(); else addEventListener('load', start, { once: true }); + // A box above a closed block is typed in before its table is ever near + // the screen: it is wired now. + for (const table of tables) if (table.closest('details:not([open])') && filterControls(table)) filterTable(table); let timer = 0; addEventListener('resize', () => { clearTimeout(timer); @@ -743,6 +757,18 @@ const followMovedCard = () => { if (match && !target) location.replace(new URL(`files/?sha1=${match[1]}`, siteRoot())); }; +// A link to a card folded in a table row (emulators with many entries) +// opens its row. Chrome does it on its own. Where the browser left the +// target hidden, the same address is followed again once the row is open, +// so the browser brings the card into view as for any link. +const openTargetBlock = () => { + const id = decodeURIComponent(location.hash.slice(1)); + const block = id ? document.getElementById(id)?.closest('details') : null; + if (!block || block.open) return; + block.open = true; + location.replace(location.hash); +}; + // The site search holds no hashes. A hash typed in it gets "Look up this // hash" as its first result, and Enter opens the lookup. The pages Pagefind // finds for the same characters (a file name that holds them, a word that @@ -948,6 +974,8 @@ const notFound = () => { resolveFileAddress(); followMovedCard(); +openTargetBlock(); +addEventListener('hashchange', openTargetBlock); enhanceTables(); breakPaths(); hashSearch(); diff --git a/website/src/styles/retrobios.css b/website/src/styles/retrobios.css index 77831fc28..e25a289b5 100644 --- a/website/src/styles/retrobios.css +++ b/website/src/styles/retrobios.css @@ -530,6 +530,44 @@ body:has(.rb-sys-file, .rb-file-entry) { --sl-content-width: var(--rb-grid-width } :is(.rb-sys-file, .rb-file-entry):has(> .rb-file-title:target) { background: var(--rb-surface); } +/* Entries as table rows (scripts/siteentries.py): the name takes the room + and its card opens under it. On a phone the row stacks like any table + with a hash column, the name on the full width. */ +body:has(table.rb-entries) { --sl-content-width: var(--rb-grid-width); } +/* A row the filter leaves out stays out on a phone, where a stacked row is + a block: `display: block` outweighed the hidden attribute, and every + stacked table showed all its rows under "11 of 1,037 rows". */ +.sl-markdown-content table[data-rb-stack] tr[hidden] { display: none; } +@media (min-width: 50rem) { + .sl-markdown-content table.rb-wanted td.rb-hash { min-width: 10rem; } +} +.sl-markdown-content .rb-entries-wrap { container-type: inline-size; } +.sl-markdown-content table.rb-entries { width: 100%; } +@media (min-width: 50rem) { + .sl-markdown-content table.rb-entries { display: table; } + .sl-markdown-content table.rb-entries td:first-child { width: 100%; } + /* An open card takes the table's width, not its column's: the other + cells of its row keep to one line, so the room under them is free. */ + .sl-markdown-content table.rb-entries tr:has(> td > details[open]) > td:not(:first-child) { white-space: nowrap; } + .sl-markdown-content details.rb-entry[open] { contain: inline-size; } + .sl-markdown-content details.rb-entry[open] > :not(summary) { width: 100cqi; } +} +.sl-markdown-content details.rb-entry { border: 0; padding: 0; } +.sl-markdown-content details.rb-entry > summary { min-height: 24px; font-weight: 400; } +.sl-markdown-content details.rb-entry > summary > .rb-file-title { display: inline; margin: 0; } +.sl-markdown-content details.rb-entry[open] > summary { margin-bottom: 0.4rem; } +.sl-markdown-content details.rb-entry ul { margin: 0.3rem 0 0; padding-left: 1.2rem; font-size: 0.85rem; } +.sl-markdown-content details.rb-entry li { margin: 0.1rem 0; } +.sl-markdown-content details.rb-entry:has(.rb-file-title:target) { background: var(--rb-surface); } +@media (max-width: 50rem) { + .sl-markdown-content table.rb-entries td:first-child { padding-left: 0.6rem; font-size: 1rem; } + .sl-markdown-content table.rb-entries td:first-child::before { content: none; } + /* "required for Data East DECO Cassette System" ran 12 px past a phone + beside its label: a long badge wraps in its cell. */ + .sl-markdown-content table.rb-entries .rb-badge { white-space: normal; } + .sl-markdown-content details.rb-entry ul { font-size: 0.75rem; } +} + /* Collapsible blocks are quiet: a closed block must not outweigh the command it explains. Colour is kept for caution and danger. */ .sl-markdown-content details.rb-details { diff --git a/website/tools/search_gate.py b/website/tools/search_gate.py index ee8d04060..e20676687 100644 --- a/website/tools/search_gate.py +++ b/website/tools/search_gate.py @@ -112,6 +112,13 @@ QUERIES: dict[str, tuple[tuple[str, ...], int]] = { "r36s": (("platforms/", "which-pack/"), 3), "steam deck": (("platforms/emudeck/", "platforms/retrodeck/", "which-pack/"), 1), "knulli": (("platforms/knulli/",), 1), + # A card folded in a table row, on the last part of MAME's cards. + "bbc_flop_torch.xml": (("emulators/mame-files-",), 3), + # FinalBurn Neo has no pgm.zip entry: the archive is named by the badge + # of each member it reads there, in the Status column of its table. + "pgm.zip": (("emulators/fbneo/",), 3), + # A dump name in a wanted list folded under its filter box. + "speeddos": (("provenance-tosec/",), 1), } diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 84c1bd8cf..6add39e2b 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -61,7 +61,7 @@ Every module in `tests/`, by the area it guards. | Real-condition campaign | synthetic profiles, records and content folders, a fake host for load, disk, clock and commands, and whole passes against a libretro core built from C when a compiler and strace exist | `test_campaign` | | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture
"), 31) + for entry in self.missing: + self.assertIn(f"
{entry['name']}{entry["sha1"]}