From 74b49e8c0427d5fe547fbc2ec982dc03fa1a0280 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:43:59 +0100 Subject: [PATCH 1/3] refactor: return source ref parts with roles source_ref_items() gives each part of a source_ref with the role of the code it cites. _source_ref_markdown() joins them as before, so every other page renders the same text. --- scripts/sitesources.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/scripts/sitesources.py b/scripts/sitesources.py index c0270a5e9..55dc2f5e9 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -425,6 +425,24 @@ def _two_repositories(profile: dict) -> bool: def _source_ref_markdown(profile: dict, value: str | dict | None) -> str: """Render source_ref values as pinned links when their forge is known. + Each part is followed by whose code it cites, in words, when the profile + reads two repositories. source_ref_items() gives the same parts with + that role apart, for a page that names the roles once in a legend. + """ + rendered_groups = [] + for label, items in source_ref_items(profile, value): + text = ", ".join(f"{part} ({role})" if role else part for part, role in items) + if label: + text = f"**{label}:** {text}" + rendered_groups.append(text) + return ", ".join(rendered_groups) + + +def source_ref_items(profile: dict, value: str | dict | None) -> list[tuple[str, list[tuple[str, str]]]]: + """Each group of source_ref values: its label and its parts, each part + as Markdown with the role_label() of the code it cites ("" when the + profile reads one repository). + A profile can declare two repositories: the libretro port in ``source`` and the original emulator in ``upstream``. A path links into the one whose tree SOURCE_PATHS says holds it. Any other path is rendered as plain @@ -432,18 +450,18 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str: link to the wrong repository is a false citation. An address inside a shipped binary reads as one (offset 0x8094-0x80ac in libnes.so). """ - rendered_groups: list[str] = [] + groups: list[tuple[str, list[tuple[str, str]]]] = [] path_re = re.compile(r"^[A-Za-z0-9_.@/+~-]+$") for label, refs in source_ref_values(value): candidates = _forge_sources(profile, label) by_role = role_sources(profile, label) - rendered: list[str] = [] + rendered: list[tuple[str, str]] = [] current_role = "" binary = "" for part in split_source_ref(refs): if is_binary_citation(part.path): text, binary = _binary_citation(part.path, binary) - rendered.append(text) + rendered.append((text, "")) continue binary = "" role, bare = ref_role(part.path) @@ -473,22 +491,19 @@ def _source_ref_markdown(profile: dict, value: str | dict | None) -> str: else: selected = _holder(candidates, part.path) - tag = (f" ({role_label(profile, current_role, part.path)})" - if current_role and _two_repositories(profile) else "") + role = (role_label(profile, current_role, part.path) + if current_role and _two_repositories(profile) else "") if selected is None: - rendered.append(f"`{display}`{tag}") + rendered.append((f"`{display}`", role)) else: repo, pin = selected url = _source_permalink( repo, pin, real_path, part.start, part.end ) - rendered.append(f"[`{display}`]({url}){tag}") + rendered.append((f"[`{display}`]({url})", role)) - text = ", ".join(rendered) if rendered else f"`{refs}`" - if label: - text = f"**{label}:** {text}" - rendered_groups.append(text) - return ", ".join(rendered_groups) + groups.append((label, rendered or [(f"`{refs}`", "")])) + return groups def _binary_citation(path: str, previous: str) -> tuple[str, str]: From 20dc488e41a806b97f442e4f55bf1a8f2442550e Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:44:05 +0100 Subject: [PATCH 2/3] feat: list platform files as table rows Each file of a platform list is one row: name, folder, status chip, a short hash on a button that copies it whole, and a details cell. The details hold every declared hash, the size, the served copy and the readers. A reference's role is one word (glue, port, upstream, build) defined once in a legend instead of a phrase after every reference. Visible words: Batocera 2,433 to 2,332, AmberELEC 1,301 to 1,299, RomM 1,607 to 1,596, RetroArch 2,207 to 2,088, RetroDECK arcade part 2 19,127 to 2,104. Rows weigh more Markdown, so 38 more systems move to pages of their own under the existing budgets. --- scripts/generate_site.py | 273 ++++++++++++++++++++-------- tests/test_site_exports.py | 4 +- tests/test_site_platform_density.py | 244 +++++++++++++++++++++++++ tests/test_site_validation.py | 2 +- website/src/scripts/site.ts | 34 +++- website/src/styles/retrobios.css | 61 +++++++ wiki/testing-guide.md | 2 +- 7 files changed, 535 insertions(+), 85 deletions(-) create mode 100644 tests/test_site_platform_density.py diff --git a/scripts/generate_site.py b/scripts/generate_site.py index dd04f5697..0174f9035 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3253,13 +3253,16 @@ PLATFORM_REFERENCE_BUDGET = 250_000 def _file_runs(body: list[str], budget: int) -> list[list[str]]: """A system's file list in runs of at most ``budget`` characters. - The lines before the first file (counts, emulators) open every run. A - file is never cut from its hash and evidence lines. + The lines before the first file (counts, emulators, a table's header) + open every run. A file is never cut from its hash and evidence lines, + and a table row is one line. """ - first = next((i for i, line in enumerate(body) if line.startswith("- ")), len(body)) + rows = next((i + 1 for i, line in enumerate(body) if line.startswith("|--")), None) + first = rows if rows is not None else next( + (i for i, line in enumerate(body) if line.startswith("- ")), len(body)) head, items = body[:first], [] for line in body[first:]: - if line.startswith("- ") or not items: + if line.startswith(("- ", "| ")) or not items: items.append([line]) else: items[-1].append(line) @@ -3423,6 +3426,139 @@ def _launcher_refs(platform: str, config: dict) -> dict[str, list[str]]: return found +PLATFORM_FILE_HEADER = [ + "| File | Path | Status | Hash | Details |", + "|---|---|---|---|---|", +] +# The characters of a hash a row shows, as many as the served copy's sha1. +SHORT_HASH = 8 +# The hash a row shows first, when the list does not say which it checks. +_ROW_HASHES = ("sha1", "md5", "sha256", "crc32") +# An untested entry whose reason is a hash the bytes do not match. +_WRONG_HASH = re.compile(r"hash|mismatch|expected|must match", re.IGNORECASE) + + +def _status_chip(f: dict) -> str: + """The row's status as a chip, its word and its colour saying the same.""" + status = f.get("status") + if status == "ok": + word, tone = "OK", "success" + elif status == "missing": + word, tone = "Missing", "danger" + elif status == "untested": + word = "Wrong hash" if _WRONG_HASH.search(str(f.get("reason") or "")) else "Untested" + tone = "warning" + else: + word, tone = str(status), "muted" + return f'{word}' + + +def _copy_hash(algorithm: str, value: str | list) -> str: + """A declared hash cut to SHORT_HASH characters on a button that copies it whole. + A list of accepted values (Recalbox) shows its first and how many more.""" + from hashing import parse_hash_list # noqa: PLC0415 + + values = parse_hash_list(value) + if not values: + return "" + first = html.escape(values[0]) + more = f" +{len(values) - 1}" if len(values) > 1 else "" + return (f'{more}') + + +def _cell(text: str) -> str: + """Text for one cell of a Markdown table: a pipe would end the cell.""" + return text.replace("|", "\\|") + + +_MARKDOWN_PUNCTUATION = re.compile(r"([\\`*_\[\]~])") +_CODE_LINK = re.compile(r"\[`([^`]+)`\]\(") +_SHORT_SHA1 = re.compile(r"\(sha1 `([0-9a-f]+)`\)") + + +def _plain(text: str) -> str: + """Text written as itself in Markdown: no emphasis, link or tag.""" + return _MARKDOWN_PUNCTUATION.sub(r"\\\1", html.escape(text, quote=False)) + + +def _uncoded(text: str) -> str: + """A linked path as plain link text, a short sha1 as plain text. + + Each code element is one more element of the page. With one in every + name, path and reference, Batocera's page measured 10,479 elements, over + validate_site's budget of 10,000. The link and the value say the same + without it. + """ + text = _CODE_LINK.sub(lambda m: f"[{_plain(m[1])}](", text) + return _SHORT_SHA1.sub(r"(sha1 \1)", text) + + +def _platform_file_row(f: dict, cfg_entry: dict, hash_type: str = "", + launched: list[str] | None = None) -> str: + """One file of a platform's list as a table row: its name, where it goes, + its status and a short hash at a glance. The details hold every hash the + platform declares, the size, the copy that serves it and its readers. + """ + fname = f["name"] + hashes = { + "sha1": cfg_entry.get("sha1", f.get("sha1", "")), + "md5": cfg_entry.get("md5", f.get("expected_md5", "")), + # The crc32 is what System.dat and clrmamepro print. RetroDECK + # declares sha256 too (dsifirmware.bin). + "crc32": str(cfg_entry.get("crc32") or ""), + "sha256": str(cfg_entry.get("sha256") or ""), + } + size = cfg_entry.get("size", f.get("size", 0)) + shown = next((alg for alg in (hash_type, *_ROW_HASHES) if alg in hashes and hashes[alg]), "") + + facts: list[str] = [] + # One code element per value: Recalbox lists accepted md5s joined by + # commas, and each is a value a reader compares a dump with. + facts.extend(f"{alg.upper()}: {_hash_spans(hashes[alg])}" + for alg in ("sha1", "md5", "crc32", "sha256") if hashes[alg]) + # The size closes the hash lines: a line of its own was one more + # element per row. + if size and facts: + facts[-1] += f" ({_fmt_size(size)})" + elif size: + facts.append(f"Size: {_fmt_size(size)}") + if f.get("status") == "untested" and f.get("reason"): + facts.append(f"Not confirmed: {f['reason']}") + # Not every entry is firmware: the platform may list a game's data or a + # file no core it runs opens. + kind = {"game_data": "Game data, not firmware", + "unread": "No core the platform runs opens it"}.get(f.get("category", "")) + if kind: + facts.append(kind) + served = _served_line(f) + if served: + facts.append(_uncoded(served)) + if f.get("category_evidence"): + facts.append(_uncoded(_category_evidence_md(f["category_evidence"], launched))) + + # Where the file goes: three GameCube IPL.bin differ only by GC/USA, + # GC/EUR and GC/JAP. A destination that ends with the name shows its + # folder only, the name being in the cell before it. Both are plain + # text set in the code font by the table's style: a code element in + # each cell was two more elements per row. + destination = str(cfg_entry.get("destination") or "") + if destination == fname or destination.endswith("/" + fname): + destination = destination[:len(destination) - len(fname)] + summary = f'Details' + details = (f'
{summary}{"
".join(facts)}
' + if facts else "") + cells = [ + _plain(fname), + _plain(destination), + _status_chip(f), + _copy_hash(shown, hashes[shown]) if shown else "", + details, + ] + return "| " + " | ".join(_cell(cell) for cell in cells) + " |" + + def _render_platform_systems( by_system: dict, config_files: dict, emulator_files: dict, platform: tuple[str, str] | None = None, moved: dict[str, str] | None = None, @@ -3457,81 +3593,20 @@ def _render_platform_systems( body.append(f"Emulators: {emu_links}") body.append("") - # File listing. The sort is stable, so same-named entries keep the - # order verify reported them in, which is the declaration order. + # One row per file, the evidence in its details. The sort is stable, + # so same-named entries keep the order verify reported them in, + # which is the declaration order. + body.extend(PLATFORM_FILE_HEADER) seen_names: dict[str, int] = {} for f in sorted(files, key=lambda x: x["name"]): - status = f["status"] fname = f["name"] declared = config_files.get((sys_id, fname), []) nth = seen_names.get(fname, 0) seen_names[fname] = nth + 1 cfg_entry = declared[nth] if nth < len(declared) else {} - sha1 = cfg_entry.get("sha1", f.get("sha1", "")) - md5 = cfg_entry.get("md5", f.get("expected_md5", "")) - size = cfg_entry.get("size", f.get("size", 0)) - - if status == "ok": - status_display = "OK" - elif status == "untested": - reason = f.get("reason", "") - status_display = f"untested: {reason}" if reason else "untested" - elif status == "missing": - status_display = "**missing**" - else: - status_display = status - # Not every entry is firmware: the platform may list a game's data - # or a file no core it runs opens. - kind = {"game_data": "game data", - "unread": "no core the platform runs opens it"}.get(f.get("category", "")) - if kind: - status_display += f", {kind}" - - size_str = _fmt_size(size) if size else "" - details = [status_display] - if size_str: - details.append(size_str) - - # Where the file goes, when its name alone does not say: three - # GameCube IPL.bin differ only by GC/USA, GC/EUR and GC/JAP. - destination = str(cfg_entry.get("destination") or "") - where = f" at `{destination}`" if destination and destination != fname else "" - body.append(f"- `{fname}`{where} - {', '.join(details)}") - item_line = len(body) - 1 - facts: list[str] = [] - # Every hash the platform declares, whole: the crc32 is what - # System.dat and clrmamepro print, and pasting it found nothing - # for a pinned file the collection holds under other bytes. - crc32 = str(cfg_entry.get("crc32") or "") - # RetroDECK declares sha256 too: dsifirmware.bin's was in the CSV - # exports only, and pasting it into the search found nothing. - sha256 = str(cfg_entry.get("sha256") or "") - if sha1 or md5 or crc32 or sha256: - hash_parts = [] - # One code span per value: Recalbox lists accepted md5s joined - # by commas, and the search indexed the joined run as a single - # word, so the second and later values were never found. - if sha1: - hash_parts.append(f"SHA1: {_hash_spans(sha1)}") - if md5: - hash_parts.append(f"MD5: {_hash_spans(md5)}") - if crc32: - hash_parts.append(f"CRC32: `{crc32}`") - if sha256: - hash_parts.append(f"SHA256: `{sha256}`") - facts.append(" | ".join(hash_parts)) - served = _served_line(f) - if served: - facts.append(served) - if f.get("category_evidence"): - facts.append(_category_evidence_md( - f["category_evidence"], launched.get(fname.rsplit("/", 1)[-1].casefold()))) - # One line each, hashes, the copy that serves it, its readers: as - # continuation lines they ran into one paragraph. - if facts: - body[item_line] += "\\" - body.extend(f" {fact}" + ("\\" if n < len(facts) - 1 else "") - for n, fact in enumerate(facts)) + body.append(_platform_file_row( + f, cfg_entry, str((config or {}).get("hash_type") or ""), + launched.get(fname.rsplit("/", 1)[-1].casefold()))) # Measured as the Markdown the split was tuned on, a collapsible # block indented by four, so both engines split the same systems. @@ -3591,15 +3666,18 @@ def _render_platform_systems( *([""] + [", ".join( f"[part {i}]({other}.md)" if other != stem else f"part {i}" for i, other in enumerate(stems, 1))] if len(runs) > 1 else []), - "", "## Files", "", + "", "## Files", "", *markup.details("note", ROLE_LEGEND_TITLE, [ROLE_LEGEND]), *search_excluded(run), "", _date_line(), ]) + "\n" - where = (f"[a page of their own]({stems[0]}.md)" if len(stems) == 1 else - "pages of their own: " + ", ".join( - f"[part {i}]({stem}.md)" for i, stem in enumerate(stems, 1))) + # The count and status a folded system shows in its summary, + # linked to the page that lists the files. + tag = "OK" if ok_count == total else _count(total - ok_count, "issue") + count = f"{_count(total, 'file')} ({tag})" lines.extend([ - f"{_count(total, 'file')}, {ok_count} verified: listed on {where}.", "", + f"[{count}]({stems[0]}.md)" if len(stems) == 1 else f"{count}: " + ", ".join( + f"[part {i}]({stem}.md)" for i, stem in enumerate(stems, 1)), + "", ]) continue # The section's heading stays searchable, its file list with every @@ -3635,6 +3713,40 @@ def _site_profiles() -> dict[str, dict]: return load_emulator_profiles(str(_REPO_ROOT / "emulators")) +# Whose code a source reference cites, in one word: a platform page names +# each role once in ROLE_LEGEND. Written after every reference, "(emulator +# code, libretro port)" ran through Batocera's 14,000 words. +ROLE_WORDS = { + "libretro glue": "glue", + "emulator code, libretro port": "port", + "upstream emulator": "upstream", + "this build": "build", +} +ROLE_LEGEND_TITLE = "Source reference words" +ROLE_LEGEND = ( + "In a file's details, a word before its source references says whose code they are. " + "**glue** means libretro glue, **port** means emulator code in the libretro port, " + "**upstream** means the upstream emulator and **build** means this build's own code." +) + + +def _refs_by_role(profile: dict, value: str | dict | None) -> str: + """A source_ref's parts grouped by whose code they cite, each role named + once by its ROLE_WORDS word: `glue a.c:1, b.c:2 and port c.c:3`.""" + from sitesources import source_ref_items + + texts = [] + for label, items in source_ref_items(profile, value): + by_role: dict[str, list[str]] = {} + for part, role in items: + by_role.setdefault(role, []).append(part) + text = " and ".join( + (f"{ROLE_WORDS.get(role, role)} " if role else "") + ", ".join(parts) + for role, parts in by_role.items()) + texts.append(f"**{label}:** {text}" if label else text) + return ", ".join(texts) + + def _category_evidence_md(evidence: dict, launched: list[str] | None = None) -> str: """What establishes an entry's category, as the page shows it. @@ -3650,7 +3762,7 @@ def _category_evidence_md(evidence: dict, launched: list[str] | None = None) -> # plain code with the role word ("source") left in front. profile = profiles.get(reader["emulator"], {}) refs = [ - (f"`{e['name']}` at {_source_ref_markdown(profile, e['source_ref'])}" + (f"`{e['name']}` at {_refs_by_role(profile, e['source_ref'])}" if e.get("source_ref") else f"`{e['name']}`") + (" (by name)" if e.get("by") == "name" else "") for e in reader.get("entries", []) @@ -4173,8 +4285,7 @@ def generate_platform_page( "", level_label("reference"), "", - "For checking the work: the counts, where the list comes from, and " - "every file with its hashes.", + "For checking the work: counts, the list's source and each file's hashes.", "", *(line for note in (GUIDES[name].reference_notes if name in GUIDES else ()) for line in (note, "")), @@ -4305,6 +4416,8 @@ def generate_platform_page( f"| {ok_count}/{total} | {status} | {emu_str} |" ) lines.append("") + if by_system: + lines.extend(markup.details("note", ROLE_LEGEND_TITLE, [ROLE_LEGEND])) lines.extend( _render_platform_systems(by_system, config_files, emulator_files, diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index efc0490b9..80db75abb 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -340,7 +340,9 @@ class APlatformPageSaysWhereEachFileGoes(unittest.TestCase): lines = generate_site._render_platform_systems( {"nintendo-gamecube": files}, {("nintendo-gamecube", "IPL.bin"): declared}, {}) for region in regions: - self.assertIn(f"- `IPL.bin` at `dolphin-emu/Sys/GC/{region}/IPL.bin` - OK", + # The folder: the name is in the cell before it. + self.assertIn(f"| IPL.bin | dolphin-emu/Sys/GC/{region}/ | " + 'OK |', "\n".join(lines)) diff --git a/tests/test_site_platform_density.py b/tests/test_site_platform_density.py new file mode 100644 index 000000000..5585319de --- /dev/null +++ b/tests/test_site_platform_density.py @@ -0,0 +1,244 @@ +"""A platform page lists one table row per file, the evidence on demand. + +Batocera, AmberELEC and RomM ran to 14,000 to 16,000 words: every file +spelled its full MD5, the copy serving it and each source reference of +each reader, with "(emulator code, libretro port)" after every one. A row +now shows the name, the folder, the status and a short hash; its details +hold every hash, the served copy and the readers, the role of each +reference named by one word that a legend defines once. +""" + +from __future__ import annotations + +import re +import sys +import unittest +from html.parser import HTMLParser +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site # noqa: E402 + +_SKIPPED = frozenset({"script", "style", "template"}) +_VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", + "link", "meta", "source", "track", "wbr"}) + + +class _Words(HTMLParser): + """Words in
: all of them, and those a reader sees with every + details block closed (a closed block shows its summary only).""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.stack: list[tuple[str, bool]] = [] + self.visible = 0 + self.total = 0 + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag not in _VOID: + closed = tag == "details" and "open" not in dict(attrs) + self.stack.append((tag, closed)) + + def handle_endtag(self, tag: str) -> None: + for index in range(len(self.stack) - 1, -1, -1): + if self.stack[index][0] == tag: + del self.stack[index:] + return + + def _counted(self) -> bool: + names = {name for name, _ in self.stack} + return "main" in names and not names & _SKIPPED + + def _shown(self) -> bool: + for index, (_name, closed) in enumerate(self.stack): + if closed: + rest = self.stack[index + 1:] + if not rest or rest[0][0] != "summary": + return False + return True + + def handle_data(self, data: str) -> None: + if not self._counted(): + return + count = len(data.split()) + self.total += count + if self._shown(): + self.visible += count + + +def page_words(html: str) -> tuple[int, int]: + """(visible words, all words) in a page's
.""" + parser = _Words() + parser.feed(html) + return parser.visible, parser.total + + +class VisibleWordsAreCounted(unittest.TestCase): + def test_a_closed_block_shows_its_summary_only(self): + page = ("

one two

threefour five" + "
sixseven
" + "
eightnine
" + "

outside

") + self.assertEqual(page_words(page), (5, 9)) + + +class ARowPerFile(unittest.TestCase): + def _row(self, f: dict, entry: dict, hash_type: str = "md5") -> str: + return generate_site._platform_file_row(f, entry, hash_type) + + def test_the_row_at_a_glance(self): + md5, sha1 = "a" * 32, "b" * 40 + row = self._row({"name": "bios_a.bin", "status": "ok", "size": 2048}, + {"name": "bios_a.bin", "md5": md5, "sha1": sha1, "crc32": "0badf00d", + "destination": "bios/bios_a.bin"}) + cells = re.split(r"(?OK') + # The hash the platform checks, short, on a button that copies it whole. + self.assertIn(f'data-copy="{md5}"', short) + self.assertIn(f">{md5[:8]}", short) + # The accessible name holds the visible text (WCAG 2.5.3). + self.assertIn(f'aria-label="Copy MD5 {md5[:8]}"', short) + # Every hash whole in the details, with the size. + for value in (md5, sha1, "0badf00d"): + self.assertIn(value, details) + self.assertIn("2.0 KB", details) + self.assertTrue(details.startswith('
' + 'Details')) + + def test_status_words(self): + cases = { + ("missing", ""): ("Missing", "danger"), + ("untested", "declared hash contradicted by the local file"): ("Wrong hash", "warning"), + ("untested", "expected 1234... got 5678..."): ("Wrong hash", "warning"), + ("untested", "bios/x read error"): ("Untested", "warning"), + } + for (status, reason), (word, tone) in cases.items(): + row = self._row({"name": "x", "status": status, "reason": reason}, {"md5": "c" * 32}) + self.assertIn(f'{word}', row) + if reason: + self.assertIn(f"Not confirmed: {reason}", row) + + def test_accepted_values_all_in_the_details(self): + values = ["1" * 32, "2" * 32, "3" * 32] + row = self._row({"name": "x", "status": "ok"}, {"md5": ",".join(values)}) + self.assertIn(f'data-copy="{values[0]}"', row) + self.assertIn(" +2", row) + for value in values: + self.assertIn(f'{value}', row) + + def test_a_destination_under_another_name_stays_whole(self): + row = self._row({"name": "a.bin", "status": "ok"}, {"destination": "x/b.bin"}) + self.assertIn("| a.bin | x/b.bin |", row) + + def test_a_pipe_does_not_end_the_cell(self): + row = self._row({"name": "a|b", "status": "ok"}, {}) + self.assertIn(r"a\|b", row) + self.assertEqual(len(re.split(r"(? str: + return re.sub(r"\\([\\`*_\[\]~|])", r"\1", text) + + +@unittest.skipUnless((REPO_ROOT / "database.json").is_file(), "database.json absent") +class NothingIsLost(unittest.TestCase): + """Every fact a file row carried is on the page or its parts: each + declared hash whole, the served copy, every reader's references.""" + + @classmethod + def setUpClass(cls) -> None: + from common import load_database + from generate_readme import compute_coverage + + db = load_database(str(REPO_ROOT / "database.json")) + cls.pages = {} + for name in ("batocera", "retroarch"): + cov = compute_coverage(name, str(REPO_ROOT / "platforms"), db) + moved: dict[str, str] = {} + page = generate_site.generate_platform_page(name, cov, {}, {}, moved) + cls.pages[name] = (cov, page, moved) + + def test_every_fact_is_there(self): + for name, (cov, page, moved) in self.pages.items(): + text = _unescaped("\n".join([page, *moved.values()])) + systems = {d.get("system") for d in cov["details"]} + for sys_id, system in cov["config"].get("systems", {}).items(): + if sys_id not in systems: + continue + for entry in system.get("files", []): + for algorithm in ("sha1", "md5", "crc32", "sha256"): + for value in re.split(r"[,\s]+", str(entry.get(algorithm) or "")): + if value: + self.assertIn(value, text, (name, entry.get("name"), algorithm)) + for detail in cov["details"]: + self.assertIn(detail["name"], text) + if detail.get("path") and detail.get("status") != "missing": + self.assertIn(detail["path"], text, (name, detail["name"])) + readers = (detail.get("category_evidence") or {}).get("read_by", []) + # Cores of other systems that share the name are named, not + # cited, when a core of this system reads the file. + running = [reader for reader in readers if reader.get("runs_system")] + for reader in running or readers: + for url in re.findall(r"\]\((https://[^)]+)\)", generate_site._category_evidence_md( + {"read_by": [reader]})): + self.assertIn(url, text, (name, detail["name"], reader["emulator"])) + + def test_the_role_is_a_word_defined_once(self): + for name, (_cov, page, moved) in self.pages.items(): + for text in (page, *moved.values()): + self.assertNotIn("(emulator code, libretro port)", text, name) + self.assertEqual(text.count(generate_site.ROLE_LEGEND), 1, name) + + +# Visible words (outside closed details) measured on the built site, +# 2026-10-27. A ceiling is the measured value plus 5 %, never above the +# value before the rows were tables (Batocera 2,433, AmberELEC 1,301, RomM +# 1,607, RetroArch 2,207, the largest main page 2,459, the largest part +# 19,127: RetroDECK's arcade list). +PAGE_CEILINGS = {"batocera": 2433, "amberelec": 1301, "romm": 1607, "retroarch": 2192} +MAIN_PAGE_CEILING = 2459 +PART_PAGE_CEILING = 2306 + +_SITE = REPO_ROOT / "website" / "dist" / "platforms" + + +@unittest.skipUnless(_SITE.is_dir(), "website/dist not built") +class VisibleWordsStayUnderTheirCeiling(unittest.TestCase): + def test_platform_pages(self): + from common import list_registered_platforms + + mains = set(list_registered_platforms(str(REPO_ROOT / "platforms"), include_archived=True)) + for page in sorted(_SITE.glob("*/index.html")): + name = page.parent.name + visible, _total = page_words(page.read_text(encoding="utf-8")) + ceiling = PAGE_CEILINGS.get(name, MAIN_PAGE_CEILING if name in mains else PART_PAGE_CEILING) + self.assertLessEqual(visible, ceiling, name) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py index ad953d906..35acb41e7 100644 --- a/tests/test_site_validation.py +++ b/tests/test_site_validation.py @@ -377,7 +377,7 @@ class PlatformPagesPrintTheDeclaredCrc32(unittest.TestCase): path.read_text(encoding="utf-8") for path in sitepages.pages() if sitepages.name(path) == "platforms/retroarch.md" or sitepages.name(path).startswith("platforms/retroarch-")) - self.assertIn("CRC32: `91e93107`", text) + self.assertIn('CRC32: 91e93107', text) class PlainHouseStyle(unittest.TestCase): diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 3c7315a1a..69d46c288 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -1,8 +1,8 @@ // Progressive enhancement of the generated pages: table captions, filters // and keyboard scrolling, the hash lookup on files/ (a pasted hash or a file // hashed on this device), moved cards, a hash typed in the search, glossary -// terms on a tap, the OS tab of the visitor's system, the current page in -// the sidebar, and the 404 page's hints. +// terms on a tap, hash copy buttons, the OS tab of the visitor's system, the +// current page in the sidebar, and the 404 page's hints. // Which tables stack on a phone is decided when the page is built // (src/plugins/rehype-stack-tables.mjs), so the layout is right on first paint. @@ -931,6 +931,35 @@ const glossaryTerms = () => { document.addEventListener('click', close); }; +// A hash cut short on a button (button.rb-copy): a click copies the whole +// value from data-copy, says so to a screen reader, and marks the button +// for two seconds. The full value is also in the row's details, so a +// browser without the clipboard API loses nothing. +const copyButtons = () => { + if (!document.querySelector('button.rb-copy')) return; + let status: HTMLElement | null = null; + document.addEventListener('click', async (event) => { + const button = (event.target as Element | null)?.closest('button.rb-copy'); + const value = button?.dataset.copy; + if (!button || !value || !navigator.clipboard) return; + try { + await navigator.clipboard.writeText(value); + } catch { + // Clipboard refused (no permission, insecure context): nothing copied. + return; + } + if (!status) { + status = document.createElement('p'); + status.className = 'rb-visually-hidden'; + status.setAttribute('role', 'status'); + document.body.append(status); + } + status.textContent = `Copied ${value}`; + button.dataset.copied = ''; + setTimeout(() => delete button.dataset.copied, 2000); + }); +}; + // The 404 page: a search button, and the list a moved page belongs to. const notFound = () => { document.querySelector('[data-rb-open-search]')?.addEventListener('click', () => { @@ -952,6 +981,7 @@ enhanceTables(); breakPaths(); hashSearch(); glossaryTerms(); +copyButtons(); osTabs(); showCurrentPage(); notFound(); diff --git a/website/src/styles/retrobios.css b/website/src/styles/retrobios.css index 77831fc28..f10b70d3b 100644 --- a/website/src/styles/retrobios.css +++ b/website/src/styles/retrobios.css @@ -551,6 +551,67 @@ body:has(.rb-sys-file, .rb-file-entry) { --sl-content-width: var(--rb-grid-width .sl-markdown-content details.rb-details > summary::before { background-color: var(--rb-text-secondary); } .sl-markdown-content details.rb-details[open] > summary { margin-bottom: 0.6rem; } +/* A platform file row: a short hash on a button that copies it whole + (site.ts), and the row's evidence in a details cell. */ +.sl-markdown-content button.rb-copy { + display: inline-flex; + align-items: center; + gap: 0.3em; + min-height: 24px; + padding: 0 0.35em; + border: 1px solid var(--rb-border-strong); + border-radius: 3px; + background: var(--rb-surface); + color: var(--sl-color-white); + font: 0.8125rem/1.4 var(--sl-font-mono); + cursor: pointer; + white-space: nowrap; +} +.sl-markdown-content button.rb-copy::after { + content: ''; + width: 0.9em; + height: 0.9em; + background: var(--rb-text-secondary); + mask: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath d='M5 1h8a2 2 0 0 1 2 2v8h-2V3H5zM3 5h7a2 2 0 0 1 2 2v7a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V7a2 2 0 0 1 2-2zm0 2v7h7V7z'/%3E%3C/svg%3E") center / contain no-repeat; +} +.sl-markdown-content button.rb-copy[data-copied]::after { + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath d='M6 11.2 2.8 8 1.4 9.4 6 14l8.6-8.6-1.4-1.4z'/%3E%3C/svg%3E"); + background: var(--rb-ok-fg); +} +.sl-markdown-content button.rb-copy:hover { border-color: var(--rb-link); } +/* The file name and its destination, written as plain text to spare an + element per cell. */ +.sl-markdown-content table:has(:is(button.rb-copy, details.rb-row-details)) td:is(:nth-child(1), :nth-child(2)) { + font-family: var(--sl-font-mono); + font-size: 0.8125rem; + overflow-wrap: anywhere; +} +.sl-markdown-content details.rb-row-details { + margin: 0; + padding: 0; + border: 0; +} +.sl-markdown-content details.rb-row-details > summary { + min-height: 24px; + color: var(--rb-link); + font-size: 0.875rem; + cursor: pointer; +} +.sl-markdown-content details.rb-row-details[open] { + font-size: 0.8125rem; + line-height: 1.5; + overflow-wrap: anywhere; +} +/* Open beside four narrow columns, the evidence keeps a readable width: + the other cells give up their side padding for it. */ +@media (min-width: 50rem) { + .sl-markdown-content table:has(details.rb-row-details) :is(th, td) { padding-inline: 0.5rem; } + /* A long name or path wraps before the table outgrows the column. */ + .sl-markdown-content table:has(details.rb-row-details) td:first-child { min-width: 9rem; } + .sl-markdown-content table:has(details.rb-row-details) td:nth-child(2) { min-width: 4rem; } + .sl-markdown-content details.rb-row-details[open] { min-width: 15rem; } +} + /* Asides: a neutral box for notes and tips, no side stripe, no icon. Caution and danger keep their colour, as a 1 px border all round. */ .sl-markdown-content .starlight-aside { diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 84c1bd8cf..8265bc62e 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -61,7 +61,7 @@ Every module in `tests/`, by the area it guards. | Real-condition campaign | synthetic profiles, records and content folders, a fake host for load, disk, clock and commands, and whole passes against a libretro core built from C when a compiler and strace exist | `test_campaign` | | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture From e7d33caa3f45e3cb1712eaa9da134e5a5e831b89 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:41:00 +0100 Subject: [PATCH 3/3] feat: platform file rows that open full width Each file is a details whose summary is the row: name, folder when it is not the bare name, status chip. The short hash sits on one copy button beside it, since a summary is itself a control. Opened, the evidence runs across the list. A list with no folder drops the column. Code spans in the evidence are plain text. The lists split by the share of a page they render to (elements against 10,000, bytes against 600,000) instead of Markdown length. With its parts, Batocera goes from 12 pages, 18,854 elements and 1.21 MB to 10 pages, 13,671 and 1.11 MB. A part that folds back into its platform page redirects there, from the record in website/src/data/platform-parts.json. --- scripts/generate_site.py | 270 ++++++++++++++++----------- scripts/sitestarlight.py | 14 +- tests/test_site_exports.py | 6 +- tests/test_site_platform_density.py | 110 ++++++++--- tests/test_site_r2_reference.py | 17 +- tests/test_site_r3_reference.py | 2 +- tests/test_site_weight.py | 2 +- website/src/data/platform-parts.json | 82 ++++++++ website/src/scripts/site.ts | 33 ++++ website/src/styles/retrobios.css | 96 ++++++---- 10 files changed, 440 insertions(+), 192 deletions(-) create mode 100644 website/src/data/platform-parts.json diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 0174f9035..540f64ed5 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3238,44 +3238,35 @@ def with_setup_table(page: str, coverages: dict, registry: dict | None = None) - return f"{page.rstrip()}\n\n{table}" -# A platform system whose file list passes this much Markdown gets a page -# of its own: RetroDECK's arcade list was 750 KB of a 1.1 MB page. -PLATFORM_SECTION_BUDGET = 200_000 -# A system's own page in runs of at most this much Markdown: RetroDECK's -# arcade list, each file with its served copy and pinned readers, rendered -# 1.4 MB of HTML on one page. -PLATFORM_PART_BUDGET = 350_000 -# The Markdown of a platform page's file lists, past which the largest +# A platform's file lists are split by the share of a page they render to, +# in thousandths: their elements against validate_site's MAX_PAGE_ELEMENTS, +# their bytes against the 600,000 of test_site_r2_gate, whichever is +# larger. Measured in Markdown characters, the lists split anew whenever +# their markup changed: rows written as tables moved 38 more systems to +# pages of their own. +PAGE_ELEMENTS = 10_000 +PAGE_BYTES = 600_000 +# A system whose list passes this share gets a page of its own. +PLATFORM_SECTION_BUDGET = 400 +# A system's own page in runs of at most this share. +PLATFORM_PART_BUDGET = 700 +# The share of a platform page's file lists, past which the largest # systems move to pages of their own. -PLATFORM_REFERENCE_BUDGET = 250_000 +PLATFORM_REFERENCE_BUDGET = 600 -def _file_runs(body: list[str], budget: int) -> list[list[str]]: - """A system's file list in runs of at most ``budget`` characters. +# Every part page a platform list has had, with the system it held: +# platforms/batocera-atari-st.md -> platforms/batocera.md#atari-st. When a +# budget or a list changes, a part can fold back into its platform page, +# and its address then redirects there instead of ending on the 404 page. +PLATFORM_PARTS_FILE = (Path(__file__).resolve().parent.parent + / "website" / "src" / "data" / "platform-parts.json") - The lines before the first file (counts, emulators, a table's header) - open every run. A file is never cut from its hash and evidence lines, - and a table row is one line. - """ - rows = next((i + 1 for i, line in enumerate(body) if line.startswith("|--")), None) - first = rows if rows is not None else next( - (i for i, line in enumerate(body) if line.startswith("- ")), len(body)) - head, items = body[:first], [] - for line in body[first:]: - if line.startswith(("- ", "| ")) or not items: - items.append([line]) - else: - items[-1].append(line) - runs: list[list[str]] = [[]] - size = 0 - for item in items: - weight = sum(len(line) + 1 for line in item) - if runs[-1] and size + weight > budget: - runs.append([]) - size = 0 - runs[-1].extend(item) - size += weight - return [head + run for run in runs] + +def _published_platform_parts() -> dict[str, str]: + if not PLATFORM_PARTS_FILE.is_file(): + return {} + return json.loads(PLATFORM_PARTS_FILE.read_text(encoding="utf-8")) # A list moved off a reference page (gaps, provenance) is cut in parts of @@ -3283,13 +3274,18 @@ def _file_runs(body: list[str], budget: int) -> list[list[str]]: LIST_PART_BUDGET = 300_000 -def _blocks_in_runs(blocks: list[list[str]], budget: int) -> list[list[str]]: - """Blocks (table rows, folded lists) in runs of at most ``budget`` - characters, a block never cut.""" +def _characters(lines: list[str]) -> int: + return sum(len(line) + 1 for line in lines) + + +def _blocks_in_runs(blocks: list[list[str]], budget: int, + weigh: Callable[[list[str]], int] = _characters) -> list[list[str]]: + """Blocks (table rows, folded lists) in runs of at most ``budget``, by + default in characters, a block never cut.""" runs: list[list[str]] = [[]] size = 0 for block in blocks: - weight = sum(len(line) + 1 for line in block) + weight = weigh(block) if runs[-1] and size + weight > budget: runs.append([]) size = 0 @@ -3426,16 +3422,15 @@ def _launcher_refs(platform: str, config: dict) -> dict[str, list[str]]: return found -PLATFORM_FILE_HEADER = [ - "| File | Path | Status | Hash | Details |", - "|---|---|---|---|---|", -] # The characters of a hash a row shows, as many as the served copy's sha1. SHORT_HASH = 8 # The hash a row shows first, when the list does not say which it checks. _ROW_HASHES = ("sha1", "md5", "sha256", "crc32") # An untested entry whose reason is a hash the bytes do not match. _WRONG_HASH = re.compile(r"hash|mismatch|expected|must match", re.IGNORECASE) +# A list this long gets a filter box above it, as a table of RECORD_ROWS +# rows does (website/src/plugins/rehype-stack-tables.mjs). +FILTER_ROWS = 20 def _status_chip(f: dict) -> str: @@ -3450,12 +3445,13 @@ def _status_chip(f: dict) -> str: tone = "warning" else: word, tone = str(status), "muted" - return f'{word}' + return f'{html.escape(word)}' def _copy_hash(algorithm: str, value: str | list) -> str: - """A declared hash cut to SHORT_HASH characters on a button that copies it whole. - A list of accepted values (Recalbox) shows its first and how many more.""" + """A declared hash cut to SHORT_HASH characters on a button that copies + it whole. A list of accepted values (Recalbox) copies its first. Outside + a form a button submits nothing, so it carries no type.""" from hashing import parse_hash_list # noqa: PLC0415 values = parse_hash_list(value) @@ -3463,43 +3459,68 @@ def _copy_hash(algorithm: str, value: str | list) -> str: return "" first = html.escape(values[0]) more = f" +{len(values) - 1}" if len(values) > 1 else "" - return (f'{more}') + return (f'') -def _cell(text: str) -> str: - """Text for one cell of a Markdown table: a pipe would end the cell.""" - return text.replace("|", "\\|") +# Characters Markdown or MDX would read inside the raw HTML of a row: an +# MDX page parses the text of its HTML as Markdown, a .md page does not. +_INERT = str.maketrans({"*": "*", "_": "_", "`": "`", "[": "[", "]": "]"}) +_INLINE_MD = re.compile( + r"\[(?P