From 1e5e79506006d3a9e96ea0e7a83c0f585848913a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:49:52 +0200 Subject: [PATCH 1/2] feat: say what each emulator page needs first Romset row and line on MAME and FinalBurn pages, files an app ships under their own badge and section, collection counts under the table, empty or constant summary columns dropped, short hashes with the full value in the title, the profile card and cards past Required folded, data directory pages kept out of the search. --- scripts/generate_site.py | 277 +++++++++++++++++++++++--------- tests/test_audit_regressions.py | 7 +- tests/test_site_r2_emulators.py | 4 +- tests/test_site_r3_emulators.py | 197 +++++++++++++++++++++++ tests/test_site_weight.py | 2 +- wiki/testing-guide.md | 2 +- 6 files changed, 406 insertions(+), 83 deletions(-) create mode 100644 tests/test_site_r3_emulators.py diff --git a/scripts/generate_site.py b/scripts/generate_site.py index df4b4a93..681c7ea8 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4729,6 +4729,15 @@ def generate_emulators_index(profiles: dict) -> str: return "\n".join(lines) + "\n" +def comes_with_app(f: dict) -> bool: + """The emulator installs the file in every build the profile documents. + + ``bundled`` narrowed by ``mode`` is installed by one build only, and the + other build still reads it from the user's folder. + """ + return f.get("bundled") is True and not f.get("mode") + + def _file_badges(f: dict, in_repo: bool) -> list[str]: """The status chips shown beside a file name. @@ -4748,7 +4757,14 @@ def _file_badges(f: dict, in_repo: bool) -> list[str]: archive = f.get("archive", "") ftype = f.get("type", "") badges = [] - if required: + shipped = comes_with_app(f) + if shipped: + # NetherSX2's three interface fonts sat beside the BIOS as + # "required" and "missing": the app ships them, nobody fetches them. + badges.append( + 'comes with the app' + ) + elif required: badges.append( 'required' ) @@ -4756,18 +4772,18 @@ def _file_badges(f: dict, in_repo: bool) -> list[str]: badges.append( 'optional' ) - if in_repo: - badges.append( - 'in repo' - ) + # Whether the collection holds a file the app ships says nothing to + # the reader: no status chip. + if shipped: + status = "" + elif in_repo: + status = 'in repo' elif f.get("unsourceable"): - badges.append( - 'unsourceable' - ) + status = 'unsourceable' else: - badges.append( - 'missing' - ) + status = 'missing' + if status: + badges.append(status) if hle: badges.append( 'HLE fallback' @@ -4789,7 +4805,7 @@ def _file_badges(f: dict, in_repo: bool) -> list[str]: badges.append( f'{storage}' ) - if bundled: + if bundled and not shipped: # Said in words: the pack leaves the file out because the emulator, # in the build the entry names, already installs it. build = f" {f['mode']}" if f.get("mode") in ("standalone", "libretro") else "" @@ -5175,6 +5191,32 @@ def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]: f"([browse]({tree}))"] +ROMSET_FAQ = "../wiki/technical-faq.md#which-mame-version-do-the-arcade-bios-sets-match" +_MAME_RELEASE = re.compile(r"^0\.\d") + + +def emulator_romset(profile: dict) -> str: + """The romset an arcade emulator's sets must match, "MAME 0.78", or "". + + ``mame_version`` where the profile has one (MAME4droid 2024 is app + 1.38.1 on MAME 0.289), else the version of a MAME build or of a + FinalBurn core, whose sets follow its own release. + """ + if profile.get("type") in ("alias", "launcher", "test"): + return "" + emulator = str(profile.get("emulator", "")) + version = str(profile.get("core_version") or "") + if profile.get("mame_version"): + return f"MAME {profile['mame_version']}" + if "MAME" in emulator and _MAME_RELEASE.match(version): + return f"MAME {version}" + if emulator.startswith("FinalBurn Neo") and version: + return f"FinalBurn Neo {version}" + if (emulator.startswith("FB Alpha") or "FBA" in emulator) and version: + return f"FB Alpha {version}" + return "" + + def _render_metadata_rows(profile: dict) -> list[str]: """Scalar profile fields, one table row each. @@ -5182,6 +5224,9 @@ def _render_metadata_rows(profile: dict) -> list[str]: row: a dict rendered into a table cell is unreadable. """ lines: list[str] = [] + romset = emulator_romset(profile) + if romset: + lines.append(f"| Romset | [{romset}]({ROMSET_FAQ}) |") for field, label in [ ("core", "Core ID"), ("core_name", "Core name"), @@ -5203,7 +5248,12 @@ def _render_metadata_rows(profile: dict) -> list[str]: val = profile.get(field) if val is None or val == "" or isinstance(val, (dict, list)): continue - if field == "bios_size" and isinstance(val, int): + if field == "bios_directory" and profile.get("type") == "standalone" \ + and not str(val).startswith(("/", "~", "$")) and ":" not in str(val): + # A bare `bios/` read as a place on the device: it is a folder + # inside the app's own data folder, which the notes describe. + lines.append(f"| {label} | {_code(val)}, inside the app's own data folder |") + elif field == "bios_size" and isinstance(val, int): lines.append(f"| {label} | {_fmt_size(val)} ({val:,} bytes) |") elif isinstance(val, str) and val.startswith("http"): lines.append(f"| {label} | [{val}]({val}) |") @@ -5460,45 +5510,42 @@ def _availability_check( return _file_available -def _emulator_file_summary( - files: list[dict], available: Callable[[dict], bool] -) -> list[str]: - """The counts above an emulator's file table, and its categories. - - Counted by slot, as verify --emulator and the gaps page count: raze - lists every GRP its code accepts under one name, and holding one fills - the slot. Counted per entry, the page announced 46 missing where 12 - slots are empty. - """ +def _emulator_slots(files: list[dict]) -> list[list[dict]]: + """The entries of each slot, as verify --emulator and the gaps page + count them, leaving out the slots the app ships.""" from cross_reference import slot_key slots: dict[tuple, list[dict]] = {} for f in files: slots.setdefault(slot_key(f), []).append(f) - in_repo_count = sum(1 for group in slots.values() if any(available(f) for f in group)) - unsourceable_count = sum( - 1 for group in slots.values() - if not any(available(f) for f in group) and all(f.get("unsourceable") for f in group) - ) - missing_count = len(slots) - in_repo_count - unsourceable_count - req_count = sum(1 for group in slots.values() if any(f.get("required") for f in group)) - hle_count = sum(1 for group in slots.values() if any(f.get("hle_fallback") for f in group)) + return [group for group in slots.values() if not all(comes_with_app(f) for f in group)] + + +def _emulator_file_summary(files: list[dict]) -> list[str]: + """What the emulator reads, above its file table, and its categories. + + Counted by slot, as verify --emulator and the gaps page count: raze + lists every GRP its code accepts under one name, and holding one fills + the slot. Counted per entry, the page announced 46 missing where 12 + slots are empty. The files the app ships are counted apart: NetherSX2 + said "4 required" for one BIOS and three fonts in its package. + """ + slots = _emulator_slots(files) + shipped = sum(1 for f in files if comes_with_app(f)) + req_count = sum(1 for group in slots if any(f.get("required") for f in group)) + hle_count = sum(1 for group in slots if any(f.get("hle_fallback") for f in group)) # Sentences, one number format: the line read "1,563 files | 164 # required, 1399 optional | 1563 in repo". - held = [f"{in_repo_count:,} in repo", f"{missing_count:,} missing"] - if unsourceable_count: - held.append(f"{unsourceable_count:,} unsourceable") - if hle_count: - held.append(f"{hle_count:,} with HLE fallback") total = f"**{_count(len(slots), 'file')}**" - if len(slots) != len(files): - total += f" ({len(files):,} accepted versions)" - lines = [ - f"{total}: {req_count:,} required, {len(slots) - req_count:,} optional. " - f"{_said_as_list(held)}.", - "", - ] + if sum(len(group) for group in slots) != len(slots): + total += f" ({sum(len(group) for group in slots):,} accepted versions)" + line = f"{total}: {req_count:,} required, {len(slots) - req_count:,} optional." + if hle_count: + line += f" {hle_count:,} {'has' if hle_count == 1 else 'have'} an HLE fallback." + if shipped: + line += f" {_count(shipped, 'more file')} {'comes' if shipped == 1 else 'come'} with the app." + lines = [line, ""] categories = [f.get("category", "bios") for f in files] if "game_data" in categories or "bios_zip" in categories: @@ -5513,6 +5560,23 @@ def _emulator_file_summary( return lines +def _emulator_held_line(files: list[dict], available: Callable[[dict], bool]) -> list[str]: + """What this collection holds of them, one quiet line under the table: + a reader comes for what the emulator needs, the project's counts after.""" + slots = _emulator_slots(files) + if not slots: + return [] + in_repo = sum(1 for group in slots if any(available(f) for f in group)) + unsourceable = sum( + 1 for group in slots + if not any(available(f) for f in group) and all(f.get("unsourceable") for f in group) + ) + held = [f"{in_repo:,} in repo", f"{len(slots) - in_repo - unsourceable:,} missing"] + if unsourceable: + held.append(f"{unsourceable:,} unsourceable") + return [f"

In this collection: {_said_as_list(held)}.

", ""] + + def _said_as_list(items: list[str]) -> str: """"a, b and c".""" return ", ".join(items[:-1]) + f" and {items[-1]}" if len(items) > 1 else "".join(items) @@ -5567,8 +5631,18 @@ def _data_tree_page(ref: str, entry: dict, pages: list[tuple[str, list[str]]], part = f" (part {index + 1} of {len(pages)})" if len(pages) > 1 else "" readers = _said_as_list([f"[{_escape_tags(display)}]({key}.md#data-directories)" for key, display in _DATA_READERS.get(ref, [])]) + # A file list out of the search: "dolphin" found three parts of the + # dolphin-sys tree, on their title, before the Dolphin page. + front: dict = {"level": "reference", "search": "exclude"} + # The reader the tree is named after (dolphin for dolphin-sys), else + # the first by name. + readers = _DATA_READERS.get(ref) or [] + owner = max(readers, default=None, + key=lambda reader: len(reader[0]) if ref.startswith(f"{reader[0]}-") else 0) + if owner: + front["parent"] = {"label": owner[1], "link": f"{owner[0]}.md"} lines = [ - "---", "level: reference", "---", "", + "---", yaml.safe_dump(front, sort_keys=False, allow_unicode=True).rstrip("\n"), "---", "", f"# The {ref} data directory{part}", "", f"{entry.get('description') or 'A data directory'}: " f"{_count(len(members), 'file')}, as the packs ship them." @@ -5593,7 +5667,8 @@ EMULATOR_CARDS_PER_PAGE = 220 # The headings an emulator's cards go under, in page order: what it needs # first. DuckStation's three required BIOS were cards 12 to 14 under one # heading, and MAME opened on 25 floppy sound samples. -CARD_SECTIONS = (("required", "Required"), ("optional", "Optional"), ("game_data", "Game data")) +CARD_SECTIONS = (("required", "Required"), ("optional", "Optional"), ("game_data", "Game data"), + ("with_app", "Comes with the app")) _SECTION_RANK = {key: rank for rank, (key, _title) in enumerate(CARD_SECTIONS)} # Below this many cards the page is one screen of them, and a table above @@ -5602,6 +5677,8 @@ EMULATOR_SUMMARY_MIN = 6 def _card_section(f: dict) -> str: + if comes_with_app(f): + return "with_app" if f.get("category") == "game_data": return "game_data" return "required" if f.get("required") else "optional" @@ -5625,35 +5702,64 @@ def emulator_parts(name: str, files: list) -> list[tuple[str, list[int]]]: for index, start in enumerate(range(0, len(files), size))] -def _sectioned(files: list, indexes: list[int], render: Callable[[list[int]], list[str]]) -> list[str]: - """Each section's heading, then ``render`` of its entries among ``indexes``.""" +def _sectioned(files: list, indexes: list[int], render: Callable[[list[int]], list[str]], + fold: bool = False) -> list[str]: + """Each section's heading, then ``render`` of its entries among ``indexes``. + + ``fold``: every section after Required in one closed block, for a page + whose table above already lists each entry. + """ lines: list[str] = [] for key, title in CARD_SECTIONS: group = [i for i in indexes if _card_section(files[i]) == key] - if group: - lines.extend([f"## {title}", "", *render(group)]) + if not group: + continue + body = render(group) + if fold and key != "required": + body = markup.details("note", _count(len(group), "file"), body) + lines.extend([f"## {title}", "", *body]) return lines def _hash_short(f: dict) -> str: - """The shortest way to tell two dumps apart in a table: an md5 or crc32 prefix.""" + """The shortest way to tell two dumps apart in a table: the first 12 + characters of an md5, crc32 or sha1, the whole value in its title.""" for field in ("md5", "crc32", "sha1"): value = f.get(field) value = value[0] if isinstance(value, list) and value else value if value: - short = str(value).lower().split(",")[0].strip().removeprefix("0x")[:8] + full = html.escape(str(value).lower().split(",")[0].strip().removeprefix("0x")) # Kept on one line: the label above the value doubled each row. - return f"{field.upper()}\u00a0{_code(short)}" + return f'{field.upper()}\u00a0{full[:12]}' return "" -def _summary_row(f: dict, link: str, ids: str = "") -> str: - shown = str(f.get("path") or f.get("name", "")) - return (f"| {ids}[{_code(shown)}]({link}) | {_region_words(f.get('region'), 3) or '-'} " - f"| {'yes' if f.get('required') else 'no'} | {_hash_short(f) or '-'} |") +def _summary_table(files: list[dict], indexes: list[int], cell: Callable[[int, str], str]) -> list[str]: + """The table of an emulator's entries, one row each, ``cell`` writing + the linked name of entry i from its shown path. - -_SUMMARY_HEAD = ["| File | Region | Required | Hash |", "|------|--------|----------|------|"] + A column every row leaves empty is not written: AetherSX2's Region + column held "-" on every row and pushed Required off a phone screen. + Nor is Required when every row gives the same answer, which the + section heading or the count above already says: in MAME's index it + turned 1,564 rows into label and value pairs on a phone. + """ + rows = [] + for i in indexes: + f = files[i] + needed = "with the app" if comes_with_app(f) else "yes" if f.get("required") else "no" + rows.append((cell(i, str(f.get("path") or f.get("name", ""))), + _region_words(f.get("region"), 3), needed, _hash_short(f))) + keep = [0] + [column for column in (1, 3) if any(row[column] for row in rows)] + if len({row[2] for row in rows}) > 1: + keep.append(2) + keep.sort() + heads = ("File", "Region", "Required", "Hash") + return [ + "| " + " | ".join(heads[c] for c in keep) + " |", + "|" + "|".join("------" for _c in keep) + "|", + *("| " + " | ".join(row[c] or "-" for c in keep) + " |" for row in rows), + ] def plan_emulator_pages(profiles: dict) -> None: @@ -5697,7 +5803,8 @@ def _emulator_intro(emu_name: str, profile: dict) -> str: label = system_name(sys_id) if label not in names: names.append(label) - files = [f for f in profile.get("files") or [] if isinstance(f, dict)] + # What the app installs itself is no file for the reader to get. + files = [f for f in profile.get("files") or [] if isinstance(f, dict) and not comes_with_app(f)] required = [f for f in files if f.get("required") is True] sentences = [] if names: @@ -5797,10 +5904,19 @@ def emulator_pages( part_pages: dict[str, str] = {} intro = _emulator_intro(emu_name, profile) + romset = emulator_romset(profile) lines = [ f"# {emu_name}", "", *([intro, ""] if intro else []), + ] + if romset and any(isinstance(f, dict) and f.get("category") == "bios_zip" for f in files): + # The commonest arcade failure is a set from another MAME version, + # and its answer sat on a Reference page no arcade page linked. + lines.extend([f"Its BIOS ZIPs must come from the {romset} romset. " + f"[Which version each core reads]({ROMSET_FAQ}).", ""]) + card_start = len(lines) + lines += [ markup.block_open("div", 'class="rb-meta-card"'), "", "| Detail | Value |", @@ -5864,9 +5980,6 @@ def emulator_pages( if systems: sys_links = list(dict.fromkeys(_named_system_link(s, "../") for s in systems)) lines.append(f"| Systems | {', '.join(sys_links)} |") - mame_ver = profile.get("mame_version", "") - if mame_ver: - lines.append(f"| MAME version | {mame_ver} |") author = profile.get("author", "") if author: lines.append(f"| Author | {author} |") @@ -5877,6 +5990,9 @@ def emulator_pages( lines.append("") lines.append("") lines.append("") + # Reference for the few who check a revision: stacked on a phone, the + # card took two screens above the files. + lines[card_start:] = markup.details("note", "About this profile", lines[card_start:]) lines.extend(_install_block(name, profile)) @@ -5897,7 +6013,8 @@ def emulator_pages( _file_available = _availability_check(db, data_names) lines.extend(["## Files", ""]) - lines.extend(_emulator_file_summary(files, _file_available)) + lines.extend(_emulator_file_summary(files)) + held = _emulator_held_line(files, _file_available) anchors = emulator_file_anchors(files) aliases = emulator_file_aliases(files, anchors) @@ -5908,15 +6025,21 @@ def emulator_pages( parts = emulator_parts(name, files) if not parts: order = _display_order(files) - if len(files) >= EMULATOR_SUMMARY_MIN: - lines.extend(search_excluded([ - *_SUMMARY_HEAD, - *(_summary_row(files[i], f"#{anchors[i]}") for i in order), - ])) + indexed = len(files) >= EMULATOR_SUMMARY_MIN + if indexed: + lines.extend(search_excluded(_summary_table( + files, order, lambda i, shown: f"[{_code(shown)}](#{anchors[i]})"))) + lines.append("") + lines.extend(held) + # Below the table, which indexes them, the cards past the + # required ones fold: DuckStation's page was 59 phone screens, + # 102 optional cards after its three BIOS. lines.extend(_sectioned( - files, order, lambda group: [line for i in group for line in cards[i]])) + files, order, lambda group: [line for i in group for line in cards[i]], + fold=indexed)) else: lines.extend(_card_index(emu_name, files, anchors, aliases, parts)) + lines.extend(held) for number, (stem, entries) in enumerate(parts, 1): part_pages[stem] = _emulator_part_page( name, emu_name, number, parts, [cards[i] for i in entries], @@ -5996,16 +6119,18 @@ def _card_index( """ page_of = {index: stem for stem, entries in parts for index in entries} + def cell(i: int, shown: str) -> str: + # The id sits on the link itself, one element fewer per row: MAME's + # 1,564 rows put 13,300 elements on its page. + href = markup.site_href(f"emulators/{page_of[i]}/#{anchors[i]}") + alias = f'' if aliases[i] else "" + return (f'{alias}' + f"{html.escape(shown, quote=False)}") + def table(group: list[int]) -> list[str]: - rows = [ - _summary_row( - files[i], f"{page_of[i]}.md#{anchors[i]}", - "".join(f'' for a in (anchors[i], aliases[i]) if a)) - for i in group - ] return [ markup.block_open("section", f'class="rb-card-index" {markup.search_exclude()}'), - "", *_SUMMARY_HEAD, *rows, "", "", "", + "", *_summary_table(files, group, cell), "", "", "", ] return [ diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index f66bf9a8..80e23b07 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -3967,16 +3967,17 @@ class TheSiteCountsSlotsNotVersions(unittest.TestCase): 59 files, 46 missing, where verify --emulator said 23 slots, 12 missing.""" def test_one_held_version_fills_the_slot(self): - from scripts.generate_site import _emulator_file_summary + from scripts.generate_site import _emulator_file_summary, _emulator_held_line files = [ {"name": "DUKE3D.GRP", "crc32": "1"}, {"name": "DUKE3D.GRP", "crc32": "2"}, {"name": "BLOOD.RFF", "crc32": "3"}, ] - line = _emulator_file_summary(files, lambda f: f.get("crc32") == "2")[0] + line = _emulator_file_summary(files)[0] self.assertIn("**2 files** (3 accepted versions)", line) - self.assertIn("1 in repo and 1 missing", line) + held = _emulator_held_line(files, lambda f: f.get("crc32") == "2")[0] + self.assertIn("1 in repo and 1 missing", held) diff --git a/tests/test_site_r2_emulators.py b/tests/test_site_r2_emulators.py index 48957536..54fdf528 100644 --- a/tests/test_site_r2_emulators.py +++ b/tests/test_site_r2_emulators.py @@ -124,7 +124,7 @@ class RequiredFilesComeFirst(unittest.TestCase): row = next(line for line in page.splitlines() if "req.bin" in line and "| yes |" in line) self.assertIn("(#file-req-bin", row) self.assertIn("| Japan |", row) - self.assertIn("MD5\u00a0`abababab`", row) + self.assertIn(f'MD5\u00a0{"ab" * 6}', row) # The required row is the first one. table = page.index("| File | Region | Required | Hash |") self.assertLess(page.index("req.bin", table), page.index("f000.bin", table)) @@ -152,7 +152,7 @@ class TheSummaryLineIsProse(unittest.TestCase): {"name": "a.pak", "category": "game_data"}, {"name": "b.pak", "category": "game_data"}] for f in files[:164]: f["required"] = True - text = "\n".join(gs._emulator_file_summary(files, lambda _f: True)) + text = "\n".join(gs._emulator_file_summary(files)) self.assertNotIn(" | ", text) self.assertNotIn("1399", text) self.assertIn("1,401 optional", text) diff --git a/tests/test_site_r3_emulators.py b/tests/test_site_r3_emulators.py new file mode 100644 index 00000000..fb28497a --- /dev/null +++ b/tests/test_site_r3_emulators.py @@ -0,0 +1,197 @@ +"""Emulator pages say what to get, and fit (round 3, package 7). + +NetherSX2 counted three interface fonts its app ships among "4 required" +files and opened its file list on the project's counts. AetherSX2's summary +held an empty Region column that pushed Required off a phone. No MAME or +FinalBurn page said which romset its sets must match. DuckStation's page was +59 phone screens of optional cards, and MAME's index put 13,300 elements on +one page. +""" + +from __future__ import annotations + +import re +import sys +import tempfile +import unittest +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site as gs # noqa: E402 +import validate_site # noqa: E402 + +EMPTY_DB = {"files": {}, "indexes": {"by_name": {}}} +DIST = REPO_ROOT / "website" / "dist" + + +def _profile(name: str) -> dict: + return yaml.safe_load((REPO_ROOT / "emulators" / f"{name}.yml").read_text(encoding="utf-8")) + + +def _page(name: str, profile: dict | None = None) -> str: + return gs.generate_emulator_page(name, profile or _profile(name), EMPTY_DB, None, set()) + + +def _files(count: int, **fields: object) -> list[dict]: + return [{"name": f"f{i:03d}.bin", **fields} for i in range(count)] + + +class EachArcadePageNamesItsRomset(unittest.TestCase): + def test_the_romset_by_profile(self): + self.assertEqual(gs.emulator_romset(_profile("mame2003")), "MAME 0.78") + # The app's version is 1.38.1, its MAME is 0.289. + self.assertEqual(gs.emulator_romset(_profile("mame4droid-2024")), "MAME 0.289") + self.assertEqual(gs.emulator_romset(_profile("mame2003_plus")), "MAME 0.78 (plus backports)") + self.assertTrue(gs.emulator_romset(_profile("fbneo")).startswith("FinalBurn Neo ")) + self.assertTrue(gs.emulator_romset(_profile("fbalpha2012")).startswith("FB Alpha ")) + self.assertEqual(gs.emulator_romset(_profile("duckstation")), "") + # HBMAME publishes no MAME release number. + self.assertEqual(gs.emulator_romset(_profile("hbmame")), "") + + def test_a_row_and_a_line_under_the_intro(self): + page = _page("mame2003") + self.assertIn(f"| Romset | [MAME 0.78]({gs.ROMSET_FAQ}) |", page) + self.assertNotIn("| MAME version |", _page("mame2016")) + line = page.index("Its BIOS ZIPs must come from the MAME 0.78 romset.") + self.assertLess(line, page.index("About this profile")) + self.assertIn(f"[Which version each core reads]({gs.ROMSET_FAQ})", page) + + def test_the_faq_answer_exists(self): + faq = (REPO_ROOT / "wiki" / "technical-faq.md").read_text(encoding="utf-8") + self.assertIn("## Which MAME version do the arcade BIOS sets match?", faq) + + def test_no_line_without_bios_zips(self): + self.assertNotIn("Its BIOS ZIPs", _page("duckstation")) + + +class WhatTheAppShipsIsNoFileToGet(unittest.TestCase): + def setUp(self) -> None: + self.profile = _profile("nethersx2") + self.page = _page("nethersx2", self.profile) + + def test_the_badge(self): + font = next(f for f in self.profile["files"] if f["name"] == "Roboto-Regular.ttf") + badges = " ".join(gs._file_badges(font, False)) + self.assertIn(">comes with the app", badges) + self.assertNotIn(">required", badges) + self.assertNotIn(">missing", badges) + # Installed by one build only: the other still reads it from the folder. + dual = " ".join(gs._file_badges({"name": "a", "bundled": True, "mode": "standalone", + "required": True}, False)) + self.assertIn(">required", dual) + self.assertIn("installed with the standalone emulator", dual) + + def test_counts_leave_them_out(self): + shipped = sum(1 for f in self.profile["files"] if gs.comes_with_app(f)) + self.assertGreater(shipped, 0) + summary = self.page[self.page.index("## Files"):] + self.assertIn(f"{shipped} more files come with the app.", summary) + self.assertIn(": 1 required,", summary) + self.assertIn("It reads 8 files, 1 of them required", self.page) + + def test_their_own_section_after_the_others(self): + heads = re.findall(r"^## (.+)$", self.page, re.M) + self.assertEqual(heads[heads.index("Files") + 1:heads.index("Files") + 3], + ["Required", "Optional"]) + self.assertIn("Comes with the app", heads) + section = self.page[self.page.index("## Required"):self.page.index("## Optional")] + self.assertNotIn("Roboto-Regular.ttf", section) + + def test_the_collection_counts_go_under_the_table(self): + held = self.page.index("

In this collection:") + table = self.page.index("| File |") + self.assertLess(table, held) + self.assertLess(held, self.page.index("## Required")) + self.assertNotIn("in repo", self.page[self.page.index("## Files"):table]) + + def test_the_folder_is_the_apps_own(self): + self.assertIn("| BIOS directory | `bios/`, inside the app's own data folder |", self.page) + + +class TheSummaryTableFitsItsContent(unittest.TestCase): + def test_an_empty_region_column_goes(self): + page = _page("aethersx2") + table = next(line for line in page.splitlines() if line.startswith("| File |")) + self.assertNotIn("Region", table) + + def test_a_constant_required_column_goes(self): + rows = gs._summary_table(_files(3, required=True), [0, 1, 2], lambda _i, s: s) + self.assertEqual(rows[0], "| File |") + + def test_hashes_are_short_with_the_whole_value_in_the_title(self): + md5 = "0123456789abcdef" * 2 + cell = gs._hash_short({"md5": md5}) + self.assertEqual(cell, f'MD5 {md5[:12]}') + + def test_the_split_index_holds_each_id_on_its_link(self): + files = _files(gs.EMULATOR_CARDS_PER_PAGE + 1, category="bios_zip") + main = gs.emulator_pages("big", {"emulator": "Big", "type": "standalone", + "files": files}, {"files": {}})["big"] + anchor = gs.emulator_file_anchors(files)[0] + self.assertIn(f'', main) + self.assertNotIn(f'', main) + + +class LongPagesFold(unittest.TestCase): + def test_optional_cards_fold_below_the_table(self): + files = _files(gs.EMULATOR_SUMMARY_MIN) + [{"name": "req.bin", "required": True}] + page = _page("x", {"type": "standalone", "files": files}) + required = page[page.index("## Required"):page.index("## Optional")] + self.assertNotIn("{gs.EMULATOR_SUMMARY_MIN} files", optional) + + def test_a_few_cards_stay_open(self): + page = _page("x", {"type": "standalone", "files": _files(2)}) + self.assertNotIn("

2 files", page) + + def test_the_profile_card_folds(self): + page = _page("duckstation") + start = page.index("About this profile") + self.assertLess(start, page.index('
')) + self.assertLess(page.index("| Type |"), page.index("", start)) + + +class DataDirectoryPagesStayOutOfSearch(unittest.TestCase): + def test_front_matter(self): + saved = dict(gs._DATA_MEMBERS), dict(gs._DATA_READERS) + self.addCleanup(lambda: (gs._DATA_MEMBERS.clear(), gs._DATA_MEMBERS.update(saved[0]), + gs._DATA_READERS.clear(), gs._DATA_READERS.update(saved[1]))) + gs._DATA_MEMBERS["dolphin-sys"] = {"a.bin": {"sha1": "a" * 40, "size": 1}} + gs._DATA_READERS["dolphin-sys"] = [("dolphin-mmjr", "Dolphin MMJR"), ("dolphin", "Dolphin")] + page = gs._data_tree_page("dolphin-sys", {}, [("data-dolphin-sys", ["a.bin"])], 0) + front = yaml.safe_load(page.split("---\n")[1]) + self.assertEqual(front["search"], "exclude") + self.assertEqual(front["parent"], {"label": "Dolphin", "link": "dolphin.md"}) + + @unittest.skipUnless((DIST / "emulators" / "data-dolphin-sys").is_dir(), "site not built") + def test_the_built_page_is_not_indexed(self): + html = (DIST / "emulators" / "data-dolphin-sys" / "index.html").read_text(encoding="utf-8") + self.assertNotIn("data-pagefind-body", html) + + +class AnElementBudget(unittest.TestCase): + def test_validate_site_reports_a_crowded_page(self): + self.assertEqual(validate_site.MAX_PAGE_ELEMENTS, 10_000) + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + page = Path(tmp) / "crowded" / "index.html" + page.parent.mkdir() + page.write_text("" + "" * 10_001 + + "") + issues = validate_site.validate_site(Path(tmp)) + self.assertTrue(any("elements, over the 10,000 budget" in issue for issue in issues)) + + @unittest.skipUnless((DIST / "emulators" / "mame" / "index.html").is_file(), "site not built") + def test_mame_is_under_it(self): + page = validate_site._parse_page(DIST / "emulators" / "mame" / "index.html") + self.assertLess(page.elements, validate_site.MAX_PAGE_ELEMENTS) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_weight.py b/tests/test_site_weight.py index 745f1e16..90eef5d5 100644 --- a/tests/test_site_weight.py +++ b/tests/test_site_weight.py @@ -100,7 +100,7 @@ class AnEmulatorWithTooManyCardsIsSplit(unittest.TestCase): main = pages["big"] anchors = generate_site.emulator_file_anchors(files) for anchor in anchors: - self.assertIn(f'', main) + self.assertIn(f' Date: Fri, 23 Oct 2026 10:49:55 +0200 Subject: [PATCH 2/2] feat: cap a page at 10,000 elements --- scripts/validate_site.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/validate_site.py b/scripts/validate_site.py index 2328d0d3..c5205df0 100644 --- a/scripts/validate_site.py +++ b/scripts/validate_site.py @@ -41,6 +41,7 @@ class Page: empty_headers: int = 0 runtime_diagrams: int = 0 size: int = 0 + elements: int = 0 # A moved page's old address: a meta refresh to the new one, no content. redirect: str = "" @@ -52,6 +53,15 @@ class Page: # The largest page was 10.5 MB (systems/game-engines/) and took 21 s of # main-thread work on a throttled phone. Split pages now top out at 0.9 MB. MAX_PAGE_BYTES = 1_000_000 +# Elements outside scripts and styles: the browser lays out and styles each. +# emulators/mame held 13,300 and blocked the main thread for 235 ms. Its +# index now carries each card's id on the link itself. +MAX_PAGE_ELEMENTS = 10_000 +# Pages allowed past it, each with the reason, removed when it comes under. +OVER_ELEMENT_BUDGET = { + # One table of every file to source, 10,193 elements on 2026-10-23. + "gaps-files-to-source/index.html": "the file table is not split", +} # Pagefind loads its index in pieces: the metadata, then the chunks the words # typed fall in (stored compressed). 59 KB for the largest chunk and 7.5 KB of # metadata on 2026-10-22, 5.2 MB for the whole index on disk. @@ -77,6 +87,8 @@ class _PageParser(HTMLParser): def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: data = {key: value or "" for key, value in attrs} + if tag not in ("script", "style"): + self.page.elements += 1 if tag in _LITERAL_TAGS: self._literal_depth += 1 if re.fullmatch(r"h[1-6]", tag): @@ -282,6 +294,9 @@ def _check_pages( report(path, f"{page.runtime_diagrams} diagram(s) drawn by a script at runtime") if page.size > MAX_PAGE_BYTES: report(path, f"{page.size:,} bytes of HTML, over the {MAX_PAGE_BYTES:,} budget") + if page.elements > MAX_PAGE_ELEMENTS \ + and path.relative_to(site).as_posix() not in OVER_ELEMENT_BUDGET: + report(path, f"{page.elements:,} elements, over the {MAX_PAGE_ELEMENTS:,} budget") jumps = _heading_jumps(page.headings) if jumps: report(path, f"heading levels skipped: {sorted(set(jumps))}")