chore: merge platform file rows

This commit is contained in:
Abdessamad Derraz committed 2026-10-27 21:46:44 +01:00
commit 917ab7c94b
13 files changed
+867 -159

No files matched your search

+292 -125
View File
@@ -3242,41 +3242,35 @@ def with_setup_table(page: str, coverages: dict, registry: dict | None = None) -
return f"{page.rstrip()}\n\n{table}"
# A platform system whose file list passes this much Markdown gets a page
# of its own: RetroDECK's arcade list was 750 KB of a 1.1 MB page.
PLATFORM_SECTION_BUDGET = 200_000
# A system's own page in runs of at most this much Markdown: RetroDECK's
# arcade list, each file with its served copy and pinned readers, rendered
# 1.4 MB of HTML on one page.
PLATFORM_PART_BUDGET = 350_000
# The Markdown of a platform page's file lists, past which the largest
# A platform's file lists are split by the share of a page they render to,
# in thousandths: their elements against validate_site's MAX_PAGE_ELEMENTS,
# their bytes against the 600,000 of test_site_r2_gate, whichever is
# larger. Measured in Markdown characters, the lists split anew whenever
# their markup changed: rows written as tables moved 38 more systems to
# pages of their own.
PAGE_ELEMENTS = 10_000
PAGE_BYTES = 600_000
# A system whose list passes this share gets a page of its own.
PLATFORM_SECTION_BUDGET = 400
# A system's own page in runs of at most this share.
PLATFORM_PART_BUDGET = 700
# The share of a platform page's file lists, past which the largest
# systems move to pages of their own.
PLATFORM_REFERENCE_BUDGET = 250_000
PLATFORM_REFERENCE_BUDGET = 600
def _file_runs(body: list[str], budget: int) -> list[list[str]]:
"""A system's file list in runs of at most ``budget`` characters.
# Every part page a platform list has had, with the system it held:
# platforms/batocera-atari-st.md -> platforms/batocera.md#atari-st. When a
# budget or a list changes, a part can fold back into its platform page,
# and its address then redirects there instead of ending on the 404 page.
PLATFORM_PARTS_FILE = (Path(__file__).resolve().parent.parent
/ "website" / "src" / "data" / "platform-parts.json")
The lines before the first file (counts, emulators) open every run. A
file is never cut from its hash and evidence lines.
"""
first = next((i for i, line in enumerate(body) if line.startswith("- ")), len(body))
head, items = body[:first], []
for line in body[first:]:
if line.startswith("- ") or not items:
items.append([line])
else:
items[-1].append(line)
runs: list[list[str]] = [[]]
size = 0
for item in items:
weight = sum(len(line) + 1 for line in item)
if runs[-1] and size + weight > budget:
runs.append([])
size = 0
runs[-1].extend(item)
size += weight
return [head + run for run in runs]
def _published_platform_parts() -> dict[str, str]:
if not PLATFORM_PARTS_FILE.is_file():
return {}
return json.loads(PLATFORM_PARTS_FILE.read_text(encoding="utf-8"))
# A list moved off a reference page (gaps, provenance) is cut in parts of
@@ -3284,13 +3278,18 @@ def _file_runs(body: list[str], budget: int) -> list[list[str]]:
LIST_PART_BUDGET = 300_000
def _blocks_in_runs(blocks: list[list[str]], budget: int) -> list[list[str]]:
"""Blocks (table rows, folded lists) in runs of at most ``budget``
characters, a block never cut."""
def _characters(lines: list[str]) -> int:
return sum(len(line) + 1 for line in lines)
def _blocks_in_runs(blocks: list[list[str]], budget: int,
weigh: Callable[[list[str]], int] = _characters) -> list[list[str]]:
"""Blocks (table rows, folded lists) in runs of at most ``budget``, by
default in characters, a block never cut."""
runs: list[list[str]] = [[]]
size = 0
for block in blocks:
weight = sum(len(line) + 1 for line in block)
weight = weigh(block)
if runs[-1] and size + weight > budget:
runs.append([])
size = 0
@@ -3427,6 +3426,188 @@ def _launcher_refs(platform: str, config: dict) -> dict[str, list[str]]:
return found
# The characters of a hash a row shows, as many as the served copy's sha1.
SHORT_HASH = 8
# The hash a row shows first, when the list does not say which it checks.
_ROW_HASHES = ("sha1", "md5", "sha256", "crc32")
# An untested entry whose reason is a hash the bytes do not match.
_WRONG_HASH = re.compile(r"hash|mismatch|expected|must match", re.IGNORECASE)
# A list this long gets a filter box above it, as a table of RECORD_ROWS
# rows does (website/src/plugins/rehype-stack-tables.mjs).
FILTER_ROWS = 20
def _status_chip(f: dict) -> str:
"""The row's status as a chip, its word and its colour saying the same."""
status = f.get("status")
if status == "ok":
word, tone = "OK", "success"
elif status == "missing":
word, tone = "Missing", "danger"
elif status == "untested":
word = "Wrong hash" if _WRONG_HASH.search(str(f.get("reason") or "")) else "Untested"
tone = "warning"
else:
word, tone = str(status), "muted"
return f'<span class="rb-badge rb-badge-{tone}">{html.escape(word)}</span>'
def _copy_hash(algorithm: str, value: str | list) -> str:
"""A declared hash cut to SHORT_HASH characters on a button that copies
it whole. A list of accepted values (Recalbox) copies its first. Outside
a form a button submits nothing, so it carries no type."""
from hashing import parse_hash_list # noqa: PLC0415
values = parse_hash_list(value)
if not values:
return ""
first = html.escape(values[0])
more = f" +{len(values) - 1}" if len(values) > 1 else ""
return (f'<button class="rb-copy" data-copy="{first}" '
f'aria-label="Copy {algorithm.upper()} {first[:SHORT_HASH]}{more}">'
f'{first[:SHORT_HASH]}{more}</button>')
# Characters Markdown or MDX would read inside the raw HTML of a row: an
# MDX page parses the text of its HTML as Markdown, a .md page does not.
_INERT = str.maketrans({"*": "&#42;", "_": "&#95;", "`": "&#96;", "[": "&#91;", "]": "&#93;"})
_INLINE_MD = re.compile(
r"\[(?P<label>(?:`[^`]*`|[^\]`])+)\]\((?P<url>[^)\s]+)\)"
r"|`(?P<code>[^`]+)`"
r"|\*\*(?P<bold>[^*]+)\*\*"
)
def _text_html(text: str) -> str:
"""Plain text as HTML that neither Markdown nor MDX rewrites."""
return html.escape(html.unescape(text), quote=False).translate(_INERT)
def _inline_html(markdown: str) -> str:
"""The inline Markdown of a row's evidence (links, code, bold) as HTML.
Code spans become plain text and a linked path loses its code element:
one per reference and per reader was the bulk of the elements of a
platform page, and the words say the same without them.
"""
out: list[str] = []
position = 0
for match in _INLINE_MD.finditer(markdown):
out.append(_text_html(markdown[position:match.start()]))
if match["url"]:
out.append(f'<a href="{html.escape(match["url"])}">'
f'{_text_html(match["label"].replace("`", ""))}</a>')
elif match["code"]:
out.append(_text_html(match["code"]))
else:
out.append(f"<strong>{_text_html(match['bold'])}</strong>")
position = match.end()
out.append(_text_html(markdown[position:]))
return "".join(out)
def _row_path(fname: str, cfg_entry: dict) -> str:
"""Where the file goes, when its name alone does not say: three GameCube
IPL.bin differ only by GC/USA, GC/EUR and GC/JAP. A destination that is
the name says nothing, and one that ends with it shows its folder."""
destination = str(cfg_entry.get("destination") or "")
if destination.endswith("/" + fname):
return destination[:len(destination) - len(fname)]
return "" if destination == fname else destination
def _platform_file_row(f: dict, cfg_entry: dict, hash_type: str = "",
launched: list[str] | None = None) -> str:
"""One file of a platform's list as one row, written as HTML.
The row is a details element whose summary holds the name, the folder
when there is one and the status, followed by the short hash on its copy
button: a button cannot sit in a summary, which is itself a control.
The details hold every hash the platform declares, the size, the copy
that serves it and its readers, across the width of the list.
"""
fname = f["name"]
hashes = {
"sha1": cfg_entry.get("sha1", f.get("sha1", "")),
"md5": cfg_entry.get("md5", f.get("expected_md5", "")),
# The crc32 is what System.dat and clrmamepro print. RetroDECK
# declares sha256 too (dsifirmware.bin).
"crc32": str(cfg_entry.get("crc32") or ""),
"sha256": str(cfg_entry.get("sha256") or ""),
}
size = cfg_entry.get("size", f.get("size", 0))
shown = next((alg for alg in (hash_type, *_ROW_HASHES) if alg in hashes and hashes[alg]), "")
facts: list[str] = []
# One code element per value: Recalbox lists accepted md5s joined by
# commas, and each is a value a reader compares a dump with.
facts.extend(f"{alg.upper()}: {_hash_spans(hashes[alg])}"
for alg in ("sha1", "md5", "crc32", "sha256") if hashes[alg])
# The size closes the hash lines: a line of its own was one more
# element per row.
if size and facts:
facts[-1] += f" ({_fmt_size(size)})"
elif size:
facts.append(f"Size: {_fmt_size(size)}")
if f.get("status") == "untested" and f.get("reason"):
facts.append(_text_html(f"Not confirmed: {f['reason']}"))
# Not every entry is firmware: the platform may list a game's data or a
# file no core it runs opens.
kind = {"game_data": "Game data, not firmware",
"unread": "No core the platform runs opens it"}.get(f.get("category", ""))
if kind:
facts.append(kind)
served = _served_line(f)
if served:
facts.append(_inline_html(served))
if f.get("category_evidence"):
facts.append(_inline_html(_category_evidence_md(f["category_evidence"], launched)))
path = _row_path(fname, cfg_entry)
summary = (_text_html(fname) + (f"<code>{_text_html(path)}</code>" if path else "")
+ _status_chip(f))
copy = _copy_hash(shown, hashes[shown]) if shown else ""
return (f'<details class="rb-file"><summary>{summary}</summary>'
f'{"<br>".join(facts)}</details>{copy}')
def _platform_file_list(rows: list[str], has_path: bool, anchor: str) -> list[str]:
"""A system's rows as one HTML block, with a filter box when long.
The list drops its path column when no row has one, so a blank cell
never reads as missing data.
"""
lines: list[str] = []
if len(rows) >= FILTER_ROWS:
field = f"rb-files-filter-{anchor}"
lines += [
'<div class="rb-table-filter">'
f'<label for="{field}">Filter files</label>'
f'<input id="{field}" type="search" autocomplete="off" placeholder="File, path, status, hash...">'
f'<span class="rb-table-filter-count" aria-live="polite">{len(rows):,} rows</span></div>',
"",
]
kind = "rb-files rb-files-path" if has_path else "rb-files"
return [*lines, f'<div class="{kind}">', *rows, "</div>"]
# The elements a line puts on the page: its HTML tags, and the links and
# code spans of the Markdown before a list.
_ELEMENT = re.compile(r"<[a-zA-Z]|\]\(|`[^`]+`")
def _element_weight(lines: list[str]) -> int:
"""How many elements the lines render to."""
return sum(len(_ELEMENT.findall(line)) for line in lines)
def _page_share(lines: list[str]) -> float:
"""The share of a page the lines render to, in thousandths: the rows
are HTML already, so their length is their weight in bytes."""
return max(_element_weight(lines) * 1000 / PAGE_ELEMENTS,
_characters(lines) * 1000 / PAGE_BYTES)
def _render_platform_systems(
by_system: dict, config_files: dict, emulator_files: dict,
platform: tuple[str, str] | None = None, moved: dict[str, str] | None = None,
@@ -3454,102 +3635,42 @@ def _render_platform_systems(
sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id))
anchor = sys_id.replace(" ", "-")
body = [f"{ok_count}/{total} files verified"]
head = [f"{ok_count}/{total} files verified"]
if sys_emus:
emu_links = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus))
body[-1] += "\\"
body.append(f"Emulators: {emu_links}")
body.append("")
head[-1] += "\\"
head.append(f"Emulators: {emu_links}")
head.append("")
# File listing. The sort is stable, so same-named entries keep the
# order verify reported them in, which is the declaration order.
# One row per file, the evidence in its details. The sort is stable,
# so same-named entries keep the order verify reported them in,
# which is the declaration order.
rows: list[str] = []
has_path = False
seen_names: dict[str, int] = {}
for f in sorted(files, key=lambda x: x["name"]):
status = f["status"]
fname = f["name"]
declared = config_files.get((sys_id, fname), [])
nth = seen_names.get(fname, 0)
seen_names[fname] = nth + 1
cfg_entry = declared[nth] if nth < len(declared) else {}
sha1 = cfg_entry.get("sha1", f.get("sha1", ""))
md5 = cfg_entry.get("md5", f.get("expected_md5", ""))
size = cfg_entry.get("size", f.get("size", 0))
has_path = has_path or bool(_row_path(fname, cfg_entry))
rows.append(_platform_file_row(
f, cfg_entry, str((config or {}).get("hash_type") or ""),
launched.get(fname.rsplit("/", 1)[-1].casefold())))
body = head + _platform_file_list(rows, has_path, anchor)
if status == "ok":
status_display = "OK"
elif status == "untested":
reason = f.get("reason", "")
status_display = f"untested: {reason}" if reason else "untested"
elif status == "missing":
status_display = "**missing**"
else:
status_display = status
# Not every entry is firmware: the platform may list a game's data
# or a file no core it runs opens.
kind = {"game_data": "game data",
"unread": "no core the platform runs opens it"}.get(f.get("category", ""))
if kind:
status_display += f", {kind}"
size_str = _fmt_size(size) if size else ""
details = [status_display]
if size_str:
details.append(size_str)
# Where the file goes, when its name alone does not say: three
# GameCube IPL.bin differ only by GC/USA, GC/EUR and GC/JAP.
destination = str(cfg_entry.get("destination") or "")
where = f" at `{destination}`" if destination and destination != fname else ""
body.append(f"- `{fname}`{where} - {', '.join(details)}")
item_line = len(body) - 1
facts: list[str] = []
# Every hash the platform declares, whole: the crc32 is what
# System.dat and clrmamepro print, and pasting it found nothing
# for a pinned file the collection holds under other bytes.
crc32 = str(cfg_entry.get("crc32") or "")
# RetroDECK declares sha256 too: dsifirmware.bin's was in the CSV
# exports only, and pasting it into the search found nothing.
sha256 = str(cfg_entry.get("sha256") or "")
if sha1 or md5 or crc32 or sha256:
hash_parts = []
# One code span per value: Recalbox lists accepted md5s joined
# by commas, and the search indexed the joined run as a single
# word, so the second and later values were never found.
if sha1:
hash_parts.append(f"SHA1: {_hash_spans(sha1)}")
if md5:
hash_parts.append(f"MD5: {_hash_spans(md5)}")
if crc32:
hash_parts.append(f"CRC32: `{crc32}`")
if sha256:
hash_parts.append(f"SHA256: `{sha256}`")
facts.append(" | ".join(hash_parts))
served = _served_line(f)
if served:
facts.append(served)
if f.get("category_evidence"):
facts.append(_category_evidence_md(
f["category_evidence"], launched.get(fname.rsplit("/", 1)[-1].casefold())))
# One line each, hashes, the copy that serves it, its readers: as
# continuation lines they ran into one paragraph.
if facts:
body[item_line] += "\\"
body.extend(f" {fact}" + ("\\" if n < len(facts) - 1 else "")
for n, fact in enumerate(facts))
# Measured as the Markdown the split was tuned on, a collapsible
# block indented by four, so both engines split the same systems.
size = sum(len(line) + 1 for line in body)
# Measured in the share of a page the list renders to.
size = _page_share(body)
if use_collapsible:
status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue")
title = f"{_count(total, 'file')} ({status_tag})"
size += len(f'??? note "{title}"') + 2 + 4 * sum(1 for line in body if line)
block = markup.details("note", title, body)
block = markup.details("note", f"{_count(total, 'file')} ({status_tag})", body)
while block and not block[-1]:
block.pop()
else:
block = body
sections.append({"sys_id": sys_id, "anchor": anchor, "body": body, "block": block,
sections.append({"sys_id": sys_id, "anchor": anchor, "head": head, "rows": rows,
"has_path": has_path, "block": block,
"size": size, "total": total, "ok": ok_count})
# A list longer than its budget goes to a page of its own, then the
@@ -3568,7 +3689,7 @@ def _render_platform_systems(
kept -= sec["size"]
for sec in sections:
sys_id, anchor, body = sec["sys_id"], sec["anchor"], sec["body"]
sys_id, anchor = sec["sys_id"], sec["anchor"]
total, ok_count = sec["total"], sec["ok"]
# Every system is a heading with the same anchor in both layouts:
# emulator cards link here, and the site search splits a page at its
@@ -3578,7 +3699,9 @@ def _render_platform_systems(
lines.append("")
if moved is not None and platform and sys_id in own_page:
key, display = platform
runs = _file_runs(body, PLATFORM_PART_BUDGET)
runs = [sec["head"] + _platform_file_list(run, sec["has_path"], anchor)
for run in _blocks_in_runs([[row] for row in sec["rows"]],
PLATFORM_PART_BUDGET, _page_share)]
stems = [f"{key}-{anchor}" + (f"-{n}" if n > 1 else "")
for n in range(1, len(runs) + 1)]
for n, (stem, run) in enumerate(zip(stems, runs), 1):
@@ -3595,15 +3718,18 @@ def _render_platform_systems(
*([""] + [", ".join(
f"[part {i}]({other}.md)" if other != stem else f"part {i}"
for i, other in enumerate(stems, 1))] if len(runs) > 1 else []),
"", "## Files", "",
"", "## Files", "", *markup.details("note", ROLE_LEGEND_TITLE, [ROLE_LEGEND]),
*search_excluded(run),
"", _date_line(),
]) + "\n"
where = (f"[a page of their own]({stems[0]}.md)" if len(stems) == 1 else
"pages of their own: " + ", ".join(
f"[part {i}]({stem}.md)" for i, stem in enumerate(stems, 1)))
# The count and status a folded system shows in its summary,
# linked to the page that lists the files.
tag = "OK" if ok_count == total else _count(total - ok_count, "issue")
count = f"{_count(total, 'file')} ({tag})"
lines.extend([
f"{_count(total, 'file')}, {ok_count} verified: listed on {where}.", "",
f"[{count}]({stems[0]}.md)" if len(stems) == 1 else f"{count}: " + ", ".join(
f"[part {i}]({stem}.md)" for i, stem in enumerate(stems, 1)),
"",
])
continue
# The section's heading stays searchable, its file list with every
@@ -3639,6 +3765,40 @@ def _site_profiles() -> dict[str, dict]:
return load_emulator_profiles(str(_REPO_ROOT / "emulators"))
# Whose code a source reference cites, in one word: a platform page names
# each role once in ROLE_LEGEND. Written after every reference, "(emulator
# code, libretro port)" ran through Batocera's 14,000 words.
ROLE_WORDS = {
"libretro glue": "glue",
"emulator code, libretro port": "port",
"upstream emulator": "upstream",
"this build": "build",
}
ROLE_LEGEND_TITLE = "Source reference words"
ROLE_LEGEND = (
"In a file's details, a word before its source references says whose code they are. "
"**glue** means libretro glue, **port** means emulator code in the libretro port, "
"**upstream** means the upstream emulator and **build** means this build's own code."
)
def _refs_by_role(profile: dict, value: str | dict | None) -> str:
"""A source_ref's parts grouped by whose code they cite, each role named
once by its ROLE_WORDS word: `glue a.c:1, b.c:2 and port c.c:3`."""
from sitesources import source_ref_items
texts = []
for label, items in source_ref_items(profile, value):
by_role: dict[str, list[str]] = {}
for part, role in items:
by_role.setdefault(role, []).append(part)
text = " and ".join(
(f"{ROLE_WORDS.get(role, role)} " if role else "") + ", ".join(parts)
for role, parts in by_role.items())
texts.append(f"**{label}:** {text}" if label else text)
return ", ".join(texts)
def _category_evidence_md(evidence: dict, launched: list[str] | None = None) -> str:
"""What establishes an entry's category, as the page shows it.
@@ -3654,7 +3814,7 @@ def _category_evidence_md(evidence: dict, launched: list[str] | None = None) ->
# plain code with the role word ("source") left in front.
profile = profiles.get(reader["emulator"], {})
refs = [
(f"`{e['name']}` at {_source_ref_markdown(profile, e['source_ref'])}"
(f"`{e['name']}` at {_refs_by_role(profile, e['source_ref'])}"
if e.get("source_ref") else f"`{e['name']}`")
+ (" (by name)" if e.get("by") == "name" else "")
for e in reader.get("entries", [])
@@ -4177,8 +4337,7 @@ def generate_platform_page(
"",
level_label("reference"),
"",
"For checking the work: the counts, where the list comes from, and "
"every file with its hashes.",
"For checking the work: counts, the list's source and each file's hashes.",
"",
*(line for note in (GUIDES[name].reference_notes if name in GUIDES else ())
for line in (note, "")),
@@ -4309,6 +4468,8 @@ def generate_platform_page(
f"| <span data-pagefind-ignore>{ok_count}/{total}</span> | {status} | {emu_str} |"
)
lines.append("")
if by_system:
lines.extend(markup.details("note", ROLE_LEGEND_TITLE, [ROLE_LEGEND]))
lines.extend(
_render_platform_systems(by_system, config_files, emulator_files,
@@ -11749,6 +11910,7 @@ def _generate(args: argparse.Namespace) -> None:
)
if HANDHELD_GUIDE:
write_if_changed(str(docs / HANDHELD_GUIDE.path), generate_handheld_guide(HANDHELD_GUIDE))
published_parts = _published_platform_parts()
for name, cov in coverages.items():
# The emulators this platform runs, not every one of the system.
runs = resolve_platform_cores(cov["config"], profiles)
@@ -11763,6 +11925,9 @@ def _generate(args: argparse.Namespace) -> None:
if stem in coverages:
raise RuntimeError(f"a system page takes the platform {stem}'s address")
PART_PAGES.add(f"platforms/{stem}.md")
anchor = re.search(rf"\]\({re.escape(name)}\.md#([^)\s]+)\)", page)
if anchor:
published_parts[f"platforms/{stem}.md"] = f"platforms/{name}.md#{anchor[1]}"
# One console of one setup's list, a reference: "playstation
# bios" found five setups' lists before the console's page.
# Weighed down, they still won on their title, which the page
@@ -11934,6 +12099,8 @@ def _generate(args: argparse.Namespace) -> None:
stale += _sweep_top_pages(docs)
print("Adding page metadata and structured data...")
write_if_changed(str(PLATFORM_PARTS_FILE),
json.dumps(dict(sorted(published_parts.items())), indent=1) + "\n")
lastmod = decorate_markdown_pages(docs)
nav = generate_nav(coverages, manufacturers, profiles, registry,
@@ -11950,7 +12117,7 @@ def _generate(args: argparse.Namespace) -> None:
json.dumps(sitestarlight.sidebar(nav, present, _overview_anchors(nav, docs)),
indent=1) + "\n")
write_if_changed(str(generated / "redirects.json"),
json.dumps(sitestarlight.redirects(profiles, present), indent=1,
json.dumps(sitestarlight.redirects(profiles, present, published_parts), indent=1,
sort_keys=True) + "\n")
write_if_changed(str(generated / "lastmod.json"),
json.dumps(dict(sorted(lastmod.items())), indent=1) + "\n")