From d2974660ebd5f7765389853c4372df3c8ac52419 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Thu, 22 Oct 2026 15:21:52 +0200
Subject: [PATCH 1/6] refactor: emit site blocks through sitemarkup
Callouts, collapsible blocks, tabs, buttons, card grids and search
exclusion go through one module that writes the engine's syntax. The
MkDocs output is unchanged apart from whitespace-only lines.
---
scripts/generate_readme.py | 18 +-
scripts/generate_site.py | 512 ++++++++++++++++---------------------
scripts/sitediagrams.py | 6 +-
scripts/siteexports.py | 52 ++--
scripts/siteguides.py | 23 +-
scripts/sitemarkup.py | 183 +++++++++++++
6 files changed, 444 insertions(+), 350 deletions(-)
create mode 100644 scripts/sitemarkup.py
diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py
index 3723ed55..f8651907 100644
--- a/scripts/generate_readme.py
+++ b/scripts/generate_readme.py
@@ -230,14 +230,16 @@ def release_totals(platform_name: str, record: dict) -> tuple[int | None, int |
def download_links(
- platform_name: str, record: dict, repo: str, label: str, attrs: str = ""
+ platform_name: str, record: dict, repo: str, label: str, attrs: str = "",
+ button: Callable[[str, str], str] | None = None,
) -> str:
"""Markdown links to the files of a platform's pack in the recorded release.
A pack published whole gets one link labelled *label*. A pack in parts
gets one link per part, since every part is needed. *repo* is the
repository URL, absolute or relative, and *attrs* a MkDocs attribute list
- put after each link. Without a recorded release the link is the latest
+ put after each link, or *button* the function that draws each link as a
+ button on the site. Without a recorded release the link is the latest
release page. A release recorded without this platform's pack says so and
names the installer, which reads the file lists on main: a button to the
release served six platforms a page with no pack for them.
@@ -245,18 +247,22 @@ def download_links(
pack = release_record.pack_for(platform_name, record)
tag = record.get("tag")
assets = pack.get("assets") if pack else None
+ def link(text: str, url: str) -> str:
+ return button(text, url) if button else f"[{text}]({url}){attrs}"
+
if not tag:
- return f"[{label}]({repo}/releases/latest){attrs}"
+ return link(label, f"{repo}/releases/latest")
if not assets:
return f"Not in {tag}: [use the installer]({SITE_URL}wiki/installer/)"
urls = [f"{repo}/releases/download/{tag}/{asset}" for asset in assets]
if len(urls) == 1:
- return f"[{label}]({urls[0]}){attrs}"
+ return link(label, urls[0])
+ styled = bool(attrs or button)
parts = [
- f"[{'P' if attrs else 'p'}art {number} of {len(urls)}]({url}){attrs}"
+ link(f"{'P' if styled else 'p'}art {number} of {len(urls)}", url)
for number, url in enumerate(urls, 1)
]
- if attrs:
+ if styled:
return " ".join(parts)
return f"{label}: " + ", ".join(parts)
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index 45b29724..f8e6da82 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -73,6 +73,7 @@ from generate_readme import (
release_totals,
)
from provenance_report import build_report, distinct_missing
+import sitemarkup as markup
from siteexports import generate_exports_page, load_export_reports
from siteguides import (
ALSO_RUNS,
@@ -384,7 +385,7 @@ def _timestamp() -> str:
def _stat(value: str, label: str) -> list[str]:
return [
- '
",
@@ -402,7 +403,7 @@ def _group_members(coverages: dict, registry: dict, group: str) -> list[str]:
def _next_block(links: list[str]) -> list[str]:
"""Where to go from here, at the end of a page."""
- return ['!!! tip "Next"', "", *(f" - {link}" for link in links), ""]
+ return markup.aside("tip", "Next", [f"- {link}" for link in links])
def generate_home(
@@ -431,8 +432,8 @@ def generate_home(
"the **BIOS**. Your emulator needs a copy of it. We put the right copy "
"in the right folder.",
"",
- "[Get your files](which-pack.md){ .md-button .md-button--primary } "
- "[What is a BIOS?](wiki/what-are-bios-files.md){ .md-button }",
+ markup.button("Get your files", "which-pack.md", primary=True) + " "
+ + markup.button("What is a BIOS?", "wiki/what-are-bios-files.md"),
"",
f"Free. No account. Works with {works_with}.",
"",
@@ -446,30 +447,28 @@ def generate_home(
"",
"## Three steps",
"",
- "1. **Pick your device.** Its page shows your BIOS folder.",
- "2. **Run one command**, or download a ZIP file.",
- "3. **Start a game.**",
- "",
+ *(markup.steps([
+ "**Pick your device.** Its page shows your BIOS folder.",
+ "**Run one command**, or download a ZIP file.",
+ "**Start a game.**",
+ ]) if markup.starlight() else [
+ "1. **Pick your device.** Its page shows your BIOS folder.",
+ "2. **Run one command**, or download a ZIP file.",
+ "3. **Start a game.**",
+ "",
+ ]),
"## Pick your device",
"",
- '
',
- "",
]
- for group, (label, _description) in DEVICE_GROUPS.items():
- members = _group_members(coverages, registry, group)
- if not members:
- continue
- lines += [
- f"- **{label}**",
- "",
- " " + ", ".join(
- f"[{coverages[name]['platform']}](platforms/{name}.md)" for name in members
- ),
- "",
- ]
+ lines += markup.card_grid([
+ (label, [", ".join(
+ f"[{coverages[name]['platform']}](platforms/{name}.md)"
+ for name in _group_members(coverages, registry, group)
+ )])
+ for group, (label, _description) in DEVICE_GROUPS.items()
+ if _group_members(coverages, registry, group)
+ ])
lines += [
- "
",
- "",
"## Is it safe?",
"",
"- It writes only in your BIOS folder.",
@@ -501,43 +500,23 @@ def generate_explore_index(db: dict, coverages: dict, profiles: dict) -> str:
"One console, one emulator, or one installer option: each has its page, "
"with the files it needs and the code that reads them.",
"",
- '
',
- "",
- "- **Systems**",
- "",
- " What each console and computer needs, file by file.",
- "",
- " [Browse the systems](systems/index.md)",
- "",
- "- **Emulators**",
- "",
- " What each emulator reads, with the source code that reads it.",
- "",
- " [Browse the emulators](emulators/index.md)",
- "",
- "- **Installer options**",
- "",
- " Fewer files, one region, another folder, a check without download.",
- "",
- " [All installer options](wiki/installer.md)",
- "",
- "- **How files are checked**",
- "",
- " Each setup checks its files its own way: by name, by MD5 or by SHA1.",
- "",
- " Reference: [How each platform checks files](wiki/verification-modes.md)",
- "",
- "- **Data and API**",
- "",
- " Every file and hash as JSON, CSV and SQLite.",
- "",
- " Reference: [Data and API](data.md)",
- "",
- "
",
- "",
+ *markup.card_grid([
+ ("Systems", ["What each console and computer needs, file by file.", "",
+ "[Browse the systems](systems/index.md)"]),
+ ("Emulators", ["What each emulator reads, with the source code that reads it.", "",
+ "[Browse the emulators](emulators/index.md)"]),
+ ("Installer options", [
+ "Fewer files, one region, another folder, a check without download.", "",
+ "[All installer options](wiki/installer.md)"]),
+ ("How files are checked", [
+ "Each setup checks its files its own way: by name, by MD5 or by SHA1.", "",
+ "Reference: [How each platform checks files](wiki/verification-modes.md)"]),
+ ("Data and API", ["Every file and hash as JSON, CSV and SQLite.", "",
+ "Reference: [Data and API](data.md)"]),
+ ]),
"## The collection",
"",
- '
',
+ markup.block_open("div", 'class="rb-stats"'),
"",
*_stat(f"{total_files:,}", COLLECTION_FILES_LABEL),
*_stat(str(len(coverages)), "Setups supported"),
@@ -582,7 +561,7 @@ def _pack_buttons(platform_name: str, label: str = "Pack") -> str:
release_record.load_record(),
REPO_URL,
label,
- "{ .md-button .md-button--primary }",
+ button=lambda text, url: markup.button(text, url, primary=True),
)
@@ -1295,9 +1274,11 @@ def generate_data_page(exports: list[dict]) -> str:
"RetroBIOS publishes the same metadata used by its verifier, pack builder "
"and website as versioned static files. No account or API key is required.",
"",
- "[API catalog](api/v1/catalog.json){ .md-button .md-button--primary } "
- "[SQLite snapshot](downloads/retrobios.sqlite){ .md-button } "
- "[Collection DAT](downloads/retrobios.dat){ .md-button }",
+ " ".join([
+ markup.button("API catalog", "api/v1/catalog.json", primary=True),
+ markup.button("SQLite snapshot", "downloads/retrobios.sqlite"),
+ markup.button("Collection DAT", "downloads/retrobios.dat"),
+ ]),
"",
"The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game "
"per `bios///` folder of the collection, named "
@@ -1497,7 +1478,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
stripped = paragraph.strip()
if (
not stripped
- or stripped.startswith(("#", "|", "```", "???", "<", "- ", "* "))
+ or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* "))
):
continue
plain = _plain_markdown(stripped)
@@ -1977,16 +1958,11 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st
members = _group_members(coverages, registry, group)
if not members:
continue
- lines += [f"## {label}", "", description, "", '
", ""]
+ lines += [f"## {label}", "", description, ""]
+ lines += markup.card_grid([
+ (f"[{coverages[name]['platform']}]({name}.md)", [GUIDES[name].summary])
+ for name in members
+ ])
lines += [
"## Reference: every platform at a glance { #reference }",
@@ -2088,24 +2064,14 @@ def _render_platform_systems(
# emulator cards link here, and the site search splits a page at its
# headings. Folded into blocks under one heading, every hash pasted
# in the search led to the systems overview of a 100-block page.
- block_start = len(lines) + 2
- if use_collapsible:
- status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue")
- lines.append(f"### {sys_link} {{ #{anchor} }}")
- lines.append("")
- lines.append(f'??? note "{_count(total, "file")} ({status_tag})"')
- lines.append("")
- pad = " "
- else:
- lines.append(f"### {sys_link} {{ #{anchor} }}")
- lines.append("")
- pad = ""
-
- lines.append(f"{pad}{ok_count}/{total} files verified")
+ lines.append(f"### {sys_link} {{ #{anchor} }}")
+ lines.append("")
+ block_start = len(lines)
+ body = [f"{ok_count}/{total} files verified"]
if sys_emus:
emu_links = ", ".join(_emulator_link(e, "../") for e in sorted(sys_emus))
- lines.append(f"{pad}Emulators: {emu_links}")
- lines.append("")
+ body.append(f"Emulators: {emu_links}")
+ body.append("")
# File listing. The sort is stable, so same-named entries keep the
# order verify reported them in, which is the declaration order.
@@ -2146,7 +2112,7 @@ def _render_platform_systems(
# GameCube IPL.bin differ only by GC/USA, GC/EUR and GC/JAP.
destination = str(cfg_entry.get("destination") or "")
where = f" at `{destination}`" if destination and destination != fname else ""
- lines.append(f"{pad}- `{fname}`{where} - {', '.join(details)}")
+ body.append(f"- `{fname}`{where} - {', '.join(details)}")
# Every hash the platform declares, whole: the crc32 is what
# System.dat and clrmamepro print, and pasting it found nothing
# for a pinned file the collection holds under other bytes.
@@ -2167,23 +2133,32 @@ def _render_platform_systems(
hash_parts.append(f"CRC32: `{crc32}`")
if sha256:
hash_parts.append(f"SHA256: `{sha256}`")
- lines.append(f"{pad} {' | '.join(hash_parts)}")
+ body.append(f" {' | '.join(hash_parts)}")
if f.get("category_evidence"):
- lines.append(f"{pad} {_category_evidence_md(f['category_evidence'])}")
+ body.append(f" {_category_evidence_md(f['category_evidence'])}")
- block = lines[block_start:]
- if moved is not None and platform and sum(len(line) + 1 for line in block) > PLATFORM_SECTION_BUDGET:
+ # Measured as the Markdown the split was tuned on, a collapsible
+ # block indented by four, so both engines split the same systems.
+ size = sum(len(line) + 1 for line in body)
+ if use_collapsible:
+ status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue")
+ title = f"{_count(total, 'file')} ({status_tag})"
+ size += len(f'??? note "{title}"') + 2 + 4 * sum(1 for line in body if line)
+ block = markup.details("note", title, body)
+ while block and not block[-1]:
+ block.pop()
+ else:
+ block = body
+ if moved is not None and platform and size > PLATFORM_SECTION_BUDGET:
key, display = platform
stem = f"{key}-{anchor}"
- body = block[2:] if use_collapsible else block
moved[stem] = "\n".join([
f"# {display}: {sys_id}", "",
f"Part of the [{display} page]({key}.md#{anchor}), whose other systems "
f"are listed there. Each file below is as {display} declares it, with "
"its status in the pack.",
"", "## Files", "",
- *search_excluded(
- [line[len(pad):] if line.startswith(pad) else line for line in body]),
+ *search_excluded(body),
"", f"*Generated on {_timestamp()}*",
]) + "\n"
lines[block_start:] = [
@@ -2193,7 +2168,7 @@ def _render_platform_systems(
continue
# The section's heading stays searchable, its file list with every
# hash is the lookup's: retrodeck's arcade list was 495 KB of index.
- lines[block_start:] = search_excluded(lines[block_start:])
+ lines[block_start:] = search_excluded(block)
return lines
@@ -2340,14 +2315,12 @@ def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list:
for line in (note, "")]
if guide.more_folders:
detected = _detected_systems(registry, name) if guide.more_folders_detected else []
- lines += [
- f'??? note "{guide.more_folders_title}"',
+ lines += markup.details("note", guide.more_folders_title, [
+ *_folder_rows(guide.more_folders),
+ *(f"| {system} | `{folder}` |" for system, folder in detected),
"",
- *_folder_rows(guide.more_folders, " "),
- *(f" | {system} | `{folder}` |" for system, folder in detected),
- "",
- *([f" {guide.more_folders_note}", ""] if guide.more_folders_note else []),
- ]
+ *([guide.more_folders_note, ""] if guide.more_folders_note else []),
+ ])
lines += [
*(["", ""] if guide.on_card else []),
"## Install",
@@ -2454,14 +2427,9 @@ def generate_platform_page(
]
if registry.get(name, {}).get("status") == "archived":
- lines.extend(
- [
- '!!! warning "Archived platform"',
- "",
- f" {display}'s list is kept as last read. Its pack is still built.",
- "",
- ]
- )
+ lines.extend(markup.aside("warning", "Archived platform", [
+ f"{display}'s list is kept as last read. Its pack is still built.",
+ ]))
summary = GUIDES[name].summary if name in GUIDES else ""
if summary:
@@ -2485,45 +2453,23 @@ def generate_platform_page(
if no_list:
# MinUI and its forks hand each core a folder and name no file: a
# 0/0 coverage band read as "this device is not supported".
- lines.extend(
- [
- '!!! info "No BIOS list of its own"',
- "",
- f" {display} names no BIOS file itself: it gives each emulator core "
- "its own folder and the core looks for its files there. The pack "
- "places every file the cores it runs need into those folders. The "
- "core files are listed further down this page.",
- "",
- ]
- )
+ lines.extend(markup.aside("info", "No BIOS list of its own", [
+ f"{display} names no BIOS file itself: it gives each emulator core "
+ "its own folder and the core looks for its files there. The pack "
+ "places every file the cores it runs need into those folders. The "
+ "core files are listed further down this page.",
+ ]))
band = [] if no_list else [
- '
', "",
+ lines.extend([*band, markup.block_open("div", 'class="rb-meta-card"'), "",
"| Detail | Value |", "|---|---|"])
if hash_type:
lines.append(f"| Hash type | {hash_type} |")
@@ -3150,9 +3096,7 @@ def generate_provenance_page(
lines.extend([f"### {label}", ""])
for dat in sorted(by_dat):
entries = by_dat[dat]
- lines.extend([f'??? note "{dat} ({len(entries)})"', ""])
- lines.append(" | Name | Description | SHA1 |")
- lines.append(" |------|-------------|------|")
+ table = ["| Name | Description | SHA1 |", "|------|-------------|------|"]
for entry in sorted(entries, key=lambda e: e["name"]):
sha1 = entry.get("sha1") or "-"
# A pipe in catalog text would split the markdown table row
@@ -3164,8 +3108,8 @@ def generate_provenance_page(
f" Ships in {_ROMSET_SOURCES.get(hit['source'], hit['source'])} "
f"`{hit['archive']}` as `{hit['member']}`."
)
- lines.append(f" | `{name}` | {desc} | `{sha1}` |")
- lines.append("")
+ table.append(f"| `{name}` | {desc} | `{sha1}` |")
+ lines.extend(markup.details("note", f"{dat} ({len(entries)})", table))
if derived_entries:
lines.extend(
@@ -3302,11 +3246,11 @@ def _system_file_card(
anchor = ""
else:
anchored.add(anchor)
- lines = ['
', ""]
+ lines = [markup.block_open("div", 'class="rb-sys-file"'), ""]
# The heading's own link is this page's, the second link is the file's
# own address, resolved by content wherever its card moves.
permalink = (
- f' permalink'
) if anchor else ""
lines.append(_card_title(
@@ -3373,8 +3317,8 @@ def tables_out_of_search(markdown: str, threshold: int = SEARCH_TABLE_BYTES) ->
end += 1
table = lines[index:end]
if sum(len(line) for line in table) > threshold:
- out += ["", f"{indent}", "", *table,
- "", f"{indent}", ""]
+ out += ["", indent + markup.block_open("section", markup.search_exclude()), "",
+ *table, "", f"{indent}", ""]
else:
out += table
index = end
@@ -3393,7 +3337,8 @@ def search_excluded(lines: list[str]) -> list[str]:
the first div closed inside an excluded div (a code block's) ended the
exclusion and the rest of the block was indexed.
"""
- return ["", "", "", *lines, "", "", ""]
+ return ["", markup.block_open("section", markup.search_exclude()), "",
+ *lines, "", "", ""]
# Card lines the site search leaves out. They list where a file goes and
@@ -3418,7 +3363,8 @@ def _search_skipped_apart(lines: list[str]) -> list[str]:
for line in lines:
skip = line.startswith(SEARCH_SKIPPED) or (skipping and line.startswith(" "))
if skip and not skipping:
- out.extend(["", '
', ""])
+ out.extend(["", markup.block_open(
+ "div", f'class="rb-search-skip" {markup.search_exclude()}'), ""])
elif skipping and not skip:
out.extend(["", "
", ""])
skipping = skip
@@ -3438,6 +3384,13 @@ def _card_title(text: str, anchor: str, searchable: bool = True) -> str:
of cards stops above it (mkdocs_hooks.py), and a card without an
anchor keeps the id the toc gives it.
"""
+ if markup.starlight():
+ # Raw HTML: the card links to itself already, and a Markdown heading
+ # would add the engine's anchor link and icon to every card.
+ title = re.sub(r"`([^`]+)`", lambda m: f"{html.escape(m[1], quote=False)}", text)
+ ident = f' id="{anchor}"' if anchor else ""
+ ignore = "" if searchable else f" {markup.search_exclude()}"
+ return f'
{title}
'
attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title"
if not searchable:
attributes += " data-search-exclude"
@@ -3509,11 +3462,8 @@ def _cards_markdown(cards: list[Card]) -> list[str]:
# A variant gets the same card as a primary: the revision a
# platform pins is the one a reader comes to find, and showed as
# a name and a truncated MD5, nothing said who pins it.
- lines.append(f'??? note "Variants ({len(variants)})"')
- lines.append("")
- for card in variants:
- lines.extend(f" {line}" if line else "" for line in card)
- lines.append("")
+ lines.extend(markup.details(
+ "note", f"Variants ({len(variants)})", [line for card in variants for line in card]))
return lines
@@ -4019,7 +3969,7 @@ def _render_emulator_file(
badges = _file_badges(f, in_repo)
badge_str = " ".join(badges)
- lines.append('
')
+ lines.append(markup.block_open("div", 'class="rb-file-entry"'))
lines.append("")
marker = f'' if alias else ""
lines.append(_card_title(f"{marker}{_code(fname)} {badge_str}", anchor))
@@ -4307,14 +4257,12 @@ def _install_block(name: str, profile: dict) -> list[str]:
"""
if profile.get("type") in ("alias", "launcher", "test"):
return []
- return search_excluded([
- '??? tip "Install this emulator\'s files"',
- "",
- " The installer finds your platform and its BIOS folder, then downloads "
+ return search_excluded(markup.details("tip", "Install this emulator's files", [
+ "The installer finds your platform and its BIOS folder, then downloads "
"only the files this emulator reads there and checks each one.",
"",
- *install_commands(f"--emulator {name}", " "),
- ])
+ *install_commands(f"--emulator {name}"),
+ ]))
def _platform_install_block(name: str, display: str, on_card: bool = False) -> list[str]:
@@ -4325,37 +4273,32 @@ def _platform_install_block(name: str, display: str, on_card: bool = False) -> l
computer it is mounted on, whose root is named.
"""
if on_card:
- return [
- '???+ tip "Install with one command"',
- "",
- f" With the {display} SD card mounted, the installer downloads the files "
+ return markup.details("tip", "Install with one command", [
+ f"With the {display} SD card mounted, the installer downloads the files "
"of the pack that are missing or differ from it into `Bios//` on "
"the card, checks each one, and resumes where it stopped. `--dest` "
"names the card's root.",
"",
- " Linux, macOS:",
+ "Linux, macOS:",
"",
- " ```sh",
- f" {install_command('sh', f'--platform {name} --dest /path/to/card')}",
- " ```",
+ "```sh",
+ install_command("sh", f"--platform {name} --dest /path/to/card"),
+ "```",
"",
- " Windows (PowerShell, E: being the card):",
+ "Windows (PowerShell, E: being the card):",
"",
- " ```powershell",
- " " + install_command("powershell", f"--platform {name} --dest E:\\"),
- " ```",
- "",
- ]
- return [
- '???+ tip "Install with one command"',
- "",
- f" The installer downloads the files of the {display} pack that are "
+ "```powershell",
+ install_command("powershell", f"--platform {name} --dest E:\\"),
+ "```",
+ ], opened=True)
+ return markup.details("tip", "Install with one command", [
+ f"The installer downloads the files of the {display} pack that are "
"missing or differ from it, checks each one, and resumes where it "
"stopped. Add `--system` with the ids of the systems overview below "
"to install fewer systems.",
"",
- *install_commands(f"--platform {name}", " "),
- ]
+ *install_commands(f"--platform {name}"),
+ ], opened=True)
def _console_install_block(console: str, declared: list[tuple[str, str]]) -> list[str]:
@@ -4363,19 +4306,17 @@ def _console_install_block(console: str, declared: list[tuple[str, str]]) -> lis
if not declared:
return []
lines = [
- f'??? tip "Install the {console} files"',
- "",
- " Each platform names the system its own way. On Windows, run the "
+ "Each platform names the system its own way. On Windows, run the "
"same options through `&([scriptblock]::Create((irm "
f"{INSTALL_BASE}/install.ps1)))`.",
"",
- " ```sh",
+ "```sh",
]
lines += [
- f" {install_command('sh', f'--platform {platform} --system {shlex.quote(system)}')}"
+ install_command("sh", f"--platform {platform} --system {shlex.quote(system)}")
for platform, system in declared
]
- return search_excluded(lines + [" ```", ""])
+ return search_excluded(markup.details("tip", f"Install the {console} files", [*lines, "```"]))
def _render_platform_details(profile: dict) -> list[str]:
@@ -4383,16 +4324,16 @@ def _render_platform_details(profile: dict) -> list[str]:
lines: list[str] = []
platform_details = profile.get("platform_details")
if platform_details and isinstance(platform_details, dict):
- lines.extend(['???+ info "Platform details"', ""])
+ body: list[str] = []
for pk, pv in platform_details.items():
if isinstance(pv, dict):
- lines.extend([f" - **{pk}:**", ""])
+ body.extend([f"- **{pk}:**", ""])
for sk, sv in pv.items():
- lines.append(f" - {sk}: {_detail_value(profile, sk, sv)}")
- lines.append("")
+ body.append(f" - {sk}: {_detail_value(profile, sk, sv)}")
+ body.append("")
else:
- lines.append(f" - **{pk}:** {_detail_value(profile, pk, pv)}")
- lines.append("")
+ body.append(f"- **{pk}:** {_detail_value(profile, pk, pv)}")
+ lines.extend(markup.details("info", "Platform details", body, opened=True))
return lines
@@ -4466,13 +4407,11 @@ def _render_structured_blocks(profile: dict) -> list[str]:
val = profile.get(field)
if val is None:
continue
- lines.append(f'???+ abstract "{label}"')
- lines.append("")
- start = len(lines)
- _render_yaml_value(lines, val, indent=4,
+ body: list[str] = []
+ _render_yaml_value(body, val, indent=0,
field_text=_structured_field_text(profile))
- lines[start:] = [_escape_tags(line) for line in lines[start:]]
- lines.append("")
+ lines.extend(markup.details(
+ "abstract", label, [_escape_tags(line) for line in body], opened=True))
return lines
@@ -4728,7 +4667,7 @@ def emulator_pages(
f"# {emu_name}",
"",
*([intro, ""] if intro else []),
- '
',
+ markup.block_open("div", 'class="rb-meta-card"'),
"",
"| Detail | Value |",
"|---|---|",
@@ -4812,19 +4751,14 @@ def emulator_pages(
lines.extend(_render_structured_blocks(profile))
if notes:
- indented = _admonition_body(link_prose_citations(profile, notes))
- lines.extend(['??? note "Technical notes"', f" {indented}", ""])
+ body = _admonition_body(link_prose_citations(profile, notes)).replace("\n ", "\n")
+ lines.extend(markup.details("note", "Technical notes", body.split("\n")))
if not files:
lines.append("No BIOS or firmware files required.")
if exclusion:
- lines.extend(
- [
- "",
- '!!! info "Why no files"',
- f" {_escape_tags(str(exclusion))}",
- ]
- )
+ lines.append("")
+ lines.extend(markup.aside("info", "Why no files", [_escape_tags(str(exclusion))])[:-1])
else:
_file_available = _availability_check(db, data_names)
@@ -4851,7 +4785,11 @@ def emulator_pages(
if data_dirs:
# The member tables are a file list with hashes: out of the search,
# dolphin, primehack and ishiiruka each put 210 KB of it there.
- lines.extend(["## Data directories { data-search-exclude }", ""])
+ lines.extend([
+ "## Data directories" + (" { data-search-exclude }" if not markup.starlight() else ""),
+ "",
+ ])
+ trees_start = len(lines)
member_anchors: set[str] = set()
# Each tree with what it holds, where it comes from, which build reads
# it and the code that does: these trees put thousands of files in
@@ -4881,13 +4819,7 @@ def emulator_pages(
lines.append(f" - Source: {_source_ref_markdown(profile, dd['source_ref'])}")
members = _DATA_MEMBERS.get(ref) or {}
if members:
- lines.extend([
- "",
- f' ??? info "{_count(len(members), "file")}, as the packs ship them"',
- "",
- " | File | Size | SHA1 |",
- " |------|-----:|------|",
- ])
+ table = ["| File | Size | SHA1 |", "|------|-----:|------|"]
for member, info in sorted(members.items()):
sha1 = str(info.get("sha1", ""))
# One id per page: two members can hold the same bytes
@@ -4895,12 +4827,19 @@ def emulator_pages(
anchor = f"sha1-{sha1[:12]}"
span = "" if anchor in member_anchors else f''
member_anchors.add(anchor)
- lines.append(
- f" | {span}{_code(member)} "
+ table.append(
+ f"| {span}{_code(member)} "
f"| {_fmt_size(int(info.get('size', 0)))} | {_code(sha1)} |"
)
lines.append("")
+ lines.extend(markup.details(
+ "info", f"{_count(len(members), 'file')}, as the packs ship them",
+ table, indent=" "))
lines.append("")
+ if markup.starlight():
+ # The member tables are the lookup's, as the heading's attribute
+ # keeps them under MkDocs.
+ lines[trees_start:] = search_excluded(lines[trees_start:])
lines.extend(_next_block([
"[Get your files](../which-pack.md), for the setup you play on",
@@ -4927,7 +4866,7 @@ def _card_index(
+ ", ".join(f"[{number}]({stem}.md)" for number, (stem, _e) in enumerate(parts, 1))
+ ".",
"",
- '',
+ markup.block_open("section", f'class="rb-card-index" {markup.search_exclude()}'),
"",
"| File | Required | Category |",
"|------|----------|----------|",
@@ -5302,9 +5241,11 @@ def generate_gap_analysis(
"\"Files to source\", [adding it](contributing.md) fills the gap for "
"every platform at once.",
"",
- "[Download gaps CSV](downloads/gaps.csv){ .md-button } "
- "[Open gaps API](api/v1/gaps.json){ .md-button } "
- "[All data exports](data.md){ .md-button }",
+ " ".join([
+ markup.button("Download gaps CSV", "downloads/gaps.csv"),
+ markup.button("Open gaps API", "api/v1/gaps.json"),
+ markup.button("All data exports", "data.md"),
+ ]),
"",
_platform_lists_sentence(
total_files_verify, total_verified, total_untested, total_missing_verify,
@@ -5733,13 +5674,7 @@ def _render_xref_by_platform(
display = cov["platform"]
config = cov["config"]
- lines.append(f'??? abstract "{display}"')
- lines.append("")
- lines.append(
- f" [Open {display} platform profile](platforms/{pname}.md)"
- "{ .md-button }"
- )
- lines.append("")
+ body = [markup.button(f"Open {display} platform profile", f"platforms/{pname}.md"), ""]
# The cores the builder and verify resolve: a hand copy here dropped
# the libretro set a list naming retroarch pulls in, and the system
@@ -5750,11 +5685,11 @@ def _render_xref_by_platform(
if key in unique
}
- lines.append(f" **{len(matched)} cores**{_core_scope_note(config, unique)}")
- lines.append("")
+ body.append(f"**{len(matched)} cores**{_core_scope_note(config, unique)}")
+ body.append("")
- lines.append(" | Core | Classification | Systems | Files | Upstream |")
- lines.append(" |------|---------------|---------|-------|----------|")
+ body.append("| Core | Classification | Systems | Files | Upstream |")
+ body.append("|------|---------------|---------|-------|----------|")
for emu_name in sorted(matched.keys()):
p = matched[emu_name]
@@ -5798,12 +5733,12 @@ def _render_xref_by_platform(
elif source:
upstream_display = source
- lines.append(
- f" | [{emu_display}](emulators/{emu_name}.md) | {cls} | "
+ body.append(
+ f"| [{emu_display}](emulators/{emu_name}.md) | {cls} | "
f"{sys_str} | {file_str} | {upstream_display} |"
)
- lines.append("")
+ lines.extend(markup.details("abstract", display, body))
return lines
@@ -5892,9 +5827,11 @@ def generate_cross_reference(
"The files a platform's emulators load and its own list does not name are "
"counted on [What the packs do not hold](gaps.md).",
"",
- "[Download cross-reference CSV](downloads/cross-reference.csv){ .md-button } "
- "[Open emulator API](api/v1/emulators.json){ .md-button } "
- "[All data exports](data.md){ .md-button }",
+ " ".join([
+ markup.button("Download cross-reference CSV", "downloads/cross-reference.csv"),
+ markup.button("Open emulator API", "api/v1/emulators.json"),
+ markup.button("All data exports", "data.md"),
+ ]),
"",
]
@@ -5915,13 +5852,11 @@ def generate_contributing() -> str:
# The end of a reference page: back to the index, or on to contributing.
-REFERENCE_NEXT = "\n" + "\n".join([
- '!!! tip "Next"',
- "",
- " - [Reference index](wiki/index.md), every reference page in one list",
- " - For contributors: [How to contribute](contributing.md)",
- "",
-])
+def reference_next() -> str:
+ return "\n" + "\n".join(markup.aside("tip", "Next", [
+ "- [Reference index](wiki/index.md), every reference page in one list",
+ "- For contributors: [How to contribute](contributing.md)",
+ ]))
# Wiki pages
@@ -6238,34 +6173,26 @@ def generate_which_pack() -> str:
"",
*terminal_tabs(("windows", "macos", "linux", "steamdeck", "android"),
dict.fromkeys(TERMINALS, "--check"), steps=False),
- '??? note "Fill an SD card from a computer"',
- "",
- " Name your setup and the BIOS folder on the card. For RetroArch on a "
- "Nintendo Switch:",
- "",
- " ```sh",
- " " + install_command(
- "sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system"),
- " ```",
- "",
+ *markup.details("note", "Fill an SD card from a computer", [
+ "Name your setup and the BIOS folder on the card. For RetroArch on a "
+ "Nintendo Switch:",
+ "",
+ "```sh",
+ install_command(
+ "sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system"),
+ "```",
+ ]),
"Details: [All installer options](wiki/installer.md).",
"",
"## Pick your setup",
"",
- '
',
- "",
]
- for group, (label, _description) in DEVICE_GROUPS.items():
- members = _group_keys(registry, group, display)
- if not members:
- continue
- lines += [
- f"- **{label}**",
- "",
- f" [{', '.join(display[k] for k in members)}](#{GROUP_ANCHORS[group]})",
- "",
- ]
- lines += ["
", ""]
+ lines += markup.card_grid([
+ (label, [f"[{', '.join(display[k] for k in members)}](#{GROUP_ANCHORS[group]})"])
+ for group, (label, _description) in DEVICE_GROUPS.items()
+ for members in [_group_keys(registry, group, display)]
+ if members
+ ])
for group, (label, description) in DEVICE_GROUPS.items():
members = _group_keys(registry, group, display)
if not members:
@@ -6294,18 +6221,15 @@ def generate_which_pack() -> str:
)
lines.append("")
if group == "handheld":
- lines += [
- '??? note "Other handhelds that run RetroArch"',
- "",
- " They use the RetroArch files, in the folder below. The command "
+ lines += markup.details("note", "Other handhelds that run RetroArch", [
+ "They use the RetroArch files, in the folder below. The command "
"finds it.",
"",
- " | System | Runs | BIOS folder |",
- " |--------|------|-------------|",
- *(f" | {name} | [RetroArch](platforms/retroarch.md) | `{folder}` |"
+ "| System | Runs | BIOS folder |",
+ "|--------|------|-------------|",
+ *(f"| {name} | [RetroArch](platforms/retroarch.md) | `{folder}` |"
for name, folder in _detected_systems(registry, "retroarch")),
- "",
- ]
+ ])
lines += [
"## Download a ZIP { #manual-download }",
"",
@@ -6699,7 +6623,7 @@ def _generate(args: argparse.Namespace) -> None:
exports = generate_data_exports(
docs, db, coverages, profiles, stats, gap_report
)
- write_if_changed(str(docs / "data.md"), generate_data_page(exports) + REFERENCE_NEXT)
+ write_if_changed(str(docs / "data.md"), generate_data_page(exports) + reference_next())
PAGE_DATA["data.md"] = dataset_structured_data(exports, db)
export_reports = load_export_reports()
@@ -6774,7 +6698,7 @@ def _generate(args: argparse.Namespace) -> None:
print("Generating cross-reference page...")
write_if_changed(
str(docs / "cross-reference.md"),
- tables_out_of_search(generate_cross_reference(coverages, profiles)) + REFERENCE_NEXT,
+ tables_out_of_search(generate_cross_reference(coverages, profiles)) + reference_next(),
)
print("Generating gap analysis page...")
@@ -6782,7 +6706,7 @@ def _generate(args: argparse.Namespace) -> None:
str(docs / "gaps.md"),
tables_out_of_search(generate_gap_analysis(
profiles, coverages, db, suppl_names, registry, gap_report
- )) + REFERENCE_NEXT,
+ )) + reference_next(),
)
print("Generating updated platform files page...")
@@ -6791,7 +6715,7 @@ def _generate(args: argparse.Namespace) -> None:
generate_exports_page(
export_reports,
{name: cov["platform"] for name, cov in coverages.items()},
- ) + REFERENCE_NEXT,
+ ) + reference_next(),
)
print("Generating dump provenance page...")
@@ -6800,7 +6724,7 @@ def _generate(args: argparse.Namespace) -> None:
str(docs / "provenance.md"),
tables_out_of_search(
generate_provenance_page(db, provenance_report, load_derivable())
- ) + REFERENCE_NEXT,
+ ) + reference_next(),
)
# Wiki pages: copy manually maintained sources + generate dynamic ones
diff --git a/scripts/sitediagrams.py b/scripts/sitediagrams.py
index 99f28365..4d5d5f15 100644
--- a/scripts/sitediagrams.py
+++ b/scripts/sitediagrams.py
@@ -36,6 +36,8 @@ from xml.sax.saxutils import escape
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
+import sitemarkup as markup # noqa: E402
+
PLACEHOLDER = re.compile(r"(?m)^([ \t]*)[ \t]*$")
PLACEHOLDER_ANYWHERE = re.compile(r"")
@@ -605,12 +607,12 @@ def figure_html(image_id: str, occurrence: int = 1, inline: bool = False) -> str
anchor = f"rb-fig-{image_id}{suffix}"
if inline:
html = (
- f''
+ f''
f'{drawings}{caption}'
)
return html.translate(_INLINE_ESCAPES)
return (
- f'\n'
+ f'\n'
f"{drawings}\n"
f"{caption}\n"
""
diff --git a/scripts/siteexports.py b/scripts/siteexports.py
index bf85862b..5b02b6e9 100644
--- a/scripts/siteexports.py
+++ b/scripts/siteexports.py
@@ -11,6 +11,8 @@ from __future__ import annotations
import json
from pathlib import Path
+import sitemarkup as markup
+
EXPORTS_DIR = "exports"
REPO_BLOB = "https://github.com/Abdess/retrobios/blob/main"
REPO_TREE = "https://github.com/Abdess/retrobios/tree/main"
@@ -73,12 +75,8 @@ def requirement_lines(report: dict) -> list[str]:
]
if not items:
return []
- return [
- f'??? note "Required flags aligned with the code ({len(items)})"',
- "",
- *[f" - {item}" for item in items],
- "",
- ]
+ return markup.details("note", f"Required flags aligned with the code ({len(items)})",
+ [f"- {item}" for item in items])
def _plural(count: int, noun: str, plural: str = "") -> str:
@@ -141,12 +139,8 @@ def candidate_lines(candidates: list[dict]) -> list[str]:
title = (reason[:1].upper() + reason[1:]).replace('"', "'")
items = [f"`{c['system']}` `{c['destination']}`" for c in entries]
if len(entries) > LONG_LIST:
- lines.extend([
- f'??? note "{title} ({len(entries)})"',
- "",
- *[f" - {item}" for item in items],
- "",
- ])
+ lines.extend(markup.details("note", f"{title} ({len(entries)})",
+ [f"- {item}" for item in items]))
else:
lines.extend([f"{title}:", "", *[f"- {item}" for item in items], ""])
return lines
@@ -263,35 +257,29 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
])
lines.extend(candidate_lines(report.get("removal_candidates") or []))
if report.get("invalid"):
- lines.extend([
- '!!! warning "The updated file does not pass the platform\'s own checks"',
- "",
- *[f" - {issue}" for issue in report["invalid"]],
- "",
- ])
+ lines.extend(markup.aside(
+ "warning", "The updated file does not pass the platform's own checks",
+ [f"- {issue}" for issue in report["invalid"]]))
originals = report.get("originals") or {}
files = report["files"]
raw = [f"{REPO_RAW}/{EXPORTS_DIR}/{platform}/{name}" for name in files]
if len(files) == 1:
lines.extend([
- f"[Download the updated file]({raw[0]}){{ .md-button .md-button--primary }} "
- f"[See it on GitHub]({REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}){{ .md-button }}",
+ markup.button("Download the updated file", raw[0], primary=True) + " "
+ + markup.button("See it on GitHub", f"{REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}"),
"",
])
else:
lines.extend([
- f"[Browse the {len(files)} updated files]"
- f"({REPO_TREE}/{EXPORTS_DIR}/{platform}){{ .md-button .md-button--primary }}",
+ markup.button(f"Browse the {len(files)} updated files",
+ f"{REPO_TREE}/{EXPORTS_DIR}/{platform}", primary=True),
"",
])
if len(originals) > 1:
- lines.extend([
- f'??? note "The {len(originals)} original files it is based on"',
- "",
- *[f" - [`{name}`]({url})" for name, url in sorted(originals.items())],
- "",
- ])
+ lines.extend(markup.details(
+ "note", f"The {len(originals)} original files it is based on",
+ [f"- [`{name}`]({url})" for name, url in sorted(originals.items())]))
elif originals:
(name, url), = originals.items()
lines.extend([f"Based on [`{name}`]({url}).", ""])
@@ -301,10 +289,6 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
items = change_items(report, key)
if not items:
continue
- lines.extend([
- f'??? note "{title} ({len(items)})"',
- "",
- *[f" - {item}" for item in items],
- "",
- ])
+ lines.extend(markup.details("note", f"{title} ({len(items)})",
+ [f"- {item}" for item in items]))
return "\n".join(lines)
diff --git a/scripts/siteguides.py b/scripts/siteguides.py
index 990d435c..f0f4fe5b 100644
--- a/scripts/siteguides.py
+++ b/scripts/siteguides.py
@@ -15,6 +15,7 @@ import textwrap
from dataclasses import dataclass, fields
from pathlib import Path
+import sitemarkup as markup
from safeparse import require_yaml
PLATFORMS_DIR = Path(__file__).resolve().parent.parent / "platforms"
@@ -263,15 +264,12 @@ def terminal_tabs(
"""Content tabs, one per place the command runs, each with how to open
the terminal there. Tab labels are shared across the site, so the reader's
choice carries from page to page."""
- lines: list[str] = []
+ items = []
for key in tabs:
label, how, shell = TERMINALS[key]
- lines += [f'{indent}=== "{label}"', ""]
- if steps:
- lines += [f"{indent} {how}", ""]
- lines += _fence(shell, install_command(shell, args.get(key, "")), indent + " ")
- lines.append("")
- return lines
+ body = [how, ""] if steps else []
+ items.append((label, body + _fence(shell, install_command(shell, args.get(key, "")), "")))
+ return markup.tabs(items, indent)
def platform_args(name: str, guide: Guide) -> dict[str, str]:
@@ -290,11 +288,8 @@ def platform_args(name: str, guide: Guide) -> dict[str, str]:
def trust_box(prefix: str, indent: str = "") -> list[str]:
"""Is it safe, under every command on a guide page: five plain facts."""
- return [
- f'{indent}??? note "Is it safe?"',
+ return markup.details("note", "Is it safe?", [
+ *(f"- {item}" for item in TRUST_ITEMS),
"",
- *(f"{indent} - {item}" for item in TRUST_ITEMS),
- "",
- f"{indent} [Read the script first]({prefix}wiki/faq.md#is-piping-the-installer-into-a-shell-safe)",
- "",
- ]
+ f"[Read the script first]({prefix}wiki/faq.md#is-piping-the-installer-into-a-shell-safe)",
+ ], indent)
diff --git a/scripts/sitemarkup.py b/scripts/sitemarkup.py
new file mode 100644
index 00000000..2116b556
--- /dev/null
+++ b/scripts/sitemarkup.py
@@ -0,0 +1,183 @@
+"""The page constructs the generator writes, in the syntax of the site engine.
+
+Every block that is not plain Markdown (a callout, a collapsible block, OS
+tabs, a button, a card grid) is written through one of these functions, so
+the generator holds no engine syntax of its own. Bodies are passed without
+indentation: the function indents them as the engine reads them.
+
+The engine is Astro Starlight. MkDocs Material is kept while the two builds
+are compared page by page, selected with set_engine("mkdocs").
+"""
+
+from __future__ import annotations
+
+import html
+import re
+
+ENGINES = ("mkdocs", "starlight")
+_engine = "mkdocs"
+
+# Material's callout kinds, by the role Starlight gives them.
+ASIDE_ROLES = {
+ "note": "note", "info": "note", "abstract": "note", "example": "note",
+ "question": "note", "quote": "note", "tip": "tip", "success": "tip",
+ "warning": "caution", "failure": "danger", "danger": "danger", "bug": "danger",
+}
+
+# The attribute that keeps an element out of the site search.
+SEARCH_EXCLUDE = {"mkdocs": "data-search-exclude", "starlight": "data-pagefind-ignore"}
+
+
+def set_engine(name: str) -> None:
+ global _engine
+ if name not in ENGINES:
+ raise ValueError(f"unknown site engine {name!r}, expected one of {ENGINES}")
+ _engine = name
+
+
+def engine() -> str:
+ return _engine
+
+
+def starlight() -> bool:
+ return _engine == "starlight"
+
+
+def search_exclude() -> str:
+ return SEARCH_EXCLUDE[_engine]
+
+
+def inline_html(text: str) -> str:
+ """Inline Markdown (code, bold, links) for text placed inside raw HTML,
+ where Markdown is not read."""
+ parts = re.split(r"(`[^`]+`)", text)
+ out = []
+ for part in parts:
+ if part.startswith("`") and part.endswith("`") and len(part) > 1:
+ out.append(f"{html.escape(part[1:-1], quote=False)}")
+ continue
+ part = html.escape(part, quote=False)
+ part = re.sub(r"\*\*([^*]+)\*\*", r"\1", part)
+ part = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", r'\1', part)
+ out.append(part)
+ return "".join(out)
+
+
+def _indented(lines: list[str], pad: str) -> list[str]:
+ return [f"{pad}{line}" if line.strip() else "" for line in lines]
+
+
+def _trim(lines: list[str]) -> list[str]:
+ start, end = 0, len(lines)
+ while start < end and not lines[start].strip():
+ start += 1
+ while end > start and not lines[end - 1].strip():
+ end -= 1
+ return list(lines[start:end])
+
+
+def aside(kind: str, title: str, body: list[str], indent: str = "") -> list[str]:
+ """A callout that is always open."""
+ body = _trim(body)
+ if _engine == "mkdocs":
+ return [f'{indent}!!! {kind} "{title}"', "", *_indented(body, indent + " "), ""]
+ label = title.replace("[", "\\[").replace("]", "\\]")
+ fence = "::::" if any(line.lstrip().startswith(":::") for line in body) else ":::"
+ return [f"{indent}{fence}{ASIDE_ROLES.get(kind, 'note')}[{label}]",
+ *_indented(body, indent), f"{indent}{fence}", ""]
+
+
+def details(kind: str, title: str, body: list[str], indent: str = "", opened: bool = False) -> list[str]:
+ """A collapsible block, closed unless *opened*."""
+ body = _trim(body)
+ if _engine == "mkdocs":
+ marker = "???+" if opened else "???"
+ return [f'{indent}{marker} {kind} "{title}"', "", *_indented(body, indent + " "), ""]
+ role = ASIDE_ROLES.get(kind, "note")
+ state = " open" if opened else ""
+ return [
+ f'{indent}',
+ f"{indent}{inline_html(title)}",
+ "",
+ *_indented(body, indent),
+ "",
+ f"{indent}",
+ "",
+ ]
+
+
+def tabs(items: list[tuple[str, list[str]]], indent: str = "", sync: str = "os") -> list[str]:
+ """Content tabs. Tabs that share a label share the reader's choice
+ across pages."""
+ lines: list[str] = []
+ if _engine == "mkdocs":
+ for label, body in items:
+ lines += [f'{indent}=== "{label}"', "", *_indented(_trim(body), indent + " "), ""]
+ return lines
+ lines.append(f'{indent}')
+ for label, body in items:
+ lines += [f'{indent}', "",
+ *_indented(_trim(body), indent), "", f"{indent}"]
+ return [*lines, f"{indent}", ""]
+
+
+def button(text: str, target: str, primary: bool = False) -> str:
+ """A link drawn as a button."""
+ if _engine == "mkdocs":
+ classes = ".md-button .md-button--primary" if primary else ".md-button"
+ return f"[{text}]({target}){{ {classes} }}"
+ classes = "rb-button rb-button-primary" if primary else "rb-button"
+ return f'{inline_html(text)}'
+
+
+def image(alt: str, src: str, **attributes: str) -> str:
+ """An image with HTML attributes (width, align)."""
+ if _engine == "mkdocs":
+ attrs = " ".join(f"{key}={value}" for key, value in attributes.items())
+ return f"{{ {attrs} }}"
+ attrs = "".join(f' {key}="{html.escape(str(value))}"' for key, value in attributes.items())
+ return f''
+
+
+def block_open(tag: str, attributes: str = "") -> str:
+ """The opening tag of an HTML block whose content is Markdown."""
+ attributes = f" {attributes.strip()}" if attributes.strip() else ""
+ if _engine == "mkdocs":
+ return f"<{tag}{attributes} markdown>"
+ return f"<{tag}{attributes}>"
+
+
+def card_grid(cards: list[tuple[str, list[str]]]) -> list[str]:
+ """Cards in a grid: a bold title line and a body each."""
+ lines = [block_open("div", 'class="grid cards"' if _engine == "mkdocs" else 'class="rb-cards"'), ""]
+ for title, body in cards:
+ if _engine == "mkdocs":
+ lines += [f"- **{title}**", "", *_indented(_trim(body), " "), ""]
+ else:
+ lines += ['
', "", f"**{title}**", "", *_trim(body), "", "
", ""]
+ return [*lines, "
", ""]
+
+
+def steps(items: list[str]) -> list[str]:
+ """An ordered list drawn as numbered steps."""
+ return [block_open("div", 'class="rb-steps"'), "",
+ *(f"{number}. {item}" for number, item in enumerate(items, 1)), "", "
", ""]
+
+
+def heading_attributes(anchor: str = "", classes: str = "", search: bool = True) -> str:
+ """The attribute block of a heading: its id, classes, and whether the
+ search reads it."""
+ parts = [f"#{anchor}" if anchor else "", *(f".{c}" for c in classes.split())]
+ if not search:
+ parts.append(search_exclude())
+ parts = [part for part in parts if part]
+ return f" {{ {' '.join(parts)} }}" if parts else ""
+
+
+def site_href(path: str) -> str:
+ """A link to a site address written in raw HTML (``files/?sha1=...``),
+ which no Markdown resolution reaches: rooted at the site under Starlight,
+ relative to a page two levels deep under MkDocs."""
+ if _engine == "mkdocs":
+ return f"../../{path}"
+ return f"/retrobios/{path}"
From 342d0f727135692afbfe3ceb73f8c0b7988d5387 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Thu, 22 Oct 2026 15:41:39 +0200
Subject: [PATCH 2/6] feat: write the site for Starlight behind a flag
generate_site.py --engine starlight writes the pages, navigation and
data files of an Astro Starlight project in website/. Links resolve to
site addresses, every heading keeps its MkDocs id, and pages split by
size are measured in the MkDocs Markdown so both builds cut the same
files. The 839 pages and their 45,613 ids match the MkDocs build.
---
.github/workflows/deploy-site.yml | 2 +
scripts/generate_site.py | 250 +-
scripts/sitemarkup.py | 14 +
scripts/siterender.py | 4 +-
scripts/sitestarlight.py | 572 ++
website/.gitignore | 7 +
website/astro.config.mjs | 85 +
website/package-lock.json | 5403 ++++++++++++++++++
website/package.json | 17 +
website/src/assets/logo-dark.svg | 1 +
website/src/assets/logo-light.svg | 1 +
website/src/components/FilePicker.astro | 186 +
website/src/components/Footer.astro | 10 +
website/src/components/MarkdownContent.astro | 21 +
website/src/components/PageTitle.astro | 40 +
website/src/content.config.ts | 22 +
website/src/env.d.ts | 7 +
website/src/routeData.ts | 96 +
website/src/scripts/site.ts | 275 +
website/src/styles/retrobios.css | 557 ++
website/tools/axe.mjs | 24 +
website/tools/check_tools.mjs | 44 +
website/tools/lighthouse.py | 57 +
website/tools/measure.mjs | 106 +
website/tools/serve.mjs | 51 +
website/tools/shots.mjs | 25 +
26 files changed, 7817 insertions(+), 60 deletions(-)
create mode 100644 scripts/sitestarlight.py
create mode 100644 website/.gitignore
create mode 100644 website/astro.config.mjs
create mode 100644 website/package-lock.json
create mode 100644 website/package.json
create mode 100644 website/src/assets/logo-dark.svg
create mode 100644 website/src/assets/logo-light.svg
create mode 100644 website/src/components/FilePicker.astro
create mode 100644 website/src/components/Footer.astro
create mode 100644 website/src/components/MarkdownContent.astro
create mode 100644 website/src/components/PageTitle.astro
create mode 100644 website/src/content.config.ts
create mode 100644 website/src/env.d.ts
create mode 100644 website/src/routeData.ts
create mode 100644 website/src/scripts/site.ts
create mode 100644 website/src/styles/retrobios.css
create mode 100644 website/tools/axe.mjs
create mode 100644 website/tools/check_tools.mjs
create mode 100644 website/tools/lighthouse.py
create mode 100644 website/tools/measure.mjs
create mode 100644 website/tools/serve.mjs
create mode 100644 website/tools/shots.mjs
diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml
index 22c6dd54..27ca1287 100644
--- a/.github/workflows/deploy-site.yml
+++ b/.github/workflows/deploy-site.yml
@@ -45,9 +45,11 @@ on:
- "scripts/sect233r1.py"
- "scripts/sitediagrams.py"
- "scripts/siteguides.py"
+ - "scripts/sitemarkup.py"
- "scripts/siteexports.py"
- "scripts/siterender.py"
- "scripts/sitesources.py"
+ - "scripts/sitestarlight.py"
- "scripts/sitewrite.py"
- "scripts/slot.py"
- "scripts/slots.py"
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index f8e6da82..bd41ec6c 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -74,6 +74,7 @@ from generate_readme import (
)
from provenance_report import build_report, distinct_missing
import sitemarkup as markup
+import sitestarlight
from siteexports import generate_exports_page, load_export_reports
from siteguides import (
ALSO_RUNS,
@@ -91,11 +92,16 @@ from siteguides import (
import release_record
DOCS_DIR = "docs"
+# The Starlight project, and where its pages go inside it.
+STARLIGHT_DIR = "website"
+STARLIGHT_CONTENT = "src/content/docs"
SITE_NAME = "RetroBIOS"
REPO_URL = "https://github.com/Abdess/retrobios"
RELEASE_URL = f"{REPO_URL}/releases/latest"
SITE_URL = "https://abdess.github.io/retrobios/"
GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads"]
+# Generated directories of files served as they are, not pages.
+PUBLIC_DIRS = ("api", "downloads")
@@ -1873,8 +1879,15 @@ def _front_matter(markdown: str) -> dict:
return data if isinstance(data, dict) else {}
-def decorate_markdown_pages(docs: Path) -> None:
- """Add per-page descriptions and machine-readable structured data."""
+def decorate_markdown_pages(docs: Path) -> dict[str, str]:
+ """Add per-page descriptions and machine-readable structured data.
+
+ Under Starlight each page is written in its final form (front matter,
+ site addresses, pinned heading ids) and the dates a page states are
+ returned by address, for the sitemap.
+ """
+ lastmod: dict[str, str] = {}
+ terms = sitestarlight.glossary_terms(_REPO_ROOT / WIKI_SRC_DIR / "glossary.md")
for path in sorted(docs.rglob("*.md")):
relative = path.relative_to(docs)
if any(part in LOCAL_NOTE_DIRS for part in relative.parts[:-1]):
@@ -1918,6 +1931,23 @@ def decorate_markdown_pages(docs: Path) -> None:
},
**PAGE_DATA.get(relative.as_posix(), {}),
}
+ if markup.starlight():
+ target, text = sitestarlight.finalize_page(
+ relative.as_posix(), markdown, level=level,
+ boost=SEARCH_BOOST.get(level), browser_title=browser_title,
+ description=description, structured=structured, terms=terms,
+ search=authored.get("search") != "exclude",
+ )
+ write_decorated(docs / target, text)
+ # One file per page: a page that gained or lost its components
+ # changed extension, and the other one would claim its address.
+ stem = relative.with_suffix("").as_posix()
+ for suffix in (".md", ".mdx"):
+ if f"{stem}{suffix}" != target:
+ (docs / f"{stem}{suffix}").unlink(missing_ok=True)
+ if structured.get("dateModified"):
+ lastmod[sitestarlight.page_url(target)] = str(structured["dateModified"])
+ continue
structured_json = json.dumps(
structured, ensure_ascii=False, separators=(",", ":")
).replace("", "<\\/")
@@ -1937,6 +1967,7 @@ def decorate_markdown_pages(docs: Path) -> None:
+ (f"{level_label(level)}\n\n" if level else "")
)
write_decorated(path, front_matter + markdown)
+ return lastmod
# Platform pages
@@ -2408,7 +2439,7 @@ def generate_platform_page(
logo_url = registry.get(name, {}).get("logo", "")
logo_md = (
- f"{{ width=48 align=right }}\n\n" if logo_url else ""
+ markup.image(display, logo_url, width="48", align="right") + "\n\n" if logo_url else ""
)
homepage = config.get("homepage", "")
@@ -3467,12 +3498,13 @@ def _cards_markdown(cards: list[Card]) -> list[str]:
return lines
-def _parts(cards: list[Card], budget: int) -> list[list[Card]]:
- """Cards in runs of at most ``budget`` bytes of Markdown, in order."""
+def _parts(cards: list[Card], budget: int, sizes: list[int] | None = None) -> list[list[Card]]:
+ """Cards in runs of at most ``budget`` bytes of Markdown, in order.
+ ``sizes`` gives each card's weight when it is not its own Markdown."""
parts: list[list[Card]] = [[]]
size = 0
- for card in cards:
- weight = sum(len(line) + 1 for line in card[1])
+ for index, card in enumerate(cards):
+ weight = sizes[index] if sizes else sum(len(line) + 1 for line in card[1])
if parts[-1] and size + weight > budget:
parts.append([])
size = 0
@@ -3507,11 +3539,11 @@ def system_pages(
"""
slug = _slug(manufacturer)
json_link = (f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json)")
- whole_anchors: set[str] = set()
- sections = {
- console: _console_cards(manufacturer, consoles[console], consumers, whole_anchors)
- for console in sorted(consoles)
- }
+ def all_sections() -> dict[str, list[Card]]:
+ anchors: set[str] = set()
+ return {console: _console_cards(manufacturer, consoles[console], consumers, anchors)
+ for console in sorted(consoles)}
+
intro = (
f"Every BIOS file the collection holds for {manufacturer} machines, one "
"card per file. A card gives the file's fingerprints, the emulators that "
@@ -3520,29 +3552,40 @@ def system_pages(
"first, the others under \"Variants\". To install the files of one "
"console, open its \"Install\" box."
)
- lines = [
- f"# {manufacturer}", "", intro, "",
- f"The files below by console, with every hash, are also published as {json_link}.",
- "",
- ]
- for console, cards in sections.items():
- lines.append(f"## {system_icon_markdown(manufacturer, console)}{console}")
- lines.append("")
- lines.extend(_console_install_block(console, (installs or {}).get(console, [])))
- lines.extend(_cards_markdown(cards))
- lines.append("")
- lines.extend(_next_block(_NEXT_FROM_SYSTEMS))
- lines.append(f'
Generated on {_timestamp()}.
')
- whole = "\n".join(lines) + "\n"
- if len(whole) <= budget:
- return {slug: (whole, consoles)}
+ def whole_page(sections: dict[str, list[Card]]) -> str:
+ lines = [
+ f"# {manufacturer}", "", intro, "",
+ f"The files below by console, with every hash, are also published as {json_link}.",
+ "",
+ ]
+ for console, cards in sections.items():
+ lines.append(f"## {system_icon_markdown(manufacturer, console)}{console}")
+ lines.append("")
+ lines.extend(_console_install_block(console, (installs or {}).get(console, [])))
+ lines.extend(_cards_markdown(cards))
+ lines.append("")
+ lines.extend(_next_block(_NEXT_FROM_SYSTEMS))
+ lines.append(f'
Generated on {_timestamp()}.
')
+ return "\n".join(lines) + "\n"
+
+ def card_weight(card: list[str]) -> int:
+ return sum(len(line) + 1 for line in card)
+
+ # Sizes are those of the MkDocs Markdown the budget was tuned on, so
+ # every engine splits the same pages at the same cards.
+ with markup.using("mkdocs"):
+ measured = all_sections()
+ measured_text = whole_page(measured)
+ if len(measured_text) <= budget:
+ text = whole_page(all_sections()) if markup.starlight() else measured_text
+ return {slug: (text, consoles)}
# The heaviest sections move to pages of their own until what stays on
# the manufacturer's page is half the budget: a section of one file
# keeps its place, and a reader does not open 91 pages to browse Other.
weights = {
- console: sum(len(line) + 1 for _variant, card, _sha1 in cards for line in card)
- for console, cards in sections.items()
+ console: sum(card_weight(card) for _variant, card, _sha1 in cards)
+ for console, cards in measured.items()
}
moved: set[str] = set()
for console in sorted(weights, key=lambda c: (-weights[c], c)):
@@ -3570,7 +3613,10 @@ def system_pages(
# Cards are rendered again for their own page: the anchors a page
# holds are its own.
cards = _console_cards(manufacturer, consoles[console], consumers, set())
- parts = _parts(cards, budget)
+ with markup.using("mkdocs"):
+ sizes = [card_weight(card) for _variant, card, _sha1
+ in _console_cards(manufacturer, consoles[console], consumers, set())]
+ parts = _parts(cards, budget, sizes)
base = f"{slug}-{_slugify_anchor(console) or 'files'}"
stems = [base if index == 0 else f"{base}-{index + 1}" for index in range(len(parts))]
record_console_pages(slug, console, stems, [
@@ -3616,7 +3662,11 @@ def system_pages(
def _part_range(part: list[Card]) -> str:
"""The first and last file names of a part, as the cards show them."""
- names = [re.search(r"`([^`]*)`", card[2]).group(1) for _variant, card, _sha1 in part]
+ names = []
+ for _variant, card, _sha1 in part:
+ # A Markdown heading under MkDocs, an HTML one under Starlight.
+ match = re.search(r"`([^`]*)`|(.*?)", card[2])
+ names.append(match[1] if match[1] is not None else html.unescape(match[2]))
if len(names) == 1:
return f"`{names[0]}`"
return f"{_count(len(names), 'file')}, `{names[0]}` to `{names[-1]}`"
@@ -5852,6 +5902,31 @@ def generate_contributing() -> str:
# The end of a reference page: back to the index, or on to contributing.
+def generate_not_found() -> str:
+ """The page a stale link lands on. Pages have moved before (file cards
+ refiled between manufacturers), so it offers the search, the two pages
+ a visitor most often wants, and the list a moved page belongs to."""
+ return "\n".join([
+ "---",
+ "search: exclude",
+ "---",
+ "",
+ "# Page not found",
+ "",
+ '
This address has no page. It may have moved since the link was written.
',
+ "",
+ '',
+ "",
+ "- [Start here](index.md): what BIOS files are and how to install them",
+ "- [Get your files](which-pack.md): the pack or the command for your device",
+ "- Browse [platforms](platforms/index.md), [systems](systems/index.md) "
+ "or [emulators](emulators/index.md)",
+ "",
+ '',
+ "",
+ ])
+
+
def reference_next() -> str:
return "\n" + "\n".join(markup.aside("tip", "Next", [
"- [Reference index](wiki/index.md), every reference page in one list",
@@ -6248,6 +6323,13 @@ def generate_which_pack() -> str:
"",
"Run the command and press **c** when it asks.",
"",
+ *([
+ "Or choose your setup and consoles here. You get the list of files "
+ "and a command that installs only those.",
+ "",
+ "",
+ "",
+ ] if markup.starlight() else []),
*_next_block([
"[First install, step by step](wiki/first-install.md)",
"[When a game does not start](wiki/troubleshooting.md)",
@@ -6467,10 +6549,23 @@ def main() -> None:
help="emulator profile directory (default: %(default)s)",
)
parser.add_argument(
- "--docs-dir", default=DOCS_DIR,
- help="directory the site sources are written to (default: %(default)s)",
+ "--engine", choices=markup.ENGINES, default="mkdocs",
+ help="site engine the pages are written for (default: %(default)s)",
+ )
+ parser.add_argument(
+ "--site-dir", default=STARLIGHT_DIR,
+ help="the Starlight project, with --engine starlight (default: %(default)s)",
+ )
+ parser.add_argument(
+ "--docs-dir", default=None,
+ help="directory the pages are written to (default: docs, or the "
+ "Starlight project's content collection)",
)
args = parser.parse_args()
+ markup.set_engine(args.engine)
+ if args.docs_dir is None:
+ args.docs_dir = (str(Path(args.site_dir) / STARLIGHT_CONTENT)
+ if markup.starlight() else DOCS_DIR)
try:
# Two runs over one docs/ delete each other's pages and scratch
# files in the sweep, so the second is refused instead.
@@ -6492,34 +6587,42 @@ def _generate(args: argparse.Namespace) -> None:
db = load_database(args.db)
docs = Path(args.docs_dir)
+ # Files the site serves as they are (data, downloads, images): beside the
+ # pages under MkDocs, in the project's public folder under Starlight.
+ public = Path(args.site_dir) / "public" if markup.starlight() else docs
for d in GENERATED_DIRS:
- (docs / d).mkdir(parents=True, exist_ok=True)
+ (public / d if d in PUBLIC_DIRS else docs / d).mkdir(parents=True, exist_ok=True)
- css_src = Path("docs_assets") / "extra.css"
- css_dest = docs / "stylesheets" / "extra.css"
- if css_src.exists():
- css_dest.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(css_src, css_dest)
- js_src = Path("docs_assets") / "site.js"
- js_dest = docs / "javascripts" / "site.js"
- if js_src.exists():
- js_dest.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(js_src, js_dest)
+ if not markup.starlight():
+ css_src = Path("docs_assets") / "extra.css"
+ css_dest = docs / "stylesheets" / "extra.css"
+ if css_src.exists():
+ css_dest.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(css_src, css_dest)
+ js_src = Path("docs_assets") / "site.js"
+ js_dest = docs / "javascripts" / "site.js"
+ if js_src.exists():
+ js_dest.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(js_src, js_dest)
# No robots.txt: a crawler reads it only at the host root (RFC 9309), and
# this project site lives under /retrobios/. Every page's head points at
# the sitemap instead (overrides/main.html).
- (docs / "robots.txt").unlink(missing_ok=True)
+ (public / "robots.txt").unlink(missing_ok=True)
- images_dest = docs / "assets" / "images"
+ images_dest = public / "assets" / "images"
images_dest.mkdir(parents=True, exist_ok=True)
# Logos, system icons and sharing cards are served from the site itself
# (docs_assets/*/sources.json records where each came from).
- for folder in ("logos", "system-icons", "og"):
+ folders = [("logos", "logos"), ("system-icons", "system-icons"), ("og", "og")]
+ if markup.starlight():
+ # The theme's own folder under MkDocs: the mark, favicons, touch icon.
+ folders.append(("overrides/assets/brand", "brand"))
+ for folder, dest in folders:
source = Path("docs_assets") / folder
if source.is_dir():
- shutil.copytree(source, docs / "assets" / folder, dirs_exist_ok=True)
+ shutil.copytree(source, public / "assets" / dest, dirs_exist_ok=True)
assets_src = Path(".github") / "assets"
for name, dest_name in [("logo.png", "logo.png"), ("favicon.png", "favicon.png")]:
src = assets_src / name
@@ -6572,7 +6675,7 @@ def _generate(args: argparse.Namespace) -> None:
stats = compute_stats(db, coverages, profiles)
stats["composition"] = compute_composition(db)
- write_if_changed(str(docs / "stats.json"), generate_stats(stats))
+ write_if_changed(str(public / "stats.json"), generate_stats(stats))
# Computed once: the gap analysis page and the published gaps export must
# report the same files.
@@ -6611,7 +6714,7 @@ def _generate(args: argparse.Namespace) -> None:
PART_PAGES.add(f"systems/{stem}.md")
write_if_changed(str(docs / "systems" / f"{stem}.md"), page)
write_single_item(
- docs / "api" / "v1" / "systems" / f"{slug}.json", "systems",
+ public / "api" / "v1" / "systems" / f"{slug}.json", "systems",
system_export_item(slug, mfr, consoles),
str(db.get("generated_at") or _timestamp()),
)
@@ -6621,7 +6724,7 @@ def _generate(args: argparse.Namespace) -> None:
print("Generating static API and bulk data exports...")
exports = generate_data_exports(
- docs, db, coverages, profiles, stats, gap_report
+ public, db, coverages, profiles, stats, gap_report
)
write_if_changed(str(docs / "data.md"), generate_data_page(exports) + reference_next())
PAGE_DATA["data.md"] = dataset_structured_data(exports, db)
@@ -6662,7 +6765,7 @@ def _generate(args: argparse.Namespace) -> None:
for name, profile in profiles.items()
if profile.get("type") not in ("alias", "test")
}
- (docs / "api" / "v1" / "emulators").mkdir(parents=True, exist_ok=True)
+ (public / "api" / "v1" / "emulators").mkdir(parents=True, exist_ok=True)
for name, profile in public_profiles.items():
# The profile itself, one small document per core: reusing one
# meant downloading emulators.json (5.9 MB) and filtering it.
@@ -6670,7 +6773,7 @@ def _generate(args: argparse.Namespace) -> None:
# profile validates against the published schema, which an "id"
# beside its fields broke on all 530, and schema_version is there.
write_if_changed(
- str(docs / "api" / "v1" / "emulators" / f"{name}.json"),
+ str(public / "api" / "v1" / "emulators" / f"{name}.json"),
json.dumps(
{
"schema_version": 1,
@@ -6734,7 +6837,12 @@ def _generate(args: argparse.Namespace) -> None:
wiki_src = Path(WIKI_SRC_DIR)
if wiki_src.is_dir():
for src_file in wiki_src.glob("*.md"):
- shutil.copy2(src_file, wiki_dest / src_file.name)
+ if markup.starlight():
+ # Written in Material syntax until the wiki is written for Starlight.
+ write_if_changed(str(wiki_dest / src_file.name),
+ sitestarlight.wiki_page(src_file.read_text(encoding="utf-8")))
+ else:
+ shutil.copy2(src_file, wiki_dest / src_file.name)
_record(wiki_dest / src_file.name)
# data-model.md is generated (contains live DB stats)
write_if_changed(
@@ -6750,15 +6858,39 @@ def _generate(args: argparse.Namespace) -> None:
# Swept before the decoration, which rewrites every page it finds: a
# page no generator produced any more (a console moved back onto its
# manufacturer's page) was decorated, recorded, and kept for ever.
- stale = _sweep_generated(docs, GENERATED_DIRS)
+ stale = _sweep_generated(docs, [d for d in GENERATED_DIRS if d not in PUBLIC_DIRS])
+ stale += _sweep_generated(public, list(PUBLIC_DIRS))
+
+ if markup.starlight():
+ write_if_changed(str(docs / "404.md"), generate_not_found())
print("Adding page metadata and structured data...")
- decorate_markdown_pages(docs)
+ lastmod = decorate_markdown_pages(docs)
+
+ nav = generate_mkdocs_nav(coverages, manufacturers, profiles, registry,
+ Path(WIKI_SRC_DIR))
+ if markup.starlight():
+ print("Writing the Starlight navigation and the picker's data...")
+ site = Path(args.site_dir)
+ present = {
+ path.relative_to(docs).with_suffix(".md").as_posix()
+ for path in docs.rglob("*.md*") if path.suffix in (".md", ".mdx")
+ } - {"404.md"}
+ generated = site / "src" / "generated"
+ generated.mkdir(parents=True, exist_ok=True)
+ write_if_changed(str(generated / "sidebar.json"),
+ json.dumps(sitestarlight.sidebar(nav, present), indent=1) + "\n")
+ write_if_changed(str(generated / "lastmod.json"),
+ json.dumps(dict(sorted(lastmod.items())), indent=1) + "\n")
+ sitestarlight.picker_data(_REPO_ROOT / "install", public / "data" / "picker")
+ total_pages = len(present)
+ print(f"\nGenerated {total_pages} pages in {args.docs_dir}/")
+ if stale:
+ print(f"Removed {stale} page(s) nothing produced this run")
+ return
# Update mkdocs.yml nav section only (avoid yaml.dump round-trip mangling quotes)
print("Updating mkdocs.yml nav...")
- nav = generate_mkdocs_nav(coverages, manufacturers, profiles, registry,
- Path(WIKI_SRC_DIR))
nav_yaml = yaml.dump(
{"nav": nav}, default_flow_style=False, sort_keys=False, allow_unicode=True
)
diff --git a/scripts/sitemarkup.py b/scripts/sitemarkup.py
index 2116b556..14362806 100644
--- a/scripts/sitemarkup.py
+++ b/scripts/sitemarkup.py
@@ -13,6 +13,7 @@ from __future__ import annotations
import html
import re
+from contextlib import contextmanager
ENGINES = ("mkdocs", "starlight")
_engine = "mkdocs"
@@ -39,6 +40,19 @@ def engine() -> str:
return _engine
+@contextmanager
+def using(name: str):
+ """Write in another engine's syntax for a while. A page split by size
+ is measured in the MkDocs Markdown its budgets were tuned on, so both
+ engines cut the same pages at the same files."""
+ previous = _engine
+ set_engine(name)
+ try:
+ yield
+ finally:
+ set_engine(previous)
+
+
def starlight() -> bool:
return _engine == "starlight"
diff --git a/scripts/siterender.py b/scripts/siterender.py
index 0683ad43..16d48ec9 100644
--- a/scripts/siterender.py
+++ b/scripts/siterender.py
@@ -7,6 +7,8 @@ import urllib.parse
from collections.abc import Callable
from pathlib import Path
+import sitemarkup as markup
+
WIKI_SRC_DIR = "wiki" # manually maintained wiki sources
SYSTEM_ICON_BASE = "https://raw.githubusercontent.com/libretro/retroarch-assets/master/xmb/systematic/png"
@@ -89,7 +91,7 @@ def system_icon_markdown(manufacturer: str, console_name: str) -> str:
name = _icon_name(manufacturer, console_name)
if not _icon_available.get(name):
return ""
- return f"}){{ width=24 }} "
+ return markup.image(console_name, _icon_url(name), width="24") + " "
# Global index: maps system_id -> (manufacturer_slug, console_name) for cross-linking
_system_page_map: dict[str, tuple[str, str]] = {}
diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py
new file mode 100644
index 00000000..324ae18c
--- /dev/null
+++ b/scripts/sitestarlight.py
@@ -0,0 +1,572 @@
+"""The site as an Astro Starlight project: pages, sidebar and data files.
+
+generate_site.py writes each page body; this module gives it the form
+Starlight reads. A page's links are written as the docs tree holds them
+(``../systems/sony.md``, MkDocs' convention, the one a Markdown editor
+follows), and the site serves them as addresses under /retrobios/, so
+each target is resolved against the page's own place here. Every heading
+keeps the id the MkDocs site gave it, so the anchors cited since stay
+valid.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+import re
+import unicodedata
+import urllib.parse
+from html import unescape
+from pathlib import Path, PurePosixPath
+
+BASE = "/retrobios/"
+SITE_URL = "https://abdess.github.io/retrobios/"
+
+# Components a page body may use, and where each comes from.
+COMPONENTS = {
+ "Tabs": "@astrojs/starlight/components",
+ "TabItem": "@astrojs/starlight/components",
+ "FilePicker": "components/FilePicker.astro",
+}
+
+# Pages of file cards: level 3 headings by the hundred, so the table of
+# contents stops at level 2.
+CARD_PAGES = ("systems/", "emulators/", "platforms/")
+
+FENCE = re.compile(r"(^[ \t]*(`{3,}|~{3,}).*?^[ \t]*\2[ \t]*$)", re.M | re.S)
+CODE_SPAN = re.compile(r"(`+)(?!`)(.+?)(?!?)\[(?P(?:[^\[\]\n]|\[[^\]\n]*\])*)\]\((?P[^)\s]+)(?P \"[^\"]*\")?\)")
+HTML_URL = re.compile(r"(?P\b(?:href|src))=\"(?P[^\"]*)\"")
+ATTR_BLOCK = re.compile(r"[ \t]*\{(?P[^{}]*)\}[ \t]*$")
+ID_ATTR = re.compile(r"\bid=\"(?P[^\"]+)\"")
+IDCOUNT = re.compile(r"^(.*)_([0-9]+)$")
+
+
+# Addresses
+
+
+def page_url(rel: str) -> str:
+ """The address of a page, by its path under the docs tree."""
+ stem = rel.rsplit(".", 1)[0] if rel.endswith((".md", ".mdx")) else rel
+ if stem == "index":
+ return BASE
+ if stem.endswith("/index"):
+ stem = stem[: -len("/index")]
+ return f"{BASE}{stem}/"
+
+
+def resolve(rel_page: str, target: str) -> str:
+ """A link target written relative to the page's file, as a site address."""
+ if not target or re.match(r"^(?:[a-z][a-z0-9+.-]*:|#|/)", target):
+ return target
+ path, hash_mark, fragment = target.partition("#")
+ path, question, query = path.partition("?")
+ parts: list[str] = []
+ for piece in (PurePosixPath(rel_page).parent / path).as_posix().split("/"):
+ if piece in ("", "."):
+ continue
+ if piece == "..":
+ if parts:
+ parts.pop()
+ else:
+ parts.append(piece)
+ joined = "/".join(parts)
+ if joined.endswith((".md", ".mdx")):
+ url = page_url(joined)
+ elif path.endswith("/") or not path:
+ url = f"{BASE}{joined}/" if joined else BASE
+ else:
+ url = f"{BASE}{joined}"
+ if question:
+ url += f"?{query}"
+ return url + (f"#{fragment}" if hash_mark else "")
+
+
+# Text outside code
+
+
+def outside_fences(text: str, fn) -> str:
+ parts = FENCE.split(text)
+ # FENCE has two groups: text, fence, marker, text, ...
+ out = []
+ for index in range(0, len(parts), 3):
+ out.append(fn(parts[index]))
+ if index + 1 < len(parts):
+ out.append(parts[index + 1])
+ return "".join(out)
+
+
+def outside_code(text: str, fn) -> str:
+ def chunk(part: str) -> str:
+ out, position = [], 0
+ for match in CODE_SPAN.finditer(part):
+ out.append(fn(part[position:match.start()]))
+ out.append(match.group(0))
+ position = match.end()
+ out.append(fn(part[position:]))
+ return "".join(out)
+ return outside_fences(text, chunk)
+
+
+def rewrite_links(text: str, rel: str) -> str:
+ def markdown(match: re.Match) -> str:
+ return (f"{match['bang']}[{match['text']}]({resolve(rel, match['url'])}"
+ f"{match['title'] or ''})")
+
+ def attribute(match: re.Match) -> str:
+ return f'{match["attr"]}="{resolve(rel, match["url"])}"'
+
+ text = outside_fences(text, lambda chunk: MD_LINK.sub(markdown, chunk))
+ return outside_fences(text, lambda chunk: HTML_URL.sub(attribute, chunk))
+
+
+# Heading ids, as Python-Markdown's toc extension gives them
+
+
+def toc_slug(text: str) -> str:
+ text = re.sub(r"<[^>]+>", "", text)
+ text = re.sub(r"!\[[^\]]*\]\([^)]*\)(\{[^}]*\})?", "", text)
+ text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
+ text = unescape(text.replace("`", "").replace("**", "").replace("\\", ""))
+ text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii")
+ text = re.sub(r"[^\w\s-]", "", text).strip().lower()
+ return re.sub(r"[-\s]+", "-", text)
+
+
+def _unique(ident: str, used: set[str]) -> str:
+ while ident in used or not ident:
+ match = IDCOUNT.match(ident)
+ ident = f"{match[1]}_{int(match[2]) + 1}" if match else f"{ident}_1"
+ used.add(ident)
+ return ident
+
+
+def pin_heading_ids(text: str, title_id: str = "") -> str:
+ """Every heading with the id the MkDocs build gave it. *title_id* is
+ the page title's, taken first."""
+ used: set[str] = {title_id} if title_id else set()
+ explicit = set()
+ for match in re.finditer(r"\{[^{}\n]*#([^\s}]+)[^{}\n]*\}[ \t]*$", text, re.M):
+ explicit.add(match[1])
+ for match in ID_ATTR.finditer(text):
+ explicit.add(match["id"])
+ used |= explicit
+
+ def markdown(match: re.Match) -> str:
+ heading = match["mdtext"]
+ attrs = ATTR_BLOCK.search(heading)
+ body = heading[: attrs.start()] if attrs else heading
+ tokens = attrs["attrs"].split() if attrs else []
+ if any(token.startswith("#") for token in tokens):
+ return match.group(0)
+ ident = _unique(toc_slug(body), used)
+ return f"{match['hashes']} {body.rstrip()} {{ {' '.join(['#' + ident, *tokens])} }}"
+
+ def raw(match: re.Match) -> str:
+ if ID_ATTR.search(match["attrs"]):
+ return match.group(0)
+ level, text = match["level"], match["htmltext"]
+ ident = _unique(toc_slug(text), used)
+ return f'{text}'
+
+ def pick(match: re.Match) -> str:
+ return markdown(match) if match["hashes"] else raw(match)
+
+ # In document order: one pass over either kind of heading.
+ return outside_fences(text, lambda chunk: _ANY_HEADING.sub(pick, chunk))
+
+
+_ANY_HEADING = re.compile(
+ r"^(?P#{2,6})[ \t]+(?P.+?)[ \t]*$"
+ r"|[2-6])(?P(?:\s[^>]*)?)>(?P.*?)",
+ re.M | re.S,
+)
+
+
+# Glossary tooltips
+
+
+def glossary_terms(path: Path) -> dict[str, str]:
+ """Each term the glossary defines, with its definition as plain text."""
+ terms: dict[str, str] = {}
+ if not path.is_file():
+ return terms
+ text = path.read_text(encoding="utf-8")
+ for match in re.finditer(r"^(?P\*\*.+)\n: (?P.+(?:\n .+)*)", text, re.M):
+ definition = re.sub(r"\s+", " ", match["body"]).strip()
+ definition = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", definition).replace("`", "")
+ for term in re.findall(r"\*\*([^*]+)\*\*", match["term"]):
+ terms.setdefault(term, definition)
+ return terms
+
+
+_SKIP_TAGS = re.compile(r"<(a|code|pre|abbr|h[1-6]|script|style|summary|svg|figure)\b.*?\1>", re.S | re.I)
+
+
+def add_tooltips(text: str, terms: dict[str, str]) -> str:
+ """The first use of each glossary term on a page, with its definition
+ a hover or a tap away. Headings, links, code and HTML are left as they are."""
+ if not terms:
+ return text
+ pending = dict(terms)
+ pattern = re.compile(
+ r"(? str:
+ if not pending:
+ return chunk
+ out = []
+ for line in chunk.split("\n"):
+ stripped = line.lstrip()
+ if (not pending or stripped.startswith(("#", "|", "<", ":::", "[", "!"))
+ or not stripped):
+ out.append(line)
+ continue
+ out.append(_tooltip_line(line, pattern, pending))
+ return "\n".join(out)
+
+ return outside_code(text, prose)
+
+
+def _tooltip_line(line: str, pattern: re.Pattern, pending: dict[str, str]) -> str:
+ # Inline HTML, links and emphasis markers stay whole.
+ protected = re.compile(r"<[^>]+>|\[[^\]]*\]\([^)]*\)|\*\*[^*]+\*\*|\{[^}]*\}")
+ out, position = [], 0
+ for match in protected.finditer(line):
+ out.append(_tooltip_text(line[position:match.start()], pattern, pending))
+ out.append(match.group(0))
+ position = match.end()
+ out.append(_tooltip_text(line[position:], pattern, pending))
+ return "".join(out)
+
+
+def _tooltip_text(text: str, pattern: re.Pattern, pending: dict[str, str]) -> str:
+ def swap(match: re.Match) -> str:
+ term = match.group(1)
+ if term not in pending:
+ return term
+ definition = pending.pop(term)
+ return f'{term}'
+ return pattern.sub(swap, text)
+
+
+# MDX
+
+
+_VOID = re.compile(r"<(br|img|input|hr|source|meta|link|wbr)\b([^>]*?)(?")
+_HEADING_ATTRS = re.compile(r"(?m)(?<=\S)[ \t]+\{[ \t]*#[^{}\n]*\}[ \t]*$")
+_COMMENT = re.compile(r"", re.S)
+
+
+def mdx_safe(text: str) -> str:
+ """What MDX would read as code in prose: braces, a stray ``<``,
+ comments, unclosed void tags and ```` links."""
+ def fix(chunk: str) -> str:
+ chunk = _COMMENT.sub("", chunk)
+ chunk = re.sub(r"<(https?://[^>\s]+)>", r"[\1](\1)", chunk)
+ held: list[str] = []
+
+ def hold(match: re.Match) -> str:
+ held.append(match.group(0))
+ return f"\x00{len(held) - 1}\x00"
+ chunk = _HEADING_ATTRS.sub(hold, chunk)
+ chunk = chunk.replace("{", "{").replace("}", "}")
+ chunk = re.sub(r"\x00(\d+)\x00", lambda m: held[int(m[1])], chunk)
+ chunk = re.sub(r"<(?![A-Za-z/])", "<", chunk)
+ return _VOID.sub(lambda m: f"<{m[1]}{m[2].rstrip()} />", chunk)
+ return outside_code(text, fix)
+
+
+def used_components(body: str) -> list[str]:
+ return [name for name in COMPONENTS if re.search(rf"<{name}[\s/>]", body)]
+
+
+# Pages
+
+
+def _front_matter(data: dict) -> str:
+ import yaml
+
+ return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=1000) + "---\n\n"
+
+
+def og_section(rel: str, level: str) -> str:
+ """The sharing card a page takes: its level, or what its address says."""
+ url = page_url(rel)[len(BASE):]
+ if not url:
+ return "home"
+ if level in ("start", "setup", "explore", "reference", "contribute"):
+ return level
+ if url.startswith(("which-pack", "wiki/first-install", "wiki/faq", "wiki/getting-started")):
+ return "start"
+ if url.startswith("platforms"):
+ return "setup"
+ if url.startswith(("systems", "emulators", "files")):
+ return "explore"
+ if url.startswith(("contributing", "wiki/adding", "wiki/profiling", "wiki/testing", "wiki/release")):
+ return "contribute"
+ return "reference"
+
+
+def page_head(rel: str, level: str, title: str, browser_title: str, structured: dict) -> list[dict]:
+ """What a page adds to its head: its own window title, the sharing
+ card of its section, and its structured data."""
+ section = og_section(rel, level)
+ image = f"{SITE_URL}assets/og/{section}.png"
+ head: list[dict] = []
+ if browser_title != title:
+ head.append({"tag": "title", "content": f"{browser_title} - RetroBIOS"})
+ head += [
+ {"tag": "meta", "attrs": {"property": "og:image", "content": image}},
+ {"tag": "meta", "attrs": {"property": "og:image:width", "content": "1200"}},
+ {"tag": "meta", "attrs": {"property": "og:image:height", "content": "630"}},
+ {"tag": "meta", "attrs": {"property": "og:image:alt", "content": f"RetroBIOS: {browser_title}"}},
+ {"tag": "meta", "attrs": {"name": "twitter:image", "content": image}},
+ {"tag": "script", "attrs": {"type": "application/ld+json"},
+ "content": json.dumps(structured, ensure_ascii=False, separators=(",", ":")).replace("", "<\\/")},
+ ]
+ return head
+
+
+def split_title(markdown: str) -> tuple[str, str, str]:
+ """The page's H1 as plain text, the id MkDocs gave it, and the body
+ without it."""
+ match = re.search(r"^# (.+?)[ \t]*$\n?", markdown, re.M)
+ if not match:
+ return "", "", markdown
+ explicit = re.search(r"\{[^}]*#([^\s}]+)[^}]*\}\s*$", match[1])
+ title = re.sub(r"\s*\{[^}]*\}\s*$", "", match[1])
+ ident = explicit[1] if explicit else toc_slug(title)
+ title = re.sub(r"<[^>]+>", "", title)
+ title = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", title)
+ title = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", title).replace("`", "")
+ return unescape(title).strip(), ident, markdown[: match.start()] + markdown[match.end():]
+
+
+def finalize_page(
+ rel: str,
+ markdown: str,
+ *,
+ level: str,
+ boost: int | None,
+ browser_title: str,
+ description: str,
+ structured: dict,
+ terms: dict[str, str],
+ search: bool = True,
+) -> tuple[str, str]:
+ """The Starlight file for a page body: its path under the content
+ collection and its text."""
+ title, title_id, body = split_title(markdown)
+ body = body.lstrip("\n")
+ body = rewrite_links(body, rel)
+ body = pin_heading_ids(body, title_id)
+ if title_id:
+ # The title is the theme's, with an id of its own: the one MkDocs
+ # gave it stays an anchor at the top of the text.
+ body = f'\n\n' + body
+ if rel not in ("wiki/glossary.md", "wiki/reference-terms.md"):
+ body = add_tooltips(body, terms)
+ components = used_components(body)
+ stem = rel.rsplit(".", 1)[0]
+ if components:
+ body = mdx_safe(body)
+ imports: dict[str, list[str]] = {}
+ for name in components:
+ imports.setdefault(COMPONENTS[name], []).append(name)
+ # Our own components are imported from src/, two levels above the
+ # content collection plus the page's own folders.
+ up = "../" * (rel.count("/") + 2)
+ body = "".join(f"import {{ {', '.join(names)} }} from '{source}';\n"
+ if source.startswith("@") else f"import {names[0]} from '{up}{source}';\n"
+ for source, names in imports.items()) + "\n" + body
+ front: dict = {"title": title or browser_title, "description": description}
+ # Starlight's slugger drops dots and plus signs: the address stays MkDocs'.
+ if re.search(r"[^a-z0-9_/-]", stem):
+ front["slug"] = stem
+ if level:
+ front["level"] = level
+ if boost:
+ front["boost"] = boost
+ if rel.startswith(CARD_PAGES):
+ front["tableOfContents"] = {"minHeadingLevel": 2, "maxHeadingLevel": 2}
+ if not search:
+ front["pagefind"] = False
+ front["head"] = page_head(rel, level, front["title"], browser_title, structured)
+ if structured.get("dateModified"):
+ front["lastmod"] = str(structured["dateModified"])
+ return stem + (".mdx" if components else ".md"), _front_matter(front) + body
+
+
+# Material syntax in the hand-written wiki pages, read until they are
+# written for Starlight.
+
+_WIKI_ADMONITION = re.compile(r'^(?P *)(?P!!!|\?\?\?\+?) *(?P[a-z]+)(?: +"(?P[^"]*)")? *$')
+_WIKI_TAB = re.compile(r'^(?P *)=== +"(?P
", ""])
return lines
@@ -3405,14 +3421,14 @@ def _search_skipped_apart(lines: list[str]) -> list[str]:
return out
-def _card_title(text: str, anchor: str, searchable: bool = True) -> str:
+def _card_title(text: str, anchor: str, searchable: bool = True, weight: int = 10) -> str:
"""A file card's heading, so the site search lands on the card.
The search splits a page at its headings: a card titled in bold sent a
pasted md5 to the top of a page of a hundred cards. Level 3, under
the console or the Files heading: a level 4 after a level 2 skipped
one for screen readers on 551 pages. The table of contents of a page
- of cards stops above it (mkdocs_hooks.py), and a card without an
+ of cards stops above it (sitestarlight.py), and a card without an
anchor keeps the id the toc gives it.
"""
if markup.starlight():
@@ -3420,11 +3436,12 @@ def _card_title(text: str, anchor: str, searchable: bool = True) -> str:
# would add the engine's anchor link and icon to every card.
title = re.sub(r"`([^`]+)`", lambda m: f"{html.escape(m[1], quote=False)}", text)
ident = f' id="{anchor}"' if anchor else ""
- # The card is the file's own page in all but name: its title weighs
- # as a page title does, so a file name typed in the search finds
- # the card before a guide that cites it as an example.
- weight = ' data-pagefind-weight="7"' if searchable else f" {markup.search_exclude()}"
- return f'
{title}
'
+ # A system page's card is the file's own page in all but name: its
+ # title weighs the most a word can, so a file name typed in the
+ # search finds the card before a guide that cites it as an example.
+ # An emulator's card says who reads the file, and weighs half.
+ search = f' data-pagefind-weight="{weight}"' if searchable else f" {markup.search_exclude()}"
+ return f'
{title}
'
attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title"
if not searchable:
attributes += " data-search-exclude"
@@ -3574,8 +3591,8 @@ def system_pages(
def card_weight(card: list[str]) -> int:
return sum(len(line) + 1 for line in card)
- # Sizes are those of the MkDocs Markdown the budget was tuned on, so
- # every engine splits the same pages at the same cards.
+ # Sizes are those of the MkDocs Markdown the budget was set on, so the
+ # pages split where they always have and no card changes address.
with markup.using("mkdocs"):
measured = all_sections()
measured_text = whole_page(measured)
@@ -3667,7 +3684,7 @@ def _part_range(part: list[Card]) -> str:
"""The first and last file names of a part, as the cards show them."""
names = []
for _variant, card, _sha1 in part:
- # A Markdown heading under MkDocs, an HTML one under Starlight.
+ # An HTML heading on the site, a Markdown one in the size yardstick.
match = re.search(r"`([^`]*)`|(.*?)", card[2])
names.append(match[1] if match[1] is not None else html.unescape(match[2]))
if len(names) == 1:
@@ -4025,7 +4042,7 @@ def _render_emulator_file(
lines.append(markup.block_open("div", 'class="rb-file-entry"'))
lines.append("")
marker = f'' if alias else ""
- lines.append(_card_title(f"{marker}{_code(fname)} {badge_str}", anchor))
+ lines.append(_card_title(f"{marker}{_code(fname)} {badge_str}", anchor, weight=5))
lines.append("")
if desc:
lines.extend([_escape_tags(str(desc)), ""])
@@ -4845,7 +4862,7 @@ def emulator_pages(
# The member tables are a file list with hashes: out of the search,
# dolphin, primehack and ishiiruka each put 210 KB of it there.
lines.extend([
- "## Data directories" + (" { data-search-exclude }" if not markup.starlight() else ""),
+ "## Data directories",
"",
])
trees_start = len(lines)
@@ -4895,10 +4912,8 @@ def emulator_pages(
"info", f"{_count(len(members), 'file')}, as the packs ship them",
table, indent=" "))
lines.append("")
- if markup.starlight():
- # The member tables are the lookup's, as the heading's attribute
- # keeps them under MkDocs.
- lines[trees_start:] = search_excluded(lines[trees_start:])
+ # The member tables are the lookup's.
+ lines[trees_start:] = search_excluded(lines[trees_start:])
lines.extend(_next_block([
"[Get your files](../which-pack.md), for the setup you play on",
@@ -6125,23 +6140,7 @@ def _system_emulators(
if names & data.get("names", set())]
-def _mkdocs_config(docs: Path, text: str) -> tuple[Path, str]:
- """Where the site's config goes, and its text for that place.
-
- The tracked mkdocs.yml belongs to docs/. A preview into another folder
- rewrote it anyway, with the navigation of work in progress, and mkdocs
- kept reading docs/. Another folder gets its own config beside it, its
- paths taken from there.
- """
- if os.path.normpath(str(docs)) == DOCS_DIR:
- return Path("mkdocs.yml"), text
- where = docs.parent
- for relative in ("scripts/mkdocs_hooks.py", "docs_assets/overrides"):
- text = text.replace(relative, os.path.relpath(relative, where).replace(os.sep, "/"))
- return where / f"{docs.name}.mkdocs.yml", f"docs_dir: {docs.name}\n" + text
-
-
-# mkdocs.yml nav generator
+# Navigation
_OS_NAMES = {"windows": "Windows", "darwin": "macOS", "linux": "Linux"}
@@ -6332,13 +6331,10 @@ def generate_which_pack() -> str:
"",
"Run the command and press **c** when it asks.",
"",
- *([
- "Or choose your setup and consoles here. You get the list of files "
- "and a command that installs only those.",
- "",
- "",
- "",
- ] if markup.starlight() else []),
+ "Or pick them here, and get the files and the command.",
+ "",
+ "",
+ "",
*_next_block([
"[First install, step by step](wiki/first-install.md)",
"[When a game does not start](wiki/troubleshooting.md)",
@@ -6486,7 +6482,7 @@ def _nav_target(entry: object) -> str:
return entry if isinstance(entry, str) else str(next(iter(entry.values())))
-def generate_mkdocs_nav(
+def generate_nav(
coverages: dict,
manufacturers: dict,
profiles: dict,
@@ -6543,7 +6539,7 @@ def generate_mkdocs_nav(
def main() -> None:
parser = argparse.ArgumentParser(
allow_abbrev=False,
- description="Generate MkDocs site from project data"
+ description="Generate the documentation site from project data"
)
parser.add_argument(
"--db", default="database.json",
@@ -6557,24 +6553,18 @@ def main() -> None:
"--emulators-dir", default="emulators",
help="emulator profile directory (default: %(default)s)",
)
- parser.add_argument(
- "--engine", choices=markup.ENGINES, default="mkdocs",
- help="site engine the pages are written for (default: %(default)s)",
- )
parser.add_argument(
"--site-dir", default=STARLIGHT_DIR,
- help="the Starlight project, with --engine starlight (default: %(default)s)",
+ help="the Astro Starlight project the site is written into (default: %(default)s)",
)
parser.add_argument(
"--docs-dir", default=None,
- help="directory the pages are written to (default: docs, or the "
- "Starlight project's content collection)",
+ help="directory the pages are written to (default: the project's "
+ "content collection, src/content/docs)",
)
args = parser.parse_args()
- markup.set_engine(args.engine)
if args.docs_dir is None:
- args.docs_dir = (str(Path(args.site_dir) / STARLIGHT_CONTENT)
- if markup.starlight() else DOCS_DIR)
+ args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT)
try:
# Two runs over one docs/ delete each other's pages and scratch
# files in the sweep, so the second is refused instead.
@@ -6596,38 +6586,25 @@ def _generate(args: argparse.Namespace) -> None:
db = load_database(args.db)
docs = Path(args.docs_dir)
- # Files the site serves as they are (data, downloads, images): beside the
- # pages under MkDocs, in the project's public folder under Starlight.
- public = Path(args.site_dir) / "public" if markup.starlight() else docs
+ # Files the site serves as they are (data, downloads, images), in the
+ # project's public folder.
+ public = Path(args.site_dir) / "public"
for d in GENERATED_DIRS:
(public / d if d in PUBLIC_DIRS else docs / d).mkdir(parents=True, exist_ok=True)
- if not markup.starlight():
- css_src = Path("docs_assets") / "extra.css"
- css_dest = docs / "stylesheets" / "extra.css"
- if css_src.exists():
- css_dest.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(css_src, css_dest)
- js_src = Path("docs_assets") / "site.js"
- js_dest = docs / "javascripts" / "site.js"
- if js_src.exists():
- js_dest.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(js_src, js_dest)
-
# No robots.txt: a crawler reads it only at the host root (RFC 9309), and
# this project site lives under /retrobios/. Every page's head points at
- # the sitemap instead (overrides/main.html).
+ # the sitemap instead (Starlight's head links it).
(public / "robots.txt").unlink(missing_ok=True)
images_dest = public / "assets" / "images"
images_dest.mkdir(parents=True, exist_ok=True)
# Logos, system icons and sharing cards are served from the site itself
# (docs_assets/*/sources.json records where each came from).
- folders = [("logos", "logos"), ("system-icons", "system-icons"), ("og", "og")]
- if markup.starlight():
- # The theme's own folder under MkDocs: the mark, favicons, touch icon.
- folders.append(("overrides/assets/brand", "brand"))
+ # brand: the mark, favicons and touch icon (scripts/make_site_icons.py).
+ folders = [("logos", "logos"), ("system-icons", "system-icons"), ("og", "og"),
+ ("brand", "brand")]
for folder, dest in folders:
source = Path("docs_assets") / folder
if source.is_dir():
@@ -6799,6 +6776,9 @@ def _generate(args: argparse.Namespace) -> None:
data_dirs=page_data_directories(name, profiles),
)
PAGE_DATA[f"emulators/{name}.md"] = emulator_structured_data(profile)
+ if (profile.get("type") in ("launcher", "alias")
+ or profile.get("core_classification") in ("launcher", "alias", "frozen_snapshot")):
+ SEARCH_DEMOTED.add(f"emulators/{name}.md")
files = profile.get("files") or []
for stem, entries in emulator_parts(name, files):
PAGE_DATA[f"emulators/{stem}.md"] = emulator_structured_data(
@@ -6846,12 +6826,7 @@ def _generate(args: argparse.Namespace) -> None:
wiki_src = Path(WIKI_SRC_DIR)
if wiki_src.is_dir():
for src_file in wiki_src.glob("*.md"):
- if markup.starlight():
- # Written in Material syntax until the wiki is written for Starlight.
- write_if_changed(str(wiki_dest / src_file.name),
- sitestarlight.wiki_page(src_file.read_text(encoding="utf-8")))
- else:
- shutil.copy2(src_file, wiki_dest / src_file.name)
+ shutil.copy2(src_file, wiki_dest / src_file.name)
_record(wiki_dest / src_file.name)
# data-model.md is generated (contains live DB stats)
write_if_changed(
@@ -6870,163 +6845,32 @@ def _generate(args: argparse.Namespace) -> None:
stale = _sweep_generated(docs, [d for d in GENERATED_DIRS if d not in PUBLIC_DIRS])
stale += _sweep_generated(public, list(PUBLIC_DIRS))
- if markup.starlight():
- write_if_changed(str(docs / "404.md"), generate_not_found())
+ write_if_changed(str(docs / "404.md"), generate_not_found())
print("Adding page metadata and structured data...")
lastmod = decorate_markdown_pages(docs)
- nav = generate_mkdocs_nav(coverages, manufacturers, profiles, registry,
+ nav = generate_nav(coverages, manufacturers, profiles, registry,
Path(WIKI_SRC_DIR))
- if markup.starlight():
- print("Writing the Starlight navigation and the picker's data...")
- site = Path(args.site_dir)
- present = {
- path.relative_to(docs).with_suffix(".md").as_posix()
- for path in docs.rglob("*.md*") if path.suffix in (".md", ".mdx")
- } - {"404.md"}
- generated = site / "src" / "generated"
- generated.mkdir(parents=True, exist_ok=True)
- write_if_changed(str(generated / "sidebar.json"),
- json.dumps(sitestarlight.sidebar(nav, present), indent=1) + "\n")
- write_if_changed(str(generated / "lastmod.json"),
- json.dumps(dict(sorted(lastmod.items())), indent=1) + "\n")
- sitestarlight.picker_data(_REPO_ROOT / "install", public / "data" / "picker")
- total_pages = len(present)
- print(f"\nGenerated {total_pages} pages in {args.docs_dir}/")
- if stale:
- print(f"Removed {stale} page(s) nothing produced this run")
- return
-
- # Update mkdocs.yml nav section only (avoid yaml.dump round-trip mangling quotes)
- print("Updating mkdocs.yml nav...")
- nav_yaml = yaml.dump(
- {"nav": nav}, default_flow_style=False, sort_keys=False, allow_unicode=True
- )
-
- # Rewrite mkdocs.yml entirely (static config + generated nav)
- mkdocs_static = """\
-site_name: RetroBIOS
-site_description: Source-verified BIOS and firmware packs for RetroArch, Batocera,
- Recalbox, Lakka, RetroPie, EmuDeck, RetroBat, RetroDECK, RomM, BizHawk, ROCKNIX,
- KNULLI, AmberELEC, EmuELEC and MiSTer FPGA.
-site_url: https://abdess.github.io/retrobios/
-repo_url: https://github.com/Abdess/retrobios
-repo_name: Abdess/retrobios
-# Almost every page is generated from platforms/, emulators/ and database.json,
-# so a per-page edit link would point at a file that does not exist.
-edit_uri: ''
-# lastmod in the sitemap from each page's own date, or none at all.
-hooks:
- - scripts/mkdocs_hooks.py
-# Working notes kept under docs/ during development are not part of the site.
-exclude_docs: |
- plans/
- specs/
-copyright: MIT for the tooling. BIOS and firmware files are third-party system
- software, preserved for personal backup, archival and interoperability.
-theme:
- name: material
- # Open Graph tags live in overrides/main.html: the social plugin that
- # would otherwise emit them needs Pillow and CairoSVG only to render a
- # preview image the pages do not need.
- custom_dir: docs_assets/overrides
- palette:
- - media: (prefers-color-scheme)
- toggle:
- icon: material/brightness-auto
- name: Switch to light mode
- - media: '(prefers-color-scheme: light)'
- scheme: default
- toggle:
- icon: material/brightness-7
- name: Switch to dark mode
- - media: '(prefers-color-scheme: dark)'
- scheme: slate
- toggle:
- icon: material/brightness-4
- name: Switch to auto
- font: false
- logo: assets/brand/logo.svg
- favicon: assets/brand/favicon-32.png
- icon:
- logo: material/chip
- features:
- - navigation.instant
- - navigation.instant.prefetch
- - navigation.instant.progress
- - navigation.tabs
- - navigation.tabs.sticky
- - navigation.sections
- - navigation.top
- - navigation.tracking
- - navigation.indexes
- # Breadcrumbs above the title: on a deep page they say which tab it is in.
- - navigation.path
- # 400+ pages: pruning keeps the navigation out of every page's HTML.
- - navigation.prune
- - navigation.footer
- - search.suggest
- - search.highlight
- - search.share
- - content.code.copy
- - content.tabs.link
- - toc.follow
-extra_css:
-- stylesheets/extra.css
-extra_javascript:
-- javascripts/site.js
-extra:
- social:
- - icon: fontawesome/brands/github
- link: https://github.com/Abdess/retrobios
- name: RetroBIOS on GitHub
-markdown_extensions:
-- abbr
-- admonition
-- attr_list
-- def_list
-- footnotes
-- meta
-- md_in_html
-- tables
-- toc:
- permalink: true
- # Level 3 is the wiki's subsections. On pages of file cards, also level
- # 3, the hook keeps the table of contents to level 2.
- toc_depth: 3
-- pymdownx.details
-- pymdownx.highlight:
- anchor_linenums: true
-- pymdownx.inlinehilite
-- pymdownx.keys
-# No mermaid fence: Material drew it in the browser with a script from a
-# CDN. Diagrams are SVG files written by scripts/sitediagrams.py.
-- pymdownx.superfences
-- pymdownx.tabbed:
- alternate_style: true
-plugins:
-- search
-# Link rot fails the build: deploy-site.yml runs `mkdocs build --strict`.
-# omitted_files stays at its default (info) so a stale local docs/ does not
-# break a local build. CI regenerates docs/ from scratch anyway.
-validation:
- absolute_links: warn
- unrecognized_links: warn
- anchors: warn
-"""
- config_path, config_text = _mkdocs_config(Path(docs), mkdocs_static + nav_yaml)
- write_if_changed(str(config_path), config_text)
-
- # Counted on disk: the split pages made a sum of page kinds wrong.
- total_pages = sum(
- 1 for path in docs.rglob("*.md")
- if not any(part in LOCAL_NOTE_DIRS for part in path.relative_to(docs).parts[:-1])
- )
+ print("Writing the navigation and the picker's data...")
+ site = Path(args.site_dir)
+ present = {
+ path.relative_to(docs).with_suffix(".md").as_posix()
+ for path in docs.rglob("*.md*") if path.suffix in (".md", ".mdx")
+ } - {"404.md"}
+ generated = site / "src" / "generated"
+ generated.mkdir(parents=True, exist_ok=True)
+ write_if_changed(str(generated / "sidebar.json"),
+ json.dumps(sitestarlight.sidebar(nav, present), indent=1) + "\n")
+ write_if_changed(str(generated / "lastmod.json"),
+ json.dumps(dict(sorted(lastmod.items())), indent=1) + "\n")
+ sitestarlight.picker_data(_REPO_ROOT / "install", public / "data" / "picker")
+ total_pages = len(present)
print(f"\nGenerated {total_pages} pages in {args.docs_dir}/")
if stale:
print(f"Removed {stale} page(s) nothing produced this run")
+
from sitediagrams import prepare_page, unresolved_images # noqa: E402,F401
from sitewrite import ( # noqa: E402,F401
_produced,
diff --git a/scripts/make_site_icons.py b/scripts/make_site_icons.py
index a4f1cdca..fbf94609 100644
--- a/scripts/make_site_icons.py
+++ b/scripts/make_site_icons.py
@@ -11,8 +11,8 @@ four pins per side and the pixel glyph on a 4 by 4 grid. Three variants:
shapes rasterised by a headless Chromium through Playwright, for browsers
that take no SVG icon and for home-screen shortcuts.
-The files land in ``docs_assets/overrides/assets/brand/``, which MkDocs copies
-with the theme. The site build never runs this script.
+The files land in ``docs_assets/brand/``, which generate_site.py copies into
+the site. The site build never runs this script.
python scripts/make_site_icons.py
"""
@@ -22,7 +22,7 @@ from __future__ import annotations
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
-OUT = ROOT / "docs_assets" / "overrides" / "assets" / "brand"
+OUT = ROOT / "docs_assets" / "brand"
HEADER = "#3b3f7a"
GLYPH = "#8be0f7"
diff --git a/scripts/mkdocs_hooks.py b/scripts/mkdocs_hooks.py
deleted file mode 100644
index 11cef35d..00000000
--- a/scripts/mkdocs_hooks.py
+++ /dev/null
@@ -1,37 +0,0 @@
-"""MkDocs hooks for the generated site.
-
-The sitemap gave all 664 pages the build date as lastmod, so the field told
-a crawler nothing. A page whose content carries its own date (an emulator
-page: the day its profile was last read against the source) states it in
-its front matter. The rest get no lastmod rather than an inexact one.
-"""
-
-from __future__ import annotations
-
-from typing import Any
-
-# Pages made of file cards, level 3 headings by the hundred: their table of
-# contents stops at level 2, the consoles of a system page, the Files of an
-# emulator, the sections of a platform page (which listed every system id).
-CARD_PAGES = ("systems/", "emulators/", "platforms/")
-
-
-def on_page_markdown(markdown: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001
- """Take lastmod from the page's own date, or leave it out."""
- page.update_date = str(page.meta.get("date_modified") or "")
- return markdown
-
-
-def _prune(items: list, depth: int) -> None:
- for item in items:
- if item.level >= depth:
- item.children = []
- else:
- _prune(item.children, depth)
-
-
-def on_page_content(html: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001
- """Keep the table of contents of a page of cards to level 2."""
- if page.file.src_uri.startswith(CARD_PAGES):
- _prune(page.toc.items, 2)
- return html
diff --git a/scripts/pipeline.py b/scripts/pipeline.py
index 536b0306..f1daf953 100644
--- a/scripts/pipeline.py
+++ b/scripts/pipeline.py
@@ -24,8 +24,8 @@ Steps, numbered as the run prints them:
5 consistency check verify == pack: counts, paths, every emulator pack
6 --verify-packs pack integrity, extraction and hashes
7 generate_readme.py README.md and CONTRIBUTING.md
- 8 generate_site.py MkDocs pages
- 8b mkdocs build --strict rendered as the site deploy renders it, then validate_site.py
+ 8 generate_site.py site pages, into the Starlight project in website/
+ 8b npm run build (website/) rendered as the site deploy renders it, then validate_site.py
--offline skips 2 to 2b2 and the network side of 2e and 4: release assets
come from .cache/large/ only.
@@ -1102,13 +1102,15 @@ def main() -> None:
if args.skip_docs:
skip_notice("render_site", "--skip-docs")
results["render_site"] = SKIPPED
- elif shutil.which("mkdocs") is None:
- skip_notice("render_site", "mkdocs not installed")
+ elif shutil.which("npm") is None or not Path("website/node_modules").is_dir():
+ skip_notice("render_site", "Node not set up: npm ci --prefix website")
results["render_site"] = SKIPPED
else:
- ok, _ = run(["mkdocs", "build", "--strict"], STEP_LABELS["render_site"])
+ # Starlight's link check runs inside the build and fails it.
+ ok, _ = run(["npm", "run", "build", "--prefix", "website"], STEP_LABELS["render_site"])
if ok:
- ok, _ = run([sys.executable, "scripts/validate_site.py"], "8b validate site")
+ ok, _ = run([sys.executable, "scripts/validate_site.py", "--site-dir", "website/dist"],
+ "8b validate site")
results["render_site"] = ok
all_ok = all_ok and ok
diff --git a/scripts/siteexports.py b/scripts/siteexports.py
index 5b02b6e9..2e1f691a 100644
--- a/scripts/siteexports.py
+++ b/scripts/siteexports.py
@@ -75,8 +75,15 @@ def requirement_lines(report: dict) -> list[str]:
]
if not items:
return []
- return markup.details("note", f"Required flags aligned with the code ({len(items)})",
- [f"- {item}" for item in items])
+ return _lookup_only(markup.details(
+ "note", f"Required flags aligned with the code ({len(items)})",
+ [f"- {item}" for item in items]))
+
+
+def _lookup_only(block: list[str]) -> list[str]:
+ """A list of entries changed, out of the site search: 129 "md5" in them
+ put this page above the one that explains MD5 checks."""
+ return ["", markup.block_open("section", markup.search_exclude()), "", *block, "", "", ""]
def _plural(count: int, noun: str, plural: str = "") -> str:
@@ -139,8 +146,8 @@ def candidate_lines(candidates: list[dict]) -> list[str]:
title = (reason[:1].upper() + reason[1:]).replace('"', "'")
items = [f"`{c['system']}` `{c['destination']}`" for c in entries]
if len(entries) > LONG_LIST:
- lines.extend(markup.details("note", f"{title} ({len(entries)})",
- [f"- {item}" for item in items]))
+ lines.extend(_lookup_only(markup.details(
+ "note", f"{title} ({len(entries)})", [f"- {item}" for item in items])))
else:
lines.extend([f"{title}:", "", *[f"- {item}" for item in items], ""])
return lines
@@ -289,6 +296,6 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str:
items = change_items(report, key)
if not items:
continue
- lines.extend(markup.details("note", f"{title} ({len(items)})",
- [f"- {item}" for item in items]))
+ lines.extend(_lookup_only(markup.details(
+ "note", f"{title} ({len(items)})", [f"- {item}" for item in items])))
return "\n".join(lines)
diff --git a/scripts/sitemarkup.py b/scripts/sitemarkup.py
index f404a2fd..7218e880 100644
--- a/scripts/sitemarkup.py
+++ b/scripts/sitemarkup.py
@@ -5,18 +5,22 @@ tabs, a button, a card grid) is written through one of these functions, so
the generator holds no engine syntax of its own. Bodies are passed without
indentation: the function indents them as the engine reads them.
-The engine is Astro Starlight. MkDocs Material is kept while the two builds
-are compared page by page, selected with set_engine("mkdocs").
+The site is built with Astro Starlight. The MkDocs Material syntax the site
+was first written in stays as one thing only: the yardstick of the size
+budgets that split long pages (system pages, platform sections). Those
+budgets were set on that Markdown, and measuring it keeps every page cut
+at the same file, so no card changes address.
"""
from __future__ import annotations
import html
import re
+from collections.abc import Iterator
from contextlib import contextmanager
ENGINES = ("mkdocs", "starlight")
-_engine = "mkdocs"
+_engine = "starlight"
# Material's callout kinds, by the role Starlight gives them.
ASIDE_ROLES = {
@@ -41,10 +45,9 @@ def engine() -> str:
@contextmanager
-def using(name: str):
- """Write in another engine's syntax for a while. A page split by size
- is measured in the MkDocs Markdown its budgets were tuned on, so both
- engines cut the same pages at the same files."""
+def using(name: str) -> Iterator[None]:
+ """Write in another syntax for a while: a page split by size is
+ measured in the MkDocs Markdown its budgets were set on."""
previous = _engine
set_engine(name)
try:
diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py
index 60783b53..55654522 100644
--- a/scripts/sitestarlight.py
+++ b/scripts/sitestarlight.py
@@ -16,6 +16,7 @@ import json
import re
import unicodedata
import urllib.parse
+from collections.abc import Callable, Iterator
from html import unescape
from pathlib import Path, PurePosixPath
@@ -85,7 +86,7 @@ def resolve(rel_page: str, target: str) -> str:
# Text outside code
-def outside_fences(text: str, fn) -> str:
+def outside_fences(text: str, fn: Callable[[str], str]) -> str:
parts = FENCE.split(text)
# FENCE has two groups: text, fence, marker, text, ...
out = []
@@ -96,7 +97,7 @@ def outside_fences(text: str, fn) -> str:
return "".join(out)
-def outside_code(text: str, fn) -> str:
+def outside_code(text: str, fn: Callable[[str], str]) -> str:
def chunk(part: str) -> str:
out, position = [], 0
for match in CODE_SPAN.finditer(part):
@@ -345,12 +346,27 @@ def split_title(markdown: str) -> tuple[str, str, str]:
return unescape(title).strip(), ident, markdown[: match.start()] + markdown[match.end():]
+_CARD_NAME = re.compile(r'
]*class="rb-file-title"(?![^>]*data-pagefind-ignore)[^>]*>([^<]+)')
+
+
+def file_names_meta(body: str) -> str:
+ """The names of the files a system page holds cards for, as a search
+ field of the page. A page of 250 cards is long, and the search weighs a
+ long page down: the Sony page, where scph5501.bin is downloaded, came
+ after every short page citing the name. The field is weighted in the
+ site's ranking (astro.config.mjs, metaWeights.files)."""
+ names = sorted({unescape(name) for name in _CARD_NAME.findall(body)})
+ if not names:
+ return ""
+ return f'\n{html.escape(" ".join(names))}\n'
+
+
def finalize_page(
rel: str,
markdown: str,
*,
level: str,
- boost: int | None,
+ weight: float | None,
browser_title: str,
description: str,
structured: dict,
@@ -375,6 +391,8 @@ def finalize_page(
# is a name, and stays: it is how "neogeo.zip" finds the cores reading it.
body = re.sub(r' *)(?P!!!|\?\?\?\+?) *(?P[a-z]+)(?: +"(?P[^"]*)")? *$')
-_WIKI_TAB = re.compile(r'^(?P *)=== +"(?P[^"]+)" *$')
-_WIKI_BUTTON = re.compile(r"\[(?P[^\]]+)\]\((?P[^)\s]+)\)\{ *(?P[^}]*)\}")
-
-
-def _indented_block(lines: list[str], start: int, indent: int) -> tuple[list[str], int]:
- body: list[str] = []
- index = start
- pad = " " * (indent + 4)
- while index < len(lines) and (not lines[index].strip() or lines[index].startswith(pad)):
- body.append(lines[index][indent + 4:] if lines[index].startswith(pad) else "")
- index += 1
- while body and not body[-1].strip():
- body.pop()
- index -= 1
- return body, index
-
-
-def wiki_page(text: str) -> str:
- """A wiki page in Material syntax, its blocks written for Starlight."""
- import sitemarkup as markup
-
- def blocks(lines: list[str]) -> list[str]:
- out: list[str] = []
- index = 0
- while index < len(lines):
- line = lines[index]
- admonition = _WIKI_ADMONITION.match(line)
- tab = _WIKI_TAB.match(line)
- if admonition:
- indent = len(admonition["indent"])
- body, index = _indented_block(lines, index + 1, indent)
- title = admonition["title"] if admonition["title"] is not None else admonition["type"].capitalize()
- pad = " " * indent
- if admonition["kind"] == "!!!":
- out += markup.aside(admonition["type"], title, blocks(body), pad)
- else:
- out += markup.details(admonition["type"], title, blocks(body), pad,
- opened=admonition["kind"].endswith("+"))
- continue
- if tab:
- indent = len(tab["indent"])
- items = []
- while index < len(lines):
- tab = _WIKI_TAB.match(lines[index])
- if not tab or len(tab["indent"]) != indent:
- break
- body, index = _indented_block(lines, index + 1, indent)
- items.append((tab["label"], blocks(body)))
- while index < len(lines) and not lines[index].strip() and index + 1 < len(lines) \
- and _WIKI_TAB.match(lines[index + 1]):
- index += 1
- out += markup.tabs(items, " " * indent)
- continue
- out.append(line)
- index += 1
- return out
-
- def button(match: re.Match) -> str:
- classes = match["attrs"].split()
- if ".md-button" not in classes:
- return match.group(0)
- return markup.button(match["text"], match["url"], primary=".md-button--primary" in classes)
-
- text = "\n".join(blocks(text.split("\n")))
- return outside_fences(text, lambda chunk: _WIKI_BUTTON.sub(button, chunk))
-
-
# Navigation
@@ -497,7 +444,7 @@ def sidebar(nav: list, present: set[str]) -> dict:
"""
owner: dict[str, str] = {}
- def item(entry, topic: str) -> dict | None:
+ def item(entry: object, topic: str) -> dict | None:
if isinstance(entry, str):
owner[entry] = topic
return {"label": "Overview", "link": _sidebar_link(entry)} if entry in present else None
@@ -536,7 +483,7 @@ def sidebar(nav: list, present: set[str]) -> dict:
return {"topics": topics, "options": {"topics": extra, "exclude": exclude}}
-def _links(items: list[dict]):
+def _links(items: list[dict]) -> Iterator[dict]:
for entry in items:
if "link" in entry:
yield entry
diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py
index d7964f81..fe9162fb 100644
--- a/scripts/validate_schemas.py
+++ b/scripts/validate_schemas.py
@@ -439,11 +439,11 @@ def main() -> int:
if release_path.exists():
errors.extend(_validate_json_files([release_path], "release.schema.json"))
- stats_path = ROOT / "docs" / "stats.json"
+ stats_path = ROOT / "website" / "public" / "stats.json"
if stats_path.exists():
errors.extend(_validate_json_files([stats_path], "stats.schema.json"))
- api_dir = ROOT / "docs" / "api" / "v1"
+ api_dir = ROOT / "website" / "public" / "api" / "v1"
api_database = api_dir / "database.json"
if api_database.exists():
errors.extend(
diff --git a/scripts/validate_site.py b/scripts/validate_site.py
index fe21d2d7..6f76ca6f 100644
--- a/scripts/validate_site.py
+++ b/scripts/validate_site.py
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
"""Validate the built RetroBIOS site as a navigable, accessible data product.
-This complements the engine's own link check (``mkdocs build --strict`` or
-Starlight's link validator) with checks on the rendered HTML: metadata,
+This complements Starlight's own link check (starlight-links-validator,
+which fails the build) with checks on the rendered HTML: metadata,
heading structure, image alternatives, duplicate IDs, structured data, local
resources, links and fragments, and the budgets of what a visitor downloads.
"""
@@ -50,10 +50,6 @@ class Page:
# The largest page was 10.5 MB (systems/game-engines/) and took 21 s of
# main-thread work on a throttled phone. Split pages now top out at 0.9 MB.
MAX_PAGE_BYTES = 1_000_000
-# The search index is downloaded by every visitor who opens the search:
-# 21.0 MB raw and 4.88 MB gzip with hashes in it, 7.3 MB and 1.44 MB without.
-MAX_SEARCH_INDEX_BYTES = 8_500_000
-MAX_SEARCH_INDEX_GZIP_BYTES = 1_700_000
# Pagefind loads its index in pieces: the metadata, then the chunks the words
# typed fall in (stored compressed). 59 KB for the largest chunk and 7.5 KB of
# metadata on 2026-10-22, 5.2 MB for the whole index on disk.
@@ -192,39 +188,6 @@ def _check_pagefind(site: Path) -> list[str]:
return issues
-def _check_search_index(site: Path) -> list[str]:
- """The index every visitor who opens the search downloads, against its budget."""
- import gzip
-
- if (site / "pagefind").is_dir():
- return _check_pagefind(site)
- path = site / "search" / "search_index.json"
- if not path.is_file():
- return []
- raw = path.read_bytes()
- packed = len(gzip.compress(raw, 6))
- issues = []
- if len(raw) > MAX_SEARCH_INDEX_BYTES:
- issues.append(f"search/search_index.json: {len(raw):,} bytes, "
- f"over the {MAX_SEARCH_INDEX_BYTES:,} budget")
- if packed > MAX_SEARCH_INDEX_GZIP_BYTES:
- issues.append(f"search/search_index.json: {packed:,} bytes gzip, "
- f"over the {MAX_SEARCH_INDEX_GZIP_BYTES:,} budget")
- return issues
-
-
-def _base_path(config_path: Path) -> str:
- """Read site_url without constructing executable YAML extension tags."""
- config = config_path.read_text(encoding="utf-8")
- match = re.search(
- r"(?m)^site_url:\s*['\"]?([^\s#'\"]+)",
- config,
- )
- site_url = match.group(1) if match else ""
- path = urlsplit(site_url).path or "/"
- return "/" + path.strip("/") + "/" if path.strip("/") else "/"
-
-
def _local_target(
href: str, page_path: Path, site: Path, base_path: str
) -> tuple[list[Path], str] | None:
@@ -370,12 +333,11 @@ def _check_links(pages: dict, site: Path, base_path: str) -> list[str]:
return issues
-def validate_site(site: Path, config_path: Path | None = None, base: str = "/retrobios/") -> list[str]:
+def validate_site(site: Path, base_path: str = "/retrobios/") -> list[str]:
site = site.resolve()
if not site.is_dir():
return [f"site directory does not exist: {site}"]
- base_path = _base_path(config_path) if config_path and config_path.is_file() else base
html_paths = sorted(site.rglob("*.html"))
if not html_paths:
return [f"no HTML pages found in {site}"]
@@ -394,7 +356,7 @@ def validate_site(site: Path, config_path: Path | None = None, base: str = "/ret
issues.append(f"duplicate {label} {value!r}: {rendered}")
issues.extend(_check_links(pages, site, base_path))
- issues.extend(_check_search_index(site))
+ issues.extend(_check_pagefind(site))
return issues
@@ -406,13 +368,9 @@ def main() -> None:
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
- "--site-dir", default="site",
+ "--site-dir", default="website/dist",
help="the built site (default: %(default)s)",
)
- parser.add_argument(
- "--config", default=None,
- help="an MkDocs configuration to read the base path from",
- )
parser.add_argument(
"--base", default="/retrobios/",
help="the path the site is served under (default: %(default)s)",
@@ -424,7 +382,7 @@ def main() -> None:
args = parser.parse_args()
try:
- issues = validate_site(Path(args.site_dir), Path(args.config) if args.config else None, args.base)
+ issues = validate_site(Path(args.site_dir), args.base)
except (OSError, RuntimeError, ValueError) as exc:
print(f"Site validation failed: {exc}", file=sys.stderr)
raise SystemExit(1) from exc
diff --git a/tests/sitepages.py b/tests/sitepages.py
new file mode 100644
index 00000000..ee8897a5
--- /dev/null
+++ b/tests/sitepages.py
@@ -0,0 +1,46 @@
+"""The generated site pages, for the tests that read them.
+
+generate_site.py writes each page into the Starlight project's content
+collection, as .md or as .mdx when it uses a component. A test names a
+page by its docs path (``which-pack.md``) and gets whichever file holds it.
+"""
+
+from __future__ import annotations
+
+import os
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+CONTENT = REPO_ROOT / "website" / "src" / "content" / "docs"
+
+
+def require() -> Path:
+ """The content folder, or a skip when the site was not generated (a
+ failure in CI, which generates it before the suite)."""
+ if not (CONTENT / "index.md").is_file():
+ if os.environ.get("CI"):
+ raise AssertionError("CI generates the site before the suite")
+ raise unittest.SkipTest("site not generated (python scripts/generate_site.py)")
+ return CONTENT
+
+
+def page(name: str) -> Path:
+ """The file of a page named by its docs path."""
+ stem = name[:-3] if name.endswith(".md") else name
+ for suffix in (".md", ".mdx"):
+ candidate = CONTENT / f"{stem}{suffix}"
+ if candidate.is_file():
+ return candidate
+ return CONTENT / f"{stem}.md"
+
+
+def pages() -> list[Path]:
+ """Every page file, sorted."""
+ root = require()
+ return sorted(p for p in root.rglob("*") if p.suffix in (".md", ".mdx"))
+
+
+def name(path: Path) -> str:
+ """A page file's docs path, the name tests use for it."""
+ return path.relative_to(CONTENT).with_suffix(".md").as_posix()
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index 78388d20..99ae6883 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -258,7 +258,7 @@ class WorkflowRegressions(unittest.TestCase):
and acted on afterwards rather than swallowed. A best-effort side
action such as adding a label is not a check and keeps its `|| true`.
"""
- checks = ("python scripts/", "unittest", "mkdocs build")
+ checks = ("python scripts/", "unittest", "npm run build")
for name in ("validate.yml", "deploy-site.yml"):
workflow = self._workflow(name)
for job_name, job in workflow["jobs"].items():
@@ -4311,7 +4311,7 @@ class AFileCardSaysAllItKnows(unittest.TestCase):
))
# The card is a heading with this id, linked by the heading's own
# permalink.
- self.assertIn("{ #file-neogeo-zip .rb-file-title }", text)
+ self.assertIn('
{value}", text)
@@ -4975,17 +4975,11 @@ class NullReadsAsNone(unittest.TestCase):
self.assertIn("none: the source is the project itself", page)
def test_the_rendered_site_has_no_python_none(self):
- docs = ROOT / "docs"
- if not docs.is_dir():
- if os.environ.get("CI"):
- self.fail("CI generates docs/ before the suite")
- self.skipTest("docs/ not generated")
- pattern = re.compile(r"\| None \||: None$|: None \|", re.M)
- from scripts.generate_site import LOCAL_NOTE_DIRS
+ from tests import sitepages
- found = [p.relative_to(docs).as_posix() for p in sorted(docs.rglob("*.md"))
- if not set(p.relative_to(docs).parts[:-1]) & set(LOCAL_NOTE_DIRS)
- and pattern.search(p.read_text(encoding="utf-8"))]
+ pattern = re.compile(r"\| None \||: None$|: None \|", re.M)
+ found = [sitepages.name(p) for p in sitepages.pages()
+ if pattern.search(p.read_text(encoding="utf-8"))]
self.assertEqual(found, [])
@@ -5029,7 +5023,7 @@ class VersionAnchorsFollowContent(unittest.TestCase):
"raze", {"emulator": "Raze", "type": "standalone", "files": files},
{"files": {}, "indexes": {}},
)
- return re.findall(r"\{ #(file-[^ ]+) \.rb-file-title \}", page)
+ return re.findall(r'
str:
# The site ends each page with its next step in the reading order. That
# is navigation, not contributing text, and a repository file has none.
- text = text.split('\n\n!!! tip "Next"\n', 1)[0].rstrip("\n")
+ text = text.split("\n\n:::tip[Next]\n", 1)[0].rstrip("\n")
text = re.sub(r"\]\([^)]*\)", "](link)", text)
return "\n".join(text.splitlines()[1:]).replace("on this site", "on the documentation site")
@@ -73,17 +73,14 @@ class BeforeOpeningAPullRequest(unittest.TestCase):
workflow = (REPO_ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8")
self.assertIn("validate_schemas.py --source-only", workflow)
- def test_it_installs_the_packages_the_site_render_needs(self):
- """Without mkdocs-material and pymdown-extensions the pipeline's
- render step (8b) is skipped for every contributor."""
- deploy = (REPO_ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(
- encoding="utf-8"
- )
- specs = re.findall(r'"((?:mkdocs-material|pymdown-extensions)[^"]*)"', deploy)
- self.assertEqual(len(specs), 2)
+ def test_it_installs_what_the_checks_need_and_no_site_package(self):
+ """The site is built by Node from website/package-lock.json: a pip
+ line naming the old theme installed packages nothing reads."""
text = generate_readme.generate_contributing()
- for spec in specs:
- self.assertIn(f'"{spec}"', text)
+ section = text.split("## Before opening a PR", 1)[1].split("\n## ", 1)[0]
+ self.assertIn('pip install pyyaml "jsonschema[format-nongpl]" markdown', section)
+ self.assertNotIn("mkdocs", section)
+ self.assertTrue((REPO_ROOT / "website" / "package-lock.json").is_file())
class OneFileThroughTheBrowser(unittest.TestCase):
diff --git a/tests/test_export_report.py b/tests/test_export_report.py
index 7008c4c8..26c25fc4 100644
--- a/tests/test_export_report.py
+++ b/tests/test_export_report.py
@@ -85,22 +85,22 @@ class CorrectedFilesPage(unittest.TestCase):
def test_single_file_is_downloadable_raw(self):
self.assertIn(
- "(https://raw.githubusercontent.com/Abdess/retrobios/main/exports/recalbox/es_bios.xml)",
+ 'href="https://raw.githubusercontent.com/Abdess/retrobios/main/exports/recalbox/es_bios.xml"',
self.page,
)
def test_several_files_link_to_their_directory(self):
self.assertIn("tree/main/exports/retropie", self.page)
- self.assertIn('??? note "The 2 original files it is based on"', self.page)
+ self.assertIn("The 2 original files it is based on", self.page)
def test_every_change_is_listed(self):
- self.assertIn('??? note "Hashes updated to the emulator source (12)"', self.page)
+ self.assertIn("Hashes updated to the emulator source (12)", self.page)
for i in range(12):
- self.assertIn(f" - `sys/file{i}.bin md5`", self.page)
+ self.assertIn(f"\n- `sys/file{i}.bin md5`", self.page)
def test_a_failing_file_is_said_to_fail(self):
- self.assertIn(" - bash -n failed on a.sh", self.page)
- self.assertIn(" - a size the format has no field for", self.page)
+ self.assertIn("\n- bash -n failed on a.sh", self.page)
+ self.assertIn("\n- a size the format has no field for", self.page)
def test_nothing_published_says_how_to_build(self):
self.assertIn("export_native.py --all --fetch", generate_exports_page([], {}))
@@ -412,16 +412,16 @@ class ThePageShowsEachValueBeforeAndAfter(unittest.TestCase):
def test_a_corrected_hash_shows_both_values(self):
self.assertIn(
- f" - `psx/scph5501.bin` md5 `5626{'0' * 28}` becomes `c523{'1' * 28}`",
+ f"\n- `psx/scph5501.bin` md5 `5626{'0' * 28}` becomes `c523{'1' * 28}`",
self.page,
)
def test_a_requirement_names_the_entry_and_its_value(self):
- self.assertIn('??? note "Required flags aligned with the code (1)"', self.page)
- self.assertIn(" - `psx/scph5501.bin` becomes optional", self.page)
+ self.assertIn("Required flags aligned with the code (1)", self.page)
+ self.assertIn("\n- `psx/scph5501.bin` becomes optional", self.page)
def test_a_filled_hash_shows_its_value(self):
- self.assertIn(f" - `psx/scph5501.bin` sha1 `{'f' * 40}` added", self.page)
+ self.assertIn(f"\n- `psx/scph5501.bin` sha1 `{'f' * 40}` added", self.page)
def test_no_arrow_glyph(self):
for glyph in (chr(0x2192), "->", "=>"):
diff --git a/tests/test_prose_style.py b/tests/test_prose_style.py
index a1312290..df24a41f 100644
--- a/tests/test_prose_style.py
+++ b/tests/test_prose_style.py
@@ -56,6 +56,9 @@ def _tracked() -> list[Path]:
def _semicolons(text: str) -> list[str]:
+ # An HTML character reference (& {) ends in a semicolon that
+ # joins no clauses.
+ text = re.sub(r"&(?:#\d+|#x[0-9a-fA-F]+|[a-zA-Z]+);", "&", text)
return [
line.strip()
for line in BACKTICKS.sub("``", text).splitlines()
@@ -356,18 +359,17 @@ class GeneratedSiteProse(unittest.TestCase):
sys.path.insert(0, str(ROOT / "scripts"))
from generate_site import LOCAL_NOTE_DIRS # noqa: PLC0415
- docs = ROOT / "docs"
- pages = [
- p for p in docs.rglob("*.md")
- if not set(p.relative_to(docs).parts[:-1]) & set(LOCAL_NOTE_DIRS)
- ] if docs.is_dir() else []
- if not pages:
- if os.environ.get("CI"):
- self.fail("CI generates docs/ before the suite")
- self.skipTest("docs/ not generated (python scripts/generate_site.py)")
+ from tests import sitepages # noqa: PLC0415
+
+ pages = [p for p in sitepages.pages()
+ if not set(p.relative_to(sitepages.CONTENT).parts[:-1]) & set(LOCAL_NOTE_DIRS)]
offenders = []
for page in pages:
- text = re.sub(r"", "", page.read_text(encoding="utf-8"), flags=re.S)
+ text = page.read_text(encoding="utf-8")
+ # The front matter carries the page's structured data, not prose.
+ if text.startswith("---\n"):
+ text = text[text.index("\n---\n", 4) + 5:]
+ text = re.sub(r"", "", text, flags=re.S)
text = FENCED.sub("", text)
for number, line in enumerate(text.splitlines(), 1):
if GLYPHS.search(line) or _semicolons(line) or ONE_PLURAL.search(line):
diff --git a/tests/test_schema_semantic_gate.py b/tests/test_schema_semantic_gate.py
index a8022cc6..6c06b3d0 100644
--- a/tests/test_schema_semantic_gate.py
+++ b/tests/test_schema_semantic_gate.py
@@ -42,7 +42,7 @@ class SemanticManifestValidation(unittest.TestCase):
(install / "synthetic.json").write_text(
json.dumps({"files": None}), encoding="utf-8"
)
- api = root / "docs" / "api" / "v1"
+ api = root / "website" / "public" / "api" / "v1"
api.mkdir(parents=True)
(api / "catalog.json").write_text(
json.dumps({"count": 1, "items": None}), encoding="utf-8"
@@ -58,7 +58,7 @@ class SemanticManifestValidation(unittest.TestCase):
text = output.getvalue().replace("\\", "/")
self.assertIn("database.json", text)
self.assertIn("install/synthetic.json", text)
- self.assertIn("docs/api/v1/catalog.json", text)
+ self.assertIn("website/public/api/v1/catalog.json", text)
self.assertIn("files", text)
self.assertIn("items", text)
self.assertNotIn("Traceback", text)
diff --git a/tests/test_site_content_data.py b/tests/test_site_content_data.py
index fc77a2d3..295ea1ba 100644
--- a/tests/test_site_content_data.py
+++ b/tests/test_site_content_data.py
@@ -148,7 +148,7 @@ def _wiki_copy(tmp: str) -> Path:
class NavFromFrontMatter(unittest.TestCase):
def _nav(self, wiki: Path) -> dict:
- nav = generate_site.generate_mkdocs_nav({}, {}, {}, {}, wiki)
+ nav = generate_site.generate_nav({}, {}, {}, {}, wiki)
return {next(iter(tab)): next(iter(tab.values())) for tab in nav}
def test_the_repository_nav_matches_the_pages(self):
diff --git a/tests/test_site_copy.py b/tests/test_site_copy.py
index d44a673a..eece3067 100644
--- a/tests/test_site_copy.py
+++ b/tests/test_site_copy.py
@@ -21,7 +21,10 @@ from dataclasses import dataclass, field
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
-DOCS = REPO_ROOT / "docs"
+sys.path.insert(0, str(REPO_ROOT))
+from tests import sitepages # noqa: E402
+
+DOCS = sitepages.CONTENT
# Level 0: the Start here pages, with their own ceiling on words.
START_PAGES = {
@@ -142,15 +145,19 @@ def guide_part(markdown: str) -> str:
def parse(markdown: str) -> Page:
- """Blocks of prose a reader sees, code and HTML left out."""
+ """Blocks of prose a reader sees, code and HTML left out.
+
+ A collapsed block () counts, but not before the first action:
+ the reader opens it on purpose. Of a group of tabs the first is read and
+ the others only stand in for it.
+ """
text = guide_part(markdown)
text = re.sub(r"", "", text, flags=re.S)
text = re.sub(r"", "", text, flags=re.S)
page = Page()
seen_words = 0
- collapsed_indent: int | None = None
- tab_indent: int | None = None # the tabs of a group a reader picks one of
- hidden_indent: int | None = None
+ details = 0 # depth of collapsed blocks
+ tab = 0 # the tab of a group being read, 0 outside a group
fence: str | None = None
current: list[str] = []
kind = "paragraph"
@@ -162,8 +169,7 @@ def parse(markdown: str) -> Page:
body = strip_inline(" ".join(current))
if WORD.search(body):
term = page.blocks[-1].text.lower() if kind == "definition" and page.blocks else ""
- block = Block(kind, body, collapsed_indent is not None, term,
- hidden_indent is not None)
+ block = Block(kind, body, details > 0, term, tab > 1)
page.blocks.append(block)
if (page.before_action is None and not block.collapsed
and not block.hidden and kind not in ("table", "heading")):
@@ -185,45 +191,52 @@ def parse(markdown: str) -> Page:
if stripped.startswith(fence):
fence = None
continue
- if stripped and tab_indent is not None and indent <= tab_indent and not stripped.startswith("=== "):
- flush()
- tab_indent = hidden_indent = None
if stripped.startswith(("```", "~~~")):
fence = stripped[:3]
- if hidden_indent is None:
+ if tab < 2:
act()
continue
- if collapsed_indent is not None and stripped and indent <= collapsed_indent:
- flush()
- collapsed_indent = None
if not stripped:
flush()
continue
- if re.match(r"(\?\?\?\+?|!!!) ", stripped):
- flush()
- if stripped.startswith("??? "):
- collapsed_indent = indent
+ if stripped.startswith("import ") and stripped.endswith(";"):
continue
- if stripped.startswith("=== "):
- flush()
- if tab_indent == indent:
- hidden_indent = indent
- else:
- tab_indent, hidden_indent = indent, None
- continue
- if stripped.startswith("<") or stripped.startswith(""):
+ if stripped.startswith(":::"):
flush()
continue
- if ".md-button" in stripped:
+ if stripped.startswith(""):
+ flush()
+ details = max(0, details - 1)
+ continue
+ if stripped.startswith(""):
+ flush()
+ tab = 0
+ continue
+ if "rb-button" in stripped or stripped.startswith(" list[tuple[str, str, int]]:
"""(relative path, level, ceiling) for every page the budgets hold."""
pages = [(name, "start", ceiling) for name, ceiling in START_PAGES.items()]
pages += [
- (str(path.relative_to(docs)), "setup", SETUP_CEILING)
- for path in sorted((docs / "platforms").glob("*.md"))
- if path.name != "index.md"
+ (sitepages.name(path), "setup", SETUP_CEILING)
+ for path in sorted((docs / "platforms").glob("*.md*"))
+ if path.stem != "index"
and "\n## Where the files go" in path.read_text(encoding="utf-8")
]
pages.append(("platforms/index.md", "setup", SETUP_INDEX_CEILING))
- return [(name, level, ceiling) for name, level, ceiling in pages if (docs / name).is_file()]
+ return [(name, level, ceiling) for name, level, ceiling in pages
+ if sitepages.page(name).is_file()]
def _docs() -> Path:
- if not (DOCS / "index.md").is_file():
- if os.environ.get("CI"):
- raise AssertionError("CI generates docs/ before the suite")
- raise unittest.SkipTest("docs/ not generated (python scripts/generate_site.py)")
- return DOCS
+ return sitepages.require()
def _load(name: str) -> Page:
- return parse((_docs() / name).read_text(encoding="utf-8"))
+ _docs()
+ return parse(sitepages.page(name).read_text(encoding="utf-8"))
class CopyBudgets(unittest.TestCase):
@@ -400,8 +411,10 @@ class TheMeasureItself(unittest.TestCase):
SAMPLE = (
"---\nlevel: start\n---\n\n# Title\n\nOne two three. Four five.\n\n"
"- Item **one**.\n- Item [two](x.md).\n\n```sh\nnot counted at all\n```\n\n"
- "??? note \"Hidden\"\n\n Collapsed words here.\n\n"
- "=== \"A\"\n\n First tab.\n\n=== \"B\"\n\n Second tab words.\n\n"
+ '\nHidden\n\n'
+ "Collapsed words here.\n\n\n\n"
+ '\n\n\nFirst tab.\n\n\n'
+ '\n\nSecond tab words.\n\n\n\n\n'
"| A | B |\n|---|---|\n| cell | cell |\n\n## Reference: deep\n\nNot read.\n"
)
@@ -429,7 +442,7 @@ def report(docs: Path) -> str:
rows = ["| Page | Words | Before action | Max block | Median / max sentence |",
"|---|---:|---:|---:|---|"]
for name, _level, _ceiling in held_pages(docs):
- m = measure(parse((docs / name).read_text(encoding="utf-8")))
+ m = measure(parse(sitepages.page(name).read_text(encoding="utf-8")))
before = "none" if m["before_action"] is None else m["before_action"]
rows.append(f"| {name} | {m['words']} | {before} | {m['max_block']} | "
f"{m['median_sentence']:g} / {m['max_sentence']} |")
diff --git a/tests/test_site_cores.py b/tests/test_site_cores.py
index b60fb636..0133e397 100644
--- a/tests/test_site_cores.py
+++ b/tests/test_site_cores.py
@@ -36,7 +36,8 @@ class SiteCoreCounts(unittest.TestCase):
page = generate_site.generate_cross_reference(coverages, profiles)
shown = {
m.group(1): int(m.group(2))
- for m in re.finditer(r'\?\?\? abstract "([^"]+)"\n\n.*\n\n {4}\*\*(\d+) cores', page)
+ for m in re.finditer(
+ r'([^<]+)\n\n.*\n\n\*\*(\d+) cores', page)
}
for name, cov in coverages.items():
with self.subTest(platform=name):
diff --git a/tests/test_site_emulator_page.py b/tests/test_site_emulator_page.py
index 9c5df978..0342897b 100644
--- a/tests/test_site_emulator_page.py
+++ b/tests/test_site_emulator_page.py
@@ -86,7 +86,7 @@ class ACardKeepsItsAddress(unittest.TestCase):
files = [{"name": "DUKE3D.GRP", "md5": "aaaa1111bbbb"}]
page = _page("raze", {"type": "standalone", "files": files})
self.assertIn('', page)
- self.assertIn("{ #file-duke3d-grp-aaaa1111 .rb-file-title }", page)
+ self.assertIn('
", text)
+ self.assertIn(f'{md5}', text)
class AMismatchNamesTheDumpTheCollectionHolds(unittest.TestCase):
@@ -924,12 +925,14 @@ class NestedProfileBlocksRenderAsLists(unittest.TestCase):
lines = generate_site._render_platform_details(
{"platform_details": {"megacd": {"root": "x"}, "note": "y"}}
)
- lines[0] = lines[0].replace("???+", "!!!")
try:
import markdown
except ImportError:
self.skipTest("markdown is not installed")
- html = markdown.markdown("\n".join(lines), extensions=["admonition"])
+ # The block's body, read as the engine reads Markdown inside HTML.
+ body = lines[lines.index("Platform details") + 1:
+ lines.index("")]
+ html = markdown.markdown("\n".join(body))
self.assertIn("
root: x
", html)
self.assertNotIn("**", html)
@@ -959,7 +962,7 @@ class TheOneLinerIsWhereUsersLand(unittest.TestCase):
"verified": 0, "missing": 0, "details": [], "config": {"systems": {}}}
page = generate_site.generate_platform_page("batocera", cov)
self.assertIn("install.sh | sh -s -- --platform batocera", page)
- self.assertLess(page.index("--platform batocera"), page.index("md-button"))
+ self.assertLess(page.index("--platform batocera"), page.index("rb-button"))
# Batocera runs the command on the device itself. A platform detected
# on Windows gets the PowerShell form.
cov = dict(cov, platform="RetroArch", mode="existence")
diff --git a/tests/test_site_funnel.py b/tests/test_site_funnel.py
index a6a2e92e..bf2234a1 100644
--- a/tests/test_site_funnel.py
+++ b/tests/test_site_funnel.py
@@ -19,12 +19,14 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import generate_site # noqa: E402
+from tests import sitepages # noqa: E402
from common import list_registered_platforms, load_platform_config, load_platform_registry # noqa: E402
from siteguides import DEVICE_GROUPS, GROUP_ANCHORS, GUIDES # noqa: E402
-DOCS = REPO_ROOT / "docs"
+DOCS = sitepages.CONTENT
ORDER = list(generate_site.LEVELS)
-LINK = re.compile(r"\[[^\]]*\]\(([^)\s#]+\.md)(?:#[^)]*)?\)")
+# Pages link to site addresses once written for the site.
+LINK = re.compile(r"\[[^\]]*\]\((/retrobios/[^)\s#?]*)(?:[#?][^)]*)?\)")
FENCED = re.compile(r"^\s*```.*?^\s*```", re.S | re.M)
@@ -38,13 +40,7 @@ def _which_pack() -> str:
def _docs_pages() -> list[Path]:
- if not DOCS.is_dir() or not (DOCS / "index.md").is_file():
- if os.environ.get("CI"):
- raise AssertionError("CI generates docs/ before the suite")
- raise unittest.SkipTest("docs/ not generated (python scripts/generate_site.py)")
- skipped = set(generate_site.LOCAL_NOTE_DIRS)
- return [p for p in sorted(DOCS.rglob("*.md"))
- if not set(p.relative_to(DOCS).parts[:-1]) & skipped]
+ return [p for p in sitepages.pages() if p.name not in ("404.md",)]
def _level(page: Path) -> str:
@@ -55,14 +51,14 @@ class FiveTabs(unittest.TestCase):
"""Twelve tabs put the walkthrough at position eleven, under "Wiki"."""
def test_the_tabs_in_reading_order(self):
- nav = generate_site.generate_mkdocs_nav({}, {}, {}, {})
+ nav = generate_site.generate_nav({}, {}, {}, {})
self.assertEqual(
[next(iter(tab)) for tab in nav],
["Start here", "Your setup", "Explore", "Reference", "Contribute"],
)
def test_each_tab_opens_on_an_index_or_a_first_page(self):
- nav = generate_site.generate_mkdocs_nav({}, {}, {}, {})
+ nav = generate_site.generate_nav({}, {}, {}, {})
tabs = {next(iter(tab)): next(iter(tab.values())) for tab in nav}
self.assertEqual(tabs["Start here"][0], "index.md")
self.assertEqual(tabs["Your setup"][0], "platforms/index.md")
@@ -70,8 +66,10 @@ class FiveTabs(unittest.TestCase):
self.assertEqual(tabs["Contribute"][0], "contributing.md")
def test_the_breadcrumbs_are_on(self):
- source = (REPO_ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")
- self.assertIn(" - navigation.path\n", source)
+ title = (REPO_ROOT / "website" / "src" / "components" / "PageTitle.astro").read_text(encoding="utf-8")
+ route = (REPO_ROOT / "website" / "src" / "routeData.ts").read_text(encoding="utf-8")
+ self.assertIn('aria-label="Breadcrumb"', title)
+ self.assertIn("rbBreadcrumbs", route)
class EveryPlatformOnThePicker(unittest.TestCase):
@@ -100,8 +98,17 @@ class EveryPlatformOnThePicker(unittest.TestCase):
of the nav is reachable only by a link and MkDocs says so at info
level. getting-started.md is kept out on purpose: it only links the
pages that took its content, for the old links that point to it."""
- # mkdocs.yml is what generate_site writes, and CI checks it is current.
- nav = (Path(__file__).resolve().parents[1] / "mkdocs.yml").read_text(encoding="utf-8")
+ def targets(entries) -> list[str]:
+ found = []
+ for entry in entries:
+ if isinstance(entry, str):
+ found.append(entry)
+ else:
+ for value in entry.values():
+ found += targets(value) if isinstance(value, list) else [value]
+ return found
+
+ nav = " ".join(targets(generate_site.generate_nav({}, {}, {}, {})))
kept_out = {"getting-started.md"}
wiki = Path(__file__).resolve().parents[1] / "wiki"
missing = sorted(
@@ -116,7 +123,7 @@ class EveryPlatformOnThePicker(unittest.TestCase):
key: {"platform": load_platform_config(key, str(REPO_ROOT / "platforms"))["platform"]}
for key in _registered()
}
- nav = generate_site.generate_mkdocs_nav(coverages, {}, {}, registry)
+ nav = generate_site.generate_nav(coverages, {}, {}, registry)
setup = next(tab["Your setup"] for tab in nav if "Your setup" in tab)
groups = {next(iter(item)): next(iter(item.values()))
for item in setup if isinstance(item, dict)}
@@ -166,7 +173,12 @@ class EveryPageHasALevel(unittest.TestCase):
level = _level(page)
with self.subTest(page=str(page.relative_to(DOCS))):
self.assertIn(level, generate_site.LEVELS)
- self.assertIn(generate_site.level_label(level), text)
+ self.assertIn(f"\nlevel: {level}\n", text)
+ # The label above each title comes from the page's level.
+ component = (REPO_ROOT / "website" / "src" / "components" / "PageTitle.astro").read_text()
+ for level, label in generate_site.LEVELS.items():
+ with self.subTest(level=level):
+ self.assertIn(f"{level}: '{label}'", component)
def test_the_decoration_keeps_an_authored_level(self):
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
@@ -177,9 +189,8 @@ class EveryPageHasALevel(unittest.TestCase):
generate_site.decorate_markdown_pages(Path(tmp))
text = page.read_text()
self.assertEqual(text.count("level: start"), 1)
- self.assertEqual(text.count(generate_site.level_label("start")), 1)
- self.assertIn(" boost: 5\n", text)
- self.assertIn("# FAQ\n", text)
+ self.assertIn("searchWeight: 10\n", text)
+ self.assertIn("title: FAQ\n", text)
class DeepLinksCarryALabel(unittest.TestCase):
@@ -189,8 +200,11 @@ class DeepLinksCarryALabel(unittest.TestCase):
its line, or in the heading of its section ("## Reference: ...")."""
def test_start_and_setup_pages(self):
+ import sitestarlight # noqa: PLC0415
+
pages = _docs_pages()
levels = {page: _level(page) for page in pages}
+ by_url = {sitestarlight.page_url(sitepages.name(page)): page for page in pages}
offenders = []
for page, level in levels.items():
if level not in ("start", "setup"):
@@ -201,7 +215,7 @@ class DeepLinksCarryALabel(unittest.TestCase):
if re.match(r"#{1,3} ", line):
heading = line.lstrip("#").strip()
for match in LINK.finditer(line):
- target = (page.parent / match.group(1)).resolve()
+ target = by_url.get(match.group(1))
if target not in levels:
continue
deeper = levels[target]
@@ -218,18 +232,20 @@ class GuidePagesEndOnANextStep(unittest.TestCase):
for page in ("index.md", "which-pack.md", "platforms/batocera.md",
"wiki/first-install.md", "wiki/troubleshooting.md",
"wiki/what-are-bios-files.md", "wiki/faq.md"):
- path = DOCS / page if page.startswith(("index", "which", "platforms")) else REPO_ROOT / page
- if not path.is_file():
- _docs_pages()
+ if page.startswith(("index", "which", "platforms")):
+ sitepages.require()
+ path = sitepages.page(page)
+ else:
+ path = REPO_ROOT / page
with self.subTest(page=page):
text = path.read_text(encoding="utf-8")
- self.assertTrue('!!! tip "Next"' in text or "## What comes next" in text)
+ self.assertTrue(":::tip[Next]" in text or "## What comes next" in text)
def test_the_platform_page_opens_on_its_guide(self):
"""The folder came after four counters and a metadata table, and was
the pack-relative `bios/` where Batocera reads /userdata/bios/."""
- text = (DOCS / "platforms" / "batocera.md").read_text(encoding="utf-8") \
- if (DOCS / "platforms" / "batocera.md").is_file() else _docs_pages()
+ sitepages.require()
+ text = sitepages.page("platforms/batocera.md").read_text(encoding="utf-8")
order = [text.index(h) for h in (
"## Where the files go", "## Install", "## Download", "## How to check",
"## Reference: every file by system",
diff --git a/tests/test_site_images.py b/tests/test_site_images.py
index 9abf635f..2084bc56 100644
--- a/tests/test_site_images.py
+++ b/tests/test_site_images.py
@@ -200,9 +200,15 @@ class SharingCards(unittest.TestCase):
ET.fromstring(sitediagrams.og_card_svg(section, b"png"))
def test_the_template_names_every_section(self):
- template = (ROOT / "docs_assets" / "overrides" / "main.html").read_text(encoding="utf-8")
- named = set(re.findall(r"section = '([a-z]+)'", template))
+ import sitestarlight # noqa: PLC0415
+
+ named = {sitestarlight.og_section("index.md", "")} | {
+ sitestarlight.og_section("wiki/page.md", level)
+ for level in ("start", "setup", "explore", "reference", "contribute")}
self.assertEqual(named, set(sitediagrams.OG_SECTIONS))
+ for section in named:
+ with self.subTest(section=section):
+ self.assertTrue((ROOT / "docs_assets" / "og" / f"{section}.png").is_file())
class ImagesStillNeeded(unittest.TestCase):
diff --git a/tests/test_site_render.py b/tests/test_site_render.py
index a5b4d6f0..83c4e69b 100644
--- a/tests/test_site_render.py
+++ b/tests/test_site_render.py
@@ -126,16 +126,18 @@ class StructuredBlocks(unittest.TestCase):
page = self.render(self.profile("firmware_titles", [
{"id": "0005001b-10042400", "name": "Shared fonts",
"content": ["CafeCn.ttf", "CafeKr.ttf"]}]))
- self.assertIn(" - id: 0005001b-10042400, name: Shared fonts\n", page)
- self.assertIn(" - **content:**", page)
- self.assertIn(" - CafeKr.ttf", page)
+ self.assertIn("\n- id: 0005001b-10042400, name: Shared fonts\n", page)
+ self.assertIn("\n - **content:**", page)
+ self.assertIn("\n - CafeKr.ttf", page)
def test_a_folded_note_stays_inside_its_block(self):
page = self.render(self.profile("analysis", {"tmss": {
"note": "The core points\nsystem/tmss_path at the system directory."}}))
self.assertIn("- **note:** The core points system/tmss_path at the", page)
- body = page.split("\n")[1:]
- self.assertEqual([line for line in body if line and not line.startswith(" ")], [])
+ # Nothing of the note is left after the block closes.
+ lines = page.split("\n")
+ end = lines.index("")
+ self.assertEqual([line for line in lines[end + 1:] if line.strip()], [])
def test_a_long_note_is_not_cut(self):
note = "word " * 80 + "end."
@@ -315,7 +317,7 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
self.assertEqual(len(headings), 11)
self.assertTrue(headings[0].endswith("{ #system-00 }"))
self.assertNotIn('', lines)
- self.assertIn('??? note "1 file (OK)"', lines)
+ self.assertIn("1 file (OK)", lines)
def test_an_emulator_card_is_a_heading_with_its_anchor(self):
import generate_site
@@ -323,8 +325,8 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
entry = {"name": "scph5501.bin", "required": True}
lines = generate_site._render_emulator_file(
entry, {"files": [entry]}, None, [entry], lambda f: True, "file-scph5501-bin")
- self.assertTrue(lines[2].startswith("### `scph5501.bin` "))
- self.assertTrue(lines[2].endswith("{ #file-scph5501-bin .rb-file-title }"))
+ self.assertTrue(lines[2].startswith('
",
"- Note",
])
@@ -391,8 +397,8 @@ class TheSearchSkipsLinkLists(unittest.TestCase):
short = "| a |\n|---|\n| 1 |"
long = " | a |\n |---|\n" + "\n".join(f" | {'x' * 50} |" for _ in range(80))
text = generate_site.tables_out_of_search(f"# T\n\n{short}\n\n{long}\n")
- self.assertEqual(text.count("data-search-exclude"), 1)
- self.assertIn(" \n\n | a |", text)
+ self.assertEqual(text.count("data-pagefind-ignore"), 1)
+ self.assertIn(" \n\n | a |", text)
self.assertIn(short, text)
diff --git a/tests/test_site_system_links.py b/tests/test_site_system_links.py
index 9e1edf78..2cc38c43 100644
--- a/tests/test_site_system_links.py
+++ b/tests/test_site_system_links.py
@@ -143,7 +143,7 @@ class AnIdNamesOneCardPerPage(unittest.TestCase):
"path": "bios/Nintendo/DS/a.bin"}
page = gs.generate_system_page(
"Nintendo", {"DS": [entry], "DSi": [entry]}, FileConsumers({"files": {}}))
- self.assertEqual(page.count("#sha1-aaaaaaaaaaaa .rb-file-title"), 1)
+ self.assertEqual(page.count('id="sha1-aaaaaaaaaaaa" class="rb-file-title"'), 1)
self.assertEqual(page.count("rb-sys-file"), 2)
diff --git a/tests/test_site_validation.py b/tests/test_site_validation.py
index 4c897af3..b553ac43 100644
--- a/tests/test_site_validation.py
+++ b/tests/test_site_validation.py
@@ -101,9 +101,9 @@ class PlatformDetails(unittest.TestCase):
"root": "/x",
}}}
)
- self.assertIn(" - extensions_tried: .bin, .zip", lines)
- self.assertIn(" - hle_available: no", lines)
- self.assertIn(" - root: <system_dir>/x", lines)
+ self.assertIn(" - extensions_tried: .bin, .zip", lines)
+ self.assertIn(" - hle_available: no", lines)
+ self.assertIn(" - root: <system_dir>/x", lines)
class RenderedSiteValidation(unittest.TestCase):
@@ -112,10 +112,8 @@ class RenderedSiteValidation(unittest.TestCase):
self.root = Path(self.temp.name)
self.site = self.root / "site"
self.site.mkdir()
- self.config = self.root / "mkdocs.yml"
- self.config.write_text(
- "site_url: https://example.test/retrobios/\n", encoding="utf-8"
- )
+ # The path the site is served under.
+ self.config = "/retrobios/"
def tearDown(self) -> None:
self.temp.cleanup()
@@ -174,8 +172,7 @@ class TheSiteStaysWithinItsBudgets(unittest.TestCase):
self.temp = tempfile.TemporaryDirectory(dir=TMP_ROOT)
self.site = Path(self.temp.name) / "site"
self.site.mkdir()
- self.config = Path(self.temp.name) / "mkdocs.yml"
- self.config.write_text("site_url: https://example.test/retrobios/\n", encoding="utf-8")
+ self.config = "/retrobios/"
def tearDown(self) -> None:
self.temp.cleanup()
@@ -214,17 +211,16 @@ class TheSiteStaysWithinItsBudgets(unittest.TestCase):
self.assertTrue(any("drawn by a script at runtime" in issue for issue in issues))
def test_a_search_index_over_its_budget(self):
- import random
-
import validate_site as module
- (self.site / "search").mkdir()
- rng = random.Random(1)
- noise = "".join(rng.choice("0123456789abcdef") for _ in range(module.MAX_SEARCH_INDEX_BYTES + 1))
- (self.site / "search" / "search_index.json").write_text(noise, encoding="utf-8")
+ index = self.site / "pagefind" / "index"
+ index.mkdir(parents=True)
+ (index / "en_big.pf_index").write_bytes(b"x" * (module.MAX_PAGEFIND_CHUNK_BYTES + 1))
+ (self.site / "pagefind" / "pagefind.en_x.pf_meta").write_bytes(
+ b"x" * (module.MAX_PAGEFIND_META_BYTES + 1))
issues = self._issues("")
- self.assertTrue(any("search_index.json" in issue and "gzip" not in issue for issue in issues))
- self.assertTrue(any("search_index.json" in issue and "gzip" in issue for issue in issues))
+ self.assertTrue(any("en_big.pf_index" in issue for issue in issues), issues)
+ self.assertTrue(any(".pf_meta" in issue for issue in issues), issues)
class UnrenderedMarkdown(unittest.TestCase):
@@ -327,9 +323,12 @@ class TheSitemapIsAnnouncedWhereCrawlersLook(unittest.TestCase):
never found that way."""
def test_the_head_names_it_and_no_robots_is_written(self):
+ # Starlight links the sitemap from every page's head when the site
+ # has an address, and the config keeps sitemap.xml at its old path.
root = Path(__file__).resolve().parents[1]
- override = (root / "docs_assets" / "overrides" / "main.html").read_text()
- self.assertIn('rel="sitemap"', override)
+ config = (root / "website" / "astro.config.mjs").read_text()
+ self.assertIn("site: SITE", config)
+ self.assertIn("sitemap.xml", config)
source = (root / "scripts" / "generate_site.py").read_text()
self.assertNotIn('"robots.txt"),\n "User-agent', source)
self.assertNotIn("Sitemap: https://", source)
@@ -351,11 +350,10 @@ class PlatformPagesPrintTheDeclaredCrc32(unittest.TestCase):
only its sha1 and md5, so the crc a clrmamepro user pastes found nothing."""
def test_the_rendered_page(self):
- page = ROOT / "docs" / "platforms" / "retroarch.md"
- if not page.is_file():
- if os.environ.get("CI"):
- self.fail("CI generates docs/ before the suite")
- self.skipTest("site not generated")
+ from tests import sitepages # noqa: PLC0415
+
+ sitepages.require()
+ page = sitepages.page("platforms/retroarch.md")
self.assertIn("CRC32: `91e93107`", page.read_text(encoding="utf-8"))
@@ -378,7 +376,7 @@ class PlainHouseStyle(unittest.TestCase):
}
def test_the_stylesheet_has_none_of_the_patterns(self):
- css = (ROOT / "docs_assets" / "extra.css").read_text(encoding="utf-8")
+ css = (ROOT / "website" / "src" / "styles" / "retrobios.css").read_text(encoding="utf-8")
for label, pattern in self.PATTERNS.items():
with self.subTest(pattern=label):
self.assertIsNone(pattern.search(css), label)
@@ -392,21 +390,3 @@ class PlainHouseStyle(unittest.TestCase):
first = page.read_text(encoding="utf-8").splitlines()[0]
with self.subTest(page=page.name):
self.assertFalse(first.endswith(" - RetroBIOS"), first)
-
-
-class APreviewLeavesTheTrackedConfig(unittest.TestCase):
- """--docs-dir moved the pages but still rewrote the tracked mkdocs.yml."""
-
- def test_another_folder_gets_its_own_config(self):
- from generate_site import _mkdocs_config
-
- path, text = _mkdocs_config(Path("docs"), "hooks:\n - scripts/mkdocs_hooks.py\n")
- self.assertEqual(path, Path("mkdocs.yml"))
- path, text = _mkdocs_config(Path("tmp/preview"), "hooks:\n - scripts/mkdocs_hooks.py\n")
- self.assertEqual(path, Path("tmp/preview.mkdocs.yml"))
- self.assertTrue(text.startswith("docs_dir: preview\n"))
- self.assertIn("../scripts/mkdocs_hooks.py", text)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_site_weight.py b/tests/test_site_weight.py
index a524d7ed..9cc55b7a 100644
--- a/tests/test_site_weight.py
+++ b/tests/test_site_weight.py
@@ -18,8 +18,9 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import generate_site # noqa: E402
-import mkdocs_hooks # noqa: E402
+import sitestarlight # noqa: E402
import siterender # noqa: E402
+import sitemarkup as markup # noqa: E402
from cross_reference import FileConsumers # noqa: E402
@@ -54,7 +55,7 @@ class ASystemPageOverItsBudgetIsSplit(unittest.TestCase):
self.assertIn("## Big", summary)
self.assertIn("## Small", summary)
# The small section keeps its cards in place.
- self.assertIn("{ #sha1-" + f"{900:012x}", summary)
+ self.assertIn('id="sha1-' + f"{900:012x}", summary)
parts = [stem for stem in pages if stem.startswith("maker-big")]
self.assertGreater(len(parts), 1)
self.assertEqual(parts[0], "maker-big")
@@ -73,7 +74,9 @@ class ASystemPageOverItsBudgetIsSplit(unittest.TestCase):
def test_a_console_on_one_page_is_linked_directly(self):
self.consoles["Big"] = self.consoles["Big"][:12]
- whole = self._pages(10**9)["maker"][0]
+ # The budget is measured in the Markdown it was set on.
+ with markup.using("mkdocs"):
+ whole = self._pages(10**9)["maker"][0]
pages = self._pages(len(whole) - 1)
self.assertIn("maker-big", pages)
self.assertNotIn("maker-big-2", pages)
@@ -99,8 +102,8 @@ class AnEmulatorWithTooManyCardsIsSplit(unittest.TestCase):
for anchor in anchors:
self.assertIn(f'', main)
self.assertIn('class="rb-card-index"', main)
- self.assertNotIn(".rb-file-title", main)
- self.assertIn(f"{{ #{anchors[0]} .rb-file-title }}", pages["big-files-1"])
+ self.assertNotIn("rb-file-title", main)
+ self.assertIn(f'id="{anchors[0]}" class="rb-file-title"', pages["big-files-1"])
def test_cards_are_linked_on_the_page_that_holds_them(self):
files = [{"name": f"set{i:04d}.zip"} for i in range(generate_site.EMULATOR_CARDS_PER_PAGE + 1)]
@@ -147,21 +150,27 @@ class TheNavigationOpensOneInitial(unittest.TestCase):
class TheTableOfContentsOfACardPageStopsAtLevelTwo(unittest.TestCase):
@staticmethod
- def _item(level: int, children: list | None = None) -> SimpleNamespace:
- return SimpleNamespace(level=level, children=children or [])
+ @staticmethod
+ def _contents(rel: str, sections: int):
+ import yaml # noqa: PLC0415
+
+ body = "# T\n\n" + "".join(f"## S{i}\n\n### card\n\n" for i in range(sections))
+ _target, text = sitestarlight.finalize_page(
+ rel, body, level="", weight=None, browser_title="T", description="d",
+ structured={}, terms={})
+ return yaml.safe_load(text.split("---\n")[1]).get("tableOfContents", "default")
def test_cards_leave_the_toc_and_wiki_pages_keep_level_three(self):
- def page(uri: str) -> SimpleNamespace:
- h2 = self._item(2, [self._item(3)])
- return SimpleNamespace(file=SimpleNamespace(src_uri=uri),
- toc=SimpleNamespace(items=[self._item(1, [h2])]))
-
- for uri, kept in (("systems/sony.md", 0), ("platforms/retroarch.md", 0),
- ("emulators/mame.md", 0), ("wiki/faq.md", 1)):
- current = page(uri)
- mkdocs_hooks.on_page_content("", current, None, None)
- with self.subTest(uri=uri):
- self.assertEqual(len(current.toc.items[0].children[0].children), kept)
+ level_two = {"minHeadingLevel": 2, "maxHeadingLevel": 2}
+ for rel, sections, expected in (
+ ("systems/sony.md", 3, level_two), ("platforms/retroarch.md", 5, level_two),
+ ("emulators/mame.md", 3, level_two), ("wiki/faq.md", 3, "default"),
+ # One or two headings over hundreds of cards: no table at all,
+ # whose script watched every block of the page.
+ ("emulators/mame-files-2.md", 1, False), ("systems/arcade-arcade.md", 2, False),
+ ):
+ with self.subTest(rel=rel):
+ self.assertEqual(self._contents(rel, sections), expected)
if __name__ == "__main__":
diff --git a/tests/test_workflow_paths.py b/tests/test_workflow_paths.py
index 88636e12..f9929b4b 100644
--- a/tests/test_workflow_paths.py
+++ b/tests/test_workflow_paths.py
@@ -79,15 +79,8 @@ class DeploySiteTriggers(unittest.TestCase):
}
)
self.assertIn("generate_site.py", entry_points)
- # mkdocs build runs the hooks mkdocs.yml names: a script of the build.
- # The file carries Python tags safe_load refuses, but the list is plain.
- mkdocs = (REPO_ROOT / "mkdocs.yml").read_text(encoding="utf-8")
- hooks = re.search(r"^hooks:\n((?: - .+\n)+)", mkdocs, re.M)
- entry_points += [
- line.strip()[2:].removeprefix("scripts/")
- for line in (hooks.group(1).splitlines() if hooks else [])
- if line.strip()[2:].startswith("scripts/")
- ]
+ # The Starlight project is an input of its own build.
+ self.assertIn("website/**", patterns)
uncovered = sorted(
path
for path in closure(entry_points)
@@ -109,9 +102,9 @@ class DeploySiteTriggers(unittest.TestCase):
class ThePullRequestBuildsTheSite(unittest.TestCase):
- """mkdocs was installed by deploy-site.yml alone, which runs after the
- merge, and validate.yml did not watch wiki/: a dead link or a line read as
- a second heading passed every check and failed the deploy on main."""
+ """The site build ran in deploy-site.yml alone, after the merge, and
+ validate.yml did not watch wiki/: a dead link or a line read as a second
+ heading passed every check and failed the deploy on main."""
@staticmethod
def _workflow(name: str) -> dict:
@@ -124,17 +117,23 @@ class ThePullRequestBuildsTheSite(unittest.TestCase):
runs = "\n".join(str(step.get("run", "")) for step in steps)
for command in (
"python scripts/generate_site.py",
- "mkdocs build --strict",
- "python scripts/validate_site.py",
+ "npm ci --prefix website",
+ "npm run build --prefix website",
+ "python scripts/validate_site.py --site-dir website/dist",
):
with self.subTest(command=command):
self.assertIn(command, runs)
- for package in ("mkdocs-material", "pymdown-extensions"):
- with self.subTest(package=package):
- self.assertIn(package, runs)
+ # The same Node, from the official action pinned by commit, in both.
+ uses = {str(step.get("uses", "")) for step in steps}
+ deploy = {str(step.get("uses", ""))
+ for step in self._workflow("deploy-site.yml")["jobs"]["build"]["steps"]}
+ node = {use for use in uses if use.startswith("actions/setup-node@")}
+ self.assertEqual(len(node), 1)
+ self.assertTrue(node <= deploy)
+ self.assertRegex(next(iter(node)), r"@[0-9a-f]{40}$")
def test_the_pages_exist_before_the_suite_reads_them(self):
- """docs/ is gitignored. Built after the suite, the tests that read
+ """The pages are gitignored. Built after the suite, the tests that read
the rendered pages found none and skipped on every CI run."""
steps = self._workflow("validate.yml")["jobs"]["run-tests"]["steps"]
runs = [str(step.get("run", "")) for step in steps]
diff --git a/website/astro.config.mjs b/website/astro.config.mjs
index 9c2a04ed..03b72b92 100644
--- a/website/astro.config.mjs
+++ b/website/astro.config.mjs
@@ -66,12 +66,21 @@ export default defineConfig({
expressiveCode: { defaultProps: { wrap: true } },
lastUpdated: false,
pagination: true,
- // Measured on the ten queries of tools/search_gate.py, inside a range
- // where all ten answer: Starlight's defaults (0.1, 0.1, 2, 9) let a guide
- // that cites scph5501 once outrank the file's own card. A long page
- // is penalised a little more, and how often a word appears counts.
+ // Measured on the queries of tools/search_gate.py, at the centre of a
+ // range where all of them answer. How often a word appears counts more
+ // than in Starlight's defaults (0.1, 0.1, 2, 9), a page's title weighs
+ // ten times its text, so "dolphin" finds Dolphin, and the file names of
+ // a system page (its `files` field, written by sitestarlight.py) count
+ // half, so "scph5501" finds the Sony page without a file name winning
+ // every other query.
pagefind: {
- ranking: { pageLength: 0.3, termFrequency: 0.5, termSaturation: 1.4, termSimilarity: 9 },
+ ranking: {
+ pageLength: 0.1,
+ termFrequency: 0.5,
+ termSaturation: 1.4,
+ termSimilarity: 9,
+ metaWeights: { title: 10, files: 0.5 },
+ },
},
plugins: [
starlightSidebarTopics(sidebar.topics, sidebar.options),
diff --git a/website/src/components/MarkdownContent.astro b/website/src/components/MarkdownContent.astro
index 44cb19e3..b0e31a6d 100644
--- a/website/src/components/MarkdownContent.astro
+++ b/website/src/components/MarkdownContent.astro
@@ -1,14 +1,15 @@
---
-// Search ranking: Pagefind multiplies the score of words inside a weighted
-// element, so a Start here page outranks the reference page that names it.
+// Search weight: Pagefind multiplies the score of words inside a weighted
+// element, so a Start here page outranks the reference page that names it,
+// and an emulator outranks its launcher or its frozen copy.
import Default from '@astrojs/starlight/components/MarkdownContent.astro';
-const { boost } = Astro.locals.starlightRoute.entry.data;
+const { searchWeight } = Astro.locals.starlightRoute.entry.data;
---
{
- boost ? (
-
+ searchWeight ? (
+
) : (
diff --git a/website/src/content.config.ts b/website/src/content.config.ts
index 15a91e1b..b926e385 100644
--- a/website/src/content.config.ts
+++ b/website/src/content.config.ts
@@ -12,8 +12,8 @@ export const collections = {
// Who the page is written for, from someone who knows nothing
// (start) to someone adding to the project (contribute).
level: z.enum(['start', 'setup', 'explore', 'reference', 'contribute']).optional(),
- // Search ranking of the first two levels.
- boost: z.number().min(1).max(10).optional(),
+ // How much the site search weighs the page's words (1 by default).
+ searchWeight: z.number().min(0.1).max(10).optional(),
// The date the page's content carries, for the sitemap.
lastmod: z.string().optional(),
}),
diff --git a/website/src/styles/retrobios.css b/website/src/styles/retrobios.css
index 662d6298..2c00d08c 100644
--- a/website/src/styles/retrobios.css
+++ b/website/src/styles/retrobios.css
@@ -367,7 +367,8 @@
/* Collapsible blocks keep the callout colour of their role */
.sl-markdown-content details.rb-details {
- border-inline-start: 4px solid var(--sl-color-blue);
+ border: 1px solid var(--sl-color-blue);
+ border-radius: 4px;
background: var(--sl-color-blue-low);
padding: 0.6rem 1rem;
}
diff --git a/website/tools/search_gate.py b/website/tools/search_gate.py
index bc068d61..1a3a8a6a 100644
--- a/website/tools/search_gate.py
+++ b/website/tools/search_gate.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
-"""The search gate: ten queries a visitor types, and the pages that answer them.
+"""The search gate: queries a visitor types, and the pages that answer them.
-Runs measure.mjs against a served build and counts a query as answered when
-one of its expected pages is in the first three results of the site's own
-search box.
+Runs measure.mjs against a served build, through the site's own search box,
+and checks each query's expected pages against the rank it must reach: the
+first result, or the first three.
python website/tools/search_gate.py http://127.0.0.1:4321/retrobios/
"""
@@ -17,21 +17,24 @@ from pathlib import Path
TOOLS = Path(__file__).resolve().parent
-# Query, then the pages that answer it (path prefixes under the site).
-QUERIES: dict[str, tuple[str, ...]] = {
- "playstation bios": ("systems/sony/",),
- # The file itself: its card on the system page, or a core that reads it.
- "scph5501": ("systems/sony/", "emulators/beetle_psx/", "emulators/duckstation/",
- "emulators/duckstation-legacy/", "emulators/pcsx1/", "emulators/swanstation/",
- "emulators/ares/", "emulators/pcsx_rearmed/", "files/"),
- "batocera folder": ("platforms/batocera/",),
- "neogeo.zip": ("emulators/fbneo", "emulators/mame", "systems/snk/", "systems/arcade"),
- "duckstation": ("emulators/duckstation/", "emulators/duckstation-legacy/"),
- "steam deck": ("platforms/emudeck/", "platforms/retrodeck/", "which-pack/"),
- "missing bios": ("wiki/troubleshooting/", "wiki/what-are-bios-files/", "wiki/faq/"),
- "md5": ("wiki/verification-modes/",),
- "retroarch system folder": ("platforms/retroarch/",),
- "knulli": ("platforms/knulli/",),
+# Query: (pages that answer it, as path prefixes under the site, and the rank one
+# of them must reach). An emulator name must find the emulator itself first,
+# not its launcher or a frozen copy.
+QUERIES: dict[str, tuple[tuple[str, ...], int]] = {
+ "batocera folder": (("platforms/batocera/",), 1),
+ "missing bios": (("wiki/troubleshooting/", "wiki/what-are-bios-files/", "wiki/faq/"), 1),
+ "neogeo.zip": (("emulators/fbneo", "emulators/mame", "systems/snk/", "systems/arcade"), 1),
+ "retroarch system folder": (("platforms/retroarch/",), 1),
+ "installer": (("wiki/installer/", "wiki/first-install/"), 1),
+ # The file's card on its system page, where it is downloaded.
+ "scph5501": (("systems/sony/",), 3),
+ "md5": (("wiki/verification-modes/",), 1),
+ "where do bios files go": (("which-pack/", "wiki/first-install/"), 1),
+ "dolphin": (("emulators/dolphin/",), 1),
+ "duckstation": (("emulators/duckstation/",), 1),
+ "playstation bios": (("systems/sony/",), 3),
+ "steam deck": (("platforms/emudeck/", "platforms/retrodeck/", "which-pack/"), 1),
+ "knulli": (("platforms/knulli/",), 1),
}
@@ -48,12 +51,14 @@ def main() -> int:
if "query" not in result:
print(f"first search: {result['first_search_kb']} KB in {result['first_search_requests']} requests")
continue
- top = [entry.split(" | ")[1][len(root):] for entry in result["top"][:3]]
- hit = any(page.startswith(QUERIES[result["query"]]) for page in top)
+ pages, rank = QUERIES[result["query"]]
+ top = [entry.split(" | ") for entry in result["top"][:3]]
+ hit = any(href[len(root):].startswith(pages) for _title, href in top[:rank])
answered += hit
- print(f"{'ok ' if hit else 'MISS'} {result['query']!r}: {', '.join(top)}")
- print(f"{answered}/{len(QUERIES)} queries answered in the top 3")
- return 0 if answered >= 9 else 1
+ titles = "; ".join(title for title, _href in top)
+ print(f"{'ok ' if hit else 'MISS'} {result['query']!r} (top {rank}): {titles}")
+ print(f"{answered}/{len(QUERIES)} queries answered")
+ return 0 if answered == len(QUERIES) else 1
if __name__ == "__main__":
diff --git a/website/tools/search_scores.mjs b/website/tools/search_scores.mjs
index 8e3f9a69..4673be14 100644
--- a/website/tools/search_scores.mjs
+++ b/website/tools/search_scores.mjs
@@ -6,18 +6,18 @@ const [base, query, ranking] = process.argv.slice(2);
const browser = await chromium.launch({ executablePath: '/usr/bin/google-chrome' });
const page = await browser.newPage();
await page.goto(base, { waitUntil: 'networkidle' });
-const rows = await page.evaluate(async ({ root, query, ranking }) => {
+const rows = await page.evaluate(async ({ root, query, ranking, top }) => {
const pagefind = await import(`${root}pagefind/pagefind.js`);
if (ranking) await pagefind.options({ ranking: JSON.parse(ranking) });
const search = await pagefind.search(query);
- const top = search.results.slice(0, 8);
- const data = await Promise.all(top.map((result) => result.data()));
- return top.map((result, index) => ({
+ const shown = search.results.slice(0, top);
+ const data = await Promise.all(shown.map((result) => result.data()));
+ return shown.map((result, index) => ({
score: result.score.toFixed(3),
url: data[index].url,
words: data[index].word_count,
hits: result.words.length,
}));
-}, { root: base, query, ranking });
+}, { root: base, query, ranking, top: Number(process.env.TOP || 8) });
for (const row of rows) console.log(row.score, row.words, row.hits, row.url);
await browser.close();
diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md
index d564b04c..384a04f5 100644
--- a/wiki/adding-a-platform.md
+++ b/wiki/adding-a-platform.md
@@ -652,7 +652,7 @@ This executes in sequence:
5. Consistency check: verify and the pack agree on counts and on the paths each archive holds
6. Pack integrity - extract ZIPs and verify hashes per platform mode
7. `generate_readme.py` - regenerate README and CONTRIBUTING
-8. `generate_site.py` - regenerate documentation site and `mkdocs.yml`
+8. `generate_site.py` - regenerate the documentation site and its navigation
A new platform is archived-by-default nowhere: `status: active` puts it in
`list_platforms.py`, the CI matrix, and the scrapers. Set
diff --git a/wiki/architecture.md b/wiki/architecture.md
index 102ad182..43afbff3 100644
--- a/wiki/architecture.md
+++ b/wiki/architecture.md
@@ -39,7 +39,8 @@ data/ cached data directories (not BIOS, fetched at build)
schemas/ JSON schemas for platform and emulator YAML (checked in CI)
tests/ test suite with synthetic fixtures
wiki/ hand-written documentation sources
-docs/ generated MkDocs site (gitignored, rebuilt in CI)
+website/ Astro Starlight project; pages, data and navigation
+ are generated into it (gitignored, rebuilt in CI)
_mame_clones.json MAME parent/clone set mappings
database.json file index built from bios/ (SHA1 primary key)
dist/ generated packs (gitignored)
@@ -55,7 +56,8 @@ heading above the match. Every file card is therefore a level 3 heading with
the card's anchor, under its console or a Files heading, and every platform
system a heading of its own: a file name typed in the search opens that card,
not the top of the page. On pages of cards (systems, emulators, platforms) the
-table of contents stops at level 2 (`scripts/mkdocs_hooks.py`).
+table of contents stops at level 2, and a page of cards under one or two
+headings has none (`scripts/sitestarlight.py`).
The browser downloads and indexes the whole search index before it shows a
first result, so the index holds names and prose only. Hashes, lists of other
@@ -450,7 +452,7 @@ pattern and how to add a test.
| Workflow | File | Trigger | Role |
|----------|------|---------|------|
-| Deploy Site | `deploy-site.yml` | push to main on any input of the site build, and manual | validate contracts, generate site, build with MkDocs, validate rendered HTML, deploy to Pages |
+| Deploy Site | `deploy-site.yml` | push to main on any input of the site build, and manual | validate contracts, generate site, build with Astro Starlight, validate rendered HTML, deploy to Pages |
| Validation | `validate.yml` | pull request and push to main on the collection, profiles, schemas, scripts, tests, the installer and every input of the site build | schema check and full test suite on both, BIOS hash validation and auto-label on pull requests |
Each workflow's `paths:` list is the exact set of files that starts it.
diff --git a/wiki/community-tools.md b/wiki/community-tools.md
index baa3bdb3..7718f7d7 100644
--- a/wiki/community-tools.md
+++ b/wiki/community-tools.md
@@ -21,6 +21,7 @@ See the project's [README](https://github.com/monster-penguin/BIOS-Preservation-
---
-!!! note "Add your project"
+:::note[Add your project]
+Open an issue with a link to your repository.
+:::
- Open an issue with a link to your repository.
diff --git a/wiki/faq.md b/wiki/faq.md
index 78b03233..092a1cc6 100644
--- a/wiki/faq.md
+++ b/wiki/faq.md
@@ -104,7 +104,8 @@ and name the file. A file is removed when the owner of its rights asks.
Files disappear when stores close, like the 3DS and Wii U shops in 2023. We keep
them while they can still be found.
-!!! tip "Next"
+:::tip[Next]
+- [Get your files](../which-pack.md)
+- [Words used on this site](glossary.md)
+:::
- - [Get your files](../which-pack.md)
- - [Words used on this site](glossary.md)
diff --git a/wiki/first-install.md b/wiki/first-install.md
index 7746aa0a..fb31ea97 100644
--- a/wiki/first-install.md
+++ b/wiki/first-install.md
@@ -15,39 +15,47 @@ From an empty BIOS folder to games that start.
A **terminal** is a window where you type commands. Paste the line for your
system, then press Enter.
-=== "Windows"
+
+
- Open the Start menu. Type **PowerShell** and open it.
+Open the Start menu. Type **PowerShell** and open it.
- ```powershell
- irm https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1 | iex
- ```
+```powershell
+irm https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1 | iex
+```
-=== "Mac"
+
+
- Open **Terminal**. It is in Applications, then Utilities.
+Open **Terminal**. It is in Applications, then Utilities.
- ```sh
- curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh
- ```
+```sh
+curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh
+```
-=== "Linux and Steam Deck"
+
+
- On a Steam Deck, switch to Desktop Mode and open **Konsole**.
+On a Steam Deck, switch to Desktop Mode and open **Konsole**.
- ```sh
- curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh
- ```
+```sh
+curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh
+```
-=== "Android"
+
+
- Install **Termux**, a terminal app. Run these three lines in it.
+Install **Termux**, a terminal app. Run these three lines in it.
+
+```sh
+pkg install python
+termux-setup-storage
+curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh
+```
+
+
+
- ```sh
- pkg install python
- termux-setup-storage
- curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh
- ```
On Batocera, Recalbox and other game systems, follow
[your setup's page](../which-pack.md).
@@ -145,8 +153,9 @@ drive.
See [When a game does not start](troubleshooting.md).
-!!! tip "Next"
+:::tip[Next]
+- [Your setup's page](../which-pack.md): its folder and how it checks files
+- [When a game does not start](troubleshooting.md)
+- Details: [Reading the installer's output, line by line](installer.md#reading-a-run)
+:::
- - [Your setup's page](../which-pack.md): its folder and how it checks files
- - [When a game does not start](troubleshooting.md)
- - Details: [Reading the installer's output, line by line](installer.md#reading-a-run)
diff --git a/wiki/getting-started.md b/wiki/getting-started.md
index 2d2cc6a5..ec50eee6 100644
--- a/wiki/getting-started.md
+++ b/wiki/getting-started.md
@@ -7,7 +7,7 @@ nav: false
Three pages take you from nothing to games that start.
-[Get your files](../which-pack.md){ .md-button .md-button--primary }
+Get your files
## What is a BIOS?
diff --git a/wiki/glossary.md b/wiki/glossary.md
index b7b11749..e19275e4 100644
--- a/wiki/glossary.md
+++ b/wiki/glossary.md
@@ -43,8 +43,9 @@ nav_title: Glossary
**Game data**
: Files from your own copy of a PC game, like Doom. Not a BIOS.
-!!! tip "Next"
+:::tip[Next]
+- [What is a BIOS?](what-are-bios-files.md)
+- [Get your files](../which-pack.md)
+- Reference: [Terms of the reference pages](reference-terms.md)
+:::
- - [What is a BIOS?](what-are-bios-files.md)
- - [Get your files](../which-pack.md)
- - Reference: [Terms of the reference pages](reference-terms.md)
diff --git a/wiki/legal.md b/wiki/legal.md
index a0e282bf..f2524f56 100644
--- a/wiki/legal.md
+++ b/wiki/legal.md
@@ -41,7 +41,8 @@ This reflects the project's good-faith understanding, not legal advice.
A rights holder, or anyone acting for one, can [open an issue](https://github.com/Abdess/retrobios/issues) identifying the file by path or hash. A file whose removal is requested by its rights holder is removed from the repository and from the next release. No formal notice is required and none will be demanded. State the file and the basis of the claim. The same channel is open to anyone else with a concern about a specific file.
-!!! tip "Next"
+:::tip[Next]
+- [FAQ](faq.md)
+- [What is a BIOS?](what-are-bios-files.md)
+:::
- - [FAQ](faq.md)
- - [What is a BIOS?](what-are-bios-files.md)
diff --git a/wiki/reference-terms.md b/wiki/reference-terms.md
index ddcdde12..9ed2edfe 100644
--- a/wiki/reference-terms.md
+++ b/wiki/reference-terms.md
@@ -149,7 +149,8 @@ everyday words (BIOS, emulator, core, hash) are in the [glossary](glossary.md).
takes a file from (the collection, a data folder, a release asset) is its
source, not its provenance.
-!!! tip "Next"
+:::tip[Next]
+- [How each platform checks files](verification-modes.md)
+- [Glossary](glossary.md)
+:::
- - [How each platform checks files](verification-modes.md)
- - [Glossary](glossary.md)
diff --git a/wiki/release-process.md b/wiki/release-process.md
index e6353421..7d366126 100644
--- a/wiki/release-process.md
+++ b/wiki/release-process.md
@@ -18,8 +18,8 @@ Budget target: ~175 minutes/month on the GitHub free tier.
| Workflow | File | Trigger |
|----------|------|---------|
-| Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, mkdocs.yml), manual |
-| Validation | `validate.yml` | PR and push to main touching `bios/**`, `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh`, `install.ps1`, or any other input of the site build (`wiki/**`, `docs_assets/**`, `provenance/**`, `exports/**`, `install/**`, `database.json`, `release.json`, `derivable.json`, `mkdocs.yml`) |
+| Deploy Site | `deploy-site.yml` | Push to main (platforms, emulators, provenance, wiki, scripts, database.json, website), manual |
+| Validation | `validate.yml` | PR and push to main touching `bios/**`, `platforms/**`, `emulators/**`, `schemas/**`, `scripts/**`, `tests/**`, `install.py`, `install.sh`, `install.ps1`, or any other input of the site build (`wiki/**`, `docs_assets/**`, `provenance/**`, `exports/**`, `install/**`, `database.json`, `release.json`, `derivable.json`, `website/**`) |
Upstream BIOS lists are not scraped on a schedule. A maintainer runs the
scrapers by hand (see [adding a scraper](adding-a-scraper.md)), reviews the
@@ -32,7 +32,8 @@ hosted runner should rebuild and re-upload.
**Trigger.** Push to `main` when any of these paths change: `platforms/`,
`emulators/`, `provenance/`, `wiki/`, `schemas/`, `tests/`, `docs_assets/`,
-`install/`, `install.py`, `database.json`, `release.json`, `mkdocs.yml`, the
+`install/`, `install.py`, `database.json`, `release.json`, `website/` (the
+Starlight project: configuration, components, styles, lockfile), the
workflow itself, and every script the build runs or imports. Also manual
dispatch.
@@ -45,19 +46,21 @@ is missing.
### Steps
1. Checkout, Python 3.12
-2. Install `pyyaml`, `jsonschema[format-nongpl]==4.26.0`,
- `mkdocs>=1.6,<2`, `mkdocs-material>=9.7.5,<10`,
- `pymdown-extensions>=10.14`
+2. Install `pyyaml` and `jsonschema[format-nongpl]==4.26.0`, then Node 24 with
+ the official `actions/setup-node` (npm cache keyed on the lockfile) and
+ `npm ci --prefix website`
3. Run `validate_schemas.py`: the data contracts, checked before anything is
generated (see below)
4. Restore large files from the `large-files` release, refresh data directories
-5. Run `generate_site.py` (converts YAML data into MkDocs pages and rewrites
- `mkdocs.yml`)
+5. Run `generate_site.py`: it writes the pages, the navigation and the data
+ files into the Astro Starlight project in `website/`
6. Run `generate_readme.py` (rebuilds README.md and CONTRIBUTING.md)
-7. `mkdocs build --strict` to produce the static site
-8. Run `validate_site.py` on the rendered HTML (metadata, headings, image
- alternatives, duplicate ids, local links and fragments, page and search
- index budgets)
+7. `npm run build --prefix website` to produce the static site in
+ `website/dist/`. The build runs `starlight-links-validator` with its
+ strict defaults, so a dead link or anchor fails it
+8. Run `validate_site.py --site-dir website/dist` on the rendered HTML
+ (metadata, headings, image alternatives, duplicate ids, every local link
+ and fragment again, page and search index budgets)
9. Require the committed README and CONTRIBUTING to match what the generator
just produced. `write_if_changed()` compares content with the timestamp line
stripped, so a run that only moves the clock leaves the files untouched and
@@ -77,14 +80,18 @@ The site is deployed via the `github-pages` environment using the official
cancelled (`cancel-in-progress: false`): cancelling one mid-flight leaves the
deployment stuck and the next runs time out waiting on it.
-`--strict` turns MkDocs warnings into failures, so a broken internal link or a
-dangling anchor fails the build instead of shipping. The `validation:` block in
-`mkdocs.yml` is what promotes unrecognized links and missing anchors to
-warnings in the first place.
+Two link checks run, and both fail the build: Starlight's validator inside
+`astro build`, and `validate_site.py` over the rendered files. They read
+different things (the content collection, the final HTML with its navigation
+and breadcrumbs), and each has caught what the other passed.
-The theme version is pinned on both sides: `>=9.7.5` because that is the
-release which caps `mkdocs < 2` (MkDocs 2.0 ships without a license), `<10`
-so a major theme release cannot change the site without a deliberate bump.
+Astro, Starlight and their plugins are pinned to exact versions in
+`website/package.json`, resolved from the committed `website/package-lock.json`
+by `npm ci`. Starlight is below 1.0 and Astro shipped two majors in 2026, so an
+upgrade is its own change, built and measured with the scripts in
+`website/tools/` (search queries, Lighthouse, axe, screenshots) before it
+merges. Node runs only inside the site build: `install.py` and the build
+scripts stay Python.
## validate.yml - Validation
@@ -119,7 +126,7 @@ declares while schemas and tests pass it. This job confronts it with its
upstream. An edited profile is not held to it.
**run-tests.** Runs `python -m unittest discover -s tests -t . -v`, then builds the
-site as Deploy Site does (`generate_site.py`, `mkdocs build --strict`,
+site as Deploy Site does (`generate_site.py`, `npm run build --prefix website`,
`validate_site.py`) whatever the suite returned. Must pass before a merge, and
again on the commit a direct push puts at the head of main.
@@ -257,7 +264,7 @@ python scripts/check_release.py --tag "v${DATE}" --sums dist/SHA256SUMS.txt \
# and pack_size describe the packs just built. git status names anything
# else the run rewrote.
git status --short
-git add release.json README.md mkdocs.yml install/*.json install/targets/*.json
+git add release.json README.md install/*.json install/targets/*.json
git commit -m "chore: record release v${DATE}" && git push
# 6. Keep only the new release plus large-files: an older pack carries hashes
diff --git a/wiki/technical-faq.md b/wiki/technical-faq.md
index 45d027e7..9b9adaf6 100644
--- a/wiki/technical-faq.md
+++ b/wiki/technical-faq.md
@@ -143,7 +143,8 @@ Running `verify.py --emulator --verbose` shows the emulator-level ground
It reaches only as far as the profiles do. Files no profile states a value for are platform-only, and nothing can be said about their content. The footer of the platform report gives that ratio as `Ground truth: N/M files have emulator validation`.
-!!! tip "Next"
+:::tip[Next]
+- [Reference index](index.md), every reference page in one list
+- For contributors: [How to contribute](../contributing.md)
+:::
- - [Reference index](index.md), every reference page in one list
- - For contributors: [How to contribute](../contributing.md)
diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md
index f0e45fa5..537f7238 100644
--- a/wiki/testing-guide.md
+++ b/wiki/testing-guide.md
@@ -287,7 +287,23 @@ The test suite is one layer of verification. The full quality gate is:
3. No unexpected CRITICAL entries in the verify output
4. Pack file counts match verification file counts (consistency check)
5. Pack integrity passes (every declared file extractable with correct hash)
-6. `mkdocs build --strict` and `python scripts/validate_site.py` pass
+6. `npm run build --prefix website` and `python scripts/validate_site.py` pass
+
+A change to the site engine, its version, its search ranking or the page
+weights is measured on a served build before it lands, with the scripts in
+`website/tools/` (they need `npm install --no-save playwright-core axe-core
+lighthouse` and Chrome):
+
+- `search_gate.py`: the queries a visitor types and the page that must answer
+ each, read from the site's own search box
+- `lighthouse.py`: Lighthouse mobile on a list of pages, the heaviest ones
+ included
+- `axe.mjs`: WCAG 2.2 AA in light and dark
+- `check_tools.mjs`: the hash lookup, the file picker, a hash typed in the
+ search, breadcrumbs, previous and next across tabs, glossary terms, tables
+ on a phone and the 404 page, in a real browser
+- `serve.mjs`: the build served under /retrobios/ with gzip, as GitHub Pages
+ serves it
A change that passes the tests but breaks the pipeline is not merged. A new
CRITICAL entry in the verify output after a change is explained before the
@@ -380,7 +396,7 @@ nav_title: Save files # only when the menu names it otherwise than its headin
- `level` picks the tab and the label shown at the top of the page.
- `order` sorts the tab. Pages are spaced by ten, so a new page fits between
two without renumbering. The generated pages (systems, emulators, the gaps
- and data pages) hold fixed places among them, set in `generate_mkdocs_nav`.
+ and data pages) hold fixed places among them, set in `generate_nav`.
- `nav_title` is the menu label. Without it the menu shows the page's `# `
heading. It is not `title`, which both MkDocs and Astro Starlight read as
the page's own title.
@@ -419,7 +435,7 @@ The `validate.yml` workflow runs `python -m unittest discover -s tests -t . -v`
roads into main: every pull request, and every direct push, that touches
`bios/`, `platforms/`, `emulators/`, `schemas/`, `scripts/`, `tests/`,
`install.py` or an input of the site build such as `wiki/`. The same job then
-builds the site with `mkdocs build --strict`, so a page that would fail the
+builds the site with `npm run build --prefix website`, so a page that would fail the
deploy fails the pull request. Work reaches main by push as often as by pull request, so a suite
wired to pull requests alone would guard the road nobody takes. The test job
(`run-tests`) runs in parallel with schema validation, and on a pull request
diff --git a/wiki/tools.md b/wiki/tools.md
index d85d1a1c..c84eabb8 100644
--- a/wiki/tools.md
+++ b/wiki/tools.md
@@ -51,7 +51,7 @@ Pipeline steps:
| 6 | Pack integrity (extract + hash) | `--skip-packs` |
| 7 | Generate README | `--skip-docs` |
| 8 | Generate site | `--skip-docs` |
-| 8b | Render the site with `mkdocs build --strict`, then `validate_site.py`, as the deploy does | `--skip-docs`, or no `mkdocs` installed |
+| 8b | Render the site with `npm run build --prefix website`, then `validate_site.py`, as the deploy does | `--skip-docs`, or Node not set up (`npm ci --prefix website`) |
Pipeline flags:
@@ -692,7 +692,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6
| `provenance_report.py` | Dump-catalog coverage and acquisition targets (see above) |
| `find_pinned_revision.py` | Walk the upstream commits that touched a file and return the revision whose bytes match a platform's pinned md5 (`--write` saves them) |
| `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database |
-| `generate_site.py` | Generate all MkDocs site pages (this documentation) |
+| `generate_site.py` | Generate the site (this documentation) into the Starlight project in `website/` |
| `validate_site.py` | Validate rendered metadata, heading order, image alternatives, JSON-LD, local resources, links and fragments, and the page size and search index budgets |
| `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held |
| `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch ` reads FBNeo at that tag, branch or commit |
@@ -732,7 +732,7 @@ Further options, by script:
| `split_pack.py` | `--max-size 1900M`, the ceiling per part (K, M, G, default 2G) |
| `torrentzip.py` | `--check ZIP` reports conformance, `--rebuild ZIP -o OUT` normalizes an archive |
| `validate_pr.py` | `--changed` takes the BIOS files the branch changes, `--markdown`, `--json` |
-| `validate_site.py` | `--site-dir` (rendered site), `--config` (mkdocs.yml), `--max-errors N` |
+| `validate_site.py` | `--site-dir` (rendered site, default `website/dist`), `--base` (path the site is served under), `--max-errors N` |
| `scraper/coreinfo_scraper.py` | `--compare-db database.json` lists the `.info` firmware the database lacks |
| `scraper/mame_hash_scraper.py` | `--release mameNNNN --profile FILE --source-commit SHA --core NAME --built-by PLATFORM` writes the profile of a MAME build a platform compiles. `--devices --listxml FILE` documents in `mame.yml` every device set with dumped ROMs that the release's `-listxml` names (`isdevice="yes"`), with `--dry-run` to preview |
| `scraper/romset_dat_importer.py` | `--cache-dir` (downloaded listxml and DATs) |
diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md
index fb1827e8..966f2df4 100644
--- a/wiki/troubleshooting.md
+++ b/wiki/troubleshooting.md
@@ -78,7 +78,8 @@ Other messages: Details: [When something goes wrong](installer.md#when-something
-!!! tip "Next"
+:::tip[Next]
+- [Your setup's page](../which-pack.md): its folder and how it checks files
+- [FAQ](faq.md)
+:::
- - [Your setup's page](../which-pack.md): its folder and how it checks files
- - [FAQ](faq.md)
diff --git a/wiki/what-are-bios-files.md b/wiki/what-are-bios-files.md
index 149fdf37..87427f84 100644
--- a/wiki/what-are-bios-files.md
+++ b/wiki/what-are-bios-files.md
@@ -9,7 +9,7 @@ nav_title: What BIOS files are
A console has a small program inside it. It starts first, then starts the game.
This program is the **BIOS**, also called **firmware**.
-[Get your files](../which-pack.md){ .md-button .md-button--primary }
+Get your files
An **emulator** is an app that acts like a console. For many consoles, it needs
a copy of the BIOS. That copy is a file: a **BIOS file**.
@@ -59,7 +59,8 @@ then glitch or do not start. The real file works best, so we include it.
Laws differ by country. [Read the project's position](faq.md#is-this-legal).
-!!! tip "Next"
+:::tip[Next]
+- [Get your files](../which-pack.md)
+- [Words used on this site](glossary.md)
+:::
- - [Get your files](../which-pack.md)
- - [Words used on this site](glossary.md)