From d2974660ebd5f7765389853c4372df3c8ac52419 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:21:52 +0200 Subject: [PATCH] refactor: emit site blocks through sitemarkup Callouts, collapsible blocks, tabs, buttons, card grids and search exclusion go through one module that writes the engine's syntax. The MkDocs output is unchanged apart from whitespace-only lines. --- scripts/generate_readme.py | 18 +- scripts/generate_site.py | 512 ++++++++++++++++--------------------- scripts/sitediagrams.py | 6 +- scripts/siteexports.py | 52 ++-- scripts/siteguides.py | 23 +- scripts/sitemarkup.py | 183 +++++++++++++ 6 files changed, 444 insertions(+), 350 deletions(-) create mode 100644 scripts/sitemarkup.py diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 3723ed55..f8651907 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -230,14 +230,16 @@ def release_totals(platform_name: str, record: dict) -> tuple[int | None, int | def download_links( - platform_name: str, record: dict, repo: str, label: str, attrs: str = "" + platform_name: str, record: dict, repo: str, label: str, attrs: str = "", + button: Callable[[str, str], str] | None = None, ) -> str: """Markdown links to the files of a platform's pack in the recorded release. A pack published whole gets one link labelled *label*. A pack in parts gets one link per part, since every part is needed. *repo* is the repository URL, absolute or relative, and *attrs* a MkDocs attribute list - put after each link. Without a recorded release the link is the latest + put after each link, or *button* the function that draws each link as a + button on the site. Without a recorded release the link is the latest release page. A release recorded without this platform's pack says so and names the installer, which reads the file lists on main: a button to the release served six platforms a page with no pack for them. @@ -245,18 +247,22 @@ def download_links( pack = release_record.pack_for(platform_name, record) tag = record.get("tag") assets = pack.get("assets") if pack else None + def link(text: str, url: str) -> str: + return button(text, url) if button else f"[{text}]({url}){attrs}" + if not tag: - return f"[{label}]({repo}/releases/latest){attrs}" + return link(label, f"{repo}/releases/latest") if not assets: return f"Not in {tag}: [use the installer]({SITE_URL}wiki/installer/)" urls = [f"{repo}/releases/download/{tag}/{asset}" for asset in assets] if len(urls) == 1: - return f"[{label}]({urls[0]}){attrs}" + return link(label, urls[0]) + styled = bool(attrs or button) parts = [ - f"[{'P' if attrs else 'p'}art {number} of {len(urls)}]({url}){attrs}" + link(f"{'P' if styled else 'p'}art {number} of {len(urls)}", url) for number, url in enumerate(urls, 1) ] - if attrs: + if styled: return " ".join(parts) return f"{label}: " + ", ".join(parts) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 45b29724..f8e6da82 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -73,6 +73,7 @@ from generate_readme import ( release_totals, ) from provenance_report import build_report, distinct_missing +import sitemarkup as markup from siteexports import generate_exports_page, load_export_reports from siteguides import ( ALSO_RUNS, @@ -384,7 +385,7 @@ def _timestamp() -> str: def _stat(value: str, label: str) -> list[str]: return [ - '
', + markup.block_open("div", 'class="rb-stat"'), f'{value}', f'{label}', "
", @@ -402,7 +403,7 @@ def _group_members(coverages: dict, registry: dict, group: str) -> list[str]: def _next_block(links: list[str]) -> list[str]: """Where to go from here, at the end of a page.""" - return ['!!! tip "Next"', "", *(f" - {link}" for link in links), ""] + return markup.aside("tip", "Next", [f"- {link}" for link in links]) def generate_home( @@ -431,8 +432,8 @@ def generate_home( "the **BIOS**. Your emulator needs a copy of it. We put the right copy " "in the right folder.", "", - "[Get your files](which-pack.md){ .md-button .md-button--primary } " - "[What is a BIOS?](wiki/what-are-bios-files.md){ .md-button }", + markup.button("Get your files", "which-pack.md", primary=True) + " " + + markup.button("What is a BIOS?", "wiki/what-are-bios-files.md"), "", f"Free. No account. Works with {works_with}.", "", @@ -446,30 +447,28 @@ def generate_home( "", "## Three steps", "", - "1. **Pick your device.** Its page shows your BIOS folder.", - "2. **Run one command**, or download a ZIP file.", - "3. **Start a game.**", - "", + *(markup.steps([ + "**Pick your device.** Its page shows your BIOS folder.", + "**Run one command**, or download a ZIP file.", + "**Start a game.**", + ]) if markup.starlight() else [ + "1. **Pick your device.** Its page shows your BIOS folder.", + "2. **Run one command**, or download a ZIP file.", + "3. **Start a game.**", + "", + ]), "## Pick your device", "", - '
', - "", ] - for group, (label, _description) in DEVICE_GROUPS.items(): - members = _group_members(coverages, registry, group) - if not members: - continue - lines += [ - f"- **{label}**", - "", - " " + ", ".join( - f"[{coverages[name]['platform']}](platforms/{name}.md)" for name in members - ), - "", - ] + lines += markup.card_grid([ + (label, [", ".join( + f"[{coverages[name]['platform']}](platforms/{name}.md)" + for name in _group_members(coverages, registry, group) + )]) + for group, (label, _description) in DEVICE_GROUPS.items() + if _group_members(coverages, registry, group) + ]) lines += [ - "
", - "", "## Is it safe?", "", "- It writes only in your BIOS folder.", @@ -501,43 +500,23 @@ def generate_explore_index(db: dict, coverages: dict, profiles: dict) -> str: "One console, one emulator, or one installer option: each has its page, " "with the files it needs and the code that reads them.", "", - '
', - "", - "- **Systems**", - "", - " What each console and computer needs, file by file.", - "", - " [Browse the systems](systems/index.md)", - "", - "- **Emulators**", - "", - " What each emulator reads, with the source code that reads it.", - "", - " [Browse the emulators](emulators/index.md)", - "", - "- **Installer options**", - "", - " Fewer files, one region, another folder, a check without download.", - "", - " [All installer options](wiki/installer.md)", - "", - "- **How files are checked**", - "", - " Each setup checks its files its own way: by name, by MD5 or by SHA1.", - "", - " Reference: [How each platform checks files](wiki/verification-modes.md)", - "", - "- **Data and API**", - "", - " Every file and hash as JSON, CSV and SQLite.", - "", - " Reference: [Data and API](data.md)", - "", - "
", - "", + *markup.card_grid([ + ("Systems", ["What each console and computer needs, file by file.", "", + "[Browse the systems](systems/index.md)"]), + ("Emulators", ["What each emulator reads, with the source code that reads it.", "", + "[Browse the emulators](emulators/index.md)"]), + ("Installer options", [ + "Fewer files, one region, another folder, a check without download.", "", + "[All installer options](wiki/installer.md)"]), + ("How files are checked", [ + "Each setup checks its files its own way: by name, by MD5 or by SHA1.", "", + "Reference: [How each platform checks files](wiki/verification-modes.md)"]), + ("Data and API", ["Every file and hash as JSON, CSV and SQLite.", "", + "Reference: [Data and API](data.md)"]), + ]), "## The collection", "", - '
', + markup.block_open("div", 'class="rb-stats"'), "", *_stat(f"{total_files:,}", COLLECTION_FILES_LABEL), *_stat(str(len(coverages)), "Setups supported"), @@ -582,7 +561,7 @@ def _pack_buttons(platform_name: str, label: str = "Pack") -> str: release_record.load_record(), REPO_URL, label, - "{ .md-button .md-button--primary }", + button=lambda text, url: markup.button(text, url, primary=True), ) @@ -1295,9 +1274,11 @@ def generate_data_page(exports: list[dict]) -> str: "RetroBIOS publishes the same metadata used by its verifier, pack builder " "and website as versioned static files. No account or API key is required.", "", - "[API catalog](api/v1/catalog.json){ .md-button .md-button--primary } " - "[SQLite snapshot](downloads/retrobios.sqlite){ .md-button } " - "[Collection DAT](downloads/retrobios.dat){ .md-button }", + " ".join([ + markup.button("API catalog", "api/v1/catalog.json", primary=True), + markup.button("SQLite snapshot", "downloads/retrobios.sqlite"), + markup.button("Collection DAT", "downloads/retrobios.dat"), + ]), "", "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game " "per `bios///` folder of the collection, named " @@ -1497,7 +1478,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: stripped = paragraph.strip() if ( not stripped - or stripped.startswith(("#", "|", "```", "???", "<", "- ", "* ")) + or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* ")) ): continue plain = _plain_markdown(stripped) @@ -1977,16 +1958,11 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st members = _group_members(coverages, registry, group) if not members: continue - lines += [f"## {label}", "", description, "", '
', ""] - for name in members: - guide = GUIDES[name] - lines += [ - f"- **[{coverages[name]['platform']}]({name}.md)**", - "", - f" {guide.summary}", - "", - ] - lines += ["
", ""] + lines += [f"## {label}", "", description, ""] + lines += markup.card_grid([ + (f"[{coverages[name]['platform']}]({name}.md)", [GUIDES[name].summary]) + for name in members + ]) lines += [ "## Reference: every platform at a glance { #reference }", @@ -2088,24 +2064,14 @@ def _render_platform_systems( # emulator cards link here, and the site search splits a page at its # headings. Folded into blocks under one heading, every hash pasted # in the search led to the systems overview of a 100-block page. - block_start = len(lines) + 2 - if use_collapsible: - status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue") - lines.append(f"### {sys_link} {{ #{anchor} }}") - lines.append("") - lines.append(f'??? note "{_count(total, "file")} ({status_tag})"') - lines.append("") - pad = " " - else: - lines.append(f"### {sys_link} {{ #{anchor} }}") - lines.append("") - pad = "" - - lines.append(f"{pad}{ok_count}/{total} files verified") + lines.append(f"### {sys_link} {{ #{anchor} }}") + lines.append("") + block_start = len(lines) + body = [f"{ok_count}/{total} files verified"] if sys_emus: emu_links = ", ".join(_emulator_link(e, "../") for e in sorted(sys_emus)) - lines.append(f"{pad}Emulators: {emu_links}") - lines.append("") + body.append(f"Emulators: {emu_links}") + body.append("") # File listing. The sort is stable, so same-named entries keep the # order verify reported them in, which is the declaration order. @@ -2146,7 +2112,7 @@ def _render_platform_systems( # GameCube IPL.bin differ only by GC/USA, GC/EUR and GC/JAP. destination = str(cfg_entry.get("destination") or "") where = f" at `{destination}`" if destination and destination != fname else "" - lines.append(f"{pad}- `{fname}`{where} - {', '.join(details)}") + body.append(f"- `{fname}`{where} - {', '.join(details)}") # Every hash the platform declares, whole: the crc32 is what # System.dat and clrmamepro print, and pasting it found nothing # for a pinned file the collection holds under other bytes. @@ -2167,23 +2133,32 @@ def _render_platform_systems( hash_parts.append(f"CRC32: `{crc32}`") if sha256: hash_parts.append(f"SHA256: `{sha256}`") - lines.append(f"{pad} {' | '.join(hash_parts)}") + body.append(f" {' | '.join(hash_parts)}") if f.get("category_evidence"): - lines.append(f"{pad} {_category_evidence_md(f['category_evidence'])}") + body.append(f" {_category_evidence_md(f['category_evidence'])}") - block = lines[block_start:] - if moved is not None and platform and sum(len(line) + 1 for line in block) > PLATFORM_SECTION_BUDGET: + # Measured as the Markdown the split was tuned on, a collapsible + # block indented by four, so both engines split the same systems. + size = sum(len(line) + 1 for line in body) + if use_collapsible: + status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue") + title = f"{_count(total, 'file')} ({status_tag})" + size += len(f'??? note "{title}"') + 2 + 4 * sum(1 for line in body if line) + block = markup.details("note", title, body) + while block and not block[-1]: + block.pop() + else: + block = body + if moved is not None and platform and size > PLATFORM_SECTION_BUDGET: key, display = platform stem = f"{key}-{anchor}" - body = block[2:] if use_collapsible else block moved[stem] = "\n".join([ f"# {display}: {sys_id}", "", f"Part of the [{display} page]({key}.md#{anchor}), whose other systems " f"are listed there. Each file below is as {display} declares it, with " "its status in the pack.", "", "## Files", "", - *search_excluded( - [line[len(pad):] if line.startswith(pad) else line for line in body]), + *search_excluded(body), "", f"*Generated on {_timestamp()}*", ]) + "\n" lines[block_start:] = [ @@ -2193,7 +2168,7 @@ def _render_platform_systems( continue # The section's heading stays searchable, its file list with every # hash is the lookup's: retrodeck's arcade list was 495 KB of index. - lines[block_start:] = search_excluded(lines[block_start:]) + lines[block_start:] = search_excluded(block) return lines @@ -2340,14 +2315,12 @@ def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list: for line in (note, "")] if guide.more_folders: detected = _detected_systems(registry, name) if guide.more_folders_detected else [] - lines += [ - f'??? note "{guide.more_folders_title}"', + lines += markup.details("note", guide.more_folders_title, [ + *_folder_rows(guide.more_folders), + *(f"| {system} | `{folder}` |" for system, folder in detected), "", - *_folder_rows(guide.more_folders, " "), - *(f" | {system} | `{folder}` |" for system, folder in detected), - "", - *([f" {guide.more_folders_note}", ""] if guide.more_folders_note else []), - ] + *([guide.more_folders_note, ""] if guide.more_folders_note else []), + ]) lines += [ *(["", ""] if guide.on_card else []), "## Install", @@ -2454,14 +2427,9 @@ def generate_platform_page( ] if registry.get(name, {}).get("status") == "archived": - lines.extend( - [ - '!!! warning "Archived platform"', - "", - f" {display}'s list is kept as last read. Its pack is still built.", - "", - ] - ) + lines.extend(markup.aside("warning", "Archived platform", [ + f"{display}'s list is kept as last read. Its pack is still built.", + ])) summary = GUIDES[name].summary if name in GUIDES else "" if summary: @@ -2485,45 +2453,23 @@ def generate_platform_page( if no_list: # MinUI and its forks hand each core a folder and name no file: a # 0/0 coverage band read as "this device is not supported". - lines.extend( - [ - '!!! info "No BIOS list of its own"', - "", - f" {display} names no BIOS file itself: it gives each emulator core " - "its own folder and the core looks for its files there. The pack " - "places every file the cores it runs need into those folders. The " - "core files are listed further down this page.", - "", - ] - ) + lines.extend(markup.aside("info", "No BIOS list of its own", [ + f"{display} names no BIOS file itself: it gives each emulator core " + "its own folder and the core looks for its files there. The pack " + "places every file the cores it runs need into those folders. The " + "core files are listed further down this page.", + ])) band = [] if no_list else [ - '
', - "", - '
', - f'{cov["present"]}/{cov["total"]}', - f'Coverage ({pct})', - "
", - "", - '
', - f'{cov["verified"]}', - 'Verified', - "
", - "", - '
', - f'{cov["missing"]}', - 'Missing', - "
", - "", - '
', - f'' - f'{mode}', - 'Checked by', - "
", + markup.block_open("div", 'class="rb-stats"'), "", + *_stat(f"{cov['present']}/{cov['total']}", f"Coverage ({pct})"), + *_stat(str(cov["verified"]), "Verified"), + *_stat(str(cov["missing"]), "Missing"), + *_stat(f'{mode}', "Checked by"), "
", "", ] - lines.extend([*band, '
', "", + lines.extend([*band, markup.block_open("div", 'class="rb-meta-card"'), "", "| Detail | Value |", "|---|---|"]) if hash_type: lines.append(f"| Hash type | {hash_type} |") @@ -3150,9 +3096,7 @@ def generate_provenance_page( lines.extend([f"### {label}", ""]) for dat in sorted(by_dat): entries = by_dat[dat] - lines.extend([f'??? note "{dat} ({len(entries)})"', ""]) - lines.append(" | Name | Description | SHA1 |") - lines.append(" |------|-------------|------|") + table = ["| Name | Description | SHA1 |", "|------|-------------|------|"] for entry in sorted(entries, key=lambda e: e["name"]): sha1 = entry.get("sha1") or "-" # A pipe in catalog text would split the markdown table row @@ -3164,8 +3108,8 @@ def generate_provenance_page( f" Ships in {_ROMSET_SOURCES.get(hit['source'], hit['source'])} " f"`{hit['archive']}` as `{hit['member']}`." ) - lines.append(f" | `{name}` | {desc} | `{sha1}` |") - lines.append("") + table.append(f"| `{name}` | {desc} | `{sha1}` |") + lines.extend(markup.details("note", f"{dat} ({len(entries)})", table)) if derived_entries: lines.extend( @@ -3302,11 +3246,11 @@ def _system_file_card( anchor = "" else: anchored.add(anchor) - lines = ['
', ""] + lines = [markup.block_open("div", 'class="rb-sys-file"'), ""] # The heading's own link is this page's, the second link is the file's # own address, resolved by content wherever its card moves. permalink = ( - f' permalink' ) if anchor else "" lines.append(_card_title( @@ -3373,8 +3317,8 @@ def tables_out_of_search(markdown: str, threshold: int = SEARCH_TABLE_BYTES) -> end += 1 table = lines[index:end] if sum(len(line) for line in table) > threshold: - out += ["", f"{indent}
", "", *table, - "", f"{indent}
", ""] + out += ["", indent + markup.block_open("section", markup.search_exclude()), "", + *table, "", f"{indent}", ""] else: out += table index = end @@ -3393,7 +3337,8 @@ def search_excluded(lines: list[str]) -> list[str]: the first div closed inside an excluded div (a code block's) ended the exclusion and the rest of the block was indexed. """ - return ["", "
", "", *lines, "", "
", ""] + return ["", markup.block_open("section", markup.search_exclude()), "", + *lines, "", "", ""] # Card lines the site search leaves out. They list where a file goes and @@ -3418,7 +3363,8 @@ def _search_skipped_apart(lines: list[str]) -> list[str]: for line in lines: skip = line.startswith(SEARCH_SKIPPED) or (skipping and line.startswith(" ")) if skip and not skipping: - out.extend(["", '
', ""]) + out.extend(["", markup.block_open( + "div", f'class="rb-search-skip" {markup.search_exclude()}'), ""]) elif skipping and not skip: out.extend(["", "
", ""]) skipping = skip @@ -3438,6 +3384,13 @@ def _card_title(text: str, anchor: str, searchable: bool = True) -> str: of cards stops above it (mkdocs_hooks.py), and a card without an anchor keeps the id the toc gives it. """ + if markup.starlight(): + # Raw HTML: the card links to itself already, and a Markdown heading + # would add the engine's anchor link and icon to every card. + title = re.sub(r"`([^`]+)`", lambda m: f"{html.escape(m[1], quote=False)}", text) + ident = f' id="{anchor}"' if anchor else "" + ignore = "" if searchable else f" {markup.search_exclude()}" + return f'{title}' attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title" if not searchable: attributes += " data-search-exclude" @@ -3509,11 +3462,8 @@ def _cards_markdown(cards: list[Card]) -> list[str]: # A variant gets the same card as a primary: the revision a # platform pins is the one a reader comes to find, and showed as # a name and a truncated MD5, nothing said who pins it. - lines.append(f'??? note "Variants ({len(variants)})"') - lines.append("") - for card in variants: - lines.extend(f" {line}" if line else "" for line in card) - lines.append("") + lines.extend(markup.details( + "note", f"Variants ({len(variants)})", [line for card in variants for line in card])) return lines @@ -4019,7 +3969,7 @@ def _render_emulator_file( badges = _file_badges(f, in_repo) badge_str = " ".join(badges) - lines.append('
') + lines.append(markup.block_open("div", 'class="rb-file-entry"')) lines.append("") marker = f'' if alias else "" lines.append(_card_title(f"{marker}{_code(fname)} {badge_str}", anchor)) @@ -4307,14 +4257,12 @@ def _install_block(name: str, profile: dict) -> list[str]: """ if profile.get("type") in ("alias", "launcher", "test"): return [] - return search_excluded([ - '??? tip "Install this emulator\'s files"', - "", - " The installer finds your platform and its BIOS folder, then downloads " + return search_excluded(markup.details("tip", "Install this emulator's files", [ + "The installer finds your platform and its BIOS folder, then downloads " "only the files this emulator reads there and checks each one.", "", - *install_commands(f"--emulator {name}", " "), - ]) + *install_commands(f"--emulator {name}"), + ])) def _platform_install_block(name: str, display: str, on_card: bool = False) -> list[str]: @@ -4325,37 +4273,32 @@ def _platform_install_block(name: str, display: str, on_card: bool = False) -> l computer it is mounted on, whose root is named. """ if on_card: - return [ - '???+ tip "Install with one command"', - "", - f" With the {display} SD card mounted, the installer downloads the files " + return markup.details("tip", "Install with one command", [ + f"With the {display} SD card mounted, the installer downloads the files " "of the pack that are missing or differ from it into `Bios//` on " "the card, checks each one, and resumes where it stopped. `--dest` " "names the card's root.", "", - " Linux, macOS:", + "Linux, macOS:", "", - " ```sh", - f" {install_command('sh', f'--platform {name} --dest /path/to/card')}", - " ```", + "```sh", + install_command("sh", f"--platform {name} --dest /path/to/card"), + "```", "", - " Windows (PowerShell, E: being the card):", + "Windows (PowerShell, E: being the card):", "", - " ```powershell", - " " + install_command("powershell", f"--platform {name} --dest E:\\"), - " ```", - "", - ] - return [ - '???+ tip "Install with one command"', - "", - f" The installer downloads the files of the {display} pack that are " + "```powershell", + install_command("powershell", f"--platform {name} --dest E:\\"), + "```", + ], opened=True) + return markup.details("tip", "Install with one command", [ + f"The installer downloads the files of the {display} pack that are " "missing or differ from it, checks each one, and resumes where it " "stopped. Add `--system` with the ids of the systems overview below " "to install fewer systems.", "", - *install_commands(f"--platform {name}", " "), - ] + *install_commands(f"--platform {name}"), + ], opened=True) def _console_install_block(console: str, declared: list[tuple[str, str]]) -> list[str]: @@ -4363,19 +4306,17 @@ def _console_install_block(console: str, declared: list[tuple[str, str]]) -> lis if not declared: return [] lines = [ - f'??? tip "Install the {console} files"', - "", - " Each platform names the system its own way. On Windows, run the " + "Each platform names the system its own way. On Windows, run the " "same options through `&([scriptblock]::Create((irm " f"{INSTALL_BASE}/install.ps1)))`.", "", - " ```sh", + "```sh", ] lines += [ - f" {install_command('sh', f'--platform {platform} --system {shlex.quote(system)}')}" + install_command("sh", f"--platform {platform} --system {shlex.quote(system)}") for platform, system in declared ] - return search_excluded(lines + [" ```", ""]) + return search_excluded(markup.details("tip", f"Install the {console} files", [*lines, "```"])) def _render_platform_details(profile: dict) -> list[str]: @@ -4383,16 +4324,16 @@ def _render_platform_details(profile: dict) -> list[str]: lines: list[str] = [] platform_details = profile.get("platform_details") if platform_details and isinstance(platform_details, dict): - lines.extend(['???+ info "Platform details"', ""]) + body: list[str] = [] for pk, pv in platform_details.items(): if isinstance(pv, dict): - lines.extend([f" - **{pk}:**", ""]) + body.extend([f"- **{pk}:**", ""]) for sk, sv in pv.items(): - lines.append(f" - {sk}: {_detail_value(profile, sk, sv)}") - lines.append("") + body.append(f" - {sk}: {_detail_value(profile, sk, sv)}") + body.append("") else: - lines.append(f" - **{pk}:** {_detail_value(profile, pk, pv)}") - lines.append("") + body.append(f"- **{pk}:** {_detail_value(profile, pk, pv)}") + lines.extend(markup.details("info", "Platform details", body, opened=True)) return lines @@ -4466,13 +4407,11 @@ def _render_structured_blocks(profile: dict) -> list[str]: val = profile.get(field) if val is None: continue - lines.append(f'???+ abstract "{label}"') - lines.append("") - start = len(lines) - _render_yaml_value(lines, val, indent=4, + body: list[str] = [] + _render_yaml_value(body, val, indent=0, field_text=_structured_field_text(profile)) - lines[start:] = [_escape_tags(line) for line in lines[start:]] - lines.append("") + lines.extend(markup.details( + "abstract", label, [_escape_tags(line) for line in body], opened=True)) return lines @@ -4728,7 +4667,7 @@ def emulator_pages( f"# {emu_name}", "", *([intro, ""] if intro else []), - '
', + markup.block_open("div", 'class="rb-meta-card"'), "", "| Detail | Value |", "|---|---|", @@ -4812,19 +4751,14 @@ def emulator_pages( lines.extend(_render_structured_blocks(profile)) if notes: - indented = _admonition_body(link_prose_citations(profile, notes)) - lines.extend(['??? note "Technical notes"', f" {indented}", ""]) + body = _admonition_body(link_prose_citations(profile, notes)).replace("\n ", "\n") + lines.extend(markup.details("note", "Technical notes", body.split("\n"))) if not files: lines.append("No BIOS or firmware files required.") if exclusion: - lines.extend( - [ - "", - '!!! info "Why no files"', - f" {_escape_tags(str(exclusion))}", - ] - ) + lines.append("") + lines.extend(markup.aside("info", "Why no files", [_escape_tags(str(exclusion))])[:-1]) else: _file_available = _availability_check(db, data_names) @@ -4851,7 +4785,11 @@ def emulator_pages( if data_dirs: # The member tables are a file list with hashes: out of the search, # dolphin, primehack and ishiiruka each put 210 KB of it there. - lines.extend(["## Data directories { data-search-exclude }", ""]) + lines.extend([ + "## Data directories" + (" { data-search-exclude }" if not markup.starlight() else ""), + "", + ]) + trees_start = len(lines) member_anchors: set[str] = set() # Each tree with what it holds, where it comes from, which build reads # it and the code that does: these trees put thousands of files in @@ -4881,13 +4819,7 @@ def emulator_pages( lines.append(f" - Source: {_source_ref_markdown(profile, dd['source_ref'])}") members = _DATA_MEMBERS.get(ref) or {} if members: - lines.extend([ - "", - f' ??? info "{_count(len(members), "file")}, as the packs ship them"', - "", - " | File | Size | SHA1 |", - " |------|-----:|------|", - ]) + table = ["| File | Size | SHA1 |", "|------|-----:|------|"] for member, info in sorted(members.items()): sha1 = str(info.get("sha1", "")) # One id per page: two members can hold the same bytes @@ -4895,12 +4827,19 @@ def emulator_pages( anchor = f"sha1-{sha1[:12]}" span = "" if anchor in member_anchors else f'' member_anchors.add(anchor) - lines.append( - f" | {span}{_code(member)} " + table.append( + f"| {span}{_code(member)} " f"| {_fmt_size(int(info.get('size', 0)))} | {_code(sha1)} |" ) lines.append("") + lines.extend(markup.details( + "info", f"{_count(len(members), 'file')}, as the packs ship them", + table, indent=" ")) lines.append("") + if markup.starlight(): + # The member tables are the lookup's, as the heading's attribute + # keeps them under MkDocs. + lines[trees_start:] = search_excluded(lines[trees_start:]) lines.extend(_next_block([ "[Get your files](../which-pack.md), for the setup you play on", @@ -4927,7 +4866,7 @@ def _card_index( + ", ".join(f"[{number}]({stem}.md)" for number, (stem, _e) in enumerate(parts, 1)) + ".", "", - '
', + markup.block_open("section", f'class="rb-card-index" {markup.search_exclude()}'), "", "| File | Required | Category |", "|------|----------|----------|", @@ -5302,9 +5241,11 @@ def generate_gap_analysis( "\"Files to source\", [adding it](contributing.md) fills the gap for " "every platform at once.", "", - "[Download gaps CSV](downloads/gaps.csv){ .md-button } " - "[Open gaps API](api/v1/gaps.json){ .md-button } " - "[All data exports](data.md){ .md-button }", + " ".join([ + markup.button("Download gaps CSV", "downloads/gaps.csv"), + markup.button("Open gaps API", "api/v1/gaps.json"), + markup.button("All data exports", "data.md"), + ]), "", _platform_lists_sentence( total_files_verify, total_verified, total_untested, total_missing_verify, @@ -5733,13 +5674,7 @@ def _render_xref_by_platform( display = cov["platform"] config = cov["config"] - lines.append(f'??? abstract "{display}"') - lines.append("") - lines.append( - f" [Open {display} platform profile](platforms/{pname}.md)" - "{ .md-button }" - ) - lines.append("") + body = [markup.button(f"Open {display} platform profile", f"platforms/{pname}.md"), ""] # The cores the builder and verify resolve: a hand copy here dropped # the libretro set a list naming retroarch pulls in, and the system @@ -5750,11 +5685,11 @@ def _render_xref_by_platform( if key in unique } - lines.append(f" **{len(matched)} cores**{_core_scope_note(config, unique)}") - lines.append("") + body.append(f"**{len(matched)} cores**{_core_scope_note(config, unique)}") + body.append("") - lines.append(" | Core | Classification | Systems | Files | Upstream |") - lines.append(" |------|---------------|---------|-------|----------|") + body.append("| Core | Classification | Systems | Files | Upstream |") + body.append("|------|---------------|---------|-------|----------|") for emu_name in sorted(matched.keys()): p = matched[emu_name] @@ -5798,12 +5733,12 @@ def _render_xref_by_platform( elif source: upstream_display = source - lines.append( - f" | [{emu_display}](emulators/{emu_name}.md) | {cls} | " + body.append( + f"| [{emu_display}](emulators/{emu_name}.md) | {cls} | " f"{sys_str} | {file_str} | {upstream_display} |" ) - lines.append("") + lines.extend(markup.details("abstract", display, body)) return lines @@ -5892,9 +5827,11 @@ def generate_cross_reference( "The files a platform's emulators load and its own list does not name are " "counted on [What the packs do not hold](gaps.md).", "", - "[Download cross-reference CSV](downloads/cross-reference.csv){ .md-button } " - "[Open emulator API](api/v1/emulators.json){ .md-button } " - "[All data exports](data.md){ .md-button }", + " ".join([ + markup.button("Download cross-reference CSV", "downloads/cross-reference.csv"), + markup.button("Open emulator API", "api/v1/emulators.json"), + markup.button("All data exports", "data.md"), + ]), "", ] @@ -5915,13 +5852,11 @@ def generate_contributing() -> str: # The end of a reference page: back to the index, or on to contributing. -REFERENCE_NEXT = "\n" + "\n".join([ - '!!! tip "Next"', - "", - " - [Reference index](wiki/index.md), every reference page in one list", - " - For contributors: [How to contribute](contributing.md)", - "", -]) +def reference_next() -> str: + return "\n" + "\n".join(markup.aside("tip", "Next", [ + "- [Reference index](wiki/index.md), every reference page in one list", + "- For contributors: [How to contribute](contributing.md)", + ])) # Wiki pages @@ -6238,34 +6173,26 @@ def generate_which_pack() -> str: "", *terminal_tabs(("windows", "macos", "linux", "steamdeck", "android"), dict.fromkeys(TERMINALS, "--check"), steps=False), - '??? note "Fill an SD card from a computer"', - "", - " Name your setup and the BIOS folder on the card. For RetroArch on a " - "Nintendo Switch:", - "", - " ```sh", - " " + install_command( - "sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system"), - " ```", - "", + *markup.details("note", "Fill an SD card from a computer", [ + "Name your setup and the BIOS folder on the card. For RetroArch on a " + "Nintendo Switch:", + "", + "```sh", + install_command( + "sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system"), + "```", + ]), "Details: [All installer options](wiki/installer.md).", "", "## Pick your setup", "", - '
', - "", ] - for group, (label, _description) in DEVICE_GROUPS.items(): - members = _group_keys(registry, group, display) - if not members: - continue - lines += [ - f"- **{label}**", - "", - f" [{', '.join(display[k] for k in members)}](#{GROUP_ANCHORS[group]})", - "", - ] - lines += ["
", ""] + lines += markup.card_grid([ + (label, [f"[{', '.join(display[k] for k in members)}](#{GROUP_ANCHORS[group]})"]) + for group, (label, _description) in DEVICE_GROUPS.items() + for members in [_group_keys(registry, group, display)] + if members + ]) for group, (label, description) in DEVICE_GROUPS.items(): members = _group_keys(registry, group, display) if not members: @@ -6294,18 +6221,15 @@ def generate_which_pack() -> str: ) lines.append("") if group == "handheld": - lines += [ - '??? note "Other handhelds that run RetroArch"', - "", - " They use the RetroArch files, in the folder below. The command " + lines += markup.details("note", "Other handhelds that run RetroArch", [ + "They use the RetroArch files, in the folder below. The command " "finds it.", "", - " | System | Runs | BIOS folder |", - " |--------|------|-------------|", - *(f" | {name} | [RetroArch](platforms/retroarch.md) | `{folder}` |" + "| System | Runs | BIOS folder |", + "|--------|------|-------------|", + *(f"| {name} | [RetroArch](platforms/retroarch.md) | `{folder}` |" for name, folder in _detected_systems(registry, "retroarch")), - "", - ] + ]) lines += [ "## Download a ZIP { #manual-download }", "", @@ -6699,7 +6623,7 @@ def _generate(args: argparse.Namespace) -> None: exports = generate_data_exports( docs, db, coverages, profiles, stats, gap_report ) - write_if_changed(str(docs / "data.md"), generate_data_page(exports) + REFERENCE_NEXT) + write_if_changed(str(docs / "data.md"), generate_data_page(exports) + reference_next()) PAGE_DATA["data.md"] = dataset_structured_data(exports, db) export_reports = load_export_reports() @@ -6774,7 +6698,7 @@ def _generate(args: argparse.Namespace) -> None: print("Generating cross-reference page...") write_if_changed( str(docs / "cross-reference.md"), - tables_out_of_search(generate_cross_reference(coverages, profiles)) + REFERENCE_NEXT, + tables_out_of_search(generate_cross_reference(coverages, profiles)) + reference_next(), ) print("Generating gap analysis page...") @@ -6782,7 +6706,7 @@ def _generate(args: argparse.Namespace) -> None: str(docs / "gaps.md"), tables_out_of_search(generate_gap_analysis( profiles, coverages, db, suppl_names, registry, gap_report - )) + REFERENCE_NEXT, + )) + reference_next(), ) print("Generating updated platform files page...") @@ -6791,7 +6715,7 @@ def _generate(args: argparse.Namespace) -> None: generate_exports_page( export_reports, {name: cov["platform"] for name, cov in coverages.items()}, - ) + REFERENCE_NEXT, + ) + reference_next(), ) print("Generating dump provenance page...") @@ -6800,7 +6724,7 @@ def _generate(args: argparse.Namespace) -> None: str(docs / "provenance.md"), tables_out_of_search( generate_provenance_page(db, provenance_report, load_derivable()) - ) + REFERENCE_NEXT, + ) + reference_next(), ) # Wiki pages: copy manually maintained sources + generate dynamic ones diff --git a/scripts/sitediagrams.py b/scripts/sitediagrams.py index 99f28365..4d5d5f15 100644 --- a/scripts/sitediagrams.py +++ b/scripts/sitediagrams.py @@ -36,6 +36,8 @@ from xml.sax.saxutils import escape ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / "scripts")) +import sitemarkup as markup # noqa: E402 + PLACEHOLDER = re.compile(r"(?m)^([ \t]*)[ \t]*$") PLACEHOLDER_ANYWHERE = re.compile(r"") @@ -605,12 +607,12 @@ def figure_html(image_id: str, occurrence: int = 1, inline: bool = False) -> str anchor = f"rb-fig-{image_id}{suffix}" if inline: html = ( - f'' + f'' f'{drawings}{caption}' ) return html.translate(_INLINE_ESCAPES) return ( - f'
\n' + f'
\n' f"{drawings}\n" f"
{caption}
\n" "
" diff --git a/scripts/siteexports.py b/scripts/siteexports.py index bf85862b..5b02b6e9 100644 --- a/scripts/siteexports.py +++ b/scripts/siteexports.py @@ -11,6 +11,8 @@ from __future__ import annotations import json from pathlib import Path +import sitemarkup as markup + EXPORTS_DIR = "exports" REPO_BLOB = "https://github.com/Abdess/retrobios/blob/main" REPO_TREE = "https://github.com/Abdess/retrobios/tree/main" @@ -73,12 +75,8 @@ def requirement_lines(report: dict) -> list[str]: ] if not items: return [] - return [ - f'??? note "Required flags aligned with the code ({len(items)})"', - "", - *[f" - {item}" for item in items], - "", - ] + return markup.details("note", f"Required flags aligned with the code ({len(items)})", + [f"- {item}" for item in items]) def _plural(count: int, noun: str, plural: str = "") -> str: @@ -141,12 +139,8 @@ def candidate_lines(candidates: list[dict]) -> list[str]: title = (reason[:1].upper() + reason[1:]).replace('"', "'") items = [f"`{c['system']}` `{c['destination']}`" for c in entries] if len(entries) > LONG_LIST: - lines.extend([ - f'??? note "{title} ({len(entries)})"', - "", - *[f" - {item}" for item in items], - "", - ]) + lines.extend(markup.details("note", f"{title} ({len(entries)})", + [f"- {item}" for item in items])) else: lines.extend([f"{title}:", "", *[f"- {item}" for item in items], ""]) return lines @@ -263,35 +257,29 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str: ]) lines.extend(candidate_lines(report.get("removal_candidates") or [])) if report.get("invalid"): - lines.extend([ - '!!! warning "The updated file does not pass the platform\'s own checks"', - "", - *[f" - {issue}" for issue in report["invalid"]], - "", - ]) + lines.extend(markup.aside( + "warning", "The updated file does not pass the platform's own checks", + [f"- {issue}" for issue in report["invalid"]])) originals = report.get("originals") or {} files = report["files"] raw = [f"{REPO_RAW}/{EXPORTS_DIR}/{platform}/{name}" for name in files] if len(files) == 1: lines.extend([ - f"[Download the updated file]({raw[0]}){{ .md-button .md-button--primary }} " - f"[See it on GitHub]({REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}){{ .md-button }}", + markup.button("Download the updated file", raw[0], primary=True) + " " + + markup.button("See it on GitHub", f"{REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}"), "", ]) else: lines.extend([ - f"[Browse the {len(files)} updated files]" - f"({REPO_TREE}/{EXPORTS_DIR}/{platform}){{ .md-button .md-button--primary }}", + markup.button(f"Browse the {len(files)} updated files", + f"{REPO_TREE}/{EXPORTS_DIR}/{platform}", primary=True), "", ]) if len(originals) > 1: - lines.extend([ - f'??? note "The {len(originals)} original files it is based on"', - "", - *[f" - [`{name}`]({url})" for name, url in sorted(originals.items())], - "", - ]) + lines.extend(markup.details( + "note", f"The {len(originals)} original files it is based on", + [f"- [`{name}`]({url})" for name, url in sorted(originals.items())])) elif originals: (name, url), = originals.items() lines.extend([f"Based on [`{name}`]({url}).", ""]) @@ -301,10 +289,6 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str: items = change_items(report, key) if not items: continue - lines.extend([ - f'??? note "{title} ({len(items)})"', - "", - *[f" - {item}" for item in items], - "", - ]) + lines.extend(markup.details("note", f"{title} ({len(items)})", + [f"- {item}" for item in items])) return "\n".join(lines) diff --git a/scripts/siteguides.py b/scripts/siteguides.py index 990d435c..f0f4fe5b 100644 --- a/scripts/siteguides.py +++ b/scripts/siteguides.py @@ -15,6 +15,7 @@ import textwrap from dataclasses import dataclass, fields from pathlib import Path +import sitemarkup as markup from safeparse import require_yaml PLATFORMS_DIR = Path(__file__).resolve().parent.parent / "platforms" @@ -263,15 +264,12 @@ def terminal_tabs( """Content tabs, one per place the command runs, each with how to open the terminal there. Tab labels are shared across the site, so the reader's choice carries from page to page.""" - lines: list[str] = [] + items = [] for key in tabs: label, how, shell = TERMINALS[key] - lines += [f'{indent}=== "{label}"', ""] - if steps: - lines += [f"{indent} {how}", ""] - lines += _fence(shell, install_command(shell, args.get(key, "")), indent + " ") - lines.append("") - return lines + body = [how, ""] if steps else [] + items.append((label, body + _fence(shell, install_command(shell, args.get(key, "")), ""))) + return markup.tabs(items, indent) def platform_args(name: str, guide: Guide) -> dict[str, str]: @@ -290,11 +288,8 @@ def platform_args(name: str, guide: Guide) -> dict[str, str]: def trust_box(prefix: str, indent: str = "") -> list[str]: """Is it safe, under every command on a guide page: five plain facts.""" - return [ - f'{indent}??? note "Is it safe?"', + return markup.details("note", "Is it safe?", [ + *(f"- {item}" for item in TRUST_ITEMS), "", - *(f"{indent} - {item}" for item in TRUST_ITEMS), - "", - f"{indent} [Read the script first]({prefix}wiki/faq.md#is-piping-the-installer-into-a-shell-safe)", - "", - ] + f"[Read the script first]({prefix}wiki/faq.md#is-piping-the-installer-into-a-shell-safe)", + ], indent) diff --git a/scripts/sitemarkup.py b/scripts/sitemarkup.py new file mode 100644 index 00000000..2116b556 --- /dev/null +++ b/scripts/sitemarkup.py @@ -0,0 +1,183 @@ +"""The page constructs the generator writes, in the syntax of the site engine. + +Every block that is not plain Markdown (a callout, a collapsible block, OS +tabs, a button, a card grid) is written through one of these functions, so +the generator holds no engine syntax of its own. Bodies are passed without +indentation: the function indents them as the engine reads them. + +The engine is Astro Starlight. MkDocs Material is kept while the two builds +are compared page by page, selected with set_engine("mkdocs"). +""" + +from __future__ import annotations + +import html +import re + +ENGINES = ("mkdocs", "starlight") +_engine = "mkdocs" + +# Material's callout kinds, by the role Starlight gives them. +ASIDE_ROLES = { + "note": "note", "info": "note", "abstract": "note", "example": "note", + "question": "note", "quote": "note", "tip": "tip", "success": "tip", + "warning": "caution", "failure": "danger", "danger": "danger", "bug": "danger", +} + +# The attribute that keeps an element out of the site search. +SEARCH_EXCLUDE = {"mkdocs": "data-search-exclude", "starlight": "data-pagefind-ignore"} + + +def set_engine(name: str) -> None: + global _engine + if name not in ENGINES: + raise ValueError(f"unknown site engine {name!r}, expected one of {ENGINES}") + _engine = name + + +def engine() -> str: + return _engine + + +def starlight() -> bool: + return _engine == "starlight" + + +def search_exclude() -> str: + return SEARCH_EXCLUDE[_engine] + + +def inline_html(text: str) -> str: + """Inline Markdown (code, bold, links) for text placed inside raw HTML, + where Markdown is not read.""" + parts = re.split(r"(`[^`]+`)", text) + out = [] + for part in parts: + if part.startswith("`") and part.endswith("`") and len(part) > 1: + out.append(f"{html.escape(part[1:-1], quote=False)}") + continue + part = html.escape(part, quote=False) + part = re.sub(r"\*\*([^*]+)\*\*", r"\1", part) + part = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", r'\1', part) + out.append(part) + return "".join(out) + + +def _indented(lines: list[str], pad: str) -> list[str]: + return [f"{pad}{line}" if line.strip() else "" for line in lines] + + +def _trim(lines: list[str]) -> list[str]: + start, end = 0, len(lines) + while start < end and not lines[start].strip(): + start += 1 + while end > start and not lines[end - 1].strip(): + end -= 1 + return list(lines[start:end]) + + +def aside(kind: str, title: str, body: list[str], indent: str = "") -> list[str]: + """A callout that is always open.""" + body = _trim(body) + if _engine == "mkdocs": + return [f'{indent}!!! {kind} "{title}"', "", *_indented(body, indent + " "), ""] + label = title.replace("[", "\\[").replace("]", "\\]") + fence = "::::" if any(line.lstrip().startswith(":::") for line in body) else ":::" + return [f"{indent}{fence}{ASIDE_ROLES.get(kind, 'note')}[{label}]", + *_indented(body, indent), f"{indent}{fence}", ""] + + +def details(kind: str, title: str, body: list[str], indent: str = "", opened: bool = False) -> list[str]: + """A collapsible block, closed unless *opened*.""" + body = _trim(body) + if _engine == "mkdocs": + marker = "???+" if opened else "???" + return [f'{indent}{marker} {kind} "{title}"', "", *_indented(body, indent + " "), ""] + role = ASIDE_ROLES.get(kind, "note") + state = " open" if opened else "" + return [ + f'{indent}
', + f"{indent}{inline_html(title)}", + "", + *_indented(body, indent), + "", + f"{indent}
", + "", + ] + + +def tabs(items: list[tuple[str, list[str]]], indent: str = "", sync: str = "os") -> list[str]: + """Content tabs. Tabs that share a label share the reader's choice + across pages.""" + lines: list[str] = [] + if _engine == "mkdocs": + for label, body in items: + lines += [f'{indent}=== "{label}"', "", *_indented(_trim(body), indent + " "), ""] + return lines + lines.append(f'{indent}') + for label, body in items: + lines += [f'{indent}', "", + *_indented(_trim(body), indent), "", f"{indent}"] + return [*lines, f"{indent}", ""] + + +def button(text: str, target: str, primary: bool = False) -> str: + """A link drawn as a button.""" + if _engine == "mkdocs": + classes = ".md-button .md-button--primary" if primary else ".md-button" + return f"[{text}]({target}){{ {classes} }}" + classes = "rb-button rb-button-primary" if primary else "rb-button" + return f'{inline_html(text)}' + + +def image(alt: str, src: str, **attributes: str) -> str: + """An image with HTML attributes (width, align).""" + if _engine == "mkdocs": + attrs = " ".join(f"{key}={value}" for key, value in attributes.items()) + return f"![{alt}]({src}){{ {attrs} }}" + attrs = "".join(f' {key}="{html.escape(str(value))}"' for key, value in attributes.items()) + return f'{html.escape(alt)}' + + +def block_open(tag: str, attributes: str = "") -> str: + """The opening tag of an HTML block whose content is Markdown.""" + attributes = f" {attributes.strip()}" if attributes.strip() else "" + if _engine == "mkdocs": + return f"<{tag}{attributes} markdown>" + return f"<{tag}{attributes}>" + + +def card_grid(cards: list[tuple[str, list[str]]]) -> list[str]: + """Cards in a grid: a bold title line and a body each.""" + lines = [block_open("div", 'class="grid cards"' if _engine == "mkdocs" else 'class="rb-cards"'), ""] + for title, body in cards: + if _engine == "mkdocs": + lines += [f"- **{title}**", "", *_indented(_trim(body), " "), ""] + else: + lines += ['
', "", f"**{title}**", "", *_trim(body), "", "
", ""] + return [*lines, "
", ""] + + +def steps(items: list[str]) -> list[str]: + """An ordered list drawn as numbered steps.""" + return [block_open("div", 'class="rb-steps"'), "", + *(f"{number}. {item}" for number, item in enumerate(items, 1)), "", "
", ""] + + +def heading_attributes(anchor: str = "", classes: str = "", search: bool = True) -> str: + """The attribute block of a heading: its id, classes, and whether the + search reads it.""" + parts = [f"#{anchor}" if anchor else "", *(f".{c}" for c in classes.split())] + if not search: + parts.append(search_exclude()) + parts = [part for part in parts if part] + return f" {{ {' '.join(parts)} }}" if parts else "" + + +def site_href(path: str) -> str: + """A link to a site address written in raw HTML (``files/?sha1=...``), + which no Markdown resolution reaches: rooted at the site under Starlight, + relative to a page two levels deep under MkDocs.""" + if _engine == "mkdocs": + return f"../../{path}" + return f"/retrobios/{path}"