mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 15:43:45 -06:00
Abdessamad Derraz
087e5b4d73
Each emulator with records gets a Tested block: claim, result, method, computer, core build and the permalink of its record at the pin. A Measurements page filters every test in the browser from the compact index, and the Data page lists measurements.sqlite with its tables. One SQLite file per emulator is linked from its page. wiki/measurements.md says how a test runs, held to the Start here budgets. Both workflows fetch the pinned commit alone with git before the build, and the deploy fails without it.
887 lines
36 KiB
Python
887 lines
36 KiB
Python
"""The site as an Astro Starlight project: pages, sidebar and data files.
|
|
|
|
generate_site.py writes each page body, and this module gives it the form
|
|
Starlight reads. A page's links are written as the docs tree holds them
|
|
(``../systems/sony.md``, MkDocs' convention, the one a Markdown editor
|
|
follows), and the site serves them as addresses under /retrobios/, so
|
|
each target is resolved against the page's own place here. Every heading
|
|
keeps the id the MkDocs site gave it, so the anchors cited since stay
|
|
valid.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
import urllib.parse
|
|
from collections.abc import Callable, Iterator, Mapping
|
|
from html import unescape
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
BASE = "/retrobios/"
|
|
SITE_URL = "https://abdess.github.io/retrobios/"
|
|
|
|
# Components a page body may use, and where each comes from.
|
|
COMPONENTS = {
|
|
"Tabs": "@astrojs/starlight/components",
|
|
"TabItem": "@astrojs/starlight/components",
|
|
"FilePicker": "components/FilePicker.astro",
|
|
"MeasurementsExplorer": "components/MeasurementsExplorer.astro",
|
|
}
|
|
|
|
# Pages of file cards: level 3 headings by the hundred, so the table of
|
|
# contents stops at level 2.
|
|
CARD_PAGES = ("systems/", "emulators/", "platforms/")
|
|
|
|
FENCE = re.compile(r"(^[ \t]*(`{3,}|~{3,}).*?^[ \t]*\2[ \t]*$)", re.M | re.S)
|
|
CODE_SPAN = re.compile(r"(`+)(?!`)(.+?)(?<!`)\1(?!`)", re.S)
|
|
MD_LINK = re.compile(r"(?P<bang>!?)\[(?P<text>(?:[^\[\]\n]|\[[^\]\n]*\])*)\]\((?P<url>[^)\s]+)(?P<title> \"[^\"]*\")?\)")
|
|
HTML_URL = re.compile(r"(?P<attr>\b(?:href|src))=\"(?P<url>[^\"]*)\"")
|
|
ATTR_BLOCK = re.compile(r"[ \t]*\{(?P<attrs>[^{}]*)\}[ \t]*$")
|
|
ID_ATTR = re.compile(r"\bid=\"(?P<id>[^\"]+)\"")
|
|
IDCOUNT = re.compile(r"^(.*)_([0-9]+)$")
|
|
|
|
|
|
# Addresses
|
|
|
|
|
|
def page_url(rel: str) -> str:
|
|
"""The address of a page, by its path under the docs tree."""
|
|
stem = rel.rsplit(".", 1)[0] if rel.endswith((".md", ".mdx")) else rel
|
|
if stem == "index":
|
|
return BASE
|
|
if stem.endswith("/index"):
|
|
stem = stem[: -len("/index")]
|
|
return f"{BASE}{stem}/"
|
|
|
|
|
|
def resolve(rel_page: str, target: str) -> str:
|
|
"""A link target written relative to the page's file, as a site address."""
|
|
if not target or re.match(r"^(?:[a-z][a-z0-9+.-]*:|#|/)", target):
|
|
return target
|
|
path, hash_mark, fragment = target.partition("#")
|
|
path, question, query = path.partition("?")
|
|
parts: list[str] = []
|
|
for piece in (PurePosixPath(rel_page).parent / path).as_posix().split("/"):
|
|
if piece in ("", "."):
|
|
continue
|
|
if piece == "..":
|
|
if parts:
|
|
parts.pop()
|
|
else:
|
|
parts.append(piece)
|
|
joined = "/".join(parts)
|
|
if joined.endswith((".md", ".mdx")):
|
|
url = page_url(joined)
|
|
elif path.endswith("/") or not path:
|
|
url = f"{BASE}{joined}/" if joined else BASE
|
|
else:
|
|
url = f"{BASE}{joined}"
|
|
if question:
|
|
url += f"?{query}"
|
|
return url + (f"#{fragment}" if hash_mark else "")
|
|
|
|
|
|
# Text outside code
|
|
|
|
|
|
def outside_fences(text: str, fn: Callable[[str], str]) -> str:
|
|
parts = FENCE.split(text)
|
|
# FENCE has two groups: text, fence, marker, text, ...
|
|
out = []
|
|
for index in range(0, len(parts), 3):
|
|
out.append(fn(parts[index]))
|
|
if index + 1 < len(parts):
|
|
out.append(parts[index + 1])
|
|
return "".join(out)
|
|
|
|
|
|
def outside_code(text: str, fn: Callable[[str], str]) -> str:
|
|
def chunk(part: str) -> str:
|
|
out, position = [], 0
|
|
for match in CODE_SPAN.finditer(part):
|
|
out.append(fn(part[position:match.start()]))
|
|
out.append(match.group(0))
|
|
position = match.end()
|
|
out.append(fn(part[position:]))
|
|
return "".join(out)
|
|
return outside_fences(text, chunk)
|
|
|
|
|
|
def rewrite_links(text: str, rel: str) -> str:
|
|
def markdown(match: re.Match) -> str:
|
|
return (f"{match['bang']}[{match['text']}]({resolve(rel, match['url'])}"
|
|
f"{match['title'] or ''})")
|
|
|
|
def attribute(match: re.Match) -> str:
|
|
return f'{match["attr"]}="{resolve(rel, match["url"])}"'
|
|
|
|
text = outside_fences(text, lambda chunk: MD_LINK.sub(markdown, chunk))
|
|
return outside_fences(text, lambda chunk: HTML_URL.sub(attribute, chunk))
|
|
|
|
|
|
# Heading ids, as Python-Markdown's toc extension gives them
|
|
|
|
|
|
def toc_slug(text: str) -> str:
|
|
text = re.sub(r"<[^>]+>", "", text)
|
|
text = re.sub(r"!\[[^\]]*\]\([^)]*\)(\{[^}]*\})?", "", text)
|
|
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
|
|
text = unescape(text.replace("`", "").replace("**", "").replace("\\", ""))
|
|
text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii")
|
|
text = re.sub(r"[^\w\s-]", "", text).strip().lower()
|
|
return re.sub(r"[-\s]+", "-", text)
|
|
|
|
|
|
def _unique(ident: str, used: set[str]) -> str:
|
|
while ident in used or not ident:
|
|
match = IDCOUNT.match(ident)
|
|
ident = f"{match[1]}_{int(match[2]) + 1}" if match else f"{ident}_1"
|
|
used.add(ident)
|
|
return ident
|
|
|
|
|
|
def pin_heading_ids(text: str, title_id: str = "") -> str:
|
|
"""Every heading with the id the MkDocs build gave it. *title_id* is
|
|
the page title's, taken first."""
|
|
used: set[str] = {title_id} if title_id else set()
|
|
explicit = set()
|
|
for match in re.finditer(r"\{[^{}\n]*#([^\s}]+)[^{}\n]*\}[ \t]*$", text, re.M):
|
|
explicit.add(match[1])
|
|
for match in ID_ATTR.finditer(text):
|
|
explicit.add(match["id"])
|
|
used |= explicit
|
|
|
|
def markdown(match: re.Match) -> str:
|
|
heading = match["mdtext"]
|
|
attrs = ATTR_BLOCK.search(heading)
|
|
body = heading[: attrs.start()] if attrs else heading
|
|
tokens = attrs["attrs"].split() if attrs else []
|
|
if any(token.startswith("#") for token in tokens):
|
|
return match.group(0)
|
|
ident = _unique(toc_slug(body), used)
|
|
return f"{match['hashes']} {body.rstrip()} {{ {' '.join(['#' + ident, *tokens])} }}"
|
|
|
|
def raw(match: re.Match) -> str:
|
|
if ID_ATTR.search(match["attrs"]):
|
|
return match.group(0)
|
|
level, text = match["level"], match["htmltext"]
|
|
ident = _unique(toc_slug(text), used)
|
|
return f'<h{level} id="{ident}"{match["attrs"]}>{text}</h{level}>'
|
|
|
|
def pick(match: re.Match) -> str:
|
|
return markdown(match) if match["hashes"] else raw(match)
|
|
|
|
# In document order: one pass over either kind of heading.
|
|
return outside_fences(text, lambda chunk: _ANY_HEADING.sub(pick, chunk))
|
|
|
|
|
|
_ANY_HEADING = re.compile(
|
|
r"^(?P<hashes>#{2,6})[ \t]+(?P<mdtext>.+?)[ \t]*$"
|
|
r"|<h(?P<level>[2-6])(?P<attrs>(?:\s[^>]*)?)>(?P<htmltext>.*?)</h(?P=level)>",
|
|
re.M | re.S,
|
|
)
|
|
|
|
|
|
# Glossary tooltips
|
|
|
|
|
|
def glossary_terms(path: Path) -> dict[str, str]:
|
|
"""Each term the glossary defines, with its definition as plain text."""
|
|
terms: dict[str, str] = {}
|
|
if not path.is_file():
|
|
return terms
|
|
text = path.read_text(encoding="utf-8")
|
|
for match in re.finditer(r"^(?P<term>\*\*.+)\n: (?P<body>.+(?:\n .+)*)", text, re.M):
|
|
definition = re.sub(r"\s+", " ", match["body"]).strip()
|
|
definition = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", definition).replace("`", "")
|
|
for term in re.findall(r"\*\*([^*]+)\*\*", match["term"]):
|
|
terms.setdefault(term, definition)
|
|
return terms
|
|
|
|
|
|
_SKIP_TAGS = re.compile(r"<(a|code|pre|abbr|h[1-6]|script|style|summary|svg|figure)\b.*?</\1>", re.S | re.I)
|
|
|
|
|
|
def add_tooltips(text: str, terms: dict[str, str]) -> str:
|
|
"""The first use of each glossary term on a page, with its definition
|
|
a hover or a tap away. Headings, links, code and HTML are left as they are."""
|
|
if not terms:
|
|
return text
|
|
pending = dict(terms)
|
|
pattern = re.compile(
|
|
r"(?<![\w/.-])(" + "|".join(re.escape(t) for t in sorted(terms, key=len, reverse=True))
|
|
+ r")(?![\w/-])"
|
|
)
|
|
|
|
def prose(chunk: str) -> str:
|
|
if not pending:
|
|
return chunk
|
|
out = []
|
|
for line in chunk.split("\n"):
|
|
stripped = line.lstrip()
|
|
if (not pending or stripped.startswith(("#", "|", "<", ":::", "[", "!"))
|
|
or not stripped):
|
|
out.append(line)
|
|
continue
|
|
out.append(_tooltip_line(line, pattern, pending))
|
|
return "\n".join(out)
|
|
|
|
return outside_code(text, prose)
|
|
|
|
|
|
def _tooltip_line(line: str, pattern: re.Pattern, pending: dict[str, str]) -> str:
|
|
# Inline HTML, links and emphasis markers stay whole.
|
|
protected = re.compile(r"<[^>]+>|\[[^\]]*\]\([^)]*\)|\*\*[^*]+\*\*|\{[^}]*\}")
|
|
out, position = [], 0
|
|
for match in protected.finditer(line):
|
|
out.append(_tooltip_text(line[position:match.start()], pattern, pending))
|
|
out.append(match.group(0))
|
|
position = match.end()
|
|
out.append(_tooltip_text(line[position:], pattern, pending))
|
|
return "".join(out)
|
|
|
|
|
|
def _tooltip_text(text: str, pattern: re.Pattern, pending: dict[str, str]) -> str:
|
|
def swap(match: re.Match) -> str:
|
|
term = match.group(1)
|
|
if term not in pending:
|
|
return term
|
|
definition = pending.pop(term)
|
|
return f'<abbr class="rb-term" title="{html.escape(definition)}">{term}</abbr>'
|
|
return pattern.sub(swap, text)
|
|
|
|
|
|
# MDX
|
|
|
|
|
|
_VOID = re.compile(r"<(br|img|input|hr|source|meta|link|wbr)\b([^>]*?)(?<!/)>")
|
|
_HEADING_ATTRS = re.compile(r"(?m)(?<=\S)[ \t]+\{[ \t]*#[^{}\n]*\}[ \t]*$")
|
|
_COMMENT = re.compile(r"<!--.*?-->", re.S)
|
|
|
|
|
|
def mdx_safe(text: str) -> str:
|
|
"""What MDX would read as code in prose: braces, a stray ``<``,
|
|
comments, unclosed void tags and ``<https://...>`` links."""
|
|
def fix(chunk: str) -> str:
|
|
chunk = _COMMENT.sub("", chunk)
|
|
chunk = re.sub(r"<(https?://[^>\s]+)>", r"[\1](\1)", chunk)
|
|
held: list[str] = []
|
|
|
|
def hold(match: re.Match) -> str:
|
|
held.append(match.group(0))
|
|
return f"\x00{len(held) - 1}\x00"
|
|
chunk = _HEADING_ATTRS.sub(hold, chunk)
|
|
chunk = chunk.replace("{", "{").replace("}", "}")
|
|
chunk = re.sub(r"\x00(\d+)\x00", lambda m: held[int(m[1])], chunk)
|
|
chunk = re.sub(r"<(?![A-Za-z/])", "<", chunk)
|
|
return _VOID.sub(lambda m: f"<{m[1]}{m[2].rstrip()} />", chunk)
|
|
return outside_code(text, fix)
|
|
|
|
|
|
def used_components(body: str) -> list[str]:
|
|
return [name for name in COMPONENTS if re.search(rf"<{name}[\s/>]", body)]
|
|
|
|
|
|
# Pages
|
|
|
|
|
|
def _front_matter(data: dict) -> str:
|
|
import yaml
|
|
|
|
return "---\n" + yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=1000) + "---\n\n"
|
|
|
|
|
|
def og_section(rel: str, level: str) -> str:
|
|
"""The sharing card a page takes: its level, or what its address says."""
|
|
url = page_url(rel)[len(BASE):]
|
|
if not url:
|
|
return "home"
|
|
if level in ("start", "setup", "explore", "reference", "contribute"):
|
|
return level
|
|
if url.startswith(("which-pack", "wiki/first-install", "wiki/faq", "wiki/getting-started")):
|
|
return "start"
|
|
if url.startswith("platforms"):
|
|
return "setup"
|
|
if url.startswith(("systems", "emulators", "files")):
|
|
return "explore"
|
|
if url.startswith(("contributing", "wiki/adding", "wiki/profiling", "wiki/testing", "wiki/release")):
|
|
return "contribute"
|
|
return "reference"
|
|
|
|
|
|
# Sharing cards of their own, made by scripts/make_og_cards.py: one per
|
|
# platform page, titled with the platform's name.
|
|
OG_DIR = Path(__file__).resolve().parent.parent / "docs_assets" / "og"
|
|
|
|
|
|
def og_image(rel: str, level: str) -> str:
|
|
"""The sharing card's address: the page's own when it has one, else
|
|
its section's."""
|
|
own = re.fullmatch(r"(platforms|systems)/([a-z0-9_-]+)\.mdx?", rel)
|
|
if own and own[2] != "index" and (OG_DIR / own[1] / f"{own[2]}.png").is_file():
|
|
return f"{SITE_URL}assets/og/{own[1]}/{own[2]}.png"
|
|
return f"{SITE_URL}assets/og/{og_section(rel, level)}.png"
|
|
|
|
|
|
NOT_FOUND = "404.md"
|
|
|
|
|
|
def page_head(rel: str, level: str, title: str, browser_title: str, structured: dict) -> list[dict]:
|
|
"""What a page adds to its head: its own window title, its sharing
|
|
card (og_image), and its structured data. The page a stale link
|
|
lands on asks not to be indexed and describes nothing."""
|
|
head: list[dict] = []
|
|
if browser_title != title:
|
|
head.append({"tag": "title", "content": f"{browser_title} - RetroBIOS"})
|
|
if rel == NOT_FOUND:
|
|
return head + [{"tag": "meta", "attrs": {"name": "robots", "content": "noindex"}}]
|
|
image = og_image(rel, level)
|
|
head += [
|
|
{"tag": "meta", "attrs": {"property": "og:image", "content": image}},
|
|
{"tag": "meta", "attrs": {"property": "og:image:width", "content": "1200"}},
|
|
{"tag": "meta", "attrs": {"property": "og:image:height", "content": "630"}},
|
|
{"tag": "meta", "attrs": {"property": "og:image:alt", "content": f"RetroBIOS: {browser_title}"}},
|
|
{"tag": "meta", "attrs": {"name": "twitter:image", "content": image}},
|
|
{"tag": "script", "attrs": {"type": "application/ld+json"},
|
|
"content": json.dumps(structured, ensure_ascii=False, separators=(",", ":")).replace("</", "<\\/")},
|
|
]
|
|
return head
|
|
|
|
|
|
def split_title(markdown: str) -> tuple[str, str, str]:
|
|
"""The page's H1 as plain text, the id MkDocs gave it, and the body
|
|
without it."""
|
|
match = re.search(r"^# (.+?)[ \t]*$\n?", markdown, re.M)
|
|
if not match:
|
|
return "", "", markdown
|
|
explicit = re.search(r"\{[^}]*#([^\s}]+)[^}]*\}\s*$", match[1])
|
|
title = re.sub(r"\s*\{[^}]*\}\s*$", "", match[1])
|
|
ident = explicit[1] if explicit else toc_slug(title)
|
|
title = re.sub(r"<[^>]+>", "", title)
|
|
title = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", title)
|
|
title = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", title).replace("`", "")
|
|
return unescape(title).strip(), ident, markdown[: match.start()] + markdown[match.end():]
|
|
|
|
|
|
_CARD_NAME = re.compile(r'<h3[^>]*class="rb-file-title"(?![^>]*data-pagefind-ignore)[^>]*><code>([^<]+)</code>')
|
|
# What separates the parts of a file name: anything but a letter or a digit.
|
|
_NAME_PART = re.compile(r"[\W_]+")
|
|
|
|
|
|
def search_field(key: str, words: list[str]) -> str:
|
|
"""A search field of the page, held in an attribute of an empty
|
|
element: the text of an element is indexed and shown as the result's
|
|
excerpt (a run of 250 file names). Pagefind splits several fields at a
|
|
comma, so a value carries none."""
|
|
value = " ".join(word.replace(",", " ") for word in words).strip()
|
|
if not value:
|
|
return ""
|
|
return f'\n<span data-pagefind-meta="{html.escape(f"{key}:{value}", quote=True)}" hidden></span>\n'
|
|
|
|
|
|
def file_names_meta(body: str) -> str:
|
|
"""The names of the files a system page holds cards for, as a search
|
|
field of the page. A page of 250 cards is long, and the search weighs a
|
|
long page down: the Sony page, where scph5501.bin is downloaded, came
|
|
after every short page citing the name. The field is weighted in the
|
|
site's ranking (astro.config.mjs, metaWeights.files).
|
|
|
|
A name is written as its parts (scph5501 bin), each once. The search
|
|
matches a word as the start of a longer one and scores every distinct
|
|
word it reaches: whole names gave "ds" twelve rare words on the DSi page
|
|
(dsi_bios7.bin, DSi_Nand_USA.bin...), which then ranked first for
|
|
"nintendo ds bios"."""
|
|
names = sorted({unescape(name) for name in _CARD_NAME.findall(body)})
|
|
parts: dict[str, str] = {}
|
|
for name in names:
|
|
for part in _NAME_PART.split(name):
|
|
if part:
|
|
parts.setdefault(part.casefold(), part)
|
|
return search_field("files", list(parts.values()))
|
|
|
|
|
|
# systems/ page -> the platforms' own ids for the consoles it holds, built
|
|
# on the first system page, once the generator has installed the sections.
|
|
_page_aliases: dict[str, list[str]] = {}
|
|
_page_aliases_built = [False]
|
|
|
|
|
|
def page_aliases(rel: str) -> list[str]:
|
|
"""The names the platforms file the consoles of a system page under
|
|
(psx, dreamcast, gba), from siterender.system_search_aliases(): "psx
|
|
bios" finds the PlayStation page though the page never says psx."""
|
|
if not _page_aliases_built[0]:
|
|
import siterender
|
|
|
|
by_page: dict[str, set[str]] = {}
|
|
for sys_id in list(siterender._system_page_map):
|
|
section = siterender.system_section(sys_id)
|
|
names = siterender.system_search_aliases(sys_id)
|
|
if not section or not names:
|
|
continue
|
|
target = siterender.console_href(*section).partition("#")[0]
|
|
by_page.setdefault(urllib.parse.unquote(target), set()).update(names)
|
|
_page_aliases.clear()
|
|
_page_aliases.update({page: sorted(names, key=str.casefold)
|
|
for page, names in by_page.items()})
|
|
_page_aliases_built[0] = bool(siterender._system_page_map)
|
|
return list(_page_aliases.get(rel, []))
|
|
|
|
|
|
def field_words(value: object) -> list[str]:
|
|
"""The words of a front matter search field: a list, or one string
|
|
with its words or phrases split at commas."""
|
|
if isinstance(value, str):
|
|
return [part.strip() for part in value.split(",") if part.strip()]
|
|
if isinstance(value, (list, tuple)):
|
|
return [str(part).strip() for part in value if str(part).strip()]
|
|
return []
|
|
|
|
|
|
def console_words(names: list[str]) -> list[str]:
|
|
"""What a visitor types for a console: its name with "BIOS". A page that
|
|
answers "No file needed" says BIOS only where the search does not read,
|
|
so "nes bios" found NES.emu and never the NES page. As a task word it
|
|
weighs like one: in the consoles field, NES.emu's title still won."""
|
|
return [f"{name} BIOS" for name in names]
|
|
|
|
|
|
_CONSOLES_META = re.compile(r'data-pagefind-meta="consoles:([^"]*)"')
|
|
|
|
|
|
def page_console_words(authored: dict, text: str) -> set[str]:
|
|
"""The words of the consoles a page answers for: its `consoles` front
|
|
matter (*authored*) before decoration, its `consoles` field after."""
|
|
names = field_words(authored.get("consoles"))
|
|
if names:
|
|
return {word.casefold() for name in names for word in name.split()}
|
|
match = _CONSOLES_META.search(text)
|
|
return set(unescape(match[1]).casefold().split()) if match else set()
|
|
|
|
|
|
def covered_names(names: list[str], page: str, words_by_page: Mapping[str, set[str]]) -> list[str]:
|
|
"""The names of *names* whose every word another page's consoles also
|
|
hold, as a word or the start of one (PlayStation on the PlayStation 2
|
|
page, Nintendo DS on the DSi page, Nintendo Super System on the
|
|
Nintendo page). The search matches a word as the start of a longer one
|
|
and ignores the words a page has that the query lacks: it cannot tell
|
|
these pages apart by words, only by a weight the other page lacks."""
|
|
held = [words for other, words in words_by_page.items() if other != page]
|
|
return [name for name in names
|
|
if any(all(any(word.startswith(part) for word in words)
|
|
for part in name.casefold().split())
|
|
for words in held)]
|
|
|
|
|
|
def _neighbour(rel: str, value: object) -> object:
|
|
"""A `prev:` or `next:` the page states, its link resolved like any
|
|
other link of the page."""
|
|
if isinstance(value, dict) and value.get("link"):
|
|
# Starlight writes the link as given: a site path gets the base.
|
|
link = resolve(rel, str(value["link"]))
|
|
if link.startswith("/") and not link.startswith(BASE):
|
|
link = BASE + link.lstrip("/")
|
|
return {**value, "link": link}
|
|
return value
|
|
|
|
|
|
_NEXT_STEPS = re.compile(r"^## Next steps\b[^\n]*\n+(?P<list>(?:[ \t]*[-*] .*\n?)+)", re.M)
|
|
|
|
|
|
def next_step(body: str) -> dict | None:
|
|
"""The first link of the page's "Next steps" list, as the theme's
|
|
`next:` front matter."""
|
|
match = _NEXT_STEPS.search(body)
|
|
if not match:
|
|
return None
|
|
link = MD_LINK.search(match["list"])
|
|
if not link or link["bang"]:
|
|
return None
|
|
return {"link": link["url"], "label": re.sub(r"[*_`]", "", link["text"]).strip()}
|
|
|
|
|
|
def finalize_page(
|
|
rel: str,
|
|
markdown: str,
|
|
*,
|
|
level: str,
|
|
weight: float | None,
|
|
browser_title: str,
|
|
description: str,
|
|
structured: dict,
|
|
terms: dict[str, str],
|
|
search: bool = True,
|
|
logo: str = "",
|
|
parent: dict | None = None,
|
|
authored: dict | None = None,
|
|
words_by_page: Mapping[str, set[str]] | None = None,
|
|
) -> tuple[str, str]:
|
|
"""The Starlight file for a page body: its path under the content
|
|
collection and its text. *parent* ({label, link}, the link written
|
|
relative to the page) is the page a part page belongs to: the route
|
|
middleware makes it the last crumb and marks it in the sidebar.
|
|
|
|
*authored* is the front matter the page was written with. Its
|
|
`searchWeight`, `prev` and `next` win over the computed ones (a part
|
|
page states its neighbours and its weight), and its `consoles` and
|
|
`terms` become search fields (astro.config.mjs, metaWeights): the
|
|
consoles a manufacturer page holds sections for, and the words a page
|
|
answers that its text does not say, as an error message words them.
|
|
*words_by_page* holds the console words of every system page, by path
|
|
without extension: a console another page's words cover is written
|
|
once more, in `exact`."""
|
|
authored = authored or {}
|
|
title, title_id, body = split_title(markdown)
|
|
body = body.lstrip("\n")
|
|
body = rewrite_links(body, rel)
|
|
body = pin_heading_ids(body, title_id)
|
|
if title_id:
|
|
# The title is the theme's, with an id of its own: the one MkDocs
|
|
# gave it stays an anchor at the top of the text.
|
|
body = f'<div id="{html.escape(title_id)}" class="rb-anchor"></div>\n\n' + body
|
|
if rel not in ("wiki/glossary.md", "wiki/reference-terms.md"):
|
|
body = add_tooltips(body, terms)
|
|
# A badge is a status (required, missing, MD5), not a word a reader
|
|
# searches for: 141 "missing" badges put an emulator page above the
|
|
# troubleshooting page for "missing bios". The archive a file sits in
|
|
# is a name, and stays: it is how "neogeo.zip" finds the cores reading it.
|
|
body = re.sub(r'<span class="(rb-badge(?![^"]*rb-badge-archive)[^"]*)"',
|
|
r'<span data-pagefind-ignore class="\1"', body)
|
|
if rel.startswith("systems/"):
|
|
body += file_names_meta(body) + search_field("aliases", page_aliases(rel))
|
|
consoles = field_words(authored.get("consoles"))
|
|
body += search_field("consoles", consoles)
|
|
body += search_field("exact", covered_names(consoles, rel.rsplit(".", 1)[0], words_by_page or {}))
|
|
body += search_field("terms", field_words(authored.get("terms")) + console_words(consoles))
|
|
components = used_components(body)
|
|
stem = rel.rsplit(".", 1)[0]
|
|
if components:
|
|
body = mdx_safe(body)
|
|
imports: dict[str, list[str]] = {}
|
|
for name in components:
|
|
imports.setdefault(COMPONENTS[name], []).append(name)
|
|
# Our own components are imported from src/, two levels above the
|
|
# content collection plus the page's own folders.
|
|
up = "../" * (rel.count("/") + 2)
|
|
body = "".join(f"import {{ {', '.join(names)} }} from '{source}';\n"
|
|
if source.startswith("@") else f"import {names[0]} from '{up}{source}';\n"
|
|
for source, names in imports.items()) + "\n" + body
|
|
front: dict = {"title": title or browser_title, "description": description}
|
|
# Starlight's slugger drops dots and plus signs: the address stays MkDocs'.
|
|
if re.search(r"[^a-z0-9_/-]", stem):
|
|
front["slug"] = stem
|
|
if level:
|
|
front["level"] = level
|
|
if logo:
|
|
front["logo"] = logo
|
|
if parent:
|
|
front["parent"] = {"label": str(parent["label"]),
|
|
"link": resolve(rel, str(parent["link"]))}
|
|
weight = authored.get("searchWeight", weight)
|
|
if weight:
|
|
front["searchWeight"] = weight
|
|
if rel == NOT_FOUND:
|
|
front["tableOfContents"] = False
|
|
elif rel.startswith(CARD_PAGES):
|
|
sections = len(re.findall(r"^## ", body, re.M))
|
|
# A page of cards under one or two headings has nothing to list, and
|
|
# the table of contents watches every block of the page to follow
|
|
# the reader: 2 x 430 ms of a phone's main thread on 400 cards.
|
|
front["tableOfContents"] = (
|
|
{"minHeadingLevel": 2, "maxHeadingLevel": 2} if sections > 2 else False)
|
|
if not search:
|
|
front["pagefind"] = False
|
|
step = next_step(body)
|
|
if rel == NOT_FOUND or not reads_in_order(rel):
|
|
front["prev"] = False
|
|
front["next"] = False
|
|
elif step:
|
|
# The page's own "Next steps" list and the next link agree.
|
|
front["next"] = step
|
|
for side in ("prev", "next"):
|
|
if rel != NOT_FOUND and side in authored:
|
|
front[side] = _neighbour(rel, authored[side])
|
|
front["head"] = page_head(rel, level, front["title"], browser_title, structured)
|
|
if structured.get("dateModified"):
|
|
front["lastmod"] = str(structured["dateModified"])
|
|
return stem + (".mdx" if components else ".md"), _front_matter(front) + body
|
|
|
|
|
|
# Navigation
|
|
|
|
|
|
def _sidebar_link(rel: str) -> str:
|
|
return page_url(rel)[len(BASE) - 1:]
|
|
|
|
|
|
# Tabs whose groups hold pages by the hundred (every system, every
|
|
# emulator): a group that does not hold the current page is drawn as one
|
|
# link to its overview, so an emulator page does not carry the links of
|
|
# every other emulator.
|
|
PRUNED_TABS = ("explore",)
|
|
|
|
# Reading runs from one tab into the next only here: the last Start here
|
|
# page leads to Your setup. Every other tab is read on its own.
|
|
CHAINED_TABS = (("start-here", "your-setup"),)
|
|
|
|
|
|
def topic_id(label: str) -> str:
|
|
return re.sub(r"[^a-z]+", "-", label.lower()).strip("-")
|
|
|
|
|
|
def heading_ids(text: str) -> set[str]:
|
|
"""The anchors a page's headings carry, as written by pin_heading_ids."""
|
|
ids = {match[1] for match in re.finditer(r"\{[^{}\n]*#([^\s}]+)[^{}\n]*\}[ \t]*$", text, re.M)}
|
|
return ids | {match["id"] for match in ID_ATTR.finditer(text)}
|
|
|
|
|
|
def reads_in_order(rel: str) -> bool:
|
|
"""A page the previous and next links pass through. A system, emulator
|
|
or platform page is reached by its name, not read after its neighbour
|
|
in the alphabet: "SNK, Sony, Sord" is no reading order."""
|
|
return not rel.startswith(CARD_PAGES) or rel.endswith("/index.md")
|
|
|
|
|
|
_DIGIT_RANGE = re.compile(r"[0-9] to [0-9]")
|
|
|
|
|
|
def by_initial(label: str, children: list[dict]) -> bool:
|
|
"""A group that files its pages by the first character of their names
|
|
(A, B, "0 to 9"): a way through a long list, not a place, so it is no
|
|
crumb."""
|
|
if not children or any("items" in child for child in children):
|
|
return False
|
|
firsts = {str(child["label"])[:1].upper() for child in children}
|
|
if len(label) == 1:
|
|
return firsts == {label}
|
|
return bool(_DIGIT_RANGE.fullmatch(label)) and not any("A" <= first <= "Z" for first in firsts)
|
|
|
|
|
|
def sidebar(nav: list, present: set[str], anchors: dict[str, set[str]] | None = None) -> dict:
|
|
"""The navigation as Starlight topics, one per tab, with what the route
|
|
middleware (website/src/routeData.ts) reads beside it.
|
|
|
|
A page the navigation does not list (the parts of a split page) joins
|
|
the topic that holds most pages of its directory. In a directory the
|
|
navigation lists nothing of (get/), it joins the topic of the pages of
|
|
the same name, and a page with neither joins the first topic: no page
|
|
is left out of every tab.
|
|
|
|
A group that files names by their first letter is marked `crumb: false`:
|
|
the breadcrumbs skip it.
|
|
|
|
`groups` gives each group, keyed by the labels from its tab down, the
|
|
page that introduces it: its own overview page, or the heading named
|
|
after it on the overview above it (`anchors` lists each page's heading
|
|
ids). A group with neither has no address, never the address of its
|
|
first page, which is a sibling of the page being read. `prune` marks the
|
|
groups drawn as one link when they do not hold the current page.
|
|
|
|
`reading` is the order of the previous and next links, one list per run
|
|
of tabs read in sequence.
|
|
"""
|
|
anchors = anchors or {}
|
|
owner: dict[str, str] = {}
|
|
groups: dict[str, dict] = {}
|
|
|
|
def label_of(entry: object) -> str:
|
|
return next(iter(entry)) if isinstance(entry, dict) else ""
|
|
|
|
def overview_of(value: list) -> str | None:
|
|
return next((v for v in value if isinstance(v, str)), None)
|
|
|
|
def item(entry: object, topic: str, trail: tuple[str, ...], parent: str | None) -> dict | None:
|
|
if isinstance(entry, str):
|
|
owner[entry] = topic
|
|
return {"label": "Overview", "link": _sidebar_link(entry)} if entry in present else None
|
|
(label, value), = entry.items()
|
|
if isinstance(value, str):
|
|
if value.startswith("http"):
|
|
return {"label": label, "link": value}
|
|
owner[value] = topic
|
|
return {"label": label, "link": _sidebar_link(value)} if value in present else None
|
|
if "/" in label:
|
|
raise ValueError(f"a navigation group label holds a slash: {label!r}")
|
|
own = overview_of(value)
|
|
here = (*trail, label)
|
|
href = None
|
|
if own and own in present:
|
|
href = page_url(own)
|
|
elif parent and parent in present:
|
|
anchor = toc_slug(label)
|
|
if anchor in anchors.get(parent, set()):
|
|
href = f"{page_url(parent)}#{anchor}"
|
|
elif topic in PRUNED_TABS:
|
|
# Drawn as a link, it needs an address: the overview it
|
|
# sits under, never one of its own pages.
|
|
href = page_url(parent)
|
|
children = [child for child in (item(v, topic, here, own or parent) for v in value) if child]
|
|
if not children:
|
|
return None
|
|
groups["/".join(here)] = {"href": href, "prune": topic in PRUNED_TABS and href is not None,
|
|
"crumb": not by_initial(label, children)}
|
|
return {"label": label, "items": children, "collapsed": True}
|
|
|
|
topics = []
|
|
for tab in nav:
|
|
(label, value), = tab.items()
|
|
topic = topic_id(label)
|
|
children = [child for child in (item(v, topic, (label,), overview_of(value)) for v in value)
|
|
if child]
|
|
if children:
|
|
topics.append({"id": topic, "label": label, "link": next(_links(children))["link"],
|
|
"items": children})
|
|
|
|
votes: dict[str, dict[str, int]] = {}
|
|
for rel, topic in owner.items():
|
|
tally = votes.setdefault(rel.rpartition("/")[0], {})
|
|
tally[topic] = tally.get(topic, 0) + 1
|
|
# The page a stale link lands on opens on the first tab: its sidebar
|
|
# is the way back in. Excluded from every tab, the theme named the tabs
|
|
# by their position, 0 to 4.
|
|
extra: dict[str, list[str]] = {topics[0]["id"]: ["/404"]} if topics else {}
|
|
# A folder the navigation lists nothing of (get/, one download page per
|
|
# setup) has no vote, and excluded from every tab its pages had the
|
|
# tabs numbered 0 to 4 too. Its pages take the tab of the pages of the
|
|
# same name (get/batocera.md, platforms/batocera.md), and its overview
|
|
# the tab its pages took.
|
|
by_name: dict[str, dict[str, int]] = {}
|
|
for rel, topic in owner.items():
|
|
tally = by_name.setdefault(PurePosixPath(rel).name, {})
|
|
tally[topic] = tally.get(topic, 0) + 1
|
|
unlisted = sorted(present - owner.keys())
|
|
adopted: dict[str, dict[str, int]] = {}
|
|
for rel in unlisted:
|
|
folder, name = rel.rpartition("/")[0], PurePosixPath(rel).name
|
|
if folder in votes or name == "index.md":
|
|
continue
|
|
tally = adopted.setdefault(folder, {})
|
|
for topic, count in by_name.get(name, {}).items():
|
|
tally[topic] = tally.get(topic, 0) + count
|
|
for rel in unlisted:
|
|
folder = rel.rpartition("/")[0]
|
|
tally = votes.get(folder) or adopted.get(folder)
|
|
if tally:
|
|
topic = max(tally, key=tally.get)
|
|
elif topics:
|
|
topic = topics[0]["id"]
|
|
else:
|
|
continue
|
|
extra.setdefault(topic, []).append("/" + (_sidebar_link(rel).strip("/") or "index"))
|
|
return {"topics": topics, "options": {"topics": extra, "exclude": []},
|
|
"groups": groups, "reading": _reading(nav, present)}
|
|
|
|
|
|
def _reading(nav: list, present: set[str]) -> list[list[dict]]:
|
|
"""The previous and next links, in runs. An overview is named after the
|
|
group or tab it opens ("Your setup", not "Overview")."""
|
|
def walk(value: list, name: str) -> Iterator[dict]:
|
|
for entry in value:
|
|
if isinstance(entry, str):
|
|
if entry in present and reads_in_order(entry):
|
|
yield {"label": name, "link": page_url(entry)}
|
|
continue
|
|
(label, target), = entry.items()
|
|
if isinstance(target, list):
|
|
yield from walk(target, label)
|
|
elif not target.startswith("http") and target in present and reads_in_order(target):
|
|
yield {"label": label, "link": page_url(target)}
|
|
|
|
tabs = {topic_id(label): list(walk(value, label))
|
|
for tab in nav for label, value in tab.items()}
|
|
runs: list[list[dict]] = []
|
|
previous = ""
|
|
for tab, pages in tabs.items():
|
|
if runs and (previous, tab) in CHAINED_TABS:
|
|
runs[-1].extend(pages)
|
|
else:
|
|
runs.append(list(pages))
|
|
previous = tab
|
|
return [run for run in runs if run]
|
|
|
|
|
|
# Addresses that once held a page, and where its content went.
|
|
MOVED_PAGES = {"wiki/getting-started.md": "which-pack.md"}
|
|
|
|
_CORE_NAME = re.compile(r"[a-z0-9_-]+")
|
|
|
|
|
|
def redirects(profiles: dict, present: set[str]) -> dict[str, str]:
|
|
"""Old and alternative addresses, for src/generated/redirects.json.
|
|
|
|
A page that moved, once its source is gone, and each name a profile's
|
|
`cores:` gives the emulator (the buildbot's `mednafen_psx` for
|
|
beetle_psx), to the profile's page. A name two profiles claim, or one
|
|
that is already a page's address, is left out. Addresses are written
|
|
under the site root, without its base."""
|
|
def address(rel: str) -> str:
|
|
return "/" + page_url(rel)[len(BASE):]
|
|
|
|
out = {address(old): address(new) for old, new in MOVED_PAGES.items()
|
|
if old not in present and new in present}
|
|
pages = {rel.casefold() for rel in present}
|
|
claims: dict[str, set[str]] = {}
|
|
for key, profile in profiles.items():
|
|
if f"emulators/{key}.md" not in present:
|
|
continue
|
|
for name in profile.get("cores") or []:
|
|
name = str(name)
|
|
if _CORE_NAME.fullmatch(name) and name != key:
|
|
claims.setdefault(name, set()).add(key)
|
|
for name, keys in sorted(claims.items()):
|
|
if len(keys) == 1 and f"emulators/{name}.md".casefold() not in pages:
|
|
out[address(f"emulators/{name}.md")] = address(f"emulators/{next(iter(keys))}.md")
|
|
return out
|
|
|
|
|
|
def _links(items: list[dict]) -> Iterator[dict]:
|
|
for entry in items:
|
|
if "link" in entry:
|
|
yield entry
|
|
else:
|
|
yield from _links(entry["items"])
|
|
|
|
|
|
# The picker's data
|
|
|
|
|
|
def picker_data(install: Path, out: Path, titles: dict[str, str]) -> int:
|
|
"""Each manifest, compacted to what the file picker reads: per file its
|
|
destination, size and systems, the systems install.py --system accepts.
|
|
`titles` names a system id the way its system page does.
|
|
"""
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
index = []
|
|
for manifest in sorted(install.glob("*.json")):
|
|
data = json.loads(manifest.read_text(encoding="utf-8"))
|
|
if not isinstance(data, dict) or "files" not in data or "platform" not in data:
|
|
continue
|
|
systems = sorted({s for f in data["files"] for s in f.get("systems") or []})
|
|
position = {s: k for k, s in enumerate(systems)}
|
|
names = data.get("system_names") or {}
|
|
payload = {
|
|
"platform": data["platform"],
|
|
"name": data.get("display_name", data["platform"]),
|
|
"folder": (data.get("detect") or [{}])[0].get("bios_path", ""),
|
|
"generated": data.get("generated", ""),
|
|
"systems": [{"id": s, "title": titles.get(s, ""), "native": names.get(s, [])}
|
|
for s in systems],
|
|
"other": any(not f.get("systems") for f in data["files"]),
|
|
"files": [[f["dest"], f.get("size", 0), [position[s] for s in f.get("systems") or []]]
|
|
for f in sorted(data["files"], key=lambda f: f["dest"])],
|
|
}
|
|
(out / f"{data['platform']}.json").write_text(
|
|
json.dumps(payload, separators=(",", ":")), encoding="utf-8")
|
|
index.append({"platform": data["platform"], "name": payload["name"],
|
|
"files": len(payload["files"])})
|
|
index.sort(key=lambda entry: entry["name"].casefold())
|
|
(out / "index.json").write_text(json.dumps(index, separators=(",", ":")), encoding="utf-8")
|
|
return len(index)
|
|
|
|
|
|
def quote_url(url: str) -> str:
|
|
return urllib.parse.quote(url, safe="/:#?=&%")
|