#!/usr/bin/env python3 """Generate the site from database.json, platform configs, and emulator profiles. The pages, navigation and data files are written into the Astro Starlight project in website/, which builds the site. Reads the same data sources as verify.py and generate_pack.py to produce a complete documentation site. Zero manual content. Usage: python scripts/generate_site.py python scripts/generate_site.py --db database.json --platforms-dir platforms """ from __future__ import annotations from siterender import SECTION_LABELS, WIKI_SRC_DIR import argparse import csv import hashlib import functools import html import io import json import os import re import shlex import shutil import sqlite3 import sys import urllib.error import urllib.parse import urllib.request import zipfile from collections import Counter from collections.abc import Callable, Iterator from datetime import datetime, timezone from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from common import ( upstream_profile_index, build_zip_contents_index, GAME_DATA_TOPS, compute_composition, count_catalog_matched, list_registered_platforms, list_release_pack_platforms, load_platform_registry, load_database, load_emulator_profiles, load_platform_config, load_provenance_snapshots, native_layer, require_yaml, resolution_is_hash_exact, resolve_local_file, resolve_platform_cores, runs_standalone, unique_emulator_profiles, yaml_load, ) from artifacts import ArtifactLockBusy, artifact_lock from artifacts import json_text as _json_text from cross_reference import FileConsumers, held_copy from nativemode import digest_algorithm, reads_file_contents yaml = require_yaml() from generate_readme import ( compute_coverage, platform_coverages, NO_PACK, asset_links, contributing_text, download_links, extract_notes, extract_targets, release_totals, ) from provenance_report import build_report, distinct_missing import sitemarkup as markup import siteentries import sitestarlight from siteexports import generate_exports_page, load_export_reports from siteguides import ( ALSO_RUNS, DEVICE_GROUPS, GROUP_ANCHORS, GROUP_TILES, GUIDES, HANDHELD_GUIDE, Guide, HandheldGuide, HandheldSystem, FENCE_TITLES, INSTALL_BASE, INSTALL_PS1, INSTALL_SH, TERMINALS, TRUST_ITEMS, detected_systems as _detected_systems, fence, folder_list, install_command, setup_count, platform_args, terminal_tabs, trust_box, ) import release_record import sitebootstrap import sitemeasurements # The Starlight project, and where its pages go inside it. STARLIGHT_DIR = "website" STARLIGHT_CONTENT = "src/content/docs" SITE_NAME = "RetroBIOS" REPO_URL = "https://github.com/Abdess/retrobios" RELEASE_URL = f"{REPO_URL}/releases/latest" # What LICENSE grants over the published data: its MIT terms name the # metadata. The BIOS files themselves are under NOTICE, never this. DATA_LICENSE = "MIT (LICENSE: scripts, tooling, schemas, documentation and metadata)" SITE_URL = "https://abdess.github.io/retrobios/" GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads", "files", release_record.GET_DIR] # Generated directories of files served as they are, not pages. PUBLIC_DIRS = ("api", "downloads") def _content_check_ceiling(profiles: dict) -> str: """How far content checking can reach across every profiled entry.""" hashed = sized = neither = 0 for profile in unique_emulator_profiles(profiles).values(): for f in profile.get("files", []) or []: if any( f.get(k) for k in ("md5", "sha1", "crc32", "sha256", "known_hash_adler32") ): hashed += 1 elif any(f.get(k) for k in ("size", "min_size", "max_size")): sized += 1 else: neither += 1 total = hashed + sized + neither if not total: return "" return ( f"Across every profiled entry, {hashed:,} of {total:,} " f"({hashed / total * 100:.0f}%) carry a hash the code checks, " f"{sized:,} ({sized / total * 100:.0f}%) only a size, and " f"{neither:,} ({neither / total * 100:.0f}%) neither. That last share " "is the ceiling of the method: where an emulator validates nothing, " "reading its source establishes which file it loads, never whether " "the content is the right dump. The provenance field answers that " "other question, for the files a dump catalog indexes." ) CLS_LABELS = { "official_port": "Official ports", "community_fork": "Community forks", "pure_libretro": "Pure libretro", "game_engine": "Game engines", "enhanced_fork": "Enhanced forks", "frozen_snapshot": "Frozen snapshots", "embedded_hle": "Embedded HLE", "launcher": "Launchers", "unclassified": "Unclassified", "other": "Other", } # The same classes said of one emulator: its page read "Classification: # Official ports". CLS_LABEL_ONE = { "official_port": "Official port", "community_fork": "Community fork", "pure_libretro": "Pure libretro", "game_engine": "Game engine", "enhanced_fork": "Enhanced fork", "frozen_snapshot": "Frozen snapshot", "embedded_hle": "Embedded HLE", "launcher": "Launcher", "unclassified": "Unclassified", "other": "Other", } def _copy_places(db: dict) -> dict[str, set[tuple[str, str, str]]]: """sha1 -> every (manufacturer, console, file name) holding a copy. The database keeps one path per content, deduplication having left one copy. by_path_suffix still records the other folders a copy is filed in, under the name that folder gives it. """ files = db.get("files", {}) bios_dir = Path(str(db.get("bios_dir") or "bios")) places: dict[str, set[tuple[str, str, str]]] = {} for sha1, entry in files.items(): parts = str(entry.get("path", "")).split("/") if len(parts) >= 3 and parts[0] == "bios": places.setdefault(sha1, set()).add((parts[1], parts[2], parts[-1])) for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items(): parts = tail.split("/") if len(parts) < 3 or parts[1].startswith(".") or len(sha1s) != 1: continue if sha1s[0] in files and (bios_dir / tail).is_file(): places.setdefault(sha1s[0], set()).add((parts[0], parts[1], parts[-1])) return places # bios/Other/ and bios/Arcade/ file a copy by # who loads it, not by machine. A copy also filed under a machine's maker # places the system there. _BUCKET_FOLDERS = frozenset({"Other", "Arcade"}) def _slug(manufacturer: str) -> str: return manufacturer.lower().replace(" ", "-") def _names_manufacturer(sys_id: str, manufacturer: str) -> bool: slug = _slug(manufacturer) return sys_id == slug or sys_id.startswith(f"{slug}-") def _copies_by_path(db: dict) -> dict[str, set[tuple[str, str, str]]]: """The copy places of each content, keyed by the path the database keeps.""" files = db.get("files", {}) return { str(files[sha1].get("path", "")): where for sha1, where in _copy_places(db).items() } def _system_folder( files: list[dict], db: dict, name_consoles: dict[str, set[tuple[str, str]]], sys_id: str = "", copies: dict[str, set[tuple[str, str, str]]] | None = None, ) -> tuple[str, str] | None: """The bios// folder a system's files live in. A content votes for every folder holding a copy under the name the system declares, not only the one deduplication kept: disksys.rom is kept as Arcade/FBNeo/fdsbios.nes and filed again under Nintendo/Famicom Disk System. A copy filed under another name, outside the manufacturer the id names, is another machine's file and votes only when no copy carries the declared name. An id that names its manufacturer (nintendo-fds) keeps the votes cast inside that manufacturer when there are any, and a maker's folder outranks the folders that file copies by emulator. """ if copies is None: copies = _copies_by_path(db) votes: dict[tuple[str, str], int] = {} for fe in files: local, status = resolve_local_file(fe, db) if not local or not resolution_is_hash_exact(status): continue held = copies.get(local, set()) if not held: parts = local.split("/") if len(parts) >= 4 and parts[0] == "bios": held = {(parts[1], parts[2], parts[-1])} declared = str(fe.get("name", "")).casefold() named = { (m, c) for m, c, base in held if base.casefold() == declared or _names_manufacturer(sys_id, m) } for place in named or {(m, c) for m, c, _ in held}: votes[place] = votes.get(place, 0) + 1 if votes: own = [place for place in votes if _names_manufacturer(sys_id, place[0])] makers = [place for place in votes if place[0] not in _BUCKET_FOLDERS] candidates = own or makers or list(votes) return max(sorted(candidates), key=lambda place: votes[place]) for fe in files: places = name_consoles.get(fe.get("name", ""), set()) if len(places) == 1: return next(iter(places)) return None def _folders_by_name(manufacturers: dict) -> dict[str, set[tuple[str, str]]]: """Normalized system name -> the console folders that spell it. nintendo-nes, nes and the NES folder normalize alike, as do sega-mastersystem and Sega/Master System. """ from common import _norm_system_id index: dict[str, set[tuple[str, str]]] = {} for mfr, consoles in manufacturers.items(): for console in consoles: spelled = console.lower().replace(" ", "-") for key in { _norm_system_id(f"{_slug(mfr)}-{spelled}"), _norm_system_id(spelled) }: index.setdefault(key, set()).add((mfr, console)) return index def _folder_named(sys_id: str, by_name: dict[str, set[tuple[str, str]]]) -> tuple[str, str] | None: from common import _norm_system_id places = by_name.get(_norm_system_id(sys_id), set()) return next(iter(places)) if len(places) == 1 else None def _system_pages_from_data( manufacturers: dict, coverages: dict, db: dict, profiles: dict | None = None, ) -> dict[str, tuple[str, str]]: """The system_id -> (manufacturer_slug, console_name) table. Uses platform file paths to trace system_id -> bios directory -> manufacturer page. """ # Every console folder a filename sits in. A name in one folder only can # place a system, while a name in several says nothing (MiSTer's boot.rom lives # under a dozen consoles, and the last one written used to win). name_consoles: dict[str, set[tuple[str, str]]] = {} for mfr, consoles in manufacturers.items(): for console, entries in consoles.items(): for entry in entries: name_consoles.setdefault(entry["name"], set()).add((mfr, console)) # Build normalized console name index for fuzzy matching console_norm: dict[str, tuple[str, str]] = {} for mfr, consoles in manufacturers.items(): slug = mfr.lower().replace(" ", "-") mfr_norm = mfr.lower().replace(" ", "-") for console in consoles: norm = console.lower().replace(" ", "-") entry = (slug, console) console_norm[norm] = entry console_norm[f"{mfr_norm}-{norm}"] = entry # Short aliases: strip common manufacturer prefix words for prefix in ( f"{mfr_norm}-", "nintendo-", "sega-", "sony-", "snk-", "nec-", ): if norm.startswith(prefix.replace(f"{mfr_norm}-", "")): pass # already covered by norm key = f"{prefix}{norm}" console_norm[key] = entry # Map system_id -> (manufacturer, console) via platform file entries copies = _copies_by_path(db) folders_by_name = _folders_by_name(manufacturers) pages: dict[str, tuple[str, str]] = {} for cov in coverages.values(): config = cov["config"] for sys_id, system in config.get("systems", {}).items(): if sys_id in pages: continue # Strategy 1: the one console folder the id spells (nintendo-nes, # NES). A copy of the disk system BIOS in the NES list does not # move the NES to the folder that copy sits in. # Strategy 2: the folder holding the files the system declares, # found by content, and most votes win. A bare name stands in only # where the collection files it in a single console. place = _folder_named(sys_id, folders_by_name) or _system_folder( system.get("files", []), db, name_consoles, sys_id, copies ) if place is not None: mfr, console = place pages[sys_id] = (mfr.lower().replace(" ", "-"), console) continue # Strategy 3: fuzzy match system_id against console directory names if sys_id in console_norm: pages[sys_id] = console_norm[sys_id] else: # Try partial match: "nintendo-wii" matches "Wii" under "Nintendo" parts = sys_id.split("-") for i in range(len(parts)): suffix = "-".join(parts[i:]) if suffix in console_norm: pages[sys_id] = console_norm[suffix] break # Systems only emulator profiles name (sega-naomi, commodore-pet): placed # by the files the profile files under them, as a platform's are. A # profile of one system files all of its entries there. for profile in (profiles or {}).values(): systems = [str(sid) for sid in profile.get("systems") or []] for sys_id in systems: if sys_id in pages: continue files = [ entry for entry in profile.get("files") or [] if isinstance(entry, dict) and (entry.get("system") == sys_id or len(systems) == 1) ] place = _folder_named(sys_id, folders_by_name) or _system_folder( files, db, name_consoles, sys_id, copies ) if place is not None: mfr, console = place pages[sys_id] = (mfr.lower().replace(" ", "-"), console) elif sys_id in console_norm: pages[sys_id] = console_norm[sys_id] return pages def _timestamp() -> str: """The build time for data files (stats, API records), ISO 8601.""" return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def data_as_of(moment: datetime | None = None) -> str: """The build date for a reader, in words, with the machine form beside it. A raw ISO stamp ("2026-10-22T22:43:08Z") closed the home page.""" moment = moment or datetime.now(timezone.utc) return (f'") def _date_line() -> str: """The date a page closes on, in the words of data_as_of().""" return f'
{data_as_of()}.
' # Home page def _stat(value: str, label: str) -> list[str]: return [ markup.block_open("div", 'class="rb-stat"'), f'{value}', f'{label}', "", "", ] def _group_members(coverages: dict, registry: dict, group: str) -> list[str]: """Registered platforms of a device group, by display name.""" return sorted( (name for name in coverages if registry.get(name, {}).get("device_group") == group), key=lambda name: coverages[name]["platform"].casefold(), ) def _next_block(links: list[str]) -> list[str]: """Where to go from here, at the end of a page: a plain heading and a list. A tinted callout with a side stripe said the same as the pager.""" return ["## Next steps", "", *(f"- {link}" for link in links), ""] def _tiles(registry: dict, members: list[str], display: dict[str, str], prefix: str = "", group: str = "", eager: int = 0) -> list[str]: """The setup tiles of one device group: logo, name and the hardware it runs on, then the group's other tiles: a kind of device that runs one of the setups (Android handhelds, Linux handhelds).""" return markup.platform_tiles([ *((display[name], f"{prefix}platforms/{name}.md", str(registry.get(name, {}).get("logo") or ""), GUIDES[name].runs_on or _runs_on(registry.get(name, {}), GUIDES[name])) for name in members), *((tile.name, f"{prefix}{tile.link}", "", tile.text) for tile in GROUP_TILES.get(group, ())), ], eager=eager) def setup_picker(registry: dict, display: dict[str, str], prefix: str = "") -> list[str]: """Every setup as a tile, under the device group it belongs to. The setups alone in their group share one row: "MiSTer FPGA" above "MiSTer FPGA" said nothing. Only the first row of logos loads with the page: the 19 logos were most of the bytes of the home page.""" lines: list[str] = [] alone: list[str] = [] for group, (label, _description) in DEVICE_GROUPS.items(): members = _group_keys(registry, group, display) if len(members) == 1 and not GROUP_TILES.get(group): alone += members elif members: eager = 0 if lines else markup.FIRST_ROW_TILES lines += [f"**{label}**", "", *_tiles(registry, members, display, prefix, group, eager)] if alone: eager = 0 if lines else markup.FIRST_ROW_TILES lines += ["**Other setups**", "", *_tiles(registry, alone, display, prefix, eager=eager)] return lines def generate_home( db: dict, coverages: dict, profiles: dict, registry: dict | None = None, platforms_dir: str = "platforms", ) -> str: """The page someone lands on knowing nothing: what a BIOS file is, whether they need one, and the three steps to get the right ones. The figures about the collection, the cards to the deeper pages and the call for help sat below the steps and doubled the page: they are on the Explore index, one tab further. No date and no "next" box: the hero buttons say where to go.""" registry = registry or load_platform_registry(platforms_dir) display = {key: load_platform_config(key, platforms_dir).get("platform", key) for key in registry} named = [n for n in ("RetroArch", "Batocera", "EmuDeck") if n in display.values()] others = setup_count() - len(named) works_with = ", ".join(named) + (f" and {others} more" if others else "") lines = [ "# BIOS files for your emulators", "", "Some consoles need a small program to start their games. It is called " "the **BIOS**. Your emulator needs a copy of it. We put the right copy " "in the right folder.", "", markup.button("Get your files", "which-pack.md", primary=True) + " " + markup.button("What is a BIOS?", "wiki/what-are-bios-files.md"), "", f"Free. No account. Works with {works_with}.", "", "- BIOS missing, even with the file there? " "[When a game does not start](wiki/troubleshooting.md)", "- Have a file? [Check it](files/index.md)", "", "## Do I need it?", "", "- **Yes**, if your emulator says a BIOS is missing.", "- **Yes** for PlayStation 1 and 2, Saturn and Neo Geo.", "- **Better with it** for Dreamcast: some games need it.", "- **Usually not** for NES, Super Nintendo, Mega Drive and Game Boy.", "", "Details: [Every console, one by one](systems/index.md)", "", "## Three steps", "", *markup.steps([ "**Pick your device.** Its page shows your BIOS folder.", "**Run one command.** Some setups have a ZIP.", "**Start a game.**", ]), "## Pick your device", "", *setup_picker(registry, display), "## Is it safe?", "", *(f"- {item}" for item in TRUST_ITEMS), "", "[How the command is checked](wiki/faq.md#is-piping-the-installer-into-a-shell-safe). " "Details: [Is it legal?](wiki/legal.md)", "", ] return "\n".join(lines) + "\n" def generate_explore_index(db: dict, coverages: dict, profiles: dict) -> str: """The Explore tab opens here: where to look for one console, one emulator or one installer option, and the figures of the collection.""" total_files = db.get("total_files", 0) total_size = db.get("total_size", 0) emulator_count = len(unique_emulator_profiles(profiles)) lines = [ "# Explore", "", "One console, one emulator, or one installer option: each has its page, " "with the files it needs and the code that reads them.", "", *markup.card_grid([ ("Systems", ["What each console and computer needs, file by file.", "", "[Browse the systems](systems/index.md)"]), ("Emulators", ["What each emulator reads, with the source code that reads it.", "", "[Browse the emulators](emulators/index.md)"]), ("Installer options", [ "Fewer files, one region, another folder, a check without download.", "", "[All installer options](wiki/installer.md)"]), ("How files are checked", [ "Each setup checks its files its own way: by name, by MD5 or by SHA1.", "", "[How each platform checks files](wiki/verification-modes.md)"]), ("Data and API", ["Every file and hash as JSON, CSV and SQLite.", "", "[Data and API](data.md)"]), ]), "## The collection", "", markup.block_open("div", 'class="rb-stats"'), "", *_stat(f"{total_files:,}", COLLECTION_FILES_LABEL), *_stat(str(setup_count()), "Setups supported"), *_stat(str(emulator_count), "Emulators documented"), *_stat(_fmt_size(total_size), "Size of the collection"), "", "", composition_sentence(db), "", "## Help the project", "", "A file you hold, an emulator profile or a correction fills a gap for " "every setup at once. For contributors: [How to contribute](contributing.md).", "", ] return "\n".join(lines) + "\n" def compute_stats(db: dict, coverages: dict, profiles: dict) -> dict: unique = unique_emulator_profiles(profiles) systems: set[str] = set() for p in unique.values(): systems.update(p.get("systems", [])) return { "schema_version": 1, "generated_at": _timestamp(), "files": db.get("total_files", 0), "size_bytes": db.get("total_size", 0), "platforms": len(coverages), "emulators": len(unique), "systems": len(systems), "catalog_matched": count_catalog_matched(db), "source": REPO_URL, "downloads": RELEASE_URL, } def _release_date(tag: str) -> str: """The date a release tag names (v2026.10.10), in words, or empty.""" match = re.fullmatch(r"v(\d{4})\.(\d{2})\.(\d{2})", tag or "") if not match: return "" moment = datetime(int(match[1]), int(match[2]), int(match[3])) return f"{moment.day} {moment.strftime('%B %Y')}" def _pack_version(pack_name: str) -> str: """The platform version a pack was built for, from its name (Batocera_43.1_BIOS_Pack.zip is 43.1).""" stem = pack_name.removesuffix("_BIOS_Pack.zip") return stem.rsplit("_", 1)[1] if "_" in stem else "" def release_line(platform_name: str, record: dict) -> list[str]: """Which release the buttons serve, with its checksums and signature, and the version gap when the page describes a newer platform than the pack.""" tag = record.get("tag") pack_name = release_record.pack_name_for(platform_name, record) if not tag or not pack_name: return [] base = f"{REPO_URL}/releases/download/{tag}" built = _release_date(tag) lines = [ f"Pack {tag}{f', built {built}' if built else ''}. " f"[SHA256SUMS.txt]({base}/SHA256SUMS.txt) and its " f"[signature]({base}/SHA256SUMS.txt.sig).", ] config = load_platform_config(platform_name, "platforms") page_version = str(config.get("version") or "") pack_version = _pack_version(pack_name) if page_version and pack_version and page_version.lstrip("v") != pack_version.lstrip("v"): display = config.get("platform", platform_name) lines.append(f"This pack was built for {display} {pack_version}. The command " f"installs the files for {display} {page_version}.") return lines # What a reader on another version of the setup does with the pack. Our data # says which list the pack was made from, not what an older list lacks. OTHER_VERSION = "Other version? Run `--check` after copying." def setup_version(platform_name: str, record: dict | None = None) -> str: """The setup and version whose BIOS list the pack was made from ("Batocera 43.1"): the version in the pack's name, else the version the platform YAML was read at. Empty when neither says, or when the platform has no YAML to read a version from.""" try: config = load_platform_config(platform_name, "platforms") except FileNotFoundError: return "" pack_name = release_record.pack_name_for(platform_name, record or {}) version = (_pack_version(pack_name) if pack_name else "") or str(config.get("version") or "") if not version: return "" return f"{config.get('platform', platform_name)} {version.removeprefix('v')}" def version_line(platform_name: str, record: dict, named: bool = False) -> str: """The version the pack fits, and what to do on another. *named* when the release line already named the pack's version.""" made_from = setup_version(platform_name, record) if named or not made_from: return OTHER_VERSION return f"Made for {made_from}. {OTHER_VERSION}" # Why a setup has no ZIP, answered once on the FAQ. NO_ZIP_FAQ = "wiki/faq.md#why-is-there-no-zip-for-my-setup" def has_release_pack(platform_name: str) -> bool: """Whether a release publishes a ZIP for the platform (`release_pack`).""" entry = load_platform_registry("platforms").get(platform_name) or {} return entry.get("release_pack") is True def no_zip_line(display: str) -> str: """The one line a setup without a release pack shows where the ZIP would be.""" return f"No ZIP for {display}. The command installs every file and checks it." def get_page_link(platform_name: str, prefix: str = "") -> str: """The platform's permanent download page, as a site link.""" return f"{prefix}{release_record.GET_DIR}/{platform_name}.md" def get_page_title(platform_name: str, display: str) -> str: """What a get/ page holds, by its name: the ZIP of a setup a release publishes one for, the files of any other. "Download KNULLI" read as the system's own download. A setup that is also an emulator (BizHawk) names it last: "BizHawk BIOS files" is the title of its emulator page.""" kind = "BIOS pack" if has_release_pack(platform_name) else "BIOS files" if (_REPO_ROOT / "emulators" / f"{platform_name}.yml").is_file(): return f"{kind} for {display}" return f"{display} {kind}" def _unzip_into(platform_name: str, display: str, extract_to: str | None, parts: int) -> list[str]: """Where the ZIP is unzipped, on the get/ page. A folder the registry writes relative to the setup's own (`system/`, `bios/`) is named as the setup's, with the way to find it.""" what = "them all" if parts > 1 else "it" if not _setup_relative(extract_to): return [f"- Unzip {what} into {_folder_words(display, extract_to)}.", ""] lead = GUIDES[platform_name].lead if platform_name in GUIDES else "" return [ f"- Unzip {what} into {_folder_words(display, extract_to)}. " f"[Where is it?](../platforms/{platform_name}.md#where-the-files-go)", "", *([lead, ""] if lead else []), ] def _setup_relative(extract_to: str | None) -> bool: """Whether the registry writes the folder relative to the setup's own (`system/`, `Firmware/`), not as a path the device reads.""" return bool(extract_to) and not extract_to.startswith(("/", "~", "<")) def _folder_words(display: str, extract_to: str | None) -> str: """The folder a pack or a copy goes into, in words: the device's path, or a folder of the setup named as the setup's.""" if not extract_to: return "your BIOS folder" if _setup_relative(extract_to): return f"{display}'s `{extract_to.rstrip('/')}` folder" return f"`{extract_to}`" def _pack_buttons(platform_name: str, label: str = "Pack", prefix: str = "../", display: str | None = None) -> str: """The way to the platform's ZIP from its setup page: a button to its get/ page, the release it comes from, and the setup version its list was read at. "Does this pack fit my version?" is the first question a frontend user asks. A setup with no release pack gets the line that says so, and why.""" if not has_release_pack(platform_name): display = display or load_platform_config(platform_name, "platforms").get( "platform", platform_name) return f"{no_zip_line(display)} [Why no ZIP?]({prefix}{NO_ZIP_FAQ})" record = release_record.load_advertised() buttons = download_links( platform_name, record, label, button=lambda text, url: markup.button(text, url, primary=True), page=get_page_link(platform_name, prefix), ) released = release_line(platform_name, record) return "\n\n".join([buttons, *released, version_line(platform_name, record, named=len(released) > 1)]) def generate_get_page(platform_name: str) -> str: """get//: the address a link outside the site can keep. Every release deletes the previous tag, and with it each asset URL: six of the eight links a wiki had to our packs went dead. This page lists the files of the pack the current release holds, read from release.json, or the command when no release publishes one. The release process never touches it: committing release.json rebuilds it.""" config = load_platform_config(platform_name, "platforms") display = config.get("platform", platform_name) record = release_record.load_advertised() pack = release_record.pack_for(platform_name, record) released = has_release_pack(platform_name) title = get_page_title(platform_name, display) if released and pack and pack.get("assets"): description = (f"The BIOS pack for {display}: {_fmt_size(pack['download_size'])} to " "download. The address stays the same and always gives the latest ZIP.") else: description = f"The BIOS files for {display}: one command installs and checks them." lines = [ "---", "search: exclude", f"description: {json.dumps(description)}", *_parent_front_matter((display, f"/platforms/{platform_name}/")), "---", "", f"# {title}", "", ] if released and pack and pack.get("assets"): parts = len(pack["assets"]) release = release_line(platform_name, record) extract_to = (load_platform_registry("platforms").get(platform_name) or {}).get("extract_to") lines += [ "This address stays the same. It always gives the latest ZIP.", "", asset_links(platform_name, record, REPO_URL, f"Download {display} Pack", button=lambda text, url: markup.button(text, url, primary=True)), "", f"- {_fmt_size(pack['download_size'])} to download, " f"{_fmt_size(pack['extracted_size'])} once unzipped.", *(["- You need both parts." if parts == 2 else f"- You need all {parts} parts."] if parts > 1 else []), *_unzip_into(platform_name, display, extract_to, parts), *(line for text in release for line in (text, "")), version_line(platform_name, record, named=len(release) > 1), "", ] elif released and record.get("tag"): lines += [ f"No {display} ZIP yet. The next release has one. " "The command installs every file and checks it.", "", *_setup_routes(platform_name, display), ] elif released: lines += [ "This address stays the same. It always gives the latest ZIP.", "", markup.button(f"Download {display} Pack", RELEASE_URL, primary=True), "", ] else: lines += [ no_zip_line(display), "", *_setup_routes(platform_name, display), f"[Why no ZIP?](../{NO_ZIP_FAQ})", "", ] lines += _next_block([ f"[{display}](../platforms/{platform_name}.md#install): the folder, " "and how to run the command on the device", "[All installer options](../wiki/installer.md)", ]) return "\n".join(lines) def generate_get_index() -> str: """get/: the permanent page of every setup, grouped as the home tiles are. A link trimmed to /get/ answered 404.""" registry = load_platform_registry("platforms") lines = [ "---", "search: exclude", 'description: "The BIOS files of every setup, one page each. Each address stays the same."', "---", "", "# BIOS files by setup", "", "One page per setup. Its address stays the same, so it is the one to share.", "", ] for group, (label, _description) in DEVICE_GROUPS.items(): names = [name for name, entry in registry.items() if entry.get("device_group") == group] if not names: continue lines += [f"## {label}", ""] for name in names: display = load_platform_config(name, "platforms").get("platform", name) lines.append(f"- [{get_page_title(name, display)}]({name}.md)") lines.append("") return "\n".join(lines) # The home tile counts the whole collection. Labelled "Files" above a table # of pack buttons, it read as what one pack contains. COLLECTION_FILES_LABEL = "Files collected" def composition_sentence(db: dict) -> str: comp = compute_composition(db) return ( f"The collection holds {comp['systems']['files']:,} console and " f"computer system files, {comp['arcade']['files']:,} arcade ROM sets " f"(`Arcade/`), and {comp['game_data']['files']:,} game and engine " "data files (`Game Engines/`, `RPG Maker/`, `ScummVM/` and the game " "publishers' trees). No pack holds " "them all: each carries what its platform's emulators load." ) def generate_stats(stats: dict) -> str: return json.dumps(stats, indent=2) + "\n" HASH_ALGORITHMS = ("sha1", "sha256", "md5", "crc32") def _hash_values(value: object) -> list[str]: """Every accepted value of one hash field, lowercase. A field holds one value, a list of accepted revisions, or (Recalbox) a comma-joined string: an exact lookup missed all but the first form. """ if value is None: return [] items = value if isinstance(value, (list, tuple)) else str(value).split(",") return sorted({str(item).strip().lower() for item in items if str(item).strip()}) def _accepted_hash_rows(platform_files: list[dict], emulator_items: list[dict]) -> list[dict]: """One row per accepted hash value, for both layers, lookup-ready.""" rows: list[dict] = [] for row in platform_files: for algorithm in HASH_ALGORITHMS: for value in _hash_values(row.get(f"{algorithm}_raw", row.get(algorithm))): rows.append({ "layer": "platform", "owner": row["platform_id"], "system": row["system"], "name": row["name"], "path": row["destination"], "algorithm": algorithm, "value": value, }) for item in emulator_items: for entry in item["profile"].get("files", []) or []: if not isinstance(entry, dict): continue for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): rows.append({ "layer": "emulator", "owner": item["id"], "system": _json_text(entry.get("system")), "name": entry.get("name", ""), "path": entry.get("path", "") or "", "algorithm": algorithm, "value": value, }) # Members of an archive: MAME and FBNeo hash the ROMs inside the # set, never the set. Left out, a loose nmk004.bin (crc32 # 83b6f611) found no row while the site search found it. archive = entry.get("name", "") for member in entry.get("contents") or []: if not isinstance(member, dict) or not member.get("name"): continue for algorithm in HASH_ALGORITHMS: for value in _hash_values(member.get(algorithm)): rows.append({ "layer": "emulator", "owner": item["id"], "system": _json_text(entry.get("system")), "name": str(member["name"]), "path": f"{archive}/{member['name']}", "algorithm": algorithm, "value": value, }) rows.sort(key=lambda r: (r["layer"], r["owner"], r["name"], r["algorithm"], r["value"])) return rows def _csv_document(fieldnames: list[str], rows: list[dict]) -> str: stream = io.StringIO(newline="") writer = csv.DictWriter( stream, fieldnames=fieldnames, extrasaction="ignore", lineterminator="\n" ) writer.writeheader() writer.writerows(rows) return stream.getvalue() def _sha256_path(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _verify_outcomes( coverage: dict, sha1_by_path: dict[str, str] ) -> Callable[[str, str], dict]: """For each declared entry, in declaration order: what verify found. Returns a function of (system, name) giving the next outcome for that pair: status, category, the collection path that answers the entry and that file's sha1. The CSV gave the declaration and nothing about it. """ details: dict[tuple[str, str], list[dict]] = {} for detail in coverage.get("details") or []: details.setdefault((detail.get("system", ""), detail.get("name", "")), []).append(detail) taken: dict[tuple[str, str], int] = {} def outcome(system: str, name: str) -> dict: found = details.get((system, name)) or [{}] nth = taken.get((system, name), 0) taken[(system, name)] = nth + 1 detail = found[min(nth, len(found) - 1)] path = str(detail.get("path") or "") if detail.get("status") != "missing" else "" return { "status": detail.get("status", ""), "category": detail.get("category", ""), "collection_path": path, "resolved_sha1": sha1_by_path.get(path, "") if path else "", } return outcome def _sha1_by_path(db: dict | None) -> dict[str, str]: return {str(e.get("path", "")): sha1 for sha1, e in ((db or {}).get("files") or {}).items()} @functools.lru_cache(maxsize=1) def _contract_keys() -> frozenset[str]: """The top-level keys platform.schema.json allows in a platform's list.""" schema = json.loads((_REPO_ROOT / "schemas" / "platform.schema.json").read_text(encoding="utf-8")) return frozenset(schema.get("properties") or {}) def _split_contract(config: dict) -> tuple[dict, dict]: """The list, valid against platform.schema.json, and the registry entry. load_platform_config merges the registry into the list: `install`, `logo`, `scraper` and `_key` made all 18 contracts fail the published schema. They stay published, beside the contract. """ allowed = _contract_keys() contract = {key: value for key, value in config.items() if key in allowed} registry = {key: value for key, value in config.items() if key not in allowed and key != "_key"} # The merged config gives a child its parent's release_pack: Lakka and # RetroPie said true in the API while every page said they had no ZIP. # The registry's own entry decides, as it does for the pages. released = config.get("_key") in _release_pack_platforms() if released or "release_pack" in registry: registry["release_pack"] = released return contract, registry @functools.lru_cache(maxsize=1) def _release_pack_platforms() -> frozenset[str]: return frozenset(list_release_pack_platforms( str(_REPO_ROOT / "platforms"), include_archived=True)) # Per-entry fields a folder check needs beyond the hashes: an md5 is a ZIP # member's when zipped_file is set, and a CSV without it judged 106 good # Batocera files wrong. ENTRY_EXPORT_FIELDS = ( "zipped_file", "hash_match_mandatory", "mandatory_declared", "curated", "shared_group", "mirror_of", "zip_digest", ) def _platform_export_rows( coverages: dict, db: dict | None = None, ) -> tuple[list[dict], list[dict]]: """Normalized platform summaries and one row per declared file.""" items: list[dict] = [] file_rows: list[dict] = [] sha1_by_path = _sha1_by_path(db) for key, coverage in sorted(coverages.items()): config = coverage["config"] outcome = _verify_outcomes(coverage, sha1_by_path) contract, registry = _split_contract(config) items.append({ "id": key, "name": coverage["platform"], "coverage": { field: coverage.get(field, 0) for field in ( "total", "present", "verified", "untested", "missing", "core_present", "core_missing", "core_unsourceable", ) }, "contract": contract, "registry": registry, }) for system_id, system in sorted(config.get("systems", {}).items()): for entry in system.get("files", []) or []: file_rows.append({ "platform_id": key, "platform": coverage["platform"], "system": system_id, "name": entry.get("name", ""), "destination": entry.get( "destination", entry.get("dest", entry.get("name", "")) ), "required": bool(entry.get("required", True)), "region": _json_text(entry.get("region")), "variant_group": _json_text(entry.get("variant_group")), "size": entry.get("size"), "sha1": _json_text(entry.get("sha1")), "sha256": _json_text(entry.get("sha256")), "md5": _json_text(entry.get("md5")), "crc32": _json_text(entry.get("crc32")), **{field: _export_flag(entry.get(field)) for field in ENTRY_EXPORT_FIELDS}, **outcome(system_id, entry.get("name", "")), }) return items, file_rows def _export_flag(value: object) -> object: """A per-entry field as CSV and SQLite hold it: booleans stay booleans, an absent field is empty.""" if value is None: return "" return value if isinstance(value, bool) else str(value) def _emulator_export_items(profiles: dict) -> list[dict]: return [ {"id": key, "profile": profile} for key, profile in sorted(profiles.items()) ] def build_emulator_gap_report( profiles: dict, coverages: dict, db: dict, data_names: set[str] | None = None, ) -> dict: """Files a core loads that no platform declares, per emulator profile. The gap analysis page and the published gaps export must not compute this twice and drift. Both read this one report. """ from common import _norm_system_id, merged_platform_declarations from cross_reference import cross_reference as run_cross_reference # Enriched with alias resolution, with each declaration's size and slot: # a name alone settled DOSBox's 32 KiB SC-55 ROM1.BIN on Galaksija's. all_declared = merged_platform_declarations( [cov["config"] for cov in coverages.values()], db ) declared: dict[str, set[str]] = {} for _name, cov in coverages.items(): config = cov["config"] for sys_id, system in config.get("systems", {}).items(): for fe in system.get("files", []): fname = fe.get("name", "") if fname: declared.setdefault(_norm_system_id(sys_id), set()).add(fname) unique_profiles = { k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test") } from common import platform_declarations, standalone_set from validation import case_folded_slots # The command line passes these too: without them a RetroBat file named # 32X_G_BIOS.BIN read as missing for a core opening 32X_G_BIOS.bin. folded = frozenset().union(*( case_folded_slots(cov["config"], platform_declarations(cov["config"], db)) for cov in coverages.values() )) return run_cross_reference( unique_profiles, declared, db, data_names=data_names, all_declared=all_declared, standalone_cores=set().union( *(standalone_set(cov["config"]) for cov in coverages.values()) ), folded_slots=folded, ) def _gap_export_rows( coverages: dict, gap_report: dict | None = None, db: dict | None = None, profiles: dict | None = None, ) -> list[dict]: """Every gap the site reports, both layers, in one table. ``platform`` rows are anomalies against a platform's own BIOS list. ``emulator`` rows are files a profiled core loads that no platform declares, which is the larger number the gap analysis page leads with. A `layer` column keeps the two apart instead of publishing only one. """ rows: list[dict] = [] sha1_by_path = _sha1_by_path(db) for key, coverage in sorted(coverages.items()): for detail in coverage.get("details", []): if detail.get("status") == "ok" and not detail.get("discrepancy"): continue path = str(detail.get("path") or "") if detail.get("status") != "missing" else "" rows.append({ "category": detail.get("category", "bios"), "collection_path": path, "resolved_sha1": sha1_by_path.get(path, "") if path else "", "layer": "platform", "platform_id": key, "platform": coverage["platform"], "emulator": "", "system": detail.get("system", ""), "name": detail.get("name", ""), "status": detail.get("status", ""), "required": bool(detail.get("required", True)), "in_repo": "", "reason": detail.get("reason", ""), "discrepancy": detail.get("discrepancy", ""), "destination": detail.get("destination", ""), "expected": "", }) for emu_key, data in sorted((gap_report or {}).items()): systems = ";".join(str(s) for s in data.get("systems", []) or []) categories = _profile_categories((profiles or {}).get(emu_key) or {}) for gap in data.get("gap_details", []) or []: rows.append({ "category": categories.get( (gap.get("name", ""), gap.get("path") or gap.get("name", "")), ""), "collection_path": "", "resolved_sha1": "", "layer": "emulator", "platform_id": "", "platform": "", "emulator": data.get("emulator", emu_key), "system": systems, "name": gap.get("name", ""), # A file the emulator's own package installs is not one to # find: its own status, as unsourceable has. "status": ( "bundled" if gap.get("bundled") and gap.get("source") == "missing" else gap.get("source", "") ), "required": bool(gap.get("required", False)), "in_repo": bool(gap.get("in_repo", False)), "reason": gap.get("note", ""), "discrepancy": "", # What a contributor needs to supply the right revision and # put it in the right place: the gaps table gave neither. "destination": gap.get("path") or gap.get("name", ""), "expected": gap.get("identity", ""), }) for entry in data.get("unsourceable", []) or []: rows.append({ "category": categories.get( (entry.get("name", ""), entry.get("path") or entry.get("name", "")), ""), "collection_path": "", "resolved_sha1": "", "layer": "emulator", "platform_id": "", "platform": "", "emulator": data.get("emulator", emu_key), "system": systems, "name": entry.get("name", ""), "status": "unsourceable", "required": bool(entry.get("required", False)), "in_repo": False, "reason": entry.get("reason", ""), "discrepancy": "", "destination": entry.get("path") or entry.get("name", ""), "expected": "", }) # Two declarations that differ only in what the row does not carry are # one row: AmberELEC's hardwareconfig.xml was printed twice. unique: dict[tuple, dict] = {} for row in rows: unique.setdefault(tuple(sorted((k, str(v)) for k, v in row.items())), row) return list(unique.values()) def _profile_categories(profile: dict) -> dict[tuple[str, str], str]: """(name, path) -> the category a profile gives the entry, bios by default.""" found: dict[tuple[str, str], str] = {} for entry in profile.get("files") or []: if isinstance(entry, dict) and entry.get("name"): name = str(entry["name"]) found.setdefault((name, str(entry.get("path") or name)), str(entry.get("category") or "bios")) return found def _cross_reference_export_rows(coverages: dict, profiles: dict) -> list[dict]: from common import resolve_platform_cores unique = { key: value for key, value in profiles.items() if value.get("type") not in ("alias", "test") } rows: list[dict] = [] for platform_id, coverage in sorted(coverages.items()): for profile_id in sorted(resolve_platform_cores(coverage["config"], unique)): profile = unique[profile_id] cores = profile.get("cores") or [profile_id] systems = profile.get("systems") or [""] for core in cores: for system in systems: rows.append({ "platform_id": platform_id, "platform": coverage["platform"], "profile_id": profile_id, "emulator": profile.get("emulator", profile_id), "core": core, "system": system, "classification": profile.get("core_classification", ""), "type": profile.get("type", ""), "source": _json_text(profile.get("source")), "upstream": _json_text(profile.get("upstream")), "profiled_commit": _json_text(profile.get("source_commit", "")), "file_count": len(profile.get("files", []) or []), }) return rows def _write_sqlite_export( destination: Path, db: dict, platform_items: list[dict], platform_files: list[dict], emulator_items: list[dict], gap_rows: list[dict], ) -> None: """Build a queryable, deterministic snapshot without embedding binaries.""" temp_dir = Path("tmp") / "site" temp_dir.mkdir(parents=True, exist_ok=True) temp_path = temp_dir / f"retrobios-{os.getpid()}.sqlite" temp_path.unlink(missing_ok=True) destination.parent.mkdir(parents=True, exist_ok=True) connection = sqlite3.connect(temp_path) try: connection.executescript(""" PRAGMA journal_mode = OFF; PRAGMA synchronous = OFF; CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL); CREATE TABLE files ( sha1 TEXT PRIMARY KEY, path TEXT NOT NULL, name TEXT NOT NULL, size INTEGER NOT NULL, md5 TEXT NOT NULL, sha256 TEXT NOT NULL, crc32 TEXT NOT NULL, adler32 TEXT NOT NULL ); CREATE TABLE file_provenance ( sha1 TEXT NOT NULL, catalog TEXT NOT NULL, details_json TEXT NOT NULL, PRIMARY KEY (sha1, catalog), FOREIGN KEY (sha1) REFERENCES files(sha1) ); CREATE TABLE platforms ( id TEXT PRIMARY KEY, name TEXT NOT NULL, total INTEGER NOT NULL, present INTEGER NOT NULL, verified INTEGER NOT NULL, missing INTEGER NOT NULL, contract_json TEXT NOT NULL ); CREATE TABLE platform_files ( platform_id TEXT NOT NULL, system TEXT NOT NULL, name TEXT NOT NULL, destination TEXT NOT NULL, required INTEGER NOT NULL, region TEXT, variant_group TEXT, size INTEGER, sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT, status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT, zipped_file TEXT, hash_match_mandatory INTEGER, mandatory_declared INTEGER, curated INTEGER, shared_group TEXT, mirror_of TEXT, zip_digest TEXT ); CREATE TABLE emulators ( id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT, classification TEXT, source TEXT, upstream TEXT, source_commit TEXT, profile_json TEXT NOT NULL ); CREATE TABLE emulator_systems ( emulator_id TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator_id, system) ); CREATE TABLE emulator_files ( emulator_id TEXT NOT NULL, system TEXT, name TEXT NOT NULL, path TEXT, required INTEGER NOT NULL, mode TEXT, region TEXT, size TEXT, sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT, source_ref TEXT ); CREATE TABLE gaps ( layer TEXT NOT NULL, platform_id TEXT, platform TEXT, emulator TEXT, system TEXT, name TEXT NOT NULL, status TEXT NOT NULL, required INTEGER NOT NULL, in_repo TEXT, reason TEXT, discrepancy TEXT, destination TEXT, expected TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT ); CREATE TABLE accepted_hashes ( layer TEXT NOT NULL, owner TEXT NOT NULL, system TEXT, name TEXT NOT NULL, path TEXT, algorithm TEXT NOT NULL, value TEXT NOT NULL ); CREATE INDEX accepted_hashes_value_idx ON accepted_hashes(algorithm, value); CREATE INDEX files_name_idx ON files(name); CREATE INDEX files_md5_idx ON files(md5); CREATE INDEX files_sha256_idx ON files(sha256); CREATE INDEX platform_files_name_idx ON platform_files(name); CREATE INDEX emulator_files_name_idx ON emulator_files(name); CREATE INDEX gaps_status_idx ON gaps(layer, status); """) commit = _source_commit() metadata = { "schema_version": "1", "generated_at": str(db.get("generated_at") or _timestamp()), **({"commit": commit} if commit else {}), "source": REPO_URL, "scope": "metadata only, no BIOS or firmware payload bytes", "license": DATA_LICENSE, } connection.executemany( "INSERT INTO metadata(key, value) VALUES (?, ?)", sorted(metadata.items()), ) for sha1, entry in sorted(db.get("files", {}).items()): connection.execute( "INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ( sha1, entry.get("path", ""), entry.get("name", ""), entry.get("size", 0), entry.get("md5", ""), entry.get("sha256", ""), entry.get("crc32", ""), entry.get("adler32", ""), ), ) for catalog, details in sorted((entry.get("provenance") or {}).items()): connection.execute( "INSERT INTO file_provenance VALUES (?, ?, ?)", (sha1, catalog, _json_text(details)), ) for item in platform_items: coverage = item["coverage"] connection.execute( "INSERT INTO platforms VALUES (?, ?, ?, ?, ?, ?, ?)", ( item["id"], item["name"], coverage.get("total", 0), coverage.get("present", 0), coverage.get("verified", 0), coverage.get("missing", 0), _json_text(item["contract"]), ), ) connection.executemany( "INSERT INTO platform_files VALUES " "(:platform_id, :system, :name, :destination, :required, :region, " ":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, " ":collection_path, :resolved_sha1, :zipped_file, :hash_match_mandatory, " ":mandatory_declared, :curated, :shared_group, :mirror_of, :zip_digest)", platform_files, ) for item in emulator_items: profile = item["profile"] connection.execute( "INSERT INTO emulators VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ( item["id"], profile.get("emulator", item["id"]), profile.get("type", ""), profile.get("core_classification", ""), _json_text(profile.get("source")), _json_text(profile.get("upstream")), # source_commit is a string or, when the builds live in # separate repositories, an object keyed by build mode. # SQLite cannot bind the object form. _json_text(profile.get("source_commit", "")), _json_text(profile), ), ) for system in sorted(set(profile.get("systems", []) or [])): connection.execute( "INSERT INTO emulator_systems VALUES (?, ?)", (item["id"], system), ) for entry in profile.get("files", []) or []: connection.execute( "INSERT INTO emulator_files VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( item["id"], _json_text(entry.get("system")), entry.get("name", ""), entry.get("path", ""), int(bool(entry.get("required", False))), entry.get("mode", ""), _json_text(entry.get("region")), _json_text(entry.get("size")), _json_text(entry.get("sha1")), _json_text(entry.get("sha256")), _json_text(entry.get("md5")), _json_text(entry.get("crc32")), _json_text(entry.get("source_ref")), ), ) connection.executemany( "INSERT INTO accepted_hashes VALUES " "(:layer, :owner, :system, :name, :path, :algorithm, :value)", _accepted_hash_rows(platform_files, emulator_items), ) connection.executemany( "INSERT INTO gaps VALUES " "(:layer, :platform_id, :platform, :emulator, :system, :name, " ":status, :required, :in_repo, :reason, :discrepancy, :destination, :expected, " ":category, :collection_path, :resolved_sha1)", gap_rows, ) connection.commit() connection.execute("VACUUM") finally: connection.close() os.replace(temp_path, destination) def write_single_item( path: Path, kind: str, item: dict, generated_at: str, commit: str = "", ) -> None: """One item in the envelope of its collection, for a page to link.""" path.parent.mkdir(parents=True, exist_ok=True) write_if_changed( str(path), json.dumps( {"schema_version": 1, "generated_at": generated_at, **({"commit": commit} if commit else {}), "kind": kind, "count": 1, "items": [item]}, indent=2, sort_keys=True, ensure_ascii=False, default=str, ) + "\n", ) def system_export_item(slug: str, manufacturer: str, consoles: dict[str, list]) -> dict: """A system page's files by console, with the hashes its cards show.""" fields = ("name", "path", "size", "sha1", "md5", "sha256", "crc32") return { "id": slug, "name": manufacturer, "consoles": { console: [ {field: entry[field] for field in fields if entry.get(field) is not None} for entry in sorted(entries, key=lambda e: (e.get("name", ""), e.get("sha1", ""))) ] for console, entries in sorted(consoles.items()) }, } @functools.lru_cache(maxsize=1) def _source_commit() -> str: """The commit of the checkout the site is built from, empty outside one.""" import subprocess try: result = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=_REPO_ROOT, capture_output=True, text=True, check=True, ) except (OSError, subprocess.CalledProcessError) as error: print(f" no commit recorded in the exports: {error}", file=sys.stderr) return "" commit = result.stdout.strip() return commit if re.fullmatch(r"[0-9a-f]{40}", commit) else "" # One row per dump a catalog describes and the collection lacks: the list # the provenance page publishes under "Wanted". WANTED_FIELDS = ("catalog", "dat", "name", "description", "size", "crc32", "md5", "sha1") def _provenance_wanted_rows(db: dict) -> list[dict]: """The Wanted list of the provenance page, as rows. A dump the collection already produces (derivable.json) is not wanted.""" derivable = load_derivable() rows: list[dict] = [] for catalog, data in build_report(db, load_provenance_snapshots()).items(): for entry in data["missing"]: if (entry.get("sha1") or "").lower() in derivable: continue for dat in entry.get("dats") or [entry.get("dat", "")]: rows.append({ "catalog": catalog, "dat": dat, **{f: _json_text(entry.get(f)) for f in WANTED_FIELDS[2:]}, }) rows.sort(key=lambda r: (r["catalog"], r["dat"], r["name"], r["sha1"])) return rows def generate_data_exports( docs: Path, db: dict, coverages: dict, profiles: dict, stats: dict, gap_report: dict | None = None, consumers: FileConsumers | None = None, ) -> list[dict]: """Create versioned static API, CSV and SQLite metadata snapshots.""" api = docs / "api" / "v1" downloads = docs / "downloads" schemas_dest = api / "schemas" for directory in (api, downloads, schemas_dest): directory.mkdir(parents=True, exist_ok=True) generated_at = str(db.get("generated_at") or _timestamp()) # The revision the data was read at: a tool comparing two downloads # could not tell which commit each came from. commit = _source_commit() stamp = {"commit": commit} if commit else {} platform_items, platform_files = _platform_export_rows(coverages, db) emulator_items = _emulator_export_items(profiles) gap_rows = _gap_export_rows(coverages, gap_report, db, profiles) cross_rows = _cross_reference_export_rows(coverages, profiles) members = data_directory_members() locations = file_location_items(db, profiles, members) envelopes = { "platforms.json": ("platforms", platform_items), "emulators.json": ("emulators", emulator_items), "gaps.json": ("verification-and-coverage-gaps", gap_rows), "file-locations.json": ("file-locations", locations), } write_lookup_shards( api, file_lookup_records(db, coverages, profiles, locations, members, consumers), generated_at, commit, ) for filename, (kind, items) in envelopes.items(): document = { "schema_version": 1, "generated_at": generated_at, **stamp, "kind": kind, "count": len(items), "items": items, } write_if_changed( str(api / filename), json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) # One platform without the others, as api/v1/emulators/.json is one # profile: platforms.json is 2.8 MB for twelve platforms. for item in platform_items: write_single_item(api / "platforms" / f"{item['id']}.json", "platforms", item, generated_at, commit) write_if_changed(str(api / "stats.json"), generate_stats(stats)) write_if_changed( str(api / "database.json"), json.dumps(db, ensure_ascii=False, indent=2) + "\n", ) # Every schema the published files are written against, the manifests' # too: an integrator had the platform's list and not what the pack writes. for schema_path in sorted((_REPO_ROOT / "schemas").glob("*.schema.json")): shutil.copy2(schema_path, schemas_dest / schema_path.name) _record(schemas_dest / schema_path.name) manifest_assets = _publish_manifests(api) for filename, (kind, items) in { "systems.json": ("system-index", system_index_items(db, coverages, profiles)), "cores.json": ("core-index", core_index_items(profiles)), }.items(): write_if_changed(str(api / filename), json.dumps({ "schema_version": 1, "generated_at": generated_at, **stamp, "kind": kind, "count": len(items), "items": items, }, ensure_ascii=False, indent=2, sort_keys=True) + "\n") file_rows = [] for sha1, entry in sorted(db.get("files", {}).items()): file_rows.append({ "sha1": sha1, "path": entry.get("path", ""), "name": entry.get("name", ""), "size": entry.get("size", 0), "md5": entry.get("md5", ""), "sha256": entry.get("sha256", ""), "crc32": entry.get("crc32", ""), "adler32": entry.get("adler32", ""), "provenance_catalogs": ";".join( sorted((entry.get("provenance") or {}).keys()) ), }) write_if_changed(str(downloads / "retrobios.dat"), collection_dat(db, generated_at)) write_if_changed( str(downloads / "files.csv"), _csv_document( [ "sha1", "path", "name", "size", "md5", "sha256", "crc32", "adler32", "provenance_catalogs", ], file_rows, ), ) write_if_changed( str(downloads / "platform-files.csv"), _csv_document( [ "platform_id", "platform", "system", "name", "destination", "required", "region", "variant_group", "size", "sha1", "sha256", "md5", "crc32", "status", "category", "collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS, ], platform_files, ), ) write_if_changed( str(downloads / "cross-reference.csv"), _csv_document( [ "platform_id", "platform", "profile_id", "emulator", "core", "system", "classification", "type", "source", "upstream", "profiled_commit", "file_count", ], cross_rows, ), ) write_if_changed( str(downloads / "gaps.csv"), _csv_document( [ "layer", "platform_id", "platform", "emulator", "system", "name", "status", "required", "in_repo", "reason", "discrepancy", "destination", "expected", "category", "collection_path", "resolved_sha1", ], gap_rows, ), ) write_if_changed( str(downloads / "accepted-hashes.csv"), _csv_document( ["layer", "owner", "system", "name", "path", "algorithm", "value"], _accepted_hash_rows(platform_files, emulator_items), ), ) _write_sqlite_export( downloads / "retrobios.sqlite", db, platform_items, platform_files, emulator_items, gap_rows, ) _record(downloads / "retrobios.sqlite") write_if_changed( str(downloads / "provenance-wanted.csv"), _csv_document(list(WANTED_FIELDS), _provenance_wanted_rows(db)), ) assets = [ (api / "database.json", "Content database", "application/json", "schemas/database.schema.json"), (api / "platforms.json", "Platform contracts", "application/json", "schemas/site-api-envelope.schema.json"), (api / "emulators.json", "Emulator profiles", "application/json", "schemas/site-api-envelope.schema.json"), (api / "gaps.json", "Verification and coverage gaps", "application/json", "schemas/site-api-envelope.schema.json"), (api / "file-locations.json", "Where each file's card is, by SHA1", "application/json", "schemas/site-api-envelope.schema.json"), (api / "stats.json", "Project statistics", "application/json", "schemas/stats.schema.json"), (api / "systems.json", "System ids, their names and sections", "application/json", "schemas/site-api-envelope.schema.json"), (api / "cores.json", "Core names and the profile each one is", "application/json", "schemas/site-api-envelope.schema.json"), *manifest_assets, (downloads / "files.csv", "File hashes CSV", "text/csv", None), (downloads / "retrobios.dat", "Collection DAT (Logiqx XML, for clrmamepro and RomVault)", "application/xml", None), (downloads / "platform-files.csv", "Platform declarations CSV", "text/csv", None), (downloads / "cross-reference.csv", "Cross-reference CSV", "text/csv", None), (downloads / "gaps.csv", "Verification and coverage gaps CSV", "text/csv", None), (downloads / "accepted-hashes.csv", "Accepted hashes, one row per value", "text/csv", None), (downloads / "provenance-wanted.csv", "Catalogued dumps the collection lacks", "text/csv", None), (downloads / "retrobios.sqlite", "SQLite snapshot", "application/vnd.sqlite3", None), *([(downloads / "measurements.sqlite", "Core probe measurements (SQLite)", "application/vnd.sqlite3", None)] if (downloads / "measurements.sqlite").is_file() else []), ] catalog_items: list[dict] = [] published: list[dict] = [] for path, title, media_type, schema in assets: relative = path.relative_to(docs).as_posix() item = { "title": title, "url": relative, "media_type": media_type, "bytes": path.stat().st_size, "sha256": _sha256_path(path), } if schema: item["schema"] = schema catalog_items.append(item) # In the catalog both references resolve against the catalog's own # URL, as RFC 3986 clients do: `url` was relative to the site root # and `schema` to api/v1/, so no single base resolved both. published.append({ **item, "url": os.path.relpath(path, api).replace(os.sep, "/"), }) catalog = { "schema_version": 1, "generated_at": generated_at, **stamp, "kind": "catalog", "license": DATA_LICENSE, "count": len(published), "items": published, } write_if_changed( str(api / "catalog.json"), json.dumps(catalog, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) # api/v1/ itself is a folder GitHub Pages answers with the 404 page: the # entry point is a document of its own, naming where to start. entry_points = [ {"title": "Catalog", "url": "catalog.json", "description": "Every published file with its size, SHA256 and schema."}, {"title": "System ids", "url": "systems.json", "description": "Each system id with its name and the section holding its files."}, {"title": "Core names", "url": "cores.json", "description": "Each core name and alias with the profile it designates."}, {"title": "Schemas", "url": "schemas/site-api-envelope.schema.json", "description": "The envelope every JSON document of api/v1 shares."}, ] write_if_changed(str(api / "index.json"), json.dumps({ "schema_version": 1, "generated_at": generated_at, **stamp, "kind": "index", "license": DATA_LICENSE, "count": len(entry_points), "items": entry_points, }, ensure_ascii=False, indent=2, sort_keys=True) + "\n") return catalog_items def write_release_advertised(generated: Path) -> None: """The release record as the pages read it, for the file picker. The picker imported release.json whole and offered the ZIP of six setups the registry no longer releases, while their pages said "No ZIP". It reads this file instead: the platforms marked release_pack only. """ generated.mkdir(parents=True, exist_ok=True) write_if_changed( str(generated / "release-advertised.json"), json.dumps(release_record.load_advertised(), ensure_ascii=False, indent=1, sort_keys=True) + "\n", ) def _publish_manifests(api: Path) -> list[tuple]: """The install manifests and the release record, beside the API. The platform's list says what a setup checks. The manifest says what its pack installs: Batocera's list holds 353 files, its pack 14,508. The only copy was install/.json on the repository's main branch, which no page named. Copied byte for byte: the SHA256 the catalog gives is the repository file's. """ folder = api / "manifests" folder.mkdir(parents=True, exist_ok=True) assets: list[tuple] = [] for source in sorted((_REPO_ROOT / "install").glob("*.json")): target = folder / source.name shutil.copy2(source, target) _record(target) assets.append((target, f"Install manifest: {source.stem}", "application/json", "schemas/install-manifest.schema.json")) # The target lists install.py --target reads: no catalog entry, and # api/v1/targets/ answered 404. targets = api / "targets" targets.mkdir(parents=True, exist_ok=True) for source in sorted((_REPO_ROOT / "install" / "targets").glob("*.json")): shutil.copy2(source, targets / source.name) _record(targets / source.name) assets.append((targets / source.name, f"Target list: {source.stem}", "application/json", "schemas/target-manifest.schema.json")) release = _REPO_ROOT / "release.json" if release.is_file(): shutil.copy2(release, api / "release.json") _record(api / "release.json") assets.append((api / "release.json", "Latest release: packs, sizes and assets", "application/json", "schemas/release.schema.json")) return assets def _site_path(markdown_target: str) -> str: """A page link as generators write it (systems/sony.md#psx) as a site-absolute path (/retrobios/systems/sony/#psx), which resolves the same from any document that holds it.""" page, _, anchor = markdown_target.partition("#") base = urllib.parse.urlsplit(SITE_URL).path page = page.removesuffix(".md") page = "" if page == "index" else page.removesuffix("/index") + "/" return f"{base}{page}" + (f"#{anchor}" if anchor else "") def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: """Each system id the platforms and profiles write, reachable by that id. api/v1/systems/ is keyed by manufacturer and its consoles by display name: a frontend keying by system id (sony-playstation) had no way in. The url is the section system_section gives, the one every page links. An id with no section of its own (tyrian, scummvm) that holds files takes the card of its first file, else the page of the emulator naming it: 28 such ids sent their hashes to no page. """ from siterender import card_page, system_ids, system_section sha1_by_path = _sha1_by_path(db) files = db.get("files") or {} by_md5 = (db.get("indexes") or {}).get("by_md5") or {} held: dict[str, set[str]] = {} for coverage in coverages.values(): for detail in coverage.get("details") or []: path = str(detail.get("path") or "") sha1 = sha1_by_path.get(path) if detail.get("status") != "missing" else None if sha1: held.setdefault(str(detail.get("system", "")), set()).add(sha1) for profile in profiles.values(): if profile.get("type") in ("alias", "test"): continue own = [str(s) for s in profile.get("systems") or []] for entry in profile.get("files") or []: if not isinstance(entry, dict): continue ids = ([str(entry["system"])] if entry.get("system") else [str(s) for s in entry.get("systems") or []] or (own if len(own) == 1 else [])) found = {v for v in _hash_values(entry.get("sha1")) if v in files} for md5 in _hash_values(entry.get("md5")): match = by_md5.get(md5) found.update([match] if isinstance(match, str) else match or []) for sys_id in ids: held.setdefault(sys_id, set()).update(found) manufacturers = {_slug(mfr): mfr for mfr in _group_by_manufacturer(db)} base = urllib.parse.urlsplit(SITE_URL).path items = [] for sys_id in sorted(system_ids(profiles, [c["config"] for c in coverages.values()])): section = system_section(sys_id) try: name = system_name(sys_id) except KeyError: name = sys_id sha1s = sorted(held.get(sys_id, set()) & files.keys()) manufacturer = manufacturers.get(section[0], "") if section else "" url = _site_path(console_href(*section)) if section else None if url is None and sha1s: manufacturer, url = _first_file_place(sys_id, sha1s, files, profiles, card_page) # The id's JSON document: api/v1/systems/ is keyed by the manufacturer # page, and sony-playstation.json answered 404. holder = section[0] if section else _slug(manufacturer) if manufacturer else "" api = f"{base}api/v1/systems/{holder}.json" if holder else None items.append({"id": sys_id, "name": name, "manufacturer": manufacturer, "url": url, "api": api, "sha1": sha1s}) return items def _first_file_place( sys_id: str, sha1s: list[str], files: dict, profiles: dict, card_page: Callable[[str, str], str], ) -> tuple[str, str | None]: """(manufacturer, url) of the card of an id's first collection file, else of the page of the first emulator naming the id.""" for sha1 in sha1s: parts = str(files[sha1].get("path", "")).split("/") if len(parts) >= 3 and parts[0] == "bios": page = card_page("/".join(parts), sha1) return parts[1], _site_path(f"systems/{page}.md#sha1-{sha1[:12]}") for key in sorted(profiles): profile = profiles[key] if profile.get("type") not in ("alias", "test") and sys_id in (profile.get("systems") or []): return "", _site_path(f"emulators/{key}.md") return "", None def core_index_items(profiles: dict) -> list[dict]: """Every name a core goes by, with the profile it designates. A frontend knows mednafen_psx_libretro, and api/v1/emulators/ had only beetle_psx.json: the reverse index resolve_platform_cores reads, aliases included, said once. A name two profiles claim (pcsx2) lists both, the one a platform list designates first. """ from common import preferred_profile, upstream_profile_index published = {name for name, p in profiles.items() if p.get("type") not in ("alias", "test")} index = upstream_profile_index(profiles) for name, profile in profiles.items(): target = str(profile.get("alias_of") or "") if profile.get("type") == "alias" and target in published: index.setdefault(name, set()).add(target) items = [] for name in sorted(index): claimants = sorted(index[name] & published) preferred = preferred_profile({name: set(claimants)}, name) for profile in sorted(claimants, key=lambda p: (p != preferred, p)): items.append({"name": name, "profile": profile, "preferred": profile == preferred}) return items def _is_manifest(item: dict) -> bool: return item["url"].startswith("api/v1/manifests/") def _is_target_list(item: dict) -> bool: return item["url"].startswith("api/v1/targets/") def _served_schemas() -> list[tuple[str, str]]: """(file name, title) of every schema api/v1/schemas/ serves, from the folder the export copies: the page linked 7 of 11.""" return [ (path.name, str(json.loads(path.read_text(encoding="utf-8")).get("title") or path.stem)) for path in sorted((_REPO_ROOT / "schemas").glob("*.schema.json")) ] # A URL template written in prose (https://...//): escaped, # the autolinker made a link of it, <ref> on screen and a 404 behind it. _URL_TEMPLATE = re.compile(r"(? list[str]: """One table row per field, its description read from the schema, so the page and the published contract say the same thing. A URL template is written as code.""" props = schema.get("properties") or {} rows = [] for key in keys: text = str((props.get(key) or {}).get("description") or "") if text: # Python before 3.12 refuses a backslash inside an f-string expression. described = _escape_tags(_URL_TEMPLATE.sub(r"`\g<0>`", text)) rows.append(f"| `{key}` | {described} |") return rows def _other_fields(schema: dict, shown: tuple[str, ...]) -> list[str]: """The fields a short table leaves out, every one with its description.""" rest = tuple(key for key in schema.get("properties") or {} if key not in shown) return ["| Field | Meaning |", "|-------|---------|", *_schema_field_rows(schema, rest)] def _nested_fields(schema: dict, prefix: str = "") -> list[str]: """A table for every object below *schema*, at any depth, and one line per code where a field lists its codes' meanings (enumDescriptions). The top-level and `files` tables left 50 nested fields undescribed, all of `omitted_files` among them: a tool could not say why a file was not installed. """ lines: list[str] = [] for key, spec in (schema.get("properties") or {}).items(): path = f"{prefix}{key}" codes = spec.get("enumDescriptions") if codes: lines.extend([f"Codes of `{path}`:", ""]) lines.extend(f"- `{code}`: {text}" for code, text in zip(spec["enum"], codes)) lines.append("") for label, child, child_path in ( ("Each entry of", spec.get("items"), f"{path}[]"), ("Each value of", spec.get("additionalProperties"), f"{path}{{}}"), ("Fields of", spec, path), ): if not isinstance(child, dict) or not child.get("properties"): continue lines.extend([ f"{label} `{path}`:", "", "| Field | Meaning |", "|-------|---------|", *_schema_field_rows(child, tuple(child["properties"])), "", ]) lines.extend(_nested_fields(child, f"{child_path}.")) return lines # The manifest fields an integrator reads first, in the order they matter. MANIFEST_TOP_FIELDS = ("platform", "version", "base_destination", "files", "omitted_files", "pack_files", "core_names", "system_cores") MANIFEST_FILE_FIELDS = ("dest", "sha1", "size", "repo_path", "release_asset", "cores", "systems", "category", "alternatives") def _short_sha256(item: dict, catalog: str) -> str: """A digest cell that fits its column: 12 characters linking the catalog that holds it, the whole value in the title. 64 characters pushed the table to 932 px in a 720 px column.""" value = str(item["sha256"]) return f'{value[:12]}' def _platforms_without_a_list(platforms_dir: Path | None = None) -> list[str]: """The names of the platforms whose registry says they keep no list.""" platforms_dir = platforms_dir or _REPO_ROOT / "platforms" names = [] for key, entry in sorted(load_platform_registry(str(platforms_dir)).items()): if entry.get("native_list") != "none": continue config = platforms_dir / str(entry.get("config") or f"{key}.yml") data = yaml_load(config.read_text(encoding="utf-8")) if config.is_file() else {} names.append(str((data or {}).get("platform") or key)) return names def _join_names(names: list[str]) -> str: return names[0] if len(names) == 1 else ", ".join(names[:-1]) + f" and {names[-1]}" def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = None) -> str: manifest_schema = json.loads( (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8")) snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") manifests = [item for item in exports if _is_manifest(item)] targets = [item for item in exports if _is_target_list(item)] no_list = _platforms_without_a_list() lines = [ "---", "terms: manifest, schema, json schema, license, check a folder, " "check bios folder script, rate limit, api", "---", "", "# Data and API", "", "RetroBIOS publishes the same metadata used by its verifier, pack builder " "and website as versioned static files. No account or API key is required.", "", " ".join([ markup.button("API catalog", "api/v1/catalog.json", primary=True), markup.button("SQLite snapshot", "downloads/retrobios.sqlite"), markup.button("Collection DAT", "downloads/retrobios.dat"), ]), "", f"**Licence:** the metadata is offered under the MIT terms of [LICENSE]({REPO_URL}/blob/main/LICENSE), " "which cover the scripts, tooling, schemas, documentation and metadata. " f"They never cover a BIOS or firmware file ([NOTICE]({REPO_URL}/blob/main/NOTICE)). " "`catalog.json` and the SQLite `metadata` table carry the same `license`.", "", "## Stable JSON endpoints", "", "Start at [`api/v1/index.json`](api/v1/index.json): it names the catalog, " "the two id indexes and the envelope schema.", "", "| Dataset | Endpoint | Schema | Size |", "|---------|----------|--------|-----:|", ] for item in exports: if (item["media_type"] != "application/json" or _is_manifest(item) or _is_target_list(item)): continue schema = ( f"[JSON Schema](api/v1/{item['schema']})" if item.get("schema") else "-" ) lines.append( f"| {item['title']} | [`{item['url']}`]({item['url']}) | " f"{schema} | {_fmt_size(item['bytes'])} |" ) lines.extend([ "", "Every envelope carries `schema_version`, `kind` and `commit`, the revision " "of this repository the data was read at. Breaking changes use a new " "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. " "Each schema's `$id` is the address it is served at.", "", "Schemas:", "", *(f"- [`{name}`](api/v1/schemas/{name}): {title}" for name, title in _served_schemas()), "", "`database.json`, `stats.json`, `release.json` and the manifests are not " "envelopes. Their revision is the `commit` of `catalog.json` that lists " "their SHA256.", "", "One emulator profile without the whole list: " "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " "[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each " "emulator page links its own. `api/v1/platforms/.json` holds one " "platform of `platforms.json`, its `contract` valid against " "[`platform.schema.json`](api/v1/schemas/platform.schema.json) and its " "`registry` the install and scraper settings beside the list. " "`api/v1/systems/.json` holds the files of one system page by console. " "Each platform and system page links its own.", "", "By id: [`systems.json`](api/v1/systems.json) gives each system id a platform " "or a profile writes (`sony-playstation`) with its name, its manufacturer, the " "`url` of the section holding its files, the `api` document of that section " "and the files' SHA1. " "[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a " "platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), " "whose document is `api/v1/emulators/.json`. A name two profiles " "claim (`pcsx2`) has one item per profile, and `preferred: true` marks the " "one a platform list naming that core gets: the profile with that very id, " "else the first id in alphabetical order.", "", "One hash without the whole list: `api/v1/lookup/.json`, where `` " "is the first two characters of the hash in lowercase, holds every SHA1, " "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` " "(the collection, a data directory, a platform's list or an emulator " "profile), the file `name`, the `url` of the page that shows it and, " "when one is known, the `system_name` the site gives its system. " "[Find or check a file](files/index.md) reads it.", "", "In a browser, the SHA1 of a file the reader picks finds its records. Web " "Crypto has no MD5, and none is needed: a setup that lists a file by MD5 " "or CRC32 is filed under the SHA1 of the collection file it names too.", "", "```js", "// The records of the file a reader picked, by its SHA1.", "async function lookUp(file) {", " const digest = await crypto.subtle.digest(\"SHA-1\", await file.arrayBuffer());", " const sha1 = [...new Uint8Array(digest)]", " .map((byte) => byte.toString(16).padStart(2, \"0\"))", " .join(\"\");", f" const base = \"{SITE_URL}api/v1/lookup/\";", " const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);", " if (!response.ok) return [];", " const shard = await response.json();", " // layer: collection, data-directory, platform or emulator. owner: which one.", " return shard.items.filter((item) => item.value === sha1);", "}", "```", "", "Two kinds of `url`:", "", "- In `catalog.json` and `index.json`, a `url` is relative to the document " "holding it, as RFC 3986 resolves it.", "- In the lookup shards, `file-locations.json` and `systems.json`, a `url` " f"is site-absolute (`/retrobios/systems/sony/#...`). Prefix " f"`{SITE_URL.removesuffix(urllib.parse.urlsplit(SITE_URL).path)}` for a full address.", "", "## Platform manifests", "", "A platform's list is what the setup checks. Its manifest is what the pack " "installs: the list's files plus every file the setup's cores load. " "`api/v1/manifests/.json` is the file `install.py` reads, byte " "for byte, valid against " "[`install-manifest.schema.json`](api/v1/schemas/install-manifest.schema.json). " "[`api/v1/release.json`](api/v1/release.json) records the packs of the latest " "release and the files each one holds.", "", "Manifests: " + ", ".join( f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})" for item in manifests) + "." if manifests else "", "", "`api/v1/targets/.json` gives each hardware target of a platform " "the cores it runs, valid against " "[`target-manifest.schema.json`](api/v1/schemas/target-manifest.schema.json). " "`install.py --target` reads the same file. Like the manifests, they are " "copied byte for byte and are not envelopes.", "", "Target lists: " + ", ".join( f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})" for item in targets) + "." if targets else "", "", "| Field | Meaning |", "|-------|---------|", *_schema_field_rows(manifest_schema, MANIFEST_TOP_FIELDS), "", "Each entry of `files`:", "", "| Field | Meaning |", "|-------|---------|", *_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), "", *markup.details("note", "All fields", [ "The manifest:", "", *_other_fields(manifest_schema, MANIFEST_TOP_FIELDS), "", "Each entry of `files`:", "", *_other_fields(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), "", *_nested_fields(manifest_schema), *_nested_fields(manifest_schema["$defs"]["file"], "files[]."), ]), "## Check a folder for one platform", "", "The job most tools come for: is this BIOS folder what the platform checks? " "Read `api/v1/platforms/.json` and apply the platform's own rules:", "", "- `verification_mode: existence`: a file at its `destination` passes.", "- An entry with no hash in the mode's field is checked on presence too.", "- `zipped_file` set: the md5 is that of this member inside the ZIP, never " "the ZIP's. Member names compare casefolded. A ZIP declared once per member " "appears once per member: Batocera lists 692 entries for 353 destinations.", "- An md5 shorter than 32 characters (Batocera) compares by its prefix. " "Recalbox writes several accepted md5 in one field, comma separated.", "- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members " "read in name order, never the archive's.", "- An entry with no `destination` (EmuDeck) names no file: the system " "passes when any file in the BIOS folder has one of its md5.", "- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. " "The script below does not apply it.", *([f"- {_join_names(no_list)} have no list of their own (`native_list: none`). " "An empty list does not mean nothing is needed."] if no_list else []), "", "```python", snippet.rstrip("\n"), "```", "", "The script is [`scripts/check_platform_folder.py`]" f"({REPO_URL}/blob/main/scripts/check_platform_folder.py). A test runs it " "against the verifier on a sample folder, so the two give the same answer. " "It exits 1 when a file is wrong or missing. For an unknown platform it " "prints the known ones.", "", "`install.py --check` answers another question by default: whether the folder " "holds what the pack installs, the manifest above. With `--declared-only`, or " "from the `declared` event, `install.py --check` answers it too.", "", "## Bulk downloads", "", "| Export | Format | Size | SHA256 |", "|--------|--------|-----:|--------|", ]) for item in exports: if item["media_type"] == "application/json": continue lines.append( f"| [{item['title']}]({item['url']}) | `{item['media_type']}` | " f"{_fmt_size(item['bytes'])} | {_short_sha256(item, 'api/v1/catalog.json')} |" ) lines.extend([ "", "Each SHA256 is cut to 12 characters. Point at it for the whole value, or " "read it in [`catalog.json`](api/v1/catalog.json).", "", "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game " "per `bios///` folder of the collection, named " "`Manufacturer\\Console` so the tools look in that subfolder, each file a " "rom named by its path below that folder. It audits a copy of the " "collection's `bios/` tree, or rebuilds that layout from your own dumps. A " "platform pack has its own layout and is checked with `install.py --check`.", "", "The SQLite file contains indexed tables for content hashes, provenance " "catalog matches, platform declarations, emulator profiles and current " "gaps. It contains metadata only, never BIOS or firmware bytes.", "", "To find who accepts a hash, query `accepted_hashes` (or " "`accepted-hashes.csv`): one lowercase row per accepted value, per " "algorithm, for platform declarations and emulator profiles alike, " "indexed on `(algorithm, value)`. A ROM inside a MAME or FBNeo set " "has its own row, its `path` naming the archive and the member " "(`nmk004.zip/nmk004.bin`). The hash columns of `platform_files` " "and `emulator_files` keep each source's own form: a Recalbox md5 may " "list several revisions separated by commas, and an emulator field " "holding several accepted values is a JSON array.", "", "```sql", "SELECT layer, owner, name FROM accepted_hashes", "WHERE algorithm = 'md5' AND value = 'c87a52c277f7952b41c639fc7bf0a43b';", "```", "", "The gaps dataset carries both layers behind one `layer` column: " "`platform` rows are anomalies against a platform's own BIOS list, " "`emulator` rows are files a profiled core loads that no platform " "declares. The two answer different questions and are not comparable " "totals. Each row is listed once.", "", "`platform-files.csv` and the gaps add what the check found: `status` " "(`ok`, `untested`, `missing`), `category` (`bios`, `game_data`, " "`unread`), and for a held file `collection_path` and `resolved_sha1`, " "the collection file that answers the declaration. " "`provenance-wanted.csv` is the Wanted list of [Dump provenance]" "(provenance.md): each dump a catalog describes and the collection lacks.", "", "`platform-files.csv` and the `platform_files` table also carry the entry " "fields a check needs:", "", "- `zipped_file` set: the md5 is the md5 of that member inside the ZIP.", "- `zip_digest` is `composite` where the md5 of a ZIP covers its members.", "- `hash_match_mandatory` and `mandatory_declared` are Recalbox's flags.", "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's " "own file does not carry.", "", *(sitemeasurements.data_page_section(measurements_sqlite) if measurements_sqlite and measurements_sqlite.is_file() else []), "## Hosting, rate limits and caching", "", "- Static files on GitHub Pages, read with a plain GET. No key, no account.", "- Any origin may read them (`Access-Control-Allow-Origin: *`), so a " "browser tool can call them directly.", "- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). " "Send the `ETag` back in `If-None-Match` to skip an unchanged download.", "- GitHub may answer HTTP 429 to a client that sends too many requests, and " "a Pages site has a soft limit of 100 GB of traffic a month ([GitHub Pages " "limits](https://docs.github.com/en/pages/getting-started-with-github-pages/" "github-pages-limits)). Fetch the one shard or document a question needs.", "- The base is the project's address, " f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.", "", "## Semantics and limits", "", "Treat four questions separately: content identity (hashes), presence in " "the collection, acceptance by a specific emulator, and catalog provenance. " "One does not imply the others. In particular, presence, a matching dump " "catalog, or emulator compatibility is not a statement about copyright, " "ownership, or redistribution rights.", "", f"The data can be newer than the [latest published pack]({RELEASE_URL}). " "A pack is published by hand, once every release check has passed.", "", "See the [data model](wiki/data-model.md), [verification modes]" "(wiki/verification-modes.md), and [methodology](wiki/architecture.md) " "before interpreting aggregate counts.", "", ]) return "\n".join(lines) def _page_title(markdown: str, path: Path) -> str: match = re.search(r"^#\s+(.+?)\s*$", markdown, flags=re.MULTILINE) if match: title = re.sub(r"<[^>]+>", "", match.group(1)) return title.replace(f" - {SITE_NAME}", "").strip() return path.stem.replace("-", " ").title() def _browser_title(relative: Path, title: str) -> str: """Return a concise, unique browser/search title for a generated page.""" key = relative.as_posix() index_titles = { # Material appends " - RetroBIOS": the home title says what the site # is, instead of "RetroBIOS - RetroBIOS". "index.md": "BIOS and firmware packs for RetroArch, Batocera and more", "platforms/index.md": "Your setup", "systems/index.md": "Systems", "emulators/index.md": "Emulators", "wiki/index.md": "Reference", "explore.md": "Explore", } if key in index_titles: return index_titles[key] # A console page is filed with the parts of its manufacturer's page, and # titled "Sega: Dreamcast". A part names its range of files. part = key in PART_PAGES and ( not key.startswith("systems/") or ": " not in title or re.search(r", part \d+ of \d+$", title)) if part: return title # The words a web search for a console or an emulator types: "dreamcast # bios", "duckstation bios". The h1 and the sidebar keep the name. if key.startswith("emulators/"): has_files = PAGE_DATA.get(key, {}).get("hasPart") # Lexaloffle, QEMU and ScummVM name a system page too. own = " emulator" if f"systems/{relative.name}" in PAGE_DATA else "" return f"{title}{own} BIOS files" if has_files else f"{title} emulator" if key.startswith("systems/"): maker, _, console = title.partition(": ") return f"{console} BIOS files ({maker})" if console else f"{title} BIOS files" return title def _plain_markdown(text: str) -> str: text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"!\[([^]]*)\]\([^)]*\)", r"\1", text) text = re.sub(r"\[([^]]+)\]\([^)]*\)", r"\1", text) # Code spans keep their text whole: validate_schemas.py read # validateschemas.py once every underscore was taken for emphasis. pieces = re.split(r"(`[^`]*`)", text) text = "".join( piece.strip("`") if piece.startswith("`") else re.sub(r"(? str: """A description cut at a word, never inside one: six snippets ended in "op..." or "Fo...".""" if len(plain) <= limit: return plain cut = plain[: limit + 1].rsplit(" ", 1)[0] if " " in plain[:limit] else plain[:limit] return cut.rstrip(" ,;:-(.") + "..." def _page_description(markdown: str, relative: Path, title: str) -> str: key = relative.as_posix() specific = { "index.md": "The BIOS files your emulators need, for RetroArch, Batocera, EmuDeck and more: the folder they go in, one command that installs and checks them, and packs to download.", "data.md": "Versioned RetroBIOS JSON API, CSV exports, SQLite snapshot, schemas, checksums and data interpretation guidance.", "cross-reference.md": "Every emulator core each retrogaming platform runs, the systems it emulates and the upstream project it comes from.", "gaps.md": "Current RetroBIOS verification gaps, platform-to-emulator divergences, missing files and documented source limitations.", "exports.md": "Each platform's own BIOS list (System.dat, batocera-systems, es_bios.xml and the rest) with the hashes and files the emulator source code shows, offered to its maintainers.", sitemeasurements.EXPLORER_PAGE: "The records of emulator cores run headless on demand, each with its claim, verdict, the BIOS it ran, method, computer and full record, filterable, and as SQLite.", "provenance.md": "Hash-based comparison of RetroBIOS metadata with No-Intro, Redump and TOSEC catalog snapshots.", "explore.md": "Systems, emulators and installer options to explore, with the figures of the BIOS collection.", "which-pack.md": "Pick your device or setup and get its BIOS files: the folder they go in, one command that installs and checks them, or a ZIP pack to extract.", "contributing.md": "Add a BIOS file the collection lacks from the browser, checked on arrival, or add a platform or an emulator profile.", } if key in specific: return specific[key] # What a search engine shows is the page's own first sentence: 707 pages # shared five templates of level 3 words ("source-pinned", "provenance"). browser = _browser_title(relative, title) held = re.search(r"\bThis part holds (.+?)\.(?: |$)", markdown, re.M) if key in PART_PAGES and held: return _shorten(f"{browser}. This part holds {_plain_markdown(held[1])}.", 155) lead = _lead(markdown) if key in PART_PAGES and re.search(r"\bpart \d+ of \d+", title): # The parts of one list open on the same sentence. first = _first_paragraph(lead) or _first_paragraph(markdown) return _shorten(f"{title}. {first}", 155) if key.startswith(f"{release_record.GET_DIR}/"): size = re.search(r"^- (.+ to download\b.*?)\.?$", markdown, re.M) if size: return _shorten(f"{title}: the BIOS pack of this setup, {_plain_markdown(size[1])}. " "The address stays the same and always gives the latest ZIP.", 155) return _shorten(_first_paragraph(lead) or title, 155) if key.startswith("systems/") and relative.stem != "index": return _system_description(markdown, browser) if key.startswith("emulators/") and relative.stem != "index": return _emulator_description(markdown, key, title, _first_paragraph(lead)) first = _first_paragraph(lead, minimum=35) questions = [_plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading)) for heading in re.findall(r"^## (.+\?)(?:[ \t]*\{[^}]*\})?[ \t]*$", markdown, re.M)] if not first and len(questions) >= 3: # A page of questions with no intro: its first paragraph is the # answer to one of them ("Run the command with --dry-run"). return _shorten(f"{title}: {' '.join(questions)}", 155) first = first or _first_paragraph(markdown, minimum=35) if first: return _shorten(first, 155) return f"{title}: RetroBIOS source-traced retrogaming firmware reference." def _lead(markdown: str) -> str: """The text a page opens with, before its first section, details block or ignored section (the install box, the profile table).""" return re.split(r"^(?:## | str: """An emulator page by its intro sentence ("DuckStation emulates Sony PlayStation. It picks the BIOS for your game's region: ..."), or by its profile table when the profile names no system.""" notes = re.search(r"Technical notes\s*\n\n(.+?)(?:\n\n|$)", markdown, re.S) if not intro and notes: sentence = re.split(r"(?<=\.)\s", _plain_markdown(notes[1]), maxsplit=1)[0] intro = f"{title}: {sentence[:1].lower()}{sentence[1:]}" if not intro: row = {name: _plain_markdown(value) for name, value in re.findall( r"^\| (Type|Classification|Version) \| (.+?) \|$", markdown, re.M)} kind = row.get("Classification") or row.get("Type") or "emulator" # The cell goes on to say where the version is read: the intro # keeps the version alone. version = row.get("Version", "").split(", ")[0] known = f", version {version}" if version and version[0].isdigit() else "" intro = f"{title}: {kind.lower()}{known}." elif title not in intro: # "It reads 1 file": the profile names no system to open on. intro = f"{title}: {intro[:1].lower()}{intro[1:]}" # An alias page sends the reader to the core whose files it shares. if PAGE_DATA.get(key, {}).get("hasPart") or "uses the same firmware as" in intro: return _shorten(intro, 155) needs = "It needs no BIOS or firmware file." if "\n## Data directories" in markdown: needs = "It needs no BIOS or firmware file, only the data folders listed." return _shorten(f"{intro} {needs}", 155) # Lines a page opens with that say where it sits, not what it holds. _PLACE_LINES = ("Part of ", "This part holds ", "On a card, ") def _first_paragraph(markdown: str, minimum: int = 1) -> str: """The first paragraph of prose of a page body, as plain text: no heading, table, list, fence, admonition or HTML block, and none that only says where the page sits.""" for paragraph in re.split(r"\n\s*\n", markdown): stripped = paragraph.strip() if ( not stripped or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* ", "import ")) ): continue plain = _plain_markdown(stripped) if plain.startswith(_PLACE_LINES): continue # A paragraph that ends on a colon introduces what follows: alone, # it describes nothing ("One file needs no clone and no local setup:"). if len(plain) >= minimum and not plain.endswith(":"): return plain return "" # Section headings of a system page that are not a console. _NOT_A_CONSOLE = {"Next steps", "Files", "Other files"} _CARD_TITLE = re.compile(r'class="rb-file-title"[^>]*>([^<]+)') def _system_description(markdown: str, browser: str) -> str: """A system page by what it holds: a manufacturer page by its consoles, a console page by its "Start with" line, else by its first files.""" consoles = [ _plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading)) for heading in re.findall(r"^## (.+?)[ \t]*$", markdown, re.M) ] consoles = [name for name in consoles if name and name not in _NOT_A_CONSOLE] if len(consoles) > 1: shown = consoles if len(consoles) <= 4 else consoles[:3] + [f"{len(consoles) - 3} more"] return _shorten(f"{browser} for {_said_as_list(shown)}, one card per file with its " "hashes and the emulators that read it.", 155) start = re.search(r"^\*\*Start with\*\*(?P.*)$(?P(?:\n+- .*)*)", markdown, re.M) if start: # "Start with the files of your machine:" introduces one line per machine. machines = [_plain_markdown(item) for item in re.findall(r"^- (.+)$", start["list"], re.M)] text = _plain_markdown(start["line"]).lstrip(": ") said = "; ".join(machines) if machines and text.endswith(":") else text return _shorten(f"{browser}. Start with {said}", 155) names = list(dict.fromkeys(html.unescape(name) for name in _CARD_TITLE.findall(markdown))) if names: shown = names if len(names) <= 3 else names[:3] + [f"{len(names) - 3} more"] return _shorten(f"{browser}: {_said_as_list(shown)}, with their hashes and the " "emulators that read them.", 155) return _shorten(f"{browser}, one card per file with its hashes and the emulators " "that read it.", 155) # Structured data a page generator knows beyond its text, by page path # relative to docs/: read by decorate_markdown_pages. PAGE_DATA: dict[str, dict] = {} # Pages that hold part of a page too long for one (a console of a split # manufacturer, a run of an emulator's cards), by path relative to docs/: # their H1 already says what they are. PART_PAGES: set[str] = set() SYSTEM_PARTS_IN_FULL = 250 def collection_dat(db: dict, generated_at: str) -> str: """The collection as a Logiqx DAT, the format clrmamepro and RomVault read. One game per bios/// folder, named Manufacturer\\Console, each file a rom named by its path below that folder, subfolders joined with a backslash as Logiqx DATs write them. An auditor checking a BIOS tree against the collection had JSON, CSV and SQLite, and a converter to write. """ import xml.etree.ElementTree as ET # noqa: PLC0415 games: dict[str, list[dict]] = {} files = db.get("files", {}) # Every copy, not the one path the database keeps: an audit of bios/ # read the 2,502 others as unknown files, and a fix moved them away # from the engines that read them by path. for sha1, paths in collection_paths(db).items(): for path in sorted(paths): parts = path.split("/") folder = parts[1:3] if len(parts) > 3 else parts[1:2] # A backslash is how a Logiqx set name writes a subfolder: named # "Sony - PlayStation", the set was looked for beside Sony/, and an # audit of a copy of bios/ found 330 sets missing. games.setdefault("\\".join(folder), []).append( {**files[sha1], "_rom": "\\".join(parts[1 + len(folder):])} ) root = ET.Element("datafile") header = ET.SubElement(root, "header") version = generated_at[:10].replace("-", "") for tag, text in ( ("name", SITE_NAME), ("description", f"{SITE_NAME} collection ({sum(map(len, games.values()))} files)"), ("version", version), ("date", generated_at[:10]), ("author", SITE_NAME), ("homepage", SITE_URL), ("url", REPO_URL), ): ET.SubElement(header, tag).text = text for game_name in sorted(games): game = ET.SubElement(root, "game", name=game_name) ET.SubElement(game, "description").text = game_name for entry in sorted(games[game_name], key=lambda e: e["_rom"]): attributes = {"name": entry["_rom"], "size": str(entry.get("size", 0))} for attribute, field in (("crc", "crc32"), ("md5", "md5"), ("sha1", "sha1")): if entry.get(field): attributes[attribute] = str(entry[field]).lower() ET.SubElement(game, "rom", attributes) ET.indent(root, space="\t") return ( '\n' '\n' + ET.tostring(root, encoding="unicode") + "\n" ) def system_structured_data(slug: str, consoles: dict[str, list]) -> dict: """A system page's files for a machine: each card as a CreativeWork. The pages that hold a card per file published a bare TechArticle with no file, hash or provenance, while emulator pages listed theirs. """ parts = [] for _console, entries in sorted(consoles.items()): for entry in entries: sha1 = str(entry.get("sha1") or "") part: dict = {"@type": "CreativeWork", "name": str(entry.get("name", ""))} if sha1: part["url"] = urllib.parse.urljoin(SITE_URL, f"systems/{slug}/#sha1-{sha1[:12]}") identifiers = [ f"{field}:{str(entry[field]).lower()}" for field in ("sha1", "md5", "crc32", "sha256") if entry.get(field) ] if identifiers: part["identifier"] = identifiers if entry.get("size"): part["contentSize"] = f"{entry['size']} B" citations = [ f"{catalog}: {record.get('dat', '')} ({record.get('name', '')})" for catalog, record in sorted((entry.get("provenance") or {}).items()) if isinstance(record, dict) ] if citations: part["citation"] = citations parts.append(part) if len(parts) > SYSTEM_PARTS_IN_FULL: # A page of cards carried up to 2.6 MB of JSON-LD (game-engines), a # quarter of its weight: past the cap a page lists its # catalogue-verified files and points at the full table. return { "hasPart": [part for part in parts if part.get("citation")], "isBasedOn": urllib.parse.urljoin(SITE_URL, "downloads/files.csv"), } return {"hasPart": parts} if parts else {} def platform_structured_data(name: str, coverage: dict) -> dict: """A platform page's declarations for a machine reader. The page printed thousands of hashes while its JSON-LD named only the page: a crawler could not learn which md5 Batocera requires for a file. The software and its version are the subject, and each declared file, with the hashes the platform checks, is a part. Past SYSTEM_PARTS_IN_FULL the parts give way to the platform-files table, as system pages do. """ config = coverage.get("config") or {} about: dict = {"@type": "SoftwareApplication", "name": str(coverage.get("platform", name))} if config.get("version"): about["softwareVersion"] = str(config["version"]) if config.get("homepage"): about["url"] = str(config["homepage"]) parts = [] for system_id, system in sorted((config.get("systems") or {}).items()): for entry in system.get("files") or []: if not isinstance(entry, dict) or not entry.get("name"): continue part: dict = { "@type": "CreativeWork", "name": str(entry.get("destination") or entry["name"]), "url": urllib.parse.urljoin(SITE_URL, f"platforms/{name}/#{system_id}"), } identifiers = [ f"{field}:{value}" for field in ("sha1", "md5", "crc32", "sha256") for value in _hash_values(entry.get(field)) ] if identifiers: part["identifier"] = identifiers parts.append(part) data: dict = {"about": about} if len(parts) > SYSTEM_PARTS_IN_FULL: data["isBasedOn"] = urllib.parse.urljoin(SITE_URL, "downloads/platform-files.csv") elif parts: data["hasPart"] = parts return data def dataset_structured_data(exports: list[dict], db: dict) -> dict: """The data page as a schema.org Dataset a harvester can index. It was a Dataset named "Data & API" with no download, licence, date or creator, so Google Dataset Search and open-data catalogues had nothing to list. The exports are metadata this project authors: MIT, as the LICENSE's scope says. """ data: dict = { "name": f"{SITE_NAME} collection data", "license": "https://opensource.org/licenses/MIT", "creator": {"@type": "Organization", "name": SITE_NAME, "url": REPO_URL}, "distribution": [ { "@type": "DataDownload", "name": item["title"], "contentUrl": urllib.parse.urljoin(SITE_URL, item["url"]), "encodingFormat": item["media_type"], "contentSize": f"{item['bytes']} B", "sha256": item["sha256"], } for item in exports ], } generated = str(db.get("generated_at") or "")[:10] if generated: data["dateModified"] = generated return data def emulator_file_anchors(files: list[dict]) -> list[str]: """The id of each file card on an emulator page, in profile order. An entry that declares a hash is addressed by it: its anchor depends on its own content, never on the entries beside it, and a second version added under the same path moves nothing. An entry without one keeps its plain anchor when its path is declared once, and otherwise takes its size, else, when even the size is shared, its description. The page and its JSON-LD both read this, so a structured part links to its card. The plain anchor of a hashed card stays reachable through emulator_file_aliases. """ bases = [ "file-" + ( re.sub( r"[^a-z0-9]+", "-", str(f.get("path") or f.get("standalone_path") or f.get("name", "")).lower(), ).strip("-") or "entry" ) for f in files ] repeated = {b for b in bases if bases.count(b) > 1} def _version_key(f: dict) -> str: digest = _first_hash_value(f) if digest: return digest[:8] return f"{f['size']}b" if f.get("size") else "" keys = [ (b, _version_key(f)) if b in repeated or _first_hash_value(f) else (b, "") for f, b in zip(files, bases) ] used: dict[str, int] = {} anchors = [] for f, (base, key) in zip(files, keys): anchor = f"{base}-{key}" if key else base if keys.count((base, key)) > 1 and f.get("description"): described = re.sub(r"[^a-z0-9]+", "-", str(f["description"]).lower()).strip("-") anchor = f"{anchor}-{described[:40].strip('-')}" used[anchor] = used.get(anchor, 0) + 1 if used[anchor] > 1: anchor = f"{anchor}-{used[anchor]}" anchors.append(anchor) return anchors def emulator_file_aliases(files: list[dict], anchors: list[str]) -> list[str]: """The plain file- id each card also answers to, or "". A link cited as emulators/raze/#file-duke3d-grp must still land once the card is addressed by its hash: the first card of that path keeps the plain id, unless another card already is it. """ taken = set(anchors) aliases = [] for f, anchor in zip(files, anchors): base = "file-" + ( re.sub( r"[^a-z0-9]+", "-", str(f.get("path") or f.get("standalone_path") or f.get("name", "")).lower(), ).strip("-") or "entry" ) if base in taken or anchor == base: aliases.append("") else: taken.add(base) aliases.append(base) return aliases # A commit id: hexadecimal with a letter and a digit, 7 to 40 long. A version # that holds one (0.8.139-2-g25e1ee2fd5) names no release. _COMMIT_ID = re.compile( r"(? dict[str, str]: """The `core_version` of a profile by build mode ("" when one covers every build), kept only where it names a release: a number and no commit id. A commit id of digits only reads as a number, so a run that starts one of the profile's own pins is one too (supermodel3's "0.3a-1481545-mm").""" pins = [str(value).casefold() for field in ("source_commit", "upstream_commit") for value in (profile[field].values() if isinstance(profile.get(field), dict) else [profile.get(field)]) if value] def names_a_commit(value: str) -> bool: return bool(_COMMIT_ID.search(value)) or any( pin.startswith(run) for run in re.findall(r"[0-9a-f]{7,40}", value.casefold()) for pin in pins) stated = profile.get("core_version") pairs = stated.items() if isinstance(stated, dict) else [("", stated)] versions = {str(mode): str(value or "").strip() for mode, value in pairs} return {mode: value for mode, value in versions.items() if re.search(r"[0-9]", value) and not names_a_commit(value)} def emulator_structured_data(profile: dict) -> dict: """What an emulator page states, for a reader of its JSON-LD. The block said only the page name: a crawler could not learn which files the core loads, their hashes, the source read or when. """ data: dict = {} # Every repository the profile reads. The source repository carries the # release the profile describes, never a commit id: a pin is no version. # The upstream of a port is another code base and states none. versions = release_versions(profile) about = [] seen: set[str] = set() for field in ("source", "upstream"): urls = profile.get(field) for mode, url in (urls.items() if isinstance(urls, dict) else [("", urls)]): if not url or not str(url).startswith("http") or str(url) in seen: continue seen.add(str(url)) entry = { "@type": "SoftwareSourceCode", "name": str(profile.get("emulator", "")), "codeRepository": str(url), } version = (versions.get(mode) or versions.get("")) if field == "source" else None if version: entry["version"] = version about.append(entry) if about: data["about"] = about[0] if len(about) == 1 else about if profile.get("profiled_date"): data["dateModified"] = str(profile["profiled_date"]) parts = [] files = [f for f in profile.get("files") or [] if isinstance(f, dict)] for f, anchor in zip(files, emulator_file_anchors(files)): if not f.get("name"): continue # One identifier per accepted value: "sha256:a,b,c,d" matched no # exact search (mesence's BS-X.bin). identifiers = [ f"{field}:{value}" for field in ("sha1", "sha256", "md5", "crc32") for value in _hash_values(f.get(field)) ] part = { "@type": "CreativeWork", "name": str(f.get("path") or f["name"]), # The card's own anchor, as system and platform parts carry. "url": f"#{anchor}", } if identifiers: part["identifier"] = identifiers # An archive is identified by its members: MAME and FBNeo hash them, # never the ZIP, and 1 338 archive parts had no identifier at all. members = [ { "@type": "CreativeWork", "name": str(member.get("name", "")), "identifier": [ f"{field}:{value}" for field in ("sha1", "crc32") for value in _hash_values(member.get(field)) ], } for member in f.get("contents") or [] if isinstance(member, dict) and member.get("name") and (member.get("sha1") or member.get("crc32")) ] if members: part["hasPart"] = members parts.append(part) if len(parts) > SYSTEM_PARTS_IN_FULL: # MAME's 1,564 entries and their members were 614 KB of JSON-LD: past # the cap the members are left to the profile's JSON. for part in parts: part.pop("hasPart", None) if parts: data["hasPart"] = parts return data # Working notes kept under docs/ during development, never part of the site, # and no generated page lives in a directory of these names. LOCAL_NOTE_DIRS = ("plans", "specs") # The funnel: who each page is written for, from someone who knows nothing # (start) to someone adding to the project (contribute). The label sits above # the title, and the search ranks the first two levels higher. LEVELS = { "start": "Start here", "setup": "Setup guide", "explore": "Details", "reference": "Reference", "contribute": "For contributors", } # How much the site search weighs the words of a page (Pagefind's # data-pagefind-weight, 1 by default). Each value was measured on the # queries of website/tools/search_gate.py. # - Start here pages answer a beginner's words first. # - A system page holds the file cards themselves: a file name finds its # card before a guide that cites the name as an example. # - A setup guide is found by its name (the page title weighs it). # - A guide behind the first two levels (reference, options, contributing) # names everything a reader types, and ranks after the page that answers. # - A launcher, an alias or a frozen copy answers a name after the emulator. # - A part of a split page (a console's files, 5 of 9) holds cards, each found # by its own heading, and ranks after the page that answers the question. SEARCH_WEIGHT = {"start": 10, "setup": 1.5, "reference": 0.5, "contribute": 0.5, "part": 0.5} SYSTEM_PAGE_WEIGHT = 5 DEEP_GUIDE_WEIGHT = 0.5 SEARCH_DEMOTED: set[str] = set() # A tool page a reader reaches by a task, not its name: "check my dump", # "md5 check" found emulator pages and never the page that checks a file. SEARCH_WEIGHT_OF_PAGE = {"files/index.md": 2} DEMOTED_WEIGHT = 0.5 def search_weight(relative: Path, level: str) -> float | None: """The search weight of a page, None for the default. A part page states its own in its front matter: search_weight(path, "part").""" if level == "part": return SEARCH_WEIGHT["part"] if relative.as_posix() in SEARCH_WEIGHT_OF_PAGE: return SEARCH_WEIGHT_OF_PAGE[relative.as_posix()] if relative.as_posix() in SEARCH_DEMOTED: return DEMOTED_WEIGHT if relative.parts[0] == "systems": return SYSTEM_PAGE_WEIGHT if relative.parts[0] == "wiki" and level not in ("start", "setup"): return DEEP_GUIDE_WEIGHT return SEARCH_WEIGHT.get(level) _REFERENCE_PAGES = { "gaps.md", "cross-reference.md", "provenance.md", "exports.md", "data.md", "files/index.md", "wiki/data-model.md", sitemeasurements.EXPLORER_PAGE, } def page_level(relative: Path) -> str: """The level of a generated page. A wiki page states its own.""" path = relative.as_posix() if path in ("index.md", "which-pack.md"): return "start" if relative.parts[0] in ("platforms", release_record.GET_DIR): return "setup" if relative.parts[0] in ("systems", "emulators") or path == "explore.md": return "explore" if path in _REFERENCE_PAGES: return "reference" if path == "contributing.md": return "contribute" return "" def level_label(level: str) -> str: """The visible label of a level, plain text for the theme to style.""" return f'{LEVELS[level]}' def _front_matter(markdown: str) -> dict: """The YAML front matter of a page, empty when it has none.""" if not markdown.startswith("---\n"): return {} end = markdown.find("\n---\n", 4) if end == -1: return {} data = yaml.safe_load(markdown[4:end]) return data if isinstance(data, dict) else {} _SHA1_BY_PREFIX: dict[str, str] = {} def _cards_elsewhere(markdown: str) -> str: """A start line links each file to its card. A console split over part pages keeps its cards on the parts, so a card this page does not hold is reached by its permanent link instead.""" links = set(re.findall(r"\]\(#(sha1-[0-9a-f]{12})\)", markdown)) away = {ident for ident in links if f'id="{ident}"' not in markdown} if not away: return markdown if not _SHA1_BY_PREFIX: db = load_database(str(_REPO_ROOT / "database.json")) _SHA1_BY_PREFIX.update({sha1[:12]: sha1 for sha1 in db["files"]}) for ident in away: full = _SHA1_BY_PREFIX[ident.removeprefix("sha1-")] markdown = markdown.replace(f"](#{ident})", f"]({markup.site_href(f'files/?sha1={full}')})") return markdown def decorate_markdown_pages(docs: Path) -> dict[str, str]: """Add per-page descriptions and machine-readable structured data. Under Starlight each page is written in its final form (front matter, site addresses, pinned heading ids) and the dates a page states are returned by address, for the sitemap. """ lastmod: dict[str, str] = {} terms = sitestarlight.glossary_terms(_REPO_ROOT / WIKI_SRC_DIR / "glossary.md") # The console words of every system page, decorated or not: a console # another page's words cover gets a search field of its own. console_words: dict[str, set[str]] = {} for path in sorted((docs / "systems").glob("*.md*")): text = path.read_text(encoding="utf-8") words = sitestarlight.page_console_words(_front_matter(text), text) if words: console_words[path.relative_to(docs).with_suffix("").as_posix()] = words for path in sorted(docs.rglob("*.md")): relative = path.relative_to(docs) if any(part in LOCAL_NOTE_DIRS for part in relative.parts[:-1]): continue original = path.read_text(encoding="utf-8") authored = _front_matter(original) if "head" in authored: # Written in its final form by an earlier pass and rewritten by # nothing since: a second pass would find no title to read. continue markdown = _cards_elsewhere(prepare_page(original, relative)) for image_id in unresolved_images(markdown): print(f" WARNING: {relative}: no diagram named {image_id!r}", file=sys.stderr) stripped = _undecorated(markdown) if stripped is not markdown: markdown = stripped elif authored: # A wiki page states its own level: keep it, wrap the body. markdown = markdown[markdown.find("\n---\n", 4) + 5:].lstrip("\n") level = authored.get("level") or page_level(relative) title = _page_title(markdown, relative) browser_title = _browser_title(relative, title) description = str(authored.get("description") or _page_description(markdown, relative, title)) if relative.name == "index.md": page_path = "" if len(relative.parts) == 1 else "/".join(relative.parts[:-1]) + "/" else: page_path = relative.with_suffix("").as_posix() + "/" # Encoded as mkdocs encodes the canonical link and the sitemap: # idtech4a++ named .../idtech4a++/ here and .../idtech4a%2B%2B/ there. canonical = urllib.parse.urljoin(SITE_URL, urllib.parse.quote(page_path)) schema_type = "WebSite" if relative.as_posix() == "index.md" else "TechArticle" if relative.as_posix() == "data.md": schema_type = "Dataset" structured = { "@context": "https://schema.org", "@type": schema_type, "name": title, "description": description, "url": canonical, "isPartOf": { "@type": "WebSite", "name": SITE_NAME, "url": SITE_URL, }, **PAGE_DATA.get(relative.as_posix(), {}), } if schema_type == "WebSite": # The home page is the site: named as the site, part of nothing. structured["name"] = SITE_NAME del structured["isPartOf"] target, text = sitestarlight.finalize_page( relative.as_posix(), markdown, level=level, weight=search_weight(relative, level), browser_title=browser_title, description=description, structured=structured, terms=terms, search=authored.get("search") != "exclude", logo=str(authored.get("logo") or ""), parent=authored.get("parent"), authored=authored, words_by_page=console_words, ) write_decorated(docs / target, text) # One file per page: a page that gained or lost its components # changed extension, and the other one would claim its address. stem = relative.with_suffix("").as_posix() for suffix in (".md", ".mdx"): if f"{stem}{suffix}" != target: (docs / f"{stem}{suffix}").unlink(missing_ok=True) if structured.get("dateModified"): lastmod[sitestarlight.page_url(target)] = str(structured["dateModified"]) return lastmod # Platform pages def generate_platform_index(coverages: dict, registry: dict | None = None) -> str: """The Your setup tab: every platform by the device it runs on, and one line to the table that sets them side by side (setup_table), which is reference matter and sits with how each setup checks its files.""" registry = registry or {} lines = [ "# Your setup", "", "Each setup's page gives the folder and the command.", "", ] for group, (label, _description) in DEVICE_GROUPS.items(): members = _group_members(coverages, registry, group) if not members: continue # The group label names the devices, and each card carries its # setup's own sentence. The kinds of device that run a setup # follow, as on the home tiles. lines += [f"## {label}", ""] lines += markup.card_grid([ *((f"[{coverages[name]['platform']}]({name}.md)", [GUIDES[name].summary]) for name in members), *((f"[{tile.name}](../{tile.link})", [tile.text]) for tile in GROUP_TILES.get(group, ())), ]) lines += [ f"Comparing setups? [Every setup side by side](../{SETUP_TABLE_LINK})", "", ] return "\n".join(lines) + "\n" # Where the side-by-side table of setups is published: with how each setup # checks its files, the column a reader compares. SETUP_TABLE_PAGE = "wiki/verification-modes.md" SETUP_TABLE_ANCHOR = "every-setup-at-a-glance" SETUP_TABLE_LINK = f"{SETUP_TABLE_PAGE}#{SETUP_TABLE_ANCHOR}" # How a setup checks a file, in the words of a reader. _HOW_IT_CHECKS = {"existence": "name only", "md5": "MD5", "sha1": "SHA-1"} def setup_table(coverages: dict, registry: dict | None = None) -> list[str]: """Every setup side by side: the files on its own list, the files in its ZIP, and how it checks them. Written under the Reference level, on the page that explains each way of checking.""" registry = registry or {} lines = [ f"## Every setup at a glance {{ #{SETUP_TABLE_ANCHOR} }}", "", "| Setup | Files on its own list | Files in the ZIP | How it checks |", "|-------|------:|------:|---------------|", ] archived = False for name, cov in sorted(coverages.items(), key=lambda item: item[1]["platform"].casefold()): status = registry.get(name, {}).get("status") archived |= status == "archived" listed = f"{cov['total']:,}" if cov["total"] else "No list of its own" files, _size = release_totals(name, release_record.load_advertised()) zipped = (f"{files:,}" if files else "No ZIP yet" if has_release_pack(name) else "No ZIP") lines.append( f"| [{cov['platform']}](../platforms/{name}.md)" f"{' (archived)' if status == 'archived' else ''} | {listed} | {zipped} | " f"{_HOW_IT_CHECKS.get(cov['mode'], cov['mode'])} |" ) lines += [ "", "The ZIP adds the files its emulators load to the files on the setup's own list.", "", ] lines += ["No ZIP: the setup is served by the command alone. " f"[Why?](../{NO_ZIP_FAQ})", ""] if archived: lines += ["An archived setup keeps its page and the command. Its list is no longer " "read again on a schedule.", ""] return lines def with_setup_table(page: str, coverages: dict, registry: dict | None = None) -> str: """The comparison table placed before the Next steps of its page, or at its end.""" table = "\n".join(setup_table(coverages, registry)) marker = "\n## Next steps" if marker in page: head, tail = page.split(marker, 1) return f"{head.rstrip()}\n\n{table}{marker}{tail}" return f"{page.rstrip()}\n\n{table}" # A platform's file lists are split by the share of a page they render to, # in thousandths: their elements against validate_site's MAX_PAGE_ELEMENTS, # their bytes against the 600,000 of test_site_r2_gate, whichever is # larger. Measured in Markdown characters, the lists split anew whenever # their markup changed: rows written as tables moved 38 more systems to # pages of their own. PAGE_ELEMENTS = 10_000 PAGE_BYTES = 600_000 # A system whose list passes this share gets a page of its own. PLATFORM_SECTION_BUDGET = 400 # A system's own page in runs of at most this share. PLATFORM_PART_BUDGET = 700 # The share of a platform page's file lists, past which the largest # systems move to pages of their own. PLATFORM_REFERENCE_BUDGET = 600 # Every part page a platform list has had, with the system it held: # platforms/batocera-atari-st.md -> platforms/batocera.md#atari-st. When a # budget or a list changes, a part can fold back into its platform page, # and its address then redirects there instead of ending on the 404 page. PLATFORM_PARTS_FILE = (Path(__file__).resolve().parent.parent / "website" / "src" / "data" / "platform-parts.json") def _published_platform_parts() -> dict[str, str]: if not PLATFORM_PARTS_FILE.is_file(): return {} return json.loads(PLATFORM_PARTS_FILE.read_text(encoding="utf-8")) # A list moved off a reference page (gaps, provenance) is cut in parts of # at most this much Markdown: about 450 KB of HTML each. LIST_PART_BUDGET = 300_000 def _characters(lines: list[str]) -> int: return sum(len(line) + 1 for line in lines) def _blocks_in_runs(blocks: list[list[str]], budget: int, weigh: Callable[[list[str]], int] = _characters) -> list[list[str]]: """Blocks (table rows, folded lists) in runs of at most ``budget``, by default in characters, a block never cut.""" runs: list[list[str]] = [[]] size = 0 for block in blocks: weight = weigh(block) if runs[-1] and size + weight > budget: runs.append([]) size = 0 runs[-1].extend(block) size += weight return runs def _parent_front_matter(parent: tuple[str, str] | None) -> list[str]: """The `parent:` front matter of a part page: the page it belongs to, by label and site path, which the breadcrumb and sidebar read.""" if parent is None: return [] label, link = parent return ["parent:", f" label: {json.dumps(label)}", f" link: {json.dumps(link)}"] def _list_pages( stem: str, title: str, intro: str, head: list[str], blocks: list[list[str]], moved: dict[str, str], budget: int = LIST_PART_BUDGET, parent: tuple[str, str] | None = None, ) -> str: """Pages of their own for a long list: ``head`` (a table header, say) opens each part, ``blocks`` are cut between parts. Returns where the list went, as the page it left says it. ``parent`` (label, site path) is the page the parts belong to, for the breadcrumb.""" runs = _blocks_in_runs(blocks, budget) stems = [stem + (f"-{n}" if n > 1 else "") for n in range(1, len(runs) + 1)] for n, (here, run) in enumerate(zip(stems, runs), 1): part = f" (part {n} of {len(runs)})" if len(runs) > 1 else "" # The page description is the opening of the intro: parts sharing # one read as duplicates to search and to validate_site. opening = f"Part {n} of {len(runs)}. {intro}" if len(runs) > 1 else intro moved[here] = "\n".join([ "---", "level: reference", *_parent_front_matter(parent), "---", "", f"# {title}{part}", "", opening, "", *([", ".join(f"[part {i}]({other}.md)" if other != here else f"part {i}" for i, other in enumerate(stems, 1)), ""] if len(runs) > 1 else []), *head, *run, "", _date_line(), ]) + "\n" if len(stems) == 1: return f"[a page of their own]({stems[0]}.md)" return "pages of their own: " + ", ".join( f"[part {i}]({other}.md)" for i, other in enumerate(stems, 1)) @functools.lru_cache(maxsize=1) def _collection_sha1_by_path() -> dict[str, str]: """The sha1 of each file the collection holds, by its path.""" with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle: files = json.load(handle).get("files", {}) return {str(entry.get("path", "")): sha1 for sha1, entry in files.items()} def _system_heading_link(sys_id: str, prefix: str = "") -> str: """A system's name, linked to its own console section when it has one. The id stays beside it in small text: it is what the platform's list writes. A system with no section of its own is named, never sent to the page a storage folder chose. """ return f"{_system_named_link(sys_id, prefix)} {_escape_tags(sys_id)}" def _system_named_link(sys_id: str, prefix: str = "") -> str: """A system's name, linked to the section system_section gives.""" from siterender import system_section name = _escape_tags(system_name(sys_id)) section = system_section(sys_id) if section is not None: slug, console = section name = f"[{name}]({console_href(slug, console, prefix)})" return name def _emulator_named_link(key: str, prefix: str = "") -> str: """An emulator by the name its page carries, linked to that page. A profile with no page (an alias, a test) is named only.""" profile = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False).get(key, {}) label = _escape_tags(str(profile.get("emulator") or key)) if profile.get("type") in ("alias", "test"): return label return f"[{label}]({prefix}emulators/{key}.md)" def _served_line(f: dict) -> str: """Which collection file answers the entry, from the resolver's result. "OK" with no path gave an expert nothing to check: Batocera's scph1001.bin is served by bios/Sony/PlayStation/scph1001_v20.bin. """ path = str(f.get("path") or "") if f.get("status") == "missing": return "Not held" if not path: # A slot the pack fills from a release asset or leaves to the user. return "" sha1 = _collection_sha1_by_path().get(path, "") label = "Served by" if f.get("status") == "ok" else "Held, not the declared hash" parts = path.split("/") if not sha1 or len(parts) < 3 or parts[0] != "bios": return f"{label}: {_code(path)}" # The card of that copy: every hash, provenance and the download. page = urllib.parse.quote(card_page(path, sha1)) return (f"{label}: [{_code(path)}](../systems/{page}.md#sha1-{sha1[:12]}) " f"(sha1 `{sha1[:8]}`)") # Platforms whose launcher records are read from the repository and revision # of their BIOS list: batocera_scraper reads configgen at its tag, # retrobat_scraper reads emulatorLauncher at the list's commit # (launcher_revision). The others read another repository. _LAUNCHERS_IN_LIST_REPO = frozenset({"batocera", "retrobat"}) def _launcher_refs(platform: str, config: dict) -> dict[str, list[str]]: """Entry name, casefolded -> each launcher record that opens it, rendered.""" from launchers import _names as launcher_names from sitesources import launcher_ref_markdown, list_source source = (list_source(str(config.get("source") or "")) if platform in _LAUNCHERS_IN_LIST_REPO else None) found: dict[str, list[str]] = {} for record in config.get("launchers") or []: if not isinstance(record, dict) or not record.get("source_ref"): continue text = (f"{_escape_tags(str(record.get('emulator', '')))} at " f"{launcher_ref_markdown(source, str(record['source_ref']))}") for name in launcher_names(record): names = found.setdefault(name.rsplit("/", 1)[-1].casefold(), []) if text not in names: names.append(text) return found # The hash a row shows first, when the list does not say which it checks. _ROW_HASHES = ("sha1", "md5", "sha256", "crc32") # An untested entry whose reason is a hash the bytes do not match. _WRONG_HASH = re.compile(r"hash|mismatch|expected|must match", re.IGNORECASE) # A list this long gets a filter box above it, as a table of RECORD_ROWS # rows does (website/src/plugins/rehype-stack-tables.mjs). FILTER_ROWS = 20 def _status_chip(f: dict) -> str: """The row's status as a chip, its word and its colour saying the same.""" status = f.get("status") if status == "ok": word, tone = "OK", "success" elif status == "missing": word, tone = "Missing", "danger" elif status == "untested": word = "Wrong hash" if _WRONG_HASH.search(str(f.get("reason") or "")) else "Untested" tone = "warning" else: word, tone = str(status), "muted" return f'{html.escape(word)}' def _copy_hash(algorithm: str, value: str | list) -> str: """A declared hash cut to SHORT_HASH characters on a button that copies it whole. A list of accepted values (Recalbox) copies its first. Outside a form a button submits nothing, so it carries no type.""" from hashing import parse_hash_list # noqa: PLC0415 values = parse_hash_list(value) if not values: return "" first = html.escape(values[0]) more = f" +{len(values) - 1}" if len(values) > 1 else "" return (f'') # Characters Markdown or MDX would read inside the raw HTML of a row: an # MDX page parses the text of its HTML as Markdown, a .md page does not. _INERT = str.maketrans({"*": "*", "_": "_", "`": "`", "[": "[", "]": "]"}) _INLINE_MD = re.compile( r"\[(?P