#!/usr/bin/env python3 """Generate the site from database.json, platform configs, and emulator profiles. The pages, navigation and data files are written into the Astro Starlight project in website/, which builds the site. Reads the same data sources as verify.py and generate_pack.py to produce a complete documentation site. Zero manual content. Usage: python scripts/generate_site.py python scripts/generate_site.py --db database.json --platforms-dir platforms """ from __future__ import annotations from siterender import SECTION_LABELS, WIKI_SRC_DIR import argparse import csv import hashlib import functools import html import io import json import os import re import shlex import shutil import sqlite3 import sys import urllib.error import urllib.parse import urllib.request import zipfile from collections.abc import Callable, Iterator from datetime import datetime, timezone from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from common import ( upstream_profile_index, build_zip_contents_index, GAME_DATA_TOPS, compute_composition, count_catalog_matched, list_registered_platforms, list_release_pack_platforms, load_platform_registry, load_database, load_emulator_profiles, load_platform_config, load_provenance_snapshots, native_layer, require_yaml, resolution_is_hash_exact, resolve_local_file, resolve_platform_cores, unique_emulator_profiles, yaml_load, ) from artifacts import ArtifactLockBusy, artifact_lock from cross_reference import FileConsumers, held_copy from nativemode import digest_algorithm, reads_file_contents yaml = require_yaml() from generate_readme import ( compute_coverage, platform_coverages, NO_PACK, asset_links, contributing_text, download_links, extract_notes, extract_targets, release_totals, ) from provenance_report import build_report, distinct_missing import sitemarkup as markup import sitestarlight from siteexports import generate_exports_page, load_export_reports from siteguides import ( ALSO_RUNS, DEVICE_GROUPS, GROUP_ANCHORS, GROUP_TILES, GUIDES, HANDHELD_GUIDE, Guide, HandheldGuide, HandheldSystem, FENCE_TITLES, INSTALL_BASE, TERMINALS, TRUST_ITEMS, detected_systems as _detected_systems, fence, folder_list, install_command, setup_count, platform_args, terminal_tabs, trust_box, ) import release_record # The Starlight project, and where its pages go inside it. STARLIGHT_DIR = "website" STARLIGHT_CONTENT = "src/content/docs" SITE_NAME = "RetroBIOS" REPO_URL = "https://github.com/Abdess/retrobios" RELEASE_URL = f"{REPO_URL}/releases/latest" # What LICENSE grants over the published data: its MIT terms name the # metadata. The BIOS files themselves are under NOTICE, never this. DATA_LICENSE = "MIT (LICENSE: scripts, tooling, schemas, documentation and metadata)" SITE_URL = "https://abdess.github.io/retrobios/" GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads", "files", release_record.GET_DIR] # Generated directories of files served as they are, not pages. PUBLIC_DIRS = ("api", "downloads") def _content_check_ceiling(profiles: dict) -> str: """How far content checking can reach across every profiled entry.""" hashed = sized = neither = 0 for profile in unique_emulator_profiles(profiles).values(): for f in profile.get("files", []) or []: if any( f.get(k) for k in ("md5", "sha1", "crc32", "sha256", "known_hash_adler32") ): hashed += 1 elif any(f.get(k) for k in ("size", "min_size", "max_size")): sized += 1 else: neither += 1 total = hashed + sized + neither if not total: return "" return ( f"Across every profiled entry, {hashed:,} of {total:,} " f"({hashed / total * 100:.0f}%) carry a hash the code checks, " f"{sized:,} ({sized / total * 100:.0f}%) only a size, and " f"{neither:,} ({neither / total * 100:.0f}%) neither. That last share " "is the ceiling of the method: where an emulator validates nothing, " "reading its source establishes which file it loads, never whether " "the content is the right dump. The provenance field answers that " "other question, for the files a dump catalog indexes." ) CLS_LABELS = { "official_port": "Official ports", "community_fork": "Community forks", "pure_libretro": "Pure libretro", "game_engine": "Game engines", "enhanced_fork": "Enhanced forks", "frozen_snapshot": "Frozen snapshots", "embedded_hle": "Embedded HLE", "launcher": "Launchers", "unclassified": "Unclassified", "other": "Other", } # The same classes said of one emulator: its page read "Classification: # Official ports". CLS_LABEL_ONE = { "official_port": "Official port", "community_fork": "Community fork", "pure_libretro": "Pure libretro", "game_engine": "Game engine", "enhanced_fork": "Enhanced fork", "frozen_snapshot": "Frozen snapshot", "embedded_hle": "Embedded HLE", "launcher": "Launcher", "unclassified": "Unclassified", "other": "Other", } def _copy_places(db: dict) -> dict[str, set[tuple[str, str, str]]]: """sha1 -> every (manufacturer, console, file name) holding a copy. The database keeps one path per content, deduplication having left one copy. by_path_suffix still records the other folders a copy is filed in, under the name that folder gives it. """ files = db.get("files", {}) bios_dir = Path(str(db.get("bios_dir") or "bios")) places: dict[str, set[tuple[str, str, str]]] = {} for sha1, entry in files.items(): parts = str(entry.get("path", "")).split("/") if len(parts) >= 3 and parts[0] == "bios": places.setdefault(sha1, set()).add((parts[1], parts[2], parts[-1])) for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items(): parts = tail.split("/") if len(parts) < 3 or parts[1].startswith(".") or len(sha1s) != 1: continue if sha1s[0] in files and (bios_dir / tail).is_file(): places.setdefault(sha1s[0], set()).add((parts[0], parts[1], parts[-1])) return places # bios/Other/ and bios/Arcade/ file a copy by # who loads it, not by machine. A copy also filed under a machine's maker # places the system there. _BUCKET_FOLDERS = frozenset({"Other", "Arcade"}) def _slug(manufacturer: str) -> str: return manufacturer.lower().replace(" ", "-") def _names_manufacturer(sys_id: str, manufacturer: str) -> bool: slug = _slug(manufacturer) return sys_id == slug or sys_id.startswith(f"{slug}-") def _copies_by_path(db: dict) -> dict[str, set[tuple[str, str, str]]]: """The copy places of each content, keyed by the path the database keeps.""" files = db.get("files", {}) return { str(files[sha1].get("path", "")): where for sha1, where in _copy_places(db).items() } def _system_folder( files: list[dict], db: dict, name_consoles: dict[str, set[tuple[str, str]]], sys_id: str = "", copies: dict[str, set[tuple[str, str, str]]] | None = None, ) -> tuple[str, str] | None: """The bios// folder a system's files live in. A content votes for every folder holding a copy under the name the system declares, not only the one deduplication kept: disksys.rom is kept as Arcade/FBNeo/fdsbios.nes and filed again under Nintendo/Famicom Disk System. A copy filed under another name, outside the manufacturer the id names, is another machine's file and votes only when no copy carries the declared name. An id that names its manufacturer (nintendo-fds) keeps the votes cast inside that manufacturer when there are any, and a maker's folder outranks the folders that file copies by emulator. """ if copies is None: copies = _copies_by_path(db) votes: dict[tuple[str, str], int] = {} for fe in files: local, status = resolve_local_file(fe, db) if not local or not resolution_is_hash_exact(status): continue held = copies.get(local, set()) if not held: parts = local.split("/") if len(parts) >= 4 and parts[0] == "bios": held = {(parts[1], parts[2], parts[-1])} declared = str(fe.get("name", "")).casefold() named = { (m, c) for m, c, base in held if base.casefold() == declared or _names_manufacturer(sys_id, m) } for place in named or {(m, c) for m, c, _ in held}: votes[place] = votes.get(place, 0) + 1 if votes: own = [place for place in votes if _names_manufacturer(sys_id, place[0])] makers = [place for place in votes if place[0] not in _BUCKET_FOLDERS] candidates = own or makers or list(votes) return max(sorted(candidates), key=lambda place: votes[place]) for fe in files: places = name_consoles.get(fe.get("name", ""), set()) if len(places) == 1: return next(iter(places)) return None def _folders_by_name(manufacturers: dict) -> dict[str, set[tuple[str, str]]]: """Normalized system name -> the console folders that spell it. nintendo-nes, nes and the NES folder normalize alike, as do sega-mastersystem and Sega/Master System. """ from common import _norm_system_id index: dict[str, set[tuple[str, str]]] = {} for mfr, consoles in manufacturers.items(): for console in consoles: spelled = console.lower().replace(" ", "-") for key in { _norm_system_id(f"{_slug(mfr)}-{spelled}"), _norm_system_id(spelled) }: index.setdefault(key, set()).add((mfr, console)) return index def _folder_named(sys_id: str, by_name: dict[str, set[tuple[str, str]]]) -> tuple[str, str] | None: from common import _norm_system_id places = by_name.get(_norm_system_id(sys_id), set()) return next(iter(places)) if len(places) == 1 else None def _system_pages_from_data( manufacturers: dict, coverages: dict, db: dict, profiles: dict | None = None, ) -> dict[str, tuple[str, str]]: """The system_id -> (manufacturer_slug, console_name) table. Uses platform file paths to trace system_id -> bios directory -> manufacturer page. """ # Every console folder a filename sits in. A name in one folder only can # place a system, while a name in several says nothing (MiSTer's boot.rom lives # under a dozen consoles, and the last one written used to win). name_consoles: dict[str, set[tuple[str, str]]] = {} for mfr, consoles in manufacturers.items(): for console, entries in consoles.items(): for entry in entries: name_consoles.setdefault(entry["name"], set()).add((mfr, console)) # Build normalized console name index for fuzzy matching console_norm: dict[str, tuple[str, str]] = {} for mfr, consoles in manufacturers.items(): slug = mfr.lower().replace(" ", "-") mfr_norm = mfr.lower().replace(" ", "-") for console in consoles: norm = console.lower().replace(" ", "-") entry = (slug, console) console_norm[norm] = entry console_norm[f"{mfr_norm}-{norm}"] = entry # Short aliases: strip common manufacturer prefix words for prefix in ( f"{mfr_norm}-", "nintendo-", "sega-", "sony-", "snk-", "nec-", ): if norm.startswith(prefix.replace(f"{mfr_norm}-", "")): pass # already covered by norm key = f"{prefix}{norm}" console_norm[key] = entry # Map system_id -> (manufacturer, console) via platform file entries copies = _copies_by_path(db) folders_by_name = _folders_by_name(manufacturers) pages: dict[str, tuple[str, str]] = {} for cov in coverages.values(): config = cov["config"] for sys_id, system in config.get("systems", {}).items(): if sys_id in pages: continue # Strategy 1: the one console folder the id spells (nintendo-nes, # NES). A copy of the disk system BIOS in the NES list does not # move the NES to the folder that copy sits in. # Strategy 2: the folder holding the files the system declares, # found by content, and most votes win. A bare name stands in only # where the collection files it in a single console. place = _folder_named(sys_id, folders_by_name) or _system_folder( system.get("files", []), db, name_consoles, sys_id, copies ) if place is not None: mfr, console = place pages[sys_id] = (mfr.lower().replace(" ", "-"), console) continue # Strategy 3: fuzzy match system_id against console directory names if sys_id in console_norm: pages[sys_id] = console_norm[sys_id] else: # Try partial match: "nintendo-wii" matches "Wii" under "Nintendo" parts = sys_id.split("-") for i in range(len(parts)): suffix = "-".join(parts[i:]) if suffix in console_norm: pages[sys_id] = console_norm[suffix] break # Systems only emulator profiles name (sega-naomi, commodore-pet): placed # by the files the profile files under them, as a platform's are. A # profile of one system files all of its entries there. for profile in (profiles or {}).values(): systems = [str(sid) for sid in profile.get("systems") or []] for sys_id in systems: if sys_id in pages: continue files = [ entry for entry in profile.get("files") or [] if isinstance(entry, dict) and (entry.get("system") == sys_id or len(systems) == 1) ] place = _folder_named(sys_id, folders_by_name) or _system_folder( files, db, name_consoles, sys_id, copies ) if place is not None: mfr, console = place pages[sys_id] = (mfr.lower().replace(" ", "-"), console) elif sys_id in console_norm: pages[sys_id] = console_norm[sys_id] return pages def _timestamp() -> str: """The build time for data files (stats, API records), ISO 8601.""" return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def data_as_of(moment: datetime | None = None) -> str: """The build date for a reader, in words, with the machine form beside it. A raw ISO stamp ("2026-10-22T22:43:08Z") closed the home page.""" moment = moment or datetime.now(timezone.utc) return (f'") def _date_line() -> str: """The date a page closes on, in the words of data_as_of().""" return f'
{data_as_of()}.
' # Home page def _stat(value: str, label: str) -> list[str]: return [ markup.block_open("div", 'class="rb-stat"'), f'{value}', f'{label}', "", "", ] def _group_members(coverages: dict, registry: dict, group: str) -> list[str]: """Registered platforms of a device group, by display name.""" return sorted( (name for name in coverages if registry.get(name, {}).get("device_group") == group), key=lambda name: coverages[name]["platform"].casefold(), ) def _next_block(links: list[str]) -> list[str]: """Where to go from here, at the end of a page: a plain heading and a list. A tinted callout with a side stripe said the same as the pager.""" return ["## Next steps", "", *(f"- {link}" for link in links), ""] def _tiles(registry: dict, members: list[str], display: dict[str, str], prefix: str = "", group: str = "", eager: int = 0) -> list[str]: """The setup tiles of one device group: logo, name and the hardware it runs on, then the group's other tiles: a kind of device that runs one of the setups (Android handhelds, Linux handhelds).""" return markup.platform_tiles([ *((display[name], f"{prefix}platforms/{name}.md", str(registry.get(name, {}).get("logo") or ""), GUIDES[name].runs_on or _runs_on(registry.get(name, {}), GUIDES[name])) for name in members), *((tile.name, f"{prefix}{tile.link}", "", tile.text) for tile in GROUP_TILES.get(group, ())), ], eager=eager) def setup_picker(registry: dict, display: dict[str, str], prefix: str = "") -> list[str]: """Every setup as a tile, under the device group it belongs to. The setups alone in their group share one row: "MiSTer FPGA" above "MiSTer FPGA" said nothing. Only the first row of logos loads with the page: the 19 logos were most of the bytes of the home page.""" lines: list[str] = [] alone: list[str] = [] for group, (label, _description) in DEVICE_GROUPS.items(): members = _group_keys(registry, group, display) if len(members) == 1 and not GROUP_TILES.get(group): alone += members elif members: eager = 0 if lines else markup.FIRST_ROW_TILES lines += [f"**{label}**", "", *_tiles(registry, members, display, prefix, group, eager)] if alone: eager = 0 if lines else markup.FIRST_ROW_TILES lines += ["**Other setups**", "", *_tiles(registry, alone, display, prefix, eager=eager)] return lines def generate_home( db: dict, coverages: dict, profiles: dict, registry: dict | None = None, platforms_dir: str = "platforms", ) -> str: """The page someone lands on knowing nothing: what a BIOS file is, whether they need one, and the three steps to get the right ones. The figures about the collection, the cards to the deeper pages and the call for help sat below the steps and doubled the page: they are on the Explore index, one tab further. No date and no "next" box: the hero buttons say where to go.""" registry = registry or load_platform_registry(platforms_dir) display = {key: load_platform_config(key, platforms_dir).get("platform", key) for key in registry} named = [n for n in ("RetroArch", "Batocera", "EmuDeck") if n in display.values()] others = setup_count() - len(named) works_with = ", ".join(named) + (f" and {others} more" if others else "") lines = [ "# BIOS files for your emulators", "", "Some consoles need a small program to start their games. It is called " "the **BIOS**. Your emulator needs a copy of it. We put the right copy " "in the right folder.", "", markup.button("Get your files", "which-pack.md", primary=True) + " " + markup.button("What is a BIOS?", "wiki/what-are-bios-files.md"), "", f"Free. No account. Works with {works_with}.", "", "- BIOS missing, even with the file there? " "[When a game does not start](wiki/troubleshooting.md)", "- Have a file? [Check it](files/index.md)", "", "## Do I need it?", "", "- **Yes**, if your emulator says a BIOS is missing.", "- **Yes** for PlayStation 1 and 2, Saturn and Neo Geo.", "- **Better with it** for Dreamcast: some games need it.", "- **Usually not** for NES, Super Nintendo, Mega Drive and Game Boy.", "", "Details: [Every console, one by one](systems/index.md)", "", "## Three steps", "", *markup.steps([ "**Pick your device.** Its page shows your BIOS folder.", "**Run one command.** Some setups have a ZIP.", "**Start a game.**", ]), "## Pick your device", "", *setup_picker(registry, display), "## Is it safe?", "", *(f"- {item}" for item in TRUST_ITEMS), "", "[How the command is checked](wiki/faq.md#is-piping-the-installer-into-a-shell-safe). " "Details: [Is it legal?](wiki/legal.md)", "", ] return "\n".join(lines) + "\n" def generate_explore_index(db: dict, coverages: dict, profiles: dict) -> str: """The Explore tab opens here: where to look for one console, one emulator or one installer option, and the figures of the collection.""" total_files = db.get("total_files", 0) total_size = db.get("total_size", 0) emulator_count = len(unique_emulator_profiles(profiles)) lines = [ "# Explore", "", "One console, one emulator, or one installer option: each has its page, " "with the files it needs and the code that reads them.", "", *markup.card_grid([ ("Systems", ["What each console and computer needs, file by file.", "", "[Browse the systems](systems/index.md)"]), ("Emulators", ["What each emulator reads, with the source code that reads it.", "", "[Browse the emulators](emulators/index.md)"]), ("Installer options", [ "Fewer files, one region, another folder, a check without download.", "", "[All installer options](wiki/installer.md)"]), ("How files are checked", [ "Each setup checks its files its own way: by name, by MD5 or by SHA1.", "", "[How each platform checks files](wiki/verification-modes.md)"]), ("Data and API", ["Every file and hash as JSON, CSV and SQLite.", "", "[Data and API](data.md)"]), ]), "## The collection", "", markup.block_open("div", 'class="rb-stats"'), "", *_stat(f"{total_files:,}", COLLECTION_FILES_LABEL), *_stat(str(setup_count()), "Setups supported"), *_stat(str(emulator_count), "Emulators documented"), *_stat(_fmt_size(total_size), "Size of the collection"), "", "", composition_sentence(db), "", "## Help the project", "", "A file you hold, an emulator profile or a correction fills a gap for " "every setup at once. For contributors: [How to contribute](contributing.md).", "", ] return "\n".join(lines) + "\n" def compute_stats(db: dict, coverages: dict, profiles: dict) -> dict: unique = unique_emulator_profiles(profiles) systems: set[str] = set() for p in unique.values(): systems.update(p.get("systems", [])) return { "schema_version": 1, "generated_at": _timestamp(), "files": db.get("total_files", 0), "size_bytes": db.get("total_size", 0), "platforms": len(coverages), "emulators": len(unique), "systems": len(systems), "catalog_matched": count_catalog_matched(db), "source": REPO_URL, "downloads": RELEASE_URL, } def _release_date(tag: str) -> str: """The date a release tag names (v2026.10.10), in words, or empty.""" match = re.fullmatch(r"v(\d{4})\.(\d{2})\.(\d{2})", tag or "") if not match: return "" moment = datetime(int(match[1]), int(match[2]), int(match[3])) return f"{moment.day} {moment.strftime('%B %Y')}" def _pack_version(pack_name: str) -> str: """The platform version a pack was built for, from its name (Batocera_43.1_BIOS_Pack.zip is 43.1).""" stem = pack_name.removesuffix("_BIOS_Pack.zip") return stem.rsplit("_", 1)[1] if "_" in stem else "" def release_line(platform_name: str, record: dict) -> list[str]: """Which release the buttons serve, with its checksums and signature, and the version gap when the page describes a newer platform than the pack.""" tag = record.get("tag") pack_name = release_record.pack_name_for(platform_name, record) if not tag or not pack_name: return [] base = f"{REPO_URL}/releases/download/{tag}" built = _release_date(tag) lines = [ f"Pack {tag}{f', built {built}' if built else ''}. " f"[SHA256SUMS.txt]({base}/SHA256SUMS.txt) and its " f"[signature]({base}/SHA256SUMS.txt.sig).", ] config = load_platform_config(platform_name, "platforms") page_version = str(config.get("version") or "") pack_version = _pack_version(pack_name) if page_version and pack_version and page_version.lstrip("v") != pack_version.lstrip("v"): display = config.get("platform", platform_name) lines.append(f"This pack was built for {display} {pack_version}. The command " f"installs the files for {display} {page_version}.") return lines # What a reader on another version of the setup does with the pack. Our data # says which list the pack was made from, not what an older list lacks. OTHER_VERSION = "Other version? Run `--check` after copying." def setup_version(platform_name: str, record: dict | None = None) -> str: """The setup and version whose BIOS list the pack was made from ("Batocera 43.1"): the version in the pack's name, else the version the platform YAML was read at. Empty when neither says, or when the platform has no YAML to read a version from.""" try: config = load_platform_config(platform_name, "platforms") except FileNotFoundError: return "" pack_name = release_record.pack_name_for(platform_name, record or {}) version = (_pack_version(pack_name) if pack_name else "") or str(config.get("version") or "") if not version: return "" return f"{config.get('platform', platform_name)} {version.removeprefix('v')}" def version_line(platform_name: str, record: dict, named: bool = False) -> str: """The version the pack fits, and what to do on another. *named* when the release line already named the pack's version.""" made_from = setup_version(platform_name, record) if named or not made_from: return OTHER_VERSION return f"Made for {made_from}. {OTHER_VERSION}" # Why a setup has no ZIP, answered once on the FAQ. NO_ZIP_FAQ = "wiki/faq.md#why-is-there-no-zip-for-my-setup" def has_release_pack(platform_name: str) -> bool: """Whether a release publishes a ZIP for the platform (`release_pack`).""" entry = load_platform_registry("platforms").get(platform_name) or {} return entry.get("release_pack") is True def no_zip_line(display: str) -> str: """The one line a setup without a release pack shows where the ZIP would be.""" return f"No ZIP for {display}. The command installs every file and checks it." def get_page_link(platform_name: str, prefix: str = "") -> str: """The platform's permanent download page, as a site link.""" return f"{prefix}{release_record.GET_DIR}/{platform_name}.md" def get_page_title(platform_name: str, display: str) -> str: """What a get/ page holds, by its name: the ZIP of a setup a release publishes one for, the files of any other. "Download KNULLI" read as the system's own download. A setup that is also an emulator (BizHawk) names it last: "BizHawk BIOS files" is the title of its emulator page.""" kind = "BIOS pack" if has_release_pack(platform_name) else "BIOS files" if (_REPO_ROOT / "emulators" / f"{platform_name}.yml").is_file(): return f"{kind} for {display}" return f"{display} {kind}" def _unzip_into(platform_name: str, display: str, extract_to: str | None, parts: int) -> list[str]: """Where the ZIP is unzipped, on the get/ page. A folder the registry writes relative to the setup's own (`system/`, `bios/`) is named as the setup's, with the way to find it.""" what = "them all" if parts > 1 else "it" if not _setup_relative(extract_to): return [f"- Unzip {what} into {_folder_words(display, extract_to)}.", ""] lead = GUIDES[platform_name].lead if platform_name in GUIDES else "" return [ f"- Unzip {what} into {_folder_words(display, extract_to)}. " f"[Where is it?](../platforms/{platform_name}.md#where-the-files-go)", "", *([lead, ""] if lead else []), ] def _setup_relative(extract_to: str | None) -> bool: """Whether the registry writes the folder relative to the setup's own (`system/`, `Firmware/`), not as a path the device reads.""" return bool(extract_to) and not extract_to.startswith(("/", "~", "<")) def _folder_words(display: str, extract_to: str | None) -> str: """The folder a pack or a copy goes into, in words: the device's path, or a folder of the setup named as the setup's.""" if not extract_to: return "your BIOS folder" if _setup_relative(extract_to): return f"{display}'s `{extract_to.rstrip('/')}` folder" return f"`{extract_to}`" def _pack_buttons(platform_name: str, label: str = "Pack", prefix: str = "../", display: str | None = None) -> str: """The way to the platform's ZIP from its setup page: a button to its get/ page, the release it comes from, and the setup version its list was read at. "Does this pack fit my version?" is the first question a frontend user asks. A setup with no release pack gets the line that says so, and why.""" if not has_release_pack(platform_name): display = display or load_platform_config(platform_name, "platforms").get( "platform", platform_name) return f"{no_zip_line(display)} [Why no ZIP?]({prefix}{NO_ZIP_FAQ})" record = release_record.load_advertised() buttons = download_links( platform_name, record, label, button=lambda text, url: markup.button(text, url, primary=True), page=get_page_link(platform_name, prefix), ) released = release_line(platform_name, record) return "\n\n".join([buttons, *released, version_line(platform_name, record, named=len(released) > 1)]) def generate_get_page(platform_name: str) -> str: """get//: the address a link outside the site can keep. Every release deletes the previous tag, and with it each asset URL: six of the eight links a wiki had to our packs went dead. This page lists the files of the pack the current release holds, read from release.json, or the command when no release publishes one. The release process never touches it: committing release.json rebuilds it.""" config = load_platform_config(platform_name, "platforms") display = config.get("platform", platform_name) record = release_record.load_advertised() pack = release_record.pack_for(platform_name, record) released = has_release_pack(platform_name) title = get_page_title(platform_name, display) if released and pack and pack.get("assets"): description = (f"The BIOS pack for {display}: {_fmt_size(pack['download_size'])} to " "download. The address stays the same and always gives the latest ZIP.") else: description = f"The BIOS files for {display}: one command installs and checks them." lines = [ "---", "search: exclude", f"description: {json.dumps(description)}", *_parent_front_matter((display, f"/platforms/{platform_name}/")), "---", "", f"# {title}", "", ] if released and pack and pack.get("assets"): parts = len(pack["assets"]) release = release_line(platform_name, record) extract_to = (load_platform_registry("platforms").get(platform_name) or {}).get("extract_to") lines += [ "This address stays the same. It always gives the latest ZIP.", "", asset_links(platform_name, record, REPO_URL, f"Download {display} Pack", button=lambda text, url: markup.button(text, url, primary=True)), "", f"- {_fmt_size(pack['download_size'])} to download, " f"{_fmt_size(pack['extracted_size'])} once unzipped.", *(["- You need both parts." if parts == 2 else f"- You need all {parts} parts."] if parts > 1 else []), *_unzip_into(platform_name, display, extract_to, parts), *(line for text in release for line in (text, "")), version_line(platform_name, record, named=len(release) > 1), "", ] elif released and record.get("tag"): lines += [ f"No {display} ZIP yet. The next release has one. " "The command installs every file and checks it.", "", *_setup_routes(platform_name, display), ] elif released: lines += [ "This address stays the same. It always gives the latest ZIP.", "", markup.button(f"Download {display} Pack", RELEASE_URL, primary=True), "", ] else: lines += [ no_zip_line(display), "", *_setup_routes(platform_name, display), f"[Why no ZIP?](../{NO_ZIP_FAQ})", "", ] lines += _next_block([ f"[{display}](../platforms/{platform_name}.md#install): the folder, " "and how to run the command on the device", "[All installer options](../wiki/installer.md)", ]) return "\n".join(lines) def generate_get_index() -> str: """get/: the permanent page of every setup, grouped as the home tiles are. A link trimmed to /get/ answered 404.""" registry = load_platform_registry("platforms") lines = [ "---", "search: exclude", 'description: "The BIOS files of every setup, one page each. Each address stays the same."', "---", "", "# BIOS files by setup", "", "One page per setup. Its address stays the same, so it is the one to share.", "", ] for group, (label, _description) in DEVICE_GROUPS.items(): names = [name for name, entry in registry.items() if entry.get("device_group") == group] if not names: continue lines += [f"## {label}", ""] for name in names: display = load_platform_config(name, "platforms").get("platform", name) lines.append(f"- [{get_page_title(name, display)}]({name}.md)") lines.append("") return "\n".join(lines) # The home tile counts the whole collection. Labelled "Files" above a table # of pack buttons, it read as what one pack contains. COLLECTION_FILES_LABEL = "Files collected" def composition_sentence(db: dict) -> str: comp = compute_composition(db) return ( f"The collection holds {comp['systems']['files']:,} console and " f"computer system files, {comp['arcade']['files']:,} arcade ROM sets " f"(`Arcade/`), and {comp['game_data']['files']:,} game and engine " "data files (`Game Engines/`, `RPG Maker/`, `ScummVM/` and the game " "publishers' trees). No pack holds " "them all: each carries what its platform's emulators load." ) def generate_stats(stats: dict) -> str: return json.dumps(stats, indent=2) + "\n" def _json_text(value: object) -> str: """Stable scalar representation for CSV and SQLite exports.""" if value is None: return "" if isinstance(value, (dict, list, tuple, bool)): return json.dumps(value, ensure_ascii=False, sort_keys=True) return str(value) HASH_ALGORITHMS = ("sha1", "sha256", "md5", "crc32") def _hash_values(value: object) -> list[str]: """Every accepted value of one hash field, lowercase. A field holds one value, a list of accepted revisions, or (Recalbox) a comma-joined string: an exact lookup missed all but the first form. """ if value is None: return [] items = value if isinstance(value, (list, tuple)) else str(value).split(",") return sorted({str(item).strip().lower() for item in items if str(item).strip()}) def _accepted_hash_rows(platform_files: list[dict], emulator_items: list[dict]) -> list[dict]: """One row per accepted hash value, for both layers, lookup-ready.""" rows: list[dict] = [] for row in platform_files: for algorithm in HASH_ALGORITHMS: for value in _hash_values(row.get(f"{algorithm}_raw", row.get(algorithm))): rows.append({ "layer": "platform", "owner": row["platform_id"], "system": row["system"], "name": row["name"], "path": row["destination"], "algorithm": algorithm, "value": value, }) for item in emulator_items: for entry in item["profile"].get("files", []) or []: if not isinstance(entry, dict): continue for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): rows.append({ "layer": "emulator", "owner": item["id"], "system": _json_text(entry.get("system")), "name": entry.get("name", ""), "path": entry.get("path", "") or "", "algorithm": algorithm, "value": value, }) # Members of an archive: MAME and FBNeo hash the ROMs inside the # set, never the set. Left out, a loose nmk004.bin (crc32 # 83b6f611) found no row while the site search found it. archive = entry.get("name", "") for member in entry.get("contents") or []: if not isinstance(member, dict) or not member.get("name"): continue for algorithm in HASH_ALGORITHMS: for value in _hash_values(member.get(algorithm)): rows.append({ "layer": "emulator", "owner": item["id"], "system": _json_text(entry.get("system")), "name": str(member["name"]), "path": f"{archive}/{member['name']}", "algorithm": algorithm, "value": value, }) rows.sort(key=lambda r: (r["layer"], r["owner"], r["name"], r["algorithm"], r["value"])) return rows def _csv_document(fieldnames: list[str], rows: list[dict]) -> str: stream = io.StringIO(newline="") writer = csv.DictWriter( stream, fieldnames=fieldnames, extrasaction="ignore", lineterminator="\n" ) writer.writeheader() writer.writerows(rows) return stream.getvalue() def _sha256_path(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _verify_outcomes( coverage: dict, sha1_by_path: dict[str, str] ) -> Callable[[str, str], dict]: """For each declared entry, in declaration order: what verify found. Returns a function of (system, name) giving the next outcome for that pair: status, category, the collection path that answers the entry and that file's sha1. The CSV gave the declaration and nothing about it. """ details: dict[tuple[str, str], list[dict]] = {} for detail in coverage.get("details") or []: details.setdefault((detail.get("system", ""), detail.get("name", "")), []).append(detail) taken: dict[tuple[str, str], int] = {} def outcome(system: str, name: str) -> dict: found = details.get((system, name)) or [{}] nth = taken.get((system, name), 0) taken[(system, name)] = nth + 1 detail = found[min(nth, len(found) - 1)] path = str(detail.get("path") or "") if detail.get("status") != "missing" else "" return { "status": detail.get("status", ""), "category": detail.get("category", ""), "collection_path": path, "resolved_sha1": sha1_by_path.get(path, "") if path else "", } return outcome def _sha1_by_path(db: dict | None) -> dict[str, str]: return {str(e.get("path", "")): sha1 for sha1, e in ((db or {}).get("files") or {}).items()} @functools.lru_cache(maxsize=1) def _contract_keys() -> frozenset[str]: """The top-level keys platform.schema.json allows in a platform's list.""" schema = json.loads((_REPO_ROOT / "schemas" / "platform.schema.json").read_text(encoding="utf-8")) return frozenset(schema.get("properties") or {}) def _split_contract(config: dict) -> tuple[dict, dict]: """The list, valid against platform.schema.json, and the registry entry. load_platform_config merges the registry into the list: `install`, `logo`, `scraper` and `_key` made all 18 contracts fail the published schema. They stay published, beside the contract. """ allowed = _contract_keys() contract = {key: value for key, value in config.items() if key in allowed} registry = {key: value for key, value in config.items() if key not in allowed and key != "_key"} # The merged config gives a child its parent's release_pack: Lakka and # RetroPie said true in the API while every page said they had no ZIP. # The registry's own entry decides, as it does for the pages. released = config.get("_key") in _release_pack_platforms() if released or "release_pack" in registry: registry["release_pack"] = released return contract, registry @functools.lru_cache(maxsize=1) def _release_pack_platforms() -> frozenset[str]: return frozenset(list_release_pack_platforms( str(_REPO_ROOT / "platforms"), include_archived=True)) # Per-entry fields a folder check needs beyond the hashes: an md5 is a ZIP # member's when zipped_file is set, and a CSV without it judged 106 good # Batocera files wrong. ENTRY_EXPORT_FIELDS = ( "zipped_file", "hash_match_mandatory", "mandatory_declared", "curated", "shared_group", "mirror_of", "zip_digest", ) def _platform_export_rows( coverages: dict, db: dict | None = None, ) -> tuple[list[dict], list[dict]]: """Normalized platform summaries and one row per declared file.""" items: list[dict] = [] file_rows: list[dict] = [] sha1_by_path = _sha1_by_path(db) for key, coverage in sorted(coverages.items()): config = coverage["config"] outcome = _verify_outcomes(coverage, sha1_by_path) contract, registry = _split_contract(config) items.append({ "id": key, "name": coverage["platform"], "coverage": { field: coverage.get(field, 0) for field in ( "total", "present", "verified", "untested", "missing", "core_present", "core_missing", "core_unsourceable", ) }, "contract": contract, "registry": registry, }) for system_id, system in sorted(config.get("systems", {}).items()): for entry in system.get("files", []) or []: file_rows.append({ "platform_id": key, "platform": coverage["platform"], "system": system_id, "name": entry.get("name", ""), "destination": entry.get( "destination", entry.get("dest", entry.get("name", "")) ), "required": bool(entry.get("required", True)), "region": _json_text(entry.get("region")), "variant_group": _json_text(entry.get("variant_group")), "size": entry.get("size"), "sha1": _json_text(entry.get("sha1")), "sha256": _json_text(entry.get("sha256")), "md5": _json_text(entry.get("md5")), "crc32": _json_text(entry.get("crc32")), **{field: _export_flag(entry.get(field)) for field in ENTRY_EXPORT_FIELDS}, **outcome(system_id, entry.get("name", "")), }) return items, file_rows def _export_flag(value: object) -> object: """A per-entry field as CSV and SQLite hold it: booleans stay booleans, an absent field is empty.""" if value is None: return "" return value if isinstance(value, bool) else str(value) def _emulator_export_items(profiles: dict) -> list[dict]: return [ {"id": key, "profile": profile} for key, profile in sorted(profiles.items()) ] def build_emulator_gap_report( profiles: dict, coverages: dict, db: dict, data_names: set[str] | None = None, ) -> dict: """Files a core loads that no platform declares, per emulator profile. The gap analysis page and the published gaps export must not compute this twice and drift. Both read this one report. """ from common import _norm_system_id, merged_platform_declarations from cross_reference import cross_reference as run_cross_reference # Enriched with alias resolution, with each declaration's size and slot: # a name alone settled DOSBox's 32 KiB SC-55 ROM1.BIN on Galaksija's. all_declared = merged_platform_declarations( [cov["config"] for cov in coverages.values()], db ) declared: dict[str, set[str]] = {} for _name, cov in coverages.items(): config = cov["config"] for sys_id, system in config.get("systems", {}).items(): for fe in system.get("files", []): fname = fe.get("name", "") if fname: declared.setdefault(_norm_system_id(sys_id), set()).add(fname) unique_profiles = { k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test") } from common import platform_declarations, standalone_set from validation import case_folded_slots # The command line passes these too: without them a RetroBat file named # 32X_G_BIOS.BIN read as missing for a core opening 32X_G_BIOS.bin. folded = frozenset().union(*( case_folded_slots(cov["config"], platform_declarations(cov["config"], db)) for cov in coverages.values() )) return run_cross_reference( unique_profiles, declared, db, data_names=data_names, all_declared=all_declared, standalone_cores=set().union( *(standalone_set(cov["config"]) for cov in coverages.values()) ), folded_slots=folded, ) def _gap_export_rows( coverages: dict, gap_report: dict | None = None, db: dict | None = None, profiles: dict | None = None, ) -> list[dict]: """Every gap the site reports, both layers, in one table. ``platform`` rows are anomalies against a platform's own BIOS list. ``emulator`` rows are files a profiled core loads that no platform declares, which is the larger number the gap analysis page leads with. A `layer` column keeps the two apart instead of publishing only one. """ rows: list[dict] = [] sha1_by_path = _sha1_by_path(db) for key, coverage in sorted(coverages.items()): for detail in coverage.get("details", []): if detail.get("status") == "ok" and not detail.get("discrepancy"): continue path = str(detail.get("path") or "") if detail.get("status") != "missing" else "" rows.append({ "category": detail.get("category", "bios"), "collection_path": path, "resolved_sha1": sha1_by_path.get(path, "") if path else "", "layer": "platform", "platform_id": key, "platform": coverage["platform"], "emulator": "", "system": detail.get("system", ""), "name": detail.get("name", ""), "status": detail.get("status", ""), "required": bool(detail.get("required", True)), "in_repo": "", "reason": detail.get("reason", ""), "discrepancy": detail.get("discrepancy", ""), "destination": detail.get("destination", ""), "expected": "", }) for emu_key, data in sorted((gap_report or {}).items()): systems = ";".join(str(s) for s in data.get("systems", []) or []) categories = _profile_categories((profiles or {}).get(emu_key) or {}) for gap in data.get("gap_details", []) or []: rows.append({ "category": categories.get( (gap.get("name", ""), gap.get("path") or gap.get("name", "")), ""), "collection_path": "", "resolved_sha1": "", "layer": "emulator", "platform_id": "", "platform": "", "emulator": data.get("emulator", emu_key), "system": systems, "name": gap.get("name", ""), # A file the emulator's own package installs is not one to # find: its own status, as unsourceable has. "status": ( "bundled" if gap.get("bundled") and gap.get("source") == "missing" else gap.get("source", "") ), "required": bool(gap.get("required", False)), "in_repo": bool(gap.get("in_repo", False)), "reason": gap.get("note", ""), "discrepancy": "", # What a contributor needs to supply the right revision and # put it in the right place: the gaps table gave neither. "destination": gap.get("path") or gap.get("name", ""), "expected": gap.get("identity", ""), }) for entry in data.get("unsourceable", []) or []: rows.append({ "category": categories.get( (entry.get("name", ""), entry.get("path") or entry.get("name", "")), ""), "collection_path": "", "resolved_sha1": "", "layer": "emulator", "platform_id": "", "platform": "", "emulator": data.get("emulator", emu_key), "system": systems, "name": entry.get("name", ""), "status": "unsourceable", "required": bool(entry.get("required", False)), "in_repo": False, "reason": entry.get("reason", ""), "discrepancy": "", "destination": entry.get("path") or entry.get("name", ""), "expected": "", }) # Two declarations that differ only in what the row does not carry are # one row: AmberELEC's hardwareconfig.xml was printed twice. unique: dict[tuple, dict] = {} for row in rows: unique.setdefault(tuple(sorted((k, str(v)) for k, v in row.items())), row) return list(unique.values()) def _profile_categories(profile: dict) -> dict[tuple[str, str], str]: """(name, path) -> the category a profile gives the entry, bios by default.""" found: dict[tuple[str, str], str] = {} for entry in profile.get("files") or []: if isinstance(entry, dict) and entry.get("name"): name = str(entry["name"]) found.setdefault((name, str(entry.get("path") or name)), str(entry.get("category") or "bios")) return found def _cross_reference_export_rows(coverages: dict, profiles: dict) -> list[dict]: from common import resolve_platform_cores unique = { key: value for key, value in profiles.items() if value.get("type") not in ("alias", "test") } rows: list[dict] = [] for platform_id, coverage in sorted(coverages.items()): for profile_id in sorted(resolve_platform_cores(coverage["config"], unique)): profile = unique[profile_id] cores = profile.get("cores") or [profile_id] systems = profile.get("systems") or [""] for core in cores: for system in systems: rows.append({ "platform_id": platform_id, "platform": coverage["platform"], "profile_id": profile_id, "emulator": profile.get("emulator", profile_id), "core": core, "system": system, "classification": profile.get("core_classification", ""), "type": profile.get("type", ""), "source": _json_text(profile.get("source")), "upstream": _json_text(profile.get("upstream")), "profiled_commit": _json_text(profile.get("source_commit", "")), "file_count": len(profile.get("files", []) or []), }) return rows def _write_sqlite_export( destination: Path, db: dict, platform_items: list[dict], platform_files: list[dict], emulator_items: list[dict], gap_rows: list[dict], ) -> None: """Build a queryable, deterministic snapshot without embedding binaries.""" temp_dir = Path("tmp") / "site" temp_dir.mkdir(parents=True, exist_ok=True) temp_path = temp_dir / f"retrobios-{os.getpid()}.sqlite" temp_path.unlink(missing_ok=True) destination.parent.mkdir(parents=True, exist_ok=True) connection = sqlite3.connect(temp_path) try: connection.executescript(""" PRAGMA journal_mode = OFF; PRAGMA synchronous = OFF; CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL); CREATE TABLE files ( sha1 TEXT PRIMARY KEY, path TEXT NOT NULL, name TEXT NOT NULL, size INTEGER NOT NULL, md5 TEXT NOT NULL, sha256 TEXT NOT NULL, crc32 TEXT NOT NULL, adler32 TEXT NOT NULL ); CREATE TABLE file_provenance ( sha1 TEXT NOT NULL, catalog TEXT NOT NULL, details_json TEXT NOT NULL, PRIMARY KEY (sha1, catalog), FOREIGN KEY (sha1) REFERENCES files(sha1) ); CREATE TABLE platforms ( id TEXT PRIMARY KEY, name TEXT NOT NULL, total INTEGER NOT NULL, present INTEGER NOT NULL, verified INTEGER NOT NULL, missing INTEGER NOT NULL, contract_json TEXT NOT NULL ); CREATE TABLE platform_files ( platform_id TEXT NOT NULL, system TEXT NOT NULL, name TEXT NOT NULL, destination TEXT NOT NULL, required INTEGER NOT NULL, region TEXT, variant_group TEXT, size INTEGER, sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT, status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT, zipped_file TEXT, hash_match_mandatory INTEGER, mandatory_declared INTEGER, curated INTEGER, shared_group TEXT, mirror_of TEXT, zip_digest TEXT ); CREATE TABLE emulators ( id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT, classification TEXT, source TEXT, upstream TEXT, source_commit TEXT, profile_json TEXT NOT NULL ); CREATE TABLE emulator_systems ( emulator_id TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator_id, system) ); CREATE TABLE emulator_files ( emulator_id TEXT NOT NULL, system TEXT, name TEXT NOT NULL, path TEXT, required INTEGER NOT NULL, mode TEXT, region TEXT, size TEXT, sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT, source_ref TEXT ); CREATE TABLE gaps ( layer TEXT NOT NULL, platform_id TEXT, platform TEXT, emulator TEXT, system TEXT, name TEXT NOT NULL, status TEXT NOT NULL, required INTEGER NOT NULL, in_repo TEXT, reason TEXT, discrepancy TEXT, destination TEXT, expected TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT ); CREATE TABLE accepted_hashes ( layer TEXT NOT NULL, owner TEXT NOT NULL, system TEXT, name TEXT NOT NULL, path TEXT, algorithm TEXT NOT NULL, value TEXT NOT NULL ); CREATE INDEX accepted_hashes_value_idx ON accepted_hashes(algorithm, value); CREATE INDEX files_name_idx ON files(name); CREATE INDEX files_md5_idx ON files(md5); CREATE INDEX files_sha256_idx ON files(sha256); CREATE INDEX platform_files_name_idx ON platform_files(name); CREATE INDEX emulator_files_name_idx ON emulator_files(name); CREATE INDEX gaps_status_idx ON gaps(layer, status); """) commit = _source_commit() metadata = { "schema_version": "1", "generated_at": str(db.get("generated_at") or _timestamp()), **({"commit": commit} if commit else {}), "source": REPO_URL, "scope": "metadata only, no BIOS or firmware payload bytes", "license": DATA_LICENSE, } connection.executemany( "INSERT INTO metadata(key, value) VALUES (?, ?)", sorted(metadata.items()), ) for sha1, entry in sorted(db.get("files", {}).items()): connection.execute( "INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ( sha1, entry.get("path", ""), entry.get("name", ""), entry.get("size", 0), entry.get("md5", ""), entry.get("sha256", ""), entry.get("crc32", ""), entry.get("adler32", ""), ), ) for catalog, details in sorted((entry.get("provenance") or {}).items()): connection.execute( "INSERT INTO file_provenance VALUES (?, ?, ?)", (sha1, catalog, _json_text(details)), ) for item in platform_items: coverage = item["coverage"] connection.execute( "INSERT INTO platforms VALUES (?, ?, ?, ?, ?, ?, ?)", ( item["id"], item["name"], coverage.get("total", 0), coverage.get("present", 0), coverage.get("verified", 0), coverage.get("missing", 0), _json_text(item["contract"]), ), ) connection.executemany( "INSERT INTO platform_files VALUES " "(:platform_id, :system, :name, :destination, :required, :region, " ":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, " ":collection_path, :resolved_sha1, :zipped_file, :hash_match_mandatory, " ":mandatory_declared, :curated, :shared_group, :mirror_of, :zip_digest)", platform_files, ) for item in emulator_items: profile = item["profile"] connection.execute( "INSERT INTO emulators VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ( item["id"], profile.get("emulator", item["id"]), profile.get("type", ""), profile.get("core_classification", ""), _json_text(profile.get("source")), _json_text(profile.get("upstream")), # source_commit is a string or, when the builds live in # separate repositories, an object keyed by build mode. # SQLite cannot bind the object form. _json_text(profile.get("source_commit", "")), _json_text(profile), ), ) for system in sorted(set(profile.get("systems", []) or [])): connection.execute( "INSERT INTO emulator_systems VALUES (?, ?)", (item["id"], system), ) for entry in profile.get("files", []) or []: connection.execute( "INSERT INTO emulator_files VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( item["id"], _json_text(entry.get("system")), entry.get("name", ""), entry.get("path", ""), int(bool(entry.get("required", False))), entry.get("mode", ""), _json_text(entry.get("region")), _json_text(entry.get("size")), _json_text(entry.get("sha1")), _json_text(entry.get("sha256")), _json_text(entry.get("md5")), _json_text(entry.get("crc32")), _json_text(entry.get("source_ref")), ), ) connection.executemany( "INSERT INTO accepted_hashes VALUES " "(:layer, :owner, :system, :name, :path, :algorithm, :value)", _accepted_hash_rows(platform_files, emulator_items), ) connection.executemany( "INSERT INTO gaps VALUES " "(:layer, :platform_id, :platform, :emulator, :system, :name, " ":status, :required, :in_repo, :reason, :discrepancy, :destination, :expected, " ":category, :collection_path, :resolved_sha1)", gap_rows, ) connection.commit() connection.execute("VACUUM") finally: connection.close() os.replace(temp_path, destination) def write_single_item( path: Path, kind: str, item: dict, generated_at: str, commit: str = "", ) -> None: """One item in the envelope of its collection, for a page to link.""" path.parent.mkdir(parents=True, exist_ok=True) write_if_changed( str(path), json.dumps( {"schema_version": 1, "generated_at": generated_at, **({"commit": commit} if commit else {}), "kind": kind, "count": 1, "items": [item]}, indent=2, sort_keys=True, ensure_ascii=False, default=str, ) + "\n", ) def system_export_item(slug: str, manufacturer: str, consoles: dict[str, list]) -> dict: """A system page's files by console, with the hashes its cards show.""" fields = ("name", "path", "size", "sha1", "md5", "sha256", "crc32") return { "id": slug, "name": manufacturer, "consoles": { console: [ {field: entry[field] for field in fields if entry.get(field) is not None} for entry in sorted(entries, key=lambda e: (e.get("name", ""), e.get("sha1", ""))) ] for console, entries in sorted(consoles.items()) }, } @functools.lru_cache(maxsize=1) def _source_commit() -> str: """The commit of the checkout the site is built from, empty outside one.""" import subprocess try: result = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=_REPO_ROOT, capture_output=True, text=True, check=True, ) except (OSError, subprocess.CalledProcessError) as error: print(f" no commit recorded in the exports: {error}", file=sys.stderr) return "" commit = result.stdout.strip() return commit if re.fullmatch(r"[0-9a-f]{40}", commit) else "" # One row per dump a catalog describes and the collection lacks: the list # the provenance page publishes under "Wanted". WANTED_FIELDS = ("catalog", "dat", "name", "description", "size", "crc32", "md5", "sha1") def _provenance_wanted_rows(db: dict) -> list[dict]: """The Wanted list of the provenance page, as rows. A dump the collection already produces (derivable.json) is not wanted.""" derivable = load_derivable() rows: list[dict] = [] for catalog, data in build_report(db, load_provenance_snapshots()).items(): for entry in data["missing"]: if (entry.get("sha1") or "").lower() in derivable: continue for dat in entry.get("dats") or [entry.get("dat", "")]: rows.append({ "catalog": catalog, "dat": dat, **{f: _json_text(entry.get(f)) for f in WANTED_FIELDS[2:]}, }) rows.sort(key=lambda r: (r["catalog"], r["dat"], r["name"], r["sha1"])) return rows def generate_data_exports( docs: Path, db: dict, coverages: dict, profiles: dict, stats: dict, gap_report: dict | None = None, consumers: FileConsumers | None = None, ) -> list[dict]: """Create versioned static API, CSV and SQLite metadata snapshots.""" api = docs / "api" / "v1" downloads = docs / "downloads" schemas_dest = api / "schemas" for directory in (api, downloads, schemas_dest): directory.mkdir(parents=True, exist_ok=True) generated_at = str(db.get("generated_at") or _timestamp()) # The revision the data was read at: a tool comparing two downloads # could not tell which commit each came from. commit = _source_commit() stamp = {"commit": commit} if commit else {} platform_items, platform_files = _platform_export_rows(coverages, db) emulator_items = _emulator_export_items(profiles) gap_rows = _gap_export_rows(coverages, gap_report, db, profiles) cross_rows = _cross_reference_export_rows(coverages, profiles) members = data_directory_members() locations = file_location_items(db, profiles, members) envelopes = { "platforms.json": ("platforms", platform_items), "emulators.json": ("emulators", emulator_items), "gaps.json": ("verification-and-coverage-gaps", gap_rows), "file-locations.json": ("file-locations", locations), } write_lookup_shards( api, file_lookup_records(db, coverages, profiles, locations, members, consumers), generated_at, commit, ) for filename, (kind, items) in envelopes.items(): document = { "schema_version": 1, "generated_at": generated_at, **stamp, "kind": kind, "count": len(items), "items": items, } write_if_changed( str(api / filename), json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) # One platform without the others, as api/v1/emulators/.json is one # profile: platforms.json is 2.8 MB for twelve platforms. for item in platform_items: write_single_item(api / "platforms" / f"{item['id']}.json", "platforms", item, generated_at, commit) write_if_changed(str(api / "stats.json"), generate_stats(stats)) write_if_changed( str(api / "database.json"), json.dumps(db, ensure_ascii=False, indent=2) + "\n", ) # Every schema the published files are written against, the manifests' # too: an integrator had the platform's list and not what the pack writes. for schema_path in sorted((_REPO_ROOT / "schemas").glob("*.schema.json")): shutil.copy2(schema_path, schemas_dest / schema_path.name) _record(schemas_dest / schema_path.name) manifest_assets = _publish_manifests(api) for filename, (kind, items) in { "systems.json": ("system-index", system_index_items(db, coverages, profiles)), "cores.json": ("core-index", core_index_items(profiles)), }.items(): write_if_changed(str(api / filename), json.dumps({ "schema_version": 1, "generated_at": generated_at, **stamp, "kind": kind, "count": len(items), "items": items, }, ensure_ascii=False, indent=2, sort_keys=True) + "\n") file_rows = [] for sha1, entry in sorted(db.get("files", {}).items()): file_rows.append({ "sha1": sha1, "path": entry.get("path", ""), "name": entry.get("name", ""), "size": entry.get("size", 0), "md5": entry.get("md5", ""), "sha256": entry.get("sha256", ""), "crc32": entry.get("crc32", ""), "adler32": entry.get("adler32", ""), "provenance_catalogs": ";".join( sorted((entry.get("provenance") or {}).keys()) ), }) write_if_changed(str(downloads / "retrobios.dat"), collection_dat(db, generated_at)) write_if_changed( str(downloads / "files.csv"), _csv_document( [ "sha1", "path", "name", "size", "md5", "sha256", "crc32", "adler32", "provenance_catalogs", ], file_rows, ), ) write_if_changed( str(downloads / "platform-files.csv"), _csv_document( [ "platform_id", "platform", "system", "name", "destination", "required", "region", "variant_group", "size", "sha1", "sha256", "md5", "crc32", "status", "category", "collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS, ], platform_files, ), ) write_if_changed( str(downloads / "cross-reference.csv"), _csv_document( [ "platform_id", "platform", "profile_id", "emulator", "core", "system", "classification", "type", "source", "upstream", "profiled_commit", "file_count", ], cross_rows, ), ) write_if_changed( str(downloads / "gaps.csv"), _csv_document( [ "layer", "platform_id", "platform", "emulator", "system", "name", "status", "required", "in_repo", "reason", "discrepancy", "destination", "expected", "category", "collection_path", "resolved_sha1", ], gap_rows, ), ) write_if_changed( str(downloads / "accepted-hashes.csv"), _csv_document( ["layer", "owner", "system", "name", "path", "algorithm", "value"], _accepted_hash_rows(platform_files, emulator_items), ), ) _write_sqlite_export( downloads / "retrobios.sqlite", db, platform_items, platform_files, emulator_items, gap_rows, ) _record(downloads / "retrobios.sqlite") write_if_changed( str(downloads / "provenance-wanted.csv"), _csv_document(list(WANTED_FIELDS), _provenance_wanted_rows(db)), ) assets = [ (api / "database.json", "Content database", "application/json", "schemas/database.schema.json"), (api / "platforms.json", "Platform contracts", "application/json", "schemas/site-api-envelope.schema.json"), (api / "emulators.json", "Emulator profiles", "application/json", "schemas/site-api-envelope.schema.json"), (api / "gaps.json", "Verification and coverage gaps", "application/json", "schemas/site-api-envelope.schema.json"), (api / "file-locations.json", "Where each file's card is, by SHA1", "application/json", "schemas/site-api-envelope.schema.json"), (api / "stats.json", "Project statistics", "application/json", "schemas/stats.schema.json"), (api / "systems.json", "System ids, their names and sections", "application/json", "schemas/site-api-envelope.schema.json"), (api / "cores.json", "Core names and the profile each one is", "application/json", "schemas/site-api-envelope.schema.json"), *manifest_assets, (downloads / "files.csv", "File hashes CSV", "text/csv", None), (downloads / "retrobios.dat", "Collection DAT (Logiqx XML, for clrmamepro and RomVault)", "application/xml", None), (downloads / "platform-files.csv", "Platform declarations CSV", "text/csv", None), (downloads / "cross-reference.csv", "Cross-reference CSV", "text/csv", None), (downloads / "gaps.csv", "Verification and coverage gaps CSV", "text/csv", None), (downloads / "accepted-hashes.csv", "Accepted hashes, one row per value", "text/csv", None), (downloads / "provenance-wanted.csv", "Catalogued dumps the collection lacks", "text/csv", None), (downloads / "retrobios.sqlite", "SQLite snapshot", "application/vnd.sqlite3", None), ] catalog_items: list[dict] = [] published: list[dict] = [] for path, title, media_type, schema in assets: relative = path.relative_to(docs).as_posix() item = { "title": title, "url": relative, "media_type": media_type, "bytes": path.stat().st_size, "sha256": _sha256_path(path), } if schema: item["schema"] = schema catalog_items.append(item) # In the catalog both references resolve against the catalog's own # URL, as RFC 3986 clients do: `url` was relative to the site root # and `schema` to api/v1/, so no single base resolved both. published.append({ **item, "url": os.path.relpath(path, api).replace(os.sep, "/"), }) catalog = { "schema_version": 1, "generated_at": generated_at, **stamp, "kind": "catalog", "license": DATA_LICENSE, "count": len(published), "items": published, } write_if_changed( str(api / "catalog.json"), json.dumps(catalog, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) # api/v1/ itself is a folder GitHub Pages answers with the 404 page: the # entry point is a document of its own, naming where to start. entry_points = [ {"title": "Catalog", "url": "catalog.json", "description": "Every published file with its size, SHA256 and schema."}, {"title": "System ids", "url": "systems.json", "description": "Each system id with its name and the section holding its files."}, {"title": "Core names", "url": "cores.json", "description": "Each core name and alias with the profile it designates."}, {"title": "Schemas", "url": "schemas/site-api-envelope.schema.json", "description": "The envelope every JSON document of api/v1 shares."}, ] write_if_changed(str(api / "index.json"), json.dumps({ "schema_version": 1, "generated_at": generated_at, **stamp, "kind": "index", "license": DATA_LICENSE, "count": len(entry_points), "items": entry_points, }, ensure_ascii=False, indent=2, sort_keys=True) + "\n") return catalog_items def write_release_advertised(generated: Path) -> None: """The release record as the pages read it, for the file picker. The picker imported release.json whole and offered the ZIP of six setups the registry no longer releases, while their pages said "No ZIP". It reads this file instead: the platforms marked release_pack only. """ generated.mkdir(parents=True, exist_ok=True) write_if_changed( str(generated / "release-advertised.json"), json.dumps(release_record.load_advertised(), ensure_ascii=False, indent=1, sort_keys=True) + "\n", ) def _publish_manifests(api: Path) -> list[tuple]: """The install manifests and the release record, beside the API. The platform's list says what a setup checks. The manifest says what its pack installs: Batocera's list holds 353 files, its pack 14,508. The only copy was install/.json on the repository's main branch, which no page named. Copied byte for byte: the SHA256 the catalog gives is the repository file's. """ folder = api / "manifests" folder.mkdir(parents=True, exist_ok=True) assets: list[tuple] = [] for source in sorted((_REPO_ROOT / "install").glob("*.json")): target = folder / source.name shutil.copy2(source, target) _record(target) assets.append((target, f"Install manifest: {source.stem}", "application/json", "schemas/install-manifest.schema.json")) # The target lists install.py --target reads: no catalog entry, and # api/v1/targets/ answered 404. targets = api / "targets" targets.mkdir(parents=True, exist_ok=True) for source in sorted((_REPO_ROOT / "install" / "targets").glob("*.json")): shutil.copy2(source, targets / source.name) _record(targets / source.name) assets.append((targets / source.name, f"Target list: {source.stem}", "application/json", "schemas/target-manifest.schema.json")) release = _REPO_ROOT / "release.json" if release.is_file(): shutil.copy2(release, api / "release.json") _record(api / "release.json") assets.append((api / "release.json", "Latest release: packs, sizes and assets", "application/json", "schemas/release.schema.json")) return assets def _site_path(markdown_target: str) -> str: """A page link as generators write it (systems/sony.md#psx) as a site-absolute path (/retrobios/systems/sony/#psx), which resolves the same from any document that holds it.""" page, _, anchor = markdown_target.partition("#") base = urllib.parse.urlsplit(SITE_URL).path page = page.removesuffix(".md") page = "" if page == "index" else page.removesuffix("/index") + "/" return f"{base}{page}" + (f"#{anchor}" if anchor else "") def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: """Each system id the platforms and profiles write, reachable by that id. api/v1/systems/ is keyed by manufacturer and its consoles by display name: a frontend keying by system id (sony-playstation) had no way in. The url is the section system_section gives, the one every page links. An id with no section of its own (tyrian, scummvm) that holds files takes the card of its first file, else the page of the emulator naming it: 28 such ids sent their hashes to no page. """ from siterender import card_page, system_ids, system_section sha1_by_path = _sha1_by_path(db) files = db.get("files") or {} by_md5 = (db.get("indexes") or {}).get("by_md5") or {} held: dict[str, set[str]] = {} for coverage in coverages.values(): for detail in coverage.get("details") or []: path = str(detail.get("path") or "") sha1 = sha1_by_path.get(path) if detail.get("status") != "missing" else None if sha1: held.setdefault(str(detail.get("system", "")), set()).add(sha1) for profile in profiles.values(): if profile.get("type") in ("alias", "test"): continue own = [str(s) for s in profile.get("systems") or []] for entry in profile.get("files") or []: if not isinstance(entry, dict): continue ids = ([str(entry["system"])] if entry.get("system") else [str(s) for s in entry.get("systems") or []] or (own if len(own) == 1 else [])) found = {v for v in _hash_values(entry.get("sha1")) if v in files} for md5 in _hash_values(entry.get("md5")): match = by_md5.get(md5) found.update([match] if isinstance(match, str) else match or []) for sys_id in ids: held.setdefault(sys_id, set()).update(found) manufacturers = {_slug(mfr): mfr for mfr in _group_by_manufacturer(db)} base = urllib.parse.urlsplit(SITE_URL).path items = [] for sys_id in sorted(system_ids(profiles, [c["config"] for c in coverages.values()])): section = system_section(sys_id) try: name = system_name(sys_id) except KeyError: name = sys_id sha1s = sorted(held.get(sys_id, set()) & files.keys()) manufacturer = manufacturers.get(section[0], "") if section else "" url = _site_path(console_href(*section)) if section else None if url is None and sha1s: manufacturer, url = _first_file_place(sys_id, sha1s, files, profiles, card_page) # The id's JSON document: api/v1/systems/ is keyed by the manufacturer # page, and sony-playstation.json answered 404. holder = section[0] if section else _slug(manufacturer) if manufacturer else "" api = f"{base}api/v1/systems/{holder}.json" if holder else None items.append({"id": sys_id, "name": name, "manufacturer": manufacturer, "url": url, "api": api, "sha1": sha1s}) return items def _first_file_place( sys_id: str, sha1s: list[str], files: dict, profiles: dict, card_page: Callable[[str, str], str], ) -> tuple[str, str | None]: """(manufacturer, url) of the card of an id's first collection file, else of the page of the first emulator naming the id.""" for sha1 in sha1s: parts = str(files[sha1].get("path", "")).split("/") if len(parts) >= 3 and parts[0] == "bios": page = card_page("/".join(parts), sha1) return parts[1], _site_path(f"systems/{page}.md#sha1-{sha1[:12]}") for key in sorted(profiles): profile = profiles[key] if profile.get("type") not in ("alias", "test") and sys_id in (profile.get("systems") or []): return "", _site_path(f"emulators/{key}.md") return "", None def core_index_items(profiles: dict) -> list[dict]: """Every name a core goes by, with the profile it designates. A frontend knows mednafen_psx_libretro, and api/v1/emulators/ had only beetle_psx.json: the reverse index resolve_platform_cores reads, aliases included, said once. A name two profiles claim (pcsx2) lists both, the one a platform list designates first. """ from common import preferred_profile, upstream_profile_index published = {name for name, p in profiles.items() if p.get("type") not in ("alias", "test")} index = upstream_profile_index(profiles) for name, profile in profiles.items(): target = str(profile.get("alias_of") or "") if profile.get("type") == "alias" and target in published: index.setdefault(name, set()).add(target) items = [] for name in sorted(index): claimants = sorted(index[name] & published) preferred = preferred_profile({name: set(claimants)}, name) for profile in sorted(claimants, key=lambda p: (p != preferred, p)): items.append({"name": name, "profile": profile, "preferred": profile == preferred}) return items def _is_manifest(item: dict) -> bool: return item["url"].startswith("api/v1/manifests/") def _is_target_list(item: dict) -> bool: return item["url"].startswith("api/v1/targets/") def _served_schemas() -> list[tuple[str, str]]: """(file name, title) of every schema api/v1/schemas/ serves, from the folder the export copies: the page linked 7 of 11.""" return [ (path.name, str(json.loads(path.read_text(encoding="utf-8")).get("title") or path.stem)) for path in sorted((_REPO_ROOT / "schemas").glob("*.schema.json")) ] # A URL template written in prose (https://...//): escaped, # the autolinker made a link of it, <ref> on screen and a 404 behind it. _URL_TEMPLATE = re.compile(r"(? list[str]: """One table row per field, its description read from the schema, so the page and the published contract say the same thing. A URL template is written as code.""" props = schema.get("properties") or {} rows = [] for key in keys: text = str((props.get(key) or {}).get("description") or "") if text: # Python before 3.12 refuses a backslash inside an f-string expression. described = _escape_tags(_URL_TEMPLATE.sub(r"`\g<0>`", text)) rows.append(f"| `{key}` | {described} |") return rows def _other_fields(schema: dict, shown: tuple[str, ...]) -> list[str]: """The fields a short table leaves out, every one with its description.""" rest = tuple(key for key in schema.get("properties") or {} if key not in shown) return ["| Field | Meaning |", "|-------|---------|", *_schema_field_rows(schema, rest)] def _nested_fields(schema: dict, prefix: str = "") -> list[str]: """A table for every object below *schema*, at any depth, and one line per code where a field lists its codes' meanings (enumDescriptions). The top-level and `files` tables left 50 nested fields undescribed, all of `omitted_files` among them: a tool could not say why a file was not installed. """ lines: list[str] = [] for key, spec in (schema.get("properties") or {}).items(): path = f"{prefix}{key}" codes = spec.get("enumDescriptions") if codes: lines.extend([f"Codes of `{path}`:", ""]) lines.extend(f"- `{code}`: {text}" for code, text in zip(spec["enum"], codes)) lines.append("") for label, child, child_path in ( ("Each entry of", spec.get("items"), f"{path}[]"), ("Each value of", spec.get("additionalProperties"), f"{path}{{}}"), ("Fields of", spec, path), ): if not isinstance(child, dict) or not child.get("properties"): continue lines.extend([ f"{label} `{path}`:", "", "| Field | Meaning |", "|-------|---------|", *_schema_field_rows(child, tuple(child["properties"])), "", ]) lines.extend(_nested_fields(child, f"{child_path}.")) return lines # The manifest fields an integrator reads first, in the order they matter. MANIFEST_TOP_FIELDS = ("platform", "version", "base_destination", "files", "omitted_files", "pack_files", "core_names", "system_cores") MANIFEST_FILE_FIELDS = ("dest", "sha1", "size", "repo_path", "release_asset", "cores", "systems", "category", "alternatives") def _short_sha256(item: dict, catalog: str) -> str: """A digest cell that fits its column: 12 characters linking the catalog that holds it, the whole value in the title. 64 characters pushed the table to 932 px in a 720 px column.""" value = str(item["sha256"]) return f'{value[:12]}' def _platforms_without_a_list(platforms_dir: Path | None = None) -> list[str]: """The names of the platforms whose registry says they keep no list.""" platforms_dir = platforms_dir or _REPO_ROOT / "platforms" names = [] for key, entry in sorted(load_platform_registry(str(platforms_dir)).items()): if entry.get("native_list") != "none": continue config = platforms_dir / str(entry.get("config") or f"{key}.yml") data = yaml_load(config.read_text(encoding="utf-8")) if config.is_file() else {} names.append(str((data or {}).get("platform") or key)) return names def _join_names(names: list[str]) -> str: return names[0] if len(names) == 1 else ", ".join(names[:-1]) + f" and {names[-1]}" def generate_data_page(exports: list[dict]) -> str: manifest_schema = json.loads( (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8")) snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") manifests = [item for item in exports if _is_manifest(item)] targets = [item for item in exports if _is_target_list(item)] no_list = _platforms_without_a_list() lines = [ "---", "terms: manifest, schema, json schema, license, check a folder, " "check bios folder script, rate limit, api", "---", "", "# Data and API", "", "RetroBIOS publishes the same metadata used by its verifier, pack builder " "and website as versioned static files. No account or API key is required.", "", " ".join([ markup.button("API catalog", "api/v1/catalog.json", primary=True), markup.button("SQLite snapshot", "downloads/retrobios.sqlite"), markup.button("Collection DAT", "downloads/retrobios.dat"), ]), "", f"**Licence:** the metadata is offered under the MIT terms of [LICENSE]({REPO_URL}/blob/main/LICENSE), " "which cover the scripts, tooling, schemas, documentation and metadata. " f"They never cover a BIOS or firmware file ([NOTICE]({REPO_URL}/blob/main/NOTICE)). " "`catalog.json` and the SQLite `metadata` table carry the same `license`.", "", "## Stable JSON endpoints", "", "Start at [`api/v1/index.json`](api/v1/index.json): it names the catalog, " "the two id indexes and the envelope schema.", "", "| Dataset | Endpoint | Schema | Size |", "|---------|----------|--------|-----:|", ] for item in exports: if (item["media_type"] != "application/json" or _is_manifest(item) or _is_target_list(item)): continue schema = ( f"[JSON Schema](api/v1/{item['schema']})" if item.get("schema") else "-" ) lines.append( f"| {item['title']} | [`{item['url']}`]({item['url']}) | " f"{schema} | {_fmt_size(item['bytes'])} |" ) lines.extend([ "", "Every envelope carries `schema_version`, `kind` and `commit`, the revision " "of this repository the data was read at. Breaking changes use a new " "URL prefix (`/api/v2/`), and fields may only be added compatibly within v1. " "Each schema's `$id` is the address it is served at.", "", "Schemas:", "", *(f"- [`{name}`](api/v1/schemas/{name}): {title}" for name, title in _served_schemas()), "", "`database.json`, `stats.json`, `release.json` and the manifests are not " "envelopes. Their revision is the `commit` of `catalog.json` that lists " "their SHA256.", "", "One emulator profile without the whole list: " "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " "[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each " "emulator page links its own. `api/v1/platforms/.json` holds one " "platform of `platforms.json`, its `contract` valid against " "[`platform.schema.json`](api/v1/schemas/platform.schema.json) and its " "`registry` the install and scraper settings beside the list. " "`api/v1/systems/.json` holds the files of one system page by console. " "Each platform and system page links its own.", "", "By id: [`systems.json`](api/v1/systems.json) gives each system id a platform " "or a profile writes (`sony-playstation`) with its name, its manufacturer, the " "`url` of the section holding its files, the `api` document of that section " "and the files' SHA1. " "[`cores.json`](api/v1/cores.json) gives each core name the buildbot or a " "platform uses (`mednafen_psx`) the profile it designates (`beetle_psx`), " "whose document is `api/v1/emulators/.json`. A name two profiles " "claim (`pcsx2`) has one item per profile, and `preferred: true` marks the " "one a platform list naming that core gets: the profile with that very id, " "else the first id in alphabetical order.", "", "One hash without the whole list: `api/v1/lookup/.json`, where `` " "is the first two characters of the hash in lowercase, holds every SHA1, " "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` " "(the collection, a data directory, a platform's list or an emulator " "profile), the file `name`, the `url` of the page that shows it and, " "when one is known, the `system_name` the site gives its system. " "[Find or check a file](files/index.md) reads it.", "", "In a browser, the SHA1 of a file the reader picks finds its records. Web " "Crypto has no MD5, and none is needed: a setup that lists a file by MD5 " "or CRC32 is filed under the SHA1 of the collection file it names too.", "", "```js", "// The records of the file a reader picked, by its SHA1.", "async function lookUp(file) {", " const digest = await crypto.subtle.digest(\"SHA-1\", await file.arrayBuffer());", " const sha1 = [...new Uint8Array(digest)]", " .map((byte) => byte.toString(16).padStart(2, \"0\"))", " .join(\"\");", f" const base = \"{SITE_URL}api/v1/lookup/\";", " const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);", " if (!response.ok) return [];", " const shard = await response.json();", " // layer: collection, data-directory, platform or emulator. owner: which one.", " return shard.items.filter((item) => item.value === sha1);", "}", "```", "", "Two kinds of `url`:", "", "- In `catalog.json` and `index.json`, a `url` is relative to the document " "holding it, as RFC 3986 resolves it.", "- In the lookup shards, `file-locations.json` and `systems.json`, a `url` " f"is site-absolute (`/retrobios/systems/sony/#...`). Prefix " f"`{SITE_URL.removesuffix(urllib.parse.urlsplit(SITE_URL).path)}` for a full address.", "", "## Platform manifests", "", "A platform's list is what the setup checks. Its manifest is what the pack " "installs: the list's files plus every file the setup's cores load. " "`api/v1/manifests/.json` is the file `install.py` reads, byte " "for byte, valid against " "[`install-manifest.schema.json`](api/v1/schemas/install-manifest.schema.json). " "[`api/v1/release.json`](api/v1/release.json) records the packs of the latest " "release and the files each one holds.", "", "Manifests: " + ", ".join( f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})" for item in manifests) + "." if manifests else "", "", "`api/v1/targets/.json` gives each hardware target of a platform " "the cores it runs, valid against " "[`target-manifest.schema.json`](api/v1/schemas/target-manifest.schema.json). " "`install.py --target` reads the same file. Like the manifests, they are " "copied byte for byte and are not envelopes.", "", "Target lists: " + ", ".join( f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})" for item in targets) + "." if targets else "", "", "| Field | Meaning |", "|-------|---------|", *_schema_field_rows(manifest_schema, MANIFEST_TOP_FIELDS), "", "Each entry of `files`:", "", "| Field | Meaning |", "|-------|---------|", *_schema_field_rows(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), "", *markup.details("note", "All fields", [ "The manifest:", "", *_other_fields(manifest_schema, MANIFEST_TOP_FIELDS), "", "Each entry of `files`:", "", *_other_fields(manifest_schema["$defs"]["file"], MANIFEST_FILE_FIELDS), "", *_nested_fields(manifest_schema), *_nested_fields(manifest_schema["$defs"]["file"], "files[]."), ]), "## Check a folder for one platform", "", "The job most tools come for: is this BIOS folder what the platform checks? " "Read `api/v1/platforms/.json` and apply the platform's own rules:", "", "- `verification_mode: existence`: a file at its `destination` passes.", "- An entry with no hash in the mode's field is checked on presence too.", "- `zipped_file` set: the md5 is that of this member inside the ZIP, never " "the ZIP's. Member names compare casefolded. A ZIP declared once per member " "appears once per member: Batocera lists 692 entries for 353 destinations.", "- An md5 shorter than 32 characters (Batocera) compares by its prefix. " "Recalbox writes several accepted md5 in one field, comma separated.", "- `zip_digest: composite` (Recalbox): the md5 of a ZIP is that of its members " "read in name order, never the archive's.", "- An entry with no `destination` (EmuDeck) names no file: the system " "passes when any file in the BIOS folder has one of its md5.", "- `hash_rule: size_and_any_hash` (RomM) wants the size and any one hash. " "The script below does not apply it.", *([f"- {_join_names(no_list)} have no list of their own (`native_list: none`). " "An empty list does not mean nothing is needed."] if no_list else []), "", "```python", snippet.rstrip("\n"), "```", "", "The script is [`scripts/check_platform_folder.py`]" f"({REPO_URL}/blob/main/scripts/check_platform_folder.py). A test runs it " "against the verifier on a sample folder, so the two give the same answer. " "It exits 1 when a file is wrong or missing. For an unknown platform it " "prints the known ones.", "", "`install.py --check` answers another question by default: whether the folder " "holds what the pack installs, the manifest above. With `--declared-only`, or " "from the `declared` event, `install.py --check` answers it too.", "", "## Bulk downloads", "", "| Export | Format | Size | SHA256 |", "|--------|--------|-----:|--------|", ]) for item in exports: if item["media_type"] == "application/json": continue lines.append( f"| [{item['title']}]({item['url']}) | `{item['media_type']}` | " f"{_fmt_size(item['bytes'])} | {_short_sha256(item, 'api/v1/catalog.json')} |" ) lines.extend([ "", "Each SHA256 is cut to 12 characters. Point at it for the whole value, or " "read it in [`catalog.json`](api/v1/catalog.json).", "", "The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game " "per `bios///` folder of the collection, named " "`Manufacturer\\Console` so the tools look in that subfolder, each file a " "rom named by its path below that folder. It audits a copy of the " "collection's `bios/` tree, or rebuilds that layout from your own dumps. A " "platform pack has its own layout and is checked with `install.py --check`.", "", "The SQLite file contains indexed tables for content hashes, provenance " "catalog matches, platform declarations, emulator profiles and current " "gaps. It contains metadata only, never BIOS or firmware bytes.", "", "To find who accepts a hash, query `accepted_hashes` (or " "`accepted-hashes.csv`): one lowercase row per accepted value, per " "algorithm, for platform declarations and emulator profiles alike, " "indexed on `(algorithm, value)`. A ROM inside a MAME or FBNeo set " "has its own row, its `path` naming the archive and the member " "(`nmk004.zip/nmk004.bin`). The hash columns of `platform_files` " "and `emulator_files` keep each source's own form: a Recalbox md5 may " "list several revisions separated by commas, and an emulator field " "holding several accepted values is a JSON array.", "", "```sql", "SELECT layer, owner, name FROM accepted_hashes", "WHERE algorithm = 'md5' AND value = 'c87a52c277f7952b41c639fc7bf0a43b';", "```", "", "The gaps dataset carries both layers behind one `layer` column: " "`platform` rows are anomalies against a platform's own BIOS list, " "`emulator` rows are files a profiled core loads that no platform " "declares. The two answer different questions and are not comparable " "totals. Each row is listed once.", "", "`platform-files.csv` and the gaps add what the check found: `status` " "(`ok`, `untested`, `missing`), `category` (`bios`, `game_data`, " "`unread`), and for a held file `collection_path` and `resolved_sha1`, " "the collection file that answers the declaration. " "`provenance-wanted.csv` is the Wanted list of [Dump provenance]" "(provenance.md): each dump a catalog describes and the collection lacks.", "", "`platform-files.csv` and the `platform_files` table also carry the entry " "fields a check needs:", "", "- `zipped_file` set: the md5 is the md5 of that member inside the ZIP.", "- `zip_digest` is `composite` where the md5 of a ZIP covers its members.", "- `hash_match_mandatory` and `mandatory_declared` are Recalbox's flags.", "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's " "own file does not carry.", "", "## Hosting, rate limits and caching", "", "- Static files on GitHub Pages, read with a plain GET. No key, no account.", "- Any origin may read them (`Access-Control-Allow-Origin: *`), so a " "browser tool can call them directly.", "- Responses are cached for 10 minutes (`Cache-Control: max-age=600`). " "Send the `ETag` back in `If-None-Match` to skip an unchanged download.", "- GitHub may answer HTTP 429 to a client that sends too many requests, and " "a Pages site has a soft limit of 100 GB of traffic a month ([GitHub Pages " "limits](https://docs.github.com/en/pages/getting-started-with-github-pages/" "github-pages-limits)). Fetch the one shard or document a question needs.", "- The base is the project's address, " f"`{SITE_URL}api/v1/`. Compare `commit` to know whether two downloads differ.", "", "## Semantics and limits", "", "Treat four questions separately: content identity (hashes), presence in " "the collection, acceptance by a specific emulator, and catalog provenance. " "One does not imply the others. In particular, presence, a matching dump " "catalog, or emulator compatibility is not a statement about copyright, " "ownership, or redistribution rights.", "", f"The data can be newer than the [latest published pack]({RELEASE_URL}). " "A pack is published by hand, once every release check has passed.", "", "See the [data model](wiki/data-model.md), [verification modes]" "(wiki/verification-modes.md), and [methodology](wiki/architecture.md) " "before interpreting aggregate counts.", "", ]) return "\n".join(lines) def _page_title(markdown: str, path: Path) -> str: match = re.search(r"^#\s+(.+?)\s*$", markdown, flags=re.MULTILINE) if match: title = re.sub(r"<[^>]+>", "", match.group(1)) return title.replace(f" - {SITE_NAME}", "").strip() return path.stem.replace("-", " ").title() def _browser_title(relative: Path, title: str) -> str: """Return a concise, unique browser/search title for a generated page.""" key = relative.as_posix() index_titles = { # Material appends " - RetroBIOS": the home title says what the site # is, instead of "RetroBIOS - RetroBIOS". "index.md": "BIOS and firmware packs for RetroArch, Batocera and more", "platforms/index.md": "Your setup", "systems/index.md": "Systems", "emulators/index.md": "Emulators", "wiki/index.md": "Reference", "explore.md": "Explore", } if key in index_titles: return index_titles[key] # A console page is filed with the parts of its manufacturer's page, and # titled "Sega: Dreamcast". A part names its range of files. part = key in PART_PAGES and ( not key.startswith("systems/") or ": " not in title or re.search(r", part \d+ of \d+$", title)) if part: return title # The words a web search for a console or an emulator types: "dreamcast # bios", "duckstation bios". The h1 and the sidebar keep the name. if key.startswith("emulators/"): has_files = PAGE_DATA.get(key, {}).get("hasPart") # Lexaloffle, QEMU and ScummVM name a system page too. own = " emulator" if f"systems/{relative.name}" in PAGE_DATA else "" return f"{title}{own} BIOS files" if has_files else f"{title} emulator" if key.startswith("systems/"): maker, _, console = title.partition(": ") return f"{console} BIOS files ({maker})" if console else f"{title} BIOS files" return title def _plain_markdown(text: str) -> str: text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"!\[([^]]*)\]\([^)]*\)", r"\1", text) text = re.sub(r"\[([^]]+)\]\([^)]*\)", r"\1", text) # Code spans keep their text whole: validate_schemas.py read # validateschemas.py once every underscore was taken for emphasis. pieces = re.split(r"(`[^`]*`)", text) text = "".join( piece.strip("`") if piece.startswith("`") else re.sub(r"(? str: """A description cut at a word, never inside one: six snippets ended in "op..." or "Fo...".""" if len(plain) <= limit: return plain cut = plain[: limit + 1].rsplit(" ", 1)[0] if " " in plain[:limit] else plain[:limit] return cut.rstrip(" ,;:-(.") + "..." def _page_description(markdown: str, relative: Path, title: str) -> str: key = relative.as_posix() specific = { "index.md": "The BIOS files your emulators need, for RetroArch, Batocera, EmuDeck and more: the folder they go in, one command that installs and checks them, and packs to download.", "data.md": "Versioned RetroBIOS JSON API, CSV exports, SQLite snapshot, schemas, checksums and data interpretation guidance.", "cross-reference.md": "Every emulator core each retrogaming platform runs, the systems it emulates and the upstream project it comes from.", "gaps.md": "Current RetroBIOS verification gaps, platform-to-emulator divergences, missing files and documented source limitations.", "exports.md": "Each platform's own BIOS list (System.dat, batocera-systems, es_bios.xml and the rest) with the hashes and files the emulator source code shows, offered to its maintainers.", "provenance.md": "Hash-based comparison of RetroBIOS metadata with No-Intro, Redump and TOSEC catalog snapshots.", "explore.md": "Systems, emulators and installer options to explore, with the figures of the BIOS collection.", "which-pack.md": "Pick your device or setup and get its BIOS files: the folder they go in, one command that installs and checks them, or a ZIP pack to extract.", "contributing.md": "Add a BIOS file the collection lacks from the browser, checked on arrival, or add a platform or an emulator profile.", } if key in specific: return specific[key] # What a search engine shows is the page's own first sentence: 707 pages # shared five templates of level 3 words ("source-pinned", "provenance"). browser = _browser_title(relative, title) held = re.search(r"\bThis part holds (.+?)\.(?: |$)", markdown, re.M) if key in PART_PAGES and held: return _shorten(f"{browser}. This part holds {_plain_markdown(held[1])}.", 155) lead = _lead(markdown) if key in PART_PAGES and re.search(r"\bpart \d+ of \d+", title): # The parts of one list open on the same sentence. first = _first_paragraph(lead) or _first_paragraph(markdown) return _shorten(f"{title}. {first}", 155) if key.startswith(f"{release_record.GET_DIR}/"): size = re.search(r"^- (.+ to download\b.*?)\.?$", markdown, re.M) if size: return _shorten(f"{title}: the BIOS pack of this setup, {_plain_markdown(size[1])}. " "The address stays the same and always gives the latest ZIP.", 155) return _shorten(_first_paragraph(lead) or title, 155) if key.startswith("systems/") and relative.stem != "index": return _system_description(markdown, browser) if key.startswith("emulators/") and relative.stem != "index": return _emulator_description(markdown, key, title, _first_paragraph(lead)) first = _first_paragraph(lead, minimum=35) questions = [_plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading)) for heading in re.findall(r"^## (.+\?)(?:[ \t]*\{[^}]*\})?[ \t]*$", markdown, re.M)] if not first and len(questions) >= 3: # A page of questions with no intro: its first paragraph is the # answer to one of them ("Run the command with --dry-run"). return _shorten(f"{title}: {' '.join(questions)}", 155) first = first or _first_paragraph(markdown, minimum=35) if first: return _shorten(first, 155) return f"{title}: RetroBIOS source-traced retrogaming firmware reference." def _lead(markdown: str) -> str: """The text a page opens with, before its first section, details block or ignored section (the install box, the profile table).""" return re.split(r"^(?:## | str: """An emulator page by its intro sentence ("DuckStation emulates Sony PlayStation. It picks the BIOS for your game's region: ..."), or by its profile table when the profile names no system.""" notes = re.search(r"Technical notes\s*\n\n(.+?)(?:\n\n|$)", markdown, re.S) if not intro and notes: sentence = re.split(r"(?<=\.)\s", _plain_markdown(notes[1]), maxsplit=1)[0] intro = f"{title}: {sentence[:1].lower()}{sentence[1:]}" if not intro: row = {name: _plain_markdown(value) for name, value in re.findall( r"^\| (Type|Classification|Version) \| (.+?) \|$", markdown, re.M)} kind = row.get("Classification") or row.get("Type") or "emulator" version = row.get("Version", "") known = f", version {version}" if version and version[0].isdigit() else "" intro = f"{title}: {kind.lower()}{known}." elif title not in intro: # "It reads 1 file": the profile names no system to open on. intro = f"{title}: {intro[:1].lower()}{intro[1:]}" # An alias page sends the reader to the core whose files it shares. if PAGE_DATA.get(key, {}).get("hasPart") or "uses the same firmware as" in intro: return _shorten(intro, 155) needs = "It needs no BIOS or firmware file." if "\n## Data directories" in markdown: needs = "It needs no BIOS or firmware file, only the data folders listed." return _shorten(f"{intro} {needs}", 155) # Lines a page opens with that say where it sits, not what it holds. _PLACE_LINES = ("Part of ", "This part holds ", "On a card, ") def _first_paragraph(markdown: str, minimum: int = 1) -> str: """The first paragraph of prose of a page body, as plain text: no heading, table, list, fence, admonition or HTML block, and none that only says where the page sits.""" for paragraph in re.split(r"\n\s*\n", markdown): stripped = paragraph.strip() if ( not stripped or stripped.startswith(("#", "|", "```", "???", ":::", "<", "- ", "* ", "import ")) ): continue plain = _plain_markdown(stripped) if plain.startswith(_PLACE_LINES): continue # A paragraph that ends on a colon introduces what follows: alone, # it describes nothing ("One file needs no clone and no local setup:"). if len(plain) >= minimum and not plain.endswith(":"): return plain return "" # Section headings of a system page that are not a console. _NOT_A_CONSOLE = {"Next steps", "Files", "Other files"} _CARD_TITLE = re.compile(r'class="rb-file-title"[^>]*>([^<]+)') def _system_description(markdown: str, browser: str) -> str: """A system page by what it holds: a manufacturer page by its consoles, a console page by its "Start with" line, else by its first files.""" consoles = [ _plain_markdown(re.sub(r"\s*\{[^}]*\}\s*$", "", heading)) for heading in re.findall(r"^## (.+?)[ \t]*$", markdown, re.M) ] consoles = [name for name in consoles if name and name not in _NOT_A_CONSOLE] if len(consoles) > 1: shown = consoles if len(consoles) <= 4 else consoles[:3] + [f"{len(consoles) - 3} more"] return _shorten(f"{browser} for {_said_as_list(shown)}, one card per file with its " "hashes and the emulators that read it.", 155) start = re.search(r"^\*\*Start with\*\*(?P.*)$(?P(?:\n+- .*)*)", markdown, re.M) if start: # "Start with the files of your machine:" introduces one line per machine. machines = [_plain_markdown(item) for item in re.findall(r"^- (.+)$", start["list"], re.M)] text = _plain_markdown(start["line"]).lstrip(": ") said = "; ".join(machines) if machines and text.endswith(":") else text return _shorten(f"{browser}. Start with {said}", 155) names = list(dict.fromkeys(html.unescape(name) for name in _CARD_TITLE.findall(markdown))) if names: shown = names if len(names) <= 3 else names[:3] + [f"{len(names) - 3} more"] return _shorten(f"{browser}: {_said_as_list(shown)}, with their hashes and the " "emulators that read them.", 155) return _shorten(f"{browser}, one card per file with its hashes and the emulators " "that read it.", 155) # Structured data a page generator knows beyond its text, by page path # relative to docs/: read by decorate_markdown_pages. PAGE_DATA: dict[str, dict] = {} # Pages that hold part of a page too long for one (a console of a split # manufacturer, a run of an emulator's cards), by path relative to docs/: # their H1 already says what they are. PART_PAGES: set[str] = set() SYSTEM_PARTS_IN_FULL = 250 def collection_dat(db: dict, generated_at: str) -> str: """The collection as a Logiqx DAT, the format clrmamepro and RomVault read. One game per bios/// folder, named Manufacturer\\Console, each file a rom named by its path below that folder, subfolders joined with a backslash as Logiqx DATs write them. An auditor checking a BIOS tree against the collection had JSON, CSV and SQLite, and a converter to write. """ import xml.etree.ElementTree as ET # noqa: PLC0415 games: dict[str, list[dict]] = {} files = db.get("files", {}) # Every copy, not the one path the database keeps: an audit of bios/ # read the 2,502 others as unknown files, and a fix moved them away # from the engines that read them by path. for sha1, paths in collection_paths(db).items(): for path in sorted(paths): parts = path.split("/") folder = parts[1:3] if len(parts) > 3 else parts[1:2] # A backslash is how a Logiqx set name writes a subfolder: named # "Sony - PlayStation", the set was looked for beside Sony/, and an # audit of a copy of bios/ found 330 sets missing. games.setdefault("\\".join(folder), []).append( {**files[sha1], "_rom": "\\".join(parts[1 + len(folder):])} ) root = ET.Element("datafile") header = ET.SubElement(root, "header") version = generated_at[:10].replace("-", "") for tag, text in ( ("name", SITE_NAME), ("description", f"{SITE_NAME} collection ({sum(map(len, games.values()))} files)"), ("version", version), ("date", generated_at[:10]), ("author", SITE_NAME), ("homepage", SITE_URL), ("url", REPO_URL), ): ET.SubElement(header, tag).text = text for game_name in sorted(games): game = ET.SubElement(root, "game", name=game_name) ET.SubElement(game, "description").text = game_name for entry in sorted(games[game_name], key=lambda e: e["_rom"]): attributes = {"name": entry["_rom"], "size": str(entry.get("size", 0))} for attribute, field in (("crc", "crc32"), ("md5", "md5"), ("sha1", "sha1")): if entry.get(field): attributes[attribute] = str(entry[field]).lower() ET.SubElement(game, "rom", attributes) ET.indent(root, space="\t") return ( '\n' '\n' + ET.tostring(root, encoding="unicode") + "\n" ) def system_structured_data(slug: str, consoles: dict[str, list]) -> dict: """A system page's files for a machine: each card as a CreativeWork. The pages that hold a card per file published a bare TechArticle with no file, hash or provenance, while emulator pages listed theirs. """ parts = [] for _console, entries in sorted(consoles.items()): for entry in entries: sha1 = str(entry.get("sha1") or "") part: dict = {"@type": "CreativeWork", "name": str(entry.get("name", ""))} if sha1: part["url"] = urllib.parse.urljoin(SITE_URL, f"systems/{slug}/#sha1-{sha1[:12]}") identifiers = [ f"{field}:{str(entry[field]).lower()}" for field in ("sha1", "md5", "crc32", "sha256") if entry.get(field) ] if identifiers: part["identifier"] = identifiers if entry.get("size"): part["contentSize"] = f"{entry['size']} B" citations = [ f"{catalog}: {record.get('dat', '')} ({record.get('name', '')})" for catalog, record in sorted((entry.get("provenance") or {}).items()) if isinstance(record, dict) ] if citations: part["citation"] = citations parts.append(part) if len(parts) > SYSTEM_PARTS_IN_FULL: # A page of cards carried up to 2.6 MB of JSON-LD (game-engines), a # quarter of its weight: past the cap a page lists its # catalogue-verified files and points at the full table. return { "hasPart": [part for part in parts if part.get("citation")], "isBasedOn": urllib.parse.urljoin(SITE_URL, "downloads/files.csv"), } return {"hasPart": parts} if parts else {} def platform_structured_data(name: str, coverage: dict) -> dict: """A platform page's declarations for a machine reader. The page printed thousands of hashes while its JSON-LD named only the page: a crawler could not learn which md5 Batocera requires for a file. The software and its version are the subject, and each declared file, with the hashes the platform checks, is a part. Past SYSTEM_PARTS_IN_FULL the parts give way to the platform-files table, as system pages do. """ config = coverage.get("config") or {} about: dict = {"@type": "SoftwareApplication", "name": str(coverage.get("platform", name))} if config.get("version"): about["softwareVersion"] = str(config["version"]) if config.get("homepage"): about["url"] = str(config["homepage"]) parts = [] for system_id, system in sorted((config.get("systems") or {}).items()): for entry in system.get("files") or []: if not isinstance(entry, dict) or not entry.get("name"): continue part: dict = { "@type": "CreativeWork", "name": str(entry.get("destination") or entry["name"]), "url": urllib.parse.urljoin(SITE_URL, f"platforms/{name}/#{system_id}"), } identifiers = [ f"{field}:{value}" for field in ("sha1", "md5", "crc32", "sha256") for value in _hash_values(entry.get(field)) ] if identifiers: part["identifier"] = identifiers parts.append(part) data: dict = {"about": about} if len(parts) > SYSTEM_PARTS_IN_FULL: data["isBasedOn"] = urllib.parse.urljoin(SITE_URL, "downloads/platform-files.csv") elif parts: data["hasPart"] = parts return data def dataset_structured_data(exports: list[dict], db: dict) -> dict: """The data page as a schema.org Dataset a harvester can index. It was a Dataset named "Data & API" with no download, licence, date or creator, so Google Dataset Search and open-data catalogues had nothing to list. The exports are metadata this project authors: MIT, as the LICENSE's scope says. """ data: dict = { "name": f"{SITE_NAME} collection data", "license": "https://opensource.org/licenses/MIT", "creator": {"@type": "Organization", "name": SITE_NAME, "url": REPO_URL}, "distribution": [ { "@type": "DataDownload", "name": item["title"], "contentUrl": urllib.parse.urljoin(SITE_URL, item["url"]), "encodingFormat": item["media_type"], "contentSize": f"{item['bytes']} B", "sha256": item["sha256"], } for item in exports ], } generated = str(db.get("generated_at") or "")[:10] if generated: data["dateModified"] = generated return data def emulator_file_anchors(files: list[dict]) -> list[str]: """The id of each file card on an emulator page, in profile order. An entry that declares a hash is addressed by it: its anchor depends on its own content, never on the entries beside it, and a second version added under the same path moves nothing. An entry without one keeps its plain anchor when its path is declared once, and otherwise takes its size, else, when even the size is shared, its description. The page and its JSON-LD both read this, so a structured part links to its card. The plain anchor of a hashed card stays reachable through emulator_file_aliases. """ bases = [ "file-" + ( re.sub( r"[^a-z0-9]+", "-", str(f.get("path") or f.get("standalone_path") or f.get("name", "")).lower(), ).strip("-") or "entry" ) for f in files ] repeated = {b for b in bases if bases.count(b) > 1} def _version_key(f: dict) -> str: digest = _first_hash_value(f) if digest: return digest[:8] return f"{f['size']}b" if f.get("size") else "" keys = [ (b, _version_key(f)) if b in repeated or _first_hash_value(f) else (b, "") for f, b in zip(files, bases) ] used: dict[str, int] = {} anchors = [] for f, (base, key) in zip(files, keys): anchor = f"{base}-{key}" if key else base if keys.count((base, key)) > 1 and f.get("description"): described = re.sub(r"[^a-z0-9]+", "-", str(f["description"]).lower()).strip("-") anchor = f"{anchor}-{described[:40].strip('-')}" used[anchor] = used.get(anchor, 0) + 1 if used[anchor] > 1: anchor = f"{anchor}-{used[anchor]}" anchors.append(anchor) return anchors def emulator_file_aliases(files: list[dict], anchors: list[str]) -> list[str]: """The plain file- id each card also answers to, or "". A link cited as emulators/raze/#file-duke3d-grp must still land once the card is addressed by its hash: the first card of that path keeps the plain id, unless another card already is it. """ taken = set(anchors) aliases = [] for f, anchor in zip(files, anchors): base = "file-" + ( re.sub( r"[^a-z0-9]+", "-", str(f.get("path") or f.get("standalone_path") or f.get("name", "")).lower(), ).strip("-") or "entry" ) if base in taken or anchor == base: aliases.append("") else: taken.add(base) aliases.append(base) return aliases def emulator_structured_data(profile: dict) -> dict: """What an emulator page states, for a reader of its JSON-LD. The block said only the page name: a crawler could not learn which files the core loads, their hashes, the source read or when. """ data: dict = {} # Every repository the profile reads, each with its own pin: xemu's # libretro build lives in another repository at another commit, and a # port's upstream_commit was never published. about = [] seen: set[str] = set() for field in ("source", "upstream"): urls = profile.get(field) pins = profile.get(f"{field}_commit") for mode, url in (urls.items() if isinstance(urls, dict) else [("", urls)]): if not url or not str(url).startswith("http") or str(url) in seen: continue seen.add(str(url)) pin = pins.get(mode) if isinstance(pins, dict) else pins entry = { "@type": "SoftwareSourceCode", "name": str(profile.get("emulator", "")), "codeRepository": str(url), "description": " ".join(filter(None, [mode, field])), } if pin: entry["version"] = str(pin) about.append(entry) if about: data["about"] = about[0] if len(about) == 1 else about if profile.get("profiled_date"): data["dateModified"] = str(profile["profiled_date"]) parts = [] files = [f for f in profile.get("files") or [] if isinstance(f, dict)] for f, anchor in zip(files, emulator_file_anchors(files)): if not f.get("name"): continue # One identifier per accepted value: "sha256:a,b,c,d" matched no # exact search (mesence's BS-X.bin). identifiers = [ f"{field}:{value}" for field in ("sha1", "sha256", "md5", "crc32") for value in _hash_values(f.get(field)) ] part = { "@type": "CreativeWork", "name": str(f.get("path") or f["name"]), # The card's own anchor, as system and platform parts carry. "url": f"#{anchor}", } if identifiers: part["identifier"] = identifiers # An archive is identified by its members: MAME and FBNeo hash them, # never the ZIP, and 1 338 archive parts had no identifier at all. members = [ { "@type": "CreativeWork", "name": str(member.get("name", "")), "identifier": [ f"{field}:{value}" for field in ("sha1", "crc32") for value in _hash_values(member.get(field)) ], } for member in f.get("contents") or [] if isinstance(member, dict) and member.get("name") and (member.get("sha1") or member.get("crc32")) ] if members: part["hasPart"] = members parts.append(part) if len(parts) > SYSTEM_PARTS_IN_FULL: # MAME's 1,564 entries and their members were 614 KB of JSON-LD: past # the cap the members are left to the profile's JSON. for part in parts: part.pop("hasPart", None) if parts: data["hasPart"] = parts return data # Working notes kept under docs/ during development, never part of the site, # and no generated page lives in a directory of these names. LOCAL_NOTE_DIRS = ("plans", "specs") # The funnel: who each page is written for, from someone who knows nothing # (start) to someone adding to the project (contribute). The label sits above # the title, and the search ranks the first two levels higher. LEVELS = { "start": "Start here", "setup": "Setup guide", "explore": "Details", "reference": "Reference", "contribute": "For contributors", } # How much the site search weighs the words of a page (Pagefind's # data-pagefind-weight, 1 by default). Each value was measured on the # queries of website/tools/search_gate.py. # - Start here pages answer a beginner's words first. # - A system page holds the file cards themselves: a file name finds its # card before a guide that cites the name as an example. # - A setup guide is found by its name (the page title weighs it). # - A guide behind the first two levels (reference, options, contributing) # names everything a reader types, and ranks after the page that answers. # - A launcher, an alias or a frozen copy answers a name after the emulator. # - A part of a split page (a console's files, 5 of 9) holds cards, each found # by its own heading, and ranks after the page that answers the question. SEARCH_WEIGHT = {"start": 10, "setup": 1.5, "reference": 0.5, "contribute": 0.5, "part": 0.5} SYSTEM_PAGE_WEIGHT = 5 DEEP_GUIDE_WEIGHT = 0.5 SEARCH_DEMOTED: set[str] = set() # A tool page a reader reaches by a task, not its name: "check my dump", # "md5 check" found emulator pages and never the page that checks a file. SEARCH_WEIGHT_OF_PAGE = {"files/index.md": 2} DEMOTED_WEIGHT = 0.5 def search_weight(relative: Path, level: str) -> float | None: """The search weight of a page, None for the default. A part page states its own in its front matter: search_weight(path, "part").""" if level == "part": return SEARCH_WEIGHT["part"] if relative.as_posix() in SEARCH_WEIGHT_OF_PAGE: return SEARCH_WEIGHT_OF_PAGE[relative.as_posix()] if relative.as_posix() in SEARCH_DEMOTED: return DEMOTED_WEIGHT if relative.parts[0] == "systems": return SYSTEM_PAGE_WEIGHT if relative.parts[0] == "wiki" and level not in ("start", "setup"): return DEEP_GUIDE_WEIGHT return SEARCH_WEIGHT.get(level) _REFERENCE_PAGES = { "gaps.md", "cross-reference.md", "provenance.md", "exports.md", "data.md", "files/index.md", "wiki/data-model.md", } def page_level(relative: Path) -> str: """The level of a generated page. A wiki page states its own.""" path = relative.as_posix() if path in ("index.md", "which-pack.md"): return "start" if relative.parts[0] in ("platforms", release_record.GET_DIR): return "setup" if relative.parts[0] in ("systems", "emulators") or path == "explore.md": return "explore" if path in _REFERENCE_PAGES: return "reference" if path == "contributing.md": return "contribute" return "" def level_label(level: str) -> str: """The visible label of a level, plain text for the theme to style.""" return f'{LEVELS[level]}' def _front_matter(markdown: str) -> dict: """The YAML front matter of a page, empty when it has none.""" if not markdown.startswith("---\n"): return {} end = markdown.find("\n---\n", 4) if end == -1: return {} data = yaml.safe_load(markdown[4:end]) return data if isinstance(data, dict) else {} def decorate_markdown_pages(docs: Path) -> dict[str, str]: """Add per-page descriptions and machine-readable structured data. Under Starlight each page is written in its final form (front matter, site addresses, pinned heading ids) and the dates a page states are returned by address, for the sitemap. """ lastmod: dict[str, str] = {} terms = sitestarlight.glossary_terms(_REPO_ROOT / WIKI_SRC_DIR / "glossary.md") for path in sorted(docs.rglob("*.md")): relative = path.relative_to(docs) if any(part in LOCAL_NOTE_DIRS for part in relative.parts[:-1]): continue original = path.read_text(encoding="utf-8") authored = _front_matter(original) if "head" in authored: # Written in its final form by an earlier pass and rewritten by # nothing since: a second pass would find no title to read. continue markdown = prepare_page(original, relative) for image_id in unresolved_images(markdown): print(f" WARNING: {relative}: no diagram named {image_id!r}", file=sys.stderr) stripped = _undecorated(markdown) if stripped is not markdown: markdown = stripped elif authored: # A wiki page states its own level: keep it, wrap the body. markdown = markdown[markdown.find("\n---\n", 4) + 5:].lstrip("\n") level = authored.get("level") or page_level(relative) title = _page_title(markdown, relative) browser_title = _browser_title(relative, title) description = str(authored.get("description") or _page_description(markdown, relative, title)) if relative.name == "index.md": page_path = "" if len(relative.parts) == 1 else "/".join(relative.parts[:-1]) + "/" else: page_path = relative.with_suffix("").as_posix() + "/" # Encoded as mkdocs encodes the canonical link and the sitemap: # idtech4a++ named .../idtech4a++/ here and .../idtech4a%2B%2B/ there. canonical = urllib.parse.urljoin(SITE_URL, urllib.parse.quote(page_path)) schema_type = "WebSite" if relative.as_posix() == "index.md" else "TechArticle" if relative.as_posix() == "data.md": schema_type = "Dataset" structured = { "@context": "https://schema.org", "@type": schema_type, "name": title, "description": description, "url": canonical, "isPartOf": { "@type": "WebSite", "name": SITE_NAME, "url": SITE_URL, }, **PAGE_DATA.get(relative.as_posix(), {}), } target, text = sitestarlight.finalize_page( relative.as_posix(), markdown, level=level, weight=search_weight(relative, level), browser_title=browser_title, description=description, structured=structured, terms=terms, search=authored.get("search") != "exclude", logo=str(authored.get("logo") or ""), parent=authored.get("parent"), authored=authored, ) write_decorated(docs / target, text) # One file per page: a page that gained or lost its components # changed extension, and the other one would claim its address. stem = relative.with_suffix("").as_posix() for suffix in (".md", ".mdx"): if f"{stem}{suffix}" != target: (docs / f"{stem}{suffix}").unlink(missing_ok=True) if structured.get("dateModified"): lastmod[sitestarlight.page_url(target)] = str(structured["dateModified"]) return lastmod # Platform pages def generate_platform_index(coverages: dict, registry: dict | None = None) -> str: """The Your setup tab: every platform by the device it runs on, and one line to the table that sets them side by side (setup_table), which is reference matter and sits with how each setup checks its files.""" registry = registry or {} lines = [ "# Your setup", "", "Each setup's page gives the folder and the command.", "", ] for group, (label, _description) in DEVICE_GROUPS.items(): members = _group_members(coverages, registry, group) if not members: continue # The group label names the devices, and each card carries its # setup's own sentence. The kinds of device that run a setup # follow, as on the home tiles. lines += [f"## {label}", ""] lines += markup.card_grid([ *((f"[{coverages[name]['platform']}]({name}.md)", [GUIDES[name].summary]) for name in members), *((f"[{tile.name}](../{tile.link})", [tile.text]) for tile in GROUP_TILES.get(group, ())), ]) lines += [ f"Comparing setups? [Every setup side by side](../{SETUP_TABLE_LINK})", "", ] return "\n".join(lines) + "\n" # Where the side-by-side table of setups is published: with how each setup # checks its files, the column a reader compares. SETUP_TABLE_PAGE = "wiki/verification-modes.md" SETUP_TABLE_ANCHOR = "every-setup-at-a-glance" SETUP_TABLE_LINK = f"{SETUP_TABLE_PAGE}#{SETUP_TABLE_ANCHOR}" # How a setup checks a file, in the words of a reader. _HOW_IT_CHECKS = {"existence": "name only", "md5": "MD5", "sha1": "SHA-1"} def setup_table(coverages: dict, registry: dict | None = None) -> list[str]: """Every setup side by side: the files on its own list, the files in its ZIP, and how it checks them. Written under the Reference level, on the page that explains each way of checking.""" registry = registry or {} lines = [ f"## Every setup at a glance {{ #{SETUP_TABLE_ANCHOR} }}", "", "| Setup | Files on its own list | Files in the ZIP | How it checks |", "|-------|------:|------:|---------------|", ] archived = False for name, cov in sorted(coverages.items(), key=lambda item: item[1]["platform"].casefold()): status = registry.get(name, {}).get("status") archived |= status == "archived" listed = f"{cov['total']:,}" if cov["total"] else "No list of its own" files, _size = release_totals(name, release_record.load_advertised()) zipped = (f"{files:,}" if files else "No ZIP yet" if has_release_pack(name) else "No ZIP") lines.append( f"| [{cov['platform']}](../platforms/{name}.md)" f"{' (archived)' if status == 'archived' else ''} | {listed} | {zipped} | " f"{_HOW_IT_CHECKS.get(cov['mode'], cov['mode'])} |" ) lines += [ "", "The ZIP adds the files its emulators load to the files on the setup's own list.", "", ] lines += ["No ZIP: the setup is served by the command alone. " f"[Why?](../{NO_ZIP_FAQ})", ""] if archived: lines += ["An archived setup keeps its page and the command. Its list is no longer " "read again on a schedule.", ""] return lines def with_setup_table(page: str, coverages: dict, registry: dict | None = None) -> str: """The comparison table placed before the Next steps of its page, or at its end.""" table = "\n".join(setup_table(coverages, registry)) marker = "\n## Next steps" if marker in page: head, tail = page.split(marker, 1) return f"{head.rstrip()}\n\n{table}{marker}{tail}" return f"{page.rstrip()}\n\n{table}" # A platform system whose file list passes this much Markdown gets a page # of its own: RetroDECK's arcade list was 750 KB of a 1.1 MB page. PLATFORM_SECTION_BUDGET = 200_000 # A system's own page in runs of at most this much Markdown: RetroDECK's # arcade list, each file with its served copy and pinned readers, rendered # 1.4 MB of HTML on one page. PLATFORM_PART_BUDGET = 350_000 # The Markdown of a platform page's file lists, past which the largest # systems move to pages of their own. PLATFORM_REFERENCE_BUDGET = 250_000 def _file_runs(body: list[str], budget: int) -> list[list[str]]: """A system's file list in runs of at most ``budget`` characters. The lines before the first file (counts, emulators) open every run. A file is never cut from its hash and evidence lines. """ first = next((i for i, line in enumerate(body) if line.startswith("- ")), len(body)) head, items = body[:first], [] for line in body[first:]: if line.startswith("- ") or not items: items.append([line]) else: items[-1].append(line) runs: list[list[str]] = [[]] size = 0 for item in items: weight = sum(len(line) + 1 for line in item) if runs[-1] and size + weight > budget: runs.append([]) size = 0 runs[-1].extend(item) size += weight return [head + run for run in runs] # A list moved off a reference page (gaps, provenance) is cut in parts of # at most this much Markdown: about 450 KB of HTML each. LIST_PART_BUDGET = 300_000 def _blocks_in_runs(blocks: list[list[str]], budget: int) -> list[list[str]]: """Blocks (table rows, folded lists) in runs of at most ``budget`` characters, a block never cut.""" runs: list[list[str]] = [[]] size = 0 for block in blocks: weight = sum(len(line) + 1 for line in block) if runs[-1] and size + weight > budget: runs.append([]) size = 0 runs[-1].extend(block) size += weight return runs def _parent_front_matter(parent: tuple[str, str] | None) -> list[str]: """The `parent:` front matter of a part page: the page it belongs to, by label and site path, which the breadcrumb and sidebar read.""" if parent is None: return [] label, link = parent return ["parent:", f" label: {json.dumps(label)}", f" link: {json.dumps(link)}"] def _list_pages( stem: str, title: str, intro: str, head: list[str], blocks: list[list[str]], moved: dict[str, str], budget: int = LIST_PART_BUDGET, parent: tuple[str, str] | None = None, ) -> str: """Pages of their own for a long list: ``head`` (a table header, say) opens each part, ``blocks`` are cut between parts. Returns where the list went, as the page it left says it. ``parent`` (label, site path) is the page the parts belong to, for the breadcrumb.""" runs = _blocks_in_runs(blocks, budget) stems = [stem + (f"-{n}" if n > 1 else "") for n in range(1, len(runs) + 1)] for n, (here, run) in enumerate(zip(stems, runs), 1): part = f" (part {n} of {len(runs)})" if len(runs) > 1 else "" # The page description is the opening of the intro: parts sharing # one read as duplicates to search and to validate_site. opening = f"Part {n} of {len(runs)}. {intro}" if len(runs) > 1 else intro moved[here] = "\n".join([ "---", "level: reference", *_parent_front_matter(parent), "---", "", f"# {title}{part}", "", opening, "", *([", ".join(f"[part {i}]({other}.md)" if other != here else f"part {i}" for i, other in enumerate(stems, 1)), ""] if len(runs) > 1 else []), *head, *run, "", _date_line(), ]) + "\n" if len(stems) == 1: return f"[a page of their own]({stems[0]}.md)" return "pages of their own: " + ", ".join( f"[part {i}]({other}.md)" for i, other in enumerate(stems, 1)) @functools.lru_cache(maxsize=1) def _collection_sha1_by_path() -> dict[str, str]: """The sha1 of each file the collection holds, by its path.""" with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle: files = json.load(handle).get("files", {}) return {str(entry.get("path", "")): sha1 for sha1, entry in files.items()} def _system_heading_link(sys_id: str, prefix: str = "") -> str: """A system's name, linked to its own console section when it has one. The id stays beside it in small text: it is what the platform's list writes. A system with no section of its own is named, never sent to the page a storage folder chose. """ return f"{_system_named_link(sys_id, prefix)} {_escape_tags(sys_id)}" def _system_named_link(sys_id: str, prefix: str = "") -> str: """A system's name, linked to the section system_section gives.""" from siterender import system_section name = _escape_tags(system_name(sys_id)) section = system_section(sys_id) if section is not None: slug, console = section name = f"[{name}]({console_href(slug, console, prefix)})" return name def _emulator_named_link(key: str, prefix: str = "") -> str: """An emulator by the name its page carries, linked to that page. A profile with no page (an alias, a test) is named only.""" profile = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False).get(key, {}) label = _escape_tags(str(profile.get("emulator") or key)) if profile.get("type") in ("alias", "test"): return label return f"[{label}]({prefix}emulators/{key}.md)" def _served_line(f: dict) -> str: """Which collection file answers the entry, from the resolver's result. "OK" with no path gave an expert nothing to check: Batocera's scph1001.bin is served by bios/Sony/PlayStation/scph1001_v20.bin. """ path = str(f.get("path") or "") if f.get("status") == "missing": return "Not held" if not path: # A slot the pack fills from a release asset or leaves to the user. return "" sha1 = _collection_sha1_by_path().get(path, "") label = "Served by" if f.get("status") == "ok" else "Held, not the declared hash" parts = path.split("/") if not sha1 or len(parts) < 3 or parts[0] != "bios": return f"{label}: {_code(path)}" # The card of that copy: every hash, provenance and the download. page = urllib.parse.quote(card_page(path, sha1)) return (f"{label}: [{_code(path)}](../systems/{page}.md#sha1-{sha1[:12]}) " f"(sha1 `{sha1[:8]}`)") # Platforms whose launcher records are read from the repository and revision # of their BIOS list: batocera_scraper reads configgen at its tag, # retrobat_scraper reads emulatorLauncher at the list's commit # (launcher_revision). The others read another repository. _LAUNCHERS_IN_LIST_REPO = frozenset({"batocera", "retrobat"}) def _launcher_refs(platform: str, config: dict) -> dict[str, list[str]]: """Entry name, casefolded -> each launcher record that opens it, rendered.""" from launchers import _names as launcher_names from sitesources import launcher_ref_markdown, list_source source = (list_source(str(config.get("source") or "")) if platform in _LAUNCHERS_IN_LIST_REPO else None) found: dict[str, list[str]] = {} for record in config.get("launchers") or []: if not isinstance(record, dict) or not record.get("source_ref"): continue text = (f"{_escape_tags(str(record.get('emulator', '')))} at " f"{launcher_ref_markdown(source, str(record['source_ref']))}") for name in launcher_names(record): names = found.setdefault(name.rsplit("/", 1)[-1].casefold(), []) if text not in names: names.append(text) return found def _render_platform_systems( by_system: dict, config_files: dict, emulator_files: dict, platform: tuple[str, str] | None = None, moved: dict[str, str] | None = None, config: dict | None = None, ) -> list[str]: """One collapsible block per system, listing every file it needs. Split out of generate_platform_page: the block is most of that function's branching, since each file renders a different set of hashes, sizes and provenance depending on what the platform declares. With ``platform`` (its key and name) and ``moved``, a list longer than PLATFORM_SECTION_BUDGET goes to a page of its own, written into ``moved`` by stem, and the system's heading stays with a link to it. """ lines: list[str] = [] # Per-system detail sections (collapsible for large platforms) use_collapsible = len(by_system) > 10 by_system_declared = _declared_by_system(config_files) launched = _launcher_refs(platform[0] if platform else "", config or {}) sections: list[dict] = [] for sys_id, files in sorted(by_system.items()): ok_count = sum(1 for f in files if f["status"] == "ok") total = len(files) sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id)) anchor = sys_id.replace(" ", "-") body = [f"{ok_count}/{total} files verified"] if sys_emus: emu_links = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus)) body[-1] += "\\" body.append(f"Emulators: {emu_links}") body.append("") # File listing. The sort is stable, so same-named entries keep the # order verify reported them in, which is the declaration order. seen_names: dict[str, int] = {} for f in sorted(files, key=lambda x: x["name"]): status = f["status"] fname = f["name"] declared = config_files.get((sys_id, fname), []) nth = seen_names.get(fname, 0) seen_names[fname] = nth + 1 cfg_entry = declared[nth] if nth < len(declared) else {} sha1 = cfg_entry.get("sha1", f.get("sha1", "")) md5 = cfg_entry.get("md5", f.get("expected_md5", "")) size = cfg_entry.get("size", f.get("size", 0)) if status == "ok": status_display = "OK" elif status == "untested": reason = f.get("reason", "") status_display = f"untested: {reason}" if reason else "untested" elif status == "missing": status_display = "**missing**" else: status_display = status # Not every entry is firmware: the platform may list a game's data # or a file no core it runs opens. kind = {"game_data": "game data", "unread": "no core the platform runs opens it"}.get(f.get("category", "")) if kind: status_display += f", {kind}" size_str = _fmt_size(size) if size else "" details = [status_display] if size_str: details.append(size_str) # Where the file goes, when its name alone does not say: three # GameCube IPL.bin differ only by GC/USA, GC/EUR and GC/JAP. destination = str(cfg_entry.get("destination") or "") where = f" at `{destination}`" if destination and destination != fname else "" body.append(f"- `{fname}`{where} - {', '.join(details)}") item_line = len(body) - 1 facts: list[str] = [] # Every hash the platform declares, whole: the crc32 is what # System.dat and clrmamepro print, and pasting it found nothing # for a pinned file the collection holds under other bytes. crc32 = str(cfg_entry.get("crc32") or "") # RetroDECK declares sha256 too: dsifirmware.bin's was in the CSV # exports only, and pasting it into the search found nothing. sha256 = str(cfg_entry.get("sha256") or "") if sha1 or md5 or crc32 or sha256: hash_parts = [] # One code span per value: Recalbox lists accepted md5s joined # by commas, and the search indexed the joined run as a single # word, so the second and later values were never found. if sha1: hash_parts.append(f"SHA1: {_hash_spans(sha1)}") if md5: hash_parts.append(f"MD5: {_hash_spans(md5)}") if crc32: hash_parts.append(f"CRC32: `{crc32}`") if sha256: hash_parts.append(f"SHA256: `{sha256}`") facts.append(" | ".join(hash_parts)) served = _served_line(f) if served: facts.append(served) if f.get("category_evidence"): facts.append(_category_evidence_md( f["category_evidence"], launched.get(fname.rsplit("/", 1)[-1].casefold()))) # One line each, hashes, the copy that serves it, its readers: as # continuation lines they ran into one paragraph. if facts: body[item_line] += "\\" body.extend(f" {fact}" + ("\\" if n < len(facts) - 1 else "") for n, fact in enumerate(facts)) # Measured as the Markdown the split was tuned on, a collapsible # block indented by four, so both engines split the same systems. size = sum(len(line) + 1 for line in body) if use_collapsible: status_tag = "OK" if ok_count == total else _count(total - ok_count, "issue") title = f"{_count(total, 'file')} ({status_tag})" size += len(f'??? note "{title}"') + 2 + 4 * sum(1 for line in body if line) block = markup.details("note", title, body) while block and not block[-1]: block.pop() else: block = body sections.append({"sys_id": sys_id, "anchor": anchor, "body": body, "block": block, "size": size, "total": total, "ok": ok_count}) # A list longer than its budget goes to a page of its own, then the # largest lists until the page's reference fits PLATFORM_REFERENCE_BUDGET: # with each file's served copy and pinned readers, Batocera's 97 systems # rendered 785 KB of HTML on one page. own_page: set[str] = set() if moved is not None and platform: own_page = {sec["sys_id"] for sec in sections if sec["size"] > PLATFORM_SECTION_BUDGET} kept = sum(sec["size"] for sec in sections if sec["sys_id"] not in own_page) for sec in sorted(sections, key=lambda sec: -sec["size"]): if kept <= PLATFORM_REFERENCE_BUDGET: break if sec["sys_id"] not in own_page: own_page.add(sec["sys_id"]) kept -= sec["size"] for sec in sections: sys_id, anchor, body = sec["sys_id"], sec["anchor"], sec["body"] total, ok_count = sec["total"], sec["ok"] # Every system is a heading with the same anchor in both layouts: # emulator cards link here, and the site search splits a page at its # headings. Folded into blocks under one heading, every hash pasted # in the search led to the systems overview of a 100-block page. lines.append(f"### {_system_heading_link(sys_id, '../')} {{ #{anchor} }}") lines.append("") if moved is not None and platform and sys_id in own_page: key, display = platform runs = _file_runs(body, PLATFORM_PART_BUDGET) stems = [f"{key}-{anchor}" + (f"-{n}" if n > 1 else "") for n in range(1, len(runs) + 1)] for n, (stem, run) in enumerate(zip(stems, runs), 1): part = f" (part {n} of {len(runs)})" if len(runs) > 1 else "" # A console query landed here with no way to the console's # own page: the list says whose it is first, linked. moved[stem] = "\n".join([ "---", *_parent_front_matter((display, f"/platforms/{key}/")), "---", "", f"# {display}: {system_name(sys_id)}{part}", "", f"{display}'s list for {_system_named_link(sys_id, '../')}. " f"Part of the [{display} page]({key}.md#{anchor}), whose other systems " f"are listed there. Each file below is as {display} declares it, with " "its status in the pack.", *([""] + [", ".join( f"[part {i}]({other}.md)" if other != stem else f"part {i}" for i, other in enumerate(stems, 1))] if len(runs) > 1 else []), "", "## Files", "", *search_excluded(run), "", _date_line(), ]) + "\n" where = (f"[a page of their own]({stems[0]}.md)" if len(stems) == 1 else "pages of their own: " + ", ".join( f"[part {i}]({stem}.md)" for i, stem in enumerate(stems, 1))) lines.extend([ f"{_count(total, 'file')}, {ok_count} verified: listed on {where}.", "", ]) continue # The section's heading stays searchable, its file list with every # hash is the lookup's: retrodeck's arcade list was 495 KB of index. lines.extend(search_excluded(sec["block"])) return lines # Platforms with an updated file published by export_native.py. _EXPORTED: set[str] = set() def _platform_data_links(name: str) -> str: """The platform's own data, machine-readable: the YAML transcribed from its upstream list, this page's JSON and the manifest the installer reads. An emulator page linked its YAML and JSON, a platform page nothing: a tool had to download all twelve platforms in platforms.json and filter. """ links = [ f"[YAML]({REPO_URL}/blob/main/platforms/{name}.yml)", f"[JSON](../api/v1/platforms/{name}.json)", ] if (_REPO_ROOT / "install" / f"{name}.json").is_file(): links.append(f"[install manifest]({REPO_URL}/blob/main/install/{name}.json)") return ", ".join(links) @functools.lru_cache(maxsize=1) def _site_profiles() -> dict[str, dict]: """Every emulator profile, read once for the citations of a page.""" return load_emulator_profiles(str(_REPO_ROOT / "emulators")) def _category_evidence_md(evidence: dict, launched: list[str] | None = None) -> str: """What establishes an entry's category, as the page shows it. ``launched`` holds the platform's launcher records that open the entry, already rendered: "(by launcher)" alone named no record and no line. """ if evidence.get("read_by"): groups: dict[bool, list[str]] = {True: [], False: []} bare: list[str] = [] profiles = _site_profiles() for reader in evidence["read_by"]: # Pinned links, the repository named in words: the refs were # plain code with the role word ("source") left in front. profile = profiles.get(reader["emulator"], {}) refs = [ (f"`{e['name']}` at {_source_ref_markdown(profile, e['source_ref'])}" if e.get("source_ref") else f"`{e['name']}`") + (" (by name)" if e.get("by") == "name" else "") for e in reader.get("entries", []) ] if reader["emulator"] == "launcher": name = "the platform's launcher" refs = list(launched or []) else: name = _emulator_link(reader["emulator"], "../") groups[bool(reader.get("runs_system"))].append( f"{name} ({', '.join(refs)})" if refs else f"{name} (by {reader['proof']})") if not reader.get("runs_system"): # A launcher record is this platform's own code: kept whole. bare.append(f"{name} ({', '.join(refs)})" if reader["emulator"] == "launcher" and refs else name) if not groups[True]: return "Read by " + ", ".join(groups[False]) text = "Read by " + ", ".join(groups[True]) # Cores of other systems that share the name: named, not cited, as # their lines describe another machine's file. if bare: text += ", also named by " + ", ".join(bare) return text examined = evidence.get("cores_examined") or [] cores = ("cores examined: " + ", ".join(_emulator_link(c, "../") for c in examined) if examined else "no core examined") if evidence.get("unsettled"): return f"No reader found, not settled: {evidence['unsettled']} ({cores})" return f"No reader found ({cores})" @functools.lru_cache(maxsize=1) def _profiles_by_core_name() -> dict[str, list[str]]: """Core name as a platform writes it -> the profiles with a page for it. MinUI writes `mednafen_vb`, the profile and its page are `beetle_vb`. """ profiles = load_emulator_profiles(str(_REPO_ROOT / "emulators")) with_page = {k for k, v in profiles.items() if v.get("type") not in ("alias", "test")} return { core: sorted(k for k in keys if k in with_page) for core, keys in upstream_profile_index(profiles).items() } def _core_name_links(cores: list[str]) -> list[str]: """A link per core to its profile page, the bare name where none exists.""" index = _profiles_by_core_name() links = [] for core in cores: keys = [core] if core in index.get(core, []) else index.get(core, []) if keys: links.extend(_emulator_link(k, "../") for k in keys) else: links.append(f"`{core}`") return links def _core_folders_section(name: str, config: dict) -> list[str]: """Each folder the frontend hands a core, and the files the pack puts in it. For a platform that names no file of its own (MinUI and its forks): the folders come from its launch scripts, the files from its install manifest. """ folders = config.get("core_system_dirs") or {} manifest = _REPO_ROOT / "install" / f"{name}.json" files = json.loads(manifest.read_text())["files"] if manifest.is_file() else [] lines = [ "### Core folders", "", "Each folder below is the one the frontend gives that core. The pack " "puts the files the core reads there.", "", "| Folder | Cores | Files |", "|--------|-------|-------|", ] for folder, info in sorted(folders.items()): held = sorted( f["dest"][len(folder) + 1:] for f in files if f.get("dest", "").startswith(folder + "/") ) cores = ", ".join(_core_name_links(info.get("cores", []))) shown = ", ".join(f"`{n}`" for n in held) if held else "none needed" lines.append(f"| `{folder}/` | {cores} | {shown} |") lines.append("") return lines def _check_sentence(display: str, mode: str, config: dict, no_list: bool) -> str: """What the frontend itself checks, from the mode its code uses.""" if no_list: return (f"{display} has no BIOS list, so it checks nothing itself. Each " "emulator looks for its own files when a game starts.") if config.get("hash_rule") == "size_and_any_hash": return f"{display} also checks each file's size and contents." if digest_algorithm(mode): return f"{display} also checks each file's contents." return f"{display} checks names only. `--check` also checks contents." def _folder_rows(rows: tuple[tuple[str, str], ...], indent: str = "") -> list[str]: return [ f"{indent}| Where | BIOS folder |", f"{indent}|-------|-------------|", *(f"{indent}| {label} | {path} |" for label, path in rows), ] # The second most asked question about BIOS files (14 % of the community # sample): the file is there and the setup still reports it missing. # `--check` lists every file whose hash is wrong by name (install.py, the # "wrong hash" lines of check_lines). MISSING_IN_PLACE = "Reported missing anyway? `--check` names a wrong file." # Under every command of a setup page: the safety list is the one the home # page shows in full, so the setup page links it. SAFE_LINK = "[Is it safe?](../index.md#is-it-safe)" # The pass line of "How to check". `--check` prints "All files up to date." # only when no file is missing or wrong (install.py, the args.check branch # after check_local). "0 wrong hash" alone is also what an empty folder # prints: 0/14508 present (0 verified, 0 wrong hash). CHECK_PASS_LINE = "All good when it says `All files up to date.`" # Under "From your computer": a device with no network installs from the ZIP # (`--from-pack`). The id is the one wiki/installer.md gives its section. OFFLINE_LINE = ("No internet on the device? " "[Install from the ZIP](../wiki/installer.md#without-a-network-offline-install).") def _network_share(name: str, guide: Guide) -> str: """The network folder a computer fills, for a setup that runs on its own device and has no ZIP: the command runs on the computer and writes there. Empty when the page cites no such folder.""" if guide.run != ("device",) or _computer_first(name, guide): return "" for label, path in guide.folders: if label.startswith("From Windows") and path.startswith("`\\\\"): return path.strip("`").rstrip("\\") return "" def _offline_line(name: str) -> list[str]: """The offline line, on a setup whose ZIP a release publishes: the installer reads that ZIP with no network (`--from-pack`).""" pack = release_record.pack_for(name, release_record.load_advertised()) return [OFFLINE_LINE, ""] if pack and pack.get("assets") else [] # The route of a setup with no ZIP when the device has no network: the # command writes to any folder named after --dest (wiki/installer.md), so a # computer fills the device's card. The FAQ answers it with an example. CARD_ROUTE_FAQ = "wiki/faq.md#fill-an-sd-card-from-a-computer" def _card_route(name: str, display: str, guide: Guide) -> list[str]: """The route with no network on the device, for a setup whose ZIP no release publishes: the command run on a computer, into the card. A setup with a ZIP has the offline line, a card setup's command already runs on the computer. The folder named after --dest is the one the command fills on the device (the registry's extract_to): AmberELEC's is `/storage/roms/`, which holds `bios/` and `neogeo/`. How a computer sees it is the reader's to name: no guide cites it.""" if guide.on_card or _offline_line(name): return [] extract_to = (load_platform_registry("platforms").get(name) or {}).get("extract_to") leaf = (extract_to or "bios").rstrip("/").rsplit("/", 1)[-1].lstrip("~") or "bios" tabs = guide.card_run or ("windows", "macos", "linux") args = {key: f"--platform {name} --dest " + (f"E:\\path\\to\\{leaf}" if key == "windows" else f"/path/to/{leaf}") for key in tabs} if guide.run != ("device",): lead = ("No network here? Run the command on a computer that has one, with a USB " f"drive's folder after `--dest`. Then copy it into {_folder_words(display, extract_to)}.") return [lead, "", *terminal_tabs(tabs, args, steps=False)] lead = ("No network? Put the device's card in your computer. After `--dest`, " f"name the folder the device calls `{extract_to}`.") return [lead, "", *terminal_tabs(tabs, args, steps=False), f"[More on SD cards](../{CARD_ROUTE_FAQ})", ""] def _get_page_line(name: str, display: str) -> list[str]: """The setup's get/ page, under the command of a setup with no ZIP: the address a guide writer keeps. A setup with a ZIP links it by its button.""" if has_release_pack(name): return [] return [f"Link to share: [{get_page_title(name, display)}](../{get_page_link(name)})", ""] def _setup_routes(name: str, display: str, page_link: bool = False) -> list[str]: """The ways to run the command for a setup no ZIP serves first, written once for its setup page and its get/ page: the network folder from a computer, the card from a computer when the device has no network, and the device's own terminal. *page_link* adds the get/ page, on the setup page only.""" guide = GUIDES[name] share = _network_share(name, guide) card = _card_route(name, display, guide) shared = _get_page_line(name, display) if page_link else [] if share or (card and guide.run == ("device",)): computer = ["## From your computer { #from-your-computer }", ""] if share: computer += [ "Needs the device on your network. Run the command on the computer.", "", *terminal_tabs(("windows",), {"windows": f"--platform {name} --dest {share}"}), SAFE_LINK, "", ] return [*computer, *card, *_guide_install(name, guide, "## On the device { #install }"), *shared] return [*_guide_install(name, guide, "## Install"), *shared, *card] # Shown in the "On the device" tab of a guide that has no `terminal:` of its own. DEVICE_TERMINAL_FALLBACK = "Or fill the folder from your computer, with the ZIP." def _computer_first(name: str, guide: Guide) -> bool: """A system that runs only on its own device, with a ZIP to download: a beginner fills its folder from a computer more easily than from a terminal on the TV. Without a ZIP the command is the only route.""" pack = release_record.pack_for(name, release_record.load_advertised()) return guide.run == ("device",) and bool(pack and pack.get("assets")) def _ships_neogeo(name: str) -> bool: """Whether the setup's files hold `neogeo.zip`, the archive a reader most often unzips by mistake: arcade BIOS sets are read zipped.""" manifest = _REPO_ROOT / "install" / f"{name}.json" if not manifest.is_file(): return False files = json.loads(manifest.read_text(encoding="utf-8")).get("files", []) return any(f.get("dest", "").rsplit("/", 1)[-1] == "neogeo.zip" for f in files) def _guide_download(name: str, display: str, guide: Guide, extract_to: str | None, heading: str) -> list[str]: """The pack: its buttons, its size, and where it is unzipped.""" lines = [heading, ""] record = release_record.load_advertised() pack = release_record.pack_for(name, record) network = any(label.startswith("From ") for label, _path in guide.folders) if pack and pack.get("assets"): parts = len(pack["assets"]) bullets = [ f"{_fmt_size(pack['download_size'])} to download, " f"{_fmt_size(pack['extracted_size'])} once unzipped." ] if parts > 1: bullets.append("You need both parts." if parts == 2 else f"You need all {parts} parts.") if _computer_first(name, guide) and network: target = "the network folder above" else: target = f"`{extract_to}`" if extract_to else "your BIOS folder" bullets.append(f"Unzip {'them all' if parts > 1 else 'it'} into {target}." + (" Files like `neogeo.zip` stay zipped." if _ships_neogeo(name) else "")) if guide.download_hint and not _computer_first(name, guide): bullets.append(guide.download_hint) lines += [ _pack_buttons(name, f"Download {display} Pack"), "", *(f"- {b}" for b in bullets), "", ] elif not has_release_pack(name): lines += [_pack_buttons(name, display=display), ""] elif record.get("tag"): made_from = setup_version(name) lines += [ f"No {display} ZIP yet. The command installs the same files" + (f", made for {made_from}." if made_from else "."), "", ] else: lines += [_pack_buttons(name, f"Download {display} Pack"), ""] return [*lines, *only_some_consoles(name, "../")] def only_some_consoles(name: str, prefix: str = "") -> list[str]: """The way to a smaller download, in the words Get your files uses.""" return [ f"Only some consoles? [Pick them]({prefix}which-pack.md?platform={name}" "#only-some-consoles), or press **c** in the command.", "", ] def _guide_install(name: str, guide: Guide, heading: str) -> list[str]: """The command, in a tab per place it runs, and whether it is safe.""" lines = [ heading, "", "It finds the folder and checks each file.", "", ] if guide.on_card: lines += [ "Put the SD card in a computer. On Windows, note its letter, like " "**E:**. Put the card's path after `--dest`.", "", ] device = guide.terminal or f"{TERMINALS['device'][1]} {DEVICE_TERMINAL_FALLBACK}" return [ *lines, *terminal_tabs(guide.run, platform_args(name, guide), how={"device": device}), SAFE_LINK, "", ] def devices_meta(devices: tuple[str, ...]) -> str: """The device names a reader types, as the `devices` search field of the page: an attribute on an empty element, commas out (the search engine splits its keys on commas), so no excerpt shows them.""" names = " ".join(name.replace(",", " ") for name in devices) return f'' # Devices named in a row of the handheld guide. The rest stay in the # page's search field and on the list the row links. HANDHELD_ROW_DEVICES = 6 # Where a handheld row with no cited folder sends the reader: the setting # that shows RetroArch's BIOS folder, the words of the RetroArch page. RETROARCH_FOLDER_SETTING = ("In RetroArch, open **Settings > Directory > System/BIOS**. " "It shows the folder.") # After a folder on a part of the card only Linux reads. LINUX_ONLY_HINT = ("Copy from a Linux computer, or check the folder in RetroArch's " "**Settings > Directory > System/BIOS**.") def _handheld_row(system: HandheldSystem) -> str: shown = ", ".join(system.devices[:HANDHELD_ROW_DEVICES]) more = len(system.devices) - HANDHELD_ROW_DEVICES devices = (f"{shown}{f' and {more} more' if more > 0 else ''} " f"([list]({system.devices_source}))" if system.devices else "") if not system.folder: folder = RETROARCH_FOLDER_SETTING else: folder = f"{system.folder} ([source]({system.folder_source}))" if system.linux_only: folder += f". {LINUX_ONLY_HINT}" return f"| {system.name} | {devices} | {folder} |" def generate_handheld_guide(guide: HandheldGuide) -> str: """The guide for the handheld systems that run a setup (ArkOS, muOS, Onion... run RetroArch): one row per system the installer detects, the folder seen on the card from a computer and the devices, each cited from the system's own pages, then that setup's ZIP and the command for a card. A handheld owner starts from the device name. The systems were named only in a closed note, with a path seen from inside the device.""" uses = GUIDES[guide.uses] display = load_platform_config(guide.uses, "platforms").get("platform", guide.uses) args = {"windows": f"--platform {guide.uses} --dest E:\\BIOS", "macos": f"--platform {guide.uses} --dest /path/to/card/BIOS", "linux": f"--platform {guide.uses} --dest /path/to/card/BIOS"} return "\n".join([ f"# {guide.title}", "", guide.summary, "", devices_meta(guide.devices), "", "[Download](#download)", "", "## Where the files go { #where-the-files-go }", "", "Put the SD card in a computer. Find your system below.", "", "| System | Devices | BIOS folder on the card |", "|--------|---------|-------------------------|", *(_handheld_row(system) for system in guide.systems), "", *_guide_download(guide.uses, display, uses, None, "## Download { #download }"), *_offline_line(guide.uses), "## Fill the card from a computer { #install }", "", "On Windows, note the card's letter, like **E:**. Put its BIOS folder after " "`--dest`, written as in the table.", "", *terminal_tabs(("windows", "macos", "linux"), args), SAFE_LINK, "", *_next_block([ f"[{display}]({guide.uses}.md): every folder and every file", "[When a game does not start](../wiki/troubleshooting.md)", ]), ]) + "\n" def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list: bool) -> list[str]: """The guide part of a platform page: folder, command, pack, check, next. A system that runs only on its own device puts the computer route (download, unzip into the network folder) before the command.""" guide = GUIDES[name] entry = registry.get(name, {}) extract_to = entry.get("extract_to") lines = ["## Where the files go", ""] if guide.lead: lines += [guide.lead, ""] lines += _folder_rows(guide.folders) if extract_to and has_release_pack(name): lines.append(f"| Unzip the pack into | `{extract_to}` |") lines.append("") for note in guide.notes: lines += [note, ""] lines += [line for note in extract_notes("platforms", released_only=True) if f"The {display} pack " in note for line in (note, "")] if guide.more_folders: detected = _detected_systems(registry, name) if guide.more_folders_detected else [] lines += markup.details("note", guide.more_folders_title, [ *_folder_rows(guide.more_folders), *(f"| {system} | `{folder}` |" for system, folder in detected), "", *([guide.more_folders_note, ""] if guide.more_folders_note else []), ]) if guide.on_card: lines += ["", ""] if _computer_first(name, guide): lines += [ *_guide_download(name, display, guide, extract_to, "## From your computer { #download }"), *_offline_line(name), *_guide_install(name, guide, "## On the device { #install }"), ] else: lines += [ *_setup_routes(name, display, page_link=True), *_guide_download(name, display, guide, extract_to, "## Download"), *_offline_line(name), ] check_args = {key: f"{args} --check" for key, args in platform_args(name, guide).items()} lines += [ "## How to check", "", "Add `--check` to the command. It changes nothing.", "", *terminal_tabs(guide.run, check_args, steps=False), CHECK_PASS_LINE, "", _check_sentence(display, cov["mode"], cov["config"], no_list), "", MISSING_IN_PLACE, "", *([guide.finish, ""] if guide.finish else []), *_next_block([ "[First install, step by step](../wiki/first-install.md)", "[When a game does not start](../wiki/troubleshooting.md)", ]), ] return lines # How a platform checks a file, in words. _CHECK_WORDS = {"existence": "name only", "md5": "MD5", "sha1": "SHA-1", "sha256": "SHA-256", "crc32": "CRC32"} def generate_platform_page( name: str, cov: dict, registry: dict | None = None, emulator_files: dict | None = None, moved: dict[str, str] | None = None, ) -> str: """A guide first (folder, command, pack, check), then every file the platform's list and its emulators need, under a reference heading. ``moved`` collects the pages of systems whose list is too long for it. """ from sitesources import list_source registry = registry or {} config = cov["config"] display = cov["platform"] mode = cov["mode"] pct = _pct(cov["present"], cov["total"]) # The mark is drawn beside the title from the front matter: floated # beside the first line at 48 px it read as a broken image. logo_url = registry.get(name, {}).get("logo", "") logo_url = local_images().get(logo_url, logo_url) homepage = config.get("homepage", "") version = config.get("version", "") hash_type = config.get("hash_type", "") base_dest = config.get("base_destination", "") mode_badge = ( "rb-badge-success" if reads_file_contents(mode) else "rb-badge-info" ) lines = [ *(["---", f"logo: {json.dumps(logo_url)}", "---", ""] if logo_url else []), f"# {display}", "", ] if registry.get(name, {}).get("status") == "archived": lines.extend(markup.aside("warning", "Archived platform", [ f"{display}'s list is kept as last read. Its pack is still built.", ])) summary = GUIDES[name].summary if name in GUIDES else "" if summary: lines += [summary + (f" [{display} website]({homepage})." if homepage else ""), ""] no_list = not cov["total"] and registry.get(name, {}).get("native_list") == "none" if name in GUIDES: lines += _platform_guide(name, display, cov, registry, no_list) lines += [ "## Reference: every file by system { #reference-every-file-by-system }", "", level_label("reference"), "", "For checking the work: the counts, where the list comes from, and " "every file with its hashes.", "", *(line for note in (GUIDES[name].reference_notes if name in GUIDES else ()) for line in (note, "")), ] if no_list: # MinUI and its forks hand each core a folder and name no file: a # 0/0 coverage band read as "this device is not supported". lines.extend(markup.aside("info", "No BIOS list of its own", [ f"{display} names no BIOS file itself: it gives each emulator core " "its own folder and the core looks for its files there. The pack " "places every file the cores it runs need into those folders. The " "core files are listed further down this page.", ])) band = [] if no_list else [ markup.block_open("div", 'class="rb-stats"'), "", *_stat(f"{cov['present']}/{cov['total']}", f"Coverage ({pct})"), *_stat(str(cov["verified"]), "Verified"), *_stat(str(cov["missing"]), "Missing"), *_stat(f'{_CHECK_WORDS.get(mode, mode)}', "Checked by"), "", "", ] lines.extend([*band, markup.block_open("div", 'class="rb-meta-card"'), "", "| Detail | Value |", "|---|---|"]) # The hash the list writes, when it is not the one the platform checks: # RetroArch checks presence and its System.dat writes sha1. if hash_type and hash_type != mode: lines.append(f"| Hashes in the list | {_CHECK_WORDS.get(hash_type, hash_type)} |") if version: lines.append(f"| Version | {version} |") # base_destination is only the root the platform's own list writes its # paths from: the pack strips it, so it is no folder to extract into. if base_dest: lines.append(f"| Paths in the platform's list start at | `{base_dest}/` |") if homepage: lines.append(f"| Homepage | [{homepage}]({homepage}) |") transcribed = str(config.get("source") or "") if transcribed: # The file every declaration on this page was read from: without it # a hash could not be traced to the revision that states it. listed = list_source(transcribed) moving = (listed.moving if listed else bool( re.search(r"/(?:master|main|db)(?:/|$)|refs/heads/", transcribed))) note = " (a branch: the revision read is not recorded)" if moving else "" # Named in words: the raw URL broke into 25 lines of 8 characters # on a phone. link = (f"[{_escape_tags(listed.label)}]({listed.browse()})" if listed else f"<{transcribed}>") lines.append(f"| Transcribed from | {link}{note} |") if name in _EXPORTED: lines.append(f"| Updated file | [Download it and see the changes](../exports.md#{name}) |") lines.append(f"| Data | {_platform_data_links(name)} |") contrib_list = registry.get(name, {}).get("contributed_by", []) if contrib_list: for cb in contrib_list: username = cb.get("username", "") contribution = cb.get("contribution", "") pr = cb.get("pr") pr_link = f" ([#{pr}]({REPO_URL}/pull/{pr}))" if pr else "" lines.append( f"| Contributed by | [@{username}](https://github.com/{username}):" f" {contribution}{pr_link} |" ) lines.extend(["", ""]) if name not in GUIDES: detect = (registry.get(name, {}).get("install") or {}).get("detect") or [] on_card = any(rule.get("method") == "card_marker" for rule in detect) lines.extend(["", *_platform_install_block(name, display, on_card)]) lines.extend(["", _pack_buttons(name, f"Download {display} Pack", display=display)]) lines.append("") # The declaration behind each detail, by system and name in declaration # order: keyed by name alone, the last of MiSTer's 23 boot.rom gave its # MD5 to every system's row. config_files: dict[tuple[str, str], list[dict]] = {} for sys_id, system in config.get("systems", {}).items(): for fe in system.get("files", []): fname = fe.get("name", "") if fname: config_files.setdefault((sys_id, fname), []).append(fe) by_system: dict[str, list] = {} for d in cov["details"]: sys_id = d.get("system", "unknown") by_system.setdefault(sys_id, []).append(d) if no_list: lines.extend(_core_folders_section(name, config)) if _own_list_only(config): lines.extend([f"**Cores:** {OWN_LIST_ONLY}", ""]) # System summary table (quick navigation) lines.extend( [] if no_list and not by_system else [ "### Systems overview", "", "| System | Files | Status | Emulators |", "|--------|-------|--------|-----------|", ] ) by_system_declared = _declared_by_system(config_files) for sys_id, files in sorted(by_system.items()): ok_count = sum(1 for f in files if f["status"] == "ok") total = len(files) non_ok = total - ok_count if non_ok == 0: status = 'OK' else: status = ( f'' f'{non_ok} issue{"s" if non_ok > 1 else ""}' ) sys_emus = _system_emulators(sys_id, emulator_files, by_system_declared.get(sys_id)) anchor = sys_id.replace(" ", "-") # Named and linked as the detail blocks below do: the column held # profile ids, none of 1,463 linked. emu_str = ", ".join(_emulator_named_link(e, "../") for e in sorted(sys_emus)[:3]) if len(sys_emus) > 3: emu_str += f" [+{len(sys_emus) - 3}](#{anchor})" # The name a reader filters on, the id the list writes beside it. # The id and the count are no words a reader searches for: indexed, # every platform page answered a system id before its system's page. lines.append( f"| [{_escape_tags(system_name(sys_id))}](#{anchor}) " f"{sys_id} " f"| {ok_count}/{total} | {status} | {emu_str} |" ) lines.append("") lines.extend( _render_platform_systems(by_system, config_files, emulator_files, (name, display), moved, config) ) lines.extend(["", _date_line()]) return "\n".join(lines) + "\n" # System pages def generate_files_page() -> str: """Find a file by any of its hashes: files/?hash=. site.js reads the one shard of api/v1/lookup/ the value names and lists every page that cites it. files/?sha1= is the card's permanent address and opens the card directly. """ return "\n".join([ "---", "terms: check my dump, md5 check, verify bios file, check hash, " "is my bios correct, is my bios right", "---", "", "# Find or check a file", "", "Check a BIOS dump: paste its hash or choose the file.", "", "A hash is a fingerprint of a file's contents ([Glossary](../wiki/glossary.md)). " "Tools such as clrmamepro and RomVault print one, and so do emulator logs " "and the BIOS lists of setups.", "", "Paste it below. The page shows the file in the collection that has it, the " "setups that list it and the emulators that check for it.", "", '
', ' ', ' ', '', "
", "", '

', "", "To try it, paste `0555c6fae8906f3f09baf5988f00e55f88e9f30b`, the SHA1 of " "`scph5501.bin`, the PlayStation BIOS v3.0. " "To look a file up by its name, use the search at the top of the page.", "", "## Permanent links", "", "A file's card lives on its manufacturer's page, and a file refiled " "between manufacturers moves with it. `files/?sha1=` is the address " "that does not move: it opens the card wherever it is now. The link icon " "beside each card's name is that address.", "", "## The index behind it", "", "The lookup reads one file of [`api/v1/lookup/`](../data.md), named by the " "first two characters of the hash: one of 256, never the whole index. " "[`api/v1/file-locations.json`](../api/v1/file-locations.json) lists every " "card by SHA1 in one file.", "", ]) _REPO_ROOT = Path(__file__).resolve().parent.parent def data_directory_members(install_dir: Path = _REPO_ROOT / "install") -> dict[str, dict[str, dict]]: """Each data directory's files with their sha1 and size, as the install manifests record them for the installer to check.""" members: dict[str, dict[str, dict]] = {} for path in sorted(install_dir.glob("*.json")): manifest = json.loads(path.read_text(encoding="utf-8")) for tree in manifest.get("data_directories") or []: members.setdefault(str(tree.get("ref", "")), {}).update(tree.get("files") or {}) return members def data_directory_revisions(data_dir: Path = _REPO_ROOT / "data") -> dict[str, dict]: """The revision each data directory was fetched at, as refresh_data_dirs records it.""" path = data_dir / ".versions.json" if not path.is_file(): return {} return json.loads(path.read_text(encoding="utf-8")) def page_data_directories(name: str, profiles: dict) -> list[dict]: """The data directories an emulator page lists. Its own, then those of the aliases that point at it: an alias has no page, and the 90 FreeIntv overlays its profile declares were linked to emulators/freeintv_ts_overlay/, a page that does not exist. """ trees = [t for t in profiles[name].get("data_directories") or [] if isinstance(t, dict)] refs = {str(t.get("ref", "")) for t in trees} for alias in sorted(profiles): profile = profiles[alias] if profile.get("type") != "alias" or profile.get("alias_of") != name: continue for tree in profile.get("data_directories") or []: ref = str(tree.get("ref", "")) if isinstance(tree, dict) and ref not in refs: refs.add(ref) trees.append(tree) return trees def _data_member_owners(profiles: dict) -> dict[str, str]: """The emulator page that lists each data directory, the first by name. Only a profile that has a page can own one. """ owners: dict[str, str] = {} for name in sorted(profiles): if profiles[name].get("type") in ("alias", "test"): continue for tree in page_data_directories(name, profiles): owners.setdefault(str(tree.get("ref", "")), name) return owners def file_location_items( db: dict, profiles: dict | None = None, members: dict | None = None ) -> list[dict]: """Each held file's card, keyed by its sha1. A card lives on its manufacturer's page, and a file refiled between manufacturers (mt32_control.rom, Commodore to Roland) moved its link with it. files/?sha1= reads this list and goes to the card wherever it is now: the address cited depends on the content only. The url is site-absolute (/retrobios/systems/...): written from the site root, it resolved against the document holding it to a page that does not exist. """ items = [] for sha1, entry in sorted(db.get("files", {}).items()): parts = entry.get("path", "").split("/") if len(parts) < 3 or parts[0] != "bios": continue page = card_page(entry.get("path", ""), sha1) items.append({"sha1": sha1, "url": _site_path(f"systems/{page}.md#sha1-{sha1[:12]}")}) # A data directory's files are listed on the page of an emulator reading # the tree: 2,534 hashes the packs ship were found nowhere on the site. owners = _data_member_owners(profiles or {}) known = {item["sha1"] for item in items} for ref, files in sorted((members or {}).items()): if ref not in owners: continue # A large tree is listed on pages of its own (data_tree_pages). page_of = {member: stem for stem, names in data_tree_pages(ref, files) for member in names} for member, info in sorted(files.items()): sha1 = str(info.get("sha1", "")) if sha1 and sha1 not in known: known.add(sha1) page = page_of.get(member, owners[ref]) items.append({"sha1": sha1, "url": _site_path(f"emulators/{page}.md#sha1-{sha1[:12]}")}) return sorted(items, key=lambda item: item["sha1"]) LOOKUP_PREFIX = 2 # hex characters naming a lookup shard: 256 files def _lookup_system_name(sys_id: str) -> str | None: """The system's name as every page prints it, None for an id the site cannot name. The Find a file page reads it, and never builds one.""" try: return system_name(sys_id) except KeyError: return None def _console_system_names(profiles: dict, coverages: dict) -> dict[tuple[str, str], str]: """(manufacturer slug, console folder) -> the name of the system whose own section that folder is: a collection file is named after the console it is filed under. A folder no id owns, or that two names own, is left out. """ from siterender import _own_section, system_ids found: dict[tuple[str, str], set[str]] = {} for sys_id in system_ids(profiles, [c["config"] for c in coverages.values()]): own = _own_section(sys_id) if own is not None: found.setdefault((_slug(own[0]), own[1]), set()).add(system_name(sys_id)) return {key: next(iter(names)) for key, names in found.items() if len(names) == 1} def file_lookup_records( db: dict, coverages: dict, profiles: dict, locations: list[dict], members: dict | None = None, consumers: FileConsumers | None = None, ) -> list[dict]: """Every hash the site knows, one record per value, for files/?hash=. The collection's own files lead to their card, a platform's declared value to the system section of its page, a profile's to the entry's card. The site search indexed these values as text, 4 MB of the 21 MB it made every visitor download: the lookup reads one shard instead. """ files = db.get("files", {}) named = _console_system_names(profiles, coverages) member_names = { str(info.get("sha1", "")): member.rsplit("/", 1)[-1] for tree in (members or {}).values() for member, info in sorted(tree.items()) } records: list[dict] = [] for item in locations: sha1 = item["sha1"] entry = files.get(sha1) if entry is None: # A data directory member: the manifests record its sha1 only. records.append({"value": sha1, "algorithm": "sha1", "layer": "data-directory", "name": member_names.get(sha1, ""), "sha1": sha1, "url": item["url"]}) continue parts = str(entry.get("path", "")).split("/") held_by = named.get((_slug(parts[1]), parts[2])) if len(parts) > 3 else None for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): records.append({ "value": value, "algorithm": algorithm, "layer": "collection", "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"], **({"system_name": held_by} if held_by else {}), }) for key, coverage in sorted(coverages.items()): for system_id, system in sorted((coverage["config"].get("systems") or {}).items()): url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}") system_label = _lookup_system_name(system_id) for entry in system.get("files") or []: for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): records.append({ "value": value, "algorithm": algorithm, "layer": "platform", "owner": key, "name": str(entry.get("name", "")), "url": url, **({"system_name": system_label} if system_label else {}), }) for key, profile in sorted(profiles.items()): if profile.get("type") in ("alias", "test"): continue entries = profile.get("files") or [] own_systems = [str(s) for s in profile.get("systems") or []] for entry, anchor in zip(entries, emulator_file_anchors(entries)): url = _site_path(f"emulators/{emulator_card_page(key, anchor)}.md#{anchor}") sys_id = str(entry.get("system") or "") or ( own_systems[0] if len(own_systems) == 1 else "") entry_label = _lookup_system_name(sys_id) if sys_id else None members = [(str(entry.get("name", "")), entry)] + [ (str(member["name"]), member) for member in entry.get("contents") or [] if isinstance(member, dict) and member.get("name") ] for name, holder in members: for algorithm in HASH_ALGORITHMS: for value in _hash_values(holder.get(algorithm)): records.append({ "value": value, "algorithm": algorithm, "layer": "emulator", "owner": key, "name": name, "url": url, **({"system_name": entry_label} if entry_label else {}), }) records += _held_under_every_hash(records, files, db.get("indexes") or {}) for record in records: if record["layer"] in ("platform", "emulator"): record.setdefault("declared", record["algorithm"]) if consumers is not None: records += _read_by_name(records, files, consumers) unique = {json.dumps(r, sort_keys=True): r for r in records} return sorted(unique.values(), key=lambda r: ( r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"])) def _read_by_name(records: list[dict], files: dict, consumers: FileConsumers) -> list[dict]: """The emulators the file's card names that no declared hash reached. The card counts every profile entry resolving to the file, by name included: scph5501.bin named 11 emulators on its card and 9 in the lookup, PCSX-ReARMed and PCSX1 reading it by name. Each is written under every hash of the file, `declared` saying no hash was. """ reached = {(r["sha1"], r["owner"]) for r in records if r["layer"] == "emulator" and r.get("sha1")} added: list[dict] = [] for sha1, emulators in sorted(consumers.emulators.items()): entry = files.get(sha1) if entry is None: continue for emulator in sorted(emulators): if (sha1, emulator) in reached: continue index = consumers.emulator_entries.get(sha1, {}).get(emulator) entries = consumers.emulator_files.get(emulator) or [] if index is None or index >= len(entries): continue anchor = _emulator_card_anchor(emulator, entries, index) url = _site_path(f"emulators/{emulator_card_page(emulator, anchor)}.md#{anchor}") name = str(entries[index].get("name", "")) for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): added.append({ "value": value, "algorithm": algorithm, "layer": "emulator", "owner": emulator, "name": name, "url": url, "sha1": sha1, "declared": "name", }) return added def _collection_sha1(files: dict, indexes: dict, algorithm: str, value: str) -> str | None: """The collection file a declared value designates, None when none does.""" if algorithm == "sha1": return value if value in files else None found = indexes.get(f"by_{algorithm}", {}).get(value) if isinstance(found, list): found = found[0] if len(found) == 1 else None return found if found in files else None def _held_under_every_hash(records: list[dict], files: dict, indexes: dict) -> list[dict]: """A platform or emulator record whose value designates a collection file, written under the file's other hashes too. Batocera lists scph5501.bin by md5: pasted by its sha1, the file showed no setup at all. Each record now carries the file's sha1, and ``declared`` names the hash its owner wrote when the record is a copy. """ copies: list[dict] = [] for record in records: if record["layer"] not in ("platform", "emulator"): continue sha1 = _collection_sha1(files, indexes, record["algorithm"], record["value"]) if sha1 is None: continue record["sha1"] = sha1 for algorithm in HASH_ALGORITHMS: for value in _hash_values(files[sha1].get(algorithm)): if value != record["value"]: copies.append({**record, "value": value, "algorithm": algorithm, "declared": record["algorithm"]}) return copies def write_lookup_shards( api: Path, records: list[dict], generated_at: str, commit: str = "", ) -> int: """One envelope per leading pair of hex characters, as the lookup asks. A value that is not hexadecimal (a typo in a profile) has no shard the lookup would read, and is left out. Returns the number of shards. """ shards: dict[str, list[dict]] = {} for record in records: if re.fullmatch(r"[0-9a-f]{8,}", record["value"]): shards.setdefault(record["value"][:LOOKUP_PREFIX], []).append(record) folder = api / "lookup" folder.mkdir(parents=True, exist_ok=True) for prefix, items in sorted(shards.items()): write_if_changed(str(folder / f"{prefix}.json"), json.dumps({ "schema_version": 1, "generated_at": generated_at, **({"commit": commit} if commit else {}), "kind": "file-lookup", "count": len(items), "items": items, }, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n") return len(shards) def collection_paths(db: dict) -> dict[str, set[str]]: """Every path under bios/ that holds each file, by sha1. The database keeps one path per content. The others are the full tails by_path_suffix records that name a file on disk: RPG Maker, engine trees and the copies filed under several consoles keep 2,502 of them. """ files = db.get("files", {}) bios_dir = Path(str(db.get("bios_dir") or "bios")) paths: dict[str, set[str]] = {} for sha1, entry in files.items(): path = str(entry.get("path", "")) if path.startswith("bios/") and path.count("/") >= 2: paths.setdefault(sha1, set()).add(path) for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items(): if tail.count("/") < 2 or len(sha1s) != 1 or sha1s[0] not in files: continue if (bios_dir / tail).is_file(): paths.setdefault(sha1s[0], set()).add(f"bios/{tail}") return paths def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]: """Group files by manufacturer -> console -> files. The database keeps one path per content, and a copy filed elsewhere was on no page of its own: the Channel F BIOS sat under Arcade > FBNeo only, and Fairchild had no system page. Every folder holding a copy lists it, read from the full paths by_path_suffix records. The game engines' files are grouped by engine (_engine_sections). """ files = db.get("files", {}) manufacturers: dict[str, dict[str, list]] = {} for sha1, where in _copy_places(db).items(): for manufacturer, console in sorted({(m, c) for m, c, _ in where}): manufacturers.setdefault(manufacturer, {}).setdefault(console, []).append(files[sha1]) return _engine_sections(manufacturers) def _engine_sections(manufacturers: dict[str, dict[str, list]]) -> dict[str, dict[str, list]]: """A game engine's files in one section headed with the engine's name. The Systems index listed ".variants", "scummvm.zip" and "wildmidi.cfg" as games, and ScummVM's files sat on three pages. A folder of variants joins the section of its primary, a file kept at a top's root joins the section of the one engine that reads it (else of the engine the top is named after), the folder under Other named after a game-engine top joins that top, and a folder named by a profile key is headed with the engine's name (SECTION_LABELS), its address unchanged. """ from siterender import _PROGRAM_TOPS, SECTION_LABELS, record_card_move # noqa: PLC0415 profiles = {key: profile for key, profile in _profiles_on_disk().items() if profile.get("type") not in ("alias", "test")} readers: dict[str, set[str]] = {} for key, profile in profiles.items(): for entry in profile.get("files") or []: if isinstance(entry, dict): for name in [entry.get("name"), *(entry.get("aliases") or [])]: if name: readers.setdefault(str(name).rsplit("/", 1)[-1], set()).add(key) def engine(key: str) -> str: return str(profiles[key].get("emulator") or key) def profile_of(name: str) -> str | None: return next((k for k in sorted(profiles) if name.casefold() in (k.casefold(), engine(k).casefold())), None) out: dict[str, dict[str, list]] = {} for mfr, consoles in manufacturers.items(): if mfr not in GAME_DATA_TOPS: out[mfr] = dict(consoles) continue own = profile_of(mfr) regrouped: dict[str, list] = {} for console, files in sorted(consoles.items(), key=lambda item: item[0].startswith(".")): for f in files: parts = str(f.get("path", "")).split("/") target = console if console.startswith(".") or (len(parts) == 3 and console == parts[-1]): # A variant answers its primary's name: scummvm.zip.a93f1c4b. base = parts[-1].rsplit(".", 1)[0] if console.startswith(".") else parts[-1] keys = sorted(readers.get(base, ())) key = keys[0] if len(keys) == 1 else own found = next((c for c, held in regrouped.items() if any(h.get("name") == base for h in held)), None) target = found or (next((c for c in consoles if c == key), None) or (engine(key) if key else base)) if target != console: record_card_move(mfr, console, str(f.get("sha1", "")), mfr, target) regrouped.setdefault(target, []).append(f) out[mfr] = regrouped # The folder under Other an engine top is named after: ScummVM's # engine data under Other/ScummVM, its themes under ScummVM/. others = out.get("Other", {}) for folder in sorted(others): top = next((t for t in out if t in GAME_DATA_TOPS and t.casefold() == folder.casefold()), None) if top is None or top in _PROGRAM_TOPS: continue sections = out[top] target = next((c for c in sections if c.casefold() == folder.casefold()), folder) held = {str(f.get("sha1", "")) for f in sections.get(target, [])} for f in others[folder]: record_card_move("Other", folder, str(f.get("sha1", "")), top, target) sections[target] = [*sections.get(target, []), *(f for f in others.pop(folder) if str(f.get("sha1", "")) not in held)] for mfr in GAME_DATA_TOPS: for console in out.get(mfr, {}): key = profiles.get(console) and console if key and engine(key) != console: SECTION_LABELS[(mfr, console)] = engine(key) return {mfr: consoles for mfr, consoles in out.items() if consoles} def _console_display(manufacturer: str, console: str) -> str: """A console section's name as the reader looks for it: the heading system_name gives its id, or the folder alone under a program's top (Other, Arcade), where "Other mame" would name nothing.""" from siterender import _PROGRAM_TOPS, _heading_name # noqa: PLC0415 return console if manufacturer in _PROGRAM_TOPS else _heading_name(manufacturer, console) def generate_systems_index(manufacturers: dict) -> str: """Every console in one table, found by its own name. The index listed 74 manufacturers and no console: someone looking for PlayStation had to know it is filed under Sony, and Other hid 91 sections. Manufacturer pages stay the containers. """ for old, new in SYSTEM_PAGES_MOVED.items(): sitestarlight.MOVED_PAGES.setdefault(old, new) total_mfr = len(manufacturers) total_consoles = sum(len(c) for c in manufacturers.values()) # A copy filed under two folders appears on both pages, counted once here. total_files = len({ f.get("sha1") for consoles in manufacturers.values() for files in consoles.values() for f in files }) lines = [ "# Systems", "", "Every BIOS file the collection holds, by console. Open a console to " "see each file, the emulators that read it and the setups that list it.", "", f"{_count(total_consoles, 'console')}, {_count(total_files, 'file')} in the " f"repository, from {_count(total_mfr, 'manufacturer')}.", "", ] # Other holds one folder per emulator: a7800 or altirra in the console # table named no console. Arcade boards are neither consoles nor # computers, and the crumb of their page says Arcade too. def kind(mfr: str) -> str: if mfr in GAME_DATA_TOPS: return "games" if mfr == "Arcade": return "arcade" return "programs" if mfr == "Other" else "consoles" for heading, group, first, note in ( ("Consoles and computers", "consoles", "Console", ""), ("Arcade", "arcade", "Name", "Arcade boards, and the sets each arcade emulator family reads."), ("Emulator folders", "programs", "Folder", "Files kept under the emulator that reads them, not under a console."), ("Game engines and PC games", "games", "Name", "Game engines and PC games hold the files a game engine reads, such " "as the data of a game, rather than the BIOS of a machine."), ): rows = sorted( ((_folder_emulator(console)[0] if group == "programs" else _section_label(mfr, console) if mfr == "Arcade" or (mfr, console) in SECTION_LABELS else _console_display(mfr, console), mfr, console, len(files)) for mfr, consoles in manufacturers.items() if kind(mfr) == group for console, files in consoles.items()), key=lambda row: (row[0].casefold(), row[1].casefold(), row[2]), ) if not rows: continue lines += [f"## {heading}", ""] if note: lines += [note, ""] if group == "programs": # Each folder by the emulator's name, linked to its page: the # rows read a7800 and azaharplus beside console names, and # every maker cell said Other. lines += ["| Emulator | Files |", "|---------|------:|"] for name, mfr, console, count in rows: href = console_href(_slug(mfr), console).removeprefix("systems/") key = _folder_emulator(console)[1] cell = name.replace("|", "-") cell = f"[{cell}](../emulators/{key}.md)" if key else cell lines.append(f"| {cell} | [{_count(count, 'file')}]({href}) |") lines.append("") continue lines += [ # The count before the maker: on a phone the last column is # the one scrolled off, and most names carry the maker already. f"| {first} | Files | Maker |", "|---------|------:|-------|", ] for name, mfr, console, count in rows: cell = name.replace("|", "-") href = console_href(_slug(mfr), console).removeprefix("systems/") lines.append(f"| [{cell}]({href}) | {count:,} | [{mfr}]({_slug(mfr)}.md) |") lines.append("") return "\n".join(lines) + "\n" def _folder_emulator(folder: str) -> tuple[str, str]: """The display name and profile of the emulator a folder under Other is named after, by its profile key or its name, else the folder and "". """ profiles = _profiles_on_disk() if folder in profiles and profiles[folder].get("type") not in ("alias", "test"): return str(profiles[folder].get("emulator") or folder), folder for key in sorted(profiles): profile = profiles[key] if profile.get("type") in ("alias", "test"): continue names = {str(profile.get(field) or "").casefold() for field in ("emulator", "display_name")} if folder.casefold() in names: return str(profile.get("emulator") or key), key return folder, "" _PROVENANCE_LABELS = {"redump": "Redump", "no-intro": "No-Intro", "tosec": "TOSEC"} _PROVENANCE_HOMES = { "redump": "http://redump.org/", "no-intro": "https://no-intro.org/", "tosec": "https://www.tosecdev.org/", } def _prov_title(data: dict) -> str: """Tooltip text for a provenance badge.""" parts = [data.get("dat", ""), data.get("description", "")] return ": ".join(p for p in parts if p).replace('"', """) # TOSEC dump flags that say the dump is not a clean copy of the chip # (TOSEC Naming Convention, "Dump info flags"); [a] alternate and [!] # verified good are not among them. _TOSEC_BAD_FLAGS = { "b": "bad dump", "o": "overdump", "h": "hacked", "cr": "cracked", "f": "fixed", "t": "trained", "p": "pirate", } _TOSEC_FLAG_RE = re.compile(r"\[(b|o|h|cr|f|t|p)\d*(?:\s[^\]]*)?\]") def _dump_flags(source: str, data: dict) -> list[str]: """What a catalog's name says is wrong with the dump, as words.""" if source != "tosec": return [] name = str(data.get("name") or data.get("description") or "") seen: list[str] = [] for flag in _TOSEC_FLAG_RE.findall(name): word = _TOSEC_BAD_FLAGS[flag] if word not in seen: seen.append(word) return seen def _provenance_lines(provenance: dict, prefix: str) -> list[str]: """One visible line per catalog: its badge and the entry's own name. The name was only a title attribute, unreadable on a touch screen and absent from the search for 793 of 803 provenances, and a TOSEC entry flagged [b] or [o] wore the same green badge as a clean dump. """ lines: list[str] = [] for source, data in sorted(provenance.items()): label = _PROVENANCE_LABELS.get(source, source) flags = _dump_flags(source, data) kind = "rb-badge-warning" if flags else "rb-badge-success" badge = f'[{label}]({prefix}provenance.md#{source})' name = str(data.get("name") or data.get("description") or "") line = f"- Catalogued: {badge} {_code(name)}" if name else f"- Catalogued: {badge}" if flags: line += f" ({', '.join(flags)}, as {label} records it)" lines.append(line) return lines def load_derivable(path: str = "derivable.json") -> dict[str, dict]: """Missing sha1 -> how the collection produces it, as the pipeline found. Empty when the pipeline has not recorded it: the page then lists every missing dump, as it did before the record existed. """ try: with open(path, encoding="utf-8") as handle: return json.load(handle) except FileNotFoundError: return {} _ROMSET_SOURCES = {"mame": "MAME", "fbneo": "FBNeo"} def _date_in_words(stamp: str) -> str: """`2026-08-07T...` as ``.""" try: day = datetime.strptime(str(stamp)[:10], "%Y-%m-%d") except ValueError: return _escape_tags(str(stamp)) return f'' def generate_provenance_page( db: dict, report: dict, derivable: dict[str, dict] | None = None, moved: dict[str, str] | None = None, ) -> str: """Page explaining the verified dump badges and listing catalog gaps. With ``moved``, each catalog's wanted list goes to a page of its own, keyed by stem: the three lists made a 694 KB page.""" matched_files = count_catalog_matched(db) total_files = compute_composition(db)["systems"]["files"] # When each catalog was read: a 76-day-old snapshot showed no age. as_of = ", ".join( f"{_PROVENANCE_LABELS.get(source, source)} {_date_in_words(data['imported_at'])}" for source, data in report.items() if data.get("imported_at") ) lines = [ "# Dump provenance", "", *([f'

Catalogs as of: {as_of}.

', ""] if as_of else []), "Provenance says whether a file is, byte for byte, a dump that Redump, " "No-Intro or TOSEC catalogues. It tells you where a dump is known from, " "not whether an emulator accepts it: that is the emulator's code, read " "in its profile. Example: `scph5501.bin` is Redump's `ps-30a.bin`.", "", f"**{matched_files:,}** of {total_files:,} system files match an entry " "in a dump-preservation catalog. Those files carry a " 'catalog badge and the ' "catalog entry's own name on the [system pages](systems/index.md), " 'orange (TOSEC) where ' "TOSEC flags the dump as bad, overdumped, hacked, cracked or fixed. " "Arcade ROM sets and engine data " "sit outside what those catalogs index, so they are excluded from the " "ratio rather than counted as failures.", "", "## What the badge means", "", "The badge says the file is byte-identical to a dump catalogued by " "No-Intro, Redump, or TOSEC. Matching is done on SHA1, falling back to " "MD5 plus size for older catalog entries that predate SHA1. Filenames " "are never used: the same dump is `fdsbios.nes` here, " "`[BIOS] Family Computer Disk System (Japan) (En) (Rev 1).bin` at " "No-Intro, and " "`Nintendo Famicom Disk System BIOS (198x)(Nintendo)(JP)(en).bin` at " "TOSEC.", "", "## What it does not mean", "", "A file without a badge is not inferior and works exactly the same. " "Emulator behaviour is decided by [verification]" "(wiki/verification-modes.md), which reads the emulator source code, " "not by catalog membership. Plenty of files this project ships are " "outside any catalog by nature: composites a core assembles for itself " "(the MiSTer X68000 `boot.rom` is `cgrom.dat` joined to `iplrom.dat`), " "modern console firmware updates, game data such as Diablo's `diabdat.mpq`, and " "arcade sets tracked by MAME driver source instead.", "", "When the two views disagree, this project follows the code, because " "that is what decides whether your emulator boots. The reasoning is in " "the [technical questions](wiki/technical-faq.md#are-these-files-verified-against-original-" "hardware-dumps).", "", "## Coverage", "", "| Catalog | In collection | Covered DATs | Snapshot |", "|---------|--------------|-------------|----------|", ] for source, data in report.items(): label = _PROVENANCE_LABELS.get(source, source) home = _PROVENANCE_HOMES.get(source, "") in_scope = data["matched"] + len(data["missing"]) pct = 100 * data["matched"] / in_scope if in_scope else 0 name = f"[{label}]({home})" if home else label lines.append( f"| {name} | {data['matched']:,}/{in_scope:,} ({pct:.0f}%) | " f"{len(data['covered_dats'])} | {_date_in_words(data['imported_at'])} |" ) lines.extend( [ "", "Coverage counts only DATs the collection already reaches: a DAT " "counts as covered once at least one of its entries is held. " "No-Intro tags every non-game dump `[BIOS]`, including digital " "title distribution such as the Wii U and 3DS CDN catalogues, " "which this project does not ship.", "", ] ) out_of_scope = {s: d["out_of_scope"] for s, d in report.items() if d["out_of_scope"]} if out_of_scope: detail = ", ".join( f"{_PROVENANCE_LABELS.get(s, s)} {c:,}" for s, c in sorted(out_of_scope.items()) ) lines.extend( [ f"Entries in DATs the collection does not cover at all are " f"excluded from those numbers ({detail}).", "", ] ) # A dump the collection already produces (the odd bytes of a Kickstart, # the halves of a character ROM) is no acquisition target: the page asked # the public for 79 of them. derivable = derivable or {} derived_entries: dict[str, dict] = {} for data in report.values(): kept = [] for entry in data["missing"]: sha1 = (entry.get("sha1") or "").lower() if sha1 in derivable: derived_entries.setdefault(sha1, entry) else: kept.append(entry) data["missing"] = kept total_missing = distinct_missing(report) lines.extend( [ "## Wanted: catalogued dumps this project does not have", "", f"The tables below list **{total_missing:,}** dumps a catalog " "describes and the collection lacks. Nothing here ships in any " "pack: it is an acquisition list, published with hashes so anyone " "can check a personal collection against it. A contribution " "matching one of these hashes is welcome: see " "[Contributing](contributing.md).", "", ] ) for source, data in report.items(): if not data["missing"]: continue label = _PROVENANCE_LABELS.get(source, source) by_dat: dict[str, list] = {} for entry in data["missing"]: for dat in entry.get("dats") or [entry.get("dat", "")]: by_dat.setdefault(dat, []).append(entry) lines.extend([f"### {label}", ""]) blocks: list[list[str]] = [] for dat in sorted(by_dat): entries = by_dat[dat] table = ["| Name | Description | SHA1 |", "|------|-------------|------|"] for entry in sorted(entries, key=lambda e: e["name"]): sha1 = entry.get("sha1") or "-" # A pipe in catalog text would split the markdown table row name = entry["name"].replace("|", "-") desc = (entry.get("description") or "").replace("|", "-") # Where an emulator's romset already carries the dump. for hit in entry.get("romsets", []): desc += ( f" Ships in {_ROMSET_SOURCES.get(hit['source'], hit['source'])} " f"`{hit['archive']}` as `{hit['member']}`." ) table.append(f"| `{name}` | {desc} | `{sha1}` |") blocks.append(markup.details("note", f"{dat} ({len(entries)})", table)) if moved is None: lines.extend(line for block in blocks for line in block) continue where = _list_pages( f"provenance-{_slugify_anchor(source)}", f"Wanted from {label}", f"Dumps {label} catalogues and the collection lacks, by DAT. Part of " f"[Dump provenance](provenance.md#{_slugify_anchor(label)}).", [], blocks, moved, parent=("Dump provenance", "/provenance/"), ) lines.extend([ f"{_count(len(data['missing']), 'dump')} in {_count(len(by_dat), 'DAT')}: " f"listed on {where}.", "", ]) if derived_entries: lines.extend( [ "## Derivable: dumps the collection already produces", "", f"**{len(derived_entries):,}** catalogued dumps are not held as " "files but come out of one that is: a byte or word split, a " "half, a swap, or two chips joined. They are not wanted.", "", "| Name | Comes from | How | SHA1 |", "|------|------------|-----|------|", ] ) for sha1, entry in sorted(derived_entries.items(), key=lambda kv: kv[1]["name"]): how = derivable[sha1] sources = " + ".join(f"`{path}`" for path in how.get("from", [])) name = entry["name"].replace("|", "-") lines.append(f"| `{name}` | {sources} | {how.get('transform', '')} | `{sha1}` |") lines.append("") lines.extend(["", _date_line()]) return "\n".join(lines) + "\n" _RELEASE_ASSET_NAMES: dict[str, str] | None = None def _fetch_url(path: str) -> str: """Where a collection file downloads from: its raw URL, or the large-files release asset when .gitignore keeps it out of git.""" global _RELEASE_ASSET_NAMES if _RELEASE_ASSET_NAMES is None: from largefiles import ( # noqa: PLC0415 LARGE_FILES_RELEASE, asset_names, load_registered_paths, published_asset_name, ) _RELEASE_ASSET_NAMES = { registered: f"{REPO_URL}/releases/download/{LARGE_FILES_RELEASE}/" + urllib.parse.quote(published_asset_name(name)) for registered, name in asset_names(load_registered_paths()).items() } return _RELEASE_ASSET_NAMES.get(path) or ( "https://raw.githubusercontent.com/Abdess/retrobios/main/" + urllib.parse.quote(path) ) _platform_names: dict[str, str] = {} def _platform_display(name: str) -> str: """A setup's name as its platform file writes it: Batocera, not batocera.""" display = _platform_names.get(name) if display is None: try: config = load_platform_config(name, str(_REPO_ROOT / "platforms")) except FileNotFoundError: config = {} display = _platform_names[name] = str(config.get("platform") or name) return display # A setup, the link to its page, and where it puts the file. Place = tuple[str, str, list[str]] def _platform_places(sha1: str, consumers: FileConsumers) -> list[Place]: """Each platform declaring the file, linked to its system section, with the destination it gives the file.""" systems_of = getattr(consumers, "platform_systems", {}).get(sha1, {}) destinations_of = getattr(consumers, "platform_destinations", {}).get(sha1, {}) places: list[Place] = [] for platform in consumers.platforms.get(sha1, ()): section = sorted(systems_of.get(platform, ())) fragment = f"#{section[0].replace(' ', '-')}" if section else "" places.append((_platform_display(platform), f"../platforms/{platform}.md{fragment}", sorted(destinations_of.get(platform, ())))) return sorted(places, key=lambda place: place[0].casefold()) def _platform_links(sha1: str, consumers: FileConsumers) -> list[str]: """Each platform declaring the file, as Markdown: its name, linked to its system section, then the destination it gives the file.""" return [ f"[{name}]({href})" + (": " + ", ".join(_code(d) for d in dests) if dests else "") for name, href, dests in _platform_places(sha1, consumers) ] _card_anchors: dict[str, tuple[list, list[str]]] = {} def _emulator_card_anchor(emulator: str, files: list, index: int) -> str: """The id of a profile entry's card on its emulator page.""" cached = _card_anchors.get(emulator) if cached is None or cached[0] is not files: cached = _card_anchors[emulator] = (files, emulator_file_anchors(files)) return cached[1][index] if 0 <= index < len(cached[1]) else "" def _emulator_links(sha1: str, consumers: FileConsumers) -> list[str]: """Each emulator reading the file, linked to the entry that reads it. The platform links landed on their system section, and the emulator links on the page top: duckstation's scph5501.bin entry, with the line that accepts it, sat 260 lines down a page of 105 entries. """ entries = getattr(consumers, "emulator_entries", {}).get(sha1, {}) files_of = getattr(consumers, "emulator_files", {}) links = [] # By the name the emulator's page carries: 19,499 card links read # melonds_ds where its page says melonDS DS. named = sorted(((_emulator_display(e), e) for e in consumers.emulators.get(sha1, ())), key=lambda pair: (pair[0].casefold(), pair[1])) for display, emulator in named: index = entries.get(emulator) anchor = (_emulator_card_anchor(emulator, files_of[emulator], index) if index is not None and emulator in files_of else "") page = emulator_card_page(emulator, anchor) if anchor else emulator fragment = f"#{anchor}" if anchor else "" # The card's facts are raw HTML: "Game & Watch" is escaped, and a # bracket would end the link the facts are parsed from. text = html.escape(display, quote=False).replace("[", "[").replace("]", "]") links.append(f"[{text}](../emulators/{page}.md{fragment})") return links def _installed_places(sha1: str, consumers: FileConsumers, declared: set[str]) -> list[Place]: """Each pack installing the file that its platform's list does not name, with the place the pack gives it.""" places = [ (_platform_display(platform), f"../platforms/{platform}.md", sorted(d for d in dests if d)) for platform, dests in getattr(consumers, "installed", {}).get(sha1, {}).items() if platform not in declared ] return sorted(places, key=lambda place: place[0].casefold()) def _places_lines(places: list[Place]) -> list[str]: """Setups grouped by where they put the file, one line per place: "`scph5501.bin`: Batocera, EmuDeck". Each setup repeating the name took five lines where one says it. A setup that names no place (EmuDeck matches by MD5) is listed on a line of its own, last.""" def plain(text: str) -> str: return html.escape(text, quote=False).replace("`", "`") by_place: dict[tuple[str, ...], list[str]] = {} for name, href, dests in places: by_place.setdefault(tuple(dests), []).append(f"[{name}]({href})") ordered = sorted(by_place.items(), key=lambda item: (not item[0], -len(item[1]), item[0])) return [ (" and ".join(f"{plain(d)}" for d in dests) + ": " if dests else "") + ", ".join(links) for dests, links in ordered ] def _inline_card_html(text: str) -> str: """A card value written in Markdown (code spans, links), as the HTML a fact list holds. HTML already in the value is kept.""" out: list[str] = [] for part in re.split(r"(`[^`]*`)", text): if len(part) > 1 and part.startswith("`") and part.endswith("`"): out.append(f"{html.escape(part[1:-1], quote=False)}") else: out.append(re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", r'\1', part)) return "".join(out) # A link icon, the way a heading anchor is drawn: the word "permalink" was # read as part of every card title, by the search and by screen readers. # Drawn once per page and used by each card: inline, its 400 bytes were # 100 KB on the Sony page. PERMALINK_SYMBOL = ( '' ) _PERMALINK_ICON = ('') def _icon_symbol() -> list[str]: """The icon's drawing, for a page of cards. None in the yardstick.""" return [PERMALINK_SYMBOL, ""] if markup.starlight() else [] RomsetMembers = tuple[tuple[str, str], ...] # A release a recipe agrees with: its place among the imported releases of # its source, and its name ("0.220", "1.0.0.03"). Release = tuple[int, str] def _release_name(source: str, label: str, versions: dict) -> str: """"MAME mame0220" as "0.220", an FBNeo DAT title as its version.""" match = re.fullmatch(r"MAME mame(\d)(\d+)", label) if source == "mame" else None return f"{match[1]}.{match[2]}" if match else str(versions.get(label) or label) @functools.lru_cache(maxsize=1) def _romset_snapshots() -> list[tuple[str, dict]]: return [(str(snapshot.get("source", path.stem)), snapshot) for path in sorted((_REPO_ROOT / "recipes").glob("*.json")) for snapshot in [json.loads(path.read_text(encoding="utf-8"))]] @functools.lru_cache(maxsize=1) def _romset_releases() -> dict[str, list[tuple[RomsetMembers, str, list[Release]]]]: """Archive name -> (members, source, the releases agreeing on them), from the MAME and FBNeo recipe snapshots under recipes/. A snapshot stores a recipe once for every version that agrees on it, so a match reads "this set's contents in those versions". """ releases: dict[str, list[tuple[RomsetMembers, str, list[Release]]]] = {} for source, snapshot in _romset_snapshots(): versions = snapshot.get("dats") or {} order = {label: index for index, label in enumerate(versions)} for entry in snapshot.get("entries") or []: members = tuple(sorted( (str(m["name"]), str(m.get("crc32") or "").lower()) for m in entry.get("members") or [] if m.get("name") is not None )) if not entry.get("name") or not members or not all(crc for _n, crc in members): continue agreed = [(order.get(label, -1), _release_name(source, str(label), versions)) for label in entry.get("dats") or [entry.get("dat")]] releases.setdefault(str(entry["name"]), []).append((members, source, agreed)) return releases def _release_runs(releases: list[Release]) -> str: """Releases as runs of consecutive imports: "0.190 to 0.195, 0.220 to 0.289". One FBNeo version shared by 15 DATs reads once.""" runs: list[list[Release]] = [] for release in sorted(set(releases)): if runs and release[1] == runs[-1][-1][1]: continue if runs and release[0] == runs[-1][-1][0] + 1: runs[-1].append(release) else: runs.append([release]) return ", ".join(run[0][1] if len(run) == 1 else f"{run[0][1]} to {run[-1][1]}" for run in runs) def _source_order(source: str) -> tuple[bool, str]: return source != "mame", source def _romset_line(f: dict) -> str: """Which MAME and FBNeo releases a collected archive's contents match. MAME and FBNeo check the members of an archive by CRC32, never its own bytes or its member names: a set works with a release when it holds every CRC32 the release's recipe lists, extra members or not. The main neogeo.zip, which every MAME from 0.190 to 0.289 loads, read "No match" when names had to agree and no member could be extra. Empty when no recipe names the archive or the file is not on disk. """ recipes = _romset_releases().get(str(f.get("name", ""))) path = _REPO_ROOT / str(f.get("path", "")) if not recipes or not path.is_file(): return "" try: with zipfile.ZipFile(path) as archive: held = tuple(sorted((info.filename, f"{info.CRC:08x}") for info in archive.infolist() if not info.is_dir())) except (OSError, zipfile.BadZipFile) as exc: print(f" WARNING: {path}: {exc}", file=sys.stderr) return "" crcs = {crc for _name, crc in held} same: dict[str, list[Release]] = {} holds: dict[str, list[Release]] = {} # source -> (fewest CRC32s missing, the latest release missing that many) closest: dict[str, tuple[int, Release]] = {} for members, source, agreed in recipes: missing = len({crc for _name, crc in members} - crcs) if members == held: same.setdefault(source, []).extend(agreed) elif not missing: holds.setdefault(source, []).extend(agreed) elif agreed: latest = max(agreed) best = closest.get(source) if best is None or (missing, -latest[0]) < (best[0], -best[1][0]): closest[source] = (missing, latest) def runs(found: dict[str, list[Release]], skip: dict[str, list[Release]]) -> str: return " and ".join( f"{_ROMSET_SOURCES.get(source, source)} {_release_runs(kept)}" for source in sorted(found, key=_source_order) for kept in [[r for r in found[source] if r not in skip.get(source, [])]] if kept) said = [] if same: said.append(f"Same contents as {runs(same, {})}") if holds and runs(holds, same): said.append(f"Holds every ROM of {runs(holds, same)}") if said: return ". ".join(said) if not closest: return f"No match in {_romset_coverage()}" return "No match: missing " + ", ".join( f"{_count(missing, 'ROM')} of {_ROMSET_SOURCES.get(source, source)} {release[1]}" for source, (missing, release) in sorted(closest.items(), key=lambda item: _source_order(item[0]))) @functools.lru_cache(maxsize=1) def _romset_coverage() -> str: """The releases the recipes cover: "MAME 0.190 to 0.289 or FBNeo 1.0.0.03".""" covered = {} for source, snapshot in _romset_snapshots(): versions = snapshot.get("dats") or {} if versions: ends = [(index, _release_name(source, label, versions)) for index, label in enumerate(versions)] # The ends of the span, read as one run. span = [ends[0], (1, ends[-1][1])] if len(ends) > 1 else ends covered[source] = f"{_ROMSET_SOURCES.get(source, source)} {_release_runs(span)}" return " or ".join(covered[source] for source in sorted(covered, key=_source_order)) ROMSET_FAQ = "../wiki/technical-faq.md#which-mame-version-do-the-arcade-bios-sets-match" def _system_file_card( f: dict, consumers: FileConsumers, anchored: set[str] | None = None, searchable: bool = True, ) -> list[str]: """One file of a system page: hashes, who reads it, its provenance. ``searchable`` false keeps even the name out of the site search: the asset trees of game engines hold thousands of 0.ogg and tile.png. """ name = f["name"] sha1_full = f.get("sha1", "unknown") md5_full = f.get("md5", "unknown") size = _fmt_size(f.get("size", 0)) # Who reads this file, not who reads a file of this name plats = sorted(consumers.platforms.get(sha1_full, ())) emus = sorted(consumers.emulators.get(sha1_full, ())) installed = _installed_places(sha1_full, consumers, set(plats)) anchor = f"sha1-{sha1_full[:12]}" if sha1_full != "unknown" else "" # A copy filed under two consoles of one maker is listed twice on its # page: the first card holds the anchor, as an id names one element. if anchored is not None: if anchor in anchored: anchor = "" else: anchored.add(anchor) skip = markup.search_exclude() lines = [markup.block_open("div", 'class="rb-sys-file"'), ""] # The heading's own link is this page's, the icon is the file's own # address, resolved by content wherever its card moves. # The yardstick keeps the word, the size the page budget was set on. icon = _PERMALINK_ICON if markup.starlight() else "permalink" permalink = ( f' {icon}' ) if anchor else "" # A file nothing reads said so nowhere: 53 of 248 Sony cards had no # emulator, platform or pack line and looked unfinished. unread = (f' No reader' if not (emus or plats or installed) else "") lines.append(_card_title( f'{_code(name)} {size}{unread}{permalink}', anchor, searchable, )) lines.append("") # Each fact: its label, its values, whether the search reads it. What a # setup does with the file comes first, the fingerprints last. facts: list[tuple[str, list[str], bool]] = [] if plats: # Each link lands on the platform's system section, and says where # that platform puts the file. Out of the search: the card's title # weighs the most a word can, and the list made the excerpt of # every card a run of setup names and paths. facts.append(("On the setup's list", _places_lines(_platform_places(sha1_full, consumers)), False)) if installed: # The packs that carry the file for their cores: a RetroBat user # looking up scph5501.bin found every platform but RetroBat, whose # pack installs it for six cores. facts.append(("Packed for its emulators by", _places_lines(installed), False)) if emus: facts.append(("Emulators", [", ".join(_emulator_links(sha1_full, consumers))], False)) romset = _romset_line(f) if romset: # The version question every arcade player asks, answered from the # recipes: the cards named emulators and no release. facts.append(("Romset", [romset], False)) if f.get("path"): # The card named a file, its hashes and provenance, but not where to # get it: a user missing scph5501.bin found the reference and no way # to the bytes. facts.append(("Collection path", [f"{_code(f['path'])} ([download]({_fetch_url(f['path'])}))"], False)) catalogued = [line.split(": ", 1)[1] for line in _provenance_lines(f.get("provenance") or {}, "../")] if catalogued: facts.append(("Catalogued", catalogued, False)) # Every hash in full: the site search indexes text, not a title # attribute, so a pasted md5 or crc32 (what clrmamepro, RomVault or an # emulator log prints) found nothing behind a 12-character prefix. # files/?hash= finds them, and the index skips them. for label, value in (("SHA1", sha1_full), ("MD5", md5_full), ("CRC32", f.get("crc32")), ("SHA256", f.get("sha256"))): if value and value != "unknown": facts.append((label, [f'{html.escape(str(value))}'], False)) if not markup.starlight(): # The size yardstick pages are cut by: the list a card was when the # budget was set, so a fact grid moves no card to another page. lines.extend(search_excluded([f"- {label}: {', '.join(values)}" for label, values, _searched in facts])) else: # A grid of facts, label then value: two bullet lists of code chips # read as a dump. What the search skips is one list with one # attribute: on each row it was 77 KB of the Sony page. read = [fact for fact in facts if fact[2] and searchable] skipped = [fact for fact in facts if fact not in read] for group, attributes in ((read, ""), (skipped, f" {skip}")): if group: lines.append(f'
') # One value cell per label: the grid pairs each dt with one dd. # A searched value keeps its label out: the excerpt ran the two # together, "On the setup's listAmberELEC". label_skip = f" {skip}" if not attributes else "" lines.extend(f"{label}
" + "
".join(_inline_card_html(v) for v in values) + "
" for label, values, _searched in group) lines.append("
") lines.extend(["", "", ""]) return lines SEARCH_TABLE_BYTES = 3000 def tables_out_of_search(markdown: str, threshold: int = SEARCH_TABLE_BYTES) -> str: """Each table longer than ``threshold`` bytes, wrapped out of the search. A search result shows the start of a section: a 600-row table as one snippet helps nobody, and the gaps, provenance and cross-reference tables were 900 KB of the index. The headings above them stay. """ lines = markdown.split("\n") out: list[str] = [] index = 0 while index < len(lines): match = re.match(r"( *)\|", lines[index]) if not match: out.append(lines[index]) index += 1 continue indent = match.group(1) end = index while end < len(lines) and re.match(rf"{indent}\|", lines[end]): end += 1 table = lines[index:end] if sum(len(line) for line in table) > threshold: out += ["", indent + markup.block_open("section", markup.search_exclude()), "", *table, "", f"{indent}", ""] else: out += table index = end return "\n".join(out) def search_excluded(lines: list[str]) -> list[str]: """Lines the site search leaves out, as a block of their own. The browser downloads and indexes the whole search index before the first result: hashes, link lists and install commands repeated on 600 pages made it 21 MB. Hashes are found by files/?hash=, names and prose stay searchable. A section, not a div: the search plugin compares elements by tag, so the first div closed inside an excluded div (a code block's) ended the exclusion and the rest of the block was indexed. """ return ["", markup.block_open("section", markup.search_exclude()), "", *lines, "", "", ""] # Card lines the site search leaves out. They list where a file goes and # who cites it, links to other pages that index the same names, and they # were 6.6 of the 24.7 MB the browser downloads and indexes before the first # result. The name, the hashes, the entry's path and the notes stay # searchable. SEARCH_SKIPPED = ( "- SHA1:", "- MD5:", "- CRC32:", "- SHA256:", "- Adler32", "- Size:", "- Validation", "- Priority:", "- Contents (", "- Platforms:", "- Also installed by:", "- Source:", "- Asked for by:", "- Collection path:", "- Collection copy:", "- Collection copies:", "- Installed by the platform's own package on:", "- Not in the platform's package", "- Never opened on these platforms", ) def _search_skipped_apart(lines: list[str]) -> list[str]: """Card list lines, each run the search skips in a block of its own.""" out: list[str] = [] skipping = False for line in lines: skip = line.startswith(SEARCH_SKIPPED) or (skipping and line.startswith(" ")) if skip and not skipping: out.extend(["", markup.block_open( "div", f'class="rb-search-skip" {markup.search_exclude()}'), ""]) elif skipping and not skip: out.extend(["", "", ""]) skipping = skip out.append(line) if skipping: out.extend(["", "", ""]) return out def _card_title(text: str, anchor: str, searchable: bool = True, weight: int = 10) -> str: """A file card's heading, so the site search lands on the card. The search splits a page at its headings: a card titled in bold sent a pasted md5 to the top of a page of a hundred cards. Level 3, under the console or the Files heading: a level 4 after a level 2 skipped one for screen readers on 551 pages. The table of contents of a page of cards stops above it (sitestarlight.py), and a card without an anchor keeps the id the toc gives it. """ if markup.starlight(): # Raw HTML: the card links to itself already, and a Markdown heading # would add the engine's anchor link and icon to every card. title = re.sub(r"`([^`]+)`", lambda m: f"{html.escape(m[1], quote=False)}", text) ident = f' id="{anchor}"' if anchor else "" # A system page's card is the file's own page in all but name: its # title weighs the most a word can, so a file name typed in the # search finds the card before a guide that cites it as an example. # An emulator's card says who reads the file, and weighs half. search = f' data-pagefind-weight="{weight}"' if searchable else f" {markup.search_exclude()}" return f'{title}' attributes = f"#{anchor} .rb-file-title" if anchor else ".rb-file-title" if not searchable: attributes += " data-search-exclude" return f"### {text} {{ {attributes} }}" def system_install_choices(coverages: dict) -> dict[str, dict[str, list[tuple[str, str]]]]: """slug -> console -> the (platform, system id) pairs linked to it. Read through the same table as every system link, so a command sits in the section the platform's system links to. Only a system the platform's install manifest records is offered: --system refuses any other name. """ from siterender import _page_by_norm # noqa: PLC0415 choices: dict[str, dict[str, list[tuple[str, str]]]] = {} for name, cov in sorted(coverages.items()): manifest = _REPO_ROOT / "install" / f"{name}.json" if not manifest.is_file(): continue document = json.loads(manifest.read_text(encoding="utf-8")) recorded = { str(system) for entry in (document.get("files") or []) + (document.get("omitted_files") or []) for system in entry.get("systems") or [] } for sys_id in sorted(set(cov["config"].get("systems") or {}) & recorded): page = _system_page_map.get(sys_id) or _page_by_norm(sys_id) if page: slug, console = page choices.setdefault(slug, {}).setdefault(console, []).append((name, sys_id)) return choices # A manufacturer page above this much Markdown becomes a summary with one # page per console, and a console above it is cut into parts. Measured as # MkDocs Markdown, a page renders to up to 1.8 times its size in HTML with # its structured data: at 500 KB the Sony page was 849 KB, and every page # stays under 600 KB from 300 KB. game-engines was one 10.5 MB page that # took 21 s of main-thread work on a phone. SYSTEM_PAGE_BUDGET = 300_000 # What the sections kept on a manufacturer's summary may weigh once # rendered, estimated from their Markdown. Fitted on the Nintendo, Sega and # Commodore summaries: highlighted install commands grow about eleven times # since each carries its setup's name, cards 1.7 times. The page budget is # 600 KB of HTML, and the frame around the content (head, navigation, # footer) is about 45 KB. CARD_HTML_RATIO = 1.7 INSTALL_HTML_RATIO = 11 SYSTEM_SUMMARY_HTML = 500_000 # A system card: whether it is a variant, its lines, its sha1. Card = tuple[bool, list[str], str] def _reading_entries(sha1: str, consumers: FileConsumers) -> list[dict]: """The profile entries that read a file, one per emulator.""" files_of = getattr(consumers, "emulator_files", {}) entries = [] for emulator, index in getattr(consumers, "emulator_entries", {}).get(sha1, {}).items(): files = files_of.get(emulator) or [] if 0 <= index < len(files) and isinstance(files[index], dict): entries.append(files[index]) return entries _NO_PRIORITY = 1 << 30 def _card_rank(f: dict, consumers: FileConsumers, start: dict[str, int] | None = None) -> tuple: """Where a file's card sits in its console: the files the section's start line names first, in its order (``start``, sha1 -> place), then the files an emulator requires, then by the emulators' search order (smallest first), then a required file by how many emulators read it, then by how many setups list the file, then by name. By byte order the PlayStation section opened on LibPS.exe and PSX_DTLH2000.bin, capitals first, and scph5501.bin was 28th. Without the reader count the Neo Geo section opened on aes.zip, which geolith alone reads. Without the start line's order, "first below" named neogeo.zip over a section opening on sfix.sfix. """ sha1 = str(f.get("sha1", "")) entries = _reading_entries(sha1, consumers) priorities = [e["priority"] for e in entries if isinstance(e.get("priority"), int)] required = any(e.get("required", True) for e in entries) return ( (start or {}).get(sha1, _NO_PRIORITY), not required, min(priorities, default=_NO_PRIORITY), -len(entries) if required else 0, -len(consumers.platforms.get(sha1, ())), f["name"].casefold(), f["name"], f.get("path", ""), ) def _by_name(f: dict) -> tuple: return f["name"].casefold(), f["name"], f.get("path", "") def _console_cards( manufacturer: str, files: list[dict], consumers: FileConsumers, anchored: set[str], by_name: bool = False, start: list[str] | None = None, ) -> list[Card]: """Each card of one console, primaries then variants, with whether it is a variant. ``start`` is the sha1s the start line names, whose cards come first in its order. ``by_name`` orders the rest by name alone, for a console cut into parts named by their first and last file: the PlayStation 2 opened on cheats_ni.zip, its BIOS far below.""" searchable = manufacturer not in GAME_DATA_TOPS place = {sha1: index for index, sha1 in enumerate(start or [])} def order(f: dict) -> tuple: if by_name: return (place.get(str(f.get("sha1", "")), _NO_PRIORITY), *_by_name(f)) return _card_rank(f, consumers, place) # Another revision of a MAME or FBNeo set is another release's set, its # Romset line saying which: it sits open beside the set of its name. # Twelve neogeo.zip were on five pages, the SNK ones closed under # Variants. romsets = _romset_releases() beside = [f for f in files if "/.variants/" in f["path"] and f["name"] in romsets] main_files = [f for f in files if "/.variants/" not in f["path"]] variant_files = [f for f in files if "/.variants/" in f["path"] and f not in beside] # Sets of one name side by side, the first ranked leading. ranked = sorted(main_files, key=order) by_set: dict[str, list[dict]] = {} for f in [*ranked, *sorted(beside, key=order)]: if f["name"] in romsets: by_set.setdefault(f["name"], []).append(f) ordered: list[dict] = [] for f in ranked: if f["name"] not in romsets: ordered.append(f) elif by_set.get(f["name"]): ordered.extend(by_set.pop(f["name"])) ordered.extend(f for files in by_set.values() for f in files) cards = [(False, _system_file_card(f, consumers, anchored, searchable), str(f.get("sha1", ""))) for f in ordered] cards += [(True, _system_file_card(v, consumers, anchored, searchable), str(v.get("sha1", ""))) for v in sorted(variant_files, key=order)] return cards # A "Start with" line names this many files at most: Game Engines' # TheXTech requires 2,014. START_WITH_NAMES = 6 def _region_order() -> dict[str, int]: from region import REGION_TREE, WORLD # noqa: PLC0415 ordered = [WORLD] for parent, members in REGION_TREE.items(): ordered += [parent, *sorted(members)] return {region: index for index, region in enumerate(ordered)} _REGION_WORDS = {"uk": "UK", "asia-ntsc": "Asia (NTSC)", "asia-pal": "Asia (PAL)"} def _region_word(region: str) -> str: return _REGION_WORDS.get(region) or region.replace("-", " ").title() def _reading_pairs(sha1: str, consumers: FileConsumers) -> list[tuple[str, dict]]: """The profile entries that read a file, each with its emulator.""" files_of = getattr(consumers, "emulator_files", {}) pairs = [] for emulator, index in sorted(getattr(consumers, "emulator_entries", {}).get(sha1, {}).items()): files = files_of.get(emulator) or [] if 0 <= index < len(files) and isinstance(files[index], dict): pairs.append((emulator, files[index])) return pairs @functools.lru_cache(maxsize=1) def _profiles_on_disk() -> dict[str, dict]: from common import load_emulator_profiles # noqa: PLC0415 return load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False) def _entry_systems(emulator: str, entry: dict) -> list[str]: """The system an entry is read for: its own, or its profile's only one.""" if entry.get("system"): return [str(entry["system"])] systems = (_profiles_on_disk().get(emulator) or {}).get("systems") or [] return [str(systems[0])] if len(systems) == 1 else [] def _emulator_display(emulator: str) -> str: return str((_profiles_on_disk().get(emulator) or {}).get("emulator") or emulator) # id(consumers) -> (consumers, section -> the entries whose system leads there). _SECTION_ENTRIES: dict[int, tuple[object, dict[tuple[str, str], list]]] = {} def _entries_by_section(consumers: FileConsumers) -> dict[tuple[str, str], list]: """Each console section with the (sha1, emulator, entry, system) whose system leads to it, read once per build.""" from siterender import system_section # noqa: PLC0415 cached = _SECTION_ENTRIES.get(id(consumers)) if cached is not None and cached[0] is consumers: return cached[1] sections: dict[str, tuple[str, str] | None] = {} index: dict[tuple[str, str], list] = {} for sha1 in sorted(getattr(consumers, "emulator_entries", {})): for emulator, entry in _reading_pairs(sha1, consumers): for sys_id in _entry_systems(emulator, entry): if sys_id not in sections: sections[sys_id] = system_section(sys_id) place = sections[sys_id] if place is not None: index.setdefault(place, []).append((sha1, emulator, entry, sys_id)) _SECTION_ENTRIES.clear() _SECTION_ENTRIES[id(consumers)] = (consumers, index) return index def _setup_name(f: dict, consumers: FileConsumers, pairs: list[tuple[str, dict]]) -> str: """The name most setups give a file, else most emulators: the collection keeps BizHawk's GBA_bios.rom where eight setups write gba_bios.bin.""" counts: dict[str, int] = {} destinations = getattr(consumers, "platform_destinations", {}).get(str(f.get("sha1", "")), {}) for dests in destinations.values(): for base in {d.replace("\\", "/").rsplit("/", 1)[-1] for d in dests if d}: counts[base] = counts.get(base, 0) + 1 if not counts: for _emulator, entry in pairs: if entry.get("name"): name = str(entry["name"]) counts[name] = counts.get(name, 0) + 1 if not counts: return f["name"] most = max(counts.values()) top = sorted((n for n, c in counts.items() if c == most), key=lambda n: (n.casefold(), n)) return f["name"] if f["name"] in top else top[0] def _without_files(emulators: list[str]) -> str: """Who runs without the files: every emulator reading one, up to three by name, more counted, as their order would rank nothing.""" names = sorted({_emulator_display(e) for e in emulators}, key=str.casefold) if len(names) > 3: return f": none of its {len(names)} emulators requires one" return " with " + (", ".join(names[:-1]) + f" or {names[-1]}" if len(names) > 1 else names[0]) def _visible(name: str) -> str: """The file name a start line shows, out of its code span and link.""" match = re.match(r"\[?`([^`]*)`", name) return match[1] if match else name def _add_name(names: list[str], name: str) -> None: """A name once per line: two copies of sfcbox.zip read as one file.""" if all(_visible(n) != _visible(name) for n in names): names.append(name) class _StartGroup: """What one system of a section asks for.""" def __init__(self) -> None: self.by_region: dict[str, list[str]] = {} self.plain: list[str] = [] # Files an emulator reads when present and replaces when absent. self.optional: list[str] = [] self.emulators: list[str] = [] # Each file an emulator requires, by its visible name, with the # emulators requiring it and the name as the line writes it. self.requires: dict[str, set[str]] = {} self.written: dict[str, str] = {} # Visible name -> the sha1 of the file it names. self.sha1s: dict[str, str] = {} # Emulators that replace a missing file with their own code. self.hle: set[str] = set() # Files an emulator reads when present and runs the same without. self.extra: list[str] = [] # By line (a region, "" for every region): visible name -> the # emulators requiring that file for the line. Three Mega CD BIOS, # one per region, share the name cd_bios.rom. self.line_readers: dict[str, dict[str, set[str]]] = {} # (line, visible name, emulator) -> the name the emulator reads. self.read_as: dict[tuple[str, str, str], str] = {} # (line, visible name) -> the file name its card shows. self.card_names: dict[tuple[str, str], str] = {} # (line, visible name) of the files most of whose readers read # them beside another one they require. self.extras: set[tuple[str, str]] = set() def require(self, name: str, emulator: str) -> None: self.written.setdefault(_visible(name), name) self.requires.setdefault(_visible(name), set()).add(emulator) def read_for(self, line: str, name: str, emulator: str, read: str, card: str) -> None: """Record that ``emulator`` requires ``name`` for ``line``, under the name ``read``, its card titled ``card``.""" visible = _visible(name) self.line_readers.setdefault(line, {}).setdefault(visible, set()).add(emulator) self.read_as.setdefault((line, visible, emulator), read) self.card_names.setdefault((line, visible), card) def absorb(self, other: _StartGroup) -> None: for region, names in other.by_region.items(): for name in names: _add_name(self.by_region.setdefault(region, []), name) for mine, theirs in ((self.plain, other.plain), (self.optional, other.optional), (self.extra, other.extra), (self.emulators, other.emulators)): for name in theirs: _add_name(mine, name) for visible, emulators in other.requires.items(): self.written.setdefault(visible, other.written[visible]) self.requires.setdefault(visible, set()).update(emulators) for visible, sha1 in other.sha1s.items(): self.sha1s.setdefault(visible, sha1) self.hle.update(other.hle) for line, readers in other.line_readers.items(): for visible, emulators in readers.items(): self.line_readers.setdefault(line, {}).setdefault(visible, set()).update(emulators) for key, value in other.read_as.items(): self.read_as.setdefault(key, value) for key, value in other.card_names.items(): self.card_names.setdefault(key, value) self.extras |= other.extras def requirers(self) -> set[str]: return set().union(*self.requires.values()) if self.requires else set() def as_read(self, visible: str, emulators: set[str], line: str | None = None) -> str: """A file as an emulator reading it instead is told to fetch it: by its card's name, with the name it reads when that differs. The line read "ares reads neo-epo.sp1" above a card titled neo-epo.bin.""" lines = [line] if line is not None else sorted(self.line_readers) for key in lines: card = self.card_names.get((key, visible)) if card is None: continue reads = {self.read_as[(key, visible, e)] for e in emulators if (key, visible, e) in self.read_as} written = self.written.get(visible, _code(card)) # A linked name (a file under Arcade) keeps its link. shown = written if _visible(written) == card else _code(card) if len(reads) == 1 and next(iter(reads)) != card: return f"{shown} (as {_code(next(iter(reads)))})" return shown return self.written.get(visible, _code(visible)) def _start_groups( files: list[dict], consumers: FileConsumers, section: tuple[str, str] | None ) -> dict[str | None, _StartGroup]: """The files of a section an emulator reads, by the system they are read for. An entry is the section's when one of its systems leads here: the DS section named five DSi files, and the NES one FBNeo's YM2608 sample, read for a system with no section of its own. An entry that names no system, in a profile of several, reads the section's own console. A file the system requires but filed under Arcade, where sets serve every machine, is named too, linked to its card: the Neo Geo BIOS. One filed under another console is that console's, whatever system an emulator files it under (BizHawk reads the Mega CD BIOS for its Genesis), and one under Other is a single emulator's, its shaders and fonts. A file the emulator ships with is no file to fetch.""" from siterender import _heading_name, _manufacturer_slug, system_name, system_section # noqa: PLC0415 here = (_manufacturer_slug(section[0]), section[1]) if section else None held = {str(f.get("sha1", "")) for f in files} primaries = [f for f in files if "/.variants/" not in f["path"]] kept: dict[str, list[tuple[str, dict, str | None]]] = {} for f in primaries: sha1 = str(f.get("sha1", "")) for emulator, entry in _reading_pairs(sha1, consumers): if entry.get("bundled"): continue system = None systems = _entry_systems(emulator, entry) if here is not None and systems: system = next((s for s in systems if system_section(s) == here), None) if system is None: continue kept.setdefault(sha1, []).append((emulator, entry, system)) candidates = list(primaries) known = (getattr(consumers, "_db", None) or {}).get("files", {}) if here is not None: for sha1, emulator, entry, system in _entries_by_section(consumers).get(here, []): record = known.get(sha1) parts = str((record or {}).get("path", "")).split("/") if (sha1 in held or not record or entry.get("bundled") or ".variants" in parts or len(parts) < 3 or parts[1] != "Arcade"): continue if sha1 not in kept: candidates.append({**record, "elsewhere": True}) kept.setdefault(sha1, []).append((emulator, entry, system)) groups: dict[str | None, _StartGroup] = {} ranked = sorted(candidates, key=lambda f: _card_rank(f, consumers)) rendered: dict[str, str] = {} for f in ranked: sha1 = str(f.get("sha1", "")) pairs = kept.get(sha1, []) name = _code(_setup_name(f, consumers, [(e, entry) for e, entry, _s in pairs])) if f.get("elsewhere"): name = f"[{name}]({markup.site_href(f'files/?sha1={sha1}')})" rendered[sha1] = name # A member read inside its archive is the archive's to name: FinalBurn # Neo reads sfix.sfix inside neogeo.zip, and the line had it read # sfix.sfix instead of neogeo.zip. archives = {_visible(name): (name, sha1) for sha1, name in reversed(rendered.items())} by_name = (getattr(consumers, "_db", None) or {}).get("indexes", {}).get("by_name", {}) def archive_named(archive: str) -> tuple[str, str]: """The archive as the line names it, linked to its card when the collection holds one, with that card's sha1.""" if archive in archives: return archives[archive] found = by_name.get(archive) or [] found = [found] if isinstance(found, str) else sorted(found) found.sort(key=lambda s: "/.variants/" in str(known.get(s, {}).get("path", ""))) if not found: return _code(archive), "" return f"[{_code(archive)}]({markup.site_href(f'files/?sha1={found[0]}')})", "" def reading(f: dict) -> Iterator[tuple[str, dict, str | None, str, str]]: """Each read of a file: the emulator, its entry, the system, the name the line writes and the sha1 of the card it names.""" sha1 = str(f.get("sha1", "")) for emulator, entry, system in kept.get(sha1, []): name, named = rendered[sha1], sha1 if entry.get("archive"): name, named = archive_named(str(entry["archive"])) yield emulator, entry, system, name, named # A file as many of whose readers require another file and read this # one only when present is an extra, not a start file: NAOMI opened on # hod2bios.zip, which 13 of the 19 emulators listing it read beside # naomi.zip for the House of the Dead 2 alone. requiring: dict[str, set[str]] = {} optional: dict[str, set[str]] = {} for f in ranked: for emulator, entry, _system, name, named in reading(f): if entry.get("hle_fallback"): continue side = requiring if entry.get("required", True) else optional side.setdefault(named or _visible(name), set()).add(emulator) needs_other = {emulator: {key for key, emulators in requiring.items() if emulator in emulators} for emulators in requiring.values() for emulator in emulators} weak = {key for key, emulators in requiring.items() if len(emulators) <= len({e for e in optional.get(key, ()) if needs_other.get(e, set()) - {key}})} for f in ranked: for emulator, entry, system, name, named in reading(f): try: key = system_name(system) if system else None except KeyError: key = None group = groups.setdefault(key, _StartGroup()) if named: group.sha1s.setdefault(_visible(name), named) if emulator not in group.emulators: group.emulators.append(emulator) if entry.get("hle_fallback"): _add_name(group.optional, name) group.hle.add(emulator) continue if not entry.get("required", True): _add_name(group.extra, name) continue group.require(name, emulator) read = str(entry.get("archive") or entry.get("name") or _visible(name)) card = str(entry["archive"]) if entry.get("archive") else str(f["name"]) regions = sorted({str(r) for r in entry.get("region") or []}) for region in regions or [""]: _add_name(group.by_region.setdefault(region, []) if region else group.plain, name) group.read_for(region, name, emulator, read, card) if (named or _visible(name)) in weak: group.extras.add((region, _visible(name))) named = [key for key in groups if key is not None] if None in groups and named: # An entry that names no system reads the section's own console. heading = _heading_name(*section) if section else None groups[heading if heading in groups else named[0]].absorb(groups.pop(None)) return groups def _start_with( files: list[dict], consumers: FileConsumers, first_below: bool = True, section: tuple[str, str] | None = None, ) -> list[str]: """The line a console section opens on: the files an emulator requires, by the region the emulator picks each for, and by system when the section holds several. ``section`` is the (manufacturer, console) the files are under: an entry read for another console's system is left to that console. Read from the profiles' system, required, hle_fallback and region fields, never typed per system. A system most of whose emulators run without its files says so, and names the few that require one. A section where no emulator reads a file gets no line. """ return _start(files, consumers, first_below, section)[0] def _start_order( files: list[dict], consumers: FileConsumers, section: tuple[str, str] | None = None, ) -> list[str]: """The sha1s of the files a section's start line names, in its order: the cards come in that order, so "first below" is true.""" return _start(files, consumers, True, section)[1] def _start( files: list[dict], consumers: FileConsumers, first_below: bool, section: tuple[str, str] | None, ) -> tuple[list[str], list[str]]: """The start line and the sha1s it names, in order.""" groups = _start_groups(files, consumers, section) sha1s: dict[str, str] = {} for group in groups.values(): for visible, sha1 in group.sha1s.items(): sha1s.setdefault(visible, sha1) held = {str(f.get("sha1", "")) for f in files} lines = _lead_file(groups) if not lines: lines = _group_lines(groups, first_below, held, sha1s, _section_label(*section) if section and section != OTHER_ARCADE else "") order: list[str] = [] for line in lines: for visible in re.findall(r"`([^`]*)`", line): sha1 = sha1s.get(visible) if sha1 and sha1 not in order: order.append(sha1) return lines, order def _group_lines( groups: dict[str | None, _StartGroup], first_below: bool, held: set[str], sha1s: dict[str, str], console: str = "", ) -> list[str]: """One block per system, as the section opens on them. ``console`` names the section in a line that picks a BIOS by region.""" keys = sorted(groups, key=lambda k: str(k).casefold()) blocks: dict[str | None, list[tuple[str, str]]] = {} for key in keys: block = _group_start(groups[key], first_below and not blocks) if block and first_below and not blocks: # "First below" holds only when the first file named is a card # of this page: the Neo Geo BIOS is filed under Arcade. first = re.search(r"`([^`]*)`", block[0][1]) if first and sha1s.get(first[1]) not in held: block = _group_start(groups[key], False) if block: blocks[key] = block if not blocks: return [] if len(blocks) == 1: return _start_lines(next(iter(blocks.values())), console) if all(lead == "*" for block in blocks.values() for lead, _text in block): # No machine of the section needs a file: the SNES said "Start with" # over two lines that each said no file is needed. lines = ["**No file needed** on any of these machines:", ""] lines += [f"- **{key}**: {text.lstrip(': ')}" for key, block in blocks.items() for _lead, text in block] return [*lines, ""] lines = ["**Start with** the files of your machine:", ""] for key, block in blocks.items(): for lead, text in block: if lead == "*": lines.append(f"- **{key}**: no file needed{text}") continue if lead == "+": # What the others read instead ends the machine's line. end = "" if lines[-1].endswith(".") else "." lines[-1] = f"{lines[-1]}{end} {text}" continue lines.append(f"- **{key}, {lead}**: {text}" if lead else f"- **{key}**: {text}") return [*lines, ""] def _display_names(emulators: set[str] | list[str]) -> list[str]: return sorted({_emulator_display(e) for e in emulators}, key=str.casefold) def _names_counted(names: list[str], most: int = 3, noun: str = "") -> str: """Names as "a, b and c", the ones past ``most`` counted: "a, b, c and 12 more". One past ``most`` is named, not counted as "1 more".""" if len(names) > most + 1: more = f"{len(names) - most} more" + (f" {noun}" if noun else "") return f"{', '.join(names[:most])} and {more}" return _said_as_list(names) def _lead_file(groups: dict[str | None, _StartGroup]) -> list[str]: """A section of several systems where most emulators require one file: that file first, and the emulators that read another instead. The Neo Geo section listed three machines, and neogeo.zip, which 26 emulators read, came after a file ares alone reads.""" if len(groups) < 2: return [] emulators: set[str] = set() requires: dict[str, set[str]] = {} written: dict[str, str] = {} holder: dict[str, _StartGroup] = {} for group in groups.values(): emulators.update(group.emulators) for visible, readers in group.requires.items(): requires.setdefault(visible, set()).update(readers) written.setdefault(visible, group.written[visible]) holder.setdefault(visible, group) if not requires: return [] top = max(requires, key=lambda v: len(requires[v])) readers = requires[top] if 2 * len(readers) <= len(emulators): return [] zipped = " Keep it zipped." if top.casefold().endswith(".zip") else "" names = _display_names(readers) said = (_names_counted(names, 2, "emulators") if len(names) > 2 else _said_as_list(names)) lines = [f"**Start with** {written[top]}.{zipped} {said} read{'s' if len(names) == 1 else ''} it.", ""] # The emulators that require other files and not this one, by the # files they require, each by its card's name. others: dict[tuple[str, ...], set[str]] = {} for emulator in emulators - readers: theirs = tuple(holder[v].as_read(v, {emulator}) for v in requires if emulator in requires[v]) if theirs: others.setdefault(theirs, set()).add(emulator) sentences = [] for files, who in sorted(others.items(), key=lambda item: (-len(item[1]), item[0])): names = _display_names(who) verb = "reads" if len(names) == 1 else "read" sentences.append(f"{_names_counted(names)} {verb} {_said_as_list(list(files))} instead.") if sentences: lines += [" ".join(sentences), ""] return lines def _group_start(group: _StartGroup, first_below: bool) -> list[tuple[str, str]]: """One system's start as (lead, text) pairs. The lead is the regions a line is for, "" the files every region needs, "*" the line of a system that runs without any file. A file is a start file when the emulators requiring it outweigh those that run without it. The Game Boy opened on dmg_boot.bin, which Emux GB alone of 13 requires, under a home page that says the Game Boy needs no BIOS.""" from region import _PARENT # noqa: PLC0415 def listed(names: list[str], below: bool = first_below) -> str: if len(names) <= START_WITH_NAMES: return ", ".join(names) # The first three by rank, the rest counted: "the 9 files" named # none, and the Neo Geo BIOS everyone reads is filed under Arcade. return f"{', '.join(names[:3])} and {_count(len(names) - 3, 'more file')}" + ( ", first below" if below else "") requirers = group.requirers() hle = group.hle - requirers running = set(group.emulators) - requirers # One that replaces the file with its own code runs, with less # accuracy, and counts half: most PlayStation cores have such a # fallback, and a reader still starts with the BIOS. if requirers and 2 * len(requirers) <= 2 * len(running - hle) + len(hle): free = _display_names(running) needing = _display_names(requirers) names = list(group.written.values()) verb = "requires" if len(needing) == 1 else "require" if all(len(readers) == len(requirers) for readers in group.requires.values()): return [("*", f" with {_names_counted(free)}. Only {_names_counted(needing)} {verb} " f"{listed(names, False)}.")] # Each requires some of the files: "bsnes, FinalBurn Neo and 3 # more require 22 files" read as 22 files for a bsnes user. these = f"these {len(names)} files" if len(names) > START_WITH_NAMES else "these files" return [("*", f" with {_names_counted(free)}. Only {_names_counted(needing)} {verb} " f"some of {these}: {listed(names, False)}.")] def instead_of(line: str, names: list[str], answered: set[str]) -> str: """What the emulators no other name answers read instead, one sentence per set of files: "FinalBurn Neo reads fdsbios.zip instead".""" readers = group.line_readers.get(line, {}) others: dict[tuple[str, ...], set[str]] = {} for emulator in sorted(set().union(*(readers.get(_visible(n), set()) for n in names)) - answered): theirs = tuple(group.as_read(_visible(n), {emulator}, line) for n in names if emulator in readers.get(_visible(n), set())) others.setdefault(theirs, set()).add(emulator) sentences = [] for files, who in sorted(others.items(), key=lambda item: (-len(item[1]), item[0])): shown = _display_names(who) verb = "reads" if len(shown) == 1 else "read" sentences.append(f"{_names_counted(shown)} {verb} {_said_as_list(list(files))} instead.") return " ".join(sentences) def split(line: str, names: list[str]) -> tuple[list[str], str, set[str]]: """A line's names as the files to start with, the sentence naming what other emulators read instead, and every emulator the line answers. Names no emulator requires beside another are other names for one slot: "fdsbios.zip, disksys.rom" read as two files to get.""" readers = group.line_readers.get(line, {}) sets = {n: readers.get(_visible(n), set()) for n in names} lead = max(names, key=lambda n: len(sets[n])) main = [n for n in names if n == lead or sets[n] <= sets[lead]] rest = [n for n in names if n not in main] # A file the lead's readers require beside it is a start file too, # unless most of its readers take it as an extra. main = [n for n in main if n == lead or (line, _visible(n)) not in group.extras] return main, instead_of(line, rest, set(sets[lead])), set().union(*sets.values()) order = _region_order() # A territory its region answers the same way adds nothing: Brazil # under Latin America. by_region = {r: names for r, names in group.by_region.items() if group.by_region.get(_PARENT.get(r, "")) != names} # Regions that need the same files share a line. regional: dict[tuple[str, ...], str] = {} answered: set[str] = set() shown: set[str] = set() for region in sorted(by_region, key=lambda r: (order.get(r, len(order)), r)): main, sentence, readers = split(region, by_region[region]) answered |= readers shown |= {_visible(n) for n in by_region[region]} text = f"{listed(main)}. {sentence}" if sentence else listed(main) same = next((rs for rs, held in regional.items() if held == text), None) if same is None: regional[(region,)] = text else: regional[(*same, region)] = regional.pop(same) out = [(", ".join(_region_word(r) for r in regions), regional[regions]) for regions in sorted(regional, key=lambda rs: (order.get(rs[0], len(order)), rs))] # A name a region line gives already is no "also", and one only # emulators no region line answers reads is an "instead": armsx1 reads # scph1001.bin where the others pick a BIOS by region. plain = [n for n in group.plain if _visible(n) not in shown] if plain and out: readers = group.line_readers.get("", {}) also = [n for n in plain if readers.get(_visible(n), set()) & answered] if also: out.append(("", listed(also))) sentence = instead_of("", [n for n in plain if n not in also], answered) if sentence: out.append(("+", sentence)) elif plain: main, sentence, _readers = split("", plain) out.append(("", listed(main))) if sentence: out.append(("+", sentence)) if not out and group.optional: out.append(("*", f"{_without_files(group.emulators)}. " f"For full accuracy: {listed(group.optional)}.")) elif not out and group.extra: # The Mega Drive's emulators read the TMSS BIOS when it is there: # the section said nothing under a home page saying none is needed. out.append(("*", f"{_without_files(group.emulators)}. " f"Optional: {listed(group.extra, False)}.")) return out def _start_lines(block: list[tuple[str, str]], console: str = "") -> list[str]: """One system's start, as the lines its section opens on. A BIOS picked by region is named with its console: "playstation bios" found the PlayStation 2 page first, its file names holding "BIOS" and the PlayStation page's text never saying "PlayStation BIOS".""" lead, text = block[0] if lead == "*": return [f"**No file needed**{text}", ""] # What the emulators no line answers read instead, after the lines. instead = " ".join(text for lead, text in block if lead == "+") block = [(lead, text) for lead, text in block if lead != "+"] if len(block) == 1 and block[0][0] == "": named, _sep, _where = block[0][1].partition(", first below") below = " They come first below." if _sep else "" return [" ".join(filter(None, [f"**Start with**: {named}.{below}", instead])), ""] bios = f"the {console} BIOS" if console else "the BIOS" lines = [f"**Start with** {bios} for the region of your games:", ""] for lead, text in block: lines.append(f"- **{lead or 'Some emulators also require'}**: {text}") if instead: lines += ["", instead] return [*lines, ""] def _cards_markdown(cards: list[Card]) -> list[str]: """Primaries, then the variants in a closed block.""" lines = [line for variant, card, _sha1 in cards if not variant for line in card] variants = [card for variant, card, _sha1 in cards if variant] if variants: # A variant gets the same card as a primary: the revision a # platform pins is the one a reader comes to find, and showed as # a name and a truncated MD5, nothing said who pins it. lines.extend(markup.details( "note", f"Variants ({len(variants)})", [line for card in variants for line in card])) return lines def _parts(cards: list[Card], budget: int, sizes: list[int] | None = None) -> list[list[Card]]: """Cards in runs of at most ``budget`` bytes of Markdown, in order. ``sizes`` gives each card's weight when it is not its own Markdown.""" parts: list[list[Card]] = [[]] size = 0 for index, card in enumerate(cards): weight = sizes[index] if sizes else sum(len(line) + 1 for line in card[1]) if parts[-1] and size + weight > budget: parts.append([]) size = 0 parts[-1].append(card) size += weight return parts def _console_heading(manufacturer: str, console: str, label: str = "") -> str: """A console's heading, its name alone. An icon on 79 of 544 headings read as missing art on the rest, set the text at two positions on one page and faded out on one theme or the other. ``label`` is the text when it differs from the folder, the id staying the folder's.""" return f"## {label or console} {{ #{console_anchor(manufacturer, console)} }}" def _romset_note(consoles: dict[str, list]) -> list[str]: """The line a page of MAME or FBNeo sets carries: an arcade player looking for the version found the answer only under Reference.""" releases = _romset_releases() if not any(str(f.get("name", "")) in releases for files in consoles.values() for f in files): return [] return ["MAME and FBNeo sets must match the emulator's version. " f"[Which version each core reads]({ROMSET_FAQ}).", ""] _NEXT_FROM_SYSTEMS = [ "[Get your files](../which-pack.md), for the setup you play on", "[When a game does not start](../wiki/troubleshooting.md)", "[Every emulator](../emulators/index.md), and the files each one reads", ] # The arcade boards the system pages give a section, by the console key # their section goes by (the board's name) -> the board's system id. Set by # arcade_board_sections() before the first system page is written. _BOARD_CONSOLES: dict[str, str] = {} # The Arcade folder of files no board claims: the folder is named after # its maker, which said nothing about what it holds. OTHER_ARCADE = ("Arcade", "Arcade") OTHER_ARCADE_LABEL = "Other arcade files" # (manufacturer, console) -> the boards whose sets left that folder's # section for the boards' own pages. The section names them. _BOARDS_FROM: dict[tuple[str, str], set[str]] = {} # System pages published under an address no page holds since, and where # their files went: the Atom and the MZ were boards, ScummVM's and SDLPAL's # files were on two pages, the Dreamcast lost its NAOMI sets. An address # redirects only while no page takes it back (sitestarlight.redirects). SYSTEM_PAGES_MOVED = { "systems/arcade-acorn-atom.md": "systems/acorn.md", "systems/arcade-sharp-mz-2000.md": "systems/sharp.md", "systems/arcade-sharp-mz-2500.md": "systems/sharp.md", "systems/arcade-miuchiz.md": "systems/arcade.md", "systems/arcade-yamaha-ym2608.md": "systems/arcade.md", "systems/other-scummvm.md": "systems/scummvm.md", "systems/other-sdlpal.md": "systems/sdlpal.md", "systems/sega-dreamcast.md": "systems/sega.md", } def arcade_board_sections( manufacturers: dict[str, dict[str, list]], installs: dict[str, dict[str, list[tuple[str, str]]]], ) -> tuple[dict[str, dict[str, list]], dict[str, dict[str, list[tuple[str, str]]]]]: """The Arcade page grouped by board, and each board's install commands. The Arcade sections were the folders of bios/Arcade/: NAOMI, PGM and ZN-1 had no section, and their names stayed plain on 568 emulator pages. Each archive goes to the board siterender.arcade_board() reads from the profiles, never from its folder, and a folder named after a board (Namco System 246) joins it, whatever maker's folder holds the copy. A set a reviewed machine owns (siterender.archive_machine) goes to that machine's section. Files neither claims stay in their folders. Each board is recorded (siterender.record_board_section) before any page links one, and a system's install commands follow its link to the board's section. Call after reset_console_pages().""" from siterender import ( # noqa: PLC0415 arcade_board, arcade_boards, archive_machine, record_board_section, record_card_move, system_section, ) _BOARD_CONSOLES.clear() _BOARDS_FROM.clear() arcade = manufacturers.get("Arcade") if not arcade: return manufacturers, installs names = arcade_boards() by_name = {name.casefold(): board for board, name in names.items()} boards: dict[str, list] = {} folders: dict[str, list] = {} seen: dict[str, set[str]] = {} # The sets a reviewed machine owns go to its section: neogeo.zip sat on # five Arcade pages, the Atom's and the MZ-2000's sets on board pages. machines: dict[tuple[str, str], list] = {} def file_on_board(board: str, f: dict) -> None: # One card per content: a copy under Arcade and one under MAME # are the same file. if str(f.get("sha1", "")) in seen.setdefault(board, set()): return seen[board].add(str(f.get("sha1", ""))) boards.setdefault(board, []).append(f) for console in sorted(arcade): joined = by_name.get(console.casefold()) for f in arcade[console]: sha1 = str(f.get("sha1", "")) board = joined or arcade_board(sha1) machine = None if board else archive_machine(sha1) if board is not None: file_on_board(board, f) elif machine is not None: machines.setdefault(machine, []).append(f) record_card_move("Arcade", console, sha1, *machine) else: folders.setdefault(console, []).append(f) # A board's set filed under a machine's folder is on the board's page # too, whatever the folder: naomi.zip under the Dreamcast was the copy # five setups list, and the NAOMI page did not name it. makers: dict[str, dict[str, list]] = {} for mfr, consoles in manufacturers.items(): if mfr == "Arcade": continue for console, files in consoles.items(): kept = [] for f in files: sha1 = str(f.get("sha1", "")) board = arcade_board(sha1) machine = None if board else archive_machine(sha1) if board is not None: file_on_board(board, f) _BOARDS_FROM.setdefault((mfr, console), set()).add(board) elif machine is not None and machine != (mfr, console): # The Atom's own set sat with the BBC Micro. machines.setdefault(machine, []).append(f) record_card_move(mfr, console, sha1, *machine) else: kept.append(f) if len(kept) != len(files): makers.setdefault(mfr, dict(consoles))[console] = kept for (mfr, console), files in sorted(machines.items()): consoles = makers.setdefault(mfr, dict(manufacturers.get(mfr, {}))) held = {str(f.get("sha1", "")) for f in consoles.get(console, [])} consoles[console] = [*consoles.get(console, []), *(f for f in files if str(f.get("sha1", "")) not in held)] for board in sorted(boards): record_board_section(board) _BOARD_CONSOLES[names[board]] = board regrouped = {**folders, **{names[board]: files for board, files in boards.items()}} makers = {mfr: {console: files for console, files in consoles.items() if files} for mfr, consoles in makers.items()} moved: dict[str, dict[str, list[tuple[str, str]]]] = {} for slug, consoles in installs.items(): for console, pairs in consoles.items(): for platform, sys_id in pairs: section = system_section(sys_id) if section and section[0] == "arcade" and section[1] in _BOARD_CONSOLES: slug_to, console_to = section else: slug_to, console_to = slug, console moved.setdefault(slug_to, {}).setdefault(console_to, []).append((platform, sys_id)) return {**manufacturers, **makers, "Arcade": regrouped}, moved def _boards_line(manufacturer: str, console: str) -> list[str]: """The line of a section whose board sets went to the boards' pages: the Dreamcast folder holds NAOMI's and Atomiswave's.""" from siterender import arcade_boards # noqa: PLC0415 boards = _BOARDS_FROM.get((manufacturer, console)) if not boards: return [] names = arcade_boards() links = [f"[{names[board]}]({console_href('arcade', names[board]).removeprefix('systems/')})" for board in sorted(boards, key=lambda b: names[b].casefold())] return [f"**Arcade board BIOS** on the boards' own pages: {', '.join(links)}.", ""] def _section_label(manufacturer: str, console: str) -> str: """The heading a section shows.""" if (manufacturer, console) == OTHER_ARCADE: return OTHER_ARCADE_LABEL return SECTION_LABELS.get((manufacturer, console), console) def _part_name(manufacturer: str, console: str) -> str: """What a part's title calls the files: "Arcade files a to b".""" if (manufacturer, console) == OTHER_ARCADE: return "Arcade" return SECTION_LABELS.get((manufacturer, console), console) def _system_front_matter( parent: tuple[str, str] | None = None, prev: tuple[str, str] | None = None, following: tuple[str, str] | None = None, weight: float | None = None, consoles: list[str] | None = None, ) -> list[str]: """The front matter of a system page: the page it belongs to, the parts before and after it (label, site path), its search weight and the consoles it holds, which the search reads as a field.""" lines = _parent_front_matter(parent) for key, link in (("prev", prev), ("next", following)): if link: lines += [f"{key}:", f" label: {json.dumps(link[0])}", f" link: {json.dumps(link[1])}"] if weight is not None: lines.append(f"searchWeight: {weight}") if consoles: lines += ["consoles:", *(f" - {json.dumps(name)}" for name in consoles)] return ["---", *lines, "---", ""] if lines else [] def _part_weight() -> float: """The search weight of a part page: its cards keep their own weight.""" return SEARCH_WEIGHT.get("part", DEMOTED_WEIGHT) @functools.lru_cache(maxsize=None) def _manifest_systems(platform: str) -> frozenset[str]: """The system ids a platform's install manifest records: --system refuses any other.""" manifest = _REPO_ROOT / "install" / f"{platform}.json" if not manifest.is_file(): return frozenset() document = json.loads(manifest.read_text(encoding="utf-8")) return frozenset( str(system) for entry in (document.get("files") or []) + (document.get("omitted_files") or []) for system in entry.get("systems") or [] ) def _install_pairs( declared: list[tuple[str, str]], lead: str | None, consumers: FileConsumers, ) -> list[tuple[str, str]]: """A section's install commands: the setups whose system leads here, then each setup that files the section's first file under a system of its own (RetroArch files neogeo.zip under arcade), with that id.""" pairs = list(declared) have = {platform for platform, _system in pairs} for platform, systems in sorted(getattr(consumers, "platform_systems", {}) .get(lead or "", {}).items()): if platform in have: continue recorded = [s for s in sorted(systems) if s in _manifest_systems(platform)] if recorded: pairs.append((platform, recorded[0])) have.add(platform) return sorted(pairs, key=lambda pair: (_platform_display(pair[0]).casefold(), pair[1])) def system_pages( manufacturer: str, consoles: dict[str, list], consumers: FileConsumers, installs: dict[str, list[tuple[str, str]]] | None = None, budget: int = SYSTEM_PAGE_BUDGET, ) -> dict[str, tuple[str, dict[str, list]]]: """A manufacturer's pages by stem under systems/: one page, or a summary and a page per console when the whole would pass ``budget``. Each page comes with the files whose cards it holds, by console. Every console keeps its heading on the summary, so a link to systems//# still lands, and site.js sends a card anchor the page no longer holds to the card's new place. The pages each card and console went to are recorded for every other link (siterender.record_console_pages). An arcade board has a page of its own, systems/arcade-/, under a heading of that id. """ from siterender import board_slug # noqa: PLC0415 slug = _slug(manufacturer) installs = installs or {} boards = {console: _BOARD_CONSOLES[console] for console in consoles if manufacturer == "Arcade" and console in _BOARD_CONSOLES} json_next = (f"[Every hash of these files, as JSON](../api/v1/systems/" f"{urllib.parse.quote(slug)}.json)") # Boards first on the Arcade page, then the folders of sets for many # machines. Such a folder starts nowhere: its line named konamigv.zip, # sys573.zip and 97 more, files of boards with lines of their own. ordered = sorted(consoles, key=lambda c: (bool(boards) and c not in boards, c)) def start_of(console: str, first_below: bool = True) -> list[str]: if boards and console not in boards: return [] return [*_start_with(consoles[console], consumers, first_below, (manufacturer, console)), *_boards_line(manufacturer, console)] starts = {console: [] if boards and console not in boards else _start_order(consoles[console], consumers, (manufacturer, console)) for console in consoles} def cards_of(console: str, anchors: set[str], by_name: bool = False) -> list[Card]: return _console_cards(manufacturer, consoles[console], consumers, anchors, by_name, starts[console]) def install_block(console: str) -> list[str]: lead = starts[console][0] if starts[console] else None return _console_install_block( _section_label(manufacturer, console), _install_pairs(installs.get(console, []), lead, consumers)) def all_sections() -> dict[str, list[Card]]: anchors: set[str] = set() return {console: cards_of(console, anchors) for console in ordered} # One line, then the legend closed: four paragraphs said the same on # every maker's page before its first console. intro = f"Every BIOS file for {manufacturer} machines, one card per file." legend = search_excluded(markup.details("note", "How to read a card", [ "A card gives the file's fingerprints, the emulators that read it, the setups " "that list it and where to download it. A file can have several versions, one " "per region or revision: the main one comes first, the others under " "\"Variants\". To install the files of one console, open its \"Install\" box.", "", "**On the setup's list** names the setups whose own BIOS list has the file, by " "the name each gives it. **Packed for its emulators by** names the setups whose " "pack adds it because one of their emulators reads it.", ])) # The consoles a page answers for, a search field: machines only. An # emulator's folder under Other put that page first for "dolphin", and # a console with a page of its own is that page's to answer for. machines = [console for console in ordered if manufacturer not in ("Other", *GAME_DATA_TOPS) and (manufacturer != "Arcade" or console in boards)] def consoles_front(kept: set[str] | None = None) -> list[str]: return _system_front_matter(consoles=[ _section_label(manufacturer, c) for c in machines if kept is None or c in kept]) front = consoles_front() def whole_page(sections: dict[str, list[Card]]) -> str: lines = [ *front, f"# {manufacturer}", "", intro, "", *legend, *_romset_note(consoles), *_icon_symbol(), ] for console, cards in sections.items(): lines.append(_console_heading(manufacturer, console, _section_label(manufacturer, console))) lines.append("") lines.extend(start_of(console)) lines.extend(install_block(console)) lines.extend(_cards_markdown(cards)) lines.append("") lines.extend(_next_block([*_NEXT_FROM_SYSTEMS, json_next])) lines.extend(["", _date_line()]) return "\n".join(lines) + "\n" def card_weight(card: list[str]) -> int: return sum(len(line) + 1 for line in card) # Sizes are those of the MkDocs Markdown the budget was set on, so the # pages split where they always have and no card changes address. with markup.using("mkdocs"): measured = all_sections() measured_text = whole_page(measured) weights = { console: sum(card_weight(card) for _variant, card, _sha1 in cards) for console, cards in measured.items() } # An install box is a few lines of Markdown and about nine times that # once highlighted: ten of them were 295 KB of the 583 KB Nintendo # page, which the card weights never counted. A page is also cut when # its estimate as it renders passes the summary's. with markup.using("mkdocs"): boxes = {console: sum(len(line) + 1 for line in install_block(console)) for console in consoles} rendered = {console: CARD_HTML_RATIO * weights[console] + INSTALL_HTML_RATIO * boxes[console] for console in weights} if (not boards and len(measured_text) <= budget and sum(rendered.values()) <= SYSTEM_SUMMARY_HTML): text = whole_page(all_sections()) if markup.starlight() else measured_text return {slug: (text, consoles)} # Every board has a page of its own. Then the heaviest sections move # to pages of their own until what stays on the manufacturer's page is # half the budget: a section of one file keeps its place, and a reader # does not open 91 pages to browse Other. moved: set[str] = set(boards) for console in sorted(weights, key=lambda c: (-weights[c], c)): if sum(w for c, w in weights.items() if c not in moved) <= budget // 2: break moved.add(console) # Then until what stays holds its estimate as it renders. for console in sorted(rendered, key=lambda c: (-rendered[c], c)): if sum(w for c, w in rendered.items() if c not in moved) <= SYSTEM_SUMMARY_HTML: break moved.add(console) pages: dict[str, tuple[str, dict[str, list]]] = {} summary = [ *consoles_front(set(consoles) - moved), f"# {manufacturer}", "", intro, "", *legend, *_romset_note(consoles), *_icon_symbol(), ] summary_anchors: set[str] = set() parent = (manufacturer, f"/systems/{slug}/") for console in ordered: label = _section_label(manufacturer, console) summary.append(_console_heading(manufacturer, console, label)) summary.append("") if console not in moved: summary.extend(start_of(console)) summary.extend(install_block(console)) summary.extend(_cards_markdown(cards_of(console, summary_anchors))) summary.append("") continue # Cards are rendered again for their own page: the anchors a page # holds are its own. def cut(by_name: bool) -> list[list[Card]]: cards = cards_of(console, set(), by_name) with markup.using("mkdocs"): sizes = [card_weight(card) for _variant, card, _sha1 in cards_of( console, set(), by_name)] parts = _parts(cards, budget, sizes) if len(parts) < 2: return parts # Even parts: cut at budget, the PlayStation 2 was 137 cards # and 4. The same number of parts, each near the mean. share = -(-sum(sizes) // len(parts)) even = _parts(cards, share, sizes) while len(even) > len(parts): share += max(1, share // 50) even = _parts(cards, share, sizes) return even # One page keeps the required files first. Parts are named by their # first and last file, so a console cut in parts goes by name, the # files the start line names first. parts = cut(False) if len(parts) > 1: parts = cut(True) board = boards.get(console) base = (f"arcade-{board_slug(board)}" if board else f"{slug}-{_slugify_anchor(console) or 'files'}") stems = [base if index == 0 else f"{base}-{index + 1}" for index in range(len(parts))] clash = set(stems) & set(pages) if clash: raise RuntimeError(f"two system sections take one page: {sorted(clash)}") record_console_pages(slug, console, stems, [ [sha1 for _variant, _card, sha1 in part] for part in parts ]) lead = set(starts[console]) # A part by the files it holds: "Arcade: Arcade, part 3 of 5" said # nothing a search result could use. A folder under Other goes by # its emulator's name: "mame files" named the folder. named = _folder_emulator(console)[0] if manufacturer == "Other" else None titles = ([f"{manufacturer}: {named or label}"] if len(stems) == 1 else [ "{} files {} to {}".format(named or _part_name(manufacturer, console), *_part_ends(part, lead)) for part in parts]) count = _count(len(consoles[console]), "file") summary.extend(start_of(console, False)) if len(stems) == 1: own = "its" if len(consoles[console]) == 1 else "their" summary.append(f"{count}, on [{own} own page]({stems[0]}.md).") else: summary.append(f"{count}, on {len(stems)} pages by file name:") summary.append("") for index, (stem, part) in enumerate(zip(stems, parts)): summary.append(f"- [Part {index + 1}]({stem}.md): {_part_range(part, lead)}") summary.append("") for index, (stem, part) in enumerate(zip(stems, parts)): neighbours = [(titles[i], f"/systems/{stems[i]}/") for i in (index - 1, index + 1) if 0 <= i < len(stems)] lines = [ *_system_front_matter( parent, prev=neighbours[0] if index > 0 else None, following=neighbours[-1] if index + 1 < len(stems) else None, weight=_part_weight() if len(stems) > 1 else None, consoles=[label] if index == 0 and console in machines else None), # The crumbs end on the maker's page: no line says it again. f"# {titles[index]}", "", *_icon_symbol(), ] if len(stems) > 1: others = ", ".join( f"[part {i + 1}]({other}.md)" for i, other in enumerate(stems) if i != index) lines.extend([f"This part holds {_part_range(part, lead)}. The others: {others}.", ""]) # A board's page is its one section: its title names it, and a # heading of the same name said it a third time. Its cards come # under "Files", as every page of one console's. if index == 0: lines.extend(start_of(console, len(stems) == 1)) lines.extend(legend) lines.extend(_romset_note({console: consoles[console]})) lines.extend(install_block(console)) lines.extend(["## Files", ""]) lines.extend(_cards_markdown(part)) lines.append("") lines.extend(_next_block([f"[{manufacturer}]({slug}.md), every section"] + _NEXT_FROM_SYSTEMS + [json_next])) lines.extend(["", _date_line()]) held = {sha1 for _variant, _card, sha1 in part} pages[stem] = ("\n".join(lines) + "\n", {console: [ f for f in consoles[console] if str(f.get("sha1", "")) in held]}) summary.extend(_next_block([*_NEXT_FROM_SYSTEMS, json_next])) summary.extend(["", _date_line()]) kept = {console: files for console, files in consoles.items() if console not in moved} return {slug: ("\n".join(summary) + "\n", kept), **pages} def _part_ends(part: list[Card], lead: set[str] | frozenset[str] = frozenset()) -> tuple[str, str]: """The first and last file names of a part, as the cards show them. The files a start line puts first are left out when the part holds others: the range is the run by name that follows them.""" named = [card for card in part if card[2] not in lead] or part names = [] for _variant, card, _sha1 in named: # An HTML heading on the site, a Markdown one in the size yardstick. match = re.search(r"`([^`]*)`|(.*?)", card[2]) names.append(match[1] if match[1] is not None else html.unescape(match[2])) return names[0], names[-1] def _part_range(part: list[Card], lead: set[str] | frozenset[str] = frozenset()) -> str: """The first and last file names of a part, with its count.""" first, last = _part_ends(part, lead) if len(part) == 1: return f"`{first}`" return f"{_count(len(part), 'file')}, `{first}` to `{last}`" def generate_system_page( manufacturer: str, consoles: dict[str, list], consumers: FileConsumers, installs: dict[str, list[tuple[str, str]]] | None = None, ) -> str: """One manufacturer's page, a section per console. ``installs`` gives, per console, the (platform, system id) pairs whose install command the section offers. A page over the budget is the summary system_pages writes, its consoles on pages of their own. """ return system_pages(manufacturer, consoles, consumers, installs)[_slug(manufacturer)][0] # Emulator pages def _named_system_link(sys_id: str, prefix: str = "") -> str: """A system by its display name, linked where _system_link links it. The emulator pages printed the id ("sony-playstation") where every other page says "Sony PlayStation". The id stays in the address only. """ from siterender import system_section # noqa: PLC0415 name = system_name(sys_id).replace("[", r"\[").replace("]", r"\]") section = system_section(sys_id) if not section: return name slug, console = section return f"[{name}]({console_href(slug, console, prefix)})" def _distinct_spellings(names: list) -> list[str]: """Names as a reader tells them apart: one per spelling, ignoring case. DuckStation's cores list `Duckstation` for EmuELEC, whose launch name is spelled so, and the page read "duckstation, Duckstation". The profile keeps both, the core lookup being case-sensitive. """ seen: dict[str, str] = {} for name in map(str, names): seen.setdefault(name.casefold(), name) return list(seen.values()) # Region slugs whose words are not the slug title-cased. _REGION_WORDS = { "uk": "UK", "asia-ntsc": "Asia (NTSC)", "asia-pal": "Asia (PAL)", "world": "every region", } def _region_words(value: object, most: int = 0) -> str: """A region: value in words. ``most`` caps the regions named, the rest counted: one Saturn BIOS serves seven.""" raw = value if isinstance(value, list) else [value] if value else [] words = [_REGION_WORDS.get(str(r), str(r).replace("-", " ").title()) for r in raw] if most and len(words) > most: words = words[:most] + [f"{len(words) - most} more"] return _said_as_list(words) _CATEGORY_WORDS = {"bios": "BIOS", "game_data": "game data", "bios_zip": "BIOS ZIP"} INITIAL_DIGITS = "0 to 9" def emulator_initial(display: str) -> str: """The letter an emulator is filed under, in the navigation and on the index alike. A name that does not open on a letter goes under "0 to 9".""" initial = display[:1].upper() return initial if "A" <= initial <= "Z" else INITIAL_DIGITS def generate_emulators_index(profiles: dict) -> str: unique = { k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test") } aliases = {k: v for k, v in profiles.items() if v.get("type") == "alias"} by_class: dict[str, list[tuple[str, dict]]] = {} for name in sorted(unique.keys()): p = unique[name] cls = p.get("core_classification", "other") by_class.setdefault(cls, []).append((name, p)) total_files = sum(len(p.get("files", [])) for p in unique.values()) lines = [ "# Emulators", "", "Each page is one emulator: the files it reads, whether it starts games " "without them, and the line of its source code that says so. The " "kind says how a core relates to the emulator it comes from, which " "tells how far its own files can differ.", "", f"**{len(unique):,}** emulator profiles, **{total_files:,}** files total, **{len(aliases):,}** aliases.", "", "| Kind | Count | Description |", "|------|-------|-------------|", ] cls_desc = { "official_port": "The same authors keep the app and the RetroArch core", "community_fork": "Ported to RetroArch by someone else", "pure_libretro": "Built for RetroArch, no app of its own", "game_engine": "Game engine reimplementation", "enhanced_fork": "Fork with added features", "frozen_snapshot": "Frozen at an old version", "embedded_hle": "Needs no BIOS file: the emulator carries its own", "launcher": "Launches an external emulator", "other": "Unclassified", } cls_order = [ "official_port", "community_fork", "pure_libretro", "game_engine", "enhanced_fork", "frozen_snapshot", "embedded_hle", "launcher", "other", ] for cls in cls_order: entries = by_class.get(cls, []) if not entries: continue label = CLS_LABELS.get(cls, cls) desc = cls_desc.get(cls, "") lines.append(f"| {label} | {len(entries)} | {desc} |") lines.append("") # One section per initial, the groups the navigation draws: its letter # links land on the section of the same name. by_initial: dict[str, list[tuple[str, dict]]] = {} for name, p in unique.items(): by_initial.setdefault(emulator_initial(str(p.get("emulator", name))), []).append((name, p)) for initial in sorted(by_initial, key=lambda g: (g != INITIAL_DIGITS, g)): lines.extend([ f"## {initial}", "", "| Emulator | Kind | Systems | Files |", "|--------|------|---------|-------|", ]) entries = sorted(by_initial[initial], key=lambda e: str(e[1].get("emulator", e[0])).casefold()) for name, p in entries: emu_name = p.get("emulator", name) cls = p.get("core_classification", "other") systems = list(dict.fromkeys( _named_system_link(s, "../") for s in p.get("systems") or [])) files = p.get("files", []) sys_str = ", ".join(systems[:3]) if len(systems) > 3: sys_str += f" and {len(systems) - 3} more" file_count = len(files) file_str = f"{file_count:,}" if file_count else "-" lines.append(f"| [{emu_name}]({name}.md) | {CLS_LABEL_ONE.get(cls, cls)} | {sys_str} | {file_str} |") lines.append("") if aliases: lines.extend(["## Aliases", ""]) lines.append("| Core | Points to |") lines.append("|------|-----------|") for name in sorted(aliases.keys()): parent = aliases[name].get( "alias_of", aliases[name].get("bios_identical_to", "unknown") ) lines.append(f"| {name} | [{parent}]({parent}.md) |") lines.append("") return "\n".join(lines) + "\n" def comes_with_app(f: dict) -> bool: """The emulator installs the file in every build the profile documents. ``bundled`` narrowed by ``mode`` is installed by one build only, and the other build still reads it from the user's folder. """ return f.get("bundled") is True and not f.get("mode") def _file_badges(f: dict, in_repo: bool, for_system: str = "") -> list[str]: """The status chips shown beside a file name. Fourteen independent optional fields, each contributing at most one chip. Kept apart from the rest of the row so the field-by-field rendering stays readable. ``for_system`` names the system a required file serves, on a profile that runs more than one. """ required = f.get("required", False) hle = f.get("hle_fallback", False) mode = f.get("mode", "") category = f.get("category", "") region = f.get("region", "") storage = f.get("storage", "") bundled = f.get("bundled", False) embedded = f.get("embedded", False) has_builtin = f.get("has_builtin", False) archive = f.get("archive", "") ftype = f.get("type", "") badges = [] shipped = comes_with_app(f) if shipped: # NetherSX2's three interface fonts sat beside the BIOS as # "required" and "missing": the app ships them, nobody fetches them. badges.append( 'comes with the app' ) elif required: needed = f"required for {html.escape(for_system)}" if for_system else "required" badges.append( f'{needed}' ) else: badges.append( 'optional' ) # Whether the collection holds a file the app ships says nothing to # the reader: no status chip. if shipped: status = "" elif in_repo: status = 'in repo' elif f.get("unsourceable"): status = 'unsourceable' else: status = 'missing' if status: badges.append(status) if hle: badges.append( 'HLE fallback' ) if mode: badges.append( f'{mode}' ) if category and category != "bios": badges.append( f'{_CATEGORY_WORDS.get(category, category)}' ) if region: # Labelled: "optional in repo japan" read as one run-on phrase. badges.append( f'Region: {_region_words(region)}' ) if storage and storage != "embedded": badges.append( f'{storage}' ) if bundled and not shipped: # Said in words: the pack leaves the file out because the emulator, # in the build the entry names, already installs it. build = f" {f['mode']}" if f.get("mode") in ("standalone", "libretro") else "" badges.append( f'installed with the{build} emulator' ) if embedded: badges.append( 'embedded' ) if has_builtin: badges.append( 'built-in fallback' ) if archive: badges.append( f'in {archive}' ) if ftype and ftype != "bios": badges.append( f'{ftype}' ) return badges def _first_hash_value(entry: dict) -> str: """The first declared hash of an entry, lowercase, for a stable anchor.""" for field in ("sha1", "md5", "crc32", "sha256"): value = entry.get(field) value = value[0] if isinstance(value, list) and value else value if value: return str(value).lower().split(",")[0].strip().removeprefix("0x") return "" _LOAD_FROM = { "system_dir": "the system (BIOS) directory", "save_dir": "the save directory, not the BIOS directory", "content_dir": "the folder of the loaded content, not the BIOS directory", "working_directory": "the working directory, not the BIOS directory", "data_dir": "the emulator's data directory, not the BIOS directory", "hdd_image": "inside the emulated hard disk image, not the BIOS directory", "internal_storage": "the app's internal storage, not the BIOS directory", } def _by_build(value: object, words: Callable[[str], str]) -> str: """A field keyed by build, said per build, or a plain value said once.""" if isinstance(value, dict): return ", ".join(f"{words(str(v))} ({mode} build)" for mode, v in value.items() if v) return words(str(value)) if value else "" def _load_from_text(value: object) -> str: """Where the code reads a file, in words. The card showed the Path alone: Dolphin's libretro core reads gba_bios.bin from its save directory, and a reader put it under system/. """ if value in (None, "", "system_dir"): return "" return _by_build(value, lambda v: _LOAD_FROM.get(v, _code(v))) def _found_by_text(value: object) -> str: """How the code finds a file: by the name it opens, or by content.""" words = { "name": "file name (the code opens this name)", "content": "content (the code scans the folder and identifies the file " "by its hash, any file name works)", } return _by_build(value, lambda v: words.get(v, _code(v))) def _on_mismatch_lines(value: object) -> list[str]: """What the code does when a declared check fails, where it is not reject.""" said = {"warn": "warns and runs with the file", "reject": "rejects the file"} if value == "warn": return [f"On a failed check: {said['warn']}"] if isinstance(value, dict): return [f"On a failed {check} check: {said.get(str(how), _code(str(how)))}" for check, how in value.items()] return [] def _build_rows(profile: dict) -> list[str]: """Meta card rows for who ships the build, how its files are found, why its revision is held and where its releases are announced.""" rows = [] built_by = profile.get("built_by") or [] if built_by: names = ", ".join(f"[{p}](../platforms/{p}.md)" for p in map(str, built_by)) rows.append(f"| Built by | {names}, under its core name. The libretro " "buildbot does not publish this build. |") found = _found_by_text(profile.get("found_by")) if found: rows.append(f"| Files found by | {found} |") frozen = profile.get("pin_frozen") if frozen: held = _escape_tags(' '.join(str(frozen).split())).replace('|', r'\|') rows.append(f"| Revision held | {held} |") announced = profile.get("version_source") if isinstance(announced, dict) and announced.get("url"): url = str(announced["url"]) row = f"| Releases announced at | [{url}]({url})" if announced.get("no_genuine_copy"): row += (f" (version {_escape_tags(str(announced['no_genuine_copy']))} " "announced there, found only as rebuilt copies)") rows.append(row + " |") return rows def _render_emulator_file( f: dict, profile: dict, consumers: FileConsumers | None, files: list, _file_available: Callable[[dict], bool], anchor: str = "", owner: str = "", alias: str = "", ) -> list[str]: """Render one row of an emulator profile's file table. `anchor` is the card's id on the page, so one file can be cited by URL, and `alias` a second id it answers to. Split out of generate_emulator_page, where it was a 228-line loop body carrying most of that function's branching: thirty-odd optional fields, each with its own badge, hash line or note. """ lines: list[str] = [] fname = f.get("name", "") in_repo = _file_available(f) source_ref = f.get("source_ref", "") aliases = f.get("aliases", []) validation = f.get("validation", []) size = f.get("size") fnote = f.get("note", f.get("notes", "")) fmd5 = f.get("md5", "") fsha1 = f.get("sha1", "") fcrc32 = f.get("crc32", "") fsha256 = f.get("sha256", "") fadler32 = f.get("known_hash_adler32", "") fmin = f.get("min_size") fmax = f.get("max_size") desc = f.get("description", "") fpath = f.get("path", "") fsystem = f.get("system", "") priority = f.get("priority") fast_boot = f.get("fast_boot") contents = f.get("contents", []) config_key = f.get("config_key", "") dest = f.get("dest", f.get("destination", "")) fpattern = f.get("pattern", "") region_check = f.get("region_check") size_note = f.get("size_note", "") size_range = f.get("size_range", "") # melonDS reads DS and DSi files: "required" alone sat on the DSi NAND # above the DS files, for an emulator most people run for DS games. for_system = (system_name(str(fsystem)) if fsystem and len(profile.get("systems") or []) > 1 else "") badges = _file_badges(f, in_repo, for_system) badge_str = " ".join(badges) lines.append(markup.block_open("div", 'class="rb-file-entry"')) lines.append("") marker = f'' if alias else "" label = _code(fname) if f is any_name_bios(profile): # Found by content: the name is an example, any image of the system works. system = system_name((profile.get("systems") or [""])[0]) label = (f"a {system} BIOS (example: {_code(fname)})" if _named_example(str(fname)) else f"a {system} BIOS, any file name") lines.append(_card_title(f"{marker}{label} {badge_str}", anchor, weight=5)) lines.append("") if desc: lines.extend([_escape_tags(str(desc)), ""]) body_start = len(lines) details = [] if fpath and fpath != fname: details.append(f"Path: `{fpath}`") standalone_path = f.get("standalone_path", "") if standalone_path and standalone_path != fpath: # The standalone build reads it elsewhere: MesenCE's dsp1.rom goes # under Firmware/, and a page naming only the core's path sent its # users to the wrong folder. details.append(f"Standalone path: `{standalone_path}`") read_from = _load_from_text(f.get("load_from")) if read_from: details.append(f"Read from: {read_from}") found = _found_by_text(f.get("found_by")) if found: details.append(f"Found by: {found}") if fsystem: details.append(f"System: {_named_system_link(fsystem, '../')}") if size: if isinstance(size, list): size_str = " / ".join(_fmt_size(s) for s in size) else: size_str = _fmt_size(size) if fmin or fmax: bounds = [] if fmin: bounds.append(f"min {_fmt_size(fmin)}") if fmax: bounds.append(f"max {_fmt_size(fmax)}") size_str += f" ({', '.join(bounds)})" details.append(f"Size: {size_str}") elif fmin or fmax: bounds = [] if fmin: bounds.append(f"min {_fmt_size(fmin)}") if fmax: bounds.append(f"max {_fmt_size(fmax)}") details.append(f"Size: {', '.join(bounds)}") if fsha1: details.append(f"SHA1: {_hash_spans(fsha1)}") if fmd5: details.append(f"MD5: {_hash_spans(fmd5)}") if fcrc32: crcs = fcrc32 if isinstance(fcrc32, list) else [fcrc32] details.append("CRC32: " + ", ".join(f"`{crc}`" for crc in crcs)) if fsha256: details.append(f"SHA256: {_hash_spans(fsha256)}") if fadler32: # Printed as the downloads spell a hash, one per code span: the list # repr read `['0x66f334fe', ...]`. Dolphin hashes its DSP ROMs over # byte-swapped 16-bit words, so the raw Adler-32 the downloads carry # for the same file differs, and the card says which one it lists. values = fadler32 if isinstance(fadler32, list) else [fadler32] shown = ", ".join(_code(str(v).lower().removeprefix("0x")) for v in values) if f.get("adler32_byteswap"): details.append( f"Adler32 over the file's 16-bit words byte-swapped: {shown} " "(the Adler-32 in the downloads is the file's own and differs)" ) else: details.append(f"Adler32: {shown}") if aliases: details.append(f"Aliases: {', '.join(f'`{a}`' for a in aliases)}") if priority is not None: details.append(f"Priority: {priority}") if fast_boot is not None: details.append(f"Fast boot: {'yes' if fast_boot else 'no'}") if validation: if isinstance(validation, list): details.append(f"Validation: {', '.join(validation)}") elif isinstance(validation, dict): for scope, checks in validation.items(): details.append(f"Validation ({scope}): {', '.join(checks)}") details.extend(_on_mismatch_lines(f.get("on_mismatch"))) if f.get("installed_on"): details.append("Installed by the platform's own package on: " + ", ".join(sorted(f["installed_on"]))) if f.get("not_bundled_on"): details.append("Not in the platform's package, so the pack brings it, on: " + ", ".join(sorted(f["not_bundled_on"]))) if f.get("unread_on"): details.append("Never opened on these platforms, whose launcher names other files: " + ", ".join(sorted(f["unread_on"]))) if source_ref: asker = requested_by(profile, source_ref) if asker: # A profile reading a libretro core and the original emulator # cited both trees alike: readers could not tell which code asks # for the file. details.append(f"Asked for by: {asker}") details.append( f"Source: {_source_ref_markdown(profile, source_ref)}" ) if consumers and f.get("type") == "directory": details.extend(_directory_copies(f, consumers)) if consumers: sha1 = consumers.sha1_of(f, f.get("path") or "", owner=owner) held = consumers.path_of(sha1) if sha1 and in_repo else "" parts = held.split("/") if len(parts) >= 3 and parts[0] == "bios": # Which copy answers this entry, linked to its collection card # (provenance, every hash, download): an entry declaring no hash # said "in repo" without naming the file. page = card_page(held, sha1) details.append( f"Collection copy: [{_code(held)}]" f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" ) elif in_repo and not any(d.startswith("Collection copies:") for d in details): copy = held_copy(f) if copy: details.append(_held_copy_line(copy, consumers)) plat_links = _platform_links(sha1, consumers) if sha1 else [] if plat_links: details.append(f"Platforms: {', '.join(plat_links)}") if dest and dest != fname and dest != fpath: details.append(f"Destination: `{dest}`") if config_key: details.append(f"Config key: `{config_key}`") if fpattern: details.append(f"Pattern: `{fpattern}`") if region_check is not None: details.append(f"Region check: {'yes' if region_check else 'no'}") if size_note: details.append(f"Size note: {size_note}") if size_range: details.append(f"Size range: {size_range}") if details: for d in details: lines.append(f"- {d}") if fnote: lines.append(f"- {_escape_tags(link_prose_citations(profile, str(fnote)))}") if not in_repo and f.get("unsourceable"): # The badge alone sent readers after a dump that cannot exist, and the # reason was only on the gaps page, which the card did not link. lines.append( f"- Not collectable: {_escape_tags(str(f['unsourceable']))} " "([acknowledged gaps](../gaps.md#acknowledged-gaps))" ) if contents: # Every member with the hashes the emulator checks: MAME and FBNeo # verify the members, never the archive, so a member's crc32 is the # value a reader compares a set against. lines.append(f"- Contents ({len(contents)} members):") for c in contents: if isinstance(c, dict): parts = [] if c.get("description"): parts.append(_escape_tags(str(c["description"]))) if c.get("size"): parts.append(_fmt_size(c["size"])) if c.get("crc32"): parts.append(f"CRC32 `{c['crc32']}`") if c.get("sha1"): parts.append(f"SHA1 `{c['sha1']}`") extra = f": {', '.join(parts)}" if parts else "" lines.append(f" - `{c.get('name', '')}`{extra}") else: lines.append(f" - {c}") lines[body_start:] = _search_skipped_apart(lines[body_start:]) lines.append("") lines.append("") lines.append("") return lines def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str: """Where a copy the database does not index sits: a ZIP member under bios/, or a file or tree of a data directory the packs fetch.""" path, member = copy parts = path.rstrip("/").split("/") if member: archive = _code(path) zsha1 = consumers.sha1_at(path) if zsha1 and len(parts) >= 3 and parts[0] == "bios": page = urllib.parse.quote(card_page(path, zsha1)) archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})" return f"Collection copy: member {_code(member)} of {archive}" if parts[0] == "data" and len(parts) >= 2: ref, inside = parts[1], "/".join(parts[2:]) if path.endswith("/"): where = _code(inside + "/") if inside else "the tree" return f"Collection copy: {where} in the {_code(ref)} data directory" info = (_DATA_MEMBERS.get(ref) or {}).get(inside) or {} shown = _code(inside) if info.get("sha1"): shown = (f'' f"{html.escape(inside)}") return f"Collection copy: {shown} in the {_code(ref)} data directory" tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(path.rstrip('/'))}" return f"Collection copy: [{_code(path)}]({tree})" def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]: """What the collection holds under a directory entry, and where. The card said "in repo" and linked none of the files: alephone's Plugins/ holds 393 of them, each with its own card on a system page. """ members = consumers.directory_members(entry) held = sorted({consumers.path_of(sha1) for sha1 in members.values()} - {""}) if not held: return [] folder = os.path.commonpath(held) if len(held) > 1 else os.path.dirname(held[0]) parts = folder.split("/") where = _code(folder + "/") if len(parts) >= 3 and parts[0] == "bios": where = f"[{where}]({console_href(_slug(parts[1]), parts[2], '../')})" tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(folder)}" return [f"Collection copies: {_count(len(members), 'file')} under {where} " f"([browse]({tree}))"] _MAME_RELEASE = re.compile(r"^0\.\d") def emulator_romset(profile: dict) -> str: """The romset an arcade emulator's sets must match, "MAME 0.78", or "". ``mame_version`` where the profile has one (MAME4droid 2024 is app 1.38.1 on MAME 0.289), else the version of a MAME build or of a FinalBurn core, whose sets follow its own release. """ if profile.get("type") in ("alias", "launcher", "test"): return "" emulator = str(profile.get("emulator", "")) version = str(profile.get("core_version") or "") if profile.get("mame_version"): return f"MAME {profile['mame_version']}" if "MAME" in emulator and _MAME_RELEASE.match(version): return f"MAME {version}" if emulator.startswith("FinalBurn Neo") and version: return f"FinalBurn Neo {version}" if (emulator.startswith("FB Alpha") or "FBA" in emulator) and version: return f"FB Alpha {version}" return "" def _render_metadata_rows(profile: dict) -> list[str]: """Scalar profile fields, one table row each. Anything structured belongs in a collapsible block below, not in a row: a dict rendered into a table cell is unreadable. """ lines: list[str] = [] romset = emulator_romset(profile) if romset: lines.append(f"| Romset | [{romset}]({ROMSET_FAQ}) |") for field, label in [ ("core", "Core ID"), ("core_name", "Core name"), ("bios_size", "BIOS size"), ("bios_directory", "BIOS directory"), ("bios_detection", "BIOS detection"), ("bios_selection", "BIOS selection"), ("firmware_file", "Firmware file"), ("firmware_source", "Firmware source"), ("firmware_install", "Firmware install"), ("firmware_detection", "Firmware detection"), ("resources_directory", "Resources directory"), ("rom_path", "ROM path"), ("game_count", "Game count"), ("verification", "Checks the file by"), ("analysis_date", "Analysis date"), ("analysis_commit", "Analysis commit"), ]: val = profile.get(field) if val is None or val == "" or isinstance(val, (dict, list)): continue if field == "bios_directory" and profile.get("type") == "standalone" \ and not str(val).startswith(("/", "~", "$")) and ":" not in str(val): # A bare `bios/` read as a place on the device: it is a folder # inside the app's own data folder, which the notes describe. lines.append(f"| {label} | {_code(val)}, inside the app's own data folder |") elif field == "bios_size" and isinstance(val, int): lines.append(f"| {label} | {_fmt_size(val)} ({val:,} bytes) |") elif isinstance(val, str) and val.startswith("http"): lines.append(f"| {label} | [{val}]({val}) |") else: lines.append(f"| {label} | {val} |") if profile.get("source_ref"): lines.append( f"| Source ref | {_source_ref_markdown(profile, profile['source_ref'])} |" ) return lines def install_commands(args: str = "", indent: str = "") -> list[str]: """The same install for each place a user runs it: one generator for the download page and the platform, system and emulator pages, so none of them loses the Android or Windows form.""" lines = [] for label, shell in ( ("Linux, macOS, Steam Deck:", "sh"), ("Windows (PowerShell):", "powershell"), ("Android (Termux, after `pkg install python` and `termux-setup-storage`):", "sh"), ): lines += [f"{indent}{label}", "", *fence(shell, install_command(shell, args), indent), ""] return lines def _install_block(name: str, profile: dict) -> list[str]: """One command that installs this emulator's files where it reads them. The installer finds the platform and its BIOS folder, then keeps what this emulator reads there: its own files and the platform's files of the systems it runs. Each file is also linked one by one below. """ if profile.get("type") in ("alias", "launcher", "test"): return [] if profile.get("type") == "standalone" and any_name_bios(profile) is not None: # `--emulator` installs into a platform's BIOS folder and turns off # the pass that fills standalone emulator folders (install.py, # `narrowing` in main): an app on its own reported "No supported # platform detected". The download step of the intro replaces it. return [] return search_excluded(markup.details("tip", "Install this emulator's files", [ "The installer finds your platform and its BIOS folder, then downloads " "only the files this emulator reads there and checks each one.", "", *install_commands(f"--emulator {name}"), ])) def _platform_install_block(name: str, display: str, on_card: bool = False) -> list[str]: """The platform's pack, installed file by file: what the ZIP holds, with resume, verification and only what is missing. ``on_card``: the files go on the frontend's SD card, installed from the computer it is mounted on, whose root is named. """ if on_card: return markup.details("tip", "Install with one command", [ f"With the {display} SD card mounted, the installer downloads the files " "of the pack that are missing or differ from it into `Bios//` on " "the card, checks each one, and resumes where it stopped. `--dest` " "names the card's root.", "", "Linux, macOS:", "", *fence("sh", install_command("sh", f"--platform {name} --dest /path/to/card")), "", "Windows (PowerShell, E: being the card):", "", *fence("powershell", install_command("powershell", f"--platform {name} --dest E:\\")), ], opened=True) return markup.details("tip", "Install with one command", [ f"The installer downloads the files of the {display} pack that are " "missing or differ from it, checks each one, and resumes where it " "stopped. Add `--system` with the ids of the systems overview below " "to install fewer systems.", "", *install_commands(f"--platform {name}"), ], opened=True) def _ps_quote(value: str) -> str: """A PowerShell argument: bare when plain, single-quoted otherwise.""" return value if re.fullmatch(r"[\w.+-]+", value) else "'" + value.replace("'", "''") + "'" def _console_install_block(console: str, declared: list[tuple[str, str]]) -> list[str]: """Each platform's command for one console, by the system id it gives it, for a terminal and for PowerShell, each under its setup's name: RetroBat had two lines and nothing said which was for the Neo Geo.""" if not declared: return [] sh = [line for platform, system in declared for line in ( f"# {_platform_display(platform)}", install_command("sh", f"--platform {platform} --system {shlex.quote(system)}"))] ps = [line for platform, system in declared for line in ( f"# {_platform_display(platform)}", install_command("powershell", f"--platform {platform} --system {_ps_quote(system)}"))] # The name last: "Install the Atari System 1 files" read as a count. A # label naming files already reads "Install these files": "Install # the files of Other arcade files". summary = ("Install these files" if console.casefold().endswith(" files") else f"Install the files of {console}") return search_excluded(markup.details("tip", summary, [ "Each platform names the system its own way.", "", "Linux, macOS, Steam Deck:", "", *fence("sh", "\n".join(sh)), "", "Windows (PowerShell):", "", *fence("powershell", "\n".join(ps)), ])) def _render_platform_details(profile: dict) -> list[str]: """The per-platform block, one level of nesting deep.""" lines: list[str] = [] platform_details = profile.get("platform_details") if platform_details and isinstance(platform_details, dict): body: list[str] = [] for pk, pv in platform_details.items(): if isinstance(pv, dict): body.extend([f"- **{pk}:**", ""]) for sk, sv in pv.items(): body.append(f" - {sk}: {_detail_value(profile, sk, sv)}") body.append("") else: body.append(f"- **{pk}:** {_detail_value(profile, pk, pv)}") lines.extend(markup.details("info", "Platform details", body, opened=True)) return lines def _detail_value(profile: dict, key: object, value: object) -> str: """A platform detail, its citations linked at the profile's pin.""" if key == "source_ref" and isinstance(value, (str, dict)): return _source_ref_markdown(profile, value) if isinstance(value, str): return _escape_tags(link_prose_citations(profile, value)) return _plain_value(value) def _plain_value(value: object) -> str: """A scalar or a list of scalars as a reader writes it, not as Python does.""" if isinstance(value, bool): return "yes" if value else "no" if value is None: # A YAML null states that there is none, but Python's `None` read as a gap. return "none" if isinstance(value, (list, tuple)): return ", ".join(_plain_value(item) for item in value) return _escape_tags(str(value)) def _structured_field_text(profile: dict) -> Callable[[object, object], str]: """Scalars of a structured block, citations linked at the profile's pin. The file entries and platform details linked their source_ref, and the analysis and firmware blocks printed 186 of them as plain code. """ def text(key: object, value: object) -> str: if key == "source_ref" and isinstance(value, str): return _source_ref_markdown(profile, value) if isinstance(value, str): return link_prose_citations(profile, _field_text(key, value)) return _field_text(key, value) return text def _render_structured_blocks(profile: dict) -> list[str]: """Every remaining structured field, as a collapsible section.""" lines: list[str] = [] _structured_blocks = [ ("analysis", "Source analysis"), ("memory_layout", "Memory layout"), ("regions", "Regions"), ("nvm_layout", "NVM layout"), ("model_kickstart_map", "Model kickstart map"), ("builtin_boot_roms", "Built-in boot ROMs"), ("common_bios_filenames", "Common BIOS filenames"), ("valid_bios_crc32", "Valid BIOS CRC32"), ("dev_flash", "dev_flash"), ("dev_flash2", "dev_flash2"), ("dev_flash3", "dev_flash3"), ("firmware_modules", "Firmware modules"), ("firmware_titles", "Firmware titles"), ("fallback_fonts", "Fallback fonts"), ("io_devices", "I/O devices"), ("partitions", "Partitions"), ("mlc_structure", "MLC structure"), ("machine_directories", "Machine directories"), ("machine_properties", "Machine properties"), ("whdload_kickstarts", "WHDLoad kickstarts"), ("bios_identical_to", "BIOS identical to"), ("pack_structure", "Pack structure"), ("firmware_version", "Firmware version"), ] for field, label in _structured_blocks: val = profile.get(field) if val is None: continue body: list[str] = [] _render_yaml_value(body, val, indent=0, field_text=_structured_field_text(profile)) block = markup.details( "abstract", label, [_escape_tags(line) for line in body], opened=True) # A block listing hashes is the lookup's, as a card's hash lines are: # izapple2's ROM table put "md5" 59 times in the search, above the # page that explains MD5 checks. if any(re.search(r"\b[0-9a-f]{32,64}\b", line) for line in body): block = search_excluded(block) lines.extend(block) return lines def _hash_spans(value: str | list) -> str: """Each hash a profile accepts, in full. Cut to 12 characters with the rest in a title attribute, 208 hashes that only a profile declares were out of reach of the site search: it indexes text, and a pasted sha1 matched nothing. """ from hashing import parse_hash_list # noqa: PLC0415 return ", ".join( markup.hash_value(v) for v in parse_hash_list(value) ) def _availability_check( db: dict, data_names: set[str] | None ) -> Callable[[dict], bool]: """A predicate answering whether the collection holds a file. It answers the way cross_reference does. A page that called a file missing while the gap report called it held would describe a different collection on two pages of the same site. """ from cross_reference import entry_source by_name = db.get("indexes", {}).get("by_name", {}) by_name_lower = {k.lower(): k for k in by_name} by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {}) index = { "by_name": by_name, "by_name_lower": by_name_lower, "by_path_suffix": by_path_suffix, "by_md5": db.get("indexes", {}).get("by_md5", {}), "by_sha256": db.get("indexes", {}).get("by_sha256", {}), "by_crc32": db.get("indexes", {}).get("by_crc32", {}), "db_files": db.get("files", {}), "data_names": data_names, } def _file_available(f: dict) -> bool: """Check if a file is available using the same resolution as cross_reference.""" # cross_reference sets these aside before resolving: the file found # under the name belongs to another game or release. if not f.get("name") or f.get("unsourceable"): return False return entry_source(f, index) is not None return _file_available def _emulator_slots(files: list[dict]) -> list[list[dict]]: """The entries of each slot, as verify --emulator and the gaps page count them, leaving out the slots the app ships.""" from cross_reference import slot_key slots: dict[tuple, list[dict]] = {} for f in files: slots.setdefault(slot_key(f), []).append(f) return [group for group in slots.values() if not all(comes_with_app(f) for f in group)] def _emulator_file_summary(files: list[dict]) -> list[str]: """What the emulator reads, above its file table, and its categories. Counted by slot, as verify --emulator and the gaps page count: raze lists every GRP its code accepts under one name, and holding one fills the slot. Counted per entry, the page announced 46 missing where 12 slots are empty. The files the app ships are counted apart: NetherSX2 said "4 required" for one BIOS and three fonts in its package. """ slots = _emulator_slots(files) shipped = sum(1 for f in files if comes_with_app(f)) req_count = sum(1 for group in slots if any(f.get("required") for f in group)) hle_count = sum(1 for group in slots if any(f.get("hle_fallback") for f in group)) # Sentences, one number format: the line read "1,563 files | 164 # required, 1399 optional | 1563 in repo". total = f"**{_count(len(slots), 'file')}**" if sum(len(group) for group in slots) != len(slots): total += f" ({sum(len(group) for group in slots):,} accepted versions)" line = f"{total}: {req_count:,} required, {len(slots) - req_count:,} optional." if hle_count: line += f" {hle_count:,} {'has' if hle_count == 1 else 'have'} an HLE fallback." if shipped: line += f" {_count(shipped, 'more file')} {'comes' if shipped == 1 else 'come'} with the app." lines = [line, ""] categories = [f.get("category", "bios") for f in files] if "game_data" in categories or "bios_zip" in categories: cats = [ f"{categories.count(key):,} {label}" for key, label in ( ("bios", "BIOS"), ("bios_zip", "BIOS ZIPs"), ("game_data", "game data") ) if categories.count(key) ] lines.extend([f"By kind: {_said_as_list(cats)}.", ""]) return lines def _emulator_held_line(files: list[dict], available: Callable[[dict], bool]) -> list[str]: """What this collection holds of them, one quiet line under the table: a reader comes for what the emulator needs, the project's counts after.""" slots = _emulator_slots(files) if not slots: return [] in_repo = sum(1 for group in slots if any(available(f) for f in group)) unsourceable = sum( 1 for group in slots if not any(available(f) for f in group) and all(f.get("unsourceable") for f in group) ) held = [f"{in_repo:,} in repo", f"{len(slots) - in_repo - unsourceable:,} missing"] if unsourceable: held.append(f"{unsourceable:,} unsourceable") return [f"

In this collection: {_said_as_list(held)}.

", ""] def _said_as_list(items: list[str]) -> str: """"a, b and c".""" return ", ".join(items[:-1]) + f" and {items[-1]}" if len(items) > 1 else "".join(items) # Data directory members and revisions, read once per build. _DATA_MEMBERS: dict[str, dict[str, dict]] = {} _DATA_REVISIONS: dict[str, dict] = {} # The emulator pages that list each data directory, by ref. _DATA_READERS: dict[str, list[tuple[str, str]]] = {} # A data directory listing more files than this goes to pages of its own, # this many files each: Dolphin's Sys tree put 2,164 rows and 16,900 # elements on the Dolphin page, read on a phone. DATA_TREE_ROWS = 400 DATA_TREE_ROWS_PER_PAGE = 1_000 def data_tree_pages(ref: str, members: dict[str, dict]) -> list[tuple[str, list[str]]]: """The pages under emulators/ listing a large data directory, with the members each holds in name order, or none when the tree is listed on the pages of the emulators reading it.""" if len(members) <= DATA_TREE_ROWS: return [] names = sorted(members) stem = f"data-{_slugify_anchor(ref)}" return [(stem + (f"-{n}" if n > 1 else ""), names[start:start + DATA_TREE_ROWS_PER_PAGE]) for n, start in enumerate(range(0, len(names), DATA_TREE_ROWS_PER_PAGE), 1)] def _data_member_table(members: dict[str, dict], names: list[str], anchors: set[str]) -> list[str]: """A data directory's files with their size and sha1, each sha1 an anchor the first time the page lists it (``anchors``, the page's ids).""" table = ["| File | Size | SHA1 |", "|------|-----:|------|"] for member in names: info = members[member] sha1 = str(info.get("sha1", "")) # One id per page: two members can hold the same bytes (every empty # file), and Dolphin's trees share Sys. anchor = f"sha1-{sha1[:12]}" span = "" if anchor in anchors else f'' anchors.add(anchor) table.append(f"| {span}{_code(member)} | {_fmt_size(int(info.get('size', 0)))} | {_code(sha1)} |") return table def _data_tree_page(ref: str, entry: dict, pages: list[tuple[str, list[str]]], index: int) -> str: """One page of a large data directory's file list.""" stem, names = pages[index] members = _DATA_MEMBERS.get(ref) or {} part = f" (part {index + 1} of {len(pages)})" if len(pages) > 1 else "" read_by = _said_as_list([f"[{_escape_tags(display)}]({key}.md#data-directories)" for key, display in _DATA_READERS.get(ref, [])]) # A file list out of the search: "dolphin" found three parts of the # dolphin-sys tree, on their title, before the Dolphin page. front: dict = {"level": "reference", "search": "exclude"} # The reader the tree is named after (dolphin for dolphin-sys), else # the first by name. readers = _DATA_READERS.get(ref) or [] owner = max(readers, default=None, key=lambda reader: len(reader[0]) if ref.startswith(f"{reader[0]}-") else 0) if owner: head = _part_front_matter( (owner[1], f"{owner[0]}.md"), [(f"The {ref} data directory, part {n}", f"{other}.md") for n, (other, _names) in enumerate(pages, 1)], index, front) else: head = ["---", yaml.safe_dump(front, sort_keys=False, allow_unicode=True).rstrip("\n"), "---", ""] lines = [ *head, f"# The {ref} data directory{part}", "", f"{entry.get('description') or 'A data directory'}: " f"{_count(len(members), 'file')}, as the packs ship them." + (f" Read by {read_by}." if read_by else ""), "", ] if len(pages) > 1: lines.extend([", ".join(f"[part {i}]({other}.md)" if other != stem else f"part {i}" for i, (other, _names) in enumerate(pages, 1)), ""]) lines.extend(search_excluded(_data_member_table(members, names, set()))) lines.extend(["", _date_line()]) return "\n".join(lines) + "\n" # An emulator page lists at most this many file cards: MAME's 1,564 made a # 3 MB page. Past it, the cards go to pages of their own and the emulator's # page keeps a table of every entry, each row holding the card's old # anchor. A count, not a size: system cards link to these cards and are # written before them. At 400 a part of MAME's cards was 805 KB of HTML. EMULATOR_CARDS_PER_PAGE = 220 # The headings an emulator's cards go under, in page order: what it needs # first. DuckStation's three required BIOS were cards 12 to 14 under one # heading, and MAME opened on 25 floppy sound samples. CARD_SECTIONS = (("required", "Required"), ("optional", "Optional"), ("game_data", "Game data"), ("with_app", "Comes with the app")) _SECTION_RANK = {key: rank for rank, (key, _title) in enumerate(CARD_SECTIONS)} # Below this many cards the page is one screen of them, and a table above # would repeat it. EMULATOR_SUMMARY_MIN = 6 def _card_section(f: dict) -> str: if comes_with_app(f): return "with_app" if f.get("category") == "game_data": return "game_data" return "required" if f.get("required") else "optional" def _display_order(files: list) -> list[int]: """Entry indexes in page order: required, optional, then game data, each in profile order.""" return sorted(range(len(files)), key=lambda i: _SECTION_RANK[_card_section(files[i])]) def emulator_parts(name: str, files: list) -> list[tuple[str, list[int]]]: """The pages an emulator's cards go to, with the entries each holds, or none when they fit on the emulator's page. Balanced, in page order.""" if len(files) <= EMULATOR_CARDS_PER_PAGE: return [] order = _display_order(files) count = -(-len(files) // EMULATOR_CARDS_PER_PAGE) size = -(-len(files) // count) return [(f"{name}-files-{index + 1}", order[start:start + size]) for index, start in enumerate(range(0, len(files), size))] def _sectioned(files: list, indexes: list[int], render: Callable[[list[int]], list[str]], fold: bool = False) -> list[str]: """Each section's heading, then ``render`` of its entries among ``indexes``. ``fold``: every section after Required in one closed block, for a page whose table above already lists each entry. """ lines: list[str] = [] for key, title in CARD_SECTIONS: group = [i for i in indexes if _card_section(files[i]) == key] if not group: continue body = render(group) if fold and key != "required": body = markup.details("note", _count(len(group), "file"), body) lines.extend([f"## {title}", "", *body]) return lines def _hash_short(f: dict) -> str: """The shortest way to tell two dumps apart in a table: the first 12 characters of an md5, crc32 or sha1, the whole value in its title.""" for field in ("md5", "crc32", "sha1"): value = f.get(field) value = value[0] if isinstance(value, list) and value else value if value: full = html.escape(str(value).lower().split(",")[0].strip().removeprefix("0x")) # Kept on one line: the label above the value doubled each row. return f'{field.upper()}\u00a0{full[:12]}' return "" def _summary_table(files: list[dict], indexes: list[int], cell: Callable[[int, str], str], multi_system: bool = False) -> list[str]: """The table of an emulator's entries, one row each, ``cell`` writing the linked name of entry i from its shown path. A column every row leaves empty is not written: AetherSX2's Region column held "-" on every row and pushed Required off a phone screen. Nor is Required when every row gives the same answer, which the section heading or the count above already says: in MAME's index it turned 1,564 rows into label and value pairs on a phone. """ rows = [] for i in indexes: f = files[i] needed = "with the app" if comes_with_app(f) else "yes" if f.get("required") else "no" if needed == "yes" and multi_system and f.get("system"): needed = f"for {system_name(str(f['system']))}" rows.append((cell(i, str(f.get("path") or f.get("name", ""))), _region_words(f.get("region"), 3), needed, _hash_short(f))) keep = [0] + [column for column in (1, 3) if any(row[column] for row in rows)] if len({row[2] for row in rows}) > 1: keep.append(2) keep.sort() heads = ("File", "Region", "Required", "Hash") return [ "| " + " | ".join(heads[c] for c in keep) + " |", "|" + "|".join("------" for _c in keep) + "|", *("| " + " | ".join(row[c] or "-" for c in keep) + " |" for row in rows), ] def plan_emulator_pages(profiles: dict) -> None: """Record which page holds each emulator card, before any page links one.""" stems = set() for name, profile in sorted(profiles.items()): if profile.get("type") in ("alias", "test"): continue files = profile.get("files") or [] anchors = emulator_file_anchors(files) for stem, entries in emulator_parts(name, files): stems.add(stem) record_emulator_cards(name, stem, [anchors[i] for i in entries]) clash = stems & set(profiles) if clash: raise RuntimeError(f"an emulator's card page takes a profile's address: {sorted(clash)}") def generate_emulator_page( name: str, profile: dict, db: dict, consumers: FileConsumers | None = None, data_names: set[str] | None = None, data_registry: dict | None = None, data_dirs: list[dict] | None = None, ) -> str: """One emulator's page. ``data_dirs`` defaults to the profile's own.""" return emulator_pages( name, profile, db, consumers, data_names, data_registry, data_dirs)[name] def any_name_bios(profile: dict) -> dict | None: """The one BIOS an emulator needs under any name, or None. A profile found by content (`found_by: content`, `bios_mode: agnostic`) for one system, with one required file of no declared region: the code identifies the image by what it holds, so the entry's name is an example. AetherSX2's page showed `bios/ps2-0230a-20080220.bin` as the file to get, while its notes say no name and no hash is matched. """ if profile.get("found_by") != "content" and profile.get("bios_mode") != "agnostic": return None if len(profile.get("systems") or []) != 1: return None required = [f for f in profile.get("files") or [] if isinstance(f, dict) and f.get("required") is True and not comes_with_app(f)] if len(required) != 1 or required[0].get("region") or required[0].get("category") not in (None, "bios"): return None return required[0] def _collection_card_href(entry: dict, consumers: FileConsumers | None, owner: str) -> str: """The collection card of the copy that answers an emulator entry, or "".""" if consumers is None: return "" sha1 = consumers.sha1_of(entry, entry.get("path") or "", owner=owner) held = consumers.path_of(sha1) if sha1 else "" if not held.startswith("bios/"): return "" return f"../systems/{urllib.parse.quote(card_page(held, sha1))}.md#sha1-{sha1[:12]}" def _named_example(name: str) -> bool: """A real file name, not a placeholder such as `.bin`.""" return bool(name) and "<" not in name def _any_name_folder(profile: dict, entry: dict) -> str: """Where the one BIOS goes, as the profile states it, or "".""" folder = str(profile.get("bios_directory") or "") if not folder: path = str(entry.get("path") or "") folder = path.rsplit("/", 1)[0] + "/" if "/" in path else "" if not folder: return "" if profile.get("type") == "standalone": # A bare `bios/` is a folder inside the app's own data folder. return f"the {_code(folder.strip('/'))} folder inside the app's own data folder" return f"{_code(folder)}, inside your BIOS folder" def _any_name_steps(emu_name: str, profile: dict, entry: dict, href: str) -> str: """The intro of an emulator that takes one BIOS of any name: get one, put it in its folder, any name works.""" system = system_name((profile.get("systems") or [""])[0]) name = str(entry.get("name") or "") if _named_example(name) and href: get = f"Download [{_code(name)}]({href})." else: get = f"Get one from the {_named_system_link(profile['systems'][0], '../')} files." steps = [get] folder = _any_name_folder(profile, entry) if folder: steps.append(f"Put it in {folder}.") reader = "The app" if profile.get("type") == "standalone" else "The emulator" return "\n".join([ f"{emu_name} needs one {system} BIOS file, any region.", "", *(f"{number}. {step}" for number, step in enumerate(steps, 1)), "", f"Any file name works. {reader} reads what the file holds.", ]) def _emulator_intro(emu_name: str, profile: dict, href: str = "") -> str: """What the emulator needs, in plain words, from fields every profile has. The page opened on a twelve-row table: "Type", "Version", "Checked by". DuckStation needs one PlayStation BIOS of any name, and the reader had to infer it from the "Files found by" row. ``href`` links the collection's copy of the one BIOS an emulator takes under any name. """ entry = any_name_bios(profile) if entry is not None: return _any_name_steps(emu_name, profile, entry, href) names: list[str] = [] for sys_id in profile.get("systems") or []: label = system_name(sys_id) if label not in names: names.append(label) # What the app installs itself is no file for the reader to get. files = [f for f in profile.get("files") or [] if isinstance(f, dict) and not comes_with_app(f)] required = [f for f in files if f.get("required") is True] sentences = [] if names: shown = names if len(names) <= 4 else names[:3] + [f"{len(names) - 3} more systems"] sentences.append(f"{emu_name} emulates {_said_as_list(shown)}.") by_region = _regional_choice(required) if by_region: # DuckStation needs the BIOS of the game's region, not all three: the # page said "3 of them required" and read as "get all of these". others = len(files) - len(required) sentences.append(f"It picks the BIOS for your game's region: {by_region}.") if others: sentences.append( f"The {_count(others, 'other file')} {'is' if others == 1 else 'are'} optional.") elif files: sentences.append( f"It reads {_count(len(files), 'file')}, {len(required):,} of them required: " "without a required file it does not start games." if required else f"It reads {_count(len(files), 'file')}, none of them required: it starts " "games without them." ) if required and len(names) > 1: sentences.append("A required file is needed only by the machines that use it.") if profile.get("found_by") == "content": sentences.append("It recognises a file by its contents, so any file name works.") if any(f.get("source_ref") for f in files): sentences.append("Each file below cites the line of source code that reads it.") return " ".join(sentences) def _regional_choice(required: list[dict], most: int = 6) -> str: """The required files as one choice by region, or "" when they are not one. Only when every required entry carries a region: and they name at least two: region: is the territory the code selects the file for, so the reader needs the one of the game's region. """ if not required or len(required) > most or not all(f.get("region") for f in required): return "" if len({_region_words(f["region"]) for f in required}) < 2: return "" choices = [f"{f.get('name', '')} ({_region_words(f['region'], 3)})" for f in required] return ", ".join(choices[:-1]) + f" or {choices[-1]}" def emulator_pages( name: str, profile: dict, db: dict, consumers: FileConsumers | None = None, data_names: set[str] | None = None, data_registry: dict | None = None, data_dirs: list[dict] | None = None, ) -> dict[str, str]: """An emulator's page by stem under emulators/, and the pages its cards went to when there are more than EMULATOR_CARDS_PER_PAGE.""" if profile.get("type") == "alias": parent = profile.get("alias_of", profile.get("bios_identical_to", "unknown")) return {name: ( f"# {name}\n\n" f"This core uses the same firmware as **{parent}**.\n\n" f"See [{parent}]({parent}.md) for details.\n" )} emu_name = profile.get("emulator", name) emu_type = profile.get("type", "unknown") classification = profile.get("core_classification", "") source_raw = profile.get("source", "") source = str(source_raw) if not isinstance(source_raw, dict) else "" # A key set to null says there is none (the project is its own upstream, # it publishes no version), and an absent key says nobody wrote it down. The # page printed `None` for the first, which read as the second. upstream_raw = profile.get("upstream", "") upstream_none = "upstream" in profile and upstream_raw is None upstream = str(upstream_raw) if isinstance(upstream_raw, str) else "" version_raw = profile.get("core_version", "unknown") if version_raw is None: version = "none published" elif isinstance(version_raw, dict): # Keyed by build mode like source: each build's version, named, # where the page printed the dict's Python repr. version = ", ".join(f"{v} ({mode})" for mode, v in version_raw.items() if v) else: version = version_raw profiled = profile.get("profiled_date", "unknown") systems = profile.get("systems", []) cores = profile.get("cores", [name]) files = profile.get("files", []) notes_raw = profile.get("notes", profile.get("note", "")) notes = ( str(notes_raw).strip() if notes_raw and not isinstance(notes_raw, dict) else "" ) exclusion = profile.get("exclusion_note", "") if data_dirs is None: data_dirs = profile.get("data_directories", []) part_pages: dict[str, str] = {} one_bios = any_name_bios(profile) intro = _emulator_intro(emu_name, profile, _collection_card_href( one_bios, consumers, name) if one_bios else "") romset = emulator_romset(profile) lines = [ f"# {emu_name}", "", *([intro, ""] if intro else []), ] if romset: # The commonest arcade failure is a set from another MAME version, # and its answer sat on a Reference page no arcade page linked. # FinalBurn profiles list the members of their archives, not the # archives, and MAME 2003-Plus none at all: the line went missing # on both. lines.extend([f"Its BIOS and game sets must come from the {romset} romset. " f"[Which version each core reads]({ROMSET_FAQ}).", ""]) card_start = len(lines) lines += [ markup.block_open("div", 'class="rb-meta-card"'), "", "| Detail | Value |", "|---|---|", f"| Type | {emu_type} |", ] if classification: cls_display = CLS_LABEL_ONE.get(classification, classification) lines.append(f"| Classification | {cls_display} |") if isinstance(source_raw, dict): parts = [] for k, v in source_raw.items(): if isinstance(v, str) and v.startswith("http"): parts.append(f"[{k}]({v})") else: parts.append(f"{k}: {v}") lines.append(f"| Source | {', '.join(parts)} |") elif source: if source.startswith("http"): lines.append(f"| Source | [{source}]({source}) |") else: lines.append(f"| Source | {source} |") if isinstance(upstream_raw, dict): parts = [] for k, v in upstream_raw.items(): if isinstance(v, str) and v.startswith("http"): parts.append(f"[{k}]({v})") else: parts.append(f"{k}: {v}") lines.append(f"| Upstream | {', '.join(parts)} |") elif upstream_none: lines.append("| Upstream | none: the source is the project itself |") elif upstream and upstream != source: if upstream.startswith("http"): lines.append(f"| Upstream | [{upstream}]({upstream}) |") else: lines.append(f"| Upstream | {upstream} |") gone = profile.get("upstream_gone") if gone: # The links above answer 451 or reach a domain someone else now # owns: said beside them, with the date the profile observed it. lines.append(f"| Source status | gone: {_escape_tags(str(gone))} |") mirror = profile.get("source_mirror") if mirror: lines.append(f"| Mirror | [{mirror}]({mirror}) |" if str(mirror).startswith("http") else f"| Mirror | {mirror} |") lines.append(f"| Version | {version} |") revision = revision_markdown(profile) if revision: lines.append(f"| Revision | {revision} |") lines.extend(_build_rows(profile)) lines.append(f"| Profiled | {profiled} |") # The YAML is the reusable document this site renders, and the JSON is the # same profile for a program. lines.append( f"| Profile | [YAML]({REPO_URL}/blob/main/emulators/{name}.yml), " f"[JSON](../api/v1/emulators/{name}.json) |" ) if cores: lines.append(f"| Cores | {', '.join(_distinct_spellings(cores))} |") if systems: sys_links = list(dict.fromkeys(_named_system_link(s, "../") for s in systems)) lines.append(f"| Systems | {', '.join(sys_links)} |") author = profile.get("author", "") if author: lines.append(f"| Author | {author} |") based_on = profile.get("based_on", "") if based_on: lines.append(f"| Based on | {based_on} |") lines.extend(_render_metadata_rows(profile)) lines.append("") lines.append("") lines.append("") # Reference for the few who check a revision: stacked on a phone, the # card took two screens above the files. lines[card_start:] = markup.details("note", "About this profile", lines[card_start:]) lines.extend(_install_block(name, profile)) lines.extend(_render_platform_details(profile)) lines.extend(_render_structured_blocks(profile)) if notes: body = _admonition_body(link_prose_citations(profile, notes)).replace("\n ", "\n") lines.extend(markup.details("note", "Technical notes", body.split("\n"))) if not files: lines.append("No BIOS or firmware files required.") if exclusion: lines.append("") lines.extend(markup.aside("info", "Why no files", [_escape_tags(str(exclusion))])[:-1]) else: _file_available = _availability_check(db, data_names) lines.extend(["## Files", ""]) lines.extend(_emulator_file_summary(files)) held = _emulator_held_line(files, _file_available) anchors = emulator_file_anchors(files) aliases = emulator_file_aliases(files, anchors) cards = [ _render_emulator_file(f, profile, consumers, files, _file_available, anchor, name, alias) for f, anchor, alias in zip(files, anchors, aliases) ] parts = emulator_parts(name, files) if not parts: order = _display_order(files) indexed = len(files) >= EMULATOR_SUMMARY_MIN if indexed: lines.extend(search_excluded(_summary_table( files, order, lambda i, shown: f"[{_code(shown)}](#{anchors[i]})", len(systems or []) > 1))) lines.append("") lines.extend(held) # Below the table, which indexes them, the cards past the # required ones fold: DuckStation's page was 59 phone screens, # 102 optional cards after its three BIOS. lines.extend(_sectioned( files, order, lambda group: [line for i in group for line in cards[i]], fold=indexed)) else: lines.extend(_card_index(emu_name, files, anchors, aliases, parts, len(systems or []) > 1)) lines.extend(held) for number, (stem, entries) in enumerate(parts, 1): part_pages[stem] = _emulator_part_page( name, emu_name, number, parts, [cards[i] for i in entries], [files[i] for i in entries]) if data_dirs: # The member tables are a file list with hashes: out of the search, # dolphin, primehack and ishiiruka each put 210 KB of it there. lines.extend([ "## Data directories", "", ]) trees_start = len(lines) member_anchors: set[str] = set() # Each tree with what it holds, where it comes from, which build reads # it and the code that does: these trees put thousands of files in # the packs, and the page said only `dolphin-sys` > `Sys`. for dd in data_dirs: ref = dd.get("ref", "") dest = dd.get("destination", "") entry = (data_registry or {}).get(ref, {}) line = f"- `{ref}` > `{dest}`" if entry.get("description"): line += f": {entry['description']}" lines.append(line) if entry.get("source_url"): lines.append(f" - Fetched from <{entry['source_url']}>") fetched = _DATA_REVISIONS.get(ref) or {} if fetched.get("version") or fetched.get("sha"): # The URL serves whatever is current: the revision says # which content the packs were built from. revision = str(fetched.get("sha", "")).strip('"') lines.append(f" - Revision: {fetched.get('version', '')}" + (f" ({_code(revision)})" if revision else "")) if dd.get("mode"): lines.append(f" - Read by the {dd['mode']} build only") if dd.get("bundled"): lines.append(" - Installed with the emulator, so packs leave it out") if dd.get("source_ref"): lines.append(f" - Source: {_source_ref_markdown(profile, dd['source_ref'])}") members = _DATA_MEMBERS.get(ref) or {} tree_pages = data_tree_pages(ref, members) if tree_pages: where = (f"[a page of their own]({tree_pages[0][0]}.md)" if len(tree_pages) == 1 else "pages of their own: " + ", ".join( f"[part {i}]({stem}.md)" for i, (stem, _names) in enumerate(tree_pages, 1))) lines.append(f" - {_count(len(members), 'file')}, listed on {where}") for index, (stem, _names) in enumerate(tree_pages): part_pages[stem] = _data_tree_page(ref, entry, tree_pages, index) elif members: table = _data_member_table(members, sorted(members), member_anchors) lines.append("") lines.extend(markup.details( "info", f"{_count(len(members), 'file')}, as the packs ship them", table, indent=" ")) lines.append("") # The member tables are the lookup's. lines[trees_start:] = search_excluded(lines[trees_start:]) lines.extend(_next_block([ "[Get your files](../which-pack.md), for the setup you play on", "[Every system](../systems/index.md), and the files each one needs", "[When a game does not start](../wiki/troubleshooting.md)", ])) lines.extend(["", _date_line()]) return {name: "\n".join(lines) + "\n", **part_pages} def _card_index( emu_name: str, files: list[dict], anchors: list[str], aliases: list[str], parts: list[tuple[str, range]], multi_system: bool = False, ) -> list[str]: """Every entry of a split emulator page, one row each, linked to its card. Each row holds the ids the card had on this page, so a link cited before the split lands on the row, and site.js follows it to the card. """ page_of = {index: stem for stem, entries in parts for index in entries} def cell(i: int, shown: str) -> str: # The id sits on the link itself, one element fewer per row: MAME's # 1,564 rows put 13,300 elements on its page. href = markup.site_href(f"emulators/{page_of[i]}/#{anchors[i]}") alias = f'' if aliases[i] else "" return (f'{alias}' f"{html.escape(shown, quote=False)}") def table(group: list[int]) -> list[str]: return [ markup.block_open("section", f'class="rb-card-index" {markup.search_exclude()}'), "", *_summary_table(files, group, cell, multi_system), "", "", "", ] return [ f"{emu_name} reads more files than one page holds: each entry below " f"links to its card, on {_count(len(parts), 'page')}: " + ", ".join(f"[{number}]({stem}.md)" for number, (stem, _e) in enumerate(parts, 1)) + ".", "", *_sectioned(files, _display_order(files), table), ] def _part_front_matter(parent: tuple[str, str], siblings: list[tuple[str, str]], index: int, front: dict | None = None) -> list[str]: """The front matter of part ``index`` of ``siblings`` ((label, link) each): the page it belongs to, its neighbours and the part weight. MAME's eight card pages had no parent, so the crumbs skipped MAME, and `prev: false`, so part 5 reached part 6 only through a sentence. The first part's previous page is the parent, the last part has no next. """ data = dict(front or {}) data["parent"] = {"label": parent[0], "link": parent[1]} before = siblings[index - 1] if index > 0 else parent data["prev"] = {"label": before[0], "link": before[1]} data["next"] = ({"label": siblings[index + 1][0], "link": siblings[index + 1][1]} if index + 1 < len(siblings) else False) data["searchWeight"] = SEARCH_WEIGHT.get("part", DEMOTED_WEIGHT) return ["---", yaml.safe_dump(data, sort_keys=False, allow_unicode=True).rstrip("\n"), "---", ""] def _emulator_part_page( name: str, emu_name: str, number: int, parts: list[tuple[str, range]], cards: list[list[str]], files: list[dict], ) -> str: """One page of a split emulator's cards.""" first = str(files[0].get("path") or files[0].get("name", "")) last = str(files[-1].get("path") or files[-1].get("name", "")) others = ", ".join( f"[{n}]({stem}.md)" for n, (stem, _e) in enumerate(parts, 1) if n != number) lines = [ *_part_front_matter((emu_name, f"{name}.md"), [(f"{emu_name} files, part {n}", f"{stem}.md") for n, (stem, _e) in enumerate(parts, 1)], number - 1), f"# {emu_name} files, part {number} of {len(parts)}", "", f"Part of [{emu_name}]({name}.md), whose page has the profile, the install " f"command and every entry. This part holds {_count(len(files), 'entry', 'entries')}, " f"{_code(first)} to {_code(last)}. The other parts: {others}.", "", *_sectioned(files, list(range(len(files))), lambda group: [line for i in group for line in cards[i]]), ] lines.extend(_next_block([f"[{emu_name}]({name}.md), the profile and every entry"])) lines.extend(["", _date_line()]) return "\n".join(lines) + "\n" # Contributing page _ACCEPTED_RE = re.compile(r"\b(sha1|sha256|md5|crc32) mismatch: got \S+, accepted \[([^\]]*)\]") def _held_accepted(discrepancy: str, db: dict | None) -> str: """Where the collection keeps a dump the emulator accepts, if it does. The table printed the hash the emulator wants and stopped there, while the collection held that exact dump for every one of the ten rows. """ match = _ACCEPTED_RE.search(discrepancy or "") if not match or not db: return "" algo, accepted = match.group(1), match.group(2) indexes = db.get("indexes", {}) for value in re.split(r"[\s,]+", accepted.lower()): if algo == "sha1": sha1 = value if value in db.get("files", {}) else None else: sha1 = indexes.get(f"by_{algo}", {}).get(value) path = db.get("files", {}).get(sha1 or "", {}).get("path") if path: return path return "" _PROBLEM_HASHES = ("sha1", "md5", "sha256", "crc32") def _declared_hashes(config: dict, detail: dict) -> dict[str, list[str]]: """The hashes the platform declares for a reported file, by algorithm. The untested table said "declared hash contradicted by the local file" and named neither: the reader could not look for the dump RetroDECK asks for, nor report the difference. """ from hashing import parse_hash_list # noqa: PLC0415 system = (config.get("systems") or {}).get(detail.get("system", "")) or {} for entry in system.get("files") or []: if isinstance(entry, dict) and entry.get("name") == detail.get("name"): return { algo: parse_hash_list(entry.get(algo)) for algo in _PROBLEM_HASHES if parse_hash_list(entry.get(algo)) } return {} def short_hash(value: str) -> str: """A hash in a table cell: its first 12 characters, the whole value in the title. Full MD5 and SHA1 values pushed the gaps tables past the column on every screen.""" value = str(value) if len(value) <= 12: return _code(value) return f'{html.escape(value[:12])}' def _hash_cell(hashes: dict[str, list[str]]) -> str: return "
".join( f"{algo} " + ", ".join(short_hash(v) for v in values) for algo, values in hashes.items() ) def _reason_cell(text: str) -> str: """A reason in a gaps table, in the class that keeps its column readable.""" return f'{text}' if text else "-" def _copy_cell(path: str, algos: list[str], db: dict | None) -> str: """The file a path holds, with its hashes in the algorithms declared.""" if not path: return "-" record = next( (rec | {"sha1": sha1} for sha1, rec in (db or {}).get("files", {}).items() if rec.get("path") == path), {}, ) hashes = {algo: [str(record[algo]).lower()] for algo in algos if record.get(algo)} shown = f"[{_code(path)}]({_fetch_url(path)})" return f"{shown}
{_hash_cell(hashes)}" if hashes else shown def _render_problem_files( platform_problems: list, settled: list | None = None, db: dict | None = None ) -> list[str]: """Files a platform declares that its own check then rejects. Split out of generate_gap_analysis, which reached complexity 56. This section is the one that stands alone: it reads platform_problems and nothing else, where the sections around it share gap_report and the resolved core list. """ lines: list[str] = [] # ---- Section 2: Problem files ---- missing_files: dict[str, dict] = {} untested_files: dict[str, dict] = {} mismatch_files: dict[str, dict] = {} for p in platform_problems: fname = p["name"] if p["status"] == "missing": entry = missing_files.setdefault(fname, { "name": fname, "required": p["required"], "platforms": [], "reason": p["reason"], "categories": set(), }) entry["platforms"].append(p["platform"]) entry["categories"].add(p.get("category", "bios")) if p["required"]: entry["required"] = True elif p["status"] == "untested": # Keyed by what is declared and what is held: two platforms # asking two dumps under one name are two rows. declared = p.get("declared") or {} key = f"{fname}\0{sorted(declared.items())}\0{p.get('path', '')}" entry = untested_files.setdefault(key, { "name": fname, "required": p["required"], "platforms": [], "reason": p["reason"], "declared": declared, "path": p.get("path", ""), }) entry["platforms"].append(p["platform"]) if p.get("discrepancy"): entry = mismatch_files.setdefault(fname, { "name": fname, "platforms": [], "discrepancy": p["discrepancy"], }) entry["platforms"].append(p["platform"]) total_problems = len(missing_files) + len(untested_files) + len(mismatch_files) if total_problems > 0: lines.extend([ "## Problem files", "", f"{len(missing_files)} missing, {len(untested_files)} untested, " f"{len(mismatch_files)} discrepancy.", "", ]) if missing_files: lines.extend([ f'### Missing ' f"{_count(len(missing_files), 'file')}", "", "Required is the platform's own flag. Category says what the " "platform's emulators do with the file: `bios` is firmware they " "load, `game_data` comes from a copy of the game, `unread` is " "opened by none of them. Only a missing `bios` entry stops a " "console from starting.", "", "| File | Category | Required | Platforms |", "|------|----------|----------|-----------|", ]) for fname in sorted(missing_files): f = missing_files[fname] req = "yes" if f["required"] else "no" plats = ", ".join(sorted(set(f["platforms"]))) cats = f["categories"] category = "bios" if "bios" in cats else ", ".join(sorted(cats)) lines.append(f"| `{fname}` | {category} | {req} | {plats} |") lines.append("") if untested_files: lines.extend([ f'### Untested ' f"{_count(len(untested_files), 'file')}", "", "Present, but not the expected hash. Declared is what the platform " "asks for, the collection copy is what the path holds.", "", "| File | Platforms | Reason | Declared | Collection copy |", "|------|----------|--------|----------|-----------------|", ]) for key in sorted(untested_files): f = untested_files[key] plats = ", ".join(sorted(set(f["platforms"]))) declared = f["declared"] lines.append( f"| {_code(f['name'])} | {plats} | {_reason_cell(f['reason'])} " f"| {_hash_cell(declared) or '-'} " f"| {_copy_cell(f['path'], list(declared) or ['md5'], db)} |" ) lines.append("") if mismatch_files: lines.extend([ f'### Discrepancy ' f"{_count(len(mismatch_files), 'file')}", "", "The platform's own check passes, but the emulator's size or " "hash check fails. Each row names the check that fails.", "", "| File | Platforms | Discrepancy | Accepted dump in the collection |", "|------|----------|-------------|-------------------|", ]) for fname in sorted(mismatch_files): f = mismatch_files[fname] plats = ", ".join(sorted(set(f["platforms"]))) held = _held_accepted(f["discrepancy"], db) cell = f"[{_code(held)}]({_fetch_url(held)})" if held else "-" lines.append(f"| `{fname}` | {plats} | {_reason_cell(f['discrepancy'])} | {cell} |") lines.append("") if settled: lines.extend([ "### Other revisions of a held destination", "", "A platform can declare one path twice with different hashes. The " "path holds the file one declaration accepts, so it is counted once, " "as held. The other declaration cannot be met at the same time.", "", "| File | Platform | Status | Path held for system |", "|------|----------|--------|--------------|", ]) for p in sorted(settled, key=lambda row: (row["name"], row["platform"])): lines.append( f"| {_code(p['name'])} | {_escape_tags(p['platform'])} | {p['status']} " f"| {_code(p['held_by'])} |" ) lines.append("") return lines def _platform_lists_sentence( total: int, verified: int, untested: int, missing: int, not_firmware: dict[str, int] | None = None, ) -> str: """The platforms' own lists in one sentence, every count said. Four tiles opened the page: "Verified (100%)" for 5,437 of 5,438 and "Missing 0", above a table of red cells, on the page about what is missing. The gaps are in the files the emulators load, and the sentence says so. The missing count then read as missing firmware, while every entry behind it was game data or a file no emulator opens: the split by category is said with it. """ def _is(count: int) -> str: return "is" if count == 1 else "are" not_firmware = {k: v for k, v in (not_firmware or {}).items() if v} parts = [f"{verified:,} {_is(verified)} held and pass the platform's own check"] if untested: parts.append(f"{untested:,} {_is(untested)} held but not verified " "(listed under Untested below)") parts.append(f"{missing:,} {_is(missing)} missing") listed = ", ".join(parts[:-1]) + f" and {parts[-1]}" split = "" if missing and not_firmware: labels = {"game_data": "game data", "unread": "opened by none of the platform's emulators"} firmware = missing - sum(not_firmware.values()) others = [ f"{count:,} {'is' if count == 1 else 'are'} {labels.get(category, category)}" for category, count in sorted(not_firmware.items()) ] others_text = others[0] if len(others) == 1 else ", ".join(others[:-1]) + f" and {others[-1]}" head = (f"None of the missing is firmware: {others_text}" if not firmware else f"Of the missing, {firmware:,} {'is' if firmware == 1 else 'are'} " f"firmware, {others_text}") split = f" {head}. Only missing firmware stops a console from starting." return ( f"Of the {total:,} files on the platforms' own BIOS lists, {listed}.{split} " "The files the emulators load beyond those lists are counted per " "platform in the table below." ) def _missing_required_firmware(cov: dict) -> int: """Entries of a platform's list that are missing, required and firmware. Only those stop a console from starting. Game data and files no core opens are counted apart, and red on every short row carried no signal. """ return len({ (d.get("system", ""), d.get("name", "")) for d in cov.get("details") or [] if d.get("status") == "missing" and d.get("required", True) and d.get("category", "bios") == "bios" and not d.get("settled_by") }) def _ratio_cell(done: int, total: int, alarm: int = 0) -> str: """A held-over-needed cell: the numbers, a bar, and a badge only when required firmware is missing.""" if not total: return "-" share = done / total * 100 cell = (f'' f"{done:,}/{total:,}") if alarm: cell += f' {alarm:,} required missing' return cell # GitHub's upload page for a folder of bios/: a browser upload by someone # without write access lands on a fork and opens a pull request. UPLOAD_URL = f"{REPO_URL}/upload/main/bios" # Rows of Files to source on one page: 601 on one page made a phone page # 274,509 px long. FILES_TO_SOURCE_PART = 100 def _upload_folder(profile: dict) -> str: """The bios/ folder a file the emulator loads is uploaded into: the console folder of its first system that has one, else none.""" from siterender import _own_section for sys_id in profile.get("systems") or []: own = _own_section(str(sys_id)) if own is not None: return "/".join(own) return "" def generate_gap_analysis( profiles: dict, coverages: dict, db: dict, data_names: set[str] | None = None, registry: dict | None = None, gap_report: dict | None = None, moved: dict[str, str] | None = None, ) -> str: """Generate a unified gap analysis page. With ``moved``, the list of files to source goes to pages of its own, keyed by stem: on the page it made 662 KB of HTML. Combines verification results (from coverages/verify.py) with source provenance (from cross_reference) into a single truth dashboard. Sections: 1. Verification status, aggregated across all platforms 2. Problem files: missing, untested, hash mismatch 3. Core complement: emulator files not declared by any platform """ from common import resolve_platform_cores # ---- Section 1: aggregate verify results across all platforms ---- total_verified = 0 total_untested = 0 total_missing_verify = 0 total_files_verify = 0 not_firmware_totals: dict[str, int] = {} platform_problems: list[dict] = [] settled_problems: list[dict] = [] for pname, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): total_verified += cov["verified"] total_untested += cov["untested"] total_missing_verify += cov["missing"] total_files_verify += cov["total"] for category, count in (cov.get("missing_not_firmware") or {}).items(): not_firmware_totals[category] = not_firmware_totals.get(category, 0) + count for d in cov["details"]: if d.get("settled_by") and d["status"] != "ok": # Its path holds another declaration's file: the headline, # counted by destination, rightly leaves it out, and listing # it under the same word made the page contradict itself. settled_problems.append({ "platform": cov["platform"], "name": d["name"], "status": d["status"], "held_by": d["settled_by"], }) continue if d["status"] != "ok" or d.get("discrepancy"): platform_problems.append({ "platform": cov["platform"], "platform_key": pname, "name": d["name"], "status": d["status"], "required": d.get("required", True), "category": d.get("category", "bios"), "reason": d.get("reason", ""), "discrepancy": d.get("discrepancy", ""), "system": d.get("system", ""), "declared": _declared_hashes(cov.get("config") or {}, d), "path": d.get("path") or "", }) lines = [ "# What the packs do not hold", "", "What each pack still cannot give you, and why. Most gaps are not " "firmware: a platform's list can also name game files, such as " "Diablo's `diabdat.mpq`, which you copy from your own game, or files " "no emulator of that platform opens. Those are counted apart. A missing " "firmware file means a console that does not start, and each one is " "named below with the platforms that list it.", "", "Each file here is a request: if you hold one of the files under " "\"Files to source\", [adding it](contributing.md) fills the gap for " "every platform at once.", "", " ".join([ markup.button("Download gaps CSV", "downloads/gaps.csv"), markup.button("Open gaps API", "api/v1/gaps.json"), markup.button("All data exports", "data.md"), ]), "", _platform_lists_sentence( total_files_verify, total_verified, total_untested, total_missing_verify, not_firmware_totals, ), "", ] # ---- Verification per platform ---- lines.extend([ "## What each pack contains", "", "| Platform | On its BIOS list (destinations) | Files its emulators load | Checked by |", "|----------|-----------------:|-------------------------:|------------|", ]) mode_labels = { "md5": "MD5 hash", "sha1": "SHA1 hash", "existence": "file presence", } for pname, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): display = cov["platform"] core_total = ( cov["core_present"] + cov["core_missing"] + cov["core_unsourceable"] ) lines.append( f"| [{display}](platforms/{pname}.md) " f"| {_ratio_cell(cov['present'], cov['total'], _missing_required_firmware(cov))} " f"| {_ratio_cell(cov['core_present'], core_total)} " f"| {mode_labels.get(cov['mode'], cov['mode'])} |" ) lines.extend([ "", "Each fraction is what the pack has over what is needed, counting " "required and optional files alike since both ship. The first column " "is the BIOS list the platform publishes, counted by destination: a " "path the list declares twice is one file to place. A red badge marks " "a list that is short of required firmware. The second counts files its " "emulators load that this list never mentions, found by reading their " "source code, and it is often several times larger. What is short " "is named in the sections below. That second " "number is a floor, not a ceiling: an emulator that accepts any file " "handed to it names none in its code, so nothing there can be counted. " "Checked by is the test the platform runs on its own, replicated here " "from its source code " "([how each one works](wiki/verification-modes.md)).", "", "## How much of each list the emulator code confirms { #corroboration-against-emulator-source }", "", "| Platform | On its BIOS list | Documented in a profile | Content the code checks |", "|----------|-----------------:|------------------------:|------------------------:|", ]) for pname, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): gt = cov["ground_truth"] if not gt.get("applicable", True): prof_cell = gt_cell = "-" elif gt["total"]: prof_pct = f"{gt.get('with_profile', 0) / gt['total'] * 100:.0f}%" prof_cell = f"{gt.get('with_profile', 0)} ({prof_pct})" gt_pct = f"{gt['with_validation'] / gt['total'] * 100:.0f}%" gt_cell = f"{gt['with_validation']} ({gt_pct})" else: prof_cell = gt_cell = "0" lines.append( f"| [{cov['platform']}](platforms/{pname}.md) " f"| {gt['total']:,} | {prof_cell} | {gt_cell} |" ) lines.extend([ "", "Both columns count the files on the platform's own BIOS list, the " "same number as the first column of the table above. Documented in a " "profile means an emulator profile, written from source, describes the " "file. Content the code checks is stricter: the emulator verifies a " "size or hash for it, and this tool repeats that check. The gap " "between the two is " "not a defect, it is what the emulator code does: many emulators load " "a file without ever checking its content, and no amount of profiling " "can invent a check the code does not perform. A dash means no " "profiled emulator applies to the platform, whose own source is then " "the only authority.", "", _content_check_ceiling(profiles), "", ]) # ---- Section 1b: platform lists vs emulator source ---- from truth import diff_platform_truth, generate_platform_truth div_rows = [] for pname, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): truth_data = generate_platform_truth( pname, cov["config"], (registry or {}).get(pname, {}), profiles, db ) s = diff_platform_truth(truth_data, native_layer(cov["config"]))["summary"] if s["systems_compared"] == 0: div_rows.append( f"| [{cov['platform']}](platforms/{pname}.md) | - | - | - | - |" ) else: div_rows.append( f"| [{cov['platform']}](platforms/{pname}.md) " f"| {s['total_missing']} " f"| {s['total_extra_phantom'] + s['total_extra_unprofiled']} " f"| {s['total_hash_mismatch']} " f"| {s['total_required_mismatch']} |" ) lines.extend([ "## Platform lists vs emulator source", "", "Platform file lists are scraped as-is from each upstream project. " "Emulator profiles are read from source code. The two do not always " "agree, and this table counts the differences. Packs follow the " "platform contract, and these numbers show where that contract diverges " "from what the code loads.", "", "| Platform | Missing from list | Not read by profiles | Hash conflict | Required status |", "|----------|------------------:|--------:|--------------:|----------------:|", *div_rows, "", "- **Missing from list**: a profiled emulator loads the file, the platform list does not mention it", "- **Not read by profiles**: on the platform list, loaded by no emulator profiled so far (a profile may still be incomplete)", "- **Hash conflict**: the platform list and the emulator source expect different hashes", "- **Required status**: required/optional differs between list and code", "", "A dash means no profiled emulator overlaps the platform's systems, " "so there is nothing to compare: the platform's own source is the " "only authority for its files.", "", "The same comparison drives `scripts/exporter/`: each platform's " "own file can be regenerated in its native format with these " "differences applied (System.dat, es_bios.xml, " "batocera-systems.json, ...).", "", ]) lines.extend(_render_problem_files(platform_problems, settled_problems, db)) # ---- Section 3: Core complement (cross-reference provenance) ---- unique_profiles = { k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test") } relevant_set: set[str] = set() for _name, cov in coverages.items(): matched = resolve_platform_cores(cov["config"], unique_profiles) relevant_set.update(matched) if gap_report is None: gap_report = build_emulator_gap_report(profiles, coverages, db, data_names) report_all = gap_report src_totals: dict[str, int] = { "bios": 0, "data": 0, "large_file": 0, "bundled": 0, "missing": 0, } total_undeclared = 0 emulator_gaps = [] for emu_name, data in sorted(report_all.items()): if data["gaps"] == 0: continue total_undeclared += data["gaps"] for key, count in _gap_sources(data).items(): src_totals[key] += count emulator_gaps.append((emu_name, data)) if total_undeclared > 0: total_available = ( src_totals["bios"] + src_totals["data"] + src_totals["large_file"] ) pct_available = ( f"{total_available / total_undeclared * 100:.0f}%" if total_undeclared else "0%" ) lines.extend([ "## Files emulators load that no platform lists { #core-complement }", "", f"Files loaded by emulators but not declared by any platform. " f"{total_undeclared:,} files across {_count(len(emulator_gaps), 'emulator')}, " f"{total_available:,} available ({pct_available}), " f"{src_totals['bundled']:,} installed with their emulator, " f"{src_totals['missing']:,} to source.", "", "This counts every profiled emulator, including those no platform " "ships yet, so it reaches past what the pack tables above measure: " "those cover only the emulators each platform actually ships. " "Whatever is not in the collection is an acquisition target, named " "per emulator below.", "", "### Where the files come from", "", "| Source | Count | Description |", "|--------|------:|-------------|", f"| bios/ | {src_totals['bios']} | In repository (database.json) |", f"| data/ | {src_totals['data']} | Data directories (buildbot, GitHub) |", f"| release | {src_totals['large_file']} " "| GitHub release assets (large files) |", f"| bundled | {src_totals['bundled']} " "| Installed with the emulator's own package, not collected |", f"| missing | {src_totals['missing']} | Not available, needs sourcing |", "", "### Per emulator", "", "| Emulator | Undeclared | bios | data | release | Bundled | Missing |", "|----------|----------:|-----:|-----:|--------:|--------:|--------:|", ]) for emu_name, data in sorted(emulator_gaps, key=lambda x: -x[1]["gaps"]): display = data["emulator"] sources = _gap_sources(data) m = sources["missing"] missing_str = ( f'{m}' if m > 0 else '0' ) lines.append( f"| [{display}](emulators/{emu_name}.md) " f"| {data['gaps']} " f"| {sources['bios']} " f"| {sources['data']} " f"| {sources['large_file']} " f"| {sources['bundled']} " f"| {missing_str} |" ) lines.append("") # List truly missing files with platform impact emu_to_platforms: dict[str, set[str]] = {} unique_profiles = { k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test") } for pname in coverages: config = coverages[pname]["config"] matched = resolve_platform_cores(config, unique_profiles) for emu_name in matched: emu_to_platforms.setdefault(emu_name, set()).add(pname) # One row per file to find: a name two emulators share is one file # only when both declare the same hash. Keyed by name alone, Fallout # 2's master.dat vanished behind Fallout 1's, and the three Sonic # decompilations' Data.rsdk showed as one. src_missing_details: list[dict] = [] by_identity: dict[tuple[str, str], dict] = {} entries = 0 bundled_rows: set[tuple[str, str]] = set() for emu_name, data in emulator_gaps: for g in data["gap_details"]: if g["source"] != "missing": continue if g.get("bundled"): # It arrives with the emulator's own package, which is # why the collection documents it and does not hold it: # asking readers to source it sent them after a file # every install already has. bundled_rows.add((data["emulator"], g["name"])) continue entries += 1 key = (g["name"], g.get("identity") or f"emulator:{emu_name}") row = by_identity.get(key) if row is None: row = by_identity[key] = { "name": g["name"], "path": g.get("path") or g["name"], "identity": g.get("identity", ""), "emulators": [data["emulator"]], "emu_keys": [emu_name], "required": g["required"], "source_ref": g["source_ref"], } src_missing_details.append(row) elif emu_name not in row["emu_keys"]: row["emulators"].append(data["emulator"]) row["emu_keys"].append(emu_name) row["required"] = row["required"] or g["required"] if src_missing_details: req_src = [m for m in src_missing_details if m["required"]] counted = ( f"{_count(len(src_missing_details), 'file')}, {len(req_src)} required." + ( f" They come from {_count(entries, 'entry', 'entries')}: a file " "declared with the same hash at several places is listed once." if entries != len(src_missing_details) else "" ) ) head = [ "| File | Destination | Expected | Emulator | Required | Affects platforms | Source ref |", "|------|-------------|----------|----------|----------|------------------|-----------|", ] rows: list[str] = [] for m in sorted( src_missing_details, key=lambda x: (not x["required"], x["name"]), ): plats = sorted(set().union( *(emu_to_platforms.get(key, set()) for key in m["emu_keys"]) )) plat_badges = ( " ".join( f'{p}' for p in plats ) if plats else "-" ) req = "yes" if m["required"] else "no" algorithm, _, value = m["identity"].partition(":") expected = f"{algorithm} {short_hash(value)}" if value else "-" destination = _code(m["path"]) if m["path"] != m["name"] else "-" emulators = ", ".join( f"[{_escape_tags(display)}](emulators/{key}.md)" for display, key in zip(m["emulators"], m["emu_keys"]) ) folder = _upload_folder(profiles[m["emu_keys"][0]]) upload = f"{UPLOAD_URL}/{urllib.parse.quote(folder)}" if folder else UPLOAD_URL rows.append( f"| `{m['name']}` [Have it? Upload]({upload}) | {destination} | {expected} " f"| {emulators} | {req} | {plat_badges} | " f"{_source_ref_markdown(profiles[m['emu_keys'][0]], m['source_ref'])} |" ) lines.extend(["### Files to source", "", counted, ""]) if moved is None: lines.extend([*head, *rows, ""]) else: where = _list_pages( "gaps-files-to-source", "Files to source", f"{counted} Each is a file an emulator loads that no platform lists and " "the collection does not hold. Part of " "[What the packs do not hold](gaps.md#files-to-source).", head, [rows[i:i + FILES_TO_SOURCE_PART] for i in range( 0, len(rows), FILES_TO_SOURCE_PART)], moved, budget=1, parent=("What the packs do not hold", "/gaps/"), ) lines.extend([f"Listed on {where}.", ""]) if bundled_rows: emulators = sorted({emulator for emulator, _name in bundled_rows}) lines.extend([ f"{len(bundled_rows)} further files arrive with their emulator's own " f"package ({_escape_tags(', '.join(emulators))}): documented in the profile, not " "collected, and not listed here.", "", ]) # ---- Section 4: Acknowledged gaps (unsourceable files) ---- all_unsourceable: list[dict] = [] for emu_name, data in sorted(report_all.items()): for u in data.get("unsourceable", []): all_unsourceable.append({ "name": u["name"], "emulator": data["emulator"], "emu_key": emu_name, "reason": u["reason"], "source_ref": u.get("source_ref", ""), }) if all_unsourceable: lines.extend([ "## Acknowledged gaps", "", f"{len(all_unsourceable)} files documented as unsourceable " "(verified from source code).", "", "| File | Emulator | Reason | Source ref |", "|------|----------|--------|-----------|", ]) for u in sorted(all_unsourceable, key=lambda x: x["name"]): lines.append( f"| `{u['name']}` | {u['emulator']} | {_reason_cell(u['reason'])} " f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} |" ) lines.append("") lines.extend(["", _date_line()]) return "\n".join(lines) + "\n" def _core_scope_note(config: dict, profiles: dict) -> str: """" (all libretro)" when the platform resolves every libretro core. Read from what resolve_platform_cores returns, not from the shape of ``cores:``: RetroPie and RetroDECK name ``retroarch`` in a list, which pulls in every libretro core as ``all_libretro`` does, and the label showed for RetroArch and Lakka only. """ every_libretro = resolve_platform_cores({"cores": "all_libretro"}, profiles) resolved = resolve_platform_cores(config, profiles) if not every_libretro or not every_libretro <= resolved: return "" others = len(resolved - every_libretro) if not others: return " (all libretro)" return f" (all libretro, plus {_count(others, 'other core')})" def _gap_sources(data: dict) -> dict[str, int]: """Where each undeclared file of one emulator comes from. A file the emulator's own package installs is not one to find. The report's gap_missing counts it, and the gaps page announced 1,356 files to source where 609 are, with a red badge on emulators that have none. """ counts = {"bios": 0, "data": 0, "large_file": 0, "bundled": 0, "missing": 0} for gap in data.get("gap_details") or []: source = gap.get("source", "") if source == "missing" and gap.get("bundled"): source = "bundled" if source in counts: counts[source] += 1 return counts def _xref_upstream(profile: dict) -> str: """The project a core comes from: its upstream, else its own source.""" raw = profile.get("upstream") or profile.get("source", "") return raw if isinstance(raw, str) else "" _FORGES = frozenset({ "github.com", "gitlab.com", "codeberg.org", "bitbucket.org", "git.eden-emu.dev", "git.ryujinx.app", }) def upstream_project(url: str) -> tuple[str, str]: """(project key, revision or path inside it) for an upstream URL. MAME was twelve rows: mamedev/mame and one row per release tree (mamedev/mame/tree/mame0289 and the rest). On a forge the project is owner/repo, a GitLab project every group before `/-/`, and the rest is a revision. Elsewhere the address without its scheme, `www.` and trailing slash is the project. A note written after the address (`... (sys/src/games/nes/)`) is a path inside it. """ url, _, note = url.strip().partition(" ") note = note.strip().strip("()") if not url.startswith(("http://", "https://")): return " ".join(filter(None, (url, note))), "" parsed = urllib.parse.urlsplit(url) host = parsed.netloc.lower().removeprefix("www.") parts = [p for p in parsed.path.split("/") if p] if host == "gitlab.com" and len(parts) >= 2: cut = parts.index("-") if "-" in parts else len(parts) rest = parts[cut + 1:] if rest[:1] in (["tree"], ["blob"]): rest = rest[1:] return f"{host}/{'/'.join(parts[:cut])}", "/".join(filter(None, ("/".join(rest), note))) if host in _FORGES and len(parts) >= 2: repo = parts[1].removesuffix(".git") rest = parts[2:] if rest[:1] in (["tree"], ["blob"], ["src"]): rest = rest[1:] return f"{host}/{parts[0]}/{repo}", "/".join(filter(None, ("/".join(rest), note))) return "/".join([host, *parts]), note def _upstream_label(key: str) -> str: """owner/repo on GitHub, host/owner/repo on another forge, and a Software Heritage revision by its short id.""" swh = re.fullmatch(r"archive\.softwareheritage\.org/browse/revision/([0-9a-f]{12})[0-9a-f]*", key) if swh: return f"Software Heritage revision {swh[1]}" return key.removeprefix("github.com/") def _breakable(label: str) -> str: """A project address that may wrap after each slash: unbroken, the longest pushed three of five columns off a 1,440 px screen.""" return "/".join(_escape_tags(part) for part in label.split("/")) def _xref_platform_rows(config: dict, unique: dict) -> tuple[int, list[str]]: """The cores a platform pulls in, as table rows, with their count.""" # The cores the builder and verify resolve: a hand copy here dropped # the libretro set a list naming retroarch pulls in, and the system # fallback EmuDeck relies on. matched = sorted(key for key in resolve_platform_cores(config, unique) if key in unique) rows = [] for emu_name in matched: p = unique[emu_name] cls_raw = p.get("core_classification", "-") systems = p.get("systems", []) or [] sys_str = ", ".join(_escape_tags(system_name(s)) for s in systems[:3]) if len(systems) > 3: sys_str += f" +{len(systems) - 3}" files = p.get("files", []) or [] libretro_only = sum(1 for f in files if f.get("mode") == "libretro") standalone_only = sum(1 for f in files if f.get("mode") == "standalone") file_str = str(len(files)) parts = [f"{n} {label}" for n, label in ((libretro_only, "libretro-only"), (standalone_only, "standalone-only")) if n] if parts: file_str += f" ({', '.join(parts)})" upstream = _xref_upstream(p) key, _rest = upstream_project(upstream) upstream_display = (f"[{_breakable(_upstream_label(key))}]({upstream.split()[0]})" if upstream.startswith("http") else (_escape_tags(upstream) or "-")) rows.append( f"| [{_escape_tags(p.get('emulator', emu_name))}](emulators/{emu_name}.md) " f"| {CLS_LABEL_ONE.get(cls_raw, cls_raw)} | {sys_str or '-'} | {file_str} " f"| {upstream_display} |" ) return len(matched), rows # What a platform that declares no core (cores: []) is told, on its page # and in the cross-reference. OWN_LIST_ONLY = "None. Its own list is the whole truth: no core profile adds a file." def _own_list_only(config: dict) -> bool: """The platform declares `cores: []`: its own list decides alone.""" return isinstance(config.get("cores"), list) and not config["cores"] def _render_xref_by_platform( coverages: dict, unique: dict, moved: dict[str, str] | None = None, ) -> list[str]: """Which cores each platform pulls in: counts here, a page per platform. One page held every platform's table, 30,551 nodes: the overview keeps the counts and each platform's table moves to ``moved`` by stem. """ lines = [ "## By platform", "", "| Platform | Cores | Core list |", "|----------|------:|-----------|", ] for pname in sorted(coverages, key=lambda x: coverages[x]["platform"]): cov = coverages[pname] display = cov["platform"] config = cov["config"] count, rows = _xref_platform_rows(config, unique) stem = f"cross-reference-{pname}" if _own_list_only(config): # cores: [] (MiSTer, BizHawk): no profile describes what they # read, and a page of cores would be an empty table. lines.append(f"| [{_escape_tags(display)}](platforms/{pname}.md) | 0 " f"| {OWN_LIST_ONLY} |") continue lines.append( f"| [{_escape_tags(display)}](platforms/{pname}.md) | {count}" f"{_core_scope_note(config, unique)} | [Cores on {_escape_tags(display)}]({stem}.md) |" ) if moved is not None: moved[stem] = "\n".join([ "---", "level: reference", *_parent_front_matter(("Cores by platform", "/cross-reference/")), "---", "", f"# Cores on {display}", "", f"{_count(count, 'core')}{_core_scope_note(config, unique)} that " f"{display} runs, the systems each emulates and the project it comes " f"from. Part of [Cores by platform](cross-reference.md). The files " f"are listed on the [{display} page](platforms/{pname}.md).", "", "| Core | Classification | Systems | Files | Upstream |", "|------|---------------|---------|-------|----------|", *rows, "", ]) + "\n" lines.append("") return lines def _render_xref_by_upstream( coverages: dict, unique: dict ) -> list[str]: """The same relation read the other way: one row per upstream project.""" lines: list[str] = [ "## By upstream emulator", "", "One row per project: the releases and folders of one repository are " "listed in the Revisions column. Platforms counts the platforms that run " "at least one of its cores.", "", "| Cores | Upstream | Revisions | Classification | Platforms |", "|-------|----------|-----------|----------------|----------:|", ] by_upstream: dict[str, list[tuple[str, str, str]]] = {} for emu_name, p in sorted(unique.items()): # A null upstream is the project's own source: grouping it under # the string "None" put eight unrelated engines in one row. up_str = _xref_upstream(p) if up_str: key, rest = upstream_project(up_str) by_upstream.setdefault(key, []).append((emu_name, up_str, rest)) platform_membership: dict[str, set[str]] = {} for pname, cov in coverages.items(): for key in resolve_platform_cores(cov["config"], unique): platform_membership.setdefault(key, set()).add(cov["platform"]) for key in sorted(by_upstream, key=lambda k: _upstream_label(k).casefold()): members = by_upstream[key] cores = sorted(c for c, _url, _rest in members) classifications = sorted({ CLS_LABEL_ONE.get(unique[c].get("core_classification", "-"), unique[c].get("core_classification", "-")) for c in cores }) plats = sorted(set().union(*(platform_membership.get(c, set()) for c in cores))) revisions = sorted({rest for _c, _url, rest in members if rest}) urls = sorted({url.split()[0] for _c, url, _rest in members}) # The project's own address: the bare one when a core cites it. home = next((u for u in urls if not upstream_project(u)[1]), urls[0]) upstream_cell = (f"[{_breakable(_upstream_label(key))}]({home})" if home.startswith("http") else _escape_tags(key)) lines.append( "| " + ", ".join(f"[{c}](emulators/{c}.md)" for c in cores) + f" | {upstream_cell} | {', '.join(_breakable(r) for r in revisions) or '-'} " f"| {', '.join(classifications)} | {len(plats)} |" ) return lines def generate_cross_reference( coverages: dict, profiles: dict, moved: dict[str, str] | None = None, ) -> str: """Cross-reference: counts per platform, then cores by upstream project. ``moved`` collects one page per platform with its core table. """ unique = { k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test") } total_cores = len(unique) total_upstreams = len({ upstream_project(_xref_upstream(p))[0] for p in unique.values() if _xref_upstream(p) }) lines = [ "# Cores by platform", "", "Every core each platform runs, the systems it emulates, and the project " "it comes from. A core built for RetroArch is often a port of another " "emulator, and the two can differ in the files they read and the checks " "they make, so each core names its upstream project. " f"{total_cores} cores across {len(coverages)} platforms come from " f"{total_upstreams} upstream projects.", "", "The files a platform's emulators load and its own list does not name are " "counted on [What the packs do not hold](gaps.md).", "", " ".join([ markup.button("Download cross-reference CSV", "downloads/cross-reference.csv"), markup.button("Open emulator API", "api/v1/emulators.json"), markup.button("All data exports", "data.md"), ]), "", ] lines.extend(_render_xref_by_platform(coverages, unique, moved)) lines.extend(_render_xref_by_upstream(coverages, unique)) lines.extend(["", _date_line()]) return "\n".join(lines) + "\n" def generate_contributing() -> str: return contributing_text( lambda page: f"{page}.md", "# Contributing", "on this site" ) + "\n" + "\n".join(_next_block([ "[Profiling an emulator](wiki/profiling.md), the first task most " "contributions start from", ])) # The end of a reference page: back to the index, or on to contributing. def generate_not_found() -> str: """The page a stale link lands on. Pages have moved before (file cards refiled between manufacturers), so it offers the search, the two pages a visitor most often wants, and the list a moved page belongs to.""" return "\n".join([ "---", "search: exclude", "---", "", "# Page not found", "", '

This address has no page. It may have moved since the link was written.

', "", '

', "", "- [Start here](index.md): what BIOS files are and how to install them", "- [Get your files](which-pack.md): the pack or the command for your device", "- Browse [platforms](platforms/index.md), [systems](systems/index.md) " "or [emulators](emulators/index.md)", "", '', "", ]) def reference_next() -> str: return "\n" + "\n".join(_next_block([ "[Reference index](wiki/index.md), every reference page in one list", "For contributors: [How to contribute](contributing.md)", ])) # Wiki pages # index, architecture, tools, profiling are maintained as wiki/ sources # and copied verbatim by main(). Only data-model is generated dynamically. def generate_wiki_data_model(db: dict, profiles: dict) -> str: """Generate data model documentation from actual database structure.""" files_count = len(db.get("files", {})) by_md5 = len(db.get("indexes", {}).get("by_md5", {})) by_name = len(db.get("indexes", {}).get("by_name", {})) by_crc32 = len(db.get("indexes", {}).get("by_crc32", {})) by_path = len(db.get("indexes", {}).get("by_path_suffix", {})) by_sha256 = len(db.get("indexes", {}).get("by_sha256", {})) # A real entry with real values: the PlayStation BIOS v3.0 (A). sample_key = "0555c6fae8906f3f09baf5988f00e55f88e9f30b" files = db.get("files", {}) sample = files.get(sample_key) or next(iter(files.values()), {}) lines = [ "# Data model", "", "Three kinds of file hold everything this project knows. `database.json` " "lists every file the collection holds, keyed by its SHA1. A platform YAML " "copies the BIOS list a setup publishes. An emulator profile records what " "one emulator opens, read from its code. The sections below show the " "fields of each, with a real entry, so you can read the data or build on it.", "", "Example: `scph5501.bin` is one entry of `database.json` (key " "`0555c6fa...`), one entry of `platforms/batocera.yml` (its MD5) and one " "entry of `emulators/duckstation.yml` (its MD5, region and priority).", "", "## database.json", "", f"Primary key: SHA1. **{files_count}** file entries.", "", f"The entry of `{sample.get('name', '')}`:", "", "```json", *json.dumps(sample, indent=2, ensure_ascii=False).splitlines(), "```", "", "`provenance` maps each catalog that lists the file to the DAT and entry " "it was matched against. It is present only when the file matches a " "snapshot under `provenance/`. The join runs by SHA1 first, then by " "MD5 + size. See [dump provenance](../provenance.md).", "", "### Indexes", "", "| Index | Entries | Purpose |", "|-------|---------|---------|", f"| `by_md5` | {by_md5} | MD5 to SHA1 lookup (Batocera, Recalbox verification) |", f"| `by_name` | {by_name} | filename to SHA1 list (name-based resolution) |", f"| `by_crc32` | {by_crc32} | CRC32 to SHA1 lookup |", f"| `by_path_suffix` | {by_path} | relative path to SHA1 (regional variant disambiguation) |", f"| `by_sha256` | {by_sha256} | SHA256 to SHA1 lookup (emulator profile validation) |", "", "### File resolution order", "", "`resolve_local_file` finds the file an entry describes by its hashes " "first, then by its path, then by its name, and a filename or " "destination never overrides a declared hash. The steps and their " "return codes are written once, in " "[the file resolution chain](verification-modes.md#file-resolution-chain).", "", "## Platform YAML", "", "Scraped from upstream sources. Structure:", "", "```yaml", "platform: Batocera", "verification_mode: md5 # how the platform checks files", "hash_type: md5 # hash type in file entries", "base_destination: bios # root directory for BIOS files", "systems:", " system-id:", " files:", " - name: filename", " destination: path/in/bios/dir", " md5: expected_hash", " sha1: expected_hash", " required: true", "```", "", "Supports inheritance (`inherits: retroarch`) and shared groups", "(`includes: [group_name]` referencing `_shared.yml`).", "", "`base_destination` is the prefix the pack applies to every entry. It is", "empty when the upstream destinations already carry their own root, which", "is why the RetroDECK pack ships `bios/` and `roms/` at its top level.", "Core files then go under `core_destination` when the platform states one", "(`bios` for AmberELEC and EmuELEC, whose lists also name `neogeo/` and", "`ports/`), else under the root nearly every destination shares.", "", "## Emulator YAML", "", f"**{len(profiles)}** profile files, **{len(unique_emulator_profiles(profiles))}** " "distinct emulators once aliases are folded in. Source-verified from " "emulator code.", "", "See the [profiling guide](profiling.md) for the full field reference.", "", "## Static API and bulk exports", "", "The website publishes versioned JSON, CSV and SQLite metadata generated " "from these same structures. Start with the [Data and API](../data.md) " "catalog. Each downloadable artifact carries a SHA256 in " "`api/v1/catalog.json`.", "", ] return "\n".join(lines) + "\n" # Build cross-reference indexes def _build_file_consumers(db: dict, coverages: dict, profiles: dict) -> FileConsumers: """Which platforms and emulators read each collected file.""" consumers = FileConsumers(db, build_zip_contents_index(db)) consumers.add_platforms({name: cov["config"] for name, cov in coverages.items()}) consumers.add_emulators(profiles) consumers.add_manifests(Path(__file__).resolve().parent.parent / "install") return consumers def _build_emulator_file_index(profiles: dict) -> dict[str, dict]: """Map emulator_name -> {systems, norm}, for the platform pages. norm holds the systems as _norm_system_id reads them: a platform files the Mega Drive as sega-mega-drive and genesis_plus_gx as sega-megadrive, and an exact match left the RetroArch page with no emulator for it. """ from common import _norm_system_id return { name: { "systems": set(profile.get("systems", [])), "norm": {_norm_system_id(str(s)) for s in profile.get("systems", [])}, # Archive members are read inside their archive, never by name. "names": { str(entry["name"]).lower() for entry in profile.get("files") or [] if isinstance(entry, dict) and entry.get("name") and not entry.get("archive") }, } for name, profile in profiles.items() if profile.get("type") != "alias" } def _declared_by_system(config_files: dict) -> dict[str, list[dict]]: """The platform's own entries of each system, hashes included.""" by_system: dict[str, list[dict]] = {} for (sys_id, _name), entries in config_files.items(): by_system.setdefault(sys_id, []).extend(entries) return by_system def _system_emulators( sys_id: str, emulator_files: dict | None, files: list[dict] | None = None ) -> list[str]: """The emulators a platform runs for one of its systems. A platform can group machines no profile names as one system: RetroArch files NAOMI and Atomiswave as sega-dreamcast-arcade, the Enterprise 64 and 128 as enterprise-64-128. When no system id matches, the emulators are the ones that read a file the system lists, by its name. A member of an archive does not count: fbneo's boot.bin is read inside bubsys.zip, and RomM's Dreamcast boot.bin is not that file. """ from common import _norm_system_id norm = _norm_system_id(sys_id) by_id = [name for name, data in (emulator_files or {}).items() if sys_id in data["systems"] or norm in data["norm"]] if by_id or not files: return by_id names = {str(f.get("name", "")).lower() for f in files} - {""} return [name for name, data in (emulator_files or {}).items() if names & data.get("names", set())] # Navigation _OS_NAMES = {"windows": "Windows", "darwin": "macOS", "linux": "Linux"} def _runs_on(entry: dict, guide: Guide) -> str: """Where a setup runs, read from the systems the installer detects it on.""" if guide.on_card: return "SD card, filled from a computer" if "server" in guide.run: return "A server" if "device" in guide.run: return "Its own system" systems = {rule.get("os") for rule in (entry.get("install") or {}).get("detect") or []} names = [label for key, label in _OS_NAMES.items() if key in systems] if "steamdeck" in guide.run: names.append("Steam Deck") if "android" in guide.run: names.append("Android") return ", ".join(names) def _folder_cell(name: str, guide: Guide, prefix: str = "") -> str: """The BIOS folder in a table cell, or a link where it depends on the system.""" if len(guide.folders) > 3: return f"Depends on the system: [listed on its page]({prefix}platforms/{name}.md#where-the-files-go)" return "
".join(f"{label}: {path}" for label, path in guide.folders) def _pack_cell(name: str, record: dict, prefix: str = "") -> str: """The pack's parts and size, or the command where the release has none.""" if not has_release_pack(name): return f"No ZIP. [{NO_PACK}]({prefix}platforms/{name}.md#install)." pack = release_record.pack_for(name, record) if record.get("tag") and not (pack and pack.get("assets")): return f"No ZIP yet. [{NO_PACK}]({prefix}platforms/{name}.md#install)." # One small button to the setup's get/ page, which lists every part, and # the size after it. link = download_links(name, record, "ZIP", page=get_page_link(name, prefix), button=lambda text, url: markup.button(text, url)) size = pack.get("download_size") if pack else None return f"{link} ({_fmt_size(size)})" if size else link def _group_keys(registry: dict, group: str, display: dict[str, str]) -> list[str]: return sorted( (k for k in registry if registry[k].get("device_group") == group), key=lambda k: display[k].casefold(), ) def _own_root_folders(registry: dict) -> dict[str, str]: """Platforms whose pack holds its own top folder, with where it unzips. The same rule as extract_notes: a list written from an empty base destination carries its root in every path.""" folders = {} for key, entry in registry.items(): config = load_platform_config(key, "platforms") if config.get("base_destination") == "" and entry.get("extract_to"): folders[key] = entry["extract_to"] return folders def _folder_row(key: str, display: dict[str, str], registry: dict, record: dict, own_root: dict[str, str]) -> str: """One setup's row: where it runs, its BIOS folder and its ZIP. A setup whose ZIP holds its own top folder says where to unzip it, and the BIOS folder itself when it has no ZIP to unzip.""" guide = GUIDES[key] pack = release_record.pack_for(key, record) zipped = bool(pack and pack.get("assets")) folder = (f"Unzip into `{own_root[key]}`" if key in own_root and zipped else _folder_cell(key, guide)) return (f"| [{display[key]}](platforms/{key}.md) | " f"{guide.runs_on or _runs_on(registry[key], guide)} | {folder} | " f"{_pack_cell(key, record)} |") def _folder_tables(registry: dict, record: dict, display: dict[str, str]) -> list[str]: """Every setup's folder and ZIP, one table per device group, the groups of one setup together under "Other setups" as on the tiles. A row for each setup that runs another (LaunchBox, Android) under its group.""" own_root = _own_root_folders(registry) header = ["| Setup | Runs on | BIOS folder | ZIP |", "|-------|---------|-------------|-----|"] lines: list[str] = [] alone: list[str] = [] for group, (label, _description) in DEVICE_GROUPS.items(): members = _group_keys(registry, group, display) if len(members) == 1 and not ALSO_RUNS.get(group): alone += members continue if not members: continue lines += [f'**{label}**', "", *header] lines += [_folder_row(key, display, registry, record, own_root) for key in members] lines += [ f"| {setup} | {runs_on} | {folder}, same as " f"[{display[uses]}](platforms/{uses}.md) | {_pack_cell(uses, record)} |" for setup, _summary, uses, runs_on, folder in ALSO_RUNS.get(group, ()) ] lines.append("") if alone: anchors = "".join(f'' for key in alone) lines += [f"{anchors}**Other setups**", "", *header, *(_folder_row(key, display, registry, record, own_root) for key in alone), ""] return lines # iPhone and iPad: no terminal, so each app is filled its own way. Every # step is the app's own: RetroArch's iOS guide (Files app, On My iPhone > # RetroArch) and platform_darwin.m (system under Documents/RetroArch), the # DS BIOS rows of Delta's Settings.storyboard, Provenance's import guide. IOS_SOURCES = { "retroarch_docs": "https://github.com/libretro/docs/blob/" "cfa4c1a17e586c01d3a48e9eda12b6c989f8954f/docs/guides/install-ios.md?plain=1#L94", "retroarch_code": "https://github.com/libretro/RetroArch/blob/" "5333a7c5a59e268470834a96f86047d3cb6139a9/frontend/drivers/platform_darwin.m#L438-L459", "delta": "https://github.com/rileytestut/Delta/blob/" "c1d3d068e019e6493eed45654569db3cc5beb86a/Delta/Base.lproj/Settings.storyboard#L2448-L2500", "provenance": "https://wiki.provenance-emu.com/using-provenance/importing-roms", } IOS_DEVICES = ("iPhone", "iPad", "iOS", "Delta", "Provenance") # The same names as which-pack's search terms, the section's meta field # alone leaving "ios" unanswered. IOS_TERMS = tuple(name.lower() for name in IOS_DEVICES) def ios_section() -> list[str]: """The section the iPhone and iPad tile opens: the consoles picked on a computer, then each app's own way in.""" return [ "## iPhone and iPad { #iphone-and-ipad }", "", devices_meta(IOS_DEVICES), "", "[Pick the consoles](which-pack.md?platform=retroarch#only-some-consoles) " "on a computer. Copy them to the app.", "", "- **RetroArch**: Files app, **On My iPhone > RetroArch**, then " f"`RetroArch/system/` ([docs]({IOS_SOURCES['retroarch_docs']}), " f"[code]({IOS_SOURCES['retroarch_code']})).", "- **Delta**, for DS games: import `bios7.bin`, `bios9.bin` and `firmware.bin` " f"in **Settings**, under **Nintendo DS** ([code]({IOS_SOURCES['delta']})).", "- **Provenance**: share the file, then **Copy to Provenance** " f"([docs]({IOS_SOURCES['provenance']})).", "", ] def generate_which_pack() -> str: """The device picker: every registered platform as a tile under the device it runs on, the ZIP advice with every folder and ZIP in one closed table, the console picker, and the command for a setup not listed. The tiles are the choice. Open tables under them listed every setup a second time and made the page 11,800 px tall on a phone.""" registry = load_platform_registry("platforms") record = release_record.load_advertised() display = {key: load_platform_config(key, "platforms").get("platform", key) for key in registry} all_tabs = ("windows", "macos", "linux", "steamdeck", "android") lines = [ "---", f"terms: {', '.join(IOS_TERMS)}", "---", "", "# Get your files", "", "Pick your setup. Its page gives the folder and the command.", "", "## Pick your setup", "", *setup_picker(registry, display), "## Folders and ZIP files { #manual-download }", "", "- A big ZIP comes in parts, like `Pack.part1of2.zip`. Download every part.", "- Unzip them into your BIOS folder. Put the files straight in it, " "not in a new folder.", "- Leave the ZIPs inside, like `neogeo.zip`, as they are.", "- A few ZIPs hold their own top folder. Their setup's page says where " "to unzip them.", "", *markup.details("note", "Every folder and ZIP in one table", _folder_tables(registry, record, display)), ] pending = any(has_release_pack(key) and not (pack and pack.get("assets")) for key, pack in ((key, release_record.pack_for(key, record)) for key in registry)) if record.get("tag") and pending: lines += ["Setups added after the last release have no ZIP until the next one.", ""] lines += ["No ZIP on a row: the command serves that setup. ZIPs are made for the " f"setups most people download. [Why?]({NO_ZIP_FAQ})", ""] lines += [ f"The ZIPs come from the [latest release]({RELEASE_URL}). The command may " "install newer files.", "", "## Only some consoles?", "", "Pick them here, and get the files and the command. Or run the command " "below and press **c** when it asks.", "", "", "", *ios_section(), "## Not in the list? One command finds it { #quick-install }", "", "A **terminal** is a window where you type commands.", "", *terminal_tabs(all_tabs, {}), "On Batocera, Recalbox and other game systems, run it on the device. " "Your setup's page shows how.", "", "The command asks what to install. Press **Enter** for everything.", "", "```text title=\"What it asks\" frame=\"terminal\"", " Enter) install everything", " i) only what the installed cores read", " c) choose systems, cores and regions", " q) quit", "```", "", *trust_box(""), "**Check your folder later.** Add `--check`. It changes nothing.", "", *terminal_tabs(all_tabs, dict.fromkeys(TERMINALS, "--check"), steps=False), *markup.details("note", "Fill an SD card from a computer", [ "Name your setup and the BIOS folder on the card. For RetroArch on a " "Nintendo Switch:", "", "Linux, macOS:", "", *fence("sh", install_command( "sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system")), "", "Windows (PowerShell, E: being the card):", "", *fence("powershell", install_command( "powershell", "--platform retroarch --dest E:\\retroarch\\cores\\system")), ]), *markup.details("note", "Copy the BIOS files from your old card", [ "Plug the old card in and add `--from` with its folder. The BIOS files " "found there are copied, and the rest downloaded. The old card is only read.", "", "Linux, macOS:", "", *fence("sh", install_command("sh", "--from /media/me/OLDCARD")), "", "Windows (PowerShell, E: being the old card):", "", *fence("powershell", install_command("powershell", "--from E:\\")), ]), "Details: [All installer options](wiki/installer.md).", "", *_next_block([ "[First install, step by step](wiki/first-install.md)", "[When a game does not start](wiki/troubleshooting.md)", ]), ] return "\n".join(lines) + "\n" def nav_emulators(profiles: dict) -> dict[str, dict]: """The emulators listed in the navigation, by key. A profile that emulates no system and reads no file (the libretro sample and test cores, games with nothing to load) answers no question about BIOS files. Its page stays, reachable from the overview table and the search, and the navigation is left to the cores people look up. """ return { key: profile for key, profile in unique_emulator_profiles(profiles).items() if profile.get("systems") or profile.get("files") } def _emulator_nav(profiles: dict) -> list: """Emulators by initial (A, B, C), one level deep. A name that does not open on a letter goes under "0 to 9". The route middleware draws every initial but the open one as a single link, so a page carries the links of one initial only.""" groups: dict[str, list[tuple[str, str]]] = {} for key, profile in nav_emulators(profiles).items(): display = str(profile.get("emulator", key)) groups.setdefault(emulator_initial(display), []).append((display, f"emulators/{key}.md")) nav: list = ["emulators/index.md"] for label in sorted(groups, key=lambda g: (g != INITIAL_DIGITS, g)): items = sorted(groups[label], key=lambda e: e[0].casefold()) nav.append({label: [{display: path} for display, path in items]}) return nav def _system_nav(manufacturers: dict) -> list: """Manufacturer pages, the machines apart from the arcade boards and the game and engine trees: an arcade page's crumb read "Consoles and computers".""" machines, arcade, games = [], [], [] for mfr in sorted(manufacturers, key=str.casefold): entry = {mfr: f"systems/{_slug(mfr)}.md"} if mfr in GAME_DATA_TOPS: games.append(entry) else: (arcade if mfr == "Arcade" else machines).append(entry) nav: list = ["systems/index.md", {"Consoles and computers": machines}] if arcade: nav.append({"Arcade": arcade}) if games: nav.append({"Game engines and PC games": games}) return nav def _platform_nav(coverages: dict, registry: dict) -> list: """Platforms by the kind of device they run on, as on the device picker.""" nav: list = ["platforms/index.md"] for group, (label, _description) in DEVICE_GROUPS.items(): members = [ {coverages[name]["platform"]: f"platforms/{name}.md"} for name in sorted(coverages, key=lambda n: coverages[n]["platform"].casefold()) if registry.get(name, {}).get("device_group") == group ] # The handheld guide sits with the group whose tile leads to it. members += [{HANDHELD_GUIDE.title: HANDHELD_GUIDE.path} for tile in GROUP_TILES.get(group, ()) if HANDHELD_GUIDE and tile.link == HANDHELD_GUIDE.path] if members: nav.append({label: members}) return nav # The tabs in reading order, by the level their pages declare. TABS = { "start": "Start here", "setup": "Your setup", "explore": "Explore", "reference": "Reference", "contribute": "Contribute", } class NavError(ValueError): """A wiki page does not say where it goes in the navigation.""" def _h1(markdown: str) -> str: match = re.search(r"^# (.+?)(?:\s*\{[^}]*\})?\s*$", markdown, re.M) return match.group(1).strip() if match else "" NavPlace = tuple[int, object, str] def wiki_nav(wiki_dir: Path | None = None) -> dict[str, list[NavPlace]]: """Wiki pages by tab, each at the place its front matter gives. A page states `level:` (its tab), `order:` (its place in the tab, the smallest first) and, when the navigation names it otherwise than its first heading, `nav_title:`. `group:` files it under a labelled group of its tab. `nav: false` keeps a page out on purpose. A page that says neither raises, naming what to add. """ wiki_dir = Path(wiki_dir) if wiki_dir else _REPO_ROOT / WIKI_SRC_DIR tabs: dict[str, list[NavPlace]] = {level: [] for level in TABS} for page in sorted(wiki_dir.glob("*.md")): where = f"{WIKI_SRC_DIR}/{page.name}" text = page.read_text(encoding="utf-8") meta = _front_matter(text) level = meta.get("level") if level not in TABS: raise NavError( f"{where}: front matter needs `level:`, one of {', '.join(TABS)}, " f"to place the page in a tab (found {level!r})" ) if meta.get("nav") is False: continue order = meta.get("order") if not isinstance(order, int) or isinstance(order, bool): raise NavError( f"{where}: front matter needs `order:`, a whole number giving its " f"place in the {TABS[level]} tab, smallest first (found {order!r}), " "or `nav: false` to keep it out of the navigation" ) path = f"{WIKI_SRC_DIR}/{page.name}" if page.name == "index.md": tabs[level].append((order, path, "")) continue title = meta.get("nav_title") or _h1(text) if not title: raise NavError(f"{where}: no `# ` heading and no `nav_title:` to name it") group = meta.get("group") or "" if not isinstance(group, str): raise NavError(f"{where}: `group:` names a group of the {TABS[level]} tab (found {group!r})") tabs[level].append((order, {str(title): path}, group)) return tabs def _overview_anchors(nav: list, docs: Path) -> dict[str, set[str]]: """The heading ids of each overview page the navigation names, where a group of the sidebar finds its own section.""" def overviews(value: object) -> Iterator[str]: if isinstance(value, str): yield value elif isinstance(value, list): for entry in value: yield from overviews(entry) elif isinstance(value, dict): for child in value.values(): if isinstance(child, list): yield from overviews(child) anchors: dict[str, set[str]] = {} for rel in overviews(nav): stem = rel.rsplit(".", 1)[0] page = next((docs / f"{stem}{suffix}" for suffix in (".md", ".mdx") if (docs / f"{stem}{suffix}").is_file()), None) if page: anchors[rel] = sitestarlight.heading_ids(page.read_text(encoding="utf-8")) return anchors def _nav_target(entry: object) -> str: return entry if isinstance(entry, str) else str(next(iter(entry.values()))) def generate_nav( coverages: dict, manufacturers: dict, profiles: dict, registry: dict | None = None, wiki_dir: Path | None = None, ) -> list: """The navigation, five tabs in reading order. Start here is for someone who knows nothing yet, Your setup for someone who knows their device, Explore for one system or emulator, Reference for checking the work, Contribute for adding to it. Each tab opens on its index page. The generated pages hold the places below. A wiki page takes its own place from its front matter (wiki_nav), so a new page needs no edit here. A group of a tab sits at the place of its first page, its pages in their own order. """ generated: dict[str, list[tuple]] = { "start": [(0, "index.md"), (20, {"Get your files": "which-pack.md"})], "setup": [(10 * i, item) for i, item in enumerate(_platform_nav(coverages, registry or {}))], "explore": [ (0, "explore.md"), (10, {"Systems": _system_nav(manufacturers)}), (20, {"Emulators": _emulator_nav(profiles)}), ], "reference": [ # Find a file first: the one tool a reader comes to with a file in hand. (5, {"Find or check a file": "files/index.md"}, "Checking files"), (20, {"What the packs do not hold": "gaps.md"}, "Checking files"), (30, {"Cores by platform": "cross-reference.md"}, "Checking files"), (40, {"Dump provenance": "provenance.md"}, "Checking files"), (60, {"Data and API": "data.md"}, "Data"), (64, {"Data model": "wiki/data-model.md"}, "Data"), (68, {"Updated platform files": "exports.md"}, "Data"), ], "contribute": [(0, "contributing.md")], } wiki = wiki_nav(wiki_dir) nav = [] for level, label in TABS.items(): places = [(place[0], place[1], place[2] if len(place) > 2 else "") for place in generated[level]] entries = sorted(places + wiki[level], key=lambda item: item[0]) for (first, a, _), (second, b, _) in zip(entries, entries[1:]): if first == second: raise NavError( f"{label} tab: {_nav_target(a)} and {_nav_target(b)} both take " f"order {first}; give one of them a free number" ) items: list = [] groups: dict[str, list] = {} for _order, entry, group in entries: if not group: items.append(entry) elif group in groups: groups[group].append(entry) else: groups[group] = [entry] items.append({group: groups[group]}) nav.append({label: items}) return nav # Main def main() -> None: parser = argparse.ArgumentParser( allow_abbrev=False, description="Generate the documentation site from project data" ) parser.add_argument( "--db", default="database.json", help="database file to read (default: %(default)s)", ) parser.add_argument( "--platforms-dir", default="platforms", help="platform YAML directory (default: %(default)s)", ) parser.add_argument( "--emulators-dir", default="emulators", help="emulator profile directory (default: %(default)s)", ) parser.add_argument( "--site-dir", default=STARLIGHT_DIR, help="the Astro Starlight project the site is written into (default: %(default)s)", ) parser.add_argument( "--docs-dir", default=None, help="directory the pages are written to (default: the project's " "content collection, src/content/docs)", ) args = parser.parse_args() if args.docs_dir is None: args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT) try: # Two runs over one docs/ delete each other's pages and scratch # files in the sweep, so the second is refused instead. import contextlib from common import load_data_dir_registry from refresh_data_dirs import hold_data_dirs # The coverage pages resolve against data/, held while they are read. with artifact_lock(args.docs_dir), contextlib.ExitStack() as held: hold_data_dirs(held, load_data_dir_registry(args.platforms_dir)) _generate(args) except ArtifactLockBusy as exc: print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) def _generate(args: argparse.Namespace) -> None: db = load_database(args.db) docs = Path(args.docs_dir) # Files the site serves as they are (data, downloads, images), in the # project's public folder. public = Path(args.site_dir) / "public" for d in GENERATED_DIRS: (public / d if d in PUBLIC_DIRS else docs / d).mkdir(parents=True, exist_ok=True) # No robots.txt: a crawler reads it only at the host root (RFC 9309), and # this project site lives under /retrobios/. Every page's head points at # the sitemap instead (Starlight's head links it). (public / "robots.txt").unlink(missing_ok=True) images_dest = public / "assets" / "images" images_dest.mkdir(parents=True, exist_ok=True) # Logos, system icons and sharing cards are served from the site itself # (docs_assets/*/sources.json records where each came from). # brand: the mark, favicons and touch icon (scripts/make_site_icons.py). folders = [("logos", "logos"), ("system-icons", "system-icons"), ("og", "og"), ("brand", "brand")] for folder, dest in folders: source = Path("docs_assets") / folder if source.is_dir(): shutil.copytree(source, public / "assets" / dest, dirs_exist_ok=True) assets_src = Path(".github") / "assets" for name, dest_name in [("logo.png", "logo.png"), ("favicon.png", "favicon.png")]: src = assets_src / name if src.exists(): shutil.copy2(src, images_dest / dest_name) registry_path = Path(args.platforms_dir) / "_registry.yml" registry = {} if registry_path.exists(): with open(registry_path) as f: registry = (yaml_load(f) or {}).get("platforms", {}) platform_names = list_registered_platforms( args.platforms_dir, include_archived=True ) from common import load_data_dir_registry from cross_reference import _build_supplemental_index data_registry = load_data_dir_registry(args.platforms_dir) suppl_names = _build_supplemental_index() print("Computing platform coverage...") coverages = platform_coverages( sorted(platform_names), args.platforms_dir, db, data_registry, emulators_dir=args.emulators_dir, db_path=args.db, report=lambda _name, cov: print( f" {cov['platform']}: {cov['present']}/{cov['total']} ({_pct(cov['present'], cov['total'])})" ), ) print("Loading emulator profiles...") profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False) unique_count = sum(1 for p in profiles.values() if p.get("type") != "alias") print( f" {len(profiles)} profiles ({unique_count} unique, {len(profiles) - unique_count} aliases)" ) consumers = _build_file_consumers(db, coverages, profiles) emulator_files = _build_emulator_file_index(profiles) print("Generating home page...") write_if_changed( str(docs / "index.md"), generate_home( db, coverages, profiles, registry, args.platforms_dir ) ) write_if_changed(str(docs / "explore.md"), generate_explore_index(db, coverages, profiles)) stats = compute_stats(db, coverages, profiles) stats["composition"] = compute_composition(db) write_if_changed(str(public / "stats.json"), generate_stats(stats)) # Computed once: the gap analysis page and the published gaps export must # report the same files. gap_report = build_emulator_gap_report(profiles, coverages, db, suppl_names) # Build system_id -> manufacturer page map (needed by all generators) plan_emulator_pages(profiles) print("Building system cross-reference map...") manufacturers = _group_by_manufacturer(db) set_system_page_map( _system_pages_from_data(manufacturers, coverages, db, profiles) ) print(f" {len(_system_page_map)} system IDs mapped to pages") set_console_sections(manufacturers, program_names(profiles)) from siterender import link_coverage, system_ids # noqa: PLC0415 print("Generating system pages...") prime_system_icons({ _icon_name(mfr, console) for mfr, consoles in manufacturers.items() for console in consoles }) # The arcade boards are recorded before any page links one, the # system pages first among them. reset_console_pages() sections, console_installs = arcade_board_sections( manufacturers, system_install_choices(coverages)) linked, total = link_coverage( system_ids(profiles, [cov["config"] for cov in coverages.values()])) print(f" {linked} of {total} system ids linked") write_if_changed( str(docs / "systems" / "index.md"), generate_systems_index(sections) ) (docs / "files").mkdir(parents=True, exist_ok=True) write_if_changed(str(docs / "files" / "index.md"), generate_files_page()) for mfr, consoles in sections.items(): slug = mfr.lower().replace(" ", "-") pages = system_pages(mfr, consoles, consumers, console_installs.get(slug)) for stem, (page, held) in pages.items(): PAGE_DATA[f"systems/{stem}.md"] = system_structured_data(stem, held) if stem != slug: PART_PAGES.add(f"systems/{stem}.md") write_if_changed(str(docs / "systems" / f"{stem}.md"), page) write_single_item( public / "api" / "v1" / "systems" / f"{slug}.json", "systems", # The published data keeps the collection's folders. system_export_item(slug, mfr, manufacturers[mfr]), str(db.get("generated_at") or _timestamp()), _source_commit(), ) clash = split_system_stems() & {_slug(m) for m in manufacturers} if clash: raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}") print("Generating static API and bulk data exports...") exports = generate_data_exports( public, db, coverages, profiles, stats, gap_report, consumers ) write_release_advertised(Path(args.site_dir) / "src" / "generated") write_if_changed(str(docs / "data.md"), generate_data_page(exports) + reference_next()) PAGE_DATA["data.md"] = dataset_structured_data(exports, db) export_reports = load_export_reports() _EXPORTED.update(report["platform"] for report in export_reports) print("Generating platform pages...") write_if_changed( str(docs / "platforms" / "index.md"), generate_platform_index(coverages, registry), ) if HANDHELD_GUIDE: write_if_changed(str(docs / HANDHELD_GUIDE.path), generate_handheld_guide(HANDHELD_GUIDE)) for name, cov in coverages.items(): # The emulators this platform runs, not every one of the system. runs = resolve_platform_cores(cov["config"], profiles) moved: dict[str, str] = {} write_if_changed( str(docs / "platforms" / f"{name}.md"), generate_platform_page(name, cov, registry, { emu: data for emu, data in emulator_files.items() if emu in runs }, moved), ) for stem, page in moved.items(): if stem in coverages: raise RuntimeError(f"a system page takes the platform {stem}'s address") PART_PAGES.add(f"platforms/{stem}.md") # One console of one setup's list, a reference: "playstation # bios" found five setups' lists before the console's page. # Weighed down, they still won on their title, which the page # weight does not reach, and their file list is already kept out # of the index: the title alone, found again on the setup's page # and the console's, is all the search held of them. SEARCH_DEMOTED.add(f"platforms/{stem}.md") front = _front_matter(page) if front: page = page[page.find("\n---\n", 4) + 5:].lstrip("\n") front.update(level="reference", search="exclude", parent={"label": cov["platform"], "link": f"{name}.md"}) page = "---\n" + yaml.safe_dump(front, sort_keys=False, allow_unicode=True) \ + "---\n\n" + page write_if_changed(str(docs / "platforms" / f"{stem}.md"), page) PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov) print("Generating emulator pages...") _DATA_MEMBERS.update(data_directory_members()) _DATA_REVISIONS.update(data_directory_revisions()) for key in sorted(profiles): if profiles[key].get("type") in ("alias", "test"): continue display = str(profiles[key].get("emulator", key)) for tree in page_data_directories(key, profiles): readers = _DATA_READERS.setdefault(str(tree.get("ref", "")), []) if (key, display) not in readers: readers.append((key, display)) for ref, members in _DATA_MEMBERS.items(): for stem, _names in data_tree_pages(ref, members): PART_PAGES.add(f"emulators/{stem}.md") write_if_changed( str(docs / "emulators" / "index.md"), generate_emulators_index(profiles) ) public_profiles = { name: profile for name, profile in profiles.items() if profile.get("type") not in ("alias", "test") } (public / "api" / "v1" / "emulators").mkdir(parents=True, exist_ok=True) for name, profile in public_profiles.items(): # The profile itself, one small document per core: reusing one # meant downloading emulators.json (5.9 MB) and filtering it. # The envelope and item of emulators.json, one item long: the # profile validates against the published schema, which an "id" # beside its fields broke on all 530, and schema_version is there. # The commit too, as every other envelope: 625 files lacked it. write_single_item( public / "api" / "v1" / "emulators" / f"{name}.json", "emulators", {"id": name, "profile": profile}, str(db.get("generated_at") or _timestamp()), _source_commit(), ) pages = emulator_pages( name, profile, db, consumers, suppl_names, data_registry, data_dirs=page_data_directories(name, profiles), ) PAGE_DATA[f"emulators/{name}.md"] = emulator_structured_data(profile) # A frozen copy answers after the emulator: a frozen_snapshot, and a # profile pinned to the revision one platform builds (pin_frozen). # Reicast and Flycast 2021 LE answered "dc bios missing" before Flycast. if (profile.get("type") in ("launcher", "alias") or profile.get("core_classification") in ("launcher", "alias", "frozen_snapshot") or profile.get("pin_frozen")): SEARCH_DEMOTED.add(f"emulators/{name}.md") files = profile.get("files") or [] for stem, entries in emulator_parts(name, files): PAGE_DATA[f"emulators/{stem}.md"] = emulator_structured_data( {**profile, "files": [files[i] for i in entries]}) PART_PAGES.add(f"emulators/{stem}.md") for stem, page in pages.items(): write_if_changed(str(docs / "emulators" / f"{stem}.md"), page) print("Generating cross-reference page...") xref_pages: dict[str, str] = {} write_if_changed( str(docs / "cross-reference.md"), tables_out_of_search(generate_cross_reference(coverages, profiles, xref_pages)) + reference_next(), ) for stem, page in xref_pages.items(): write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page)) for pname, cov in coverages.items(): if _own_list_only(cov["config"]): # The empty table it had is gone: its address leads to the # platform's page, which says its own list decides alone. sitestarlight.MOVED_PAGES.setdefault( f"cross-reference-{pname}.md", f"platforms/{pname}.md") print("Generating gap analysis page...") gap_pages: dict[str, str] = {} write_if_changed( str(docs / "gaps.md"), tables_out_of_search(generate_gap_analysis( profiles, coverages, db, suppl_names, registry, gap_report, gap_pages, )) + reference_next(), ) for stem, page in gap_pages.items(): write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page)) print("Generating updated platform files page...") write_if_changed( str(docs / "exports.md"), generate_exports_page( export_reports, {name: cov["platform"] for name, cov in coverages.items()}, ) + reference_next(), ) print("Generating dump provenance page...") provenance_report = build_report(db, load_provenance_snapshots()) provenance_pages: dict[str, str] = {} write_if_changed( str(docs / "provenance.md"), tables_out_of_search( generate_provenance_page(db, provenance_report, load_derivable(), provenance_pages) ) + reference_next(), ) for stem, page in provenance_pages.items(): write_if_changed(str(docs / f"{stem}.md"), tables_out_of_search(page)) # Wiki pages: copy manually maintained sources + generate dynamic ones print("Generating wiki pages...") wiki_dest = docs / "wiki" wiki_dest.mkdir(parents=True, exist_ok=True) wiki_src = Path(WIKI_SRC_DIR) if wiki_src.is_dir(): for src_file in wiki_src.glob("*.md"): shutil.copy2(src_file, wiki_dest / src_file.name) _record(wiki_dest / src_file.name) table_page = docs / SETUP_TABLE_PAGE if table_page.is_file(): write_if_changed(str(table_page), with_setup_table( table_page.read_text(encoding="utf-8"), coverages, registry)) # data-model.md is generated (contains live DB stats) write_if_changed( str(wiki_dest / "data-model.md"), generate_wiki_data_model(db, profiles) ) print("Generating which-pack page...") write_if_changed(str(docs / "which-pack.md"), generate_which_pack()) # One permanent download address per registered platform, archived ones # included: the README's table links each of them. print("Generating download pages...") get_dir = docs / release_record.GET_DIR get_dir.mkdir(parents=True, exist_ok=True) for name in platform_names: write_if_changed(str(get_dir / f"{name}.md"), generate_get_page(name)) write_if_changed(str(get_dir / "index.md"), generate_get_index()) print("Generating contributing page...") write_if_changed(str(docs / "contributing.md"), generate_contributing()) # Swept before the decoration, which rewrites every page it finds: a # page no generator produced any more (a console moved back onto its # manufacturer's page) was decorated, recorded, and kept for ever. write_if_changed(str(docs / "404.md"), generate_not_found()) stale = _sweep_generated(docs, [d for d in GENERATED_DIRS if d not in PUBLIC_DIRS]) stale += _sweep_generated(public, list(PUBLIC_DIRS)) stale += _sweep_top_pages(docs) print("Adding page metadata and structured data...") lastmod = decorate_markdown_pages(docs) nav = generate_nav(coverages, manufacturers, profiles, registry, Path(WIKI_SRC_DIR)) print("Writing the navigation and the picker's data...") site = Path(args.site_dir) present = { path.relative_to(docs).with_suffix(".md").as_posix() for path in docs.rglob("*.md*") if path.suffix in (".md", ".mdx") } - {"404.md"} generated = site / "src" / "generated" generated.mkdir(parents=True, exist_ok=True) write_if_changed(str(generated / "sidebar.json"), json.dumps(sitestarlight.sidebar(nav, present, _overview_anchors(nav, docs)), indent=1) + "\n") write_if_changed(str(generated / "redirects.json"), json.dumps(sitestarlight.redirects(profiles, present), indent=1, sort_keys=True) + "\n") write_if_changed(str(generated / "lastmod.json"), json.dumps(dict(sorted(lastmod.items())), indent=1) + "\n") system_ids = set(_system_page_map) | { sys_id for cov in coverages.values() for sys_id in cov["config"].get("systems", {}) } titles = {sys_id: system_name(sys_id) for sys_id in sorted(system_ids)} sitestarlight.picker_data(_REPO_ROOT / "install", public / "data" / "picker", titles) total_pages = len(present) print(f"\nGenerated {total_pages} pages in {args.docs_dir}/") if stale: print(f"Removed {stale} page(s) nothing produced this run") from sitediagrams import local_images, prepare_page, unresolved_images # noqa: E402,F401 from sitewrite import ( # noqa: E402,F401 _produced, _record, _undecorated, write_if_changed, write_decorated, _sweep_generated, _sweep_top_pages, ) from sitesources import ( # noqa: E402,F401 link_prose_citations, requested_by, revision_markdown, _repo_pins, _forge_sources, _source_permalink, _source_ref_markdown, ) from siterender import ( # noqa: E402,F401 _by_mode, _count, _field_text, _render_yaml_value, _slugify_anchor, _system_page_map, _system_link, set_system_page_map, set_console_sections, program_names, system_name, card_page, console_href, console_anchor, record_console_pages, record_emulator_cards, emulator_card_page, reset_console_pages, split_system_stems, _platform_link, _emulator_link, _fmt_size, _pct, _admonition_body, _code, _escape_tags, _icon_name, _icon_url, system_icon_markdown, prime_system_icons, _icon_available, SYSTEM_ICON_BASE, ICON_DIR, ) if __name__ == "__main__": from common import run_cli run_cli(main)