feat: order system cards and show their facts

# Conflicts:
#	wiki/testing-guide.md
This commit is contained in:
Abdessamad Derraz committed 2026-10-23 02:44:01 +02:00
commit 733de5f6af
6 files changed
+695 -116

No files matched your search

+381 -98
View File
@@ -2771,9 +2771,12 @@ def generate_files_page() -> str:
return "\n".join([
"# Find a file",
"",
"Paste a hash from clrmamepro, RomVault, an emulator log or a platform's "
"BIOS list. The page lists the file in the collection that has it, the "
"platforms that declare it and the emulators that check for it.",
"A hash is a fingerprint of a file's contents ([Glossary](../wiki/glossary.md)). "
"Tools such as clrmamepro and RomVault print one, and so do emulator logs "
"and the BIOS lists of setups.",
"",
"Paste it below. The page shows the file in the collection that has it, the "
"setups that list it and the emulators that check for it.",
"",
'<form class="rb-file-lookup" action="./" method="get">',
'<label for="rb-hash">SHA1, MD5, CRC32 or SHA256</label> ',
@@ -2793,8 +2796,8 @@ def generate_files_page() -> str:
"",
"A file's card lives on its manufacturer's page, and a file refiled "
"between manufacturers moves with it. `files/?sha1=<sha1>` is the address "
"that does not move: it opens the card wherever it is now. Each card's "
"**permalink** is that address.",
"that does not move: it opens the card wherever it is now. The link icon "
"beside each card's name is that address.",
"",
"## The index behind it",
"",
@@ -3027,7 +3030,22 @@ def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]:
return manufacturers
def _console_display(manufacturer: str, console: str) -> str:
"""A console section's name as the reader looks for it: the heading
system_name gives its id, or the folder alone under a program's top
(Other, Arcade), where "Other mame" would name nothing."""
from siterender import _PROGRAM_TOPS, _heading_name # noqa: PLC0415
return console if manufacturer in _PROGRAM_TOPS else _heading_name(manufacturer, console)
def generate_systems_index(manufacturers: dict) -> str:
"""Every console in one table, found by its own name.
The index listed 74 manufacturers and no console: someone looking for
PlayStation had to know it is filed under Sony, and Other hid 91
sections. Manufacturer pages stay the containers.
"""
total_mfr = len(manufacturers)
total_consoles = sum(len(c) for c in manufacturers.values())
# A copy filed under two folders appears on both pages, counted once here.
@@ -3039,38 +3057,52 @@ def generate_systems_index(manufacturers: dict) -> str:
lines = [
"# Systems",
"",
"Every BIOS file the collection holds, by manufacturer and console. Open "
"a console to see each file, the emulators that read it and the setups "
"that list it.",
"Every BIOS file the collection holds, by console. Open a console to "
"see each file, the emulators that read it and the setups that list it.",
"",
f"{total_mfr} manufacturers, {total_consoles} consoles, "
f"{total_files:,} files in the repository.",
f"{_count(total_consoles, 'console')}, {_count(total_files, 'file')} in the "
f"repository, from {_count(total_mfr, 'manufacturer')}.",
"",
]
machines = [m for m in sorted(manufacturers, key=str.casefold) if m not in GAME_DATA_TOPS]
games = [m for m in sorted(manufacturers, key=str.casefold) if m in GAME_DATA_TOPS]
for heading, names in (("Consoles and computers", machines),
("Game engines and PC games", games)):
if not names:
# Other holds one folder per emulator: a7800 or altirra in the console
# table named no console.
def kind(mfr: str) -> str:
if mfr in GAME_DATA_TOPS:
return "games"
return "programs" if mfr == "Other" else "consoles"
for heading, group, first, note in (
("Consoles and computers", "consoles", "Console", ""),
("Emulator folders", "programs", "Folder",
"Files kept under the emulator that reads them, not under a console."),
("Game engines and PC games", "games", "Name",
"Game engines and PC games hold the files a game engine reads, such "
"as the data of a game, rather than the BIOS of a machine."),
):
rows = sorted(
((_console_display(mfr, console), mfr, console, len(files))
for mfr, consoles in manufacturers.items() if kind(mfr) == group
for console, files in consoles.items()),
key=lambda row: (row[0].casefold(), row[1].casefold(), row[2]),
)
if not rows:
continue
lines += [f"## {heading}", ""]
if note:
lines += [note, ""]
lines += [
f"## {heading}",
"",
"| Manufacturer | Consoles | Files |",
"|-------------|----------|-------|",
# The count before the maker: on a phone the last column is
# the one scrolled off, and most names carry the maker already.
f"| {first} | Files | Maker |",
"|---------|------:|-------|",
]
for mfr in names:
consoles = manufacturers[mfr]
file_count = sum(len(files) for files in consoles.values())
lines.append(f"| [{mfr}]({_slug(mfr)}.md) | {len(consoles)} | {file_count} |")
for name, mfr, console, count in rows:
cell = name.replace("|", "-")
anchor = _slugify_anchor(console)
href = f"{_slug(mfr)}.md" + (f"#{anchor}" if anchor else "")
lines.append(f"| [{cell}]({href}) | {count:,} | [{mfr}]({_slug(mfr)}.md) |")
lines.append("")
if games:
lines += [
"Game engines and PC games hold the files a game engine reads, such "
"as the data of a game, rather than the BIOS of a machine.",
"",
]
return "\n".join(lines) + "\n"
@@ -3349,21 +3381,46 @@ def _fetch_url(path: str) -> str:
)
def _platform_links(sha1: str, consumers: FileConsumers) -> list[str]:
"""Each platform declaring the file: a link to its system section, and
_platform_names: dict[str, str] = {}
def _platform_display(name: str) -> str:
"""A setup's name as its platform file writes it: Batocera, not batocera."""
display = _platform_names.get(name)
if display is None:
try:
config = load_platform_config(name, str(_REPO_ROOT / "platforms"))
except FileNotFoundError:
config = {}
display = _platform_names[name] = str(config.get("platform") or name)
return display
# A setup, the link to its page, and where it puts the file.
Place = tuple[str, str, list[str]]
def _platform_places(sha1: str, consumers: FileConsumers) -> list[Place]:
"""Each platform declaring the file, linked to its system section, with
the destination it gives the file."""
systems_of = getattr(consumers, "platform_systems", {}).get(sha1, {})
destinations_of = getattr(consumers, "platform_destinations", {}).get(sha1, {})
links: list[str] = []
for platform in sorted(consumers.platforms.get(sha1, ())):
places: list[Place] = []
for platform in consumers.platforms.get(sha1, ()):
section = sorted(systems_of.get(platform, ()))
fragment = f"#{section[0].replace(' ', '-')}" if section else ""
link = f"[{platform}](../platforms/{platform}.md{fragment})"
destinations = sorted(destinations_of.get(platform, ()))
if destinations:
link += " " + ", ".join(_code(d) for d in destinations)
links.append(link)
return links
places.append((_platform_display(platform), f"../platforms/{platform}.md{fragment}",
sorted(destinations_of.get(platform, ()))))
return sorted(places, key=lambda place: place[0].casefold())
def _platform_links(sha1: str, consumers: FileConsumers) -> list[str]:
"""Each platform declaring the file, as Markdown: its name, linked to
its system section, then the destination it gives the file."""
return [
f"[{name}]({href})" + (": " + ", ".join(_code(d) for d in dests) if dests else "")
for name, href, dests in _platform_places(sha1, consumers)
]
_card_anchors: dict[str, tuple[list, list[str]]] = {}
@@ -3397,16 +3454,60 @@ def _emulator_links(sha1: str, consumers: FileConsumers) -> list[str]:
return links
def _installed_links(sha1: str, consumers: FileConsumers, declared: set[str]) -> list[str]:
def _installed_places(sha1: str, consumers: FileConsumers, declared: set[str]) -> list[Place]:
"""Each pack installing the file that its platform's list does not name,
with the place the pack gives it."""
links = []
for platform, dests in sorted(getattr(consumers, "installed", {}).get(sha1, {}).items()):
if platform in declared:
continue
where = ", ".join(_code(d) for d in sorted(dests) if d)
links.append(f"[{platform}](../platforms/{platform}.md)" + (f" {where}" if where else ""))
return links
places = [
(_platform_display(platform), f"../platforms/{platform}.md", sorted(d for d in dests if d))
for platform, dests in getattr(consumers, "installed", {}).get(sha1, {}).items()
if platform not in declared
]
return sorted(places, key=lambda place: place[0].casefold())
def _places_text(places: list[Place]) -> str:
"""Setups as "Lakka: scph1000.bin", the name linked to its page and the
destination as text: a code chip per path made the line a row of
boxes."""
def plain(text: str) -> str:
return html.escape(text, quote=False).replace("`", "&#96;")
return ", ".join(
f"[{name}]({href})" + (": " + " and ".join(plain(d) for d in dests) if dests else "")
for name, href, dests in places
)
def _inline_card_html(text: str) -> str:
"""A card value written in Markdown (code spans, links), as the HTML a
fact list holds. HTML already in the value is kept."""
out: list[str] = []
for part in re.split(r"(`[^`]*`)", text):
if len(part) > 1 and part.startswith("`") and part.endswith("`"):
out.append(f"<code>{html.escape(part[1:-1], quote=False)}</code>")
else:
out.append(re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", r'<a href="\2">\1</a>', part))
return "".join(out)
# A link icon, the way a heading anchor is drawn: the word "permalink" was
# read as part of every card title, by the search and by screen readers.
# Drawn once per page and used by each card: inline, its 400 bytes were
# 100 KB on the Sony page.
PERMALINK_SYMBOL = (
'<svg aria-hidden="true" style="display:none"><symbol id="rb-link-icon" '
'viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" '
'stroke-linecap="round" stroke-linejoin="round">'
'<path d="M10 13a5 5 0 0 0 7.5.5l3-3a5 5 0 0 0-7-7l-1.7 1.7"/>'
'<path d="M14 11a5 5 0 0 0-7.5-.5l-3 3a5 5 0 0 0 7 7l1.7-1.7"/></symbol></svg>'
)
_PERMALINK_ICON = ('<svg aria-hidden="true" width="16" height="16">'
'<use href="#rb-link-icon"/></svg>')
def _icon_symbol() -> list[str]:
"""The icon's drawing, for a page of cards. None in the yardstick."""
return [PERMALINK_SYMBOL, ""] if markup.starlight() else []
def _system_file_card(
@@ -3425,6 +3526,7 @@ def _system_file_card(
# Who reads this file, not who reads a file of this name
plats = sorted(consumers.platforms.get(sha1_full, ()))
emus = sorted(consumers.emulators.get(sha1_full, ()))
installed = _installed_places(sha1_full, consumers, set(plats))
anchor = f"sha1-{sha1_full[:12]}" if sha1_full != "unknown" else ""
# A copy filed under two consoles of one maker is listed twice on its
@@ -3434,56 +3536,87 @@ def _system_file_card(
anchor = ""
else:
anchored.add(anchor)
skip = markup.search_exclude()
lines = [markup.block_open("div", 'class="rb-sys-file"'), ""]
# The heading's own link is this page's, the second link is the file's
# own address, resolved by content wherever its card moves.
# The heading's own link is this page's, the icon is the file's own
# address, resolved by content wherever its card moves.
# The yardstick keeps the word, the size the page budget was set on.
icon = _PERMALINK_ICON if markup.starlight() else "permalink"
permalink = (
f' <a class="rb-permalink" href="{markup.site_href(f"files/?sha1={sha1_full}")}" '
f'title="Permanent link, by content">permalink</a>'
f'aria-label="Permanent link to {html.escape(name)}" '
f'title="Permanent link, by content" {skip}>{icon}</a>'
) if anchor else ""
# A file nothing reads said so nowhere: 53 of 248 Sony cards had no
# emulator, platform or pack line and looked unfinished.
unread = (f' <span class="rb-badge rb-badge-muted" {skip}>No reader</span>'
if not (emus or plats or installed) else "")
lines.append(_card_title(
f'{_code(name)} <span class="rb-badge rb-badge-muted">{size}</span>{permalink}',
f'{_code(name)} <span class="rb-badge rb-badge-muted">{size}</span>{unread}{permalink}',
anchor, searchable,
))
lines.append("")
# Every hash in full: the site search indexes text, not a title
# attribute, so a pasted md5 or crc32 (what clrmamepro, RomVault or an
# emulator log prints) found nothing behind a 12-character prefix.
for label, value in (("SHA1", sha1_full), ("MD5", md5_full),
("CRC32", f.get("crc32")), ("SHA256", f.get("sha256"))):
if value and value != "unknown":
lines.append(f'- {label}: {markup.hash_value(value)}')
if emus:
lines.append(f"- Emulators: {', '.join(_emulator_links(sha1_full, consumers))}")
provenance = f.get("provenance", {})
if provenance:
lines.extend(_provenance_lines(provenance, "../"))
if f.get("path"):
# The card named a file, its hashes and provenance, but not where to
# get it: a user missing scph5501.bin found the reference and no way
# to the bytes.
lines.append(f"- Collection path: {_code(f['path'])} ([download]({_fetch_url(f['path'])}))")
# Each fact: its label, its values, whether the search reads it. What a
# setup does with the file comes first, the fingerprints last.
facts: list[tuple[str, list[str], bool]] = []
if plats:
# Each link lands on the platform's system section, and says where
# that platform puts the file: from the card, retrodeck sent the
# reader to the top of a 2,000-file page.
lines.append(f"- Platforms: {', '.join(_platform_links(sha1_full, consumers))}")
installed = _installed_links(sha1_full, consumers, set(plats))
# that platform puts the file. The site search reads this line: it
# names the file as each setup names it, and with the name only in
# its title the card ranked below every guide citing scph5501 as an
# example.
facts.append(("On the setup's list",
[_places_text(_platform_places(sha1_full, consumers))], True))
if installed:
# The packs that carry the file for their cores: a RetroBat user
# looking up scph5501.bin found every platform but RetroBat, whose
# pack installs it for six cores.
lines.append(f"- Also installed by: {', '.join(installed)}")
# The title stays searchable: the file name is what a reader types. On
# the site the platforms line does too: it names the file as each setup
# names it, and with the name only in its title the card ranked below
# every guide citing scph5501 as an example. The size yardstick keeps
# its one block.
at = next((i for i, line in enumerate(lines) if line.startswith("- Platforms:")), None)
if markup.starlight() and searchable and at is not None:
lines[4:] = [*search_excluded(lines[4:at]), lines[at], *search_excluded(lines[at + 1:])]
facts.append(("Packed for its emulators by", [_places_text(installed)], False))
if emus:
facts.append(("Emulators", [", ".join(_emulator_links(sha1_full, consumers))], False))
if f.get("path"):
# The card named a file, its hashes and provenance, but not where to
# get it: a user missing scph5501.bin found the reference and no way
# to the bytes.
facts.append(("Collection path",
[f"{_code(f['path'])} ([download]({_fetch_url(f['path'])}))"], False))
catalogued = [line.split(": ", 1)[1]
for line in _provenance_lines(f.get("provenance") or {}, "../")]
if catalogued:
facts.append(("Catalogued", catalogued, False))
# Every hash in full: the site search indexes text, not a title
# attribute, so a pasted md5 or crc32 (what clrmamepro, RomVault or an
# emulator log prints) found nothing behind a 12-character prefix.
# files/?hash= finds them, and the index skips them.
for label, value in (("SHA1", sha1_full), ("MD5", md5_full),
("CRC32", f.get("crc32")), ("SHA256", f.get("sha256"))):
if value and value != "unknown":
facts.append((label, [f'<span class="rb-hash">{html.escape(str(value))}</span>'],
False))
if not markup.starlight():
# The size yardstick pages are cut by: the list a card was when the
# budget was set, so a fact grid moves no card to another page.
lines.extend(search_excluded([f"- {label}: {', '.join(values)}"
for label, values, _searched in facts]))
else:
lines[4:] = search_excluded(lines[4:])
# A grid of facts, label then value: two bullet lists of code chips
# read as a dump. What the search skips is one list with one
# attribute: on each row it was 77 KB of the Sony page.
read = [fact for fact in facts if fact[2] and searchable]
skipped = [fact for fact in facts if fact not in read]
for group, attributes in ((read, ""), (skipped, f" {skip}")):
if group:
lines.append(f'<dl class="rb-facts"{attributes}>')
# One value cell per label: the grid pairs each dt with one dd.
# A searched value keeps its label out: the excerpt ran the two
# together, "On the setup's listAmberELEC".
label_skip = f" {skip}" if not attributes else ""
lines.extend(f"<dt{label_skip}>{label}</dt><dd>"
+ "<br>".join(_inline_card_html(v) for v in values) + "</dd>"
for label, values, _searched in group)
lines.append("</dl>")
lines.extend(["", "</div>", ""])
return lines
@@ -3636,24 +3769,144 @@ SYSTEM_PAGE_BUDGET = 500_000
Card = tuple[bool, list[str], str]
def _reading_entries(sha1: str, consumers: FileConsumers) -> list[dict]:
"""The profile entries that read a file, one per emulator."""
files_of = getattr(consumers, "emulator_files", {})
entries = []
for emulator, index in getattr(consumers, "emulator_entries", {}).get(sha1, {}).items():
files = files_of.get(emulator) or []
if 0 <= index < len(files) and isinstance(files[index], dict):
entries.append(files[index])
return entries
_NO_PRIORITY = 1 << 30
def _card_rank(f: dict, consumers: FileConsumers) -> tuple:
"""Where a file's card sits in its console: the files an emulator
requires first, then by the emulators' search order (smallest first),
then by how many setups list the file, then by name.
By byte order the PlayStation section opened on LibPS.exe and
PSX_DTLH2000.bin, capitals first, and scph5501.bin was 28th.
"""
sha1 = str(f.get("sha1", ""))
entries = _reading_entries(sha1, consumers)
priorities = [e["priority"] for e in entries if isinstance(e.get("priority"), int)]
return (
not any(e.get("required", True) for e in entries),
min(priorities, default=_NO_PRIORITY),
-len(consumers.platforms.get(sha1, ())),
f["name"].casefold(), f["name"], f.get("path", ""),
)
def _by_name(f: dict) -> tuple:
return f["name"].casefold(), f["name"], f.get("path", "")
def _console_cards(
manufacturer: str,
files: list[dict],
consumers: FileConsumers,
anchored: set[str],
by_name: bool = False,
) -> list[Card]:
"""Each card of one console, primaries by name then variants, with
whether it is a variant."""
"""Each card of one console, primaries then variants, with whether it
is a variant. ``by_name`` orders by name alone, for a console cut into
parts named by their first and last file."""
searchable = manufacturer not in GAME_DATA_TOPS
order = _by_name if by_name else (lambda f: _card_rank(f, consumers))
main_files = [f for f in files if "/.variants/" not in f["path"]]
variant_files = [f for f in files if "/.variants/" in f["path"]]
cards = [(False, _system_file_card(f, consumers, anchored, searchable), str(f.get("sha1", "")))
for f in sorted(main_files, key=lambda x: x["name"])]
for f in sorted(main_files, key=order)]
cards += [(True, _system_file_card(v, consumers, anchored, searchable), str(v.get("sha1", "")))
for v in sorted(variant_files, key=lambda x: (x["name"], x["path"]))]
for v in sorted(variant_files, key=order)]
return cards
# A "Start with" line names this many files at most: Game Engines'
# TheXTech requires 2,014.
START_WITH_NAMES = 6
def _region_order() -> dict[str, int]:
from region import REGION_TREE, WORLD # noqa: PLC0415
ordered = [WORLD]
for parent, members in REGION_TREE.items():
ordered += [parent, *sorted(members)]
return {region: index for index, region in enumerate(ordered)}
_REGION_WORDS = {"uk": "UK", "asia-ntsc": "Asia (NTSC)", "asia-pal": "Asia (PAL)"}
def _region_word(region: str) -> str:
return _REGION_WORDS.get(region) or region.replace("-", " ").title()
def _start_with(files: list[dict], consumers: FileConsumers, first_below: bool = True) -> list[str]:
"""The line a console section opens on: the files an emulator requires,
by the region the emulator picks each for.
Read from the profiles' required and region fields, never typed per
system. A section where no emulator requires a file gets no line.
"""
from region import _PARENT # noqa: PLC0415
order = _region_order()
by_region: dict[str, list[str]] = {}
plain: list[str] = []
primaries = [f for f in files if "/.variants/" not in f["path"]]
for f in sorted(primaries, key=lambda f: _card_rank(f, consumers)):
required = [e for e in _reading_entries(str(f.get("sha1", "")), consumers)
if e.get("required", True)]
if not required:
continue
regions = {str(r) for e in required for r in e.get("region") or []}
for names in [by_region.setdefault(r, []) for r in regions] or [plain]:
if f["name"] not in names:
names.append(f["name"])
# A territory its region answers the same way adds nothing: Brazil
# under Latin America.
by_region = {r: names for r, names in by_region.items()
if by_region.get(_PARENT.get(r, "")) != names}
# Regions that need the same files share a line.
regional: dict[tuple[str, ...], list[str]] = {}
for region in sorted(by_region, key=lambda r: (order.get(r, len(order)), r)):
names = by_region[region]
same = next((rs for rs, held in regional.items() if held == names), None)
if same is None:
regional[(region,)] = names
else:
regional[(*same, region)] = regional.pop(same)
if not regional and not plain:
return []
def named(names: list[str], lead: str) -> str:
if len(names) <= START_WITH_NAMES:
return f"{lead}: {', '.join(_code(n) for n in names)}"
where = ", first below" if first_below else ""
return f"{lead}: {_count(len(names), 'more file' if regional else 'file')}{where}"
if not regional:
if len(plain) > START_WITH_NAMES:
where = " They come first below." if first_below else ""
return [f"**Start with** the {_count(len(plain), 'file')} an emulator requires.{where}",
""]
return [named(plain, "**Start with**") + ".", ""]
lines = ["**Start with** the BIOS for the region of your games:", ""]
for regions in sorted(regional, key=lambda rs: (order.get(rs[0], len(order)), rs)):
lead = ", ".join(_region_word(r) for r in regions)
lines.append(f"- **{lead}**: {', '.join(_code(n) for n in regional[regions])}")
if plain:
lines.append(named(plain, "- **Some emulators also require**"))
return [*lines, ""]
def _cards_markdown(cards: list[Card]) -> list[str]:
"""Primaries, then the variants in a closed block."""
lines = [line for variant, card, _sha1 in cards if not variant for line in card]
@@ -3682,6 +3935,17 @@ def _parts(cards: list[Card], budget: int, sizes: list[int] | None = None) -> li
return parts
def _console_heading(manufacturer: str, console: str) -> str:
"""A console's heading: its icon hidden from screen readers, then the
name with no space before it. The space read as "Section titled
“ PlayStation”" on five of seven Sony headings."""
icon = _icon_name(manufacturer, console)
if not _icon_available.get(icon):
return f"## {console}"
image = markup.image("", _icon_url(icon), width="24", **{"aria-hidden": "true"})
return f"## {image}{console}"
_NEXT_FROM_SYSTEMS = [
"[Get your files](../which-pack.md), for the setup you play on",
"[When a game does not start](../wiki/troubleshooting.md)",
@@ -3721,15 +3985,22 @@ def system_pages(
"first, the others under \"Variants\". To install the files of one "
"console, open its \"Install\" box."
)
# The two setup lines of a card differ, and only a code comment said how.
key = (
"On a card, **On the setup's list** names the setups whose own BIOS "
"list has the file. **Packed for its emulators by** names the setups "
"whose pack adds it because one of their emulators reads it."
)
def whole_page(sections: dict[str, list[Card]]) -> str:
lines = [
f"# {manufacturer}", "", intro, "",
f"# {manufacturer}", "", intro, "", key, "",
f"The files below by console, with every hash, are also published as {json_link}.",
"",
"", *_icon_symbol(),
]
for console, cards in sections.items():
lines.append(f"## {system_icon_markdown(manufacturer, console)}{console}")
lines.append(_console_heading(manufacturer, console))
lines.append("")
lines.extend(_start_with(consoles[console], consumers))
lines.extend(_console_install_block(console, (installs or {}).get(console, [])))
lines.extend(_cards_markdown(cards))
lines.append("")
@@ -3764,16 +4035,17 @@ def system_pages(
pages: dict[str, tuple[str, dict[str, list]]] = {}
summary = [
f"# {manufacturer}", "", intro, "",
f"# {manufacturer}", "", intro, "", key, "",
f"The files below by console, with every hash, are also published as {json_link}. "
f"The largest sections have a page of their own, linked from their heading.",
"",
"", *_icon_symbol(),
]
summary_anchors: set[str] = set()
for console in sorted(consoles):
summary.append(f"## {system_icon_markdown(manufacturer, console)}{console}")
summary.append(_console_heading(manufacturer, console))
summary.append("")
if console not in moved:
summary.extend(_start_with(consoles[console], consumers))
summary.extend(_console_install_block(console, (installs or {}).get(console, [])))
summary.extend(_cards_markdown(
_console_cards(manufacturer, consoles[console], consumers, summary_anchors)))
@@ -3781,11 +4053,18 @@ def system_pages(
continue
# Cards are rendered again for their own page: the anchors a page
# holds are its own.
cards = _console_cards(manufacturer, consoles[console], consumers, set())
with markup.using("mkdocs"):
sizes = [card_weight(card) for _variant, card, _sha1
in _console_cards(manufacturer, consoles[console], consumers, set())]
parts = _parts(cards, budget, sizes)
def cut(by_name: bool) -> list[list[Card]]:
cards = _console_cards(manufacturer, consoles[console], consumers, set(), by_name)
with markup.using("mkdocs"):
sizes = [card_weight(card) for _variant, card, _sha1 in _console_cards(
manufacturer, consoles[console], consumers, set(), by_name)]
return _parts(cards, budget, sizes)
# One page keeps the required files first. Parts are named by their
# first and last file, so a console cut in parts goes by name.
parts = cut(False)
if len(parts) > 1:
parts = cut(True)
base = f"{slug}-{_slugify_anchor(console) or 'files'}"
stems = [base if index == 0 else f"{base}-{index + 1}" for index in range(len(parts))]
record_console_pages(slug, console, stems, [
@@ -3793,6 +4072,7 @@ def system_pages(
])
title = f"{manufacturer}: {console}"
count = _count(len(consoles[console]), "file")
summary.extend(_start_with(consoles[console], consumers, first_below=False))
if len(stems) == 1:
summary.append(f"{count}, on [their own page]({stems[0]}.md).")
else:
@@ -3807,12 +4087,15 @@ def system_pages(
f"# {heading}", "",
f"Part of [{manufacturer}]({slug}.md), in [Systems](index.md). "
f"Every hash of these files is also published as {json_link}.",
"",
"", *_icon_symbol(),
]
if len(stems) > 1:
others = ", ".join(
f"[part {i + 1}]({other}.md)" for i, other in enumerate(stems) if i != index)
lines.extend([f"This part holds {_part_range(part)}. The others: {others}.", ""])
if index == 0:
lines.extend(_start_with(consoles[console], consumers, first_below=len(stems) == 1))
lines.extend([key, ""])
lines.extend(_console_install_block(console, (installs or {}).get(console, [])))
lines.extend(["## Files", ""])
lines.extend(_cards_markdown(part))
+1 -1
View File
@@ -4963,7 +4963,7 @@ class CollectionCardsCarryFullHashes(unittest.TestCase):
"crc32": "504086c1", "sha256": "b" * 64}
text = "\n".join(_system_file_card(entry, FileConsumers({}, {})))
for value in (entry["sha1"], entry["md5"], entry["crc32"], entry["sha256"]):
self.assertIn(f">{value}</code>", text)
self.assertIn(f'<span class="rb-hash">{value}</span>', text)
+10 -9
View File
@@ -358,10 +358,11 @@ class AVariantShowsWhoPinsIt(unittest.TestCase):
page = generate_site.generate_system_page(
"Entex", {"Adventure Vision": [primary, variant]}, consumers)
block = page[page.index("<summary>Variants (1)</summary>"):]
self.assertIn("- Collection path: `bios/Entex/Adventure Vision/.variants/advision.zip.1c086064`", block)
self.assertIn("[batocera](../platforms/batocera.md)", block)
self.assertIn("[retrobat](../platforms/retrobat.md)", block)
self.assertIn('<code class="rb-hash">' + "c" * 40 + "</code>", block)
self.assertIn("<dt>Collection path</dt><dd><code>bios/Entex/Adventure Vision/"
".variants/advision.zip.1c086064</code>", block)
self.assertIn('<a href="../platforms/batocera.md">Batocera</a>', block)
self.assertIn('<a href="../platforms/retrobat.md">RetroBat</a>', block)
self.assertIn('<span class="rb-hash">' + "c" * 40 + "</span>", block)
self.assertIn("Catalogued", block)
@@ -461,7 +462,7 @@ class ASystemCardLandsOnThePlatformsSection(unittest.TestCase):
)
self.assertEqual(
_platform_links("s", consumers),
["[retrodeck](../platforms/retrodeck.md#sony-playstation) `bios/scph5000.bin`"],
["[RetroDECK](../platforms/retrodeck.md#sony-playstation): `bios/scph5000.bin`"],
)
@@ -745,7 +746,7 @@ class ACardNamesThePacksThatInstallIt(unittest.TestCase):
import json as _json
from cross_reference import FileConsumers
from generate_site import _installed_links
from generate_site import _installed_places
with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as tmp:
Path(tmp, "retrobat.json").write_text(_json.dumps({
@@ -755,9 +756,9 @@ class ACardNamesThePacksThatInstallIt(unittest.TestCase):
}))
consumers = FileConsumers({"files": {}})
consumers.add_manifests(tmp)
links = _installed_links("a" * 40, consumers, set())
self.assertEqual(links, ["[retrobat](../platforms/retrobat.md) `scph5501.bin`"])
self.assertEqual(_installed_links("a" * 40, consumers, {"retrobat"}), [])
places = _installed_places("a" * 40, consumers, set())
self.assertEqual(places, [("RetroBat", "../platforms/retrobat.md", ["scph5501.bin"])])
self.assertEqual(_installed_places("a" * 40, consumers, {"retrobat"}), [])
class ProseCitationsLinkToThePin(unittest.TestCase):
+294
View File
@@ -0,0 +1,294 @@
"""System pages and file cards (round 2, package 6).
The Sony page opened on LibPS.exe, its card titles read "scph5501.bin
permalink" in the site search, the two setup lines of a card were defined
only in a code comment, and the systems index named no console.
"""
from __future__ import annotations
import re
import sys
import types
import unittest
from pathlib import Path
from unittest import mock
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import generate_site as gs # noqa: E402
import siterender # noqa: E402
from cross_reference import FileConsumers # noqa: E402
DIST = REPO_ROOT / "website" / "dist"
def _file(name: str, char: str, path: str = "") -> dict:
return {"name": name, "sha1": char * 40, "md5": char * 32, "size": 524288,
"path": path or f"bios/Sony/PlayStation/{name}"}
def _consumers(readers: dict[str, list[dict]], platforms: dict[str, set[str]] | None = None
) -> types.SimpleNamespace:
"""Consumers where each sha1 is read by the given profile entries, one
emulator per entry."""
files_of: dict[str, list] = {}
entries: dict[str, dict[str, int]] = {}
emulators: dict[str, set[str]] = {}
for sha1, read_by in readers.items():
for number, entry in enumerate(read_by):
emulator = f"emu{number}-{sha1[:4]}"
files_of[emulator] = [entry]
entries.setdefault(sha1, {})[emulator] = 0
emulators.setdefault(sha1, set()).add(emulator)
return types.SimpleNamespace(
platforms=platforms or {}, emulators=emulators, emulator_entries=entries,
emulator_files=files_of, installed={}, platform_systems={},
platform_destinations={},
)
class TheConsoleBiosComesFirst(unittest.TestCase):
"""Cards were sorted by byte order: capitals first, so LibPS.exe and
PSX_DTLH2000.bin opened the PlayStation section and scph5501.bin was
28th."""
def setUp(self):
self.files = [
_file("LibPS.exe", "1"), _file("PSX_DTLH2000.bin", "2"),
_file("dtlh1100.bin", "3"), _file("scph5501.bin", "4"),
_file("scph5500.bin", "5"), _file("scph1001.bin", "6"),
_file("openbios.bin", "7"),
]
self.consumers = _consumers({
"4" * 40: [{"name": "scph5501.bin", "required": True, "priority": 2,
"region": ["north-america"]}],
"5" * 40: [{"name": "scph5500.bin", "required": True, "priority": 5,
"region": ["japan"]}],
"6" * 40: [{"name": "scph1001.bin", "required": False, "priority": 1}],
"2" * 40: [{"name": "PSX_DTLH2000.bin", "required": False}],
}, platforms={"6" * 40: {"a", "b"}, "3" * 40: {"a", "b", "c"}})
def names(self) -> list[str]:
cards = gs._console_cards("Sony", self.files, self.consumers, set())
return [re.search(r"<code>(.*?)</code>", card[2])[1] for _v, card, _s in cards]
def test_required_then_priority_then_setups_then_name(self):
self.assertEqual(self.names(), [
"scph5501.bin", "scph5500.bin", # required, by priority
"scph1001.bin", # optional with a priority
"dtlh1100.bin", # listed by three setups
"LibPS.exe", "openbios.bin", "PSX_DTLH2000.bin", # by name, any case
])
def test_a_console_cut_in_parts_goes_by_name(self):
cards = gs._console_cards("Sony", self.files, self.consumers, set(), by_name=True)
names = [re.search(r"<code>(.*?)</code>", card[2])[1] for _v, card, _s in cards]
self.assertEqual(names, sorted(names, key=str.casefold))
def test_the_section_opens_on_the_bios_by_region(self):
lines = gs._start_with(self.files, self.consumers)
self.assertEqual(lines[0], "**Start with** the BIOS for the region of your games:")
self.assertIn("- **North America**: `scph5501.bin`", lines)
self.assertIn("- **Japan**: `scph5500.bin`", lines)
self.assertLess(lines.index("- **North America**: `scph5501.bin`"),
lines.index("- **Japan**: `scph5500.bin`"))
def test_regions_needing_the_same_file_share_a_line(self):
consumers = _consumers({"4" * 40: [
{"required": True, "region": ["north-america", "latin-america", "brazil"]}]})
lines = gs._start_with([_file("scph5501.bin", "4")], consumers)
# Brazil says nothing Latin America does not.
self.assertIn("- **North America, Latin America**: `scph5501.bin`", lines)
def test_a_long_list_is_counted(self):
files = [_file(f"rom{i}.bin", "abcdefgh"[i]) for i in range(8)]
consumers = _consumers({f["sha1"]: [{"required": True}] for f in files})
self.assertEqual(gs._start_with(files, consumers)[0],
"**Start with** the 8 files an emulator requires. They come first below.")
def test_no_required_file_no_line(self):
self.assertEqual(gs._start_with([_file("a.bin", "a")], _consumers({})), [])
def test_the_page_carries_the_line_under_the_console(self):
page = gs.generate_system_page("Sony", {"PlayStation": self.files}, self.consumers)
heading = page.index("## PlayStation")
self.assertLess(heading, page.index("**Start with**"))
self.assertLess(page.index("**Start with**"), page.index('class="rb-sys-file"'))
class ThePermalinkIsAnIcon(unittest.TestCase):
"""The word "permalink" was link text inside every card title: the
search showed "scph5501.bin permalink" and screen readers read it."""
def test_no_word_and_out_of_the_search(self):
lines = gs._system_file_card(_file("scph5501.bin", "4"), FileConsumers({"files": {}}))
title = lines[2]
link = re.search(r'<a class="rb-permalink"[^>]*>(.*?)</a>', title)
self.assertNotIn("permalink", re.sub(r"<[^>]+>", "", title))
self.assertIn('aria-label="Permanent link to scph5501.bin"', link[0])
self.assertIn("data-pagefind-ignore", link[0])
self.assertIn('<use href="#rb-link-icon"/>', link[1])
def test_the_page_draws_the_icon_once(self):
page = gs.generate_system_page(
"Sony", {"PlayStation": [_file("a.bin", "a"), _file("b.bin", "b")]},
FileConsumers({"files": {}}))
self.assertEqual(page.count('<symbol id="rb-link-icon"'), 1)
self.assertEqual(page.count('href="#rb-link-icon"'), 2)
class TheCardLinesSayWhatTheyMean(unittest.TestCase):
""""Platforms:" and "Also installed by:" listed different setups, as
lower-case ids, and only a code comment told them apart."""
def setUp(self):
sha1 = "4" * 40
self.consumers = types.SimpleNamespace(
platforms={sha1: {"batocera"}}, emulators={},
platform_systems={sha1: {"batocera": {"sony-playstation"}}},
platform_destinations={sha1: {"batocera": {"scph5501.bin"}}},
installed={sha1: {"retrobat": {"bios/scph5501.bin"}}},
)
self.lines = gs._system_file_card(_file("scph5501.bin", "4"), self.consumers)
def test_labels_and_display_names(self):
text = "\n".join(self.lines)
self.assertIn("<dt data-pagefind-ignore>On the setup's list</dt><dd><a href=\"../"
"platforms/batocera.md#sony-playstation\">Batocera</a>: scph5501.bin</dd>",
text)
self.assertIn("<dt>Packed for its emulators by</dt><dd><a href=\"../platforms/"
"retrobat.md\">RetroBat</a>: bios/scph5501.bin</dd>", text)
self.assertNotIn("Platforms:", text)
self.assertNotIn("Also installed by", text)
def test_the_setup_list_is_searched_and_the_rest_is_not(self):
lists = [line for line in self.lines if line.startswith("<dl")]
self.assertEqual(lists, ['<dl class="rb-facts">', '<dl class="rb-facts" data-pagefind-ignore>'])
searched = "\n".join(self.lines).split('<dl class="rb-facts">', 1)[1].split("</dl>")[0]
self.assertIn("On the setup's list", searched)
self.assertNotIn("SHA1", searched)
# The label stays out of the excerpt: the search ran the two
# together, "On the setup's listAmberELEC".
self.assertIn("<dt data-pagefind-ignore>On the setup's list</dt><dd>", searched)
def test_the_page_defines_both_lines(self):
page = gs.generate_system_page("Sony", {"PlayStation": [_file("scph5501.bin", "4")]},
self.consumers)
key = page.index("**On the setup's list** names")
self.assertLess(key, page.index('class="rb-sys-file"'))
self.assertIn("**Packed for its emulators by** names", page)
def test_a_file_nothing_reads_says_so(self):
title = gs._system_file_card(_file("GameIndex.yaml", "9"), FileConsumers({"files": {}}))[2]
self.assertIn(">No reader</span>", title)
self.assertNotIn("No reader", self.lines[2])
class FileFactsAreAGrid(unittest.TestCase):
"""Each card was two bullet lists of code chips."""
def test_one_value_cell_per_label_and_plain_hashes(self):
entry = {**_file("scph5501.bin", "4"), "crc32": "8d8cb7e4",
"provenance": {"redump": {"name": "ps-30a.bin"},
"tosec": {"name": "SCPH-5501 (1996).bin"}}}
lines = gs._system_file_card(entry, FileConsumers({"files": {}}))
facts = [line for line in lines if line.startswith("<dt>")]
self.assertTrue(facts)
for line in facts:
self.assertEqual(line.count("<dt>"), 1, line)
self.assertEqual(line.count("<dd>"), 1, line)
text = "\n".join(lines)
self.assertIn('<dt>SHA1</dt><dd><span class="rb-hash">' + "4" * 40 + "</span></dd>", text)
self.assertNotIn("<code class=\"rb-hash\">", text)
catalogued = next(line for line in facts if line.startswith("<dt>Catalogued"))
self.assertIn("<br>", catalogued)
self.assertNotRegex(text, r"(?m)^- ")
def test_collection_path_keeps_its_chip(self):
text = "\n".join(gs._system_file_card(_file("a.bin", "a"), FileConsumers({"files": {}})))
self.assertIn("<dt>Collection path</dt><dd><code>bios/Sony/PlayStation/a.bin</code>", text)
class ConsoleHeadingsStartWithTheName(unittest.TestCase):
"""The icon and a space came before the name: "Section titled
“ PlayStation”" on five of seven Sony headings."""
def test_the_icon_is_hidden_and_touches_the_name(self):
icon = siterender._icon_name("Sony", "PlayStation")
with mock.patch.dict(siterender._icon_available, {icon: True}):
heading = gs._console_heading("Sony", "PlayStation")
self.assertRegex(heading, r'^## <img [^>]*aria-hidden="true"[^>]*>PlayStation$')
self.assertIn('alt=""', heading)
def test_no_icon_no_image(self):
icon = siterender._icon_name("Nobody", "Nothing")
with mock.patch.dict(siterender._icon_available, {icon: False}):
self.assertEqual(gs._console_heading("Nobody", "Nothing"), "## Nothing")
class TheSystemsIndexListsConsoles(unittest.TestCase):
"""The index listed 74 manufacturers and no console: PlayStation had to
be known to be under Sony."""
def test_one_row_per_console_by_its_name(self):
groups = {
"Sony": {"PlayStation": [_file("a.bin", "a")], "PlayStation 2": [_file("b.bin", "b")]},
"Other": {"ScummVM": [_file("c.bin", "c")]},
"Game Engines": {"TheXTech": [_file("d.png", "d")]},
}
page = gs.generate_systems_index(groups)
self.assertIn("| [Sony PlayStation](sony.md#playstation) | 1 | [Sony](sony.md) |", page)
self.assertIn("| [Sony PlayStation 2](sony.md#playstation-2) | 1 | [Sony](sony.md) |",
page)
# A program's folder under Other is named alone, in its own table.
consoles = page.split("## Emulator folders")[0]
self.assertNotIn("ScummVM", consoles)
self.assertIn("| [ScummVM](other.md#scummvm) | 1 | [Other](other.md) |",
page.split("## Emulator folders")[1])
self.assertNotIn("TheXTech", consoles)
self.assertIn("| [TheXTech](game-engines.md#thextech)", page)
def test_rows_go_by_name_not_by_maker(self):
groups = {"Sega": {"Saturn": [_file("a.bin", "a")]},
"Atari": {"Lynx": [_file("b.bin", "b")]}}
page = gs.generate_systems_index(groups)
self.assertLess(page.index("Atari Lynx"), page.index("Sega Saturn"))
class FindAFileSaysWhatAHashIs(unittest.TestCase):
def test_the_sentence_above_the_box(self):
page = gs.generate_files_page()
intro = page.split('<form class="rb-file-lookup"')[0]
self.assertIn("A hash is a fingerprint of a file's contents", intro)
self.assertIn("(../wiki/glossary.md)", intro)
self.assertNotIn("**permalink**", page)
@unittest.skipUnless((DIST / "systems" / "sony" / "index.html").is_file(), "site not built")
class TheBuiltSonyPage(unittest.TestCase):
def setUp(self):
self.text = (DIST / "systems" / "sony" / "index.html").read_text(encoding="utf-8")
def test_no_card_title_says_permalink(self):
titles = re.findall(r'<h3[^>]*class="rb-file-title".*?</h3>', self.text, re.S)
self.assertTrue(titles)
for title in titles:
self.assertNotIn("permalink", re.sub(r"<[^>]+>", "", title))
def test_playstation_opens_on_a_bios_it_needs(self):
section = self.text.split('id="playstation"', 1)[1]
first = re.search(r'class="rb-file-title"[^>]*><code>(.*?)</code>', section)[1]
self.assertNotIn(first, {"LibPS.exe", "PSX_DTLH2000.bin"})
self.assertRegex(first, r"^scph\d+")
self.assertIn("Start with", section.split('class="rb-sys-file"', 1)[0])
def test_headings_have_no_leading_space(self):
self.assertNotRegex(self.text, r'<h2 id="playstation[^"]*"><img[^>]*>\s')
if __name__ == "__main__":
unittest.main()
+8 -7
View File
@@ -354,15 +354,16 @@ class TheSearchSkipsLinkLists(unittest.TestCase):
consumers.installed = {sha1: {"batocera": {"bios/a.bin"}}}
consumers.emulators = {sha1: {"duckstation"}}
entry = {"name": "a.bin", "sha1": sha1, "md5": "b" * 32, "size": 3}
text = "\n".join(generate_site._system_file_card(entry, consumers))
searched, _sep, skipped = text.partition("<section data-pagefind-ignore>")
self.assertIn("<code>a.bin</code>", searched)
self.assertNotIn("- MD5:", searched)
for line in ("- MD5:", "- Emulators:", "- Also installed by:"):
self.assertIn(line, skipped)
lines = generate_site._system_file_card(entry, consumers)
self.assertIn("<code>a.bin</code>", lines[2])
text = "\n".join(lines)
searched, _sep, skipped = text.partition('<dl class="rb-facts" data-pagefind-ignore>')
for label in ("MD5", "Emulators", "Packed for its emulators by"):
self.assertIn(f"<dt>{label}</dt>", skipped)
self.assertNotIn(f">{label}</dt>", searched)
# The platforms line names the file as each setup does: the site
# search reads it, so a file name finds its card.
self.assertIn("- Platforms:", searched + skipped.split("</section>", 1)[1])
self.assertRegex(searched, r'<dl class="rb-facts">\n<dt[^>]*>On the setup\'s list</dt>')
def test_a_game_data_card_skips_its_name_too(self):
import generate_site
+1 -1
View File
@@ -55,7 +55,7 @@ Every module in `tests/`, by the area it guards.
| Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
| Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_schema_semantic_gate`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
## Test architecture