still lands, and site.js sends a card
+ anchor the page no longer holds to the card's new place. The pages
+ each card and console went to are recorded for every other link
+ (siterender.record_console_pages).
+ """
+ slug = _slug(manufacturer)
+ json_link = (f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json)")
+ whole_anchors: set[str] = set()
+ sections = {
+ console: _console_cards(manufacturer, consoles[console], consumers, whole_anchors)
+ for console in sorted(consoles)
+ }
+ lines = [
+ f"# {manufacturer}", "",
+ f"The files below by console, with every hash, are also published as {json_link}.",
+ "",
+ ]
+ for console, cards in sections.items():
+ lines.append(f"## {system_icon_markdown(manufacturer, console)}{console}")
+ lines.append("")
+ lines.extend(_console_install_block(console, (installs or {}).get(console, [])))
+ lines.extend(_cards_markdown(cards))
+ lines.append("")
+ lines.extend(_next_block(_NEXT_FROM_SYSTEMS))
+ lines.append(f'Generated on {_timestamp()}.
')
+ whole = "\n".join(lines) + "\n"
+ if len(whole) <= budget:
+ return {slug: (whole, consoles)}
+
+ # The heaviest sections move to pages of their own until what stays on
+ # the manufacturer's page is half the budget: a section of one file
+ # keeps its place, and a reader does not open 91 pages to browse Other.
+ weights = {
+ console: sum(len(line) + 1 for _variant, card, _sha1 in cards for line in card)
+ for console, cards in sections.items()
+ }
+ moved: set[str] = set()
+ for console in sorted(weights, key=lambda c: (-weights[c], c)):
+ if sum(w for c, w in weights.items() if c not in moved) <= budget // 2:
+ break
+ moved.add(console)
+
+ pages: dict[str, tuple[str, dict[str, list]]] = {}
+ summary = [
+ f"# {manufacturer}", "",
+ f"The files below by console, with every hash, are also published as {json_link}. "
+ f"The largest sections have a page of their own, linked from their heading.",
+ "",
+ ]
+ summary_anchors: set[str] = set()
+ for console in sorted(consoles):
+ summary.append(f"## {system_icon_markdown(manufacturer, console)}{console}")
+ summary.append("")
+ if console not in moved:
+ summary.extend(_console_install_block(console, (installs or {}).get(console, [])))
+ summary.extend(_cards_markdown(
+ _console_cards(manufacturer, consoles[console], consumers, summary_anchors)))
+ summary.append("")
+ continue
+ # Cards are rendered again for their own page: the anchors a page
+ # holds are its own.
+ cards = _console_cards(manufacturer, consoles[console], consumers, set())
+ parts = _parts(cards, budget)
+ base = f"{slug}-{_slugify_anchor(console) or 'files'}"
+ stems = [base if index == 0 else f"{base}-{index + 1}" for index in range(len(parts))]
+ record_console_pages(slug, console, stems, [
+ [sha1 for _variant, _card, sha1 in part] for part in parts
+ ])
+ title = f"{manufacturer}: {console}"
+ count = _count(len(consoles[console]), "file")
+ if len(stems) == 1:
+ summary.append(f"{count}, on [their own page]({stems[0]}.md).")
+ else:
+ summary.append(f"{count}, on {len(stems)} pages by file name:")
+ summary.append("")
+ for index, (stem, part) in enumerate(zip(stems, parts)):
+ summary.append(f"- [Part {index + 1}]({stem}.md): {_part_range(part)}")
+ summary.append("")
+ for index, (stem, part) in enumerate(zip(stems, parts)):
+ heading = title if len(stems) == 1 else f"{title}, part {index + 1} of {len(stems)}"
+ lines = [
+ f"# {heading}", "",
+ f"Part of [{manufacturer}]({slug}.md), in [Systems](index.md). "
+ f"Every hash of these files is also published as {json_link}.",
+ "",
+ ]
+ if len(stems) > 1:
+ others = ", ".join(
+ f"[part {i + 1}]({other}.md)" for i, other in enumerate(stems) if i != index)
+ lines.extend([f"This part holds {_part_range(part)}. The others: {others}.", ""])
+ lines.extend(_console_install_block(console, (installs or {}).get(console, [])))
+ lines.extend(["## Files", ""])
+ lines.extend(_cards_markdown(part))
+ lines.append("")
+ lines.extend(_next_block([f"[{manufacturer}]({slug}.md), every section"]
+ + _NEXT_FROM_SYSTEMS))
+ lines.append(f'Generated on {_timestamp()}.
')
+ held = {sha1 for _variant, _card, sha1 in part}
+ pages[stem] = ("\n".join(lines) + "\n", {console: [
+ f for f in consoles[console] if str(f.get("sha1", "")) in held]})
+ summary.extend(_next_block(_NEXT_FROM_SYSTEMS))
+ summary.append(f'Generated on {_timestamp()}.
')
+ kept = {console: files for console, files in consoles.items() if console not in moved}
+ return {slug: ("\n".join(summary) + "\n", kept), **pages}
+
+
+def _part_range(part: list[Card]) -> str:
+ """The first and last file names of a part, as the cards show them."""
+ names = [re.search(r"`([^`]*)`", card[2]).group(1) for _variant, card, _sha1 in part]
+ if len(names) == 1:
+ return f"`{names[0]}`"
+ return f"{_count(len(names), 'file')}, `{names[0]}` to `{names[-1]}`"
+
+
def generate_system_page(
manufacturer: str,
consoles: dict[str, list],
@@ -3427,53 +3672,10 @@ def generate_system_page(
"""One manufacturer's page, a section per console.
``installs`` gives, per console, the (platform, system id) pairs whose
- install command the section offers.
+ install command the section offers. A page over the budget is the
+ summary system_pages writes, its consoles on pages of their own.
"""
- slug = manufacturer.lower().replace(" ", "-")
- lines = [
- f"# {manufacturer}",
- "",
- f"The files below by console, with every hash, are also published as "
- f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json).",
- "",
- ]
- anchored: set[str] = set()
- searchable = manufacturer not in GAME_DATA_TOPS
-
- for console_name in sorted(consoles.keys()):
- files = consoles[console_name]
- icon_md = system_icon_markdown(manufacturer, console_name)
- lines.append(f"## {icon_md}{console_name}")
- lines.append("")
- lines.extend(_console_install_block(console_name, (installs or {}).get(console_name, [])))
- main_files = [f for f in files if "/.variants/" not in f["path"]]
- variant_files = [f for f in files if "/.variants/" in f["path"]]
-
- for f in sorted(main_files, key=lambda x: x["name"]):
- lines.extend(_system_file_card(f, consumers, anchored, searchable))
-
- if variant_files:
- # A variant gets the same card as a primary: the revision a
- # platform pins is the one a reader comes to find, and showed as
- # a name and a truncated MD5, nothing said who pins it.
- lines.append(f'??? note "Variants ({len(variant_files)})"')
- lines.append("")
- for v in sorted(variant_files, key=lambda x: (x["name"], x["path"])):
- lines.extend(
- f" {line}" if line else ""
- for line in _system_file_card(v, consumers, anchored, searchable)
- )
- lines.append("")
-
- lines.append("")
-
- lines.extend(_next_block([
- "[Get your files](../which-pack.md), for the setup you play on",
- "[When a game does not start](../wiki/troubleshooting.md)",
- "[Every emulator](../emulators/index.md), and the files each one reads",
- ]))
- lines.append(f'Generated on {_timestamp()}.
')
- return "\n".join(lines) + "\n"
+ return system_pages(manufacturer, consoles, consumers, installs)[_slug(manufacturer)][0]
# Emulator pages
@@ -3914,7 +4116,7 @@ def _render_emulator_file(
# Which copy answers this entry, linked to its collection card
# (provenance, every hash, download): an entry declaring no hash
# said "in repo" without naming the file.
- page = parts[1].lower().replace(" ", "-")
+ page = card_page(held, sha1)
details.append(
f"Collection copy: [{_code(held)}]"
f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})"
@@ -3988,7 +4190,7 @@ def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str:
archive = _code(path)
zsha1 = consumers.sha1_at(path)
if zsha1 and len(parts) >= 3 and parts[0] == "bios":
- page = urllib.parse.quote(parts[1].lower().replace(" ", "-"))
+ page = urllib.parse.quote(card_page(path, zsha1))
archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})"
return f"Collection copy: member {_code(member)} of {archive}"
if parts[0] == "data" and len(parts) >= 2:
@@ -4020,8 +4222,7 @@ def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]:
parts = folder.split("/")
where = _code(folder + "/")
if len(parts) >= 3 and parts[0] == "bios":
- page = urllib.parse.quote(parts[1].lower().replace(" ", "-"))
- where = f"[{where}](../systems/{page}.md#{_slugify_anchor(parts[2])})"
+ where = f"[{where}]({console_href(_slug(parts[1]), parts[2], '../')})"
tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(folder)}"
return [f"Collection copies: {_count(len(members), 'file')} under {where} "
f"([browse]({tree}))"]
@@ -4369,6 +4570,41 @@ _DATA_MEMBERS: dict[str, dict[str, dict]] = {}
_DATA_REVISIONS: dict[str, dict] = {}
+# An emulator page lists at most this many file cards: MAME's 1,564 made a
+# 3 MB page. Past it, the cards go to pages of their own and the emulator's
+# page keeps a table of every entry, each row holding the card's old
+# anchor. A count, not a size: system cards link to these cards and are
+# written before them.
+EMULATOR_CARDS_PER_PAGE = 400
+
+
+def emulator_parts(name: str, files: list) -> list[tuple[str, range]]:
+ """The pages an emulator's cards go to, with the entries each holds,
+ or none when they fit on the emulator's page. Balanced, in profile order."""
+ if len(files) <= EMULATOR_CARDS_PER_PAGE:
+ return []
+ count = -(-len(files) // EMULATOR_CARDS_PER_PAGE)
+ size = -(-len(files) // count)
+ return [(f"{name}-files-{index + 1}", range(start, min(start + size, len(files))))
+ for index, start in enumerate(range(0, len(files), size))]
+
+
+def plan_emulator_pages(profiles: dict) -> None:
+ """Record which page holds each emulator card, before any page links one."""
+ stems = set()
+ for name, profile in sorted(profiles.items()):
+ if profile.get("type") in ("alias", "test"):
+ continue
+ files = profile.get("files") or []
+ anchors = emulator_file_anchors(files)
+ for stem, entries in emulator_parts(name, files):
+ stems.add(stem)
+ record_emulator_cards(name, stem, [anchors[i] for i in entries])
+ clash = stems & set(profiles)
+ if clash:
+ raise RuntimeError(f"an emulator's card page takes a profile's address: {sorted(clash)}")
+
+
def generate_emulator_page(
name: str,
profile: dict,
@@ -4379,13 +4615,28 @@ def generate_emulator_page(
data_dirs: list[dict] | None = None,
) -> str:
"""One emulator's page. ``data_dirs`` defaults to the profile's own."""
+ return emulator_pages(
+ name, profile, db, consumers, data_names, data_registry, data_dirs)[name]
+
+
+def emulator_pages(
+ name: str,
+ profile: dict,
+ db: dict,
+ consumers: FileConsumers | None = None,
+ data_names: set[str] | None = None,
+ data_registry: dict | None = None,
+ data_dirs: list[dict] | None = None,
+) -> dict[str, str]:
+ """An emulator's page by stem under emulators/, and the pages its cards
+ went to when there are more than EMULATOR_CARDS_PER_PAGE."""
if profile.get("type") == "alias":
parent = profile.get("alias_of", profile.get("bios_identical_to", "unknown"))
- return (
+ return {name: (
f"# {name}\n\n"
f"This core uses the same firmware as **{parent}**.\n\n"
f"See [{parent}]({parent}.md) for details.\n"
- )
+ )}
emu_name = profile.get("emulator", name)
emu_type = profile.get("type", "unknown")
@@ -4419,12 +4670,13 @@ def generate_emulator_page(
if data_dirs is None:
data_dirs = profile.get("data_directories", [])
+ part_pages: dict[str, str] = {}
lines = [
f"# {emu_name}",
"",
'',
"",
- "| | |",
+ "| Detail | Value |",
"|---|---|",
f"| Type | {emu_type} |",
]
@@ -4522,16 +4774,25 @@ def generate_emulator_page(
else:
_file_available = _availability_check(db, data_names)
+ lines.extend(["## Files", ""])
lines.extend(_emulator_file_summary(files, _file_available))
anchors = emulator_file_anchors(files)
- for f, anchor, alias in zip(files, anchors, emulator_file_aliases(files, anchors)):
- lines.extend(
- _render_emulator_file(
- f, profile, consumers, files, _file_available, anchor, name,
- alias,
- )
- )
+ aliases = emulator_file_aliases(files, anchors)
+ cards = [
+ _render_emulator_file(f, profile, consumers, files, _file_available, anchor, name, alias)
+ for f, anchor, alias in zip(files, anchors, aliases)
+ ]
+ parts = emulator_parts(name, files)
+ if not parts:
+ for card in cards:
+ lines.extend(card)
+ else:
+ lines.extend(_card_index(emu_name, files, anchors, aliases, parts))
+ for number, (stem, entries) in enumerate(parts, 1):
+ part_pages[stem] = _emulator_part_page(
+ name, emu_name, number, parts, [cards[i] for i in entries],
+ [files[i] for i in entries])
if data_dirs:
# The member tables are a file list with hashes: out of the search,
@@ -4593,6 +4854,63 @@ def generate_emulator_page(
"[When a game does not start](../wiki/troubleshooting.md)",
]))
lines.extend([f"*Generated on {_timestamp()}*"])
+ return {name: "\n".join(lines) + "\n", **part_pages}
+
+
+def _card_index(
+ emu_name: str, files: list[dict], anchors: list[str], aliases: list[str],
+ parts: list[tuple[str, range]],
+) -> list[str]:
+ """Every entry of a split emulator page, one row each, linked to its card.
+
+ Each row holds the ids the card had on this page, so a link cited
+ before the split lands on the row, and site.js follows it to the card.
+ """
+ page_of = {index: stem for stem, entries in parts for index in entries}
+ lines = [
+ f"{emu_name} reads more files than one page holds: each entry below "
+ f"links to its card, on {_count(len(parts), 'page')} by profile order "
+ + ", ".join(f"[{number}]({stem}.md)" for number, (stem, _e) in enumerate(parts, 1))
+ + ".",
+ "",
+ '',
+ "",
+ "| File | Required | Category |",
+ "|------|----------|----------|",
+ ]
+ for index, (f, anchor, alias) in enumerate(zip(files, anchors, aliases)):
+ ids = "".join(f'' for a in (anchor, alias) if a)
+ shown = str(f.get("path") or f.get("name", ""))
+ lines.append(
+ f"| {ids}[{_code(shown)}]({page_of[index]}.md#{anchor}) "
+ f"| {'yes' if f.get('required') else 'no'} | {f.get('category', 'bios')} |"
+ )
+ lines.extend(["", "", ""])
+ return lines
+
+
+def _emulator_part_page(
+ name: str, emu_name: str, number: int, parts: list[tuple[str, range]],
+ cards: list[list[str]], files: list[dict],
+) -> str:
+ """One page of a split emulator's cards."""
+ first = str(files[0].get("path") or files[0].get("name", ""))
+ last = str(files[-1].get("path") or files[-1].get("name", ""))
+ others = ", ".join(
+ f"[{n}]({stem}.md)" for n, (stem, _e) in enumerate(parts, 1) if n != number)
+ lines = [
+ f"# {emu_name} files, part {number} of {len(parts)}",
+ "",
+ f"Part of [{emu_name}]({name}.md), whose page has the profile, the install "
+ f"command and every entry. This part holds {_count(len(files), 'entry', 'entries')}, "
+ f"{_code(first)} to {_code(last)}, in the profile's order. The other parts: {others}.",
+ "",
+ "## Files",
+ "",
+ *(line for card in cards for line in card),
+ ]
+ lines.extend(_next_block([f"[{emu_name}]({name}.md), the profile and every entry"]))
+ lines.extend([f"*Generated on {_timestamp()}*"])
return "\n".join(lines) + "\n"
@@ -5947,19 +6265,35 @@ def nav_emulators(profiles: dict) -> dict[str, dict]:
def _emulator_nav(profiles: dict) -> list:
- """Emulators in alphabetical groups ("A to C"), by the name a user knows."""
- groups: dict[str, list[tuple[str, str]]] = {}
+ """Emulators in alphabetical groups ("A to C"), then by initial.
+
+ Material writes the open section of the navigation into every page: a
+ group of 96 emulators put 65 KB of links before each emulator page's
+ content. One initial is at most 60.
+ """
+ groups: dict[str, dict[str, list[tuple[str, str]]]] = {}
for key, profile in nav_emulators(profiles).items():
display = str(profile.get("emulator", key))
initial = display[:1].upper()
letters = next((g for g in EMULATOR_NAV_GROUPS if initial in g), "")
label = f"{letters[0]} to {letters[-1]}" if letters else "0 to 9"
- groups.setdefault(label, []).append((display, f"emulators/{key}.md"))
+ sub = initial if letters else label
+ groups.setdefault(label, {}).setdefault(sub, []).append((display, f"emulators/{key}.md"))
+
+ def entries(items: list[tuple[str, str]]) -> list[dict]:
+ return [{d: path} for d, path in sorted(items, key=lambda e: e[0].casefold())]
+
order = ["0 to 9"] + [f"{g[0]} to {g[-1]}" for g in EMULATOR_NAV_GROUPS]
- return ["emulators/index.md"] + [
- {label: [{d: path} for d, path in sorted(groups[label], key=lambda e: e[0].casefold())]}
- for label in order if label in groups
- ]
+ nav: list = ["emulators/index.md"]
+ for label in order:
+ if label not in groups:
+ continue
+ subs = groups[label]
+ if len(subs) == 1:
+ nav.append({label: entries(next(iter(subs.values())))})
+ else:
+ nav.append({label: [{sub: entries(subs[sub])} for sub in sorted(subs)]})
+ return nav
def _system_nav(manufacturers: dict) -> list:
@@ -6174,14 +6508,8 @@ def _generate(args: argparse.Namespace) -> None:
# report the same files.
gap_report = build_emulator_gap_report(profiles, coverages, db, suppl_names)
- print("Generating static API and bulk data exports...")
- exports = generate_data_exports(
- docs, db, coverages, profiles, stats, gap_report
- )
- write_if_changed(str(docs / "data.md"), generate_data_page(exports) + REFERENCE_NEXT)
- PAGE_DATA["data.md"] = dataset_structured_data(exports, db)
-
# Build system_id -> manufacturer page map (needed by all generators)
+ plan_emulator_pages(profiles)
print("Building system cross-reference map...")
manufacturers = _group_by_manufacturer(db)
set_system_page_map(
@@ -6189,25 +6517,6 @@ def _generate(args: argparse.Namespace) -> None:
)
print(f" {len(_system_page_map)} system IDs mapped to pages")
- export_reports = load_export_reports()
- _EXPORTED.update(report["platform"] for report in export_reports)
-
- print("Generating platform pages...")
- write_if_changed(
- str(docs / "platforms" / "index.md"),
- generate_platform_index(coverages, registry),
- )
- for name, cov in coverages.items():
- # The emulators this platform runs, not every one of the system.
- runs = resolve_platform_cores(cov["config"], profiles)
- write_if_changed(
- str(docs / "platforms" / f"{name}.md"),
- generate_platform_page(name, cov, registry, {
- emu: data for emu, data in emulator_files.items() if emu in runs
- }),
- )
- PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov)
-
print("Generating system pages...")
prime_system_icons({
@@ -6222,16 +6531,55 @@ def _generate(args: argparse.Namespace) -> None:
console_installs = system_install_choices(coverages)
(docs / "files").mkdir(parents=True, exist_ok=True)
write_if_changed(str(docs / "files" / "index.md"), generate_files_page())
+ reset_console_pages()
for mfr, consoles in manufacturers.items():
slug = mfr.lower().replace(" ", "-")
- page = generate_system_page(mfr, consoles, consumers, console_installs.get(slug))
- PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles)
+ pages = system_pages(mfr, consoles, consumers, console_installs.get(slug))
+ for stem, (page, held) in pages.items():
+ PAGE_DATA[f"systems/{stem}.md"] = system_structured_data(stem, held)
+ if stem != slug:
+ PART_PAGES.add(f"systems/{stem}.md")
+ write_if_changed(str(docs / "systems" / f"{stem}.md"), page)
write_single_item(
docs / "api" / "v1" / "systems" / f"{slug}.json", "systems",
system_export_item(slug, mfr, consoles),
str(db.get("generated_at") or _timestamp()),
)
- write_if_changed(str(docs / "systems" / f"{slug}.md"), page)
+ clash = split_system_stems() & {_slug(m) for m in manufacturers}
+ if clash:
+ raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}")
+
+ print("Generating static API and bulk data exports...")
+ exports = generate_data_exports(
+ docs, db, coverages, profiles, stats, gap_report
+ )
+ write_if_changed(str(docs / "data.md"), generate_data_page(exports) + REFERENCE_NEXT)
+ PAGE_DATA["data.md"] = dataset_structured_data(exports, db)
+
+ export_reports = load_export_reports()
+ _EXPORTED.update(report["platform"] for report in export_reports)
+
+ print("Generating platform pages...")
+ write_if_changed(
+ str(docs / "platforms" / "index.md"),
+ generate_platform_index(coverages, registry),
+ )
+ for name, cov in coverages.items():
+ # The emulators this platform runs, not every one of the system.
+ runs = resolve_platform_cores(cov["config"], profiles)
+ moved: dict[str, str] = {}
+ write_if_changed(
+ str(docs / "platforms" / f"{name}.md"),
+ generate_platform_page(name, cov, registry, {
+ emu: data for emu, data in emulator_files.items() if emu in runs
+ }, moved),
+ )
+ for stem, page in moved.items():
+ if stem in coverages:
+ raise RuntimeError(f"a system page takes the platform {stem}'s address")
+ PART_PAGES.add(f"platforms/{stem}.md")
+ write_if_changed(str(docs / "platforms" / f"{stem}.md"), page)
+ PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov)
print("Generating emulator pages...")
_DATA_MEMBERS.update(data_directory_members())
@@ -6264,12 +6612,18 @@ def _generate(args: argparse.Namespace) -> None:
indent=2, sort_keys=True, ensure_ascii=False, default=str,
) + "\n",
)
- page = generate_emulator_page(
+ pages = emulator_pages(
name, profile, db, consumers, suppl_names, data_registry,
data_dirs=page_data_directories(name, profiles),
)
PAGE_DATA[f"emulators/{name}.md"] = emulator_structured_data(profile)
- write_if_changed(str(docs / "emulators" / f"{name}.md"), page)
+ files = profile.get("files") or []
+ for stem, entries in emulator_parts(name, files):
+ PAGE_DATA[f"emulators/{stem}.md"] = emulator_structured_data(
+ {**profile, "files": [files[i] for i in entries]})
+ PART_PAGES.add(f"emulators/{stem}.md")
+ for stem, page in pages.items():
+ write_if_changed(str(docs / "emulators" / f"{stem}.md"), page)
print("Generating cross-reference page...")
write_if_changed(
@@ -6323,6 +6677,11 @@ def _generate(args: argparse.Namespace) -> None:
print("Generating contributing page...")
write_if_changed(str(docs / "contributing.md"), generate_contributing())
+ # Swept before the decoration, which rewrites every page it finds: a
+ # page no generator produced any more (a console moved back onto its
+ # manufacturer's page) was decorated, recorded, and kept for ever.
+ stale = _sweep_generated(docs, GENERATED_DIRS)
+
print("Adding page metadata and structured data...")
decorate_markdown_pages(docs)
@@ -6421,19 +6780,17 @@ markdown_extensions:
- tables
- toc:
permalink: true
- # File cards are level 4 headings: the search lands on them, and a
- # page of a hundred cards keeps a readable table of contents.
+ # Level 3 is the wiki's subsections. On pages of file cards, also level
+ # 3, the hook keeps the table of contents to level 2.
toc_depth: 3
- pymdownx.details
- pymdownx.highlight:
anchor_linenums: true
- pymdownx.inlinehilite
- pymdownx.keys
-- pymdownx.superfences:
- custom_fences:
- - name: mermaid
- class: mermaid
- format: !!python/name:pymdownx.superfences.fence_code_format
+# No mermaid fence: Material drew it in the browser with a script from a
+# CDN. Diagrams are SVG files written by scripts/sitediagrams.py.
+- pymdownx.superfences
- pymdownx.tabbed:
alternate_style: true
plugins:
@@ -6449,26 +6806,11 @@ validation:
config_path, config_text = _mkdocs_config(Path(docs), mkdocs_static + nav_yaml)
write_if_changed(str(config_path), config_text)
- total_pages = (
- 1 # home
- + 1
- + len(coverages) # platform index + detail
- + 1
- + len(manufacturers) # system index + detail
- + 1 # cross-reference
- + 1
- + sum(
- 1 for profile in profiles.values()
- if profile.get("type") not in ("alias", "test")
- ) # emulator detail pages (aliases/tests remain metadata-only)
- + 1 # gap analysis
- + 1 # which-pack
- + len(list(Path("wiki").glob("*.md"))) # wiki pages copied verbatim
- + 1 # generated wiki/data-model
- + 1 # contributing
- + 1 # data and API
+ # Counted on disk: the split pages made a sum of page kinds wrong.
+ total_pages = sum(
+ 1 for path in docs.rglob("*.md")
+ if not any(part in LOCAL_NOTE_DIRS for part in path.relative_to(docs).parts[:-1])
)
- stale = _sweep_generated(docs, GENERATED_DIRS)
print(f"\nGenerated {total_pages} pages in {args.docs_dir}/")
if stale:
print(f"Removed {stale} page(s) nothing produced this run")
@@ -6500,6 +6842,13 @@ from siterender import ( # noqa: E402,F401
_system_page_map,
_system_link,
set_system_page_map,
+ card_page,
+ console_href,
+ record_console_pages,
+ record_emulator_cards,
+ emulator_card_page,
+ reset_console_pages,
+ split_system_stems,
_platform_link,
_emulator_link,
_fmt_size,
diff --git a/scripts/mkdocs_hooks.py b/scripts/mkdocs_hooks.py
index 0028a3e1..11cef35d 100644
--- a/scripts/mkdocs_hooks.py
+++ b/scripts/mkdocs_hooks.py
@@ -10,8 +10,28 @@ from __future__ import annotations
from typing import Any
+# Pages made of file cards, level 3 headings by the hundred: their table of
+# contents stops at level 2, the consoles of a system page, the Files of an
+# emulator, the sections of a platform page (which listed every system id).
+CARD_PAGES = ("systems/", "emulators/", "platforms/")
+
def on_page_markdown(markdown: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001
"""Take lastmod from the page's own date, or leave it out."""
page.update_date = str(page.meta.get("date_modified") or "")
return markdown
+
+
+def _prune(items: list, depth: int) -> None:
+ for item in items:
+ if item.level >= depth:
+ item.children = []
+ else:
+ _prune(item.children, depth)
+
+
+def on_page_content(html: str, page: Any, config: Any, files: Any) -> str: # noqa: ARG001
+ """Keep the table of contents of a page of cards to level 2."""
+ if page.file.src_uri.startswith(CARD_PAGES):
+ _prune(page.toc.items, 2)
+ return html
diff --git a/scripts/siterender.py b/scripts/siterender.py
index f0b1cbc1..0683ad43 100644
--- a/scripts/siterender.py
+++ b/scripts/siterender.py
@@ -139,13 +139,75 @@ def _page_by_norm(sys_id: str) -> tuple[str, str] | None:
return _norm_page_map.get(_norm_system_id(sys_id))
+# A manufacturer page too large for one page is a summary, each console on
+# pages of its own: (slug, console) -> its page stems under systems/, and
+# (slug, console, sha1) -> the stem holding that card. A console of a page
+# kept whole has no entry, its section is on the manufacturer's page.
+_console_pages: dict[tuple[str, str], list[str]] = {}
+_card_pages: dict[tuple[str, str, str], str] = {}
+
+
+def reset_console_pages() -> None:
+ _console_pages.clear()
+ _card_pages.clear()
+
+
+def record_console_pages(
+ slug: str, console: str, stems: list[str], sha1s: list[list[str]]
+) -> None:
+ """Where a split manufacturer page put one console: its page stems, in
+ order, and the sha1 of each card on each of them."""
+ _console_pages[(slug, console)] = list(stems)
+ for stem, part in zip(stems, sha1s):
+ for sha1 in part:
+ _card_pages.setdefault((slug, console, sha1), stem)
+
+
+def split_system_stems() -> set[str]:
+ """Every page stem of a console moved to a page of its own."""
+ return {stem for stems in _console_pages.values() for stem in stems}
+
+
+def console_href(slug: str, console: str, prefix: str = "") -> str:
+ """Where a console's files are: its own page, or its section of the
+ manufacturer's page (also where a console cut into parts lists them)."""
+ stems = _console_pages.get((slug, console))
+ if stems and len(stems) == 1:
+ return f"{prefix}systems/{urllib.parse.quote(stems[0])}.md"
+ return f"{prefix}systems/{urllib.parse.quote(slug)}.md#{_slugify_anchor(console)}"
+
+
+def card_page(path: str, sha1: str) -> str:
+ """The systems/ page stem holding the card of the copy at ``path``
+ (bios///...)."""
+ parts = path.split("/")
+ slug = parts[1].lower().replace(" ", "-")
+ console = parts[2] if len(parts) > 2 else ""
+ return _card_pages.get((slug, console, sha1), slug)
+
+
+# Emulator cards moved off a page too long for one: (emulator, anchor) ->
+# the page stem under emulators/ that holds the card.
+_emulator_card_pages: dict[tuple[str, str], str] = {}
+
+
+def record_emulator_cards(emulator: str, stem: str, anchors: list[str]) -> None:
+ for anchor in anchors:
+ _emulator_card_pages[(emulator, anchor)] = stem
+
+
+def emulator_card_page(emulator: str, anchor: str) -> str:
+ """The emulators/ page stem holding a card: the emulator's own page, or
+ the page its cards went to."""
+ return _emulator_card_pages.get((emulator, anchor), emulator)
+
+
def _system_link(sys_id: str, prefix: str = "") -> str:
"""Generate a markdown link to a system page with anchor."""
page = _system_page_map.get(sys_id) or _page_by_norm(sys_id)
if page:
slug, console = page
- anchor = _slugify_anchor(console)
- return f"[{sys_id}]({prefix}systems/{slug}.md#{anchor})"
+ return f"[{sys_id}]({console_href(slug, console, prefix)})"
return sys_id
def _field_text(key: object, value: object) -> str:
diff --git a/tests/test_site_render.py b/tests/test_site_render.py
index 8188ece8..a5b4d6f0 100644
--- a/tests/test_site_render.py
+++ b/tests/test_site_render.py
@@ -323,7 +323,7 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
entry = {"name": "scph5501.bin", "required": True}
lines = generate_site._render_emulator_file(
entry, {"files": [entry]}, None, [entry], lambda f: True, "file-scph5501-bin")
- self.assertTrue(lines[2].startswith("#### `scph5501.bin` "))
+ self.assertTrue(lines[2].startswith("### `scph5501.bin` "))
self.assertTrue(lines[2].endswith("{ #file-scph5501-bin .rb-file-title }"))
def test_a_system_card_is_a_heading_with_its_anchor(self):
@@ -332,7 +332,7 @@ class TheSearchLandsOnTheCard(unittest.TestCase):
entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3}
lines = generate_site._system_file_card(entry, FileConsumers({"files": {}}))
- self.assertTrue(lines[2].startswith("#### `a.bin` "))
+ self.assertTrue(lines[2].startswith("### `a.bin` "))
self.assertTrue(lines[2].endswith("{ #sha1-aaaaaaaaaaaa .rb-file-title }"))
diff --git a/tests/test_site_weight.py b/tests/test_site_weight.py
new file mode 100644
index 00000000..a524d7ed
--- /dev/null
+++ b/tests/test_site_weight.py
@@ -0,0 +1,168 @@
+"""Pages a phone can open: split pages, light navigation, short tables of contents.
+
+systems/game-engines/ was one 10.5 MB page, emulators/mame/ 3 MB, and each
+of 600 emulator pages carried 65 KB of navigation. A large page is now a
+summary with pages of its own, and every address cited before the split
+still lands: a console's heading stays on the summary, a card's sha1 anchor
+is looked up by content, an emulator card's anchor sits on its row.
+"""
+
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import generate_site # noqa: E402
+import mkdocs_hooks # noqa: E402
+import siterender # noqa: E402
+from cross_reference import FileConsumers # noqa: E402
+
+
+def _file(name: str, index: int, console: str, maker: str = "Maker") -> dict:
+ sha1 = f"{index:012x}" + "0" * 28
+ return {"name": name, "sha1": sha1, "md5": f"{index:032x}", "size": 10,
+ "path": f"bios/{maker}/{console}/{name}"}
+
+
+class ASystemPageOverItsBudgetIsSplit(unittest.TestCase):
+ def setUp(self) -> None:
+ siterender.reset_console_pages()
+ self.addCleanup(siterender.reset_console_pages)
+ self.consoles = {
+ "Big": [_file(f"big{i:03d}.bin", i, "Big") for i in range(60)],
+ "Small": [_file("small.bin", 900, "Small")],
+ }
+
+ def _pages(self, budget: int) -> dict:
+ return generate_site.system_pages(
+ "Maker", self.consoles, FileConsumers({"files": {}}), budget=budget)
+
+ def test_a_page_within_its_budget_stays_whole(self):
+ pages = self._pages(10_000_000)
+ self.assertEqual(list(pages), ["maker"])
+ self.assertIn("## Big", pages["maker"][0])
+
+ def test_the_largest_section_moves_and_is_cut_in_parts(self):
+ pages = self._pages(20_000)
+ summary = pages["maker"][0]
+ # Both headings stay on the summary: systems/maker/#big still lands.
+ self.assertIn("## Big", summary)
+ self.assertIn("## Small", summary)
+ # The small section keeps its cards in place.
+ self.assertIn("{ #sha1-" + f"{900:012x}", summary)
+ parts = [stem for stem in pages if stem.startswith("maker-big")]
+ self.assertGreater(len(parts), 1)
+ self.assertEqual(parts[0], "maker-big")
+ for stem in parts:
+ self.assertIn(f"]({stem}.md)", summary)
+ self.assertLessEqual(len(pages[stem][0]), 20_000 + 5_000)
+ self.assertIn("## Files", pages[stem][0])
+ # Each card is on exactly one part, and the links follow it.
+ held = [f["sha1"] for stem in parts for f in pages[stem][1]["Big"]]
+ self.assertEqual(sorted(held), sorted(f["sha1"] for f in self.consoles["Big"]))
+ last = self.consoles["Big"][-1]
+ self.assertEqual(siterender.card_page(last["path"], last["sha1"]), parts[-1])
+ small = self.consoles["Small"][0]
+ self.assertEqual(siterender.card_page(small["path"], small["sha1"]), "maker")
+ self.assertEqual(siterender.console_href("maker", "Big"), "systems/maker.md#big")
+
+ def test_a_console_on_one_page_is_linked_directly(self):
+ self.consoles["Big"] = self.consoles["Big"][:12]
+ whole = self._pages(10**9)["maker"][0]
+ pages = self._pages(len(whole) - 1)
+ self.assertIn("maker-big", pages)
+ self.assertNotIn("maker-big-2", pages)
+ self.assertEqual(siterender.console_href("maker", "Big", "../"),
+ "../systems/maker-big.md")
+
+
+class AnEmulatorWithTooManyCardsIsSplit(unittest.TestCase):
+ def test_parts_are_balanced_and_rows_keep_the_anchors(self):
+ files = [{"name": f"set{i:04d}.zip", "required": i % 2 == 0, "category": "bios_zip"}
+ for i in range(generate_site.EMULATOR_CARDS_PER_PAGE * 2 + 1)]
+ parts = generate_site.emulator_parts("big", files)
+ self.assertEqual([stem for stem, _ in parts], ["big-files-1", "big-files-2", "big-files-3"])
+ sizes = [len(entries) for _, entries in parts]
+ self.assertLessEqual(max(sizes) - min(sizes), 1)
+ self.assertEqual(generate_site.emulator_parts("small", files[:3]), [])
+
+ pages = generate_site.emulator_pages(
+ "big", {"emulator": "Big", "type": "standalone", "files": files}, {"files": {}})
+ self.assertEqual(sorted(pages), ["big", "big-files-1", "big-files-2", "big-files-3"])
+ main = pages["big"]
+ anchors = generate_site.emulator_file_anchors(files)
+ for anchor in anchors:
+ self.assertIn(f'', main)
+ self.assertIn('class="rb-card-index"', main)
+ self.assertNotIn(".rb-file-title", main)
+ self.assertIn(f"{{ #{anchors[0]} .rb-file-title }}", pages["big-files-1"])
+
+ def test_cards_are_linked_on_the_page_that_holds_them(self):
+ files = [{"name": f"set{i:04d}.zip"} for i in range(generate_site.EMULATOR_CARDS_PER_PAGE + 1)]
+ generate_site.plan_emulator_pages({"big": {"files": files}})
+ anchors = generate_site.emulator_file_anchors(files)
+ self.assertEqual(siterender.emulator_card_page("big", anchors[-1]), "big-files-2")
+ self.assertEqual(siterender.emulator_card_page("other", "file-x"), "other")
+
+
+class APlatformSystemOverItsBudgetMoves(unittest.TestCase):
+ def test_the_heading_stays_and_the_list_moves(self):
+ files = [{"name": f"f{i:04d}.zip", "status": "ok"} for i in range(40)]
+ config = {("arcade", f["name"]): [{"name": f["name"], "md5": "a" * 32}] for f in files}
+ moved: dict[str, str] = {}
+ budget = generate_site.PLATFORM_SECTION_BUDGET
+ generate_site.PLATFORM_SECTION_BUDGET = 500
+ try:
+ lines = generate_site._render_platform_systems(
+ {"arcade": files}, config, {}, ("deck", "Deck"), moved)
+ finally:
+ generate_site.PLATFORM_SECTION_BUDGET = budget
+ text = "\n".join(lines)
+ self.assertIn("{ #arcade }", text)
+ self.assertIn("](deck-arcade.md)", text)
+ self.assertNotIn("f0001.zip", text)
+ self.assertIn("f0001.zip", moved["deck-arcade"])
+ self.assertIn("[Deck page](deck.md#arcade)", moved["deck-arcade"])
+
+
+class TheNavigationOpensOneInitial(unittest.TestCase):
+ def test_a_group_of_several_initials_is_nested(self):
+ profiles = {
+ key: {"emulator": name, "systems": ["s"]}
+ for key, name in (("a1", "Alpha"), ("b1", "Beta"), ("b2", "bravo"), ("z", "Zed"))
+ }
+ nav = generate_site._emulator_nav(profiles)
+ groups = {next(iter(g)): next(iter(g.values())) for g in nav[1:]}
+ self.assertEqual(groups["A to C"], [
+ {"A": [{"Alpha": "emulators/a1.md"}]},
+ {"B": [{"Beta": "emulators/b1.md"}, {"bravo": "emulators/b2.md"}]},
+ ])
+ self.assertEqual(groups["V to Z"], [{"Zed": "emulators/z.md"}])
+
+
+class TheTableOfContentsOfACardPageStopsAtLevelTwo(unittest.TestCase):
+ @staticmethod
+ def _item(level: int, children: list | None = None) -> SimpleNamespace:
+ return SimpleNamespace(level=level, children=children or [])
+
+ def test_cards_leave_the_toc_and_wiki_pages_keep_level_three(self):
+ def page(uri: str) -> SimpleNamespace:
+ h2 = self._item(2, [self._item(3)])
+ return SimpleNamespace(file=SimpleNamespace(src_uri=uri),
+ toc=SimpleNamespace(items=[self._item(1, [h2])]))
+
+ for uri, kept in (("systems/sony.md", 0), ("platforms/retroarch.md", 0),
+ ("emulators/mame.md", 0), ("wiki/faq.md", 1)):
+ current = page(uri)
+ mkdocs_hooks.on_page_content("", current, None, None)
+ with self.subTest(uri=uri):
+ self.assertEqual(len(current.toc.items[0].children[0].children), kept)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md
index 51bda600..f3ee044c 100644
--- a/wiki/testing-guide.md
+++ b/wiki/testing-guide.md
@@ -51,7 +51,7 @@ Every module in `tests/`, by the area it guards.
| Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` |
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
| Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
-| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
## Test architecture