')
+ lines.append('
')
+ lines.append("")
+ lines.append(_card_title(f"{_code(fname)} {badge_str}", anchor))
lines.append("")
- permalink = f'
#' if anchor else ""
- lines.append(f"**`{fname}`** {badge_str}{permalink}")
if desc:
- lines.append(f"
{_escape_tags(str(desc))}")
- lines.append("")
+ lines.extend([_escape_tags(str(desc)), ""])
+ body_start = len(lines)
details = []
if fpath and fpath != fname:
@@ -2871,7 +3000,7 @@ def _render_emulator_file(
details.append("Installed by the platform's own package on: "
+ ", ".join(sorted(f["installed_on"])))
if f.get("not_bundled_on"):
- details.append("Left out by the platform's package, so the pack brings it, on: "
+ details.append("Not in the platform's package, so the pack brings it, on: "
+ ", ".join(sorted(f["not_bundled_on"])))
if f.get("unread_on"):
details.append("Never opened on these platforms, whose launcher names other files: "
@@ -2886,6 +3015,8 @@ def _render_emulator_file(
details.append(
f"Source: {_source_ref_markdown(profile, source_ref)}"
)
+ if consumers and f.get("type") == "directory":
+ details.extend(_directory_copies(f, consumers))
if consumers:
sha1 = consumers.sha1_of(f, f.get("path") or "", owner=owner)
if sha1 and in_repo:
@@ -2949,12 +3080,34 @@ def _render_emulator_file(
lines.append(f" - `{c.get('name', '')}`{extra}")
else:
lines.append(f" - {c}")
+ lines[body_start:] = _search_skipped_apart(lines[body_start:])
lines.append("")
lines.append("
")
lines.append("")
return lines
+def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]:
+ """What the collection holds under a directory entry, and where.
+
+ The card said "in repo" and linked none of the files: alephone's
+ Plugins/ holds 393 of them, each with its own card on a system page.
+ """
+ members = consumers.directory_members(entry)
+ held = sorted({consumers.path_of(sha1) for sha1 in members.values()} - {""})
+ if not held:
+ return []
+ folder = os.path.commonpath(held) if len(held) > 1 else os.path.dirname(held[0])
+ parts = folder.split("/")
+ where = _code(folder + "/")
+ if len(parts) >= 3 and parts[0] == "bios":
+ page = urllib.parse.quote(parts[1].lower().replace(" ", "-"))
+ where = f"[{where}](../systems/{page}.md#{_slugify_anchor(parts[2])})"
+ tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(folder)}"
+ return [f"Collection copies: {_count(len(members), 'file')} under {where} "
+ f"([browse]({tree}))"]
+
+
def _render_metadata_rows(profile: dict) -> list[str]:
"""Scalar profile fields, one table row each.
@@ -3065,6 +3218,22 @@ def _plain_value(value: object) -> str:
return _escape_tags(str(value))
+def _structured_field_text(profile: dict) -> Callable[[object, object], str]:
+ """Scalars of a structured block, citations linked at the profile's pin.
+
+ The file entries and platform details linked their source_ref, and the
+ analysis and firmware blocks printed 186 of them as plain code.
+ """
+ def text(key: object, value: object) -> str:
+ if key == "source_ref" and isinstance(value, str):
+ return _source_ref_markdown(profile, value)
+ if isinstance(value, str):
+ return link_prose_citations(profile, _field_text(key, value))
+ return _field_text(key, value)
+
+ return text
+
+
def _render_structured_blocks(profile: dict) -> list[str]:
"""Every remaining structured field, as a collapsible section."""
lines: list[str] = []
@@ -3100,7 +3269,8 @@ def _render_structured_blocks(profile: dict) -> list[str]:
lines.append(f'???+ abstract "{label}"')
lines.append("")
start = len(lines)
- _render_yaml_value(lines, val, indent=4)
+ _render_yaml_value(lines, val, indent=4,
+ field_text=_structured_field_text(profile))
lines[start:] = [_escape_tags(line) for line in lines[start:]]
lines.append("")
@@ -3225,7 +3395,9 @@ def generate_emulator_page(
consumers: FileConsumers | None = None,
data_names: set[str] | None = None,
data_registry: dict | None = None,
+ data_dirs: list[dict] | None = None,
) -> str:
+ """One emulator's page. ``data_dirs`` defaults to the profile's own."""
if profile.get("type") == "alias":
parent = profile.get("alias_of", profile.get("bios_identical_to", "unknown"))
return (
@@ -3263,7 +3435,8 @@ def generate_emulator_page(
str(notes_raw).strip() if notes_raw and not isinstance(notes_raw, dict) else ""
)
exclusion = profile.get("exclusion_note", "")
- data_dirs = profile.get("data_directories", [])
+ if data_dirs is None:
+ data_dirs = profile.get("data_directories", [])
lines = [
f"# {emu_name}",
@@ -3815,8 +3988,9 @@ def generate_gap_analysis(
"only authority for its files.",
"",
"The same comparison drives `scripts/exporter/`: each platform's "
- "corrected list can be regenerated in its native format "
- "(System.dat, es_bios.xml, batocera-systems.json, ...).",
+ "own file can be regenerated in its native format with these "
+ "differences applied (System.dat, es_bios.xml, "
+ "batocera-systems.json, ...).",
"",
])
@@ -3837,7 +4011,9 @@ def generate_gap_analysis(
gap_report = build_emulator_gap_report(profiles, coverages, db, data_names)
report_all = gap_report
- src_totals: dict[str, int] = {"bios": 0, "data": 0, "large_file": 0, "missing": 0}
+ src_totals: dict[str, int] = {
+ "bios": 0, "data": 0, "large_file": 0, "bundled": 0, "missing": 0,
+ }
total_undeclared = 0
emulator_gaps = []
@@ -3845,8 +4021,8 @@ def generate_gap_analysis(
if data["gaps"] == 0:
continue
total_undeclared += data["gaps"]
- for key in src_totals:
- src_totals[key] += data.get(f"gap_{key}", 0)
+ for key, count in _gap_sources(data).items():
+ src_totals[key] += count
emulator_gaps.append((emu_name, data))
@@ -3864,9 +4040,10 @@ def generate_gap_analysis(
"## Core complement",
"",
f"Files loaded by emulators but not declared by any platform. "
- f"{total_undeclared:,} files across {len(emulator_gaps)} emulators, "
+ f"{total_undeclared:,} files across {_count(len(emulator_gaps), 'emulator')}, "
f"{total_available:,} available ({pct_available}), "
- f"{src_totals['missing']} to source.",
+ f"{src_totals['bundled']:,} installed with their emulator, "
+ f"{src_totals['missing']:,} to source.",
"",
"This counts every profiled emulator, including those no platform "
"ships yet, so it reaches past what the pack tables above measure: "
@@ -3882,17 +4059,20 @@ def generate_gap_analysis(
f"| data/ | {src_totals['data']} | Data directories (buildbot, GitHub) |",
f"| release | {src_totals['large_file']} "
"| GitHub release assets (large files) |",
+ f"| bundled | {src_totals['bundled']} "
+ "| Installed with the emulator's own package, not collected |",
f"| missing | {src_totals['missing']} | Not available, needs sourcing |",
"",
"### Per emulator",
"",
- "| Emulator | Undeclared | bios | data | release | Missing |",
- "|----------|----------:|-----:|-----:|--------:|--------:|",
+ "| Emulator | Undeclared | bios | data | release | Bundled | Missing |",
+ "|----------|----------:|-----:|-----:|--------:|--------:|--------:|",
])
for emu_name, data in sorted(emulator_gaps, key=lambda x: -x[1]["gaps"]):
display = data["emulator"]
- m = data.get("gap_missing", 0)
+ sources = _gap_sources(data)
+ m = sources["missing"]
missing_str = (
f'
{m}'
if m > 0
@@ -3901,9 +4081,10 @@ def generate_gap_analysis(
lines.append(
f"| [{display}](emulators/{emu_name}.md) "
f"| {data['gaps']} "
- f"| {data.get('gap_bios', 0)} "
- f"| {data.get('gap_data', 0)} "
- f"| {data.get('gap_large_file', 0)} "
+ f"| {sources['bios']} "
+ f"| {sources['data']} "
+ f"| {sources['large_file']} "
+ f"| {sources['bundled']} "
f"| {missing_str} |"
)
lines.append("")
@@ -4047,6 +4228,41 @@ def generate_gap_analysis(
+def _core_scope_note(config: dict, profiles: dict) -> str:
+ """" (all libretro)" when the platform resolves every libretro core.
+
+ Read from what resolve_platform_cores returns, not from the shape of
+ ``cores:``: RetroPie and RetroDECK name ``retroarch`` in a list, which
+ pulls in every libretro core as ``all_libretro`` does, and the label
+ showed for RetroArch and Lakka only.
+ """
+ every_libretro = resolve_platform_cores({"cores": "all_libretro"}, profiles)
+ resolved = resolve_platform_cores(config, profiles)
+ if not every_libretro or not every_libretro <= resolved:
+ return ""
+ others = len(resolved - every_libretro)
+ if not others:
+ return " (all libretro)"
+ return f" (all libretro, plus {_count(others, 'other core')})"
+
+
+def _gap_sources(data: dict) -> dict[str, int]:
+ """Where each undeclared file of one emulator comes from.
+
+ A file the emulator's own package installs is not one to find. The
+ report's gap_missing counts it, and the gaps page announced 1,356 files
+ to source where 609 are, with a red badge on emulators that have none.
+ """
+ counts = {"bios": 0, "data": 0, "large_file": 0, "bundled": 0, "missing": 0}
+ for gap in data.get("gap_details") or []:
+ source = gap.get("source", "")
+ if source == "missing" and gap.get("bundled"):
+ source = "bundled"
+ if source in counts:
+ counts[source] += 1
+ return counts
+
+
def _render_xref_by_platform(
coverages: dict, unique: dict
) -> list[str]:
@@ -4056,7 +4272,6 @@ def _render_xref_by_platform(
cov = coverages[pname]
display = cov["platform"]
config = cov["config"]
- platform_cores = config.get("cores", [])
lines.append(f'??? abstract "{display}"')
lines.append("")
@@ -4075,10 +4290,7 @@ def _render_xref_by_platform(
if key in unique
}
- if platform_cores == "all_libretro":
- lines.append(f" **{len(matched)} cores** (all libretro)")
- else:
- lines.append(f" **{len(matched)} cores**")
+ lines.append(f" **{len(matched)} cores**{_core_scope_note(config, unique)}")
lines.append("")
lines.append(" | Core | Classification | Systems | Files | Upstream |")
@@ -4865,7 +5077,9 @@ def _generate(args: argparse.Namespace) -> None:
# Build system_id -> manufacturer page map (needed by all generators)
print("Building system cross-reference map...")
manufacturers = _group_by_manufacturer(db)
- _build_system_page_map_from_data(manufacturers, coverages, db, profiles)
+ set_system_page_map(
+ _system_pages_from_data(manufacturers, coverages, db, profiles)
+ )
print(f" {len(_system_page_map)} system IDs mapped to pages")
export_reports = load_export_reports()
@@ -4938,7 +5152,8 @@ def _generate(args: argparse.Namespace) -> None:
) + "\n",
)
page = generate_emulator_page(
- name, profile, db, consumers, suppl_names, data_registry
+ name, profile, db, consumers, suppl_names, data_registry,
+ data_dirs=page_data_directories(name, profiles),
)
PAGE_DATA[f"emulators/{name}.md"] = emulator_structured_data(profile)
write_if_changed(str(docs / "emulators" / f"{name}.md"), page)
@@ -5088,6 +5303,9 @@ markdown_extensions:
- tables
- toc:
permalink: true
+ # File cards are level 4 headings: the search lands on them, and a
+ # page of a hundred cards keeps a readable table of contents.
+ toc_depth: 3
- pymdownx.details
- pymdownx.highlight:
anchor_linenums: true
@@ -5157,10 +5375,12 @@ from sitesources import ( # noqa: E402,F401
from siterender import ( # noqa: E402,F401
_by_mode,
_count,
+ _field_text,
_render_yaml_value,
_slugify_anchor,
_system_page_map,
_system_link,
+ set_system_page_map,
_platform_link,
_emulator_link,
_fmt_size,
diff --git a/scripts/siterender.py b/scripts/siterender.py
index 33f3afc6..3f769501 100644
--- a/scripts/siterender.py
+++ b/scripts/siterender.py
@@ -2,12 +2,18 @@
from __future__ import annotations
-import re
-
-from pathlib import Path
-from concurrent.futures import ThreadPoolExecutor
import json
+import re
+import sys
+import urllib.error
+import urllib.parse
import urllib.request
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from artifacts import write_text_atomic
+
WIKI_SRC_DIR = "wiki" # manually maintained wiki sources
SYSTEM_ICON_BASE = "https://raw.githubusercontent.com/libretro/retroarch-assets/master/xmb/systematic/png"
ICON_CACHE_PATH = Path(".cache") / "system_icons.json"
@@ -67,6 +73,27 @@ def _icon_name(manufacturer: str, console_name: str) -> str:
def _icon_url(icon_name: str) -> str:
return f"{SYSTEM_ICON_BASE}/{urllib.parse.quote(icon_name)}.png"
+def _read_icon_cache() -> dict[str, bool]:
+ """The recorded icon checks, empty when none were recorded.
+
+ An unreadable file is reported, not taken for an empty cache in silence.
+ """
+ try:
+ text = ICON_CACHE_PATH.read_text(encoding="utf-8")
+ except FileNotFoundError:
+ return {}
+ except OSError as exc:
+ print(f" WARNING: cannot read {ICON_CACHE_PATH}: {exc}", file=sys.stderr)
+ return {}
+ try:
+ data = json.loads(text)
+ except json.JSONDecodeError as exc:
+ print(f" WARNING: {ICON_CACHE_PATH} is not valid JSON ({exc}), "
+ "icons are checked again", file=sys.stderr)
+ return {}
+ return {str(k): bool(v) for k, v in data.items()} if isinstance(data, dict) else {}
+
+
def prime_system_icons(names: set[str]) -> None:
"""Record which system icons upstream actually serves.
@@ -74,14 +101,7 @@ def prime_system_icons(names: set[str]) -> None:
cannot be checked stay unavailable: the site never links an image it has
not seen answer.
"""
- cached: dict[str, bool] = {}
- if ICON_CACHE_PATH.exists():
- try:
- with open(ICON_CACHE_PATH) as f:
- cached = json.load(f)
- except (json.JSONDecodeError, OSError):
- cached = {}
-
+ cached = _read_icon_cache()
unknown = sorted(n for n in names if n not in cached)
if unknown:
def check(name: str) -> tuple[str, bool | None]:
@@ -100,13 +120,22 @@ def prime_system_icons(names: set[str]) -> None:
return name, None
print(f"Checking {len(unknown)} system icons...")
+ learned: dict[str, bool] = {}
with ThreadPoolExecutor(max_workers=8) as pool:
for name, ok in pool.map(check, unknown):
if ok is not None:
- cached[name] = ok
- ICON_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
- with open(ICON_CACHE_PATH, "w") as f:
- json.dump(cached, f, indent=2, sort_keys=True)
+ learned[name] = ok
+ cached.update(learned)
+ if learned:
+ # Another run may have recorded names meanwhile: merge with what
+ # the file holds now, and replace it whole so a reader never sees
+ # a half-written cache.
+ merged = {**_read_icon_cache(), **learned}
+ ICON_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+ write_text_atomic(
+ str(ICON_CACHE_PATH),
+ json.dumps(merged, indent=2, sort_keys=True) + "\n",
+ )
_icon_available.update(cached)
missing = sum(1 for n in names if not cached.get(n))
@@ -133,8 +162,27 @@ def _slugify_anchor(text: str) -> str:
slug = slug.strip("-")
return slug
-_norm_page_cache: dict[str, tuple[str, str] | None] = {}
-_norm_page_size = -1
+_norm_page_map: dict[str, tuple[str, str] | None] = {}
+
+
+def set_system_page_map(mapping: dict[str, tuple[str, str]]) -> None:
+ """Install the system id -> page table every link on the site reads.
+
+ The table replaces the previous one, and the index of other spellings is
+ derived from it here. Kept anywhere else, it outlived a table of the same
+ size and linked to the pages of the previous one.
+ """
+ from common import _norm_system_id
+
+ _system_page_map.clear()
+ _system_page_map.update(mapping)
+ _norm_page_map.clear()
+ for known, page in _system_page_map.items():
+ key = _norm_system_id(known)
+ if _norm_page_map.get(key, page) != page:
+ _norm_page_map[key] = None
+ else:
+ _norm_page_map[key] = page
def _page_by_norm(sys_id: str) -> tuple[str, str] | None:
@@ -144,19 +192,9 @@ def _page_by_norm(sys_id: str) -> tuple[str, str] | None:
keyed by the platforms' spellings: 2,431 of 4,195 cards had no link. A
spelling two pages share leads nowhere rather than to either.
"""
- global _norm_page_size
from common import _norm_system_id
- if _norm_page_size != len(_system_page_map):
- _norm_page_cache.clear()
- for known, page in _system_page_map.items():
- key = _norm_system_id(known)
- if _norm_page_cache.get(key, page) != page:
- _norm_page_cache[key] = None
- else:
- _norm_page_cache[key] = page
- _norm_page_size = len(_system_page_map)
- return _norm_page_cache.get(_norm_system_id(sys_id))
+ return _norm_page_map.get(_norm_system_id(sys_id))
def _system_link(sys_id: str, prefix: str = "") -> str:
@@ -175,9 +213,10 @@ def _field_text(key: object, value: object) -> str:
elif value is None:
text = "none"
else:
- text = str(value)
- if len(text) > 200:
- text = text[:200] + "..."
+ # One line: a folded note's second line started at column 0 and
+ # left the admonition it was written in. Never cut: 34 notes lost
+ # their end, and a cut citation links to another line.
+ text = " ".join(str(value).split())
return f"`{text}`" if key == "source_ref" else text
@@ -186,12 +225,19 @@ def _field_label(key: object) -> str:
return str(key).replace("_", " ")
-def _render_yaml_value(lines: list[str], val: object, indent: int = 4) -> None:
+FieldText = Callable[[object, object], str]
+
+
+def _render_yaml_value(
+ lines: list[str], val: object, indent: int = 4,
+ field_text: FieldText = _field_text,
+) -> None:
"""Render any YAML value as a nested markdown list.
Every level is a list item and its children a sublist four spaces in.
A key written as a paragraph with its children indented below it read as
an indented code block: the clownmdemu page showed the raw asterisks.
+ ``field_text`` renders each scalar, so a page can link a source_ref.
"""
pad = " " * indent
if isinstance(val, dict):
@@ -199,27 +245,32 @@ def _render_yaml_value(lines: list[str], val: object, indent: int = 4) -> None:
if isinstance(v, (dict, list)) and v:
lines.append(f"{pad}- **{_field_label(k)}:**")
lines.append("")
- _render_yaml_value(lines, v, indent + 4)
+ _render_yaml_value(lines, v, indent + 4, field_text)
else:
- lines.append(f"{pad}- **{_field_label(k)}:** {_field_text(k, v)}")
+ lines.append(f"{pad}- **{_field_label(k)}:** {field_text(k, v)}")
lines.append("")
elif isinstance(val, list):
for item in val:
if isinstance(item, dict):
- parts = [
- f"{_field_label(ik)}: {_field_text(ik, iv)}"
- for ik, iv in item.items()
- if not isinstance(iv, (dict, list))
- ]
- lines.append(f"{pad}- {', '.join(parts)}")
+ # Scalars on the item's line, lists and maps as its children:
+ # cemu's firmware titles lost the files each title carries.
+ scalars = {ik: iv for ik, iv in item.items()
+ if not (isinstance(iv, (dict, list)) and iv)}
+ nested = {ik: iv for ik, iv in item.items() if ik not in scalars}
+ parts = [f"{_field_label(ik)}: {field_text(ik, iv)}"
+ for ik, iv in scalars.items()]
+ lines.append(f"{pad}- {', '.join(parts)}".rstrip())
+ if nested:
+ lines.append("")
+ _render_yaml_value(lines, nested, indent + 4, field_text)
else:
- lines.append(f"{pad}- {_field_text(None, item)}")
+ lines.append(f"{pad}- {field_text(None, item)}")
lines.append("")
elif isinstance(val, str) and "\n" in val:
for line in val.split("\n"):
lines.append(f"{pad}{line}")
else:
- lines.append(f"{pad}{_field_text(None, val)}")
+ lines.append(f"{pad}{field_text(None, val)}")
def _platform_link(name: str, display: str, prefix: str = "") -> str:
diff --git a/scripts/sitesources.py b/scripts/sitesources.py
index 7d9e0912..f1a62110 100644
--- a/scripts/sitesources.py
+++ b/scripts/sitesources.py
@@ -6,6 +6,7 @@ was written against, so a permalink is pinned, never guessed."""
from __future__ import annotations
import re
+import urllib.parse
import urllib.request
import upstream
@@ -48,6 +49,47 @@ def _repo_pins(profile: dict, field: str, label: str) -> list[tuple[str, str, st
if raw.get(key)
]
+# Forges the pages link into but upstream.py does not read from. Each is
+# reached by browser only, so the Repo carries no API or raw base.
+_BROWSE_ONLY = {
+ "voidpoint.io": "gitlab",
+ "bitbucket.org": "bitbucket",
+ "sourceforge.net": "sourceforge",
+ "archive.softwareheritage.org": "swh",
+}
+
+
+def _browse_repo(url: str) -> upstream.Repo | None:
+ """The repository a profile URL names, on any forge the pages can link.
+
+ Profiles pinned on SourceForge, voidpoint.io, Bitbucket or Software
+ Heritage showed no revision and no linked citation: parse_repo knows
+ only the forges profile_sync fetches from.
+ """
+ repo = upstream.parse_repo(url)
+ if repo is not None or not url:
+ return repo
+ parts = urllib.parse.urlsplit(url.strip())
+ family = _BROWSE_ONLY.get(parts.netloc)
+ segments = [seg for seg in parts.path.split("/") if seg]
+ if family == "gitlab":
+ if "-" in segments:
+ segments = segments[: segments.index("-")]
+ if len(segments) < 2:
+ return None
+ owner, name = "/".join(segments[:-1]), segments[-1]
+ elif family == "bitbucket" and len(segments) >= 2:
+ owner, name = segments[0], segments[1]
+ elif family == "sourceforge" and len(segments) >= 3 and segments[0] == "p":
+ owner, name = segments[1], segments[2]
+ elif family == "swh":
+ # A revision URL names no project: the pin alone addresses the tree.
+ owner, name = "softwareheritage", "revision"
+ else:
+ return None
+ return upstream.Repo(parts.netloc, family, "", "", owner, name.removesuffix(".git"))
+
+
def _forge_sources(profile: dict, label: str = "") -> list[tuple[upstream.Repo, str]]:
"""Supported source repositories and immutable revisions for a profile."""
candidates: list[tuple[upstream.Repo, str]] = []
@@ -56,7 +98,7 @@ def _forge_sources(profile: dict, label: str = "") -> list[tuple[upstream.Repo,
for field in ("source", "upstream"):
for value, pin, fallback in _repo_pins(profile, field, label):
- repo = upstream.parse_repo(value)
+ repo = _browse_repo(value)
if repo is None:
continue
repo_key = (repo.host, repo.slug)
@@ -77,6 +119,19 @@ def _source_permalink(repo: upstream.Repo, pin: str, path: str,
"""Forge-specific browser URL for one file at one immutable revision."""
quoted_path = urllib.parse.quote(path, safe="/@:+-._~")
base = f"https://{repo.host}/{repo.owner}/{repo.name}"
+ if repo.family == "bitbucket":
+ url = f"{base}/src/{pin}/{quoted_path}"
+ if start is not None:
+ url += f"#lines-{start}" + (f":{end}" if end is not None and end != start else "")
+ return url
+ if repo.family == "sourceforge":
+ # The Allura code browser takes no line range: the file at the pin.
+ return f"https://{repo.host}/p/{repo.owner}/{repo.name}/ci/{pin}/tree/{quoted_path}"
+ if repo.family == "swh":
+ url = f"https://{repo.host}/browse/revision/{pin}/?path={quoted_path}"
+ if start is not None:
+ url += f"#L{start}" + (f"-L{end}" if end is not None and end != start else "")
+ return url
if repo.family == "github":
url = f"{base}/blob/{pin}/{quoted_path}"
elif repo.family == "gitlab":
@@ -99,6 +154,12 @@ def _commit_url(repo: upstream.Repo, pin: str) -> str:
base = f"https://{repo.host}/{repo.owner}/{repo.name}"
if repo.family == "gitlab":
return f"{base}/-/commit/{pin}"
+ if repo.family == "bitbucket":
+ return f"{base}/commits/{pin}"
+ if repo.family == "sourceforge":
+ return f"https://{repo.host}/p/{repo.owner}/{repo.name}/ci/{pin}/"
+ if repo.family == "swh":
+ return f"https://{repo.host}/browse/revision/{pin}/"
return f"{base}/commit/{pin}"
@@ -109,7 +170,8 @@ def revision_markdown(profile: dict) -> str:
refers to sat in the YAML alone, for 178 profiles.
"""
links = [
- f"[{repo.slug}@{pin[:12]}]({_commit_url(repo, pin)})"
+ f"[{'Software Heritage' if repo.family == 'swh' else repo.slug}@{pin[:12]}]"
+ f"({_commit_url(repo, pin)})"
for repo, pin in _forge_sources(profile)
]
branch = profile.get("source_branch")
@@ -143,7 +205,7 @@ def role_sources(profile: dict, label: str = "") -> dict[str, tuple[upstream.Rep
found: dict[str, tuple[upstream.Repo, str]] = {}
for field in ("source", "upstream"):
for value, pin, _fallback in _repo_pins(profile, field, label):
- repo = upstream.parse_repo(value)
+ repo = _browse_repo(value)
if repo is not None and pin and field not in found:
found[field] = (repo, pin)
return found
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
index 2b0501c0..0c392b3a 100644
--- a/tests/test_audit_regressions.py
+++ b/tests/test_audit_regressions.py
@@ -4175,8 +4175,9 @@ class AFileCardSaysAllItKnows(unittest.TestCase):
text = "\n".join(_render_emulator_file(
entry, {"files": [entry]}, None, [entry], lambda f: True, "file-neogeo-zip",
))
- self.assertIn('id="file-neogeo-zip"', text)
- self.assertIn('href="#file-neogeo-zip"', text)
+ # The card is a heading with this id, linked by the heading's own
+ # permalink.
+ self.assertIn("{ #file-neogeo-zip .rb-file-title }", text)
self.assertIn("Standalone path: `Firmware/neogeo.zip`", text)
self.assertIn("`m11.bin`", text)
self.assertIn("CRC32 `0000000b`", text)
@@ -4894,7 +4895,7 @@ class VersionAnchorsFollowContent(unittest.TestCase):
"raze", {"emulator": "Raze", "type": "standalone", "files": files},
{"files": {}, "indexes": {}},
)
- return re.findall(r'id="(file-[^"]+)"', page)
+ return re.findall(r"\{ #(file-[^ ]+) \.rb-file-title \}", page)
def test_inserting_a_version_keeps_the_others(self):
a = {"name": "DUKE3D.GRP", "path": "DUKE3D.GRP", "crc32": "bbc9ce44"}
diff --git a/tests/test_site_cores.py b/tests/test_site_cores.py
index 0be1f5fe..b60fb636 100644
--- a/tests/test_site_cores.py
+++ b/tests/test_site_cores.py
@@ -44,5 +44,30 @@ class SiteCoreCounts(unittest.TestCase):
self.assertEqual(shown[cov["platform"]], expected)
+class AllLibretroLabel(unittest.TestCase):
+ """RetroPie and RetroDECK pull in every libretro core by naming retroarch
+ in a list, and the label read "(all libretro)" for RetroArch and Lakka
+ only, the two whose cores: is the string."""
+
+ PROFILES = {
+ "a": {"type": "libretro"},
+ "b": {"type": "standalone + libretro"},
+ "sa": {"type": "standalone"},
+ }
+
+ def test_a_list_naming_retroarch_is_labelled(self):
+ note = generate_site._core_scope_note(
+ {"cores": ["retroarch", "sa"]}, self.PROFILES)
+ self.assertEqual(note, " (all libretro, plus 1 other core)")
+
+ def test_the_string_form_is_labelled(self):
+ note = generate_site._core_scope_note({"cores": "all_libretro"}, self.PROFILES)
+ self.assertEqual(note, " (all libretro)")
+
+ def test_a_selection_is_not(self):
+ self.assertEqual(
+ generate_site._core_scope_note({"cores": ["a", "sa"]}, self.PROFILES), "")
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_site_render.py b/tests/test_site_render.py
new file mode 100644
index 00000000..8f7dd3b1
--- /dev/null
+++ b/tests/test_site_render.py
@@ -0,0 +1,415 @@
+"""Renderings shared by the site pages, and the icon cache they read.
+
+No network: icon checks are answered by a stub.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import tempfile
+import unittest
+import urllib.error
+from pathlib import Path
+from unittest import mock
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import siterender # noqa: E402
+
+
+class _Served:
+ status = 200
+
+ def __enter__(self) -> "_Served":
+ return self
+
+ def __exit__(self, *exc: object) -> None:
+ return None
+
+
+class IconCacheIsReplacedWhole(unittest.TestCase):
+ """The cache was truncated then dumped in place: a concurrent reader saw
+ half a file, took it for empty, and an offline run then wrote ``{}``
+ over every recorded icon."""
+
+ def setUp(self) -> None:
+ tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(tmp.cleanup)
+ self.path = Path(tmp.name) / "system_icons.json"
+ patcher = mock.patch.object(siterender, "ICON_CACHE_PATH", self.path)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ saved = dict(siterender._icon_available)
+ self.addCleanup(lambda: (siterender._icon_available.clear(),
+ siterender._icon_available.update(saved)))
+
+ def test_an_offline_run_leaves_an_unreadable_cache_untouched(self):
+ self.path.write_text('{"Sega - Mega Drive": tr')
+ offline = mock.patch.object(
+ siterender.urllib.request, "urlopen",
+ side_effect=urllib.error.URLError("offline"))
+ with offline, mock.patch("sys.stderr"), mock.patch("builtins.print"):
+ siterender.prime_system_icons({"Sega - Mega Drive"})
+ self.assertEqual(self.path.read_text(), '{"Sega - Mega Drive": tr')
+
+ def test_names_another_run_recorded_meanwhile_are_kept(self):
+ self.path.write_text(json.dumps({"A - One": True}))
+
+ def answer(*_args: object, **_kwargs: object) -> _Served:
+ # A second build records its own check while this one runs.
+ self.path.write_text(json.dumps({"A - One": True, "C - Three": False}))
+ return _Served()
+
+ with mock.patch.object(siterender.urllib.request, "urlopen", side_effect=answer), \
+ mock.patch("builtins.print"):
+ siterender.prime_system_icons({"A - One", "B - Two"})
+ self.assertEqual(json.loads(self.path.read_text()),
+ {"A - One": True, "B - Two": True, "C - Three": False})
+ self.assertEqual(sorted(p.name for p in self.path.parent.iterdir()),
+ ["system_icons.json"])
+
+ def test_the_write_goes_through_a_scratch_file(self):
+ with mock.patch.object(siterender.urllib.request, "urlopen",
+ return_value=_Served()), \
+ mock.patch.object(siterender, "write_text_atomic",
+ wraps=siterender.write_text_atomic) as atomic, \
+ mock.patch("builtins.print"):
+ siterender.prime_system_icons({"B - Two"})
+ atomic.assert_called_once()
+
+
+class EveryPageAddressesPlatformsAsPeers(unittest.TestCase):
+ """The exports page was held to neutral wording, and the gaps page next
+ to it still called each platform's file a "corrected list". Reads the
+ text every site generator writes, docstrings and bare identifiers aside."""
+
+ WORDS = ("their file", "theirs", "of ours", "corrected", "left out", "drift")
+ SOURCES = ("generate_site.py", "siterender.py", "sitesources.py", "siteexports.py")
+
+ def test_no_page_text_judges_another_project(self):
+ import ast
+ import re
+
+ found = []
+ for name in self.SOURCES:
+ tree = ast.parse((REPO_ROOT / "scripts" / name).read_text(encoding="utf-8"))
+ docstrings = {
+ id(node.body[0].value) for node in ast.walk(tree)
+ if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef,
+ ast.AsyncFunctionDef))
+ and ast.get_docstring(node, clean=False) is not None
+ }
+ for node in ast.walk(tree):
+ if (not isinstance(node, ast.Constant) or not isinstance(node.value, str)
+ or id(node) in docstrings or re.fullmatch(r"\w+", node.value)):
+ continue
+ text = node.value.lower()
+ found += [f"{name}:{node.lineno} {word!r}"
+ for word in self.WORDS if word in text]
+ self.assertEqual(found, [])
+
+
+class BundledFilesAreNotToSource(unittest.TestCase):
+ """The gaps page summed the report's gap_missing, bundled files included:
+ 1,356 files to source where 609 were, and a red badge of 13 on ZQuest
+ Classic, whose 13 missing files all come with its own package."""
+
+ def test_the_summary_and_the_row_count_only_what_must_be_found(self):
+ import generate_site
+
+ gap = {"required": True, "source_ref": "x", "in_repo": False}
+ report = {"zq": {
+ "emulator": "ZQuest Classic", "gaps": 4, "gap_missing": 3, "gap_bios": 1,
+ "unsourceable": [],
+ "gap_details": [
+ {**gap, "name": "a.dll", "source": "missing", "bundled": True},
+ {**gap, "name": "b.dll", "source": "missing", "bundled": True},
+ {**gap, "name": "c.bin", "source": "missing"},
+ {**gap, "name": "d.bin", "source": "bios", "in_repo": True},
+ ],
+ }}
+ page = generate_site.generate_gap_analysis(
+ {"zq": {"emulator": "ZQuest Classic", "files": []}}, {},
+ {"files": {}, "indexes": {}}, gap_report=report)
+ self.assertIn("2 installed with their emulator, 1 to source.", page)
+ self.assertIn("| missing | 1 | Not available, needs sourcing |", page)
+ self.assertIn("| [ZQuest Classic](emulators/zq.md) | 4 | 1 | 0 | 0 | 2 | "
+ '
1 |', page)
+
+
+class StructuredBlocks(unittest.TestCase):
+ PIN = "1e0de94dc7e669c0925a22c0fccf6cdc837af0a0"
+
+ def profile(self, block: str, value: object) -> dict:
+ return {"source": "https://github.com/libretro/blastem",
+ "source_commit": self.PIN, block: value}
+
+ def render(self, profile: dict) -> str:
+ import generate_site
+
+ return "\n".join(generate_site._render_structured_blocks(profile))
+
+ def test_a_source_ref_in_a_block_is_a_pinned_link(self):
+ """186 citations in analysis and firmware blocks were plain code."""
+ page = self.render(self.profile("analysis", {"tmss": {
+ "source_ref": "libblastem.c:355-359"}}))
+ self.assertIn(
+ f"[`libblastem.c:355-359`](https://github.com/libretro/blastem/blob/"
+ f"{self.PIN}/libblastem.c#L355-L359)", page)
+
+ def test_a_list_inside_a_list_item_is_shown(self):
+ """cemu's firmware titles lost the files each title carries."""
+ page = self.render(self.profile("firmware_titles", [
+ {"id": "0005001b-10042400", "name": "Shared fonts",
+ "content": ["CafeCn.ttf", "CafeKr.ttf"]}]))
+ self.assertIn(" - id: 0005001b-10042400, name: Shared fonts\n", page)
+ self.assertIn(" - **content:**", page)
+ self.assertIn(" - CafeKr.ttf", page)
+
+ def test_a_folded_note_stays_inside_its_block(self):
+ page = self.render(self.profile("analysis", {"tmss": {
+ "note": "The core points\nsystem/tmss_path at the system directory."}}))
+ self.assertIn("- **note:** The core points system/tmss_path at the", page)
+ body = page.split("\n")[1:]
+ self.assertEqual([line for line in body if line and not line.startswith(" ")], [])
+
+ def test_a_long_note_is_not_cut(self):
+ note = "word " * 80 + "end."
+ page = self.render(self.profile("analysis", {"x": {"note": note}}))
+ self.assertIn("end.", page)
+
+
+class FileCardsLinkTheReadingEntry(unittest.TestCase):
+ def test_an_emulator_link_lands_on_its_entry(self):
+ """The platform links landed on their section, the emulator links on
+ the top of a page of a hundred entries."""
+ import generate_site
+ from cross_reference import FileConsumers
+
+ consumers = FileConsumers({"files": {}})
+ files = [{"name": "scph1001.bin"}, {"name": "scph5501.bin"},
+ {"name": "tmss.md", "sha1": "d0fd0c16" + "0" * 32},
+ {"name": "tmss.md", "sha1": "aa" * 20}]
+ consumers.emulators = {"s": {"duckstation", "other"}, "t": {"blastem"}}
+ consumers.emulator_entries = {"s": {"duckstation": 1}, "t": {"blastem": 2}}
+ consumers.emulator_files = {"duckstation": files, "blastem": files}
+ self.assertEqual(generate_site._emulator_links("s", consumers), [
+ "[duckstation](../emulators/duckstation.md#file-scph5501-bin)",
+ "[other](../emulators/other.md)",
+ ])
+ self.assertEqual(generate_site._emulator_links("t", consumers),
+ ["[blastem](../emulators/blastem.md#file-tmss-md-d0fd0c16)"])
+
+
+class DirectoryMembersNameTheirEmulator(unittest.TestCase):
+ """The site resolved a ``type: directory`` entry by its own name: 11,976
+ files under alephone's Plugins/, cdogs' and thextech's trees had no
+ emulator on their card, and the directory card linked none of them."""
+
+ SHA1 = "c63078a6c9ccaca7339a726a5d39410983933070"
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ from common import build_zip_contents_index, load_database, load_emulator_profiles
+ from cross_reference import FileConsumers
+
+ cls.db = load_database(str(REPO_ROOT / "database.json"))
+ if cls.SHA1 not in cls.db.get("files", {}):
+ raise unittest.SkipTest("the collection does not hold the alephone plugin")
+ cls.profile = load_emulator_profiles(str(REPO_ROOT / "emulators"))["alephone"]
+ cls.consumers = FileConsumers(cls.db, build_zip_contents_index(cls.db))
+ cls.consumers.add_emulators({"alephone": cls.profile})
+
+ def test_a_member_names_the_emulator_and_its_directory_entry(self):
+ import generate_site
+
+ files = self.profile["files"]
+ index = self.consumers.emulator_entries[self.SHA1]["alephone"]
+ self.assertEqual(files[index].get("type"), "directory")
+ self.assertEqual(generate_site._emulator_links(self.SHA1, self.consumers),
+ ["[alephone](../emulators/alephone.md#file-marathon-plugins)"])
+
+ def test_the_directory_card_points_at_its_copies(self):
+ import generate_site
+
+ entry = next(f for f in self.profile["files"]
+ if f.get("type") == "directory" and f.get("path") == "Marathon/Plugins/")
+ line = generate_site._directory_copies(entry, self.consumers)[0]
+ self.assertRegex(line, r"^Collection copies: \d+ files under "
+ r"\[`bios/Bungie/Marathon/Plugins/`\]\(\.\./systems/bungie\.md#")
+
+
+class AnAliasTreeIsListedOnItsParentsPage(unittest.TestCase):
+ """The 90 FreeIntv overlays were linked to the page of the alias that
+ declares them, which is never generated: every permalink was a 404."""
+
+ PROFILES = {
+ "freeintv": {"type": "libretro"},
+ "freeintv_ts_overlay": {
+ "type": "alias", "alias_of": "freeintv",
+ "data_directories": [{"ref": "freeintv-overlays",
+ "destination": "freeintv_overlays"}],
+ },
+ }
+
+ def test_the_permalink_goes_to_a_page_that_exists(self):
+ import generate_site
+
+ members = {"freeintv-overlays": {"a.png": {"sha1": "02" * 20}}}
+ items = generate_site.file_location_items({"files": {}}, self.PROFILES, members)
+ self.assertEqual(items, [{"sha1": "02" * 20,
+ "url": "emulators/freeintv/#sha1-020202020202"}])
+
+ def test_the_parent_page_lists_the_tree(self):
+ import generate_site
+
+ trees = generate_site.page_data_directories("freeintv", self.PROFILES)
+ self.assertEqual([t["ref"] for t in trees], ["freeintv-overlays"])
+
+
+class ForgesUpstreamDoesNotRead(unittest.TestCase):
+ """Profiles pinned on SourceForge, voidpoint.io, Bitbucket or Software
+ Heritage showed no revision, and their citations were plain code."""
+
+ def links(self, url: str, ref: str, **extra: str) -> tuple[str, str]:
+ import sitesources
+
+ profile = {"source": url, "source_commit": "f6c00044" + "0" * 32, **extra}
+ return (sitesources.revision_markdown(profile),
+ sitesources._source_ref_markdown(profile, ref))
+
+ def test_bitbucket(self):
+ revision, ref = self.links("https://bitbucket.org/linuxwolf6/splitwolf",
+ "id_lwmp.cpp:1831-1868", source_branch="scrubbed")
+ pin = "f6c00044" + "0" * 32
+ self.assertEqual(revision, f"[linuxwolf6/splitwolf@{pin[:12]}]"
+ f"(https://bitbucket.org/linuxwolf6/splitwolf/commits/{pin}), "
+ "branch `scrubbed`")
+ self.assertIn(f"(https://bitbucket.org/linuxwolf6/splitwolf/src/{pin}/"
+ "id_lwmp.cpp#lines-1831:1868)", ref)
+
+ def test_a_gitlab_instance(self):
+ _revision, ref = self.links("https://voidpoint.io/terminx/eduke32",
+ "source/duke3d/src/common.cpp:36")
+ self.assertIn("https://voidpoint.io/terminx/eduke32/-/blob/f6c00044", ref)
+ self.assertIn("/source/duke3d/src/common.cpp#L36)", ref)
+
+ def test_sourceforge(self):
+ revision, ref = self.links("https://sourceforge.net/p/dgen/dgen", "rc.cpp:12")
+ self.assertIn("(https://sourceforge.net/p/dgen/dgen/ci/f6c00044", revision)
+ self.assertRegex(ref, r"\(https://sourceforge\.net/p/dgen/dgen/ci/f6c00044\w+/tree/rc\.cpp\)")
+
+ def test_software_heritage(self):
+ revision, ref = self.links(
+ "https://archive.softwareheritage.org/browse/revision/f6c00044/",
+ "app/src/os.cpp:35-36")
+ self.assertTrue(revision.startswith("[Software Heritage@f6c000440000]"))
+ self.assertIn("/browse/revision/f6c00044" + "0" * 32 + "/?path=app/src/os.cpp#L35-L36)", ref)
+
+
+class TheDatHoldsEveryCopy(unittest.TestCase):
+ """The DAT held one path per content, the database's: auditing bios/
+ with it flagged the 2,502 other copies (RPG Maker, engine trees, a dump
+ filed under two consoles) as unknown, and a fix moved them away."""
+
+ def test_a_second_copy_is_a_rom_of_its_own_folder(self):
+ import generate_site
+
+ sha1 = "9526ef63" + "0" * 32
+ with tempfile.TemporaryDirectory() as tmp:
+ for rel in ("Atari/ST/tos104.img", "Atari/ST/tos104uk.img"):
+ (Path(tmp) / rel).parent.mkdir(parents=True, exist_ok=True)
+ (Path(tmp) / rel).write_bytes(b"x")
+ db = {
+ "bios_dir": tmp,
+ "files": {sha1: {"path": "bios/Atari/ST/tos104.img", "sha1": sha1,
+ "size": 1, "name": "tos104.img"}},
+ "indexes": {"by_path_suffix": {
+ "Atari/ST/tos104.img": [sha1], "ST/tos104.img": [sha1],
+ "Atari/ST/tos104uk.img": [sha1], "ST/tos104uk.img": [sha1],
+ }},
+ }
+ dat = generate_site.collection_dat(db, "2026-10-21T00:00:00Z")
+ self.assertIn('
', lines)
+ self.assertIn('??? note "1 file (OK)"', lines)
+
+ def test_an_emulator_card_is_a_heading_with_its_anchor(self):
+ import generate_site
+
+ entry = {"name": "scph5501.bin", "required": True}
+ lines = generate_site._render_emulator_file(
+ entry, {"files": [entry]}, None, [entry], lambda f: True, "file-scph5501-bin")
+ self.assertTrue(lines[2].startswith("#### `scph5501.bin` "))
+ self.assertTrue(lines[2].endswith("{ #file-scph5501-bin .rb-file-title }"))
+
+ def test_a_system_card_is_a_heading_with_its_anchor(self):
+ import generate_site
+ from cross_reference import FileConsumers
+
+ entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3}
+ lines = generate_site._system_file_card(entry, FileConsumers({"files": {}}))
+ self.assertTrue(lines[2].startswith("#### `a.bin` "))
+ self.assertTrue(lines[2].endswith("{ #sha1-aaaaaaaaaaaa .rb-file-title }"))
+
+
+class TheSearchSkipsLinkLists(unittest.TestCase):
+ """The search index was 24.7 MB, read and indexed before the first
+ result. Lists of platforms, copies and source citations on every card
+ made 6.6 MB of it, names already indexed on their own pages."""
+
+ def test_a_system_card_keeps_its_hashes_and_skips_its_platforms(self):
+ import generate_site
+ from cross_reference import FileConsumers
+
+ consumers = FileConsumers({"files": {}})
+ sha1 = "a" * 40
+ consumers.platforms = {sha1: {"retroarch"}}
+ consumers.installed = {sha1: {"batocera": {"bios/a.bin"}}}
+ consumers.emulators = {sha1: {"duckstation"}}
+ entry = {"name": "a.bin", "sha1": sha1, "md5": "b" * 32, "size": 3}
+ text = "\n".join(generate_site._system_file_card(entry, consumers))
+ searched, skipped = text.split('')
+ self.assertIn("- MD5:", searched)
+ self.assertIn("- Emulators:", searched)
+ self.assertIn("- Platforms:", skipped)
+ self.assertIn("- Also installed by:", skipped)
+ self.assertEqual(text.count("
"), 2)
+
+ def test_runs_alternate_in_card_order(self):
+ import generate_site
+
+ lines = ["- SHA1: x", "- Source: `a.c:1`", "- Platforms: p",
+ "- Destination: `d`", "- Contents (1 members):", " - `m`"]
+ out = [line for line in generate_site._search_skipped_apart(lines) if line]
+ self.assertEqual(out, [
+ "- SHA1: x",
+ '',
+ "- Source: `a.c:1`", "- Platforms: p", "
",
+ "- Destination: `d`", "- Contents (1 members):", " - `m`",
+ ])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_site_system_links.py b/tests/test_site_system_links.py
index c4433ed6..17f05e3d 100644
--- a/tests/test_site_system_links.py
+++ b/tests/test_site_system_links.py
@@ -69,31 +69,40 @@ class OtherSpellingsOfOneSystem(unittest.TestCase):
self.assertEqual(gs._system_emulators("sega-mega-drive", index), ["genesis_plus_gx"])
self.assertEqual(gs._system_emulators("nintendo-snes", index), [])
+ def _install(self, mapping: dict) -> None:
+ import siterender
+
+ saved = dict(siterender._system_page_map)
+ self.addCleanup(siterender.set_system_page_map, saved)
+ siterender.set_system_page_map(mapping)
+
def test_a_profile_spelling_links_to_the_platform_page(self):
import siterender
- saved = dict(siterender._system_page_map)
- self.addCleanup(lambda: (siterender._system_page_map.clear(),
- siterender._system_page_map.update(saved)))
- siterender._system_page_map.clear()
- siterender._system_page_map["sega-mega-drive"] = ("sega", "Mega Drive")
+ self._install({"sega-mega-drive": ("sega", "Mega Drive")})
self.assertIn("systems/sega.md#mega-drive", siterender._system_link("sega-megadrive"))
- siterender._system_page_map["megadrive"] = ("other", "Elsewhere")
+ self._install({"sega-mega-drive": ("sega", "Mega Drive"),
+ "megadrive": ("other", "Elsewhere")})
self.assertEqual(siterender._system_link("sega-megadrive"), "sega-megadrive")
- def test_a_system_only_profiles_name_is_placed_by_their_files(self):
+ def test_a_table_of_the_same_size_replaces_the_other_spellings(self):
+ """The index of other spellings was kept while the table kept its
+ length, so a second table of one entry linked to the first one's page."""
import siterender
- saved = dict(siterender._system_page_map)
- self.addCleanup(lambda: (siterender._system_page_map.clear(),
- siterender._system_page_map.update(saved)))
- siterender._system_page_map.clear()
+ self._install({"sega-mega-drive": ("sega", "Mega Drive")})
+ self.assertIn("systems/sega.md#", siterender._system_link("sega-megadrive"))
+ self._install({"sega-mega-drive": ("other", "Elsewhere")})
+ self.assertEqual(siterender._system_link("sega-megadrive"),
+ "[sega-megadrive](systems/other.md#elsewhere)")
+
+ def test_a_system_only_profiles_name_is_placed_by_their_files(self):
manufacturers = {"Commodore": {"PET": [{"name": "basic4.bin"}]}}
profiles = {"vice_xpet": {"systems": ["commodore-pet"],
"files": [{"name": "basic4.bin"}]}}
- gs._build_system_page_map_from_data(
+ pages = gs._system_pages_from_data(
manufacturers, {}, {"files": {}, "indexes": {}}, profiles)
- self.assertEqual(siterender._system_page_map["commodore-pet"], ("commodore", "PET"))
+ self.assertEqual(pages["commodore-pet"], ("commodore", "PET"))
class ACopyIsOnItsOwnFoldersPage(unittest.TestCase):
@@ -134,7 +143,7 @@ class AnIdNamesOneCardPerPage(unittest.TestCase):
"path": "bios/Nintendo/DS/a.bin"}
page = gs.generate_system_page(
"Nintendo", {"DS": [entry], "DSi": [entry]}, FileConsumers({"files": {}}))
- self.assertEqual(page.count('id="sha1-aaaaaaaaaaaa"'), 1)
+ self.assertEqual(page.count("#sha1-aaaaaaaaaaaa .rb-file-title"), 1)
self.assertEqual(page.count("rb-sys-file"), 2)
diff --git a/wiki/architecture.md b/wiki/architecture.md
index ab066c99..08ee1071 100644
--- a/wiki/architecture.md
+++ b/wiki/architecture.md
@@ -66,6 +66,16 @@ dist/ generated packs (gitignored)
`platforms/`, `emulators/`, `database.json`, and the `wiki/` sources.
Documentation changes belong in `wiki/` or in the generator.
+The site search splits each page at its headings and sends a result to the
+heading above the match. Every file card is therefore a level 4 heading with
+the card's anchor, and every platform system a heading of its own: a hash
+pasted in the search opens that card or that system, not the top of the page.
+Level 4 stays out of the table of contents (`toc_depth: 3`). The lines that
+list other pages (platforms, packs, source citations, collection paths) sit
+in a `data-search-exclude` block, since the browser downloads and indexes the
+whole search index before it shows a first result. File names, hashes, the
+paths an emulator reads and the notes stay searchable.
+
## Data flow
```
diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md
index 3cc4d2a6..45aaf5ea 100644
--- a/wiki/testing-guide.md
+++ b/wiki/testing-guide.md
@@ -44,7 +44,7 @@ Every module in `tests/`, by the area it guards.
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` |
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` |
| Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
-| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
+| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
## Test architecture