diff --git a/scripts/generate_site.py b/scripts/generate_site.py index f2b2afd7..1e267865 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -14,7 +14,7 @@ Usage: from __future__ import annotations -from siterender import WIKI_SRC_DIR +from siterender import SECTION_LABELS, WIKI_SRC_DIR import argparse import csv import hashlib @@ -117,7 +117,7 @@ RELEASE_URL = f"{REPO_URL}/releases/latest" # metadata. The BIOS files themselves are under NOTICE, never this. DATA_LICENSE = "MIT (LICENSE: scripts, tooling, schemas, documentation and metadata)" SITE_URL = "https://abdess.github.io/retrobios/" -GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads", +GENERATED_DIRS = ["platforms", "systems", "emulators", "wiki", "api", "downloads", "files", release_record.GET_DIR] # Generated directories of files served as they are, not pages. PUBLIC_DIRS = ("api", "downloads") @@ -1809,7 +1809,8 @@ def system_index_items(db: dict, coverages: dict, profiles: dict) -> list[dict]: def _first_file_place( - sys_id: str, sha1s: list[str], files: dict, profiles: dict, card_page, + sys_id: str, sha1s: list[str], files: dict, profiles: dict, + card_page: Callable[[str, str], str], ) -> tuple[str, str | None]: """(manufacturer, url) of the card of an id's first collection file, else of the page of the first emulator naming the id.""" @@ -4017,7 +4018,7 @@ def generate_files_page() -> str: """ return "\n".join([ "---", - "terms: check dump, md5 check, verify bios file, check hash", + "terms: check my dump, md5 check, verify bios file, check hash", "---", "", "# Find or check a file", @@ -5721,7 +5722,8 @@ def _start( held = {str(f.get("sha1", "")) for f in files} lines = _lead_file(groups) if not lines: - lines = _group_lines(groups, first_below, held, sha1s) + lines = _group_lines(groups, first_below, held, sha1s, + _section_label(*section) if section and section != OTHER_ARCADE else "") order: list[str] = [] for line in lines: for visible in re.findall(r"`([^`]*)`", line): @@ -5733,9 +5735,10 @@ def _start( def _group_lines( groups: dict[str | None, _StartGroup], first_below: bool, held: set[str], - sha1s: dict[str, str], + sha1s: dict[str, str], console: str = "", ) -> list[str]: - """One block per system, as the section opens on them.""" + """One block per system, as the section opens on them. ``console`` names + the section in a line that picks a BIOS by region.""" keys = sorted(groups, key=lambda k: str(k).casefold()) blocks: dict[str | None, list[tuple[str, str]]] = {} for key in keys: @@ -5751,7 +5754,7 @@ def _group_lines( if not blocks: return [] if len(blocks) == 1: - return _start_lines(next(iter(blocks.values()))) + return _start_lines(next(iter(blocks.values())), console) if all(lead == "*" for block in blocks.values() for lead, _text in block): # No machine of the section needs a file: the SNES said "Start with" # over two lines that each said no file is needed. @@ -5891,8 +5894,11 @@ def _group_start(group: _StartGroup, first_below: bool) -> list[tuple[str, str]] return out -def _start_lines(block: list[tuple[str, str]]) -> list[str]: - """One system's start, as the lines its section opens on.""" +def _start_lines(block: list[tuple[str, str]], console: str = "") -> list[str]: + """One system's start, as the lines its section opens on. A BIOS picked + by region is named with its console: "playstation bios" found the + PlayStation 2 page first, its file names holding "BIOS" and the + PlayStation page's text never saying "PlayStation BIOS".""" lead, text = block[0] if lead == "*": return [f"**No file needed**{text}", ""] @@ -5900,7 +5906,8 @@ def _start_lines(block: list[tuple[str, str]]) -> list[str]: named, _sep, where = text.partition(", first below") below = " They come first below." if _sep else "" return [f"**Start with**: {named}.{below}", ""] - lines = ["**Start with** the BIOS for the region of your games:", ""] + bios = f"the {console} BIOS" if console else "the BIOS" + lines = [f"**Start with** {bios} for the region of your games:", ""] for lead, text in block: lines.append(f"- **{lead or 'Some emulators also require'}**: {text}") return [*lines, ""] @@ -6031,12 +6038,16 @@ def arcade_board_sections( def _section_label(manufacturer: str, console: str) -> str: """The heading a section shows.""" - return OTHER_ARCADE_LABEL if (manufacturer, console) == OTHER_ARCADE else console + if (manufacturer, console) == OTHER_ARCADE: + return OTHER_ARCADE_LABEL + return SECTION_LABELS.get((manufacturer, console), console) def _part_name(manufacturer: str, console: str) -> str: """What a part's title calls the files: "Arcade files a to b".""" - return "Arcade" if (manufacturer, console) == OTHER_ARCADE else console + if (manufacturer, console) == OTHER_ARCADE: + return "Arcade" + return SECTION_LABELS.get((manufacturer, console), console) def _system_front_matter( @@ -6450,18 +6461,10 @@ def emulator_initial(display: str) -> str: return initial if "A" <= initial <= "Z" else INITIAL_DIGITS -# Profiles that document a format, not an emulator: `00_example` is the -# template libretro-core-info ships for core authors ("No loadable binary, -# no emulated system", its notes). Out of the index and the search, like a -# test core, and its page stays for the links that cite it. -TEMPLATE_PROFILES = frozenset({"00_example"}) -_TEMPLATE_FRONT = ["---", "search: exclude", "---", ""] - - def generate_emulators_index(profiles: dict) -> str: unique = { k: v for k, v in profiles.items() - if v.get("type") not in ("alias", "test") and k not in TEMPLATE_PROFILES + if v.get("type") not in ("alias", "test") } aliases = {k: v for k, v in profiles.items() if v.get("type") == "alias"} @@ -7482,7 +7485,7 @@ def _data_tree_page(ref: str, entry: dict, pages: list[tuple[str, list[str]]], stem, names = pages[index] members = _DATA_MEMBERS.get(ref) or {} part = f" (part {index + 1} of {len(pages)})" if len(pages) > 1 else "" - readers = _said_as_list([f"[{_escape_tags(display)}]({key}.md#data-directories)" + read_by = _said_as_list([f"[{_escape_tags(display)}]({key}.md#data-directories)" for key, display in _DATA_READERS.get(ref, [])]) # A file list out of the search: "dolphin" found three parts of the # dolphin-sys tree, on their title, before the Dolphin page. @@ -7505,7 +7508,7 @@ def _data_tree_page(ref: str, entry: dict, pages: list[tuple[str, list[str]]], f"# The {ref} data directory{part}", "", f"{entry.get('description') or 'A data directory'}: " f"{_count(len(members), 'file')}, as the packs ship them." - + (f" Read by {readers}." if readers else ""), "", + + (f" Read by {read_by}." if read_by else ""), "", ] if len(pages) > 1: lines.extend([", ".join(f"[part {i}]({other}.md)" if other != stem else f"part {i}" @@ -7848,7 +7851,6 @@ def emulator_pages( one_bios, consumers, name) if one_bios else "") romset = emulator_romset(profile) lines = [ - *(_TEMPLATE_FRONT if name in TEMPLATE_PROFILES else []), f"# {emu_name}", "", *([intro, ""] if intro else []), @@ -10293,10 +10295,10 @@ def _generate(args: argparse.Namespace) -> None: # Swept before the decoration, which rewrites every page it finds: a # page no generator produced any more (a console moved back onto its # manufacturer's page) was decorated, recorded, and kept for ever. + write_if_changed(str(docs / "404.md"), generate_not_found()) stale = _sweep_generated(docs, [d for d in GENERATED_DIRS if d not in PUBLIC_DIRS]) stale += _sweep_generated(public, list(PUBLIC_DIRS)) - - write_if_changed(str(docs / "404.md"), generate_not_found()) + stale += _sweep_top_pages(docs) print("Adding page metadata and structured data...") lastmod = decorate_markdown_pages(docs) @@ -10338,6 +10340,7 @@ from sitewrite import ( # noqa: E402,F401 write_if_changed, write_decorated, _sweep_generated, + _sweep_top_pages, ) from sitesources import ( # noqa: E402,F401 link_prose_citations, diff --git a/scripts/siterender.py b/scripts/siterender.py index 1df5f74a..c24ada69 100644 --- a/scripts/siterender.py +++ b/scripts/siterender.py @@ -629,6 +629,12 @@ _PROGRAM_TOPS = frozenset({"Other", "Arcade"}) # An Arcade heading can (Namco System 246), an Other one cannot. _NAMELESS_TOPS = frozenset({"Other"}) +# A section whose folder is a short name the code reads, headed with the +# program's name instead (bios/Arcade/fba2012 holds FB Alpha 2012's samples). +SECTION_LABELS: dict[tuple[str, str], str] = { + ("Arcade", "fba2012"): "FB Alpha 2012", +} + # Every (manufacturer, console) section the system pages carry. _section_pairs: set[tuple[str, str]] = set() # Normalized id -> the console sections spelled that way. diff --git a/scripts/sitesources.py b/scripts/sitesources.py index 62b5e26b..065544c4 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -670,4 +670,6 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + from common import run_cli + + run_cli(main) diff --git a/scripts/sitewrite.py b/scripts/sitewrite.py index fc8589ce..c90bf78d 100644 --- a/scripts/sitewrite.py +++ b/scripts/sitewrite.py @@ -69,3 +69,20 @@ def _sweep_generated(docs: Path, directories: list[str]) -> int: if path.is_dir() and not any(path.iterdir()): path.rmdir() return removed + + +def _sweep_top_pages(docs: Path) -> int: + """Delete the pages at the root of the docs tree this run did not produce. + + The root holds only generated pages, one per stem (`cross-reference-*`, + `gaps-*`), and a stem that stops being produced, such as a platform + whose core table moved to its own page, otherwise kept its old page + and claimed the address its redirect needs. + """ + removed = 0 + for path in sorted(docs.glob("*.md*")): + if (path.is_file() and path.suffix in (".md", ".mdx") + and os.path.realpath(str(path)) not in _produced): + path.unlink() + removed += 1 + return removed diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 39097b78..65795aaf 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -6724,9 +6724,6 @@ class NoArchiveRepeatsAMemberName(unittest.TestCase): self.assertEqual(repeated, []) -if __name__ == "__main__": - unittest.main() - class PipelineChildrenSkipNodeCompileCache(unittest.TestCase): """The site build's compile cache filled the 4 GB /tmp tmpfs to 3.4 GB: @@ -6735,3 +6732,7 @@ class PipelineChildrenSkipNodeCompileCache(unittest.TestCase): def test_the_run_helper_sets_it(self): source = (ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8") self.assertIn('"NODE_DISABLE_COMPILE_CACHE": "1"', source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_cores.py b/tests/test_site_cores.py index 11b14d87..61b0e14b 100644 --- a/tests/test_site_cores.py +++ b/tests/test_site_cores.py @@ -41,6 +41,12 @@ class SiteCoreCounts(unittest.TestCase): } for name, cov in coverages.items(): with self.subTest(page=name): + if generate_site._own_list_only(cov["config"]): + # cores: [] (MiSTer, BizHawk): no core table, the overview + # row points at the platform's own list instead. + self.assertNotIn(f"cross-reference-{name}", moved) + self.assertEqual(shown[cov["platform"]], 0) + continue rows = [line for line in moved[f"cross-reference-{name}"].splitlines() if line.startswith("| [")] self.assertEqual(len(rows), shown[cov["platform"]]) diff --git a/tests/test_site_r2_systems.py b/tests/test_site_r2_systems.py index 9fece26d..e883e797 100644 --- a/tests/test_site_r2_systems.py +++ b/tests/test_site_r2_systems.py @@ -289,11 +289,10 @@ class TheBuiltSonyPage(unittest.TestCase): def test_playstation_opens_on_a_bios_it_needs(self): section = self.text.split('id="playstation"', 1)[1].split("
(.*?)', section)[1]
+ # PlayStation has a page of its own, and its files open there.
+ own = (DIST / "systems" / "sony-playstation" / "index.html").read_text(encoding="utf-8")
+ self.assertIn('href="/retrobios/systems/sony-playstation/"', section)
+ first = re.search(r'class="rb-file-title"[^>]*>]*>(.*?)', own)[1]
self.assertNotIn(first, {"LibPS.exe", "PSX_DTLH2000.bin"})
self.assertRegex(first, r"^scph\d+")
diff --git a/tests/test_site_r3_nav.py b/tests/test_site_r3_nav.py
index 800a37e3..ec1c3937 100644
--- a/tests/test_site_r3_nav.py
+++ b/tests/test_site_r3_nav.py
@@ -202,7 +202,9 @@ class BuiltSite(unittest.TestCase):
def test_search_fields_hold_no_text(self) -> None:
texts = []
for path in (DIST / "systems").rglob("index.html"):
- for match in re.finditer(r"<(\w+)[^>]*data-pagefind-meta=[^>]*>([^<]*)\1>",
+ # The title meta reads the visible h1 (PageTitle.astro). The
+ # search fields are attributes on empty spans.
+ for match in re.finditer(r"<(\w+)[^>]*data-pagefind-meta=\"(?!title\")[^>]*>([^<]*)\1>",
path.read_text(encoding="utf-8")):
if match[2].strip():
texts.append(str(path.relative_to(DIST)))
diff --git a/tests/test_site_r4_emulators.py b/tests/test_site_r4_emulators.py
index 6ed8da34..95124563 100644
--- a/tests/test_site_r4_emulators.py
+++ b/tests/test_site_r4_emulators.py
@@ -142,7 +142,8 @@ class TheTemplateIsNoEmulator(unittest.TestCase):
index = gs.generate_emulators_index(profiles)
self.assertNotIn("00_example", index)
self.assertIn("melonds.md", index)
- self.assertEqual(_front(_pages("00_example")["00_example"])["search"], "exclude")
+ # A test profile gets no page at all (generate_site's public_profiles).
+ self.assertEqual(_profile("00_example")["type"], "test")
@unittest.skipUnless((DIST / "emulators" / "aethersx2" / "index.html").is_file(), "site not built")
@@ -156,9 +157,10 @@ class TheBuiltPages(unittest.TestCase):
self.assertNotIn("--emulator aethersx2", page)
def test_the_index_leaves_the_template_out(self):
+ # 00_example is `type: test`: no page, no index row.
self.assertNotIn("00_example", self._html("emulators"))
self.assertIn('data-pagefind-body', self._html("emulators/melonds"))
- self.assertNotIn('data-pagefind-body', self._html("emulators/00_example"))
+ self.assertFalse((DIST / "emulators" / "00_example").exists())
def test_mame_parts_have_a_parent_crumb(self):
page = self._html("emulators/mame-files-3")
diff --git a/tests/test_site_r4_merge.py b/tests/test_site_r4_merge.py
new file mode 100644
index 00000000..11227b62
--- /dev/null
+++ b/tests/test_site_r4_merge.py
@@ -0,0 +1,149 @@
+"""What the round 4 packages left between them once merged.
+
+A data directory page printed its readers as a Python list, every board
+page's breadcrumb read "Arcade / Arcade", the fba2012 section was headed
+with its folder name, BizHawk's tile showed a GitHub avatar and MinUI's a
+logo that is not its own, and the nodes the search gate starts wrote
+Node's compile cache into /tmp.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import re
+import sys
+import unittest
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import check_freshness # noqa: E402
+import generate_site as gs # noqa: E402
+import siterender # noqa: E402
+
+DIST = REPO_ROOT / "website" / "dist"
+
+
+def _profile(name: str) -> dict:
+ return yaml.safe_load((REPO_ROOT / "emulators" / f"{name}.yml").read_text(encoding="utf-8"))
+
+
+class DataTreeReaders(unittest.TestCase):
+ def test_read_by_names_the_emulators(self):
+ ref = "y-sys"
+ gs._DATA_MEMBERS[ref] = {"a.bin": {"size": 1, "sha1": "0" * 40}}
+ gs._DATA_READERS[ref] = [("dolphin", "Dolphin"), ("ishiiruka", "Ishiiruka")]
+ try:
+ page = gs._data_tree_page(ref, {}, [("y-sys", ["a.bin"])], 0)
+ finally:
+ del gs._DATA_MEMBERS[ref], gs._DATA_READERS[ref]
+ self.assertIn("Read by [Dolphin](dolphin.md#data-directories) and "
+ "[Ishiiruka](ishiiruka.md#data-directories).", page)
+ self.assertNotIn("('dolphin'", page)
+
+
+class EngineIntros(unittest.TestCase):
+ def test_every_intro_names_what_it_emulates(self):
+ for path in sorted(glob.glob(str(REPO_ROOT / "emulators" / "*.yml"))):
+ profile = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
+ if not isinstance(profile, dict) or profile.get("type") in ("alias", "test"):
+ continue
+ name = Path(path).stem
+ with self.subTest(profile=name):
+ intro = gs._emulator_intro(str(profile.get("emulator") or name), profile)
+ self.assertNotRegex(intro, r"emulates\s*(?:\.|game\b|and\b|,)")
+ self.assertNotIn(" ", intro)
+
+
+class SectionNames(unittest.TestCase):
+ def test_fba2012_is_headed_by_its_program(self):
+ self.assertEqual(gs._section_label("Arcade", "fba2012"), "FB Alpha 2012")
+ self.assertEqual(gs._section_label("Arcade", "Arcade"), gs.OTHER_ARCADE_LABEL)
+ self.assertEqual(gs._section_label("Sony", "PlayStation"), "PlayStation")
+ self.assertEqual(siterender.SECTION_LABELS[("Arcade", "fba2012")],
+ _profile("fbalpha2012")["emulator"])
+
+
+class PlatformLogos(unittest.TestCase):
+ def setUp(self):
+ self.registry = yaml.safe_load(
+ (REPO_ROOT / "platforms" / "_registry.yml").read_text(encoding="utf-8"))["platforms"]
+
+ def test_bizhawk_shows_its_app_icon(self):
+ self.assertTrue(self.registry["bizhawk"]["logo"].endswith(
+ "/src/BizHawk.Client.EmuHawk/images/logo.ico"))
+
+ def test_minui_has_no_logo_and_no_stale_copy(self):
+ self.assertNotIn("logo", self.registry["minui"])
+ self.assertEqual(list((REPO_ROOT / "docs_assets" / "logos").glob("minui.*")), [])
+
+ def test_every_logo_copy_has_a_registry_logo(self):
+ sources = json.loads((REPO_ROOT / "docs_assets" / "logos" / "sources.json").read_text())
+ for entry in sources["images"]:
+ with self.subTest(platform=entry["platform"]):
+ self.assertEqual(self.registry[entry["platform"]].get("logo"), entry["source"])
+
+
+class NodeCompileCache(unittest.TestCase):
+ def test_the_search_gate_starts_node_without_it(self):
+ source = (REPO_ROOT / "website" / "tools" / "search_gate.py").read_text(encoding="utf-8")
+ self.assertIn('NODE_ENV = {**os.environ, "NODE_DISABLE_COMPILE_CACHE": "1"}', source)
+ starts = [m.start() for m in re.finditer(r'subprocess\.(?:run|Popen)\(\s*\["node"', source)]
+ self.assertEqual(len(starts), 3)
+ for start in starts:
+ call = source[start:source.index("\n\n", start)]
+ self.assertIn("env=NODE_ENV", call)
+
+ def test_lighthouse_and_the_deploy(self):
+ self.assertIn('"NODE_DISABLE_COMPILE_CACHE": "1"',
+ (REPO_ROOT / "website" / "tools" / "lighthouse.py").read_text(encoding="utf-8"))
+ workflow = (REPO_ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(encoding="utf-8")
+ build = workflow[workflow.index(" build:"):workflow.index(" steps:")]
+ self.assertIn('NODE_DISABLE_COMPILE_CACHE: "1"', build)
+
+
+class SourcePathsFreshness(unittest.TestCase):
+ def test_the_committed_record_is_current(self):
+ from common import load_emulator_profiles # noqa: PLC0415
+
+ profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"), skip_aliases=False)
+ finding = check_freshness.check_source_paths(profiles)
+ self.assertEqual(finding.status, check_freshness.OK, finding.detail)
+
+ def test_a_moved_pin_is_stale(self):
+ profile = _profile("geolith")
+ moved = dict(profile, source_commit="f" * 40, upstream_commit="e" * 40)
+ finding = check_freshness.check_source_paths({"geolith": moved})
+ self.assertEqual(finding.status, check_freshness.STALE)
+ self.assertIn("sitesources.py --fetch", finding.detail)
+
+
+@unittest.skipUnless((DIST / "systems" / "arcade-acclaim-zn-1" / "index.html").is_file(),
+ "site not built")
+class BuiltCrumbs(unittest.TestCase):
+ def test_no_crumb_repeats_the_one_before(self):
+ repeated = []
+ for path in sorted(DIST.rglob("index.html")):
+ nav = re.search(r'',
+ path.read_text(encoding="utf-8"), re.S)
+ if not nav:
+ continue
+ labels = [re.sub(r"<[^>]+>", "", item).strip()
+ for item in re.findall(r"