feat: link arcade boards and program folders

scripts/siterender.py

@@ -150,8 +150,10 @@ _card_pages: dict[tuple[str, str, str], str] = {}
 
 
 def reset_console_pages() -> None:
+    """Forget where the system pages put consoles, cards and boards."""
     _console_pages.clear()
     _card_pages.clear()
+    _board_sections.clear()
 
 
 def record_console_pages(
@@ -190,7 +192,13 @@ def console_anchor(manufacturer: str, console: str) -> str:
 
 def console_href(slug: str, console: str, prefix: str = "") -> str:
     """Where a console's files are: its own page, or its section of the
-    manufacturer's page (also where a console cut into parts lists them)."""
+    manufacturer's page (also where a console cut into parts lists them).
+    An arcade board's section heads the page of its own name."""
+    if slug == "arcade" and _board_sections:
+        board = _board_ids_by_name().get(console.casefold())
+        if board in _board_sections:
+            anchor = board_slug(board)
+            return f"{prefix}systems/arcade-{urllib.parse.quote(anchor)}.md#{anchor}"
     stems = _console_pages.get((slug, console))
     if stems and len(stems) == 1:
         return f"{prefix}systems/{urllib.parse.quote(stems[0])}.md"
@@ -203,6 +211,10 @@ def card_page(path: str, sha1: str) -> str:
     parts = path.split("/")
     slug = parts[1].lower().replace(" ", "-")
     console = parts[2] if len(parts) > 2 else ""
+    if slug == "arcade" and _board_sections:
+        board = arcade_board(sha1)
+        if board in _board_sections:
+            return f"arcade-{board_slug(board)}"
     return _card_pages.get((slug, console, sha1), slug)
 
 
@@ -225,12 +237,28 @@ def emulator_card_page(emulator: str, anchor: str) -> str:
 def system_section(sys_id: str) -> tuple[str, str] | None:
     """The console section a system id leads to, as (manufacturer slug,
     console): its own section, else the parent SYSTEM_PARENTS names, else
-    the one section whose heading reads system_name(). The folder a
-    system's files sit in is no answer by itself: Emerson's Arcadia 2001
-    led to the Saturn and Sega's NAOMI to the Dreamcast.
+    the arcade board it names once record_board_section() wrote it
+    (``("arcade", <board name>)``, which console_href turns into
+    ``systems/arcade-<board slug>.md#<board slug>``), else the one section
+    whose heading reads system_name(). The folder a system's files sit in
+    is no answer by itself: Emerson's Arcadia 2001 led to the Saturn and
+    Sega's NAOMI to the Dreamcast.
 
     A heading under Arcade counts when it reads the name exactly (Namco
     System 246). One under Other never does, its headings name programs."""
+    section = _section_without_boards(sys_id)
+    if (not _board_sections or sys_id in SYSTEM_PARENTS
+            or (section is not None and section[0] != "arcade")):
+        return section
+    board = _board_of_id(sys_id)
+    if board in _board_sections:
+        return "arcade", system_name(board)
+    return section
+
+
+def _section_without_boards(sys_id: str) -> tuple[str, str] | None:
+    """system_section before arcade boards: what the folders and the
+    reviewed parents give."""
     own = _own_section(sys_id)
     if own is not None:
         return _manufacturer_slug(own[0]), own[1]
@@ -260,6 +288,236 @@ def _system_link(sys_id: str, prefix: str = "") -> str:
         return f"[{sys_id}]({console_href(slug, console, prefix)})"
     return sys_id
 
+
+# Arcade boards. A board is a system id that a profile entry with
+# ``category: bios_zip`` names, through its ``system:``/``systems:`` or the
+# profile's single system, whose archive the collection holds, and that has
+# no section under a machine's maker. The archive is found by its name and,
+# where the entry lists ``contents:``, kept only when it holds one of those
+# CRC32s: maclc3.zip names a MAME set and an unrelated 189 MB file. The
+# folder a copy sits in decides nothing.
+_arcade_source: list[tuple[dict, dict] | None] = [None]
+# sha1 -> the board id the archive is the BIOS of.
+_archive_boards: dict[str, str] = {}
+# Casefolded board name -> the board id that stands for it.
+_board_names: dict[str, str] = {}
+_boards_built = [False]
+
+
+def set_arcade_boards(profiles: dict[str, dict], db: dict) -> None:
+    """Install the profiles and database the arcade boards are read from.
+    Without it they are read from the repository on first use."""
+    _arcade_source[0] = (profiles, db)
+    _boards_built[0] = False
+
+
+def _owner_ids(entry: dict, systems: list[str]) -> set[str]:
+    """The system ids a profile entry is declared for."""
+    owners = {str(entry["system"])} if entry.get("system") else set()
+    owners.update(str(sys_id) for sys_id in entry.get("systems") or [])
+    if not owners and len(systems) == 1:
+        owners = set(systems)
+    return owners
+
+
+# _board_role for an id whose archive is its machine's: a section under the
+# machine's maker, or a reviewed parent outside Arcade.
+_MACHINE = "\0machine"
+
+
+def _board_role(sys_id: str) -> str | None:
+    """A board's name, _MACHINE for an id that names a machine with a
+    section of its own, None for an id that names neither (the generic
+    arcade id, an id with no name)."""
+    parent = SYSTEM_PARENTS.get(sys_id)
+    if parent is not None:
+        return None if parent[0] == "Arcade" else _MACHINE
+    section = _section_without_boards(sys_id)
+    if section is not None and section[0] != "arcade":
+        return _MACHINE
+    try:
+        return system_name(sys_id)
+    except KeyError:
+        return None
+
+
+def _in_arcade(sha1: str, db: dict) -> bool:
+    """Whether a copy of the content sits under bios/Arcade/."""
+    entry = db.get("files", {}).get(sha1) or {}
+    if str(entry.get("path", "")).startswith("bios/Arcade/"):
+        return True
+    return sha1 in _arcade_suffix_sha1s(db)
+
+
+_suffix_cache: list[tuple[int, frozenset[str]] | None] = [None]
+
+
+def _arcade_suffix_sha1s(db: dict) -> frozenset[str]:
+    cached = _suffix_cache[0]
+    if cached is not None and cached[0] == id(db):
+        return cached[1]
+    found = frozenset(
+        sha1s[0]
+        for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items()
+        if tail.startswith("Arcade/") and len(sha1s) == 1
+    )
+    _suffix_cache[0] = (id(db), found)
+    return found
+
+
+def _build_boards() -> None:
+    if _boards_built[0]:
+        return
+    from ziptools import zip_member_crcs
+
+    _ensure_console_sections()
+    if _arcade_source[0] is None:
+        import json
+
+        from common import load_emulator_profiles
+
+        database = _REPO_ROOT / "database.json"
+        if not database.exists():
+            _archive_boards.clear()
+            _board_names.clear()
+            _boards_built[0] = True
+            return
+        with open(database, encoding="utf-8") as handle:
+            db = json.load(handle)
+        profiles = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False)
+        _arcade_source[0] = (profiles, db)
+    profiles, db = _arcade_source[0]
+    files = db.get("files", {})
+    by_name = db.get("indexes", {}).get("by_name", {})
+
+    # sha1 -> board name -> the profiles naming it. None: a machine's own.
+    votes: dict[str, dict[str, set[str]] | None] = {}
+    ids_by_name: dict[str, set[str]] = {}
+    for key in sorted(profiles):
+        profile = profiles[key]
+        if profile.get("type") == "test":
+            continue
+        systems = [str(sys_id) for sys_id in profile.get("systems") or []]
+        for entry in profile.get("files") or []:
+            if not isinstance(entry, dict) or entry.get("category") != "bios_zip":
+                continue
+            owners = _owner_ids(entry, systems)
+            if not owners:
+                continue
+            crcs = {
+                str(member["crc32"]).lower()
+                for member in entry.get("contents") or []
+                if isinstance(member, dict) and member.get("crc32")
+            }
+            names = {str(entry.get("name", ""))}
+            names.update(str(alias) for alias in entry.get("aliases") or [])
+            archives: set[str] = set()
+            for name in names:
+                found = by_name.get(name.rsplit("/", 1)[-1]) or []
+                for sha1 in [found] if isinstance(found, str) else found:
+                    if sha1 not in files:
+                        continue
+                    if crcs:
+                        held = zip_member_crcs(str(_REPO_ROOT / files[sha1]["path"]))
+                        # An archive the build cannot read keeps its name.
+                        if held and not held & crcs:
+                            continue
+                    archives.add(sha1)
+            for owner in owners:
+                role = _board_role(owner)
+                if role == _MACHINE:
+                    for sha1 in archives:
+                        votes[sha1] = None
+                    continue
+                if role is None:
+                    continue
+                ids_by_name.setdefault(role.casefold(), set()).add(owner)
+                for sha1 in archives:
+                    ballot = votes.setdefault(sha1, {})
+                    if ballot is not None:
+                        ballot.setdefault(role.casefold(), set()).add(key)
+
+    _archive_boards.clear()
+    _board_names.clear()
+    for sha1, ballot in votes.items():
+        if not ballot:
+            continue
+        ranked = sorted(ballot.items(), key=lambda item: (-len(item[1]), item[0]))
+        # Two boards named as often: a device ROM several boards share.
+        if len(ranked) > 1 and len(ranked[0][1]) == len(ranked[1][1]):
+            continue
+        name = ranked[0][0]
+        board = sorted(ids_by_name[name])[0]
+        _archive_boards[sha1] = board
+        if _in_arcade(sha1, db):
+            _board_names[name] = board
+    # A board none of whose archives sits under Arcade has no section there.
+    for sha1, board in list(_archive_boards.items()):
+        if system_name(board).casefold() not in _board_names:
+            del _archive_boards[sha1]
+    _boards_built[0] = True
+
+
+def _board_ids_by_name() -> dict[str, str]:
+    _build_boards()
+    return _board_names
+
+
+def _board_of_id(sys_id: str) -> str | None:
+    """The board an id names, by its display name: naomi2 and sega-naomi2
+    are one board."""
+    try:
+        name = system_name(sys_id)
+    except KeyError:
+        return None
+    return _board_ids_by_name().get(name.casefold())
+
+
+def arcade_board(sha1: str) -> str | None:
+    """The system id an arcade archive is the BIOS of, or None.
+
+    Read from the profiles' ``system``/``systems`` and ``contents:``,
+    never from the folder. An archive several boards share as often (a
+    device ROM), or one a machine with its own section names (neogeo.zip
+    for the Neo Geo, cd32.zip for the CD32), has no board. Every id of a
+    board's name returns the same id, the first in sort order."""
+    _build_boards()
+    return _archive_boards.get(str(sha1))
+
+
+def board_slug(sys_id: str) -> str:
+    """The slug of a board's section: its page is systems/arcade-<slug>/
+    and its heading's id is <slug>. Read from the board's display name,
+    so every id of one board gives one slug."""
+    from sitestarlight import toc_slug
+
+    return toc_slug(system_name(str(sys_id)))
+
+
+# Boards whose section the system pages wrote. A link waits for its
+# section: before the Arcade pages were grouped by board, 8,606 links on
+# 141 pages led to pages nobody wrote and the build stopped.
+_board_sections: set[str] = set()
+
+
+def record_board_section(board: str) -> None:
+    """Mark a board's section as written, at systems/arcade-<board_slug>/
+    under the heading <board_slug>. From then on system_section() leads
+    every id of the board there, and card_page() sends its Arcade cards
+    there. The system pages call it for each board they give a section,
+    before any other page links one."""
+    canonical = _board_of_id(str(board))
+    if canonical is None:
+        raise KeyError(f"{board!r} is not an arcade board")
+    _board_sections.add(canonical)
+
+
+def arcade_boards() -> dict[str, str]:
+    """Every board, by board id -> display name."""
+    _build_boards()
+    return {board: system_name(board) for board in sorted(set(_board_names.values()))}
+
+
 def _field_text(key: object, value: object) -> str:
     """A scalar of a structured block. A source reference reads as code."""
     if isinstance(value, bool):
@@ -435,6 +693,8 @@ def set_console_sections(
     _program_names.clear()
     _program_names.update(name.casefold() for name in programs)
     _sections_installed[0] = True
+    # Which ids are boards depends on which have a section elsewhere.
+    _boards_built[0] = False
 
 
 _sections_installed = [False]
@@ -921,8 +1181,10 @@ SYSTEM_NAMES: dict[str, str] = {
     "zn1-acclaim": "Acclaim ZN-1",
     "zn1-atlus": "Atlus ZN-1",
     "zn1-capcom": "Capcom ZN-1",
-    "zn1-taito": "Taito ZN-1",
-    "zn1-tecmo": "Tecmo ZN-1",
+    # mame2003 files taitofx1.zip and tps.zip under these ids, its notes
+    # name the boards: the Taito FX-1 and the Tecmo TPS, both ZN-1 based.
+    "zn1-taito": "Taito FX-1",
+    "zn1-tecmo": "Tecmo TPS",
     "zn1-video-system": "Video System ZN-1",
     "zn2-capcom": "Capcom ZN-2",
     "zx-spectrum-128": "Sinclair ZX Spectrum 128",
@@ -1008,6 +1270,7 @@ SYSTEM_PARENTS: dict[str, tuple[str, str]] = {
     "marty": ("Fujitsu", "FM Towns"),
     "mz700": ("Sharp", "MZ"),
     "mz80k": ("Sharp", "MZ"),
+    "mz800": ("Sharp", "MZ"),
     "oricatmos": ("Tangerine", "Oric"),
     "palm-os": ("Palm", "Palm"),
     "philips-vg5k": ("Philips", "VG5000"),
@@ -1032,6 +1295,22 @@ SYSTEM_PARENTS: dict[str, tuple[str, str]] = {
     "mohaa-spearhead": ("Electronic Arts", "Medal of Honor Allied Assault"),
     "ur-quan-masters": ("Toys for Bob", "The Ur-Quan Masters"),
     "star-control-2": ("Toys for Bob", "The Ur-Quan Masters"),
+    # Systems whose files sit in the folder of the program that reads them,
+    # the only section listing them. DOS and ScummVM also read the Roland
+    # MT-32 ROMs, which their own sections do not hold.
+    "bbc-master-128": ("Other", "beebem"),
+    "bbc-master-et": ("Other", "beebem"),
+    "cave-story": ("Other", "NXEngine"),
+    "dos": ("DOS", "DOSBox"),
+    "scummvm": ("Other", "ScummVM"),
+    "sinclair-zx-spectrum-next": ("Other", "cspect"),
+    "solarus": ("Game Engines", "Solarus"),
+    "sunplus-spmp8000": ("Other", "spmp8000emu"),
+    "tomb-raider-1": ("Game Engines", "TRX"),
+    "tyrian": ("Other", "opentyrian"),
+    "vm-labs-nuon": ("Other", "nuance"),
+    "xrick": ("Other", "xrick"),
+    "zc210": ("ZC", "ZC210"),
     # The collection's general arcade section, where naomi.zip, neogeo.zip
     # and the other board sets sit.
     "arcade": ("Arcade", "Arcade"),

tests/test_site_r3_p0.py

@@ -1,11 +1,14 @@
-"""Every system id with a section leads to it (round 3, package P0).
+"""Every system id with a section leads to it (rounds 3 and 4, package P0).
 
 260 of 636 system ids linked: a variant (WonderSwan Color, Neo Geo MVS)
 never matched a heading, twin folders (DS and Nintendo DS) made the exact
 name ambiguous, and every Arcade heading was refused. siterender now reads
-a reviewed SYSTEM_PARENTS table after the id's own section, and accepts an
-Arcade heading that reads the name exactly. An id whose files the
-collection holds and that stays plain is named below, with the reason.
+a reviewed SYSTEM_PARENTS table after the id's own section, accepts an
+Arcade heading that reads the name exactly, and leads an arcade board
+(NAOMI, ZN-1, PGM) to its own section. An id counts as holding files
+through a set archive (``category: bios_zip``) its profile declares for it,
+not only through a file's own hash. An id whose files the collection holds
+and that stays plain is named below, with the reason.
 Reads the repository: skips when the database is absent.
 """
 
@@ -25,60 +28,33 @@ import siterender  # noqa: E402
 # Ids whose files the collection holds and that lead to no section.
 UNLINKED: dict[str, str] = {
     # Files only under a program's folder (Other/<emulator>, Arcade/<core>,
-    # Game Engines/<engine>, a top named after the program). The heading
-    # names the program, not the system.
-    "atari-system1": "Other > fbneo",
-    "bbc-master-128": "Other > beebem",
-    "bbc-master-et": "Other > beebem",
+    # Game Engines/<engine>, a top named after the program), a folder of
+    # several systems' files or not reviewed as the system's own. The
+    # heading names the program, not the system.
     "capcom-cps3": "Arcade > FBNeo",
-    "cave-story": "Other > NXEngine",
     "cps3": "Arcade > Arcade holds one CPS-3 set among every board's",
     "commanderx16": "Arcade > Arcade",
     "dark-forces-ii": "Game Engines > OpenJKDF2",
-    "deco-cassette": "Arcade > FBNeo",
-    "dos": "DOSBox, Roland and Other folders",
     "fpinball": "Other > future-pinball-fploader",
     "fruitmachine": "Other > mfme",
-    "igs-pgm": "Arcade > FBNeo",
-    "konami-bubsys": "Arcade > FBNeo",
     "microsoft-xbox-360": "Other > xenia-edge",
-    "midway-mcr": "Arcade > FBNeo",
     "mysteries-of-the-sith": "Game Engines > OpenJKDF2",
-    "mz2000": "Arcade > Arcade, not the Sharp MZ section",
-    "mz2500": "Arcade > Arcade, not the Sharp MZ section",
-    "mz800": "Arcade > Arcade, not the Sharp MZ section",
     "namco-na1": "Arcade > FBNeo",
     "namco-nb1": "Arcade > FBNeo",
-    "nmk16": "Arcade > FBNeo",
     "omicron": "Other > omicron",
-    "pegasus": "Arcade > MAME",
     "rpgmaker": "RPG Maker > easyrpg",
-    "scummvm": "Other > ScummVM",
     "sdlpal": "Other > sdlpal",
-    "sega-system16b": "Arcade > FBNeo",
-    "sinclair-zx-spectrum-next": "Other > cspect",
+    "sega-system16b": "Arcade > FBNeo, and isgsm.zip is also the ISG Selection Master's",
     "smbx": "Game Engines > TheXTech",
-    "solarus": "Game Engines > Solarus",
-    "sunplus-spmp8000": "Other > spmp8000emu",
-    "super-kaneko-nova": "Arcade > FBNeo",
-    "taito-cchip": "Arcade > FBNeo",
     "thomson-mo6": "Other > Clock Signal",
-    "tomb-raider-1": "Game Engines > TRX",
-    "vgmplay": "Arcade > Arcade and MAME",
-    "vm-labs-nuon": "Other > nuance",
-    "vpinball": "Arcade > PinMAME",
-    "xrick": "Other > xrick",
-    "zc210": "ZC > ZC210, the emulator's folder",
+    "vgmplay": "qsound.zip, ym2413.zip, ym2608.zip: sound chips, no board",
     # Files filed with another machine: a link would name the wrong one.
-    "atom": "atom.zip is filed with the BBC Micro",
     "commodore-c64-supercpu": "VICE's tree under the C128",
     "emerson-arcadia-2001": "ar_bios.zip is filed with the Saturn",
-    "naomi2": "naomi2.zip is filed with the Dreamcast",
-    "neogeo64": "the platform declares the Neo Geo's neogeo.zip",
-    "sega-dreamcast-arcade": "naomi.zip and awbios.zip, Arcade and Dreamcast",
+    "sega-dreamcast-arcade": "NAOMI, NAOMI 2 and Atomiswave sets, three boards",
     "spectravideo": "the SVI ROMs are filed with the MSX",
+    "taito-type-x": "mame2010 files taitogn.zip, the G-NET BIOS, under this id",
     "vvvvvv": "SDL's controller database",
-    "ym2608": "an FBNeo sound ROM, also filed with the NES",
     # Files of distinct machines or games, one section each.
     "catacomb-adventure": "three games, three sections",
     "sinclair-zx80-81": "ZX 80 and ZX 81 sections",
@@ -97,6 +73,25 @@ def _accepts(fe: dict, local: str | None, status: str) -> bool:
     return bool(path) and local.endswith("/" + str(path))
 
 
+def _archives_held(fe: dict, db: dict) -> set[str]:
+    """The copies of a set archive (``category: bios_zip``) the collection
+    holds: by its name, kept when it holds one CRC32 its ``contents:``
+    lists. MAME hashes the members, never the archive, so the entry has no
+    hash of its own and a name match is the only way to it."""
+    from ziptools import zip_member_crcs
+
+    crcs = {str(m["crc32"]).lower() for m in fe.get("contents") or []
+            if isinstance(m, dict) and m.get("crc32")}
+    names = {str(fe.get("name", ""))} | {str(a) for a in fe.get("aliases") or []}
+    held: set[str] = set()
+    for name in names:
+        for sha1 in db["indexes"]["by_name"].get(name.rsplit("/", 1)[-1], []):
+            members = zip_member_crcs(str(REPO_ROOT / db["files"][sha1]["path"]))
+            if not crcs or not members or members & crcs:
+                held.add(sha1)
+    return held
+
+
 @unittest.skipUnless((REPO_ROOT / "database.json").exists(), "database.json is not built")
 class SystemLinks(unittest.TestCase):
 
@@ -119,6 +114,12 @@ class SystemLinks(unittest.TestCase):
             profiles = load_emulator_profiles("emulators", skip_aliases=False)
             siterender.set_console_sections(
                 cls.manufacturers, siterender.program_names(profiles))
+            # The system pages give every arcade board a section of its own
+            # (round 4, 6.4) and record it. The built pages are checked by
+            # the round 4 gate.
+            siterender.reset_console_pages()
+            for board in siterender.arcade_boards():
+                siterender.record_board_section(board)
             platforms = [
                 load_platform_config(name, "platforms")
                 for name in list_registered_platforms("platforms", include_archived=True)
@@ -148,6 +149,10 @@ class SystemLinks(unittest.TestCase):
             for sys_id in cls.ids:
                 sections: set[tuple[str, str]] = set()
                 for fe in entries.get(sys_id, []):
+                    if fe.get("category") == "bios_zip":
+                        for sha1 in _archives_held(fe, db):
+                            sections.update((m, c) for m, c, _ in places.get(sha1, set()))
+                        continue
                     local, status = resolve_local_file(fe, db)
                     if not _accepts(fe, local, status):
                         continue
@@ -159,11 +164,16 @@ class SystemLinks(unittest.TestCase):
         finally:
             os.chdir(previous)
 
+    @classmethod
+    def tearDownClass(cls) -> None:
+        siterender.reset_console_pages()
+
     def test_every_id_with_files_links(self) -> None:
         linked, total = siterender.link_coverage(self.ids)
         with_files = set(self.held)
         print(f"\n  {linked} of {total} system ids linked, "
-              f"{len(with_files - set(UNLINKED))} of {len(with_files)} with files")
+              f"{len(with_files - set(UNLINKED))} of {len(with_files)} with files, "
+              f"{len(self.ids - with_files)} with no file")
         plain = sorted(
             sys_id for sys_id in with_files - set(UNLINKED)
             if siterender.system_section(sys_id) is None
@@ -240,15 +250,19 @@ class SystemLinks(unittest.TestCase):
     def test_arcade_heading_by_exact_name(self) -> None:
         self.assertEqual(
             siterender.system_section("namco-system246"), ("arcade", "Namco System 246"))
-        # No heading reads NAOMI: its sets are cards of the general section.
-        self.assertIsNone(siterender.system_section("sega-naomi"))
-        # Never a folder of copies a core loads.
-        self.assertIsNone(siterender.system_section("igs-pgm"))
+        # A board leads to its own section, never a folder of copies a core
+        # loads (Arcade > FBNeo).
+        self.assertEqual(siterender.system_section("sega-naomi"), ("arcade", "Sega NAOMI"))
+        self.assertEqual(
+            siterender.system_section("igs-pgm"), ("arcade", "IGS PolyGame Master"))
 
     def test_folder_votes_stay_out(self) -> None:
+        """A copy filed with another machine names no section: the Arcadia
+        stays plain, NAOMI 2 goes to its board, never to the Dreamcast."""
+        self.assertIsNone(siterender.system_section("emerson-arcadia-2001"))
+        self.assertEqual(siterender.system_section("naomi2"), ("arcade", "Sega NAOMI 2"))
         for sys_id in ("emerson-arcadia-2001", "naomi2"):
             with self.subTest(sys_id=sys_id):
-                self.assertIsNone(siterender.system_section(sys_id))
                 self.assertNotIn(sys_id, siterender.SYSTEM_PARENTS)
 
     def test_emulator_rows_link_the_variant(self) -> None:

tests/test_site_r4_p0.py

@@ -0,0 +1,324 @@
+"""Arcade boards and the ids that hold files (round 4, package P0).
+
+The Arcade pages were cut by storage folder, so NAOMI, ZN-1, PGM and the
+other boards had no section and stayed plain on 987 emulator rows.
+siterender.arcade_board() names the board each set archive is the BIOS of,
+read from the profiles, and system_section() leads every id of that board
+to ``systems/arcade-<board_slug>/#<board_slug>``, the section the system
+pages write. The 28 ids that systems.json gave files and no page either
+link or carry their reason below.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+import sys  # noqa: E402
+
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import siterender  # noqa: E402
+from sitestarlight import toc_slug  # noqa: E402
+
+# The boards the audit found with no section (N1, T-F3), by an id each.
+NAMED_BOARDS = {
+    "sega-naomi": "Sega NAOMI",
+    "sega-naomi2": "Sega NAOMI 2",
+    "sammy-atomiswave": "Sammy Atomiswave",
+    "capcom-zn1": "Capcom ZN-1",
+    "capcom-zn2": "Capcom ZN-2",
+    "igs-pgm": "IGS PolyGame Master",
+    "nintendo-playch10": "Nintendo PlayChoice-10",
+    "nintendo-nss": "Nintendo Super System",
+    "konami-gv": "Konami GV",
+    "deco-cassette": "Data East DECO Cassette System",
+    "super-kaneko-nova": "Kaneko Super Nova System",
+    "hyper-neogeo64": "SNK Hyper Neo Geo 64",
+    "sega-megatech": "Sega Mega-Tech",
+    "sega-megaplay": "Sega Mega Play",
+}
+
+# The ids api/v1/systems.json listed with files and no url (D P2-7).
+# Each one links, or stays plain for the reason given.
+SYSTEMS_JSON_GAPS = (
+    "atom", "bbc-master-128", "bbc-master-et", "cave-story", "dos",
+    "emerson-arcadia-2001", "fpinball", "mz2000", "mz2500", "mz800", "naomi2",
+    "neogeo64", "pegasus", "pv1000", "scummvm", "sega-dreamcast-arcade",
+    "sinclair-zx-spectrum-next", "solarus", "spectravideo", "sunplus-spmp8000",
+    "tomb-raider-1", "tyrian", "vc4000", "vgmplay", "vm-labs-nuon", "vvvvvv",
+    "xrick", "zc210",
+)
+STAYS_PLAIN = {
+    "emerson-arcadia-2001": "ar_bios.zip is the Arcadia Systems board's, filed with the Saturn",
+    "fpinball": "scripten.exe, filed with Windows",
+    "pv1000": "only MAME's software list pv1000.xml",
+    "vc4000": "only MAME's software list vc4000.xml",
+    "sega-dreamcast-arcade": "NAOMI, NAOMI 2 and Atomiswave sets, three boards",
+    "spectravideo": "the SVI ROMs are filed with the MSX",
+    "vgmplay": "sound chip sets, no board",
+    "vvvvvv": "SDL's controller database",
+}
+
+
+def _reset() -> None:
+    """Back to the repository's sections and boards, none written."""
+    siterender._arcade_source[0] = None
+    siterender._boards_built[0] = False
+    siterender._sections_installed[0] = False
+    siterender.reset_console_pages()
+
+
+def _write_every_board() -> None:
+    """What the system pages do once they give each board a section."""
+    for board in siterender.arcade_boards():
+        siterender.record_board_section(board)
+
+
+@unittest.skipUnless((REPO_ROOT / "database.json").exists(), "database.json is not built")
+class Boards(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        _reset()
+        with open(REPO_ROOT / "database.json", encoding="utf-8") as handle:
+            cls.db = json.load(handle)
+        _write_every_board()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        _reset()
+
+    def _copies(self, name: str) -> list[str]:
+        return self.db["indexes"]["by_name"][name]
+
+    def test_named_boards_exist(self) -> None:
+        boards = siterender.arcade_boards()
+        names = set(boards.values())
+        for sys_id, name in NAMED_BOARDS.items():
+            with self.subTest(sys_id=sys_id):
+                self.assertEqual(siterender.system_name(sys_id), name)
+                self.assertIn(name, names)
+
+    def test_every_copy_names_its_board(self) -> None:
+        """Every copy of a board's set, wherever it is filed: naomi.zip
+        under Sega > Dreamcast is still NAOMI's."""
+        expect = {
+            "naomi.zip": "sega-naomi",
+            "awbios.zip": "sammy-atomiswave",
+            "cpzn1.zip": "capcom-zn1",
+            "pgm.zip": "igs-pgm",
+            "playch10.zip": "nintendo-playch10",
+            "hng64.zip": "hyper-neogeo64",
+            "taitofx1.zip": "taito-fx1",
+        }
+        for name, board in expect.items():
+            for sha1 in self._copies(name):
+                with self.subTest(name=name, path=self.db["files"][sha1]["path"]):
+                    self.assertEqual(siterender.arcade_board(sha1), board)
+
+    def test_no_board(self) -> None:
+        """neogeo.zip and cd32.zip are the BIOS of machines with a section of
+        their own. mie.zip and jvs13551.zip are shared device sets."""
+        for name in ("neogeo.zip", "cd32.zip", "mie.zip", "jvs13551.zip"):
+            for sha1 in self._copies(name):
+                with self.subTest(name=name, path=self.db["files"][sha1]["path"]):
+                    self.assertIsNone(siterender.arcade_board(sha1))
+
+    def test_slug_contract(self) -> None:
+        boards = siterender.arcade_boards()
+        slugs = {siterender.board_slug(board) for board in boards}
+        self.assertEqual(len(slugs), len(boards), "two boards share a slug")
+        for board, name in boards.items():
+            with self.subTest(board=board):
+                slug = siterender.board_slug(board)
+                self.assertTrue(slug)
+                self.assertEqual(slug, toc_slug(name))
+                self.assertEqual(
+                    siterender.console_href(*siterender.system_section(board)),
+                    f"systems/arcade-{slug}.md#{slug}")
+
+    def test_aliases_share_a_board(self) -> None:
+        """Every id of one board's name leads to one section."""
+        pairs = (
+            ("sega-atomiswave", "sammy-atomiswave"),
+            ("zn1-capcom", "capcom-zn1"),
+            ("naomi2", "sega-naomi2"),
+            ("neogeo64", "hyper-neogeo64"),
+            ("zn1-taito", "taito-fx1"),
+            ("zn1-tecmo", "tecmo-tps"),
+        )
+        for alias, board in pairs:
+            with self.subTest(alias=alias):
+                self.assertEqual(siterender.board_slug(alias), siterender.board_slug(board))
+                self.assertEqual(
+                    siterender.system_section(alias), siterender.system_section(board))
+
+    def test_machines_keep_their_section(self) -> None:
+        self.assertEqual(siterender.system_section("snk-neogeo-mvs"), ("snk", "Neo Geo"))
+        self.assertEqual(siterender.system_section("sega-hikaru"), ("sega", "Hikaru"))
+        self.assertEqual(siterender.system_section("arcade"), ("arcade", "Arcade"))
+
+    def test_systems_json_gaps(self) -> None:
+        plain = sorted(i for i in SYSTEMS_JSON_GAPS if siterender.system_section(i) is None)
+        self.assertEqual(plain, sorted(STAYS_PLAIN))
+
+    def test_parents_named_by_the_plan(self) -> None:
+        expect = {
+            "dos": ("dos", "DOSBox"),
+            "scummvm": ("other", "ScummVM"),
+            "tyrian": ("other", "opentyrian"),
+            "bbc-master-128": ("other", "beebem"),
+        }
+        for sys_id, section in expect.items():
+            with self.subTest(sys_id=sys_id):
+                self.assertEqual(siterender.system_section(sys_id), section)
+
+
+@unittest.skipUnless((REPO_ROOT / "database.json").exists(), "database.json is not built")
+class LinksWaitForTheSection(unittest.TestCase):
+    """A board links once the system pages wrote its section, never to a
+    page nobody wrote: the build's link check stops on those."""
+
+    def setUp(self) -> None:
+        _reset()
+        self.addCleanup(_reset)
+
+    def test_unwritten_board_stays_plain(self) -> None:
+        self.assertIsNone(siterender.system_section("igs-pgm"))
+        self.assertIsNone(siterender.system_section("naomi2"))
+        self.assertEqual(siterender.console_href("arcade", "IGS PolyGame Master"),
+                         "systems/arcade.md#igs-polygame-master")
+
+    def test_written_board_links_and_takes_its_cards(self) -> None:
+        siterender.record_board_section("igs-pgm")
+        self.assertEqual(siterender.system_section("igs-pgm"),
+                         ("arcade", "IGS PolyGame Master"))
+        self.assertIsNone(siterender.system_section("sega-naomi"))
+        sha1 = next(s for s, e in json.loads(
+            (REPO_ROOT / "database.json").read_text(encoding="utf-8"))["files"].items()
+            if e["path"] == "bios/Arcade/Arcade/pgm.zip")
+        self.assertEqual(siterender.card_page("bios/Arcade/Arcade/pgm.zip", sha1),
+                         "arcade-igs-polygame-master")
+        siterender.reset_console_pages()
+        self.assertIsNone(siterender.system_section("igs-pgm"))
+
+    def test_an_alias_records_its_board(self) -> None:
+        siterender.record_board_section("sega-atomiswave")
+        self.assertEqual(siterender.system_section("sammy-atomiswave"),
+                         ("arcade", "Sammy Atomiswave"))
+
+    def test_no_board_refused(self) -> None:
+        with self.assertRaises(KeyError):
+            siterender.record_board_section("sony-playstation")
+
+
+class BoardRules(unittest.TestCase):
+    """The rules on a fixture: the folder decides nothing, a tie is no
+    board, a machine's own set is no board, contents: rejects a homonym."""
+
+    def setUp(self) -> None:
+        tmp_root = REPO_ROOT / "tmp"
+        tmp_root.mkdir(exist_ok=True)
+        self._tmp = tempfile.TemporaryDirectory(dir=tmp_root)
+        self.root = Path(self._tmp.name)
+        siterender.set_console_sections({("Arcade", "MAME"), ("SNK", "Neo Geo")})
+
+    def tearDown(self) -> None:
+        self._tmp.cleanup()
+        _reset()
+
+    def _zip(self, name: str, crc_member: bytes) -> str:
+        path = self.root / name
+        with zipfile.ZipFile(path, "w") as archive:
+            archive.writestr("rom.bin", crc_member)
+        return str(path)
+
+    def _install(self, entries: dict[str, list[dict]], files: dict[str, str]) -> None:
+        """``files``: sha1 -> path. Names are the paths' base names."""
+        db = {
+            "files": {sha1: {"path": path, "name": Path(path).name} for sha1, path in files.items()},
+            "indexes": {"by_name": {}, "by_path_suffix": {}},
+        }
+        for sha1, path in files.items():
+            db["indexes"]["by_name"].setdefault(Path(path).name, []).append(sha1)
+        profiles = {key: {"systems": [], "files": rows} for key, rows in entries.items()}
+        siterender.set_arcade_boards(profiles, db)
+
+    def test_folder_decides_nothing(self) -> None:
+        self._install(
+            {"core": [{"name": "pgm.zip", "category": "bios_zip", "system": "igs-pgm"}]},
+            {"a" * 40: "bios/Arcade/MAME/pgm.zip", "b" * 40: "bios/Other/x/pgm.zip"},
+        )
+        self.assertEqual(siterender.arcade_board("a" * 40), "igs-pgm")
+        self.assertEqual(siterender.arcade_board("b" * 40), "igs-pgm")
+
+    def test_tie_is_no_board(self) -> None:
+        self._install(
+            {
+                "one": [{"name": "dev.zip", "category": "bios_zip", "system": "igs-pgm"}],
+                "two": [{"name": "dev.zip", "category": "bios_zip", "system": "konami-gv"}],
+            },
+            {"c" * 40: "bios/Arcade/MAME/dev.zip"},
+        )
+        self.assertIsNone(siterender.arcade_board("c" * 40))
+
+    def test_majority_wins(self) -> None:
+        self._install(
+            {
+                "one": [{"name": "dev.zip", "category": "bios_zip", "system": "igs-pgm"}],
+                "two": [{"name": "dev.zip", "category": "bios_zip", "system": "igs-pgm"}],
+                "three": [{"name": "dev.zip", "category": "bios_zip", "system": "konami-gv"}],
+            },
+            {"c" * 40: "bios/Arcade/MAME/dev.zip"},
+        )
+        self.assertEqual(siterender.arcade_board("c" * 40), "igs-pgm")
+
+    def test_machine_set_is_no_board(self) -> None:
+        self._install(
+            {
+                "one": [{"name": "neogeo.zip", "category": "bios_zip", "system": "snk-neogeo-mvs"}],
+                "two": [{"name": "neogeo.zip", "category": "bios_zip", "system": "igs-pgm"}],
+            },
+            {"d" * 40: "bios/Arcade/MAME/neogeo.zip"},
+        )
+        self.assertIsNone(siterender.arcade_board("d" * 40))
+
+    def test_contents_reject_a_homonym(self) -> None:
+        good = self._zip("good.zip", b"good")
+        other = self.root / "other"
+        other.mkdir()
+        bad = str(other / "good.zip")
+        with zipfile.ZipFile(bad, "w") as archive:
+            archive.writestr("rom.bin", b"something else")
+        crc = f"{zipfile.ZipFile(good).getinfo('rom.bin').CRC:08x}"
+        self._install(
+            {"core": [{"name": "good.zip", "category": "bios_zip", "system": "igs-pgm",
+                       "contents": [{"name": "rom.bin", "crc32": crc}]}]},
+            {"e" * 40: good, "f" * 40: bad},
+        )
+        # Both copies filed under Arcade, as by_path_suffix records it.
+        db = siterender._arcade_source[0][1]
+        db["indexes"]["by_path_suffix"] = {
+            "Arcade/MAME/good.zip": ["e" * 40], "Arcade/FBNeo/good.zip": ["f" * 40]}
+        self.assertEqual(siterender.arcade_board("e" * 40), "igs-pgm")
+        self.assertIsNone(siterender.arcade_board("f" * 40))
+
+    def test_no_copy_under_arcade_no_section(self) -> None:
+        """A board with no set under Arcade has no section there to lead to."""
+        self._install(
+            {"core": [{"name": "pgm.zip", "category": "bios_zip", "system": "igs-pgm"}]},
+            {"a" * 40: "bios/IGS/PGM/pgm.zip"},
+        )
+        self.assertIsNone(siterender.arcade_board("a" * 40))
+        with self.assertRaises(KeyError):
+            siterender.record_board_section("igs-pgm")
+
+
+if __name__ == "__main__":
+    unittest.main()