feat: read 0.78 era arcade parents from code

mame2000, mame2003 and mame2003-plus carry a DAT their code moved past. Their parents and BIOS sets now come from src/driver.c and src/drivers/ at the pin, named by the src tree sha. The DAT is still read to report gaps. FBNeo DATs are read one namespace each, console sets keyed by driver. A BIOS with no dumped ROM is no longer listed.

scripts/scraper/arcade_parent_importer.py

@@ -5,28 +5,36 @@ and the chain ends at a BIOS set (``neogeo``, ``naomi``, ``pgm``). The archive
 the installer has to provide for a game on the card is every BIOS set on that
 chain, plus, for the MAME family, the device sets with ROMs the machine
 instantiates (``device_ref``), which a split or merged romset reads from their
-own archive. The DAT that answers is the one of the core version that runs
-the game, read at the revision its profile pins:
+own archive. The answer is read at the revision the profile pins, from the
+DAT of the core version that runs the game where that DAT is the code's own
+output, and from the driver code where it is not:
 
     python -m scripts.scraper.arcade_parent_importer --all
     python -m scripts.scraper.arcade_parent_importer --profile mame2003_plus
     python -m scripts.scraper.arcade_parent_importer --all --offline
 
-A BIOS set is ``isbios="yes"`` (MAME 0.139 and later, FBNeo) or, in the
-MAME 0.78 lineage (mame2003, mame2003-plus), which has no such attribute, a
-set marked ``runnable="no"``. flycast keeps its NAOMI and Atomiswave table in
-code: ``Games[].bios`` names the BIOS, ``naomi`` when it is null
-(core/hw/naomi/naomi_cart.cpp:195-197), and ``BIOS[].filename`` the archive
-when it differs from the name (naomi_cart.cpp:75).
+A BIOS set is ``isbios="yes"`` (MAME 0.139 and later, FBNeo). mame2000,
+mame2003 and mame2003-plus carry a DAT in metadata/ that their code has
+moved past (mame2003-plus: 186 compiled sets absent, nbajamex under a BIOS
+set the code no longer declares), so they are read from src/driver.c and
+src/drivers/: the parent is the GAME macro's PARENT argument and a BIOS set
+is flagged NOT_A_DRIVER. Their DAT is still read, to report where it
+disagrees. FBNeo's DATs are read one at a time, a console set keyed by its
+driver (``msx_1942``): the Arcade, MSX and NES DATs each carry a ``1942``.
+flycast keeps its NAOMI and Atomiswave table in code: ``Games[].bios`` names
+the BIOS, ``naomi`` when it is null (core/hw/naomi/naomi_cart.cpp:195-197),
+and ``BIOS[].filename`` the archive when it differs from the name
+(naomi_cart.cpp:75).
 
 Output: games/arcade/<profile>.json. CHD games are listed, not resolved: a
-CHD is game content. A ``romof`` pointing outside the DAT is listed with its
-target, never guessed.
+CHD is game content. A ``romof`` pointing outside the sets read is listed
+with its target, never guessed.
 """
 
 from __future__ import annotations
 
 import argparse
+import ast
 import hashlib
 import io
 import json
@@ -34,6 +42,7 @@ import re
 import sys
 import urllib.parse
 import zipfile
+from collections import Counter
 from collections.abc import Callable, Iterator
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -58,8 +67,11 @@ DEFAULT_CACHE = Path(".cache") / "dats"
 # How each profile's DAT is found. "listxml": the -listxml release asset of
 # the MAME version core_version names. "fbneo": every DAT of libretro/FBNeo's
 # dats/ at source_commit. "repo": files of the profile's source repository at
-# source_commit (a ZIP member when the file is an archive). "flycast": the
-# BIOS and Games tables of naomi_roms.cpp at source_commit.
+# source_commit (a ZIP member when the file is an archive). "source": the
+# driver code itself at source_commit (src/driver.c and src/drivers/), for the
+# cores whose committed DAT contradicts their code; "paths" then names that
+# DAT, read only to report where it disagrees. "flycast": the BIOS and Games
+# tables of naomi_roms.cpp at source_commit.
 PROFILES: dict[str, dict] = {
     "mame": {"kind": "listxml", "marker": "isbios"},
     "mame0258": {"kind": "listxml", "marker": "isbios"},
@@ -72,9 +84,9 @@ PROFILES: dict[str, dict] = {
     },
     "mame2015": {"kind": "repo", "marker": "isbios", "paths": ["metadata/mame2014-xml.zip"]},
     "mame2010": {"kind": "repo", "marker": "isbios", "paths": ["metadata/mame2010.xml"]},
-    "mame2003_plus": {"kind": "repo", "marker": "runnable", "paths": ["metadata/mame2003-plus.xml"]},
-    "mame2003": {"kind": "repo", "marker": "runnable", "paths": ["metadata/mame2003.xml"]},
-    "mame2000": {"kind": "repo", "marker": "isbios", "paths": ["metadata/MAME 0.37b5 XML.dat"]},
+    "mame2003_plus": {"kind": "source", "marker": "runnable", "paths": ["metadata/mame2003-plus.xml"]},
+    "mame2003": {"kind": "source", "marker": "runnable", "paths": ["metadata/mame2003.xml"]},
+    "mame2000": {"kind": "source", "marker": "isbios", "paths": ["metadata/MAME 0.37b5 XML.dat"]},
     "flycast": {"kind": "flycast", "paths": ["core/hw/naomi/naomi_roms.cpp"]},
 }
 
@@ -107,6 +119,7 @@ class SetEntry:
     has_roms: bool = False
     disks: bool = False
     devices: list[str] = field(default_factory=list)
+    source: str | None = None
 
 
 _ELEMENT = re.compile(rb"<(machine|game)\b([^>]*)>")
@@ -157,6 +170,7 @@ def iter_sets(stream: io.BufferedIOBase) -> Iterator[SetEntry]:
                 has_roms=bool(_machine_roms(body)),
                 disks=any(d.get("status") != "nodump" for d in disks),
                 devices=[d.decode() for d in _DEVICE_REF.findall(body)],
+                source=attrs.get("sourcefile"),
             )
             position = end + len(closing)
             keep_from = max(position, len(buffer) - _TAG_TAIL)
@@ -171,8 +185,16 @@ def is_bios(entry: SetEntry, marker: str) -> bool:
     return marker == "runnable" and not entry.runnable and not entry.isdevice
 
 
-def build_index(sets: dict[str, SetEntry], marker: str) -> dict:
-    """BIOS sets per game, device sets per game, CHD games, broken chains."""
+def build_index(
+    sets: dict[str, SetEntry], marker: str, archives: dict[str, str] | None = None
+) -> dict:
+    """BIOS sets per game, device sets per game, CHD games, broken chains.
+
+    A BIOS set none of whose ROMs was dumped is passed over: its archive holds
+    nothing, and the loader only warns about a ROM without a dump. *archives*
+    names the archive of a set whose name is not its archive's (an FBNeo
+    console driver)."""
+    archive = (archives or {}).get
     bios_names = {n for n, e in sets.items() if is_bios(e, marker)}
     device_roms = {n for n, e in sets.items() if e.isdevice and e.has_roms}
     games = {
@@ -189,10 +211,10 @@ def build_index(sets: dict[str, SetEntry], marker: str) -> dict:
             seen.add(current)
             parent = sets.get(current)
             if parent is None:
-                missing[name] = current
+                missing[name] = archive(current, current)
                 break
-            if current in bios_names:
-                bios.setdefault(current, set()).add(name)
+            if current in bios_names and parent.has_roms:
+                bios.setdefault(archive(current, current), set()).add(name)
             current = parent.romof
         for device in set(entry.devices) & device_roms:
             devices.setdefault(device, set()).add(name)
@@ -215,6 +237,87 @@ def build_index(sets: dict[str, SetEntry], marker: str) -> dict:
     }
 
 
+# --- FBNeo: one namespace per DAT ---------------------------------------------
+
+# Generated at build time from every driver and committed: the set name and the
+# source file of each driver.
+FBNEO_DRIVERLIST = "src/dep/generated/driverlist.h"
+_FBNEO_DRIVER = re.compile(r'\{\s*"([^"]+)",\s*"([^"]+\.cpp)"\s*\}')
+
+
+def fbneo_drivers(listing: str) -> dict[str, set[str]]:
+    """Driver names per source file, from driverlist.h."""
+    names: dict[str, set[str]] = {}
+    for name, path in _FBNEO_DRIVER.findall(listing):
+        names.setdefault(path, set()).add(name)
+    if not names:
+        raise ValueError(f"{FBNEO_DRIVERLIST} lists no driver")
+    return names
+
+
+def _fbneo_candidates(entry: SetEntry, names: dict[str, set[str]]) -> list[str]:
+    """The drivers a DAT set can be. A console DAT writes the archive name,
+    which the driver's GetZipName() takes by cutting the system prefix:
+    ``1942`` in the MSX DAT is the driver ``msx_1942``."""
+    drivers = names.get(entry.source or "", set())
+    if entry.name in drivers:
+        return [entry.name]
+    return sorted(n for n in drivers if "_" in n and n.split("_", 1)[1] == entry.name)
+
+
+def _fbneo_driver_names(entries: list[SetEntry], names: dict[str, set[str]]) -> dict[str, str]:
+    """Driver name per set of one DAT. Where a source file holds two systems
+    (d_nes.cpp builds the NES and the FDS drivers), the prefix the rest of
+    the DAT carries decides."""
+    candidates = {e.name: _fbneo_candidates(e, names) for e in entries}
+    prefixes = Counter(
+        found[0].split("_", 1)[0]
+        for name, found in candidates.items()
+        if len(found) == 1 and found[0] != name
+    )
+    driver: dict[str, str] = {}
+    for entry in entries:
+        found = candidates[entry.name]
+        if len(found) > 1 and prefixes:
+            dominant = prefixes.most_common(1)[0][0]
+            found = [n for n in found if n.split("_", 1)[0] == dominant]
+        if len(found) != 1:
+            raise ValueError(f"{entry.source}: {entry.name} matches drivers {found}")
+        driver[entry.name] = found[0]
+    return driver
+
+
+def fbneo_sets(
+    openers: list[Callable[[], io.BufferedIOBase]], names: dict[str, set[str]]
+) -> tuple[dict[str, SetEntry], dict[str, str]]:
+    """Every set of FBNeo's DATs keyed by its driver, and the archive of each
+    set whose driver name is not its archive name.
+
+    Each DAT is its own namespace: the Arcade, MSX, NES and ZX Spectrum DATs
+    all carry a ``1942``, and a ``romof`` inside a DAT names a set of that DAT.
+    Merged by archive name, the MSX ``1942`` that needs msx.zip was the
+    arcade one, which needs nothing. The Arcade and Neo Geo DATs both carry
+    the Neo Geo drivers: a driver read twice keeps its first reading.
+    """
+    sets: dict[str, SetEntry] = {}
+    archives: dict[str, str] = {}
+    for opener in openers:
+        with opener() as stream:
+            entries = [e for e in iter_sets(stream) if e.name]
+        driver = _fbneo_driver_names(entries, names)
+        for entry in entries:
+            name = driver[entry.name]
+            if name in sets:
+                continue
+            if name != entry.name:
+                archives[name] = entry.name
+            if entry.romof:
+                entry.romof = driver.get(entry.romof, entry.romof)
+            entry.name = name
+            sets[name] = entry
+    return sets, archives
+
+
 # --- C table reader (flycast naomi_roms.cpp) ---------------------------------
 
 _TOKEN = re.compile(r'"(?:\\.|[^"\\])*"|[{},]|[^\s{},"]+')
@@ -334,6 +437,475 @@ def flycast_index(source: str) -> dict:
     }
 
 
+# --- MAME 0.37 and 0.78 driver source (mame2000, mame2003, mame2003-plus) ----
+
+# driver.h at each pin: NAME is argument 1 and PARENT argument 2 in every
+# form, and a BIOS set carries NOT_A_DRIVER in its flags.
+_GAME_MACRO = re.compile(r"\b(GAME|GAMEX|GAMEC|GAMECX|GAMEB|GAMEBX)\s*\(")
+_DRIVER_ENTRY = re.compile(r"\bDRIVER\s*\(\s*(\w+)\s*\)")
+_ROM_START = re.compile(r"\bROM_START\s*\(\s*(\w+)\s*\)(.*?)\bROM_END\b", re.S)
+_ROM_LOAD = re.compile(r"\bROMX?_LOAD\w*\s*\(")
+_DISK_IMAGE = re.compile(r"\bDISK_IMAGE\w*\s*\(")
+_ROM_START_HEAD = re.compile(r"\bROM_START\s*\(")
+_DIRECTIVE = re.compile(r"^[ \t]*#[ \t]*(\w+)[ \t]*(.*)$")
+_MACRO_HEAD = re.compile(r"(\w+)(\([^)]*\))?[ \t]*(.*)")
+_CONDITION_TOKEN = re.compile(
+    r"\s*(?:(0[xX][0-9a-fA-F]+|\d+)[uUlL]*|(defined)|([A-Za-z_]\w*)"
+    r"|(&&|\|\||==|!=|<=|>=|<<|>>|[!<>()+\-*/%~&|^]))"
+)
+_IDENTIFIER = re.compile(r"\b[A-Za-z_]\w*\b")
+_INCLUDE = re.compile(r'"([^"]+)"')
+
+
+class DriverSourceError(ValueError):
+    """The driver source does not have the shape the reader knows."""
+
+
+def _call_args(text: str, start: int) -> list[str]:
+    """Top-level arguments of the call whose opening parenthesis ends at
+    *start*, string literals kept whole."""
+    return _call_span(text, start)[0]
+
+
+def _call_span(text: str, start: int) -> tuple[list[str], int]:
+    """The arguments of a call and the offset just past its parenthesis."""
+    args: list[str] = []
+    current: list[str] = []
+    depth, i, n = 1, start, len(text)
+    while i < n:
+        c = text[i]
+        if c == '"':
+            j = i + 1
+            while j < n and text[j] != '"':
+                j += 2 if text[j] == "\\" else 1
+            current.append(text[i : j + 1])
+            i = j + 1
+            continue
+        if c == "(":
+            depth += 1
+        elif c == ")":
+            depth -= 1
+            if depth == 0:
+                args.append("".join(current).strip())
+                return args, i + 1
+        elif c == "," and depth == 1:
+            args.append("".join(current).strip())
+            current = []
+            i += 1
+            continue
+        current.append(c)
+        i += 1
+    raise DriverSourceError("unterminated macro call")
+
+
+def _condition_value(expr: str, defines: dict[str, str], depth: int = 0) -> int:
+    """An ``#if`` expression the way the C preprocessor reads it: a name not
+    defined is 0, a defined name is its value."""
+    python: list[str] = []
+    position = 0
+    expr = expr.strip()
+    while position < len(expr):
+        match = _CONDITION_TOKEN.match(expr, position)
+        if not match:
+            raise DriverSourceError(f"#if {expr}: cannot read")
+        number, defined, name, operator = match.groups()
+        position = match.end()
+        if number:
+            python.append(str(int(number, 0)))
+        elif defined:
+            target = re.match(r"\s*\(?\s*(\w+)\s*\)?", expr[position:])
+            if not target:
+                raise DriverSourceError(f"#if {expr}: defined without a name")
+            python.append("1" if target.group(1) in defines else "0")
+            position += target.end()
+        elif name:
+            value = defines.get(name, "0").strip() or "1"
+            if depth > 8:
+                raise DriverSourceError(f"#if {expr}: {name} does not resolve")
+            python.append(str(_condition_value(value, defines, depth + 1)))
+        else:
+            python.append(
+                {"&&": " and ", "||": " or ", "!": " not ", "/": "//"}.get(operator, operator)
+            )
+    try:
+        return int(_arithmetic(ast.parse("".join(python).strip(), mode="eval").body))
+    except (SyntaxError, ZeroDivisionError, TypeError) as exc:
+        raise DriverSourceError(f"#if {expr}: {exc}") from exc
+
+
+_BINARY = {
+    ast.Add: lambda a, b: a + b, ast.Sub: lambda a, b: a - b,
+    ast.Mult: lambda a, b: a * b, ast.FloorDiv: lambda a, b: a // b,
+    ast.Mod: lambda a, b: a % b, ast.LShift: lambda a, b: a << b,
+    ast.RShift: lambda a, b: a >> b, ast.BitAnd: lambda a, b: a & b,
+    ast.BitOr: lambda a, b: a | b, ast.BitXor: lambda a, b: a ^ b,
+}
+_COMPARE = {
+    ast.Eq: lambda a, b: a == b, ast.NotEq: lambda a, b: a != b,
+    ast.Lt: lambda a, b: a < b, ast.LtE: lambda a, b: a <= b,
+    ast.Gt: lambda a, b: a > b, ast.GtE: lambda a, b: a >= b,
+}
+
+
+def _arithmetic(node: ast.AST) -> int:
+    """Integer arithmetic of an ``#if``, nothing else evaluated."""
+    if isinstance(node, ast.Constant) and isinstance(node.value, int):
+        return node.value
+    if isinstance(node, ast.UnaryOp):
+        value = _arithmetic(node.operand)
+        if isinstance(node.op, ast.Not):
+            return int(not value)
+        if isinstance(node.op, ast.USub):
+            return -value
+        if isinstance(node.op, ast.UAdd):
+            return value
+        if isinstance(node.op, ast.Invert):
+            return ~value
+    if isinstance(node, ast.BoolOp):
+        values = [_arithmetic(v) for v in node.values]
+        return int(all(values) if isinstance(node.op, ast.And) else any(values))
+    if isinstance(node, ast.BinOp) and type(node.op) in _BINARY:
+        return _BINARY[type(node.op)](_arithmetic(node.left), _arithmetic(node.right))
+    if isinstance(node, ast.Compare):
+        left = _arithmetic(node.left)
+        for op, right_node in zip(node.ops, node.comparators):
+            right = _arithmetic(right_node)
+            if type(op) not in _COMPARE or not _COMPARE[type(op)](left, right):
+                return 0
+            left = right
+        return 1
+    raise TypeError(f"{type(node).__name__} in a condition")
+
+
+class _Preprocessor:
+    """Conditional compilation over src/, enough to tell which GAME and
+    ROM_START a build compiles. Headers are read for their definitions."""
+
+    def __init__(self, headers: dict[str, str]) -> None:
+        self.headers = headers
+        self._by_basename: dict[str, list[str]] = {}
+        for path in headers:
+            self._by_basename.setdefault(path.rsplit("/", 1)[-1], []).append(path)
+
+    def _resolve(self, name: str, including: str) -> str | None:
+        folder = including.rsplit("/", 1)[0]
+        for candidate in (f"{folder}/{name}", f"src/{name}", f"src/includes/{name}"):
+            if candidate in self.headers:
+                return candidate
+        found = self._by_basename.get(name.rsplit("/", 1)[-1], [])
+        return found[0] if len(found) == 1 else None
+
+    def run(
+        self, path: str, text: str, defines: dict[str, str],
+        local: dict[str, tuple[list[str] | None, str]] | None, seen: set[str],
+    ) -> str:
+        text = _strip_comments(text).replace("\\\r\n", " ").replace("\\\n", " ")
+        out: list[str] = []
+        # Each level: (enclosing level active, a branch already taken, this branch active).
+        stack: list[tuple[bool, bool, bool]] = []
+        active = True
+        for line in text.splitlines():
+            directive = _DIRECTIVE.match(line)
+            if not directive:
+                if active:
+                    out.append(line)
+                continue
+            word, rest = directive.groups()
+            if word in ("if", "ifdef", "ifndef"):
+                if not active:
+                    stack.append((False, True, False))
+                    continue
+                if word == "if":
+                    value = bool(_condition_value(rest, defines))
+                else:
+                    value = (rest.split()[0] in defines) == (word == "ifdef")
+                stack.append((True, value, value))
+                active = value
+            elif word in ("elif", "else"):
+                if not stack:
+                    raise DriverSourceError(f"{path}: #{word} without #if")
+                outer, taken, _ = stack[-1]
+                value = (
+                    outer and not taken
+                    and (word == "else" or bool(_condition_value(rest, defines)))
+                )
+                stack[-1] = (outer, taken or value, value)
+                active = value
+            elif word == "endif":
+                if not stack:
+                    raise DriverSourceError(f"{path}: #endif without #if")
+                stack.pop()
+                active = stack[-1][2] if stack else True
+            elif not active:
+                continue
+            elif word == "define":
+                head = _MACRO_HEAD.match(rest)
+                if head:
+                    defines[head.group(1)] = head.group(3)
+                    if local is not None:
+                        params = head.group(2)
+                        local[head.group(1)] = (
+                            [a.strip() for a in params[1:-1].split(",")] if params else None,
+                            head.group(3),
+                        )
+            elif word == "undef":
+                defines.pop(rest.strip(), None)
+            elif word == "include":
+                target = _INCLUDE.match(rest)
+                header = self._resolve(target.group(1), path) if target else None
+                if header and header not in seen:
+                    seen.add(header)
+                    self.run(header, self.headers[header], defines, None, seen)
+        if stack:
+            raise DriverSourceError(f"{path}: #if without #endif")
+        return "\n".join(out)
+
+
+def _dumped(args: list[str]) -> bool:
+    """A ROM_LOAD whose ROM was dumped: 0.78 writes ``NO_DUMP`` in the hash
+    argument, 0.37 a zero CRC."""
+    if any("NO_DUMP" in a for a in args):
+        return False
+    crc = args[3] if len(args) > 3 else ""
+    return not re.fullmatch(r"0[xX]0+|0", crc.strip())
+
+
+def _expand_declarations(text: str, local: dict[str, tuple[list[str] | None, str]]) -> str:
+    """Calls of a file's own function-like macros that write a ROM_START or a
+    GAME line, expanded (pgm.c builds every DoDonPachi II set through
+    ``DDP2_SET(name, program, asic)``)."""
+    writers = {
+        name: (params, body)
+        for name, (params, body) in local.items()
+        if params is not None and (_ROM_START_HEAD.search(body) or _GAME_MACRO.search(body))
+    }
+    if not writers:
+        return text
+    call = re.compile(r"\b(" + "|".join(map(re.escape, writers)) + r")\s*\(")
+    out: list[str] = []
+    position = 0
+    for match in call.finditer(text):
+        if match.start() < position:
+            continue
+        params, body = writers[match.group(1)]
+        args, end = _call_span(text, match.end())
+        if len(args) != len(params):
+            raise DriverSourceError(f"{match.group(1)} called with {len(args)} arguments")
+        values = dict(zip(params, args))
+        out.append(text[position : match.start()])
+        out.append(_IDENTIFIER.sub(lambda m: values.get(m.group(0), m.group(0)), body))
+        position = end
+    out.append(text[position:])
+    return "".join(out)
+
+
+def compiled_drivers(driver_c: str, preprocessor: _Preprocessor) -> list[str]:
+    """The DRIVER() entries driver.c compiles into the driver list."""
+    # driver.c is read twice by the build, the second time with
+    # DRIVER_RECURSIVE defined: that pass is the list.
+    text = preprocessor.run(
+        "src/driver.c", driver_c, {"DRIVER_RECURSIVE": "", "DRIVER": ""}, None, set()
+    )
+    return [n for n in _DRIVER_ENTRY.findall(text) if n != "NAME"]
+
+
+def driver_source_sets(
+    driver_c: str, drivers: dict[str, str], headers: dict[str, str]
+) -> dict[str, SetEntry]:
+    """Every compiled set and every set on its parent chain, from the source.
+
+    *drivers* maps each path under src/drivers/ to its text and *headers*
+    every header under src/. A set is declared by a GAME-family macro whose
+    ROMs are ``ROM_START(name)``, and a set flagged ``NOT_A_DRIVER`` is a BIOS
+    set, the flag the 0.37 and 0.78 info.c use to tell a resource from a game.
+    """
+    preprocessor = _Preprocessor(headers)
+    declared: dict[str, tuple[str | None, bool]] = {}
+    roms: dict[str, tuple[bool, bool]] = {}
+    for path, source in sorted(drivers.items()):
+        local: dict[str, tuple[list[str] | None, str]] = {}
+        text = _expand_declarations(preprocessor.run(path, source, {}, local, set()), local)
+        known: dict[str, bool] = {}
+
+        def holds_roms(body: str, seen: frozenset[str] = frozenset()) -> bool:
+            for match in _ROM_LOAD.finditer(body):
+                if _dumped(_call_args(body, match.end())):
+                    return True
+            # A ROM list shared through a macro of the same file (a BIOS
+            # block every game of the board pulls in).
+            for word in set(_IDENTIFIER.findall(body)) - seen:
+                if word in local:
+                    if word not in known:
+                        known[word] = holds_roms(local[word][1], seen | {word})
+                    if known[word]:
+                        return True
+            return False
+
+        for match in _ROM_START.finditer(text):
+            body = match.group(2)
+            roms.setdefault(match.group(1), (holds_roms(body), bool(_DISK_IMAGE.search(body))))
+        for match in _GAME_MACRO.finditer(text):
+            args = _call_args(text, match.end())
+            if len(args) < 9:
+                raise DriverSourceError(f"{path}: {match.group(1)} with {len(args)} arguments")
+            name, parent = args[1], args[2]
+            if name in declared:
+                raise DriverSourceError(f"{path}: {name} declared twice")
+            declared[name] = (
+                None if parent == "0" else parent,
+                any("NOT_A_DRIVER" in a for a in args[3:]),
+            )
+    compiled = compiled_drivers(driver_c, preprocessor)
+    undeclared = [n for n in compiled if n not in declared]
+    if undeclared:
+        raise DriverSourceError(f"driver.c lists undeclared sets: {', '.join(undeclared[:10])}")
+    sets: dict[str, SetEntry] = {}
+    pending = list(compiled)
+    while pending:
+        name = pending.pop()
+        if name in sets or name not in declared:
+            continue
+        parent, bios = declared[name]
+        has_roms, disks = roms.get(name, (False, False))
+        sets[name] = SetEntry(
+            name=name, romof=parent, isbios=bios, runnable=not bios,
+            has_roms=has_roms, disks=disks,
+        )
+        if parent:
+            pending.append(parent)
+    return sets
+
+
+def git_tree_sha(entries: dict[str, tuple[str, str]]) -> str:
+    """The sha git gives a directory, from ``{path: (mode, blob sha)}`` of
+    every file under it, paths relative to it."""
+    children: dict[str, dict[str, tuple[str, str]]] = {}
+    rows: list[tuple[str, str, str]] = []
+    for path, (mode, sha) in entries.items():
+        head, sep, tail = path.partition("/")
+        if sep:
+            children.setdefault(head, {})[tail] = (mode, sha)
+        else:
+            rows.append((head, mode, sha))
+    rows += [(name, "40000", git_tree_sha(sub)) for name, sub in children.items()]
+    # git orders a tree by name, a directory compared as if it ended in "/".
+    rows.sort(key=lambda row: row[0].encode() + (b"/" if row[1] == "40000" else b""))
+    body = b"".join(
+        mode.encode() + b" " + name.encode() + b"\0" + bytes.fromhex(sha)
+        for name, mode, sha in rows
+    )
+    return hashlib.sha1(f"tree {len(body)}\0".encode() + body).hexdigest()
+
+
+@dataclass
+class DriverSource:
+    """What the reader needs from src/ at one commit, and the tree sha of
+    src/ computed from every file of it."""
+
+    driver_c: str
+    drivers: dict[str, str]
+    headers: dict[str, str]
+    tree: str
+
+
+def read_driver_source(tarball: Path) -> DriverSource:
+    """src/ of a GitHub tarball: the texts the reader needs, and the git
+    tree sha of the whole directory."""
+    import tarfile
+
+    entries: dict[str, tuple[str, str]] = {}
+    texts: dict[str, str] = {}
+    with tarfile.open(tarball, "r:gz") as archive:
+        for member in archive:
+            parts = member.name.split("/", 1)
+            if len(parts) < 2 or not parts[1].startswith("src/"):
+                continue
+            path = parts[1]
+            if member.issym():
+                entries[path[4:]] = ("120000", git_blob_sha(member.linkname.encode()))
+            elif member.isfile():
+                handle = archive.extractfile(member)
+                data = handle.read() if handle else b""
+                mode = "100755" if member.mode & 0o111 else "100644"
+                entries[path[4:]] = (mode, git_blob_sha(data))
+                if path == "src/driver.c" or path.endswith(".h") or (
+                    path.startswith("src/drivers/") and path.endswith(".c")
+                ):
+                    texts[path] = data.decode("latin-1")
+    if "src/driver.c" not in texts:
+        raise FileNotFoundError(f"{tarball} has no src/driver.c")
+    return DriverSource(
+        driver_c=texts["src/driver.c"],
+        drivers={p: t for p, t in texts.items() if p.startswith("src/drivers/") and p.endswith(".c")},
+        headers={p: t for p, t in texts.items() if p.endswith(".h")},
+        tree=git_tree_sha(entries),
+    )
+
+
+def fetch_driver_source(
+    slug: str, commit: str, cache_dir: Path, offline: bool = False
+) -> DriverSource:
+    """src/ at a commit, its tree sha checked against the repository's own
+    when online."""
+    tarball = cache_dir / "src" / slug / f"{commit}.tar.gz"
+    if offline:
+        if not tarball.is_file():
+            raise FileNotFoundError(f"{tarball} is not cached")
+    else:
+        _download(f"https://codeload.github.com/{slug}/tar.gz/{commit}", tarball)
+    source = read_driver_source(tarball)
+    if not offline:
+        listing = _api_json(f"https://api.github.com/repos/{slug}/git/trees/{commit}")
+        listed = next(
+            (
+                item.get("sha")
+                for item in (listing.get("tree", []) if isinstance(listing, dict) else [])
+                if item.get("path") == "src"
+            ),
+            None,
+        )
+        if listed != source.tree:
+            tarball.unlink()
+            raise BlobMismatchError(f"{slug}@{commit[:8]}:src is tree {listed}, not {source.tree}")
+    return source
+
+
+def dat_gaps(code: dict[str, SetEntry], dat: dict[str, SetEntry], marker: str) -> dict:
+    """Where the DAT a repository carries contradicts its own code."""
+    both = code.keys() & dat.keys()
+    return {
+        "code_only": sorted(code.keys() - dat.keys()),
+        "dat_only": sorted(dat.keys() - code.keys()),
+        "romof": sorted(n for n in both if code[n].romof != dat[n].romof),
+        "bios": sorted(n for n in both if is_bios(code[n], marker) != is_bios(dat[n], marker)),
+    }
+
+
+def _dat_cross_check(
+    spec: dict, slug: str, commit: str, cache_dir: Path, offline: bool,
+    code: dict[str, SetEntry],
+) -> dict | None:
+    """dat_gaps against the DAT the repository carries, None when the DAT
+    is not cached and the run is offline."""
+    paths: list[Path] = []
+    for path in spec.get("paths", []):
+        try:
+            local, _ = (
+                _cached_repo_file(slug, commit, path, cache_dir)
+                if offline
+                else fetch_repo_file(slug, commit, path, cache_dir)
+            )
+        except FileNotFoundError:
+            if offline:
+                return None
+            raise
+        paths.append(local)
+    if not paths:
+        return None
+    dat = read_sets([opener for p in paths for opener in _xml_streams(p)])
+    return dat_gaps(code, dat, spec["marker"])
+
+
 # --- Fetching ----------------------------------------------------------------
 
 
@@ -419,9 +991,12 @@ def _sha256(path: Path) -> str:
 
 
 def import_profile(
-    profile: str, data: dict, cache_dir: Path, offline: bool = False
+    profile: str, data: dict, cache_dir: Path, offline: bool = False,
+    report: dict | None = None,
 ) -> dict:
-    """The index document for one profile, from the DAT its pin names."""
+    """The index document for one profile, from the DAT or the code its pin
+    names. *report* receives, for a profile read from its code, where the DAT
+    the repository carries disagrees with that code."""
     spec = PROFILES[profile]
     commit = str(data.get("source_commit") or "")
     source_url = str(data.get("source") or "")
@@ -448,15 +1023,33 @@ def import_profile(
                 raise FileNotFoundError(f"{folder} is not cached")
         else:
             folder, blobs = fetch_pack("fbneo", commit, cache_dir)
+        slug = _repo_slug(source_url)
+        listing, listing_sha = (
+            _cached_repo_file(slug, commit, FBNEO_DRIVERLIST, cache_dir)
+            if offline
+            else fetch_repo_file(slug, commit, FBNEO_DRIVERLIST, cache_dir)
+        )
         source = {
             "repo": source_url,
             "commit": commit,
-            "files": {f"dats/{name}": sha for name, sha in sorted(blobs.items())},
+            "files": {
+                **{f"dats/{name}": sha for name, sha in sorted(blobs.items())},
+                FBNEO_DRIVERLIST: listing_sha,
+            },
         }
         openers = [
             (lambda p=folder / name: p.open("rb")) for name in sorted(blobs)
         ]
-        index = build_index(read_sets(openers), spec["marker"])
+        sets, archives = fbneo_sets(openers, fbneo_drivers(listing.read_text(encoding="utf-8")))
+        index = build_index(sets, spec["marker"], archives)
+    elif kind == "source":
+        slug = _repo_slug(source_url)
+        tree = fetch_driver_source(slug, commit, cache_dir, offline)
+        sets = driver_source_sets(tree.driver_c, tree.drivers, tree.headers)
+        source = {"repo": source_url, "commit": commit, "files": {"src": tree.tree}}
+        index = build_index(sets, spec["marker"])
+        if report is not None:
+            report["dat_gaps"] = _dat_cross_check(spec, slug, commit, cache_dir, offline, sets)
     else:
         slug = _repo_slug(source_url)
         files: dict[str, str] = {}
@@ -546,7 +1139,10 @@ def main() -> int:
             failed = True
             continue
         try:
-            doc = import_profile(profile, profiles[profile], args.cache_dir, args.offline)
+            report: dict = {}
+            doc = import_profile(
+                profile, profiles[profile], args.cache_dir, args.offline, report
+            )
         except (OSError, ValueError, KeyError) as exc:
             print(f"FAIL {profile}: {exc}", file=sys.stderr)
             failed = True
@@ -560,8 +1156,22 @@ def main() -> int:
             f"{profile}: {counts['games']} games, {counts['games_with_bios']} with a BIOS set, "
             f"{counts['bios_roots_used']} BIOS sets used, "
             f"{counts['games_with_devices']} with device sets, {counts['chd_games']} CHD, "
-            f"{counts['missing_parents']} parents absent from the DAT"
+            f"{counts['missing_parents']} parents absent"
         )
+        if PROFILES[profile]["kind"] == "source":
+            gaps = report.get("dat_gaps")
+            if gaps is None:
+                print("  read from the code, the committed DAT is not cached for the cross-check")
+            else:
+                print(
+                    f"  read from the code, the committed DAT disagrees: "
+                    f"{len(gaps['code_only'])} sets only in the code, "
+                    f"{len(gaps['dat_only'])} only in the DAT, "
+                    f"{len(gaps['romof'])} parents and {len(gaps['bios'])} BIOS flags differ"
+                )
+                for key in ("dat_only", "romof", "bios"):
+                    if gaps[key]:
+                        print(f"    {key}: {', '.join(gaps[key][:20])}")
         if absent:
             print(f"  BIOS sets absent from the collection: {', '.join(absent)}")
         if undeclared: