fix: list only what a knulli board builds

KNULLI's es_systems.yml is Batocera's model: it lists every emulator
its configgen knows, and its Config.in selects many of them only on
x86_64 boards, which KNULLI has none of. The platform list now holds
what at least one board selects, read by the targets scraper, plus a
program the launcher runs from the BIOS folder (lexaloffle). 59 cores
and 44 standalone emulators leave the list, with their launcher records.

The targets scraper follows Config.in and every file it sources, so a
select a package declares itself counts (Batocera's mesa3d-rpi4 gives
the Pi 4 OpenGL). A sourced file absent at the pin is named and read as
empty: KNULLI 20260511 sources package/utils/vtree/Config.in, added in
June.

scripts/scraper/batocera_scraper.py

@@ -14,7 +14,7 @@ import re
 import sys
 import urllib.error
 import urllib.request
-from collections.abc import Callable, Iterable
+from collections.abc import Callable
 from pathlib import Path
 
 
@@ -30,6 +30,8 @@ from .base_scraper import (
     requirement_entry,
 )
 from .batocera_launchers import Configgen, configs_root, extract_launchers
+from .targets.batocera_targets_scraper import BuildSelection
+from .targets.batocera_targets_scraper import Scraper as BoardTargets
 
 PLATFORM_NAME = "batocera"
 
@@ -489,76 +491,51 @@ def configgen_cores(
     for GameCube). `dropped` names what es_systems.yml files under an
     emulator without being one.
 
-    A default es_systems.yml does not offer names an emulator the build may
-    lack: KNULLI's defaults pick suyu for the Switch, and no package of its
-    tree, its batocera submodule or its buildroot builds suyu. `built` says
-    whether the build has a standalone emulator (packages_and_programs); a
-    default it answers no for is not run, nor its core.
+    `built` says whether some board builds a name (boards_build). What it
+    answers no for is not run: KNULLI's es_systems lists citron and shadps4,
+    which no KNULLI board selects, and its defaults pick suyu, which no
+    package builds. A default's core follows a standalone emulator the
+    build runs (vox_official is lexaloffle's Voxatron build).
     """
     data = yaml_load_upstream(defaults_text)
     cores, standalone = es_systems_emulators(es_systems_text)
-    offered = set(cores)
     libretro = es_systems_libretro(es_systems_text)
     if not cores:
         raise RuntimeError("es_systems.yml names no emulator")
+    if built is not None:
+        cores = {name for name in cores if built(name)}
+        standalone &= cores
     for system, cfg in data.items():
         if system == "default" or not isinstance(cfg, dict):
             continue
         emulator = cfg.get("emulator", "")
         core = cfg.get("core", "")
-        if (built is not None and emulator and emulator != "libretro"
-                and emulator not in offered and not built(emulator)):
-            continue
-        if core:
-            cores.add(core)
         if emulator and emulator != "libretro":
+            if built is not None and not built(emulator):
+                continue
             standalone.add(emulator)
             if core and core != emulator:
                 standalone.add(core)
+        elif built is not None and core and not built(core):
+            continue
+        if core:
+            cores.add(core)
     dual = (standalone & libretro) - dropped
     return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual)
 
 
-BATOCERA_REPO = "batocera-linux/batocera.linux"
+def boards_build(selection: BuildSelection, launchers: list[dict]) -> Callable[[str], bool]:
+    """Whether some board of a build runs an emulator.
 
-
-def package_names(paths: Iterable[str]) -> set[str]:
-    """The buildroot packages a tree defines: each folder holding a Config.in."""
-    return {
-        path.rsplit("/", 2)[-2]
-        for path in paths
-        if path.endswith("/Config.in") and path.count("/") >= 1
-    }
-
-
-def repo_tree(repo: str, ref: str) -> list[str]:
-    """Every path of a GitHub repository at a revision."""
-    url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
-    req = urllib.request.Request(url, headers=github_headers())
-    with urllib.request.urlopen(req, timeout=60) as resp:
-        found = json.loads(resp.read().decode("utf-8"))
-    if found.get("truncated"):
-        raise RuntimeError(f"{url}: tree truncated")
-    return [entry["path"] for entry in found.get("tree", [])]
-
-
-def submodule_revision(repo: str, path: str, ref: str) -> str:
-    """The commit a repository's submodule at `path` pins at a revision."""
-    url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}"
-    req = urllib.request.Request(url, headers=github_headers())
-    with urllib.request.urlopen(req, timeout=30) as resp:
-        found = json.loads(resp.read().decode("utf-8"))
-    if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"):
-        raise RuntimeError(f"{url}: {path} is not a submodule")
-    return str(found["sha"])
-
-
-def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]:
-    """Whether a build has a standalone emulator.
-
-    It does when a package of that name exists, or when its launcher runs a
-    program the user puts in the BIOS folder (Batocera's lexaloffle runs
-    /userdata/bios/pico-8/pico8), which no package builds.
+    es_systems.yml ties each name to the packages that build it
+    (requireAnyOf), and a name it lists is built when a board selects one of
+    them: KNULLI lists citron and shadps4, and its Config.in selects them
+    only on x86_64 boards, which it has none of. A name only
+    configgen-defaults.yml gives is built when a board selects the package
+    of that name: the symbol its Config.in defines, else the one Buildroot
+    derives from the folder (BR2_PACKAGE_<NAME>, package/pkg-generic.mk). A
+    program the launcher runs from the BIOS folder (lexaloffle,
+    /userdata/bios/pico-8/pico8) needs no package.
     """
     supplied = {
         name.strip()
@@ -567,7 +544,27 @@ def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable
         and str(record.get("search_dir") or "").startswith("/userdata/bios")
         for name in str(record.get("emulator", "")).split(",")
     }
-    return lambda emulator: emulator in packages or emulator in supplied
+
+    def built(name: str) -> bool:
+        if name in selection.emulators or name in supplied:
+            return True
+        if name in selection.listed:
+            return False
+        symbol = selection.packages.get(name) or (
+            "BR2_PACKAGE_" + name.upper().replace("-", "_"))
+        return symbol in selection.symbols
+
+    return built
+
+
+def records_run(launchers: list[dict], built: Callable[[str], bool]) -> list[dict]:
+    """The launcher records of emulators the build runs.
+
+    configgen keeps the generators of emulators a build does not select
+    (KNULLI's citron, pcsx2, shadps4): what they read is never read on the
+    device, so the records do not describe the platform.
+    """
+    return [r for r in launchers if any(built(n) for n in str(r["emulator"]).split(", "))]
 
 
 class Scraper(BaseScraper):
@@ -578,20 +575,13 @@ class Scraper(BaseScraper):
         super().__init__(url=SOURCE_URL.format(tag=self.tag))
 
     def _fetch_cores(
-        self, launchers: list[dict] | None = None
+        self, built: Callable[[str], bool] | None = None
     ) -> tuple[list[str], list[str], list[str]]:
         """Core names, standalone cores and dual-mode cores at the pinned tag.
 
-        A default es_systems.yml does not offer counts when a package of
-        Batocera's tree or its buildroot builds it, or when its launcher
-        runs a program from the BIOS folder.
+        With `built` (boards_build), a name counts when some board builds
+        it or its launcher runs a program from the BIOS folder.
         """
-        built = None
-        if launchers is not None:
-            paths = repo_tree(BATOCERA_REPO, self.tag)
-            buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag)
-            paths += repo_tree("batocera-linux/buildroot", buildroot)
-            built = packages_and_programs(package_names(paths), launchers)
         return configgen_cores(
             self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)),
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)),
@@ -633,7 +623,9 @@ class Scraper(BaseScraper):
         launchers = extract_launchers(
             lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
         )
-        cores, standalone, dual = self._fetch_cores(launchers)
+        built = boards_build(BoardTargets().build_selection(self.tag), launchers)
+        launchers = records_run(launchers, built)
+        cores, standalone, dual = self._fetch_cores(built)
         libretro_systems = es_systems_libretro_systems(
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
         )

scripts/scraper/knulli_scraper.py

@@ -37,18 +37,17 @@ from .batocera_scraper import (
 )
 from .batocera_scraper import (
     _not_cores,
+    boards_build,
     configgen_cores,
     es_systems_by_system,
     es_systems_libretro_systems,
     extract_systems_dict,
-    package_names,
-    packages_and_programs,
     platform_systems,
-    repo_tree,
+    records_run,
     requirements_from_systems,
-    submodule_revision,
 )
 from .es_emulators import attach_system_cores
+from .targets.knulli_targets_scraper import Scraper as BoardTargets
 from .batocera_launchers import configs_root
 from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers
 from .knulli_launchers import configgen as knulli_configgen
@@ -134,15 +133,11 @@ class Scraper(BaseScraper):
             lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
             lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
         )
-        # The packages KNULLI builds come from its own tree, its batocera
-        # submodule and its buildroot submodule: a default no package of
-        # them builds (suyu) is not run.
-        paths = repo_tree(REPO, self.tag)
-        paths += repo_tree("batocera-linux/batocera.linux", revision)
-        paths += repo_tree(
-            "knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag)
-        )
-        built = packages_and_programs(package_names(paths), launchers)
+        # What runs is what at least one board builds, read by the targets
+        # scraper from the Kconfig of KNULLI's tree and its batocera and
+        # buildroot submodules (boards_build).
+        built = boards_build(BoardTargets().build_selection(self.tag), launchers)
+        launchers = records_run(launchers, built)
         cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built)
         # Each system's own list (psx, es_systems.yml:640-653: libretro
         # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and

scripts/scraper/targets/batocera_targets_scraper.py

@@ -3,6 +3,8 @@
 Sources (batocera-linux/batocera.linux):
   - configs/batocera-*.board  -- board definitions, each sets BR2_PACKAGE_BATOCERA_TARGET_*
   - package/batocera/core/batocera-system/Config.in -- select PACKAGE if CONDITION lines
+  - Config.in and every Kconfig file it sources -- the selects a package
+    declares itself (advanced_drastic selects drastic)
   - package/batocera/emulationstation/batocera-es-system/es_systems.yml
     -- emulator requireAnyOf flag mapping
 Sources (batocera-linux/buildroot, at the submodule commit the tag pins):
@@ -19,6 +21,8 @@ import re
 import sys
 import urllib.error
 import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
 from datetime import datetime, timezone
 
 import yaml
@@ -79,6 +83,16 @@ def _fetch(url: str, headers: dict | None = None) -> str:
         raise RuntimeError(f"cannot fetch {url}: {e}") from e
 
 
+def _fetch_present(url: str) -> str | None:
+    """A file's text, or None when the server answers that it does not exist."""
+    try:
+        return _fetch(url)
+    except RuntimeError as e:
+        if isinstance(e.__cause__, urllib.error.HTTPError) and e.__cause__.code == 404:
+            return None
+        raise
+
+
 def _fetch_json(url: str) -> list | dict:
     text = _fetch(url, headers=github_headers())
     try:
@@ -210,6 +224,7 @@ BUILDROOT_KCONFIG = (
 )
 
 _CONFIG_LINE_RE = re.compile(r"^\s*(?:menu)?config\s+(BR2_\w+)")
+_SOURCE_RE = re.compile(r'^\s*source\s+"\$BR2_EXTERNAL_\w+_PATH/([^"]+)"')
 _OWNED_SELECT_RE = re.compile(r"^\s+select\s+(BR2_\w+)(?:\s+if\s+([^#]+))?")
 _BOARD_FLAG_RE = re.compile(r"^(BR2_\w+)=y\s*$", re.MULTILINE)
 _BOARD_INCLUDE_RE = re.compile(r"^(-?)include\s+(\S+)", re.MULTILINE)
@@ -342,6 +357,43 @@ def _arch_from_flag(flag: str, board_flags: set[str] | frozenset[str] = frozense
     return "aarch64"
 
 
+@dataclass(frozen=True)
+class BuildSelection:
+    """What the boards of a build turn on, read once from its Kconfig.
+
+    `targets` is each board's entry of the targets file. `emulators` holds
+    every es_systems name some board builds and `listed` every name
+    es_systems ties to a package, `symbols` every symbol some board ends up
+    with, and `packages` the config symbol each package folder of the tree
+    defines.
+    """
+
+    targets: dict[str, dict]
+    emulators: frozenset[str]
+    listed: frozenset[str]
+    symbols: frozenset[str]
+    packages: dict[str, str]
+
+
+def _sourced(text: str) -> list[str]:
+    """The files a Kconfig file sources, relative to the external tree."""
+    return [m.group(1) for line in text.splitlines() if (m := _SOURCE_RE.match(line))]
+
+
+def _package_symbols(files: dict[str, str]) -> dict[str, str]:
+    """package folder -> the first config symbol its Config.in defines."""
+    found: dict[str, str] = {}
+    for path, text in files.items():
+        parts = path.split("/")
+        if len(parts) < 3 or parts[-1] != "Config.in" or "package" not in parts:
+            continue
+        config = next((m.group(1) for line in text.splitlines()
+                       if (m := _CONFIG_LINE_RE.match(line))), None)
+        if config:
+            found.setdefault(parts[-2], config)
+    return found
+
+
 class Scraper(BaseTargetScraper):
     """Cross-references Batocera boards, Config.in, and es_systems to build target lists.
 
@@ -357,6 +409,9 @@ class Scraper(BaseTargetScraper):
     ES_SYSTEMS_PATH = "package/batocera/emulationstation/batocera-es-system/es_systems.yml"
     SYSTEM_SYMBOL = "BR2_PACKAGE_BATOCERA_SYSTEM"
     BUILDROOT_REPO = "batocera-linux/buildroot"
+    # Path prefix -> repository of a submodule the root Config.in sources
+    # from: KNULLI reads Batocera's packages as batocera/package/...
+    SUBMODULES: dict[str, str] = {}
 
     def __init__(self, url: str = "") -> None:
         super().__init__(url=url or f"https://github.com/{self.REPO}")
@@ -411,6 +466,45 @@ class Scraper(BaseTargetScraper):
             pending += [(inc, bool(dash)) for dash, inc in _BOARD_INCLUDE_RE.findall(text)]
         return target, flags
 
+    def _kconfig_files(self) -> dict[str, str]:
+        """The root Config.in and every file it sources, recursively.
+
+        The build reads these and nothing else: a package's own Config.in
+        can select another package (advanced_drastic selects drastic), and
+        such a select sits in no system Config.in. A sourced file absent at
+        the pin defines nothing and is named: KNULLI 20260511 sources
+        package/utils/vtree/Config.in, a package its tree gained later.
+        """
+        bases: list[tuple[str, str]] = []
+        for prefix, repo in self.SUBMODULES.items():
+            pin = _fetch_json(self._api(prefix.rstrip("/")))
+            sha = pin.get("sha") if isinstance(pin, dict) else None
+            if not sha:
+                raise RuntimeError(f"no {prefix.rstrip('/')} submodule commit at {self.tag}")
+            bases.append((prefix, f"https://raw.githubusercontent.com/{repo}/{sha}/"))
+
+        def url(path: str) -> str:
+            for prefix, base in bases:
+                if path.startswith(prefix):
+                    return base + path.removeprefix(prefix)
+            return self._raw(path)
+
+        files: dict[str, str] = {}
+        absent: list[str] = []
+        pending = ["Config.in"]
+        with ThreadPoolExecutor(max_workers=8) as pool:
+            while pending:
+                found = dict(zip(pending, pool.map(lambda path: _fetch_present(url(path)), pending)))
+                absent += [path for path, text in found.items() if text is None]
+                files.update((path, text or "") for path, text in found.items())
+                pending = sorted({src for text in found.values() if text
+                                  for src in _sourced(text)} - set(files))
+        if "Config.in" in absent or len(files) - len(absent) < 2:
+            raise RuntimeError(f"Config.in at {self.tag} sources no readable Kconfig file")
+        for path in absent:
+            print(f"  sourced but absent at {self.tag}: {path}", file=sys.stderr)
+        return files
+
     def _buildroot_rules(self) -> list[tuple[str, str, str]]:
         """Buildroot's selects for the symbols the tree tests, at its pin."""
         pin = _fetch_json(self._api("buildroot"))
@@ -423,26 +517,32 @@ class Scraper(BaseTargetScraper):
             rules += _parse_owned_selects(_fetch(f"{raw}/{path}"))
         return rules
 
-    def fetch_targets(self) -> dict:
-        """Build per-board emulator availability map."""
-        self.tag = self.pinned_revision(self.PLATFORM)
+    def build_selection(self, tag: str | None = None) -> BuildSelection:
+        """Each board's selection, from the Kconfig the build reads.
+
+        `tag` is the revision to read, by default the one the platform's
+        YAML was scraped at: a platform scraper moving to a new release
+        passes its own.
+        """
+        self.tag = tag or self.pinned_revision(self.PLATFORM)
         print("  fetching board list...", file=sys.stderr)
         boards = self._list_boards()
 
-        print("  fetching Config.in...", file=sys.stderr)
-        config_in_url = self._raw(self.CONFIG_IN_PATH)
-        config_in_text = _fetch(config_in_url)
+        print("  fetching Kconfig files...", file=sys.stderr)
+        files = self._kconfig_files()
+        config_in_text = files.get(self.CONFIG_IN_PATH)
         if not config_in_text:
-            raise RuntimeError(f"empty Config.in from {config_in_url}")
+            raise RuntimeError(f"Config.in at {self.tag} does not source {self.CONFIG_IN_PATH}")
 
         meta_rules = _parse_meta_flags(config_in_text)
-        rules = _parse_owned_selects(config_in_text)
+        rules = [rule for text in files.values() for rule in _parse_owned_selects(text)]
         print(
-            f"  parsed {len(meta_rules)} meta-flag rules, {len(rules)} select lines",
+            f"  parsed {len(meta_rules)} meta-flag rules, {len(rules)} select lines "
+            f"in {len(files)} Kconfig files",
             file=sys.stderr,
         )
-        if not rules:
-            raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {config_in_url}")
+        if not _parse_owned_selects(config_in_text):
+            raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {self.CONFIG_IN_PATH}")
         rules += self._buildroot_rules()
 
         print("  fetching es_systems.yml...", file=sys.stderr)
@@ -459,9 +559,10 @@ class Scraper(BaseTargetScraper):
             raise RuntimeError(f"no package->emulator mappings parsed from {es_systems_url}")
         images_by_flag = _parse_images(config_in_text)
         if not images_by_flag:
-            raise RuntimeError(f"no target image list parsed from {config_in_url}")
+            raise RuntimeError(f"no target image list parsed from {self.CONFIG_IN_PATH}")
 
         targets: dict[str, dict] = {}
+        symbols: set[str] = set()
         for board_name in sorted(boards):
             target_key = board_name.removeprefix(self.BOARD_PREFIX).removesuffix(".board")
             print(f"  processing {target_key}...", file=sys.stderr)
@@ -476,6 +577,7 @@ class Scraper(BaseTargetScraper):
             if self.SYSTEM_SYMBOL not in selected_packages:
                 print(f"    {board_name} builds no system image", file=sys.stderr)
                 continue
+            symbols |= selected_packages
 
             # Map selected packages to emulator names via es_systems.yml
             emulators: set[str] = set()
@@ -496,11 +598,22 @@ class Scraper(BaseTargetScraper):
                 file=sys.stderr,
             )
 
+        return BuildSelection(
+            targets=targets,
+            emulators=frozenset(c for entry in targets.values() for c in entry["cores"]),
+            listed=frozenset(n for names in package_to_emulators.values() for n in names),
+            symbols=frozenset(symbols),
+            packages=_package_symbols(files),
+        )
+
+    def fetch_targets(self) -> dict:
+        """Build per-board emulator availability map."""
+        selection = self.build_selection()
         return {
             "platform": self.PLATFORM,
             "source": f"{self.url}/tree/{self.tag}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
-            "targets": targets,
+            "targets": selection.targets,
         }
 
 

scripts/scraper/targets/knulli_targets_scraper.py

@@ -9,6 +9,8 @@ the tag platforms/knulli.yml names):
     the root Config.in in place of Batocera's batocera-system/Config.in
   - package/emulationstation/knulli-es-system/es_systems.yml
     -- emulator requireAnyOf flag mapping
+  - Config.in and what it sources, Batocera's packages read from the
+    batocera submodule at the commit the tag pins
 Sources (knulli-cfw/buildroot, at the submodule commit the tag pins):
   - arch/Config.in, package/mesa3d/Config.in, package/libglvnd/Config.in
 
@@ -32,6 +34,7 @@ class Scraper(BatoceraScraper):
     ES_SYSTEMS_PATH = "package/emulationstation/knulli-es-system/es_systems.yml"
     SYSTEM_SYMBOL = "BR2_PACKAGE_KNULLI_SYSTEM"
     BUILDROOT_REPO = "knulli-cfw/buildroot"
+    SUBMODULES = {"batocera/": "batocera-linux/batocera.linux"}
 
 
 def main() -> None: