chore: merge integ board rule

scripts/common.py

@@ -869,11 +869,24 @@ def load_target_config(
             )
         )
 
-    declared = (targets[canonical] or {}).get("cores")
+    return _target_core_set(targets[canonical] or {}, overrides, canonical, platforms_dir)
+
+
+def _target_core_set(
+    entry: dict, overrides: dict, target: str, platforms_dir: str
+) -> set[str] | None:
+    """A target's cores: what the board builds, overrides applied, and the
+    buildbot cores its RetroArch core downloader installs (`downloads`, a
+    RetroArch targets entry), which a Batocera x86_64 or armv7 board can
+    fetch from its menu."""
+    declared = entry.get("cores")
     if declared is None:
         return None
-    cores = set(str(c) for c in declared)
-    return apply_target_overrides(cores, overrides, canonical)
+    cores = apply_target_overrides({str(c) for c in declared}, overrides, target)
+    downloads = entry.get("downloads")
+    if downloads:
+        cores |= load_target_config("retroarch", str(downloads), platforms_dir) or set()
+    return cores
 
 
 def apply_target_overrides(
@@ -917,12 +930,9 @@ def list_available_targets(
     result = []
     for tname, tdata in sorted(data.get("targets", {}).items()):
         aliases = (overrides.get(tname) or {}).get("aliases", [])
-        declared = (tdata or {}).get("cores")
         # None: the target runs every core the platform declares.
-        count = (
-            None if declared is None
-            else len(apply_target_overrides({str(c) for c in declared}, overrides, tname))
-        )
+        found = _target_core_set(tdata or {}, overrides, tname, platforms_dir)
+        count = None if found is None else len(found)
         result.append(
             {
                 "name": tname,
@@ -2533,6 +2543,7 @@ def group_identical_platforms(
             cores if isinstance(cores, str) else sorted(map(str, cores or [])),
             sorted(map(str, config.get("standalone_cores") or [])),
             sorted(map(str, config.get("dual_mode_cores") or [])),
+            sorted(map(str, config.get("installable_cores") or [])),
             # One archive cannot serve a filesystem that folds case and one
             # that does not: Lakka keeps bios.col beside BIOS.col.
             bool(config.get("case_insensitive_fs")),
@@ -3226,9 +3237,11 @@ def resolve_platform_cores(
         # An emulator the platform launches standalone runs there even when
         # its own list names only the cores: Batocera's es_systems.yml files
         # lexaloffle and gsplus as standalone emulators and nowhere else.
+        # A core the platform's RetroArch installs from the buildbot runs
+        # there as well (installable_cores, Batocera x86_64 and armv7).
         core_set = {str(c) for c in cores_config} | {
             str(c) for c in config.get("standalone_cores") or []
-        }
+        } | {str(c) for c in config.get("installable_cores") or []}
         index = upstream_profile_index(profiles)
         platform = str(config.get("_key") or "")
 

scripts/scraper/batocera_scraper.py

@@ -14,11 +14,11 @@ import re
 import sys
 import urllib.error
 import urllib.request
-from collections.abc import Callable, Iterable
+from collections.abc import Callable
 from pathlib import Path
 
 
-from common import yaml_load
+from common import load_target_config, yaml_load
 from hashing import cut_md5_owner
 from safeparse import yaml_load_upstream
 
@@ -30,6 +30,8 @@ from .base_scraper import (
     requirement_entry,
 )
 from .batocera_launchers import Configgen, configs_root, extract_launchers
+from .targets.batocera_targets_scraper import BuildSelection
+from .targets.batocera_targets_scraper import Scraper as BoardTargets
 
 PLATFORM_NAME = "batocera"
 
@@ -489,76 +491,51 @@ def configgen_cores(
     for GameCube). `dropped` names what es_systems.yml files under an
     emulator without being one.
 
-    A default es_systems.yml does not offer names an emulator the build may
-    lack: KNULLI's defaults pick suyu for the Switch, and no package of its
-    tree, its batocera submodule or its buildroot builds suyu. `built` says
-    whether the build has a standalone emulator (packages_and_programs); a
-    default it answers no for is not run, nor its core.
+    `built` says whether some board builds a name (boards_build). What it
+    answers no for is not run: KNULLI's es_systems lists citron and shadps4,
+    which no KNULLI board selects, and its defaults pick suyu, which no
+    package builds. A default's core follows a standalone emulator the
+    build runs (vox_official is lexaloffle's Voxatron build).
     """
     data = yaml_load_upstream(defaults_text)
     cores, standalone = es_systems_emulators(es_systems_text)
-    offered = set(cores)
     libretro = es_systems_libretro(es_systems_text)
     if not cores:
         raise RuntimeError("es_systems.yml names no emulator")
+    if built is not None:
+        cores = {name for name in cores if built(name)}
+        standalone &= cores
     for system, cfg in data.items():
         if system == "default" or not isinstance(cfg, dict):
             continue
         emulator = cfg.get("emulator", "")
         core = cfg.get("core", "")
-        if (built is not None and emulator and emulator != "libretro"
-                and emulator not in offered and not built(emulator)):
-            continue
-        if core:
-            cores.add(core)
         if emulator and emulator != "libretro":
+            if built is not None and not built(emulator):
+                continue
             standalone.add(emulator)
             if core and core != emulator:
                 standalone.add(core)
+        elif built is not None and core and not built(core):
+            continue
+        if core:
+            cores.add(core)
     dual = (standalone & libretro) - dropped
     return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual)
 
 
-BATOCERA_REPO = "batocera-linux/batocera.linux"
+def boards_build(selection: BuildSelection, launchers: list[dict]) -> Callable[[str], bool]:
+    """Whether some board of a build runs an emulator.
 
-
-def package_names(paths: Iterable[str]) -> set[str]:
-    """The buildroot packages a tree defines: each folder holding a Config.in."""
-    return {
-        path.rsplit("/", 2)[-2]
-        for path in paths
-        if path.endswith("/Config.in") and path.count("/") >= 1
-    }
-
-
-def repo_tree(repo: str, ref: str) -> list[str]:
-    """Every path of a GitHub repository at a revision."""
-    url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
-    req = urllib.request.Request(url, headers=github_headers())
-    with urllib.request.urlopen(req, timeout=60) as resp:
-        found = json.loads(resp.read().decode("utf-8"))
-    if found.get("truncated"):
-        raise RuntimeError(f"{url}: tree truncated")
-    return [entry["path"] for entry in found.get("tree", [])]
-
-
-def submodule_revision(repo: str, path: str, ref: str) -> str:
-    """The commit a repository's submodule at `path` pins at a revision."""
-    url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}"
-    req = urllib.request.Request(url, headers=github_headers())
-    with urllib.request.urlopen(req, timeout=30) as resp:
-        found = json.loads(resp.read().decode("utf-8"))
-    if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"):
-        raise RuntimeError(f"{url}: {path} is not a submodule")
-    return str(found["sha"])
-
-
-def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]:
-    """Whether a build has a standalone emulator.
-
-    It does when a package of that name exists, or when its launcher runs a
-    program the user puts in the BIOS folder (Batocera's lexaloffle runs
-    /userdata/bios/pico-8/pico8), which no package builds.
+    es_systems.yml ties each name to the packages that build it
+    (requireAnyOf), and a name it lists is built when a board selects one of
+    them: KNULLI lists citron and shadps4, and its Config.in selects them
+    only on x86_64 boards, which it has none of. A name only
+    configgen-defaults.yml gives is built when a board selects the package
+    of that name: the symbol its Config.in defines, else the one Buildroot
+    derives from the folder (BR2_PACKAGE_<NAME>, package/pkg-generic.mk). A
+    program the launcher runs from the BIOS folder (lexaloffle,
+    /userdata/bios/pico-8/pico8) needs no package.
     """
     supplied = {
         name.strip()
@@ -567,7 +544,42 @@ def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable
         and str(record.get("search_dir") or "").startswith("/userdata/bios")
         for name in str(record.get("emulator", "")).split(",")
     }
-    return lambda emulator: emulator in packages or emulator in supplied
+
+    def built(name: str) -> bool:
+        if name in selection.emulators or name in supplied:
+            return True
+        if name in selection.listed:
+            return False
+        symbol = selection.packages.get(name) or (
+            "BR2_PACKAGE_" + name.upper().replace("-", "_"))
+        return symbol in selection.symbols
+
+    return built
+
+
+def installable_cores(selection: BuildSelection, cores: list[str]) -> list[str]:
+    """Buildbot cores a board's RetroArch core downloader installs, beyond
+    the cores the image builds.
+
+    The targets scraper names, per board, the RetroArch targets entry its
+    downloader reads (`downloads`): a Batocera x86_64 board installs any
+    core of linux-x86_64, an armv7 one any of linux-armhf.
+    """
+    found: set[str] = set()
+    for entry in selection.targets.values():
+        if entry.get("downloads"):
+            found |= load_target_config("retroarch", entry["downloads"]) or set()
+    return sorted(found - set(cores))
+
+
+def records_run(launchers: list[dict], built: Callable[[str], bool]) -> list[dict]:
+    """The launcher records of emulators the build runs.
+
+    configgen keeps the generators of emulators a build does not select
+    (KNULLI's citron, pcsx2, shadps4): what they read is never read on the
+    device, so the records do not describe the platform.
+    """
+    return [r for r in launchers if any(built(n) for n in str(r["emulator"]).split(", "))]
 
 
 class Scraper(BaseScraper):
@@ -578,20 +590,13 @@ class Scraper(BaseScraper):
         super().__init__(url=SOURCE_URL.format(tag=self.tag))
 
     def _fetch_cores(
-        self, launchers: list[dict] | None = None
+        self, built: Callable[[str], bool] | None = None
     ) -> tuple[list[str], list[str], list[str]]:
         """Core names, standalone cores and dual-mode cores at the pinned tag.
 
-        A default es_systems.yml does not offer counts when a package of
-        Batocera's tree or its buildroot builds it, or when its launcher
-        runs a program from the BIOS folder.
+        With `built` (boards_build), a name counts when some board builds
+        it or its launcher runs a program from the BIOS folder.
         """
-        built = None
-        if launchers is not None:
-            paths = repo_tree(BATOCERA_REPO, self.tag)
-            buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag)
-            paths += repo_tree("batocera-linux/buildroot", buildroot)
-            built = packages_and_programs(package_names(paths), launchers)
         return configgen_cores(
             self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)),
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)),
@@ -633,7 +638,11 @@ class Scraper(BaseScraper):
         launchers = extract_launchers(
             lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
         )
-        cores, standalone, dual = self._fetch_cores(launchers)
+        selection = BoardTargets().build_selection(self.tag)
+        built = boards_build(selection, launchers)
+        launchers = records_run(launchers, built)
+        cores, standalone, dual = self._fetch_cores(built)
+        installable = installable_cores(selection, cores)
         libretro_systems = es_systems_libretro_systems(
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
         )
@@ -653,6 +662,8 @@ class Scraper(BaseScraper):
         if dual:
             # Launched both ways: each layout gets its files.
             result["dual_mode_cores"] = dual
+        if installable:
+            result["installable_cores"] = installable
         # The systems each libretro core runs for: its save folders.
         result["libretro_systems"] = libretro_systems
         result["launchers"] = launchers

scripts/scraper/emuelec_scraper.py

@@ -209,7 +209,18 @@ class Scraper(BaseScraper):
         )
         not_cores = _not_cores(PLATFORM_NAME)
         cores, standalone, dual = es_cores(es_systems, RUNNERS, BUILDS_OF, not_cores)
+        # Each system keeps the list es_systems.json gives it: it says whose
+        # files a platform entry asks for. What runs is what an image carries.
         attach_system_cores(systems, system_cores(es_systems, RUNNERS, not_cores))
+        built = self._built()
+        unbuilt = {n for n in cores if n not in built}
+        cores, standalone, dual = ([n for n in names if n in built] for names in (cores, standalone, dual))
+        # Runners (retroarch, retrorun) and the launcher's own spellings are
+        # no ES core name: only a record of an emulator no image carries goes.
+        launchers = [
+            r for r in extract_launchers(RepoTree.github(REPO, launcher_revision(self.url)))
+            if not set(str(r["emulator"]).split(", ")) <= unbuilt
+        ]
         return {
             "platform": "EmuELEC",
             "version": self.tag.removeprefix("v"),
@@ -225,12 +236,26 @@ class Scraper(BaseScraper):
             "standalone_cores": standalone,
             "dual_mode_cores": dual,
             "systems": systems,
-            # Read at the revision the BIOS list comes from.
-            "launchers": extract_launchers(
-                RepoTree.github(REPO, launcher_revision(self.url))
-            ),
+            # Read at the revision the BIOS list comes from, for the
+            # emulators an image carries.
+            "launchers": launchers,
         }
 
+    def _built(self) -> set[str]:
+        """The cores at least one released device image carries.
+
+        Read by the targets scraper: a device keeps a core when a package it
+        builds installs it and its es_systems.cfg still offers it. The port
+        engines depend on emuelec-ports, which no package of an image
+        depends on, and flycast-dojo is in no device list (UNBUILT there).
+        """
+        try:
+            from .targets.emuelec_targets_scraper import Scraper as DeviceTargets
+        except ImportError:
+            from scraper.targets.emuelec_targets_scraper import Scraper as DeviceTargets
+        targets = DeviceTargets().device_targets(self.tag)["targets"]
+        return {name for entry in targets.values() for name in entry["cores"]}
+
 
 def main() -> None:
     from .base_scraper import scraper_cli

scripts/scraper/knulli_scraper.py

@@ -37,18 +37,18 @@ from .batocera_scraper import (
 )
 from .batocera_scraper import (
     _not_cores,
+    boards_build,
     configgen_cores,
     es_systems_by_system,
     es_systems_libretro_systems,
     extract_systems_dict,
-    package_names,
-    packages_and_programs,
+    installable_cores,
     platform_systems,
-    repo_tree,
+    records_run,
     requirements_from_systems,
-    submodule_revision,
 )
 from .es_emulators import attach_system_cores
+from .targets.knulli_targets_scraper import Scraper as BoardTargets
 from .batocera_launchers import configs_root
 from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers
 from .knulli_launchers import configgen as knulli_configgen
@@ -134,16 +134,14 @@ class Scraper(BaseScraper):
             lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
             lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
         )
-        # The packages KNULLI builds come from its own tree, its batocera
-        # submodule and its buildroot submodule: a default no package of
-        # them builds (suyu) is not run.
-        paths = repo_tree(REPO, self.tag)
-        paths += repo_tree("batocera-linux/batocera.linux", revision)
-        paths += repo_tree(
-            "knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag)
-        )
-        built = packages_and_programs(package_names(paths), launchers)
+        # What runs is what at least one board builds, read by the targets
+        # scraper from the Kconfig of KNULLI's tree and its batocera and
+        # buildroot submodules (boards_build).
+        selection = BoardTargets().build_selection(self.tag)
+        built = boards_build(selection, launchers)
+        launchers = records_run(launchers, built)
         cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built)
+        installable = installable_cores(selection, cores)
         # Each system's own list (psx, es_systems.yml:640-653: libretro
         # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and
         # duckstation-legacy).
@@ -165,6 +163,8 @@ class Scraper(BaseScraper):
             result["standalone_cores"] = standalone
         if dual:
             result["dual_mode_cores"] = dual
+        if installable:
+            result["installable_cores"] = installable
         # configgen hands a libretro core /userdata/saves/<system>, as
         # Batocera's does: the systems each core runs for are its save folders.
         result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)

scripts/scraper/rocknix_scraper.py

@@ -190,6 +190,25 @@ class Scraper(BaseScraper):
         self.tag = None if url else fetch_github_latest_version(GITHUB_REPO)
         super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.tag))
 
+    def _built_cores(self, standalone: list[str]) -> tuple[list[str], list[str]]:
+        """(cores, standalone cores) at least one device image carries.
+
+        RetroArch runs the libretro cores the packages of its emulators
+        metapackage install per device
+        (projects/ROCKNIX/packages/virtual/emulators/package.mk), read by
+        the targets scraper, where every libretro profile counted before. A
+        standalone package no device carries is not run either.
+        """
+        try:
+            from .targets.rocknix_targets_scraper import Scraper as DeviceTargets
+        except ImportError:
+            from scraper.targets.rocknix_targets_scraper import Scraper as DeviceTargets
+        builds = DeviceTargets().device_builds(self.tag or launcher_revision(self.url))
+        libretro = sorted({core for _arch, lr, _sa in builds.values() for core in lr})
+        carried = set(standalone_emulators(sorted({p for *_rest, sa in builds.values() for p in sa})))
+        standalone = [name for name in standalone if name in carried]
+        return sorted({*libretro, *standalone}), standalone
+
     def _fetch_standalone(self) -> list[str]:
         """The packages of projects/ROCKNIX/packages/emulators/standalone."""
         import json
@@ -281,7 +300,7 @@ class Scraper(BaseScraper):
             )
             entry["files"].append(requirement_entry(req))
 
-        standalone = self._fetch_standalone()
+        cores, standalone = self._built_cores(self._fetch_standalone())
         return {
             "platform": "ROCKNIX",
             "version": self.tag or fetch_github_latest_version(GITHUB_REPO),
@@ -290,9 +309,7 @@ class Scraper(BaseScraper):
             "base_destination": "bios",
             "hash_type": "md5",
             "verification_mode": "md5",
-            # RetroArch with every libretro core, plus the standalone
-            # emulators ROCKNIX packages beside it.
-            "cores": ["retroarch", *standalone],
+            "cores": cores,
             "standalone_cores": standalone,
             # Launched both ways only where package.mk also adds the
             # emulator as a RetroArch core.

scripts/scraper/targets/batocera_targets_scraper.py

@@ -3,6 +3,8 @@
 Sources (batocera-linux/batocera.linux):
   - configs/batocera-*.board  -- board definitions, each sets BR2_PACKAGE_BATOCERA_TARGET_*
   - package/batocera/core/batocera-system/Config.in -- select PACKAGE if CONDITION lines
+  - Config.in and every Kconfig file it sources -- the selects a package
+    declares itself (advanced_drastic selects drastic)
   - package/batocera/emulationstation/batocera-es-system/es_systems.yml
     -- emulator requireAnyOf flag mapping
 Sources (batocera-linux/buildroot, at the submodule commit the tag pins):
@@ -19,6 +21,8 @@ import re
 import sys
 import urllib.error
 import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
 from datetime import datetime, timezone
 
 import yaml
@@ -79,6 +83,16 @@ def _fetch(url: str, headers: dict | None = None) -> str:
         raise RuntimeError(f"cannot fetch {url}: {e}") from e
 
 
+def _fetch_present(url: str) -> str | None:
+    """A file's text, or None when the server answers that it does not exist."""
+    try:
+        return _fetch(url)
+    except RuntimeError as e:
+        if isinstance(e.__cause__, urllib.error.HTTPError) and e.__cause__.code == 404:
+            return None
+        raise
+
+
 def _fetch_json(url: str) -> list | dict:
     text = _fetch(url, headers=github_headers())
     try:
@@ -205,11 +219,14 @@ def _parse_meta_flags(text: str) -> list[tuple[str, str]]:
 # ("select BR2_PACKAGE_XROAR if BR2_PACKAGE_HAS_LIBGL") reached no board.
 BUILDROOT_KCONFIG = (
     "arch/Config.in",
+    # Each armv7-A CPU selects BR2_ARM_CPU_ARMV7A (cortex_a15_a7 included).
+    "arch/Config.in.arm",
     "package/mesa3d/Config.in",
     "package/libglvnd/Config.in",
 )
 
 _CONFIG_LINE_RE = re.compile(r"^\s*(?:menu)?config\s+(BR2_\w+)")
+_SOURCE_RE = re.compile(r'^\s*source\s+"\$BR2_EXTERNAL_\w+_PATH/([^"]+)"')
 _OWNED_SELECT_RE = re.compile(r"^\s+select\s+(BR2_\w+)(?:\s+if\s+([^#]+))?")
 _BOARD_FLAG_RE = re.compile(r"^(BR2_\w+)=y\s*$", re.MULTILINE)
 _BOARD_INCLUDE_RE = re.compile(r"^(-?)include\s+(\S+)", re.MULTILINE)
@@ -315,6 +332,51 @@ def _parse_es_systems(text: str) -> dict[str, list[str]]:
     return package_to_emulators
 
 
+# RetroArch's core downloader fetches from DEFAULT_BUILDBOT_SERVER_URL, which
+# Linux sets for x86_64, i386 and armv7 hard-float alone, and leaves empty on
+# aarch64 (config.def.h:1855-1864 at Batocera's 4b74434 and at v1.22.2).
+def buildbot_target(symbols: set[str] | frozenset[str]) -> str | None:
+    """The RetroArch targets entry a board's core downloader installs from.
+
+    `symbols` are the board's active symbols. An armv7-A CPU selects
+    BR2_ARM_CPU_ARMV7A (arch/Config.in.arm) and is hard-float unless the
+    board asks for soft-float (BR2_ARM_EABI): Buildroot picks EABIHF for a
+    CPU with a VFP, which every Cortex-A has. armv6 (bcm2835), aarch64 and
+    riscv64 get no buildbot URL.
+    """
+    if "BR2_x86_64" in symbols:
+        return "linux-x86_64"
+    if "BR2_i386" in symbols:
+        return "linux-x86"
+    if "BR2_ARM_CPU_ARMV7A" in symbols and "BR2_ARM_EABI" not in symbols:
+        return "linux-armhf"
+    return None
+
+
+# configure options and make variables that compile the downloader out.
+_NO_DOWNLOADER = ("--disable-update-cores", "--disable-online-updater", "--disable-networking",
+                  "HAVE_UPDATE_CORES=0", "HAVE_ONLINE_UPDATER=0", "HAVE_LAKKA=1")
+_BLANK_CORES_URL = re.compile(r'^\s*core_updater_buildbot_cores_url\s*=\s*""', re.MULTILINE)
+
+
+def downloader_disabled(retroarch_mk: str, shipped_cfgs: dict[str, str]) -> str | None:
+    """Why RetroArch's core downloader installs nothing, or None.
+
+    Compiled out by its build options, or left without a URL by a shipped
+    retroarch.cfg. A config that hides the menu entry alone does not stop
+    it: without HAVE_LAKKA, Settings > User Interface offers it back
+    (menu_setting.c). A config that blanks the URL for some boards only is
+    a case this reader does not split, and an error.
+    """
+    for option in _NO_DOWNLOADER:
+        if option in retroarch_mk:
+            return f"retroarch.mk sets {option}"
+    blank = sorted(path for path, text in shipped_cfgs.items() if _BLANK_CORES_URL.search(text))
+    if blank:
+        raise RuntimeError(f"{', '.join(blank)} blank the core downloader URL: split per board")
+    return None
+
+
 # Buildroot's architecture choice, as a board sets it (arch/Config.in).
 _ARCH_SYMBOLS = {
     "BR2_aarch64": "aarch64",
@@ -342,6 +404,44 @@ def _arch_from_flag(flag: str, board_flags: set[str] | frozenset[str] = frozense
     return "aarch64"
 
 
+@dataclass(frozen=True)
+class BuildSelection:
+    """What the boards of a build turn on, read once from its Kconfig.
+
+    `targets` is each board's entry of the targets file, `downloads` naming
+    the RetroArch targets entry its core downloader installs from. `emulators` holds
+    every es_systems name some board builds and `listed` every name
+    es_systems ties to a package, `symbols` every symbol some board ends up
+    with, and `packages` the config symbol each package folder of the tree
+    defines.
+    """
+
+    targets: dict[str, dict]
+    emulators: frozenset[str]
+    listed: frozenset[str]
+    symbols: frozenset[str]
+    packages: dict[str, str]
+
+
+def _sourced(text: str) -> list[str]:
+    """The files a Kconfig file sources, relative to the external tree."""
+    return [m.group(1) for line in text.splitlines() if (m := _SOURCE_RE.match(line))]
+
+
+def _package_symbols(files: dict[str, str]) -> dict[str, str]:
+    """package folder -> the first config symbol its Config.in defines."""
+    found: dict[str, str] = {}
+    for path, text in files.items():
+        parts = path.split("/")
+        if len(parts) < 3 or parts[-1] != "Config.in" or "package" not in parts:
+            continue
+        config = next((m.group(1) for line in text.splitlines()
+                       if (m := _CONFIG_LINE_RE.match(line))), None)
+        if config:
+            found.setdefault(parts[-2], config)
+    return found
+
+
 class Scraper(BaseTargetScraper):
     """Cross-references Batocera boards, Config.in, and es_systems to build target lists.
 
@@ -357,6 +457,10 @@ class Scraper(BaseTargetScraper):
     ES_SYSTEMS_PATH = "package/batocera/emulationstation/batocera-es-system/es_systems.yml"
     SYSTEM_SYMBOL = "BR2_PACKAGE_BATOCERA_SYSTEM"
     BUILDROOT_REPO = "batocera-linux/buildroot"
+    # Path prefix -> repository of a submodule the root Config.in sources
+    # from: KNULLI reads Batocera's packages as batocera/package/...
+    SUBMODULES: dict[str, str] = {}
+    RETROARCH_MK = "package/batocera/emulators/retroarch/retroarch/retroarch.mk"
 
     def __init__(self, url: str = "") -> None:
         super().__init__(url=url or f"https://github.com/{self.REPO}")
@@ -411,6 +515,62 @@ class Scraper(BaseTargetScraper):
             pending += [(inc, bool(dash)) for dash, inc in _BOARD_INCLUDE_RE.findall(text)]
         return target, flags
 
+    def _kconfig_files(self) -> dict[str, str]:
+        """The root Config.in and every file it sources, recursively.
+
+        The build reads these and nothing else: a package's own Config.in
+        can select another package (advanced_drastic selects drastic), and
+        such a select sits in no system Config.in. A sourced file absent at
+        the pin defines nothing and is named: KNULLI 20260511 sources
+        package/utils/vtree/Config.in, a package its tree gained later.
+        """
+        bases: list[tuple[str, str]] = []
+        for prefix, repo in self.SUBMODULES.items():
+            pin = _fetch_json(self._api(prefix.rstrip("/")))
+            sha = pin.get("sha") if isinstance(pin, dict) else None
+            if not sha:
+                raise RuntimeError(f"no {prefix.rstrip('/')} submodule commit at {self.tag}")
+            bases.append((prefix, f"https://raw.githubusercontent.com/{repo}/{sha}/"))
+
+        def url(path: str) -> str:
+            for prefix, base in bases:
+                if path.startswith(prefix):
+                    return base + path.removeprefix(prefix)
+            return self._raw(path)
+
+        files: dict[str, str] = {}
+        absent: list[str] = []
+        pending = ["Config.in"]
+        with ThreadPoolExecutor(max_workers=8) as pool:
+            while pending:
+                found = dict(zip(pending, pool.map(lambda path: _fetch_present(url(path)), pending)))
+                absent += [path for path, text in found.items() if text is None]
+                files.update((path, text or "") for path, text in found.items())
+                pending = sorted({src for text in found.values() if text
+                                  for src in _sourced(text)} - set(files))
+        if "Config.in" in absent or len(files) - len(absent) < 2:
+            raise RuntimeError(f"Config.in at {self.tag} sources no readable Kconfig file")
+        for path in absent:
+            print(f"  sourced but absent at {self.tag}: {path}", file=sys.stderr)
+        return files
+
+    def _shipped_retroarch_cfgs(self) -> dict[str, str]:
+        """Every retroarch.cfg the tree ships, its submodules included."""
+        repos = [("", self.REPO, self.tag)]
+        for prefix, repo in self.SUBMODULES.items():
+            pin = _fetch_json(self._api(prefix.rstrip("/")))
+            repos.append((prefix, repo, pin["sha"]))
+        found: dict[str, str] = {}
+        for prefix, repo, ref in repos:
+            listing = _fetch_json(f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1")
+            if not isinstance(listing, dict) or listing.get("truncated"):
+                raise RuntimeError(f"{repo} tree at {ref} is unreadable or truncated")
+            for entry in listing.get("tree", []):
+                if re.search(r"(^|/)retroarch\.cfg$", entry["path"]):
+                    raw = f"https://raw.githubusercontent.com/{repo}/{ref}/{entry['path']}"
+                    found[prefix + entry["path"]] = _fetch(raw)
+        return found
+
     def _buildroot_rules(self) -> list[tuple[str, str, str]]:
         """Buildroot's selects for the symbols the tree tests, at its pin."""
         pin = _fetch_json(self._api("buildroot"))
@@ -423,26 +583,32 @@ class Scraper(BaseTargetScraper):
             rules += _parse_owned_selects(_fetch(f"{raw}/{path}"))
         return rules
 
-    def fetch_targets(self) -> dict:
-        """Build per-board emulator availability map."""
-        self.tag = self.pinned_revision(self.PLATFORM)
+    def build_selection(self, tag: str | None = None) -> BuildSelection:
+        """Each board's selection, from the Kconfig the build reads.
+
+        `tag` is the revision to read, by default the one the platform's
+        YAML was scraped at: a platform scraper moving to a new release
+        passes its own.
+        """
+        self.tag = tag or self.pinned_revision(self.PLATFORM)
         print("  fetching board list...", file=sys.stderr)
         boards = self._list_boards()
 
-        print("  fetching Config.in...", file=sys.stderr)
-        config_in_url = self._raw(self.CONFIG_IN_PATH)
-        config_in_text = _fetch(config_in_url)
+        print("  fetching Kconfig files...", file=sys.stderr)
+        files = self._kconfig_files()
+        config_in_text = files.get(self.CONFIG_IN_PATH)
         if not config_in_text:
-            raise RuntimeError(f"empty Config.in from {config_in_url}")
+            raise RuntimeError(f"Config.in at {self.tag} does not source {self.CONFIG_IN_PATH}")
 
         meta_rules = _parse_meta_flags(config_in_text)
-        rules = _parse_owned_selects(config_in_text)
+        rules = [rule for text in files.values() for rule in _parse_owned_selects(text)]
         print(
-            f"  parsed {len(meta_rules)} meta-flag rules, {len(rules)} select lines",
+            f"  parsed {len(meta_rules)} meta-flag rules, {len(rules)} select lines "
+            f"in {len(files)} Kconfig files",
             file=sys.stderr,
         )
-        if not rules:
-            raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {config_in_url}")
+        if not _parse_owned_selects(config_in_text):
+            raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {self.CONFIG_IN_PATH}")
         rules += self._buildroot_rules()
 
         print("  fetching es_systems.yml...", file=sys.stderr)
@@ -459,9 +625,14 @@ class Scraper(BaseTargetScraper):
             raise RuntimeError(f"no package->emulator mappings parsed from {es_systems_url}")
         images_by_flag = _parse_images(config_in_text)
         if not images_by_flag:
-            raise RuntimeError(f"no target image list parsed from {config_in_url}")
+            raise RuntimeError(f"no target image list parsed from {self.CONFIG_IN_PATH}")
+        disabled = downloader_disabled(_fetch(self._raw(self.RETROARCH_MK)),
+                                       self._shipped_retroarch_cfgs())
+        if disabled:
+            print(f"  core downloader off: {disabled}", file=sys.stderr)
 
         targets: dict[str, dict] = {}
+        symbols: set[str] = set()
         for board_name in sorted(boards):
             target_key = board_name.removeprefix(self.BOARD_PREFIX).removesuffix(".board")
             print(f"  processing {target_key}...", file=sys.stderr)
@@ -476,6 +647,7 @@ class Scraper(BaseTargetScraper):
             if self.SYSTEM_SYMBOL not in selected_packages:
                 print(f"    {board_name} builds no system image", file=sys.stderr)
                 continue
+            symbols |= selected_packages
 
             # Map selected packages to emulator names via es_systems.yml
             emulators: set[str] = set()
@@ -491,16 +663,30 @@ class Scraper(BaseTargetScraper):
             images = sorted(set(images_by_flag.get(primary_flag, [])))
             if images and images != [target_key]:
                 targets[target_key]["images"] = images
+            downloads = None if disabled else buildbot_target(selected_packages)
+            if downloads:
+                targets[target_key]["downloads"] = downloads
             print(
                 f"    {len(emulators)} emulators ({len(selected_packages)} packages selected)",
                 file=sys.stderr,
             )
 
+        return BuildSelection(
+            targets=targets,
+            emulators=frozenset(c for entry in targets.values() for c in entry["cores"]),
+            listed=frozenset(n for names in package_to_emulators.values() for n in names),
+            symbols=frozenset(symbols),
+            packages=_package_symbols(files),
+        )
+
+    def fetch_targets(self) -> dict:
+        """Build per-board emulator availability map."""
+        selection = self.build_selection()
         return {
             "platform": self.PLATFORM,
             "source": f"{self.url}/tree/{self.tag}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
-            "targets": targets,
+            "targets": selection.targets,
         }
 
 

scripts/scraper/targets/emuelec_targets_scraper.py

@@ -610,7 +610,12 @@ class Scraper(BaseTargetScraper):
         return fetch_bytes(RAW.format(ref=self.ref, path=path)).decode("utf-8")
 
     def fetch_targets(self) -> dict:
-        self.ref = self.pinned_revision(PLATFORM_NAME)
+        return self.device_targets()
+
+    def device_targets(self, ref: str | None = None) -> dict:
+        """The targets file, read at `ref`, by default the revision
+        platforms/emuelec.yml was scraped at."""
+        self.ref = ref or self.pinned_revision(PLATFORM_NAME)
         release = json.loads(
             fetch_bytes(RELEASE_API.format(ref=self.ref), headers=github_headers())
         )

scripts/scraper/targets/knulli_targets_scraper.py

@@ -9,6 +9,8 @@ the tag platforms/knulli.yml names):
     the root Config.in in place of Batocera's batocera-system/Config.in
   - package/emulationstation/knulli-es-system/es_systems.yml
     -- emulator requireAnyOf flag mapping
+  - Config.in and what it sources, Batocera's packages read from the
+    batocera submodule at the commit the tag pins
 Sources (knulli-cfw/buildroot, at the submodule commit the tag pins):
   - arch/Config.in, package/mesa3d/Config.in, package/libglvnd/Config.in
 
@@ -32,6 +34,8 @@ class Scraper(BatoceraScraper):
     ES_SYSTEMS_PATH = "package/emulationstation/knulli-es-system/es_systems.yml"
     SYSTEM_SYMBOL = "BR2_PACKAGE_KNULLI_SYSTEM"
     BUILDROOT_REPO = "knulli-cfw/buildroot"
+    SUBMODULES = {"batocera/": "batocera-linux/batocera.linux"}
+    RETROARCH_MK = "package/retroarch/retroarch/retroarch.mk"
 
 
 def main() -> None:

scripts/scraper/targets/rocknix_targets_scraper.py

@@ -5,6 +5,8 @@ Sources (ROCKNIX/distribution, at the release tag platforms/rocknix.yml names):
     image carries, those aarch64 adds, and a `case "${DEVICE}"` block per SoC
   - projects/ROCKNIX/devices/<DEVICE>/options -- ENABLE_32BIT, which the 32-bit
     cores of a device depend on (distributions/ROCKNIX/options defaults it on)
+  - each libretro package's package.mk -- the cores its makeinstall_target
+    copies to ${INSTALL}/usr/lib/libretro (vice-lr installs ten of them)
 
 An H700 or RK3326 handheld carries neither Dolphin nor AetherSX2: without a
 target list, its pack shipped the PS2, GameCube and PS3 firmware anyway.
@@ -13,6 +15,7 @@ target list, its pack shipped the PS2, GameCube and PS3 firmware anyway.
 from __future__ import annotations
 
 import argparse
+import json
 import re
 import sys
 from datetime import datetime, timezone
@@ -89,18 +92,90 @@ def parse_device_cores(
     return libretro, standalone
 
 
-def core_names(libretro: list[str], standalone: list[str]) -> list[str]:
-    """Package names as the profiles' cores: lists spell them.
+_INSTALL_BODY_RE = re.compile(r"^makeinstall_target\(\)\s*\{\n(.*?)^\}", re.M | re.S)
+_COPY_RE = re.compile(r"\bcp\s+(.*?)\s+\$\{?INSTALL\}?/usr/lib/libretro(?:/(\S*))?")
+_CORE_FILE_RE = re.compile(r"([\w+.-]+)_libretro\.(?:so\b|\{[^}]*\bso\b[^}]*\})")
+_TEST_RE = re.compile(r'\[\s*"\$\{(\w+)\}"\s*=\s*"([^"]*)"\s*\]')
 
-    A libretro package is the core's name with `-lr` (`beetle-psx-lr`,
-    `neocd_lr`), spelled with underscores as the buildbot names it. A
-    standalone package is kept under both spellings, as the platform
-    scraper keeps it: `vice-sa` is the only name vice-standalone answers
-    to, `vice` that of the libretro profile.
+
+def installed_cores(package_mk: str, arch: str, with_32bit: bool) -> list[str]:
+    """The cores a libretro package installs on a device.
+
+    The name is that of the file makeinstall_target copies to
+    ${INSTALL}/usr/lib/libretro ($INSTALL in some), renamed when the copy names its target:
+    beetle-psx-lr copies mednafen_psx_libretro.so as beetle_psx_libretro.so,
+    vice-lr ten EMUTYPE builds. pcsx_rearmed-lr picks its file by
+    TARGET_ARCH and adds the 32-bit build when ENABLE_32BIT is on, desmume-lr
+    copies the 32-bit build on aarch64 (ARCH). A test on another variable,
+    or a package that installs no core, is an error.
     """
-    names = {re.sub(r"[-_]lr$", "", package).replace("-", "_") for package in libretro}
-    names.update(standalone_emulators(standalone))
-    return sorted(names)
+    body = _INSTALL_BODY_RE.search(package_mk)
+    if body is None:
+        raise RuntimeError("package.mk has no makeinstall_target")
+    # config/options:36 sets TARGET_ARCH="${ARCH}".
+    values = {"ARCH": arch, "TARGET_ARCH": arch, "ENABLE_32BIT": "true" if with_32bit else "false"}
+
+    def holds(condition: str) -> bool:
+        tests = _TEST_RE.findall(condition)
+        if not tests or re.sub(_TEST_RE, "", condition).strip(" ;&") not in ("", "then"):
+            raise RuntimeError(f"makeinstall_target tests {condition.strip()!r}")
+        for variable, value in tests:
+            if variable not in values:
+                raise RuntimeError(f"makeinstall_target tests ${{{variable}}}")
+            if values[variable] != value:
+                return False
+        return True
+
+    found: list[str] = []
+    stack: list[bool] = []
+    in_case = False
+    for raw in body.group(1).splitlines():
+        line = raw.strip()
+        if re.match(r"case \$\{TARGET_ARCH\} in$", line):
+            in_case = True
+            stack.append(False)
+            continue
+        if line == "esac":
+            in_case = False
+            stack.pop()
+            continue
+        if in_case and (arm := re.match(r"([\w|*]+)\)\s*(.*)$", line)):
+            stack[-1] = arch in arm.group(1).split("|") or arm.group(1) == "*"
+            line = arm.group(2)
+        if line.startswith("if "):
+            stack.append(holds(line[3:]))
+            continue
+        if line == "else":
+            stack[-1] = not stack[-1]
+            continue
+        if line == "fi":
+            stack.pop()
+            continue
+        if not all(stack):
+            continue
+        for sources, target in _COPY_RE.findall(line):
+            if named := _CORE_FILE_RE.search(target):
+                found.append(named.group(1))
+                continue
+            for source in sources.split():
+                if not source.startswith("-") and (named := _CORE_FILE_RE.search(source.rsplit("/", 1)[-1])):
+                    found.append(named.group(1))
+    if not found:
+        raise RuntimeError("makeinstall_target installs no libretro core")
+    return sorted(set(found))
+
+
+def package_paths(tree: list[str]) -> dict[str, str]:
+    """package name -> its package.mk, a ROCKNIX project package first.
+
+    The build takes projects/ROCKNIX/packages/<...>/<name> over
+    packages/<...>/<name> when both exist.
+    """
+    found: dict[str, str] = {}
+    for path in sorted(tree, key=lambda p: not p.startswith("projects/ROCKNIX/")):
+        if path.endswith("/package.mk"):
+            found.setdefault(path.rsplit("/", 2)[-2], path)
+    return found
 
 
 # A package that builds nothing itself and pulls other cores in. idtech-lr
@@ -149,13 +224,40 @@ class Scraper(BaseTargetScraper):
             raise RuntimeError(f"no device in {DEVICES_DIR} at {self.tag}")
         return sorted(devices)
 
-    def fetch_targets(self) -> dict:
-        self.tag = self.pinned_revision(PLATFORM_NAME)
+    def _tree(self) -> list[str]:
+        listing = json.loads(fetch_bytes(
+            f"https://api.github.com/repos/ROCKNIX/distribution/git/trees/{self.tag}?recursive=1",
+            headers=github_headers(),
+        ))
+        if listing.get("truncated"):
+            raise RuntimeError(f"ROCKNIX tree at {self.tag} is truncated")
+        return [entry["path"] for entry in listing.get("tree", [])]
+
+    def device_builds(self, tag: str | None = None) -> dict[str, tuple[str, list[str], list[str]]]:
+        """{device: (architecture, libretro cores, standalone packages)}.
+
+        Read at `tag`, by default the revision platforms/rocknix.yml was
+        scraped at: metapackages resolved to the packages they pull in, and
+        each libretro package to the cores it installs on the device.
+        """
+        self.tag = tag or self.pinned_revision(PLATFORM_NAME)
         package_mk = self._raw(EMULATORS_MK)
         if "LIBRETRO_CORES" not in package_mk:
             raise RuntimeError(f"no LIBRETRO_CORES in {EMULATORS_MK} at {self.tag}")
+        paths = package_paths(self._tree())
+        texts: dict[str, str] = {}
 
-        targets: dict[str, dict] = {}
+        def cores_of(package: str, arch: str, with_32bit: bool) -> list[str]:
+            if package not in paths:
+                raise RuntimeError(f"no package.mk for {package} at {self.tag}")
+            if package not in texts:
+                texts[package] = self._raw(paths[package])
+            try:
+                return installed_cores(texts[package], arch, with_32bit)
+            except RuntimeError as e:
+                raise RuntimeError(f"{paths[package]}: {e}") from e
+
+        builds: dict[str, tuple[str, list[str], list[str]]] = {}
         for device in self._devices():
             options = self._raw(f"{DEVICES_DIR}/{device}/options")
             arch = "x86_64" if device == "AMD64" else "aarch64"
@@ -165,9 +267,19 @@ class Scraper(BaseTargetScraper):
                 if meta in libretro:
                     libretro.remove(meta)
                     libretro += metapackage_cores(self._raw(path), arch)
-            cores = core_names(libretro, standalone)
-            targets[device.lower()] = {"architecture": arch, "cores": cores}
-            print(f"  {device.lower()} ({arch}): {len(cores)} cores", file=sys.stderr)
+            cores = sorted({c for package in libretro for c in cores_of(package, arch, with_32bit)})
+            builds[device.lower()] = (arch, cores, standalone)
+        return builds
+
+    def fetch_targets(self) -> dict:
+        targets: dict[str, dict] = {}
+        for device, (arch, libretro, standalone) in self.device_builds().items():
+            # A standalone package is kept under both spellings, as the
+            # platform scraper keeps it: vice-sa is the only name
+            # vice-standalone answers to, vice that of the libretro profile.
+            cores = sorted({*libretro, *standalone_emulators(standalone)})
+            targets[device] = {"architecture": arch, "cores": cores}
+            print(f"  {device} ({arch}): {len(cores)} cores", file=sys.stderr)
 
         return {
             "platform": PLATFORM_NAME,