fix: drop defaults the build has no package for

KNULLI's configgen-defaults.yml picks suyu for the Switch, which its
es_systems.yml does not offer and no package of its tree, its batocera
submodule or its buildroot builds: KNULLI does not ship suyu. A default
es_systems.yml does not offer now counts only when a package of the
build's trees has that name, or when its launcher runs a program from
the BIOS folder (Batocera's lexaloffle). Batocera's lists are unchanged
by the rule. suyu leaves KNULLI's cores, and the XDG_DATA_HOME record
that served it goes.

scripts/scraper/batocera_scraper.py

@@ -14,6 +14,7 @@ import re
 import sys
 import urllib.error
 import urllib.request
+from collections.abc import Callable, Iterable
 from pathlib import Path
 
 
@@ -168,7 +169,8 @@ def es_systems_libretro_systems(text: str, dropped: set[str]) -> dict[str, list[
 
 
 def es_systems_by_system(
-    es_systems_text: str, defaults_text: str, dropped: set[str]
+    es_systems_text: str, defaults_text: str, dropped: set[str],
+    built: Callable[[str], bool] | None = None,
 ) -> dict[str, list[str]]:
     """system -> the cores es_systems.yml offers it, in the file's order.
 
@@ -195,6 +197,10 @@ def es_systems_by_system(
         default = defaults.get(system)
         if isinstance(default, dict):
             emulator, core = default.get("emulator"), default.get("core")
+            if (built is not None and emulator and emulator != "libretro"
+                    and str(emulator) not in names and not built(str(emulator))):
+                # A default the build lacks (configgen_cores).
+                emulator = core = None
             if emulator and emulator != "libretro":
                 names.append(str(emulator))
             if core:
@@ -472,7 +478,8 @@ def platform_systems(
 
 
 def configgen_cores(
-    defaults_text: str, es_systems_text: str, dropped: set[str]
+    defaults_text: str, es_systems_text: str, dropped: set[str],
+    built: Callable[[str], bool] | None = None,
 ) -> tuple[list[str], list[str], list[str]]:
     """Core names, standalone cores and dual-mode cores of a configgen build.
 
@@ -481,9 +488,16 @@ def configgen_cores(
     dual-mode ones those es_systems.yml also offers under libretro (dolphin
     for GameCube). `dropped` names what es_systems.yml files under an
     emulator without being one.
+
+    A default es_systems.yml does not offer names an emulator the build may
+    lack: KNULLI's defaults pick suyu for the Switch, and no package of its
+    tree, its batocera submodule or its buildroot builds suyu. `built` says
+    whether the build has a standalone emulator (packages_and_programs); a
+    default it answers no for is not run, nor its core.
     """
     data = yaml_load_upstream(defaults_text)
     cores, standalone = es_systems_emulators(es_systems_text)
+    offered = set(cores)
     libretro = es_systems_libretro(es_systems_text)
     if not cores:
         raise RuntimeError("es_systems.yml names no emulator")
@@ -492,6 +506,9 @@ def configgen_cores(
             continue
         emulator = cfg.get("emulator", "")
         core = cfg.get("core", "")
+        if (built is not None and emulator and emulator != "libretro"
+                and emulator not in offered and not built(emulator)):
+            continue
         if core:
             cores.add(core)
         if emulator and emulator != "libretro":
@@ -502,6 +519,57 @@ def configgen_cores(
     return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual)
 
 
+BATOCERA_REPO = "batocera-linux/batocera.linux"
+
+
+def package_names(paths: Iterable[str]) -> set[str]:
+    """The buildroot packages a tree defines: each folder holding a Config.in."""
+    return {
+        path.rsplit("/", 2)[-2]
+        for path in paths
+        if path.endswith("/Config.in") and path.count("/") >= 1
+    }
+
+
+def repo_tree(repo: str, ref: str) -> list[str]:
+    """Every path of a GitHub repository at a revision."""
+    url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
+    req = urllib.request.Request(url, headers=github_headers())
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        found = json.loads(resp.read().decode("utf-8"))
+    if found.get("truncated"):
+        raise RuntimeError(f"{url}: tree truncated")
+    return [entry["path"] for entry in found.get("tree", [])]
+
+
+def submodule_revision(repo: str, path: str, ref: str) -> str:
+    """The commit a repository's submodule at `path` pins at a revision."""
+    url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}"
+    req = urllib.request.Request(url, headers=github_headers())
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        found = json.loads(resp.read().decode("utf-8"))
+    if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"):
+        raise RuntimeError(f"{url}: {path} is not a submodule")
+    return str(found["sha"])
+
+
+def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]:
+    """Whether a build has a standalone emulator.
+
+    It does when a package of that name exists, or when its launcher runs a
+    program the user puts in the BIOS folder (Batocera's lexaloffle runs
+    /userdata/bios/pico-8/pico8), which no package builds.
+    """
+    supplied = {
+        name.strip()
+        for record in launchers
+        if record.get("passes_to") == "exec"
+        and str(record.get("search_dir") or "").startswith("/userdata/bios")
+        for name in str(record.get("emulator", "")).split(",")
+    }
+    return lambda emulator: emulator in packages or emulator in supplied
+
+
 class Scraper(BaseScraper):
     """Scraper for batocera-systems Python dict."""
 
@@ -509,12 +577,26 @@ class Scraper(BaseScraper):
         self.tag = fetch_stable_tag()
         super().__init__(url=SOURCE_URL.format(tag=self.tag))
 
-    def _fetch_cores(self) -> tuple[list[str], list[str], list[str]]:
-        """Core names, standalone cores and dual-mode cores at the pinned tag."""
+    def _fetch_cores(
+        self, launchers: list[dict] | None = None
+    ) -> tuple[list[str], list[str], list[str]]:
+        """Core names, standalone cores and dual-mode cores at the pinned tag.
+
+        A default es_systems.yml does not offer counts when a package of
+        Batocera's tree or its buildroot builds it, or when its launcher
+        runs a program from the BIOS folder.
+        """
+        built = None
+        if launchers is not None:
+            paths = repo_tree(BATOCERA_REPO, self.tag)
+            buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag)
+            paths += repo_tree("batocera-linux/buildroot", buildroot)
+            built = packages_and_programs(package_names(paths), launchers)
         return configgen_cores(
             self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)),
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)),
             _not_cores(),
+            built,
         )
 
     def _fetch_text(self, url: str) -> str:
@@ -548,7 +630,10 @@ class Scraper(BaseScraper):
             SYSTEM_SLUG_MAP,
         )
 
-        cores, standalone, dual = self._fetch_cores()
+        launchers = extract_launchers(
+            lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
+        )
+        cores, standalone, dual = self._fetch_cores(launchers)
         libretro_systems = es_systems_libretro_systems(
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
         )
@@ -570,9 +655,7 @@ class Scraper(BaseScraper):
             result["dual_mode_cores"] = dual
         # The systems each libretro core runs for: its save folders.
         result["libretro_systems"] = libretro_systems
-        result["launchers"] = extract_launchers(
-            lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
-        )
+        result["launchers"] = launchers
         return result
 
 

scripts/scraper/knulli_launchers.py

@@ -48,7 +48,6 @@ try:
     )
     from .configgen_launchers import (
         Launcher,
-        _calls,
         LauncherShapeError,
         Source,
         _assigned_name,
@@ -74,7 +73,6 @@ except ImportError:
     )
     from configgen_launchers import (  # type: ignore[no-redef]
         Launcher,
-        _calls,
         LauncherShapeError,
         Source,
         _assigned_name,
@@ -1062,39 +1060,6 @@ EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = (
 )
 
 
-# Generators whose command sets the emulator's XDG data folder, the parent
-# of every user folder a yuzu-family emulator derives.
-DATA_HOME_GENERATORS = ("suyu.suyuGenerator",)
-
-
-def data_homes(cg: Configgen) -> list[Launcher]:
-    """Command(..., env={"XDG_DATA_HOME": <path>}) in the generators above.
-
-    The path need not lie in /userdata/bios: KNULLI's suyu keeps its data
-    in /userdata/saves/switch, which tells where its keys are not.
-    """
-    launchers = []
-    for module in DATA_HOME_GENERATORS:
-        src = cg.generator(module)
-        found = [
-            (_path_value(item, cg.paths), key)
-            for call in _calls(src.tree, attr="Command")
-            for keyword in call.keywords
-            if keyword.arg == "env" and isinstance(keyword.value, ast.Dict)
-            for key, item in zip(keyword.value.keys, keyword.value.values)
-            if isinstance(key, ast.Constant) and key.value == "XDG_DATA_HOME"
-        ]
-        if not found or found[0][0] is None:
-            src.fail("the command environment sets no XDG_DATA_HOME path")
-        value, node = found[0]
-        launchers.append(Launcher(
-            emulator=cg.emulator(module), build="standalone", kind="passes_path", names=[],
-            search_dir=value, passes_to="environment XDG_DATA_HOME",
-            source_ref=src.ref([(node.lineno, node.end_lineno)]),
-        ))
-    return launchers
-
-
 # KNULLI builds Batocera's emulator packages from this submodule, at the
 # batocera.linux commit it pins: Citron's path patch is there.
 BATOCERA_SUBMODULE = "batocera"
@@ -1114,7 +1079,6 @@ def extract_launchers(
     launchers += boot_disks(cg, read)
     for spec in GENERIC_SITES:
         launchers += bios_sites(cg, spec)
-    launchers += data_homes(cg)
     if batocera is not None:
         prefix = f"{BATOCERA_SUBMODULE}/"
         launchers += patched_paths(

scripts/scraper/knulli_scraper.py

@@ -41,8 +41,12 @@ from .batocera_scraper import (
     es_systems_by_system,
     es_systems_libretro_systems,
     extract_systems_dict,
+    package_names,
+    packages_and_programs,
     platform_systems,
+    repo_tree,
     requirements_from_systems,
+    submodule_revision,
 )
 from .es_emulators import attach_system_cores
 from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers
@@ -123,11 +127,27 @@ class Scraper(BaseScraper):
         defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag))
         es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))
         dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME)
-        cores, standalone, dual = configgen_cores(defaults, es_systems, dropped)
+        revision = self.submodule_revision()
+        launchers = extract_launchers(
+            lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
+            lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
+        )
+        # The packages KNULLI builds come from its own tree, its batocera
+        # submodule and its buildroot submodule: a default no package of
+        # them builds (suyu) is not run.
+        paths = repo_tree(REPO, self.tag)
+        paths += repo_tree("batocera-linux/batocera.linux", revision)
+        paths += repo_tree(
+            "knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag)
+        )
+        built = packages_and_programs(package_names(paths), launchers)
+        cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built)
         # Each system's own list (psx, es_systems.yml:640-653: libretro
         # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and
         # duckstation-legacy).
-        attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped))
+        attach_system_cores(
+            systems, es_systems_by_system(es_systems, defaults, dropped, built)
+        )
         result = {
             "platform": "KNULLI",
             "version": self.tag,
@@ -146,11 +166,7 @@ class Scraper(BaseScraper):
         # configgen hands a libretro core /userdata/saves/<system>, as
         # Batocera's does: the systems each core runs for are its save folders.
         result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)
-        revision = self.submodule_revision()
-        result["launchers"] = extract_launchers(
-            lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
-            lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
-        )
+        result["launchers"] = launchers
         return result