fix: drop defaults the build has no package for

KNULLI's configgen-defaults.yml picks suyu for the Switch, which its
es_systems.yml does not offer and no package of its tree, its batocera
submodule or its buildroot builds: KNULLI does not ship suyu. A default
es_systems.yml does not offer now counts only when a package of the
build's trees has that name, or when its launcher runs a program from
the BIOS folder (Batocera's lexaloffle). Batocera's lists are unchanged
by the rule. suyu leaves KNULLI's cores, and the XDG_DATA_HOME record
that served it goes.

platforms/knulli.yml

@@ -186,7 +186,6 @@ cores:
 - superbroswar
 - supermodel
 - supermodel-legacy
-- suyu
 - swanstation
 - taradino
 - tgbdual
@@ -4970,7 +4969,6 @@ standalone_cores:
 - stella
 - supermodel
 - supermodel-legacy
-- suyu
 - taradino
 - theforceengine
 - thextech
@@ -6143,14 +6141,6 @@ launchers:
   depends_on_option: not system.name == 'ti99'
   source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:431-432,\
     \ 435"
-- emulator: suyu
-  build: standalone
-  kind: passes_path
-  names: []
-  search_dir: /userdata/saves/switch
-  passes_to: environment XDG_DATA_HOME
-  fails_when_missing: false
-  source_ref: "package/system/knulli-configgen/configgen/configgen/generators/suyu/suyuGenerator.py:41"
 - emulator: citron
   build: standalone
   kind: passes_path

scripts/scraper/batocera_scraper.py

@@ -14,6 +14,7 @@ import re
 import sys
 import urllib.error
 import urllib.request
+from collections.abc import Callable, Iterable
 from pathlib import Path
 
 
@@ -168,7 +169,8 @@ def es_systems_libretro_systems(text: str, dropped: set[str]) -> dict[str, list[
 
 
 def es_systems_by_system(
-    es_systems_text: str, defaults_text: str, dropped: set[str]
+    es_systems_text: str, defaults_text: str, dropped: set[str],
+    built: Callable[[str], bool] | None = None,
 ) -> dict[str, list[str]]:
     """system -> the cores es_systems.yml offers it, in the file's order.
 
@@ -195,6 +197,10 @@ def es_systems_by_system(
         default = defaults.get(system)
         if isinstance(default, dict):
             emulator, core = default.get("emulator"), default.get("core")
+            if (built is not None and emulator and emulator != "libretro"
+                    and str(emulator) not in names and not built(str(emulator))):
+                # A default the build lacks (configgen_cores).
+                emulator = core = None
             if emulator and emulator != "libretro":
                 names.append(str(emulator))
             if core:
@@ -472,7 +478,8 @@ def platform_systems(
 
 
 def configgen_cores(
-    defaults_text: str, es_systems_text: str, dropped: set[str]
+    defaults_text: str, es_systems_text: str, dropped: set[str],
+    built: Callable[[str], bool] | None = None,
 ) -> tuple[list[str], list[str], list[str]]:
     """Core names, standalone cores and dual-mode cores of a configgen build.
 
@@ -481,9 +488,16 @@ def configgen_cores(
     dual-mode ones those es_systems.yml also offers under libretro (dolphin
     for GameCube). `dropped` names what es_systems.yml files under an
     emulator without being one.
+
+    A default es_systems.yml does not offer names an emulator the build may
+    lack: KNULLI's defaults pick suyu for the Switch, and no package of its
+    tree, its batocera submodule or its buildroot builds suyu. `built` says
+    whether the build has a standalone emulator (packages_and_programs); a
+    default it answers no for is not run, nor its core.
     """
     data = yaml_load_upstream(defaults_text)
     cores, standalone = es_systems_emulators(es_systems_text)
+    offered = set(cores)
     libretro = es_systems_libretro(es_systems_text)
     if not cores:
         raise RuntimeError("es_systems.yml names no emulator")
@@ -492,6 +506,9 @@ def configgen_cores(
             continue
         emulator = cfg.get("emulator", "")
         core = cfg.get("core", "")
+        if (built is not None and emulator and emulator != "libretro"
+                and emulator not in offered and not built(emulator)):
+            continue
         if core:
             cores.add(core)
         if emulator and emulator != "libretro":
@@ -502,6 +519,57 @@ def configgen_cores(
     return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual)
 
 
+BATOCERA_REPO = "batocera-linux/batocera.linux"
+
+
+def package_names(paths: Iterable[str]) -> set[str]:
+    """The buildroot packages a tree defines: each folder holding a Config.in."""
+    return {
+        path.rsplit("/", 2)[-2]
+        for path in paths
+        if path.endswith("/Config.in") and path.count("/") >= 1
+    }
+
+
+def repo_tree(repo: str, ref: str) -> list[str]:
+    """Every path of a GitHub repository at a revision."""
+    url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
+    req = urllib.request.Request(url, headers=github_headers())
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        found = json.loads(resp.read().decode("utf-8"))
+    if found.get("truncated"):
+        raise RuntimeError(f"{url}: tree truncated")
+    return [entry["path"] for entry in found.get("tree", [])]
+
+
+def submodule_revision(repo: str, path: str, ref: str) -> str:
+    """The commit a repository's submodule at `path` pins at a revision."""
+    url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}"
+    req = urllib.request.Request(url, headers=github_headers())
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        found = json.loads(resp.read().decode("utf-8"))
+    if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"):
+        raise RuntimeError(f"{url}: {path} is not a submodule")
+    return str(found["sha"])
+
+
+def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]:
+    """Whether a build has a standalone emulator.
+
+    It does when a package of that name exists, or when its launcher runs a
+    program the user puts in the BIOS folder (Batocera's lexaloffle runs
+    /userdata/bios/pico-8/pico8), which no package builds.
+    """
+    supplied = {
+        name.strip()
+        for record in launchers
+        if record.get("passes_to") == "exec"
+        and str(record.get("search_dir") or "").startswith("/userdata/bios")
+        for name in str(record.get("emulator", "")).split(",")
+    }
+    return lambda emulator: emulator in packages or emulator in supplied
+
+
 class Scraper(BaseScraper):
     """Scraper for batocera-systems Python dict."""
 
@@ -509,12 +577,26 @@ class Scraper(BaseScraper):
         self.tag = fetch_stable_tag()
         super().__init__(url=SOURCE_URL.format(tag=self.tag))
 
-    def _fetch_cores(self) -> tuple[list[str], list[str], list[str]]:
-        """Core names, standalone cores and dual-mode cores at the pinned tag."""
+    def _fetch_cores(
+        self, launchers: list[dict] | None = None
+    ) -> tuple[list[str], list[str], list[str]]:
+        """Core names, standalone cores and dual-mode cores at the pinned tag.
+
+        A default es_systems.yml does not offer counts when a package of
+        Batocera's tree or its buildroot builds it, or when its launcher
+        runs a program from the BIOS folder.
+        """
+        built = None
+        if launchers is not None:
+            paths = repo_tree(BATOCERA_REPO, self.tag)
+            buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag)
+            paths += repo_tree("batocera-linux/buildroot", buildroot)
+            built = packages_and_programs(package_names(paths), launchers)
         return configgen_cores(
             self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)),
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)),
             _not_cores(),
+            built,
         )
 
     def _fetch_text(self, url: str) -> str:
@@ -548,7 +630,10 @@ class Scraper(BaseScraper):
             SYSTEM_SLUG_MAP,
         )
 
-        cores, standalone, dual = self._fetch_cores()
+        launchers = extract_launchers(
+            lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
+        )
+        cores, standalone, dual = self._fetch_cores(launchers)
         libretro_systems = es_systems_libretro_systems(
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
         )
@@ -570,9 +655,7 @@ class Scraper(BaseScraper):
             result["dual_mode_cores"] = dual
         # The systems each libretro core runs for: its save folders.
         result["libretro_systems"] = libretro_systems
-        result["launchers"] = extract_launchers(
-            lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
-        )
+        result["launchers"] = launchers
         return result
 
 

scripts/scraper/knulli_launchers.py

@@ -48,7 +48,6 @@ try:
     )
     from .configgen_launchers import (
         Launcher,
-        _calls,
         LauncherShapeError,
         Source,
         _assigned_name,
@@ -74,7 +73,6 @@ except ImportError:
     )
     from configgen_launchers import (  # type: ignore[no-redef]
         Launcher,
-        _calls,
         LauncherShapeError,
         Source,
         _assigned_name,
@@ -1062,39 +1060,6 @@ EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = (
 )
 
 
-# Generators whose command sets the emulator's XDG data folder, the parent
-# of every user folder a yuzu-family emulator derives.
-DATA_HOME_GENERATORS = ("suyu.suyuGenerator",)
-
-
-def data_homes(cg: Configgen) -> list[Launcher]:
-    """Command(..., env={"XDG_DATA_HOME": <path>}) in the generators above.
-
-    The path need not lie in /userdata/bios: KNULLI's suyu keeps its data
-    in /userdata/saves/switch, which tells where its keys are not.
-    """
-    launchers = []
-    for module in DATA_HOME_GENERATORS:
-        src = cg.generator(module)
-        found = [
-            (_path_value(item, cg.paths), key)
-            for call in _calls(src.tree, attr="Command")
-            for keyword in call.keywords
-            if keyword.arg == "env" and isinstance(keyword.value, ast.Dict)
-            for key, item in zip(keyword.value.keys, keyword.value.values)
-            if isinstance(key, ast.Constant) and key.value == "XDG_DATA_HOME"
-        ]
-        if not found or found[0][0] is None:
-            src.fail("the command environment sets no XDG_DATA_HOME path")
-        value, node = found[0]
-        launchers.append(Launcher(
-            emulator=cg.emulator(module), build="standalone", kind="passes_path", names=[],
-            search_dir=value, passes_to="environment XDG_DATA_HOME",
-            source_ref=src.ref([(node.lineno, node.end_lineno)]),
-        ))
-    return launchers
-
-
 # KNULLI builds Batocera's emulator packages from this submodule, at the
 # batocera.linux commit it pins: Citron's path patch is there.
 BATOCERA_SUBMODULE = "batocera"
@@ -1114,7 +1079,6 @@ def extract_launchers(
     launchers += boot_disks(cg, read)
     for spec in GENERIC_SITES:
         launchers += bios_sites(cg, spec)
-    launchers += data_homes(cg)
     if batocera is not None:
         prefix = f"{BATOCERA_SUBMODULE}/"
         launchers += patched_paths(

scripts/scraper/knulli_scraper.py

@@ -41,8 +41,12 @@ from .batocera_scraper import (
     es_systems_by_system,
     es_systems_libretro_systems,
     extract_systems_dict,
+    package_names,
+    packages_and_programs,
     platform_systems,
+    repo_tree,
     requirements_from_systems,
+    submodule_revision,
 )
 from .es_emulators import attach_system_cores
 from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers
@@ -123,11 +127,27 @@ class Scraper(BaseScraper):
         defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag))
         es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))
         dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME)
-        cores, standalone, dual = configgen_cores(defaults, es_systems, dropped)
+        revision = self.submodule_revision()
+        launchers = extract_launchers(
+            lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
+            lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
+        )
+        # The packages KNULLI builds come from its own tree, its batocera
+        # submodule and its buildroot submodule: a default no package of
+        # them builds (suyu) is not run.
+        paths = repo_tree(REPO, self.tag)
+        paths += repo_tree("batocera-linux/batocera.linux", revision)
+        paths += repo_tree(
+            "knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag)
+        )
+        built = packages_and_programs(package_names(paths), launchers)
+        cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built)
         # Each system's own list (psx, es_systems.yml:640-653: libretro
         # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and
         # duckstation-legacy).
-        attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped))
+        attach_system_cores(
+            systems, es_systems_by_system(es_systems, defaults, dropped, built)
+        )
         result = {
             "platform": "KNULLI",
             "version": self.tag,
@@ -146,11 +166,7 @@ class Scraper(BaseScraper):
         # configgen hands a libretro core /userdata/saves/<system>, as
         # Batocera's does: the systems each core runs for are its save folders.
         result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)
-        revision = self.submodule_revision()
-        result["launchers"] = extract_launchers(
-            lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
-            lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
-        )
+        result["launchers"] = launchers
         return result
 
 

tests/test_launcher_paths.py

@@ -21,7 +21,6 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "scripts"))
 
 from launchers import handed_directories, standalone_bindings  # noqa: E402
-from scraper import knulli_launchers as kl  # noqa: E402
 from scraper import retrobat_launchers as rl  # noqa: E402
 from scraper.batocera_launchers import (  # noqa: E402
     GENERATORS,
@@ -128,35 +127,27 @@ class AFolderTheGeneratorEnters(unittest.TestCase):
         self.assertEqual(found, {"": ("outside", "")})
 
 
-class ADataFolderTheEnvironmentSets(unittest.TestCase):
-    SUYU = f"{kl.GENERATORS}/suyu/suyuGenerator.py"
-    TEXT = textwrap.dedent("""\
-        from ...batoceraPaths import CACHE, CONFIGS, SAVES
-
-        class SuyuGenerator:
-            def generate(self, system, rom):
-                commandArray = ["/usr/bin/suyu", "-f", "-g", rom]
-                return Command.Command(array=commandArray, env={
-                    "XDG_CONFIG_HOME": CONFIGS,
-                    "XDG_DATA_HOME": SAVES / "switch",
-                })
-        """)
-
-    def test_the_data_folder_is_a_record_and_binds_outside(self):
-        files = {f"{kl.CONFIGGEN}/batoceraPaths.py": PATHS,
-                 f"{kl.GENERATORS}/importer.py": IMPORTER_TEXT, self.SUYU: self.TEXT}
-        (record,) = records(kl.data_homes(kl.configgen(files.__getitem__)))
-        self.assertEqual(
-            (record["emulator"], record["search_dir"], record["passes_to"], record["source_ref"]),
-            ("suyu", "/userdata/saves/switch", "environment XDG_DATA_HOME", f"{self.SUYU}:8"),
-        )
-        config = {"base_destination": "bios", "standalone_cores": ["suyu"], "launchers": [record]}
+class AVariableADeclaredFolderIsSpelledFrom(unittest.TestCase):
+    def test_a_data_folder_outside_the_tree_binds_outside(self):
+        record = {"emulator": "emu", "build": "standalone", "kind": "passes_path",
+                  "names": [], "search_dir": "/userdata/saves/switch",
+                  "passes_to": "environment XDG_DATA_HOME", "source_ref": "x.py:1"}
+        config = {"base_destination": "bios", "standalone_cores": ["emu"], "launchers": [record]}
         found = standalone_bindings(
-            {"suyu"}, ["$XDG_DATA_HOME/suyu", "SuyuPath::KeysDir = keys"],
-            handed_directories(config),
+            {"emu"}, ["$XDG_DATA_HOME/emu"], handed_directories(config)
         )
         self.assertEqual(found, {"": ("outside", "")})
 
+    def test_a_data_folder_in_the_tree_binds_below_it(self):
+        record = {"emulator": "emu", "build": "standalone", "kind": "passes_path",
+                  "names": [], "search_dir": "/userdata/bios/data",
+                  "passes_to": "environment XDG_DATA_HOME", "source_ref": "x.py:1"}
+        config = {"base_destination": "bios", "standalone_cores": ["emu"], "launchers": [record]}
+        found = standalone_bindings(
+            {"emu"}, ["$XDG_DATA_HOME/emu"], handed_directories(config)
+        )
+        self.assertEqual(found, {"": ("", "data/emu")})
+
 
 class APortableUserFolder(unittest.TestCase):
     CITRON = """
@@ -213,5 +204,54 @@ class APortableUserFolder(unittest.TestCase):
         self.assertEqual(found, {"": ("emulators", "citron/user")})
 
 
+
+class ADefaultTheBuildLacksIsNotRun(unittest.TestCase):
+    """KNULLI's configgen-defaults.yml picks suyu for the Switch, which its
+    es_systems.yml does not offer and no package builds."""
+
+    ES = textwrap.dedent("""\
+        switch:
+          emulators:
+            citron:
+              citron: { requireAnyOf: [BR2_PACKAGE_CITRON] }
+        voxatron:
+          emulators: {}
+        """)
+    DEFAULTS = textwrap.dedent("""\
+        switch:
+          emulator: suyu
+          core: suyu
+        voxatron:
+          emulator: lexaloffle
+          core: vox_official
+        kodi:
+          emulator: kodi
+          core: kodi
+        """)
+    LAUNCHERS = [{"emulator": "lexaloffle", "kind": "fixed_name", "passes_to": "exec",
+                  "search_dir": "/userdata/bios"}]
+
+    def test_only_what_a_package_or_the_user_supplies_is_run(self):
+        from scraper import batocera_scraper as bs
+
+        paths = ["package/batocera/emulators/citron/Config.in", "package/kodi/Config.in"]
+        built = bs.packages_and_programs(bs.package_names(paths), self.LAUNCHERS)
+        cores, standalone, _dual = bs.configgen_cores(self.DEFAULTS, self.ES, set(), built)
+        self.assertEqual(sorted(cores), ["citron", "kodi", "vox_official"])
+        self.assertIn("lexaloffle", standalone)
+        self.assertNotIn("suyu", standalone)
+        by_system = bs.es_systems_by_system(self.ES, self.DEFAULTS, set(), built)
+        self.assertEqual(by_system["switch"], ["citron"])
+        # Without the packages, every default counts, as before.
+        self.assertIn("suyu", bs.configgen_cores(self.DEFAULTS, self.ES, set())[0])
+
+    def test_the_real_knulli_list_names_no_suyu(self):
+        import yaml
+
+        config = yaml.safe_load((REPO_ROOT / "platforms" / "knulli.yml").read_text())
+        self.assertNotIn("suyu", config["cores"])
+        self.assertNotIn("suyu", config.get("standalone_cores") or [])
+
+
 if __name__ == "__main__":
     unittest.main()