diff --git a/platforms/knulli.yml b/platforms/knulli.yml index 2ac54eaff..846328981 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -186,7 +186,6 @@ cores: - superbroswar - supermodel - supermodel-legacy -- suyu - swanstation - taradino - tgbdual @@ -4970,7 +4969,6 @@ standalone_cores: - stella - supermodel - supermodel-legacy -- suyu - taradino - theforceengine - thextech @@ -6143,14 +6141,6 @@ launchers: depends_on_option: not system.name == 'ti99' source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:431-432,\ \ 435" -- emulator: suyu - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/saves/switch - passes_to: environment XDG_DATA_HOME - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/suyu/suyuGenerator.py:41" - emulator: citron build: standalone kind: passes_path diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index f1e3fcdcc..399d8bd43 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -14,6 +14,7 @@ import re import sys import urllib.error import urllib.request +from collections.abc import Callable, Iterable from pathlib import Path @@ -168,7 +169,8 @@ def es_systems_libretro_systems(text: str, dropped: set[str]) -> dict[str, list[ def es_systems_by_system( - es_systems_text: str, defaults_text: str, dropped: set[str] + es_systems_text: str, defaults_text: str, dropped: set[str], + built: Callable[[str], bool] | None = None, ) -> dict[str, list[str]]: """system -> the cores es_systems.yml offers it, in the file's order. @@ -195,6 +197,10 @@ def es_systems_by_system( default = defaults.get(system) if isinstance(default, dict): emulator, core = default.get("emulator"), default.get("core") + if (built is not None and emulator and emulator != "libretro" + and str(emulator) not in names and not built(str(emulator))): + # A default the build lacks (configgen_cores). + emulator = core = None if emulator and emulator != "libretro": names.append(str(emulator)) if core: @@ -472,7 +478,8 @@ def platform_systems( def configgen_cores( - defaults_text: str, es_systems_text: str, dropped: set[str] + defaults_text: str, es_systems_text: str, dropped: set[str], + built: Callable[[str], bool] | None = None, ) -> tuple[list[str], list[str], list[str]]: """Core names, standalone cores and dual-mode cores of a configgen build. @@ -481,9 +488,16 @@ def configgen_cores( dual-mode ones those es_systems.yml also offers under libretro (dolphin for GameCube). `dropped` names what es_systems.yml files under an emulator without being one. + + A default es_systems.yml does not offer names an emulator the build may + lack: KNULLI's defaults pick suyu for the Switch, and no package of its + tree, its batocera submodule or its buildroot builds suyu. `built` says + whether the build has a standalone emulator (packages_and_programs); a + default it answers no for is not run, nor its core. """ data = yaml_load_upstream(defaults_text) cores, standalone = es_systems_emulators(es_systems_text) + offered = set(cores) libretro = es_systems_libretro(es_systems_text) if not cores: raise RuntimeError("es_systems.yml names no emulator") @@ -492,6 +506,9 @@ def configgen_cores( continue emulator = cfg.get("emulator", "") core = cfg.get("core", "") + if (built is not None and emulator and emulator != "libretro" + and emulator not in offered and not built(emulator)): + continue if core: cores.add(core) if emulator and emulator != "libretro": @@ -502,6 +519,57 @@ def configgen_cores( return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual) +BATOCERA_REPO = "batocera-linux/batocera.linux" + + +def package_names(paths: Iterable[str]) -> set[str]: + """The buildroot packages a tree defines: each folder holding a Config.in.""" + return { + path.rsplit("/", 2)[-2] + for path in paths + if path.endswith("/Config.in") and path.count("/") >= 1 + } + + +def repo_tree(repo: str, ref: str) -> list[str]: + """Every path of a GitHub repository at a revision.""" + url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1" + req = urllib.request.Request(url, headers=github_headers()) + with urllib.request.urlopen(req, timeout=60) as resp: + found = json.loads(resp.read().decode("utf-8")) + if found.get("truncated"): + raise RuntimeError(f"{url}: tree truncated") + return [entry["path"] for entry in found.get("tree", [])] + + +def submodule_revision(repo: str, path: str, ref: str) -> str: + """The commit a repository's submodule at `path` pins at a revision.""" + url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}" + req = urllib.request.Request(url, headers=github_headers()) + with urllib.request.urlopen(req, timeout=30) as resp: + found = json.loads(resp.read().decode("utf-8")) + if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"): + raise RuntimeError(f"{url}: {path} is not a submodule") + return str(found["sha"]) + + +def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]: + """Whether a build has a standalone emulator. + + It does when a package of that name exists, or when its launcher runs a + program the user puts in the BIOS folder (Batocera's lexaloffle runs + /userdata/bios/pico-8/pico8), which no package builds. + """ + supplied = { + name.strip() + for record in launchers + if record.get("passes_to") == "exec" + and str(record.get("search_dir") or "").startswith("/userdata/bios") + for name in str(record.get("emulator", "")).split(",") + } + return lambda emulator: emulator in packages or emulator in supplied + + class Scraper(BaseScraper): """Scraper for batocera-systems Python dict.""" @@ -509,12 +577,26 @@ class Scraper(BaseScraper): self.tag = fetch_stable_tag() super().__init__(url=SOURCE_URL.format(tag=self.tag)) - def _fetch_cores(self) -> tuple[list[str], list[str], list[str]]: - """Core names, standalone cores and dual-mode cores at the pinned tag.""" + def _fetch_cores( + self, launchers: list[dict] | None = None + ) -> tuple[list[str], list[str], list[str]]: + """Core names, standalone cores and dual-mode cores at the pinned tag. + + A default es_systems.yml does not offer counts when a package of + Batocera's tree or its buildroot builds it, or when its launcher + runs a program from the BIOS folder. + """ + built = None + if launchers is not None: + paths = repo_tree(BATOCERA_REPO, self.tag) + buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag) + paths += repo_tree("batocera-linux/buildroot", buildroot) + built = packages_and_programs(package_names(paths), launchers) return configgen_cores( self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)), self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores(), + built, ) def _fetch_text(self, url: str) -> str: @@ -548,7 +630,10 @@ class Scraper(BaseScraper): SYSTEM_SLUG_MAP, ) - cores, standalone, dual = self._fetch_cores() + launchers = extract_launchers( + lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}") + ) + cores, standalone, dual = self._fetch_cores(launchers) libretro_systems = es_systems_libretro_systems( self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores() ) @@ -570,9 +655,7 @@ class Scraper(BaseScraper): result["dual_mode_cores"] = dual # The systems each libretro core runs for: its save folders. result["libretro_systems"] = libretro_systems - result["launchers"] = extract_launchers( - lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}") - ) + result["launchers"] = launchers return result diff --git a/scripts/scraper/knulli_launchers.py b/scripts/scraper/knulli_launchers.py index be94eb13a..dbf5354b4 100644 --- a/scripts/scraper/knulli_launchers.py +++ b/scripts/scraper/knulli_launchers.py @@ -48,7 +48,6 @@ try: ) from .configgen_launchers import ( Launcher, - _calls, LauncherShapeError, Source, _assigned_name, @@ -74,7 +73,6 @@ except ImportError: ) from configgen_launchers import ( # type: ignore[no-redef] Launcher, - _calls, LauncherShapeError, Source, _assigned_name, @@ -1062,39 +1060,6 @@ EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = ( ) -# Generators whose command sets the emulator's XDG data folder, the parent -# of every user folder a yuzu-family emulator derives. -DATA_HOME_GENERATORS = ("suyu.suyuGenerator",) - - -def data_homes(cg: Configgen) -> list[Launcher]: - """Command(..., env={"XDG_DATA_HOME": }) in the generators above. - - The path need not lie in /userdata/bios: KNULLI's suyu keeps its data - in /userdata/saves/switch, which tells where its keys are not. - """ - launchers = [] - for module in DATA_HOME_GENERATORS: - src = cg.generator(module) - found = [ - (_path_value(item, cg.paths), key) - for call in _calls(src.tree, attr="Command") - for keyword in call.keywords - if keyword.arg == "env" and isinstance(keyword.value, ast.Dict) - for key, item in zip(keyword.value.keys, keyword.value.values) - if isinstance(key, ast.Constant) and key.value == "XDG_DATA_HOME" - ] - if not found or found[0][0] is None: - src.fail("the command environment sets no XDG_DATA_HOME path") - value, node = found[0] - launchers.append(Launcher( - emulator=cg.emulator(module), build="standalone", kind="passes_path", names=[], - search_dir=value, passes_to="environment XDG_DATA_HOME", - source_ref=src.ref([(node.lineno, node.end_lineno)]), - )) - return launchers - - # KNULLI builds Batocera's emulator packages from this submodule, at the # batocera.linux commit it pins: Citron's path patch is there. BATOCERA_SUBMODULE = "batocera" @@ -1114,7 +1079,6 @@ def extract_launchers( launchers += boot_disks(cg, read) for spec in GENERIC_SITES: launchers += bios_sites(cg, spec) - launchers += data_homes(cg) if batocera is not None: prefix = f"{BATOCERA_SUBMODULE}/" launchers += patched_paths( diff --git a/scripts/scraper/knulli_scraper.py b/scripts/scraper/knulli_scraper.py index abae8d987..80068ebd9 100644 --- a/scripts/scraper/knulli_scraper.py +++ b/scripts/scraper/knulli_scraper.py @@ -41,8 +41,12 @@ from .batocera_scraper import ( es_systems_by_system, es_systems_libretro_systems, extract_systems_dict, + package_names, + packages_and_programs, platform_systems, + repo_tree, requirements_from_systems, + submodule_revision, ) from .es_emulators import attach_system_cores from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers @@ -123,11 +127,27 @@ class Scraper(BaseScraper): defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag)) es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag)) dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME) - cores, standalone, dual = configgen_cores(defaults, es_systems, dropped) + revision = self.submodule_revision() + launchers = extract_launchers( + lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path), + lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path), + ) + # The packages KNULLI builds come from its own tree, its batocera + # submodule and its buildroot submodule: a default no package of + # them builds (suyu) is not run. + paths = repo_tree(REPO, self.tag) + paths += repo_tree("batocera-linux/batocera.linux", revision) + paths += repo_tree( + "knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag) + ) + built = packages_and_programs(package_names(paths), launchers) + cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built) # Each system's own list (psx, es_systems.yml:640-653: libretro # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and # duckstation-legacy). - attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped)) + attach_system_cores( + systems, es_systems_by_system(es_systems, defaults, dropped, built) + ) result = { "platform": "KNULLI", "version": self.tag, @@ -146,11 +166,7 @@ class Scraper(BaseScraper): # configgen hands a libretro core /userdata/saves/, as # Batocera's does: the systems each core runs for are its save folders. result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped) - revision = self.submodule_revision() - result["launchers"] = extract_launchers( - lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path), - lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path), - ) + result["launchers"] = launchers return result diff --git a/tests/test_launcher_paths.py b/tests/test_launcher_paths.py index 2f0cb8f1e..0b978bb76 100644 --- a/tests/test_launcher_paths.py +++ b/tests/test_launcher_paths.py @@ -21,7 +21,6 @@ REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT / "scripts")) from launchers import handed_directories, standalone_bindings # noqa: E402 -from scraper import knulli_launchers as kl # noqa: E402 from scraper import retrobat_launchers as rl # noqa: E402 from scraper.batocera_launchers import ( # noqa: E402 GENERATORS, @@ -128,35 +127,27 @@ class AFolderTheGeneratorEnters(unittest.TestCase): self.assertEqual(found, {"": ("outside", "")}) -class ADataFolderTheEnvironmentSets(unittest.TestCase): - SUYU = f"{kl.GENERATORS}/suyu/suyuGenerator.py" - TEXT = textwrap.dedent("""\ - from ...batoceraPaths import CACHE, CONFIGS, SAVES - - class SuyuGenerator: - def generate(self, system, rom): - commandArray = ["/usr/bin/suyu", "-f", "-g", rom] - return Command.Command(array=commandArray, env={ - "XDG_CONFIG_HOME": CONFIGS, - "XDG_DATA_HOME": SAVES / "switch", - }) - """) - - def test_the_data_folder_is_a_record_and_binds_outside(self): - files = {f"{kl.CONFIGGEN}/batoceraPaths.py": PATHS, - f"{kl.GENERATORS}/importer.py": IMPORTER_TEXT, self.SUYU: self.TEXT} - (record,) = records(kl.data_homes(kl.configgen(files.__getitem__))) - self.assertEqual( - (record["emulator"], record["search_dir"], record["passes_to"], record["source_ref"]), - ("suyu", "/userdata/saves/switch", "environment XDG_DATA_HOME", f"{self.SUYU}:8"), - ) - config = {"base_destination": "bios", "standalone_cores": ["suyu"], "launchers": [record]} +class AVariableADeclaredFolderIsSpelledFrom(unittest.TestCase): + def test_a_data_folder_outside_the_tree_binds_outside(self): + record = {"emulator": "emu", "build": "standalone", "kind": "passes_path", + "names": [], "search_dir": "/userdata/saves/switch", + "passes_to": "environment XDG_DATA_HOME", "source_ref": "x.py:1"} + config = {"base_destination": "bios", "standalone_cores": ["emu"], "launchers": [record]} found = standalone_bindings( - {"suyu"}, ["$XDG_DATA_HOME/suyu", "SuyuPath::KeysDir = keys"], - handed_directories(config), + {"emu"}, ["$XDG_DATA_HOME/emu"], handed_directories(config) ) self.assertEqual(found, {"": ("outside", "")}) + def test_a_data_folder_in_the_tree_binds_below_it(self): + record = {"emulator": "emu", "build": "standalone", "kind": "passes_path", + "names": [], "search_dir": "/userdata/bios/data", + "passes_to": "environment XDG_DATA_HOME", "source_ref": "x.py:1"} + config = {"base_destination": "bios", "standalone_cores": ["emu"], "launchers": [record]} + found = standalone_bindings( + {"emu"}, ["$XDG_DATA_HOME/emu"], handed_directories(config) + ) + self.assertEqual(found, {"": ("", "data/emu")}) + class APortableUserFolder(unittest.TestCase): CITRON = """ @@ -213,5 +204,54 @@ class APortableUserFolder(unittest.TestCase): self.assertEqual(found, {"": ("emulators", "citron/user")}) + +class ADefaultTheBuildLacksIsNotRun(unittest.TestCase): + """KNULLI's configgen-defaults.yml picks suyu for the Switch, which its + es_systems.yml does not offer and no package builds.""" + + ES = textwrap.dedent("""\ + switch: + emulators: + citron: + citron: { requireAnyOf: [BR2_PACKAGE_CITRON] } + voxatron: + emulators: {} + """) + DEFAULTS = textwrap.dedent("""\ + switch: + emulator: suyu + core: suyu + voxatron: + emulator: lexaloffle + core: vox_official + kodi: + emulator: kodi + core: kodi + """) + LAUNCHERS = [{"emulator": "lexaloffle", "kind": "fixed_name", "passes_to": "exec", + "search_dir": "/userdata/bios"}] + + def test_only_what_a_package_or_the_user_supplies_is_run(self): + from scraper import batocera_scraper as bs + + paths = ["package/batocera/emulators/citron/Config.in", "package/kodi/Config.in"] + built = bs.packages_and_programs(bs.package_names(paths), self.LAUNCHERS) + cores, standalone, _dual = bs.configgen_cores(self.DEFAULTS, self.ES, set(), built) + self.assertEqual(sorted(cores), ["citron", "kodi", "vox_official"]) + self.assertIn("lexaloffle", standalone) + self.assertNotIn("suyu", standalone) + by_system = bs.es_systems_by_system(self.ES, self.DEFAULTS, set(), built) + self.assertEqual(by_system["switch"], ["citron"]) + # Without the packages, every default counts, as before. + self.assertIn("suyu", bs.configgen_cores(self.DEFAULTS, self.ES, set())[0]) + + def test_the_real_knulli_list_names_no_suyu(self): + import yaml + + config = yaml.safe_load((REPO_ROOT / "platforms" / "knulli.yml").read_text()) + self.assertNotIn("suyu", config["cores"]) + self.assertNotIn("suyu", config.get("standalone_cores") or []) + + if __name__ == "__main__": unittest.main()