From f362a54b12f2bc57a3e1bb72b1c1bac87babdda4 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:26:09 +0100 Subject: [PATCH] fix: list only what a knulli board builds KNULLI's es_systems.yml is Batocera's model: it lists every emulator its configgen knows, and its Config.in selects many of them only on x86_64 boards, which KNULLI has none of. The platform list now holds what at least one board selects, read by the targets scraper, plus a program the launcher runs from the BIOS folder (lexaloffle). 59 cores and 44 standalone emulators leave the list, with their launcher records. The targets scraper follows Config.in and every file it sources, so a select a package declares itself counts (Batocera's mesa3d-rpi4 gives the Pi 4 OpenGL). A sourced file absent at the pin is named and read as empty: KNULLI 20260511 sources package/utils/vtree/Config.in, added in June. --- platforms/knulli.yml | 323 ------------------ scripts/scraper/batocera_scraper.py | 118 +++---- scripts/scraper/knulli_scraper.py | 21 +- .../targets/batocera_targets_scraper.py | 139 +++++++- .../scraper/targets/knulli_targets_scraper.py | 3 + tests/test_elec_scrapers.py | 10 +- tests/test_launcher_paths.py | 73 +++- tests/test_standalone_links.py | 6 +- tests/test_target_scrapers.py | 26 +- 9 files changed, 284 insertions(+), 435 deletions(-) diff --git a/platforms/knulli.yml b/platforms/knulli.yml index d2aa23488..c6923e911 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -12,55 +12,36 @@ cores: - advanced_drastic - aethersx2 - amiberry -- applewin - arduous - atari800 - azahar - beetle-saturn - bennugd -- bigpemu -- blastem - bluemsx -- boom3 - bsnes -- bsnes_hd - bstone - cannonball - cap32 -- catacombgl - cdogs - cemu - cgenius -- citra -- citron - clk - corsixth -- demul -- desmume - devilutionx - dhewm3 -- dice - dolphin -- dolphin_triforce -- dosbox - dosbox-x - dosbox_pure - dosbox_staging - dosboxx -- drastic -- duckstation -- duckstation-legacy - dxx-rebirth - easyrpg -- ecwolf - eduke32 -- eka2l1 - emuscv - etlegacy - fake08 - fallout1-ce - fallout2-ce -- fba2x - fbalpha - fbneo - fceumm @@ -68,20 +49,15 @@ cores: - flycast-xtreme - flycastvl - fmsx -- fpinball - freechaf - freeintv -- freej2me -- fsuae - fury - fuse - gambatte - gearsystem - genesisplusgx -- genesisplusgx-expanded - genesisplusgx-wide - geolith -- gong - gpsp - gsplus - gw @@ -92,15 +68,9 @@ cores: - hcl - holani - hurrican -- hypseus-singe -- ikemen -- imame4all - ioquake3 - iortcw -- jazz2-native -- kronos - lightspark -- lindbergh-loader - lowresnx - lutro - mame @@ -117,7 +87,6 @@ cores: - mesen-s - mgba - minivmac -- model2emu - moonlight - mpv - mrboom @@ -134,21 +103,14 @@ cores: - openbor7142 - openbor7530 - openjazz -- openjk -- openjkdf2 -- openlara -- openmohaa -- openmsx - opera - parallel_n64 - pce - pce_fast - pcfx -- pcsx2 - pcsx_rearmed - picodrive - play -- pocketsnes - pokemini - potator - ppsspp @@ -158,20 +120,15 @@ cores: - puae2021 - px68k - pygame -- pyxel - quasi88 - raze -- redream - reminiscence - rpcs3 -- ruffle -- ryujinx - samcoupe - same_cdi - sameduck - scummvm - sdlpop -- shadps4 - smsplus - snes9x - snes9x_next @@ -180,31 +137,21 @@ cores: - sonic2013 - sonic3-air - soniccd -- squirreljme - stella -- stella2014 - superbroswar -- supermodel -- supermodel-legacy - swanstation - taradino - tgbdual -- theforceengine - theodore - thextech - tic80 -- tr1x -- tr2x -- tsugaru - tyrian - tyrquake - uae4arm -- uqm - uzem - vb - vba-m - vecx -- vemulator - vice - vice_x128 - vice_x64 @@ -221,24 +168,18 @@ cores: - vkquake2 - vkquake3 - vox_official -- vpinball -- wasm4 - x1 - x128 - x16emu - x64 - x64dtv -- xash3d_fwgs - xemu -- xenia -- xenia-canary - xpet - xplus4 - xrick - xscpu64 - xvic - yabasanshiro -- ymir - zc210 systems: atari-400-800: @@ -4879,61 +4820,39 @@ standalone_cores: - advanced_drastic - aethersx2 - amiberry -- applewin - azahar -- bigpemu - bstone - cannonball -- catacombgl - cdogs - cemu - cgenius -- citra -- citron - clk - corsixth -- demul - devilutionx - dhewm3 - dolphin -- dolphin_triforce -- dosbox - dosbox-x - dosbox_staging - dosboxx -- drastic -- duckstation -- duckstation-legacy - dxx-rebirth - easyrpg -- ecwolf - eduke32 -- eka2l1 - etlegacy - fallout1-ce - fallout2-ce -- fba2x - flycast -- fpinball -- fsuae - fury - gsplus - gzdoom - hatari - hcl - hurrican -- hypseus-singe -- ikemen - ioquake3 - iortcw -- jazz2-native -- kodi - lexaloffle - lightspark -- lindbergh-loader - mame - melonds -- model2emu - moonlight - mpv - mupen64plus @@ -4943,71 +4862,46 @@ standalone_cores: - openbor7142 - openbor7530 - openjazz -- openjk -- openjkdf2 -- openmohaa -- openmsx -- pcsx2 - play - ppsspp - pygame -- pyxel - raze -- redream - rpcs3 -- ruffle -- ryujinx - samcoupe - scummvm - sdlpop -- shadps4 - solarus - sonic-mania - sonic2013 - sonic3-air - soniccd - stella -- supermodel -- supermodel-legacy - taradino -- theforceengine - thextech -- tr1x -- tr2x -- tsugaru - tyrian -- uqm - vice - vita3k - vkquake - vkquake2 - vkquake3 - vox_official -- vpinball - x128 - x16emu - x64 - x64dtv -- xash3d_fwgs - xemu -- xenia -- xenia-canary - xpet - xplus4 - xscpu64 - xvic - yabasanshiro -- ymir dual_mode_cores: -- applewin -- citra - dolphin - easyrpg - flycast - hatari - mame - melonds -- pcsx2 - play - ppsspp - scummvm @@ -5393,129 +5287,6 @@ libretro_systems: zc210: - zc210 launchers: -- emulator: duckstation - build: standalone - kind: names_searched - slot: NTSCU - names: - - scph101.bin - - scph1001.bin - - scph5501.bin - - scph7001.bin - - scph7501.bin - case_insensitive: true - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCU" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ - \ 589-611" -- emulator: duckstation - build: standalone - kind: names_searched - slot: PAL - names: - - scph1002.bin - - scph5502.bin - - scph5552.bin - - scph7002.bin - - scph7502.bin - - scph9002.bin - - scph102a.bin - - scph102b.bin - case_insensitive: true - search_dir: /userdata/bios - passes_to: "[BIOS] PathPAL" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ - \ 589-611" -- emulator: duckstation - build: standalone - kind: names_searched - slot: NTSCJ - names: - - scph100.bin - - scph1000.bin - - scph3000.bin - - scph3500.bin - - scph5500.bin - - scph7000.bin - - scph7003.bin - case_insensitive: true - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCJ" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ - \ 589-611" -- emulator: duckstation - build: standalone - kind: names_searched - slot: Uni - names: - - psxonpsp660.bin - - ps1_rom.bin - case_insensitive: true - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCU, PathPAL, PathNTSCJ" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ - \ 589-611" -- emulator: duckstation-legacy - build: standalone - kind: names_searched - slot: NTSCU - names: - - scph101.bin - - scph1001.bin - - scph5501.bin - - scph7001.bin - - scph7501.bin - case_insensitive: false - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCU" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" -- emulator: duckstation-legacy - build: standalone - kind: names_searched - slot: PAL - names: - - scph1002.bin - - scph5502.bin - - scph5552.bin - - scph7002.bin - - scph7502.bin - - scph9002.bin - - scph102a.bin - - scph102b.bin - case_insensitive: false - search_dir: /userdata/bios - passes_to: "[BIOS] PathPAL" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" -- emulator: duckstation-legacy - build: standalone - kind: names_searched - slot: NTSCJ - names: - - scph100.bin - - scph1000.bin - - scph3000.bin - - scph3500.bin - - scph5500.bin - - scph7000.bin - - scph7003.bin - case_insensitive: false - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCJ" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" - emulator: hatari build: standalone kind: names_searched @@ -5683,33 +5454,6 @@ launchers: passes_to: environment FLYCAST_BIOS_PATH fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/flycast/flycastGenerator.py:239" -- emulator: shadps4 - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/system/configs/shadps4 - passes_to: working directory of shadps4Generator.py - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/shadps4/shadps4Generator.py:33,\ - \ 164" -- emulator: openmsx - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/Machines - passes_to: "written: filepool add -path /userdata/bios/Machines -types system_rom\ - \ -position 1" - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/openmsx/openmsxGenerator.py:95" -- emulator: openmsx - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/openmsx - passes_to: "written: filepool add -path /userdata/bios/openmsx -types system_rom\ - \ -position 2" - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/openmsx/openmsxGenerator.py:96" - emulator: amiberry build: standalone kind: passes_path @@ -5765,15 +5509,6 @@ launchers: depends_on_option: neocd_bios source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroOptions.py:2558-2559,\ \ 2566-2569" -- emulator: pcsx2 - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/ps2 - passes_to: "inis/PCSX2.ini [Folders] Bios = ../../../bios/ps2, relative to /userdata/system/configs/PCSX2" - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/pcsx2/pcsx2Generator.py:92,\ - \ 176, 213" - emulator: aethersx2 build: standalone kind: passes_path @@ -5919,29 +5654,6 @@ launchers: fails_when_missing: false depends_on_option: "Rpcs3Generator.getFirmwareVersion() is None" source_ref: "package/system/knulli-configgen/configgen/configgen/generators/rpcs3/rpcs3Generator.py:355-358" -- emulator: ryujinx - build: standalone - kind: copies - names: - - switch/prod.keys - case_insensitive: false - search_dir: /userdata/bios - passes_to: /userdata/system/configs/Ryujinx/system/prod.keys - fails_when_missing: false - depends_on_option: "ryujinxKeys.exists()" - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/ryujinx/ryujinxGenerator.py:107-109" -- emulator: pcsx2 - build: standalone - kind: copies - names: - - patches.zip - case_insensitive: false - search_dir: /usr/share/knulli/datainit/bios/ps2 - passes_to: /userdata/bios/ps2/patches.zip - fails_when_missing: false - depends_on_option: "not pcsx2Patches.exists()" - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/pcsx2/pcsx2Generator.py:78,\ - \ 121-122" - emulator: aethersx2 build: standalone kind: copies @@ -5962,14 +5674,6 @@ launchers: passes_to: g_cfg_rom_path fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/gsplus/gsplusGenerator.py:106" -- emulator: tsugaru - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/fmtowns - passes_to: command line argument 1 - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/tsugaru/tsugaruGenerator.py:18-19" - emulator: scummvm build: standalone kind: passes_path @@ -5978,14 +5682,6 @@ launchers: passes_to: command line --extrapath fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/scummvm/scummvmGenerator.py:117-123" -- emulator: fsuae - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/amiga - passes_to: command line --kickstarts_dir - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/fsuae/fsuaeGenerator.py:63-69" - emulator: mame build: standalone kind: passes_path @@ -6039,16 +5735,6 @@ launchers: passes_to: command line -crosshairpath fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:180" -- emulator: fpinball - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios - passes_to: environment W_CACHE - fails_when_missing: false - depends_on_option: "not wsh57_done.exists()" - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/fpinball/fpinballGenerator.py:37,\ - \ 39-43" - emulator: retroarch build: libretro kind: passes_path @@ -6142,15 +5828,6 @@ launchers: depends_on_option: not system.name == 'ti99' source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:431-432,\ \ 435" -- emulator: citron - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/switch - passes_to: "CitronPath::KeysDir in src/common/fs/path_util.cpp returns it, in place\ - \ of citron_path / KEYS_DIR" - fails_when_missing: false - source_ref: "batocera/package/batocera/emulators/citron/002-adjust-paths.patch:9-10" configs_root: path: /userdata/system/configs from_bios: ../system/configs diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index 8ef10db16..84a22c2b3 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -14,7 +14,7 @@ import re import sys import urllib.error import urllib.request -from collections.abc import Callable, Iterable +from collections.abc import Callable from pathlib import Path @@ -30,6 +30,8 @@ from .base_scraper import ( requirement_entry, ) from .batocera_launchers import Configgen, configs_root, extract_launchers +from .targets.batocera_targets_scraper import BuildSelection +from .targets.batocera_targets_scraper import Scraper as BoardTargets PLATFORM_NAME = "batocera" @@ -489,76 +491,51 @@ def configgen_cores( for GameCube). `dropped` names what es_systems.yml files under an emulator without being one. - A default es_systems.yml does not offer names an emulator the build may - lack: KNULLI's defaults pick suyu for the Switch, and no package of its - tree, its batocera submodule or its buildroot builds suyu. `built` says - whether the build has a standalone emulator (packages_and_programs); a - default it answers no for is not run, nor its core. + `built` says whether some board builds a name (boards_build). What it + answers no for is not run: KNULLI's es_systems lists citron and shadps4, + which no KNULLI board selects, and its defaults pick suyu, which no + package builds. A default's core follows a standalone emulator the + build runs (vox_official is lexaloffle's Voxatron build). """ data = yaml_load_upstream(defaults_text) cores, standalone = es_systems_emulators(es_systems_text) - offered = set(cores) libretro = es_systems_libretro(es_systems_text) if not cores: raise RuntimeError("es_systems.yml names no emulator") + if built is not None: + cores = {name for name in cores if built(name)} + standalone &= cores for system, cfg in data.items(): if system == "default" or not isinstance(cfg, dict): continue emulator = cfg.get("emulator", "") core = cfg.get("core", "") - if (built is not None and emulator and emulator != "libretro" - and emulator not in offered and not built(emulator)): - continue - if core: - cores.add(core) if emulator and emulator != "libretro": + if built is not None and not built(emulator): + continue standalone.add(emulator) if core and core != emulator: standalone.add(core) + elif built is not None and core and not built(core): + continue + if core: + cores.add(core) dual = (standalone & libretro) - dropped return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual) -BATOCERA_REPO = "batocera-linux/batocera.linux" +def boards_build(selection: BuildSelection, launchers: list[dict]) -> Callable[[str], bool]: + """Whether some board of a build runs an emulator. - -def package_names(paths: Iterable[str]) -> set[str]: - """The buildroot packages a tree defines: each folder holding a Config.in.""" - return { - path.rsplit("/", 2)[-2] - for path in paths - if path.endswith("/Config.in") and path.count("/") >= 1 - } - - -def repo_tree(repo: str, ref: str) -> list[str]: - """Every path of a GitHub repository at a revision.""" - url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1" - req = urllib.request.Request(url, headers=github_headers()) - with urllib.request.urlopen(req, timeout=60) as resp: - found = json.loads(resp.read().decode("utf-8")) - if found.get("truncated"): - raise RuntimeError(f"{url}: tree truncated") - return [entry["path"] for entry in found.get("tree", [])] - - -def submodule_revision(repo: str, path: str, ref: str) -> str: - """The commit a repository's submodule at `path` pins at a revision.""" - url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}" - req = urllib.request.Request(url, headers=github_headers()) - with urllib.request.urlopen(req, timeout=30) as resp: - found = json.loads(resp.read().decode("utf-8")) - if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"): - raise RuntimeError(f"{url}: {path} is not a submodule") - return str(found["sha"]) - - -def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]: - """Whether a build has a standalone emulator. - - It does when a package of that name exists, or when its launcher runs a - program the user puts in the BIOS folder (Batocera's lexaloffle runs - /userdata/bios/pico-8/pico8), which no package builds. + es_systems.yml ties each name to the packages that build it + (requireAnyOf), and a name it lists is built when a board selects one of + them: KNULLI lists citron and shadps4, and its Config.in selects them + only on x86_64 boards, which it has none of. A name only + configgen-defaults.yml gives is built when a board selects the package + of that name: the symbol its Config.in defines, else the one Buildroot + derives from the folder (BR2_PACKAGE_, package/pkg-generic.mk). A + program the launcher runs from the BIOS folder (lexaloffle, + /userdata/bios/pico-8/pico8) needs no package. """ supplied = { name.strip() @@ -567,7 +544,27 @@ def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable and str(record.get("search_dir") or "").startswith("/userdata/bios") for name in str(record.get("emulator", "")).split(",") } - return lambda emulator: emulator in packages or emulator in supplied + + def built(name: str) -> bool: + if name in selection.emulators or name in supplied: + return True + if name in selection.listed: + return False + symbol = selection.packages.get(name) or ( + "BR2_PACKAGE_" + name.upper().replace("-", "_")) + return symbol in selection.symbols + + return built + + +def records_run(launchers: list[dict], built: Callable[[str], bool]) -> list[dict]: + """The launcher records of emulators the build runs. + + configgen keeps the generators of emulators a build does not select + (KNULLI's citron, pcsx2, shadps4): what they read is never read on the + device, so the records do not describe the platform. + """ + return [r for r in launchers if any(built(n) for n in str(r["emulator"]).split(", "))] class Scraper(BaseScraper): @@ -578,20 +575,13 @@ class Scraper(BaseScraper): super().__init__(url=SOURCE_URL.format(tag=self.tag)) def _fetch_cores( - self, launchers: list[dict] | None = None + self, built: Callable[[str], bool] | None = None ) -> tuple[list[str], list[str], list[str]]: """Core names, standalone cores and dual-mode cores at the pinned tag. - A default es_systems.yml does not offer counts when a package of - Batocera's tree or its buildroot builds it, or when its launcher - runs a program from the BIOS folder. + With `built` (boards_build), a name counts when some board builds + it or its launcher runs a program from the BIOS folder. """ - built = None - if launchers is not None: - paths = repo_tree(BATOCERA_REPO, self.tag) - buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag) - paths += repo_tree("batocera-linux/buildroot", buildroot) - built = packages_and_programs(package_names(paths), launchers) return configgen_cores( self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)), self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), @@ -633,7 +623,9 @@ class Scraper(BaseScraper): launchers = extract_launchers( lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}") ) - cores, standalone, dual = self._fetch_cores(launchers) + built = boards_build(BoardTargets().build_selection(self.tag), launchers) + launchers = records_run(launchers, built) + cores, standalone, dual = self._fetch_cores(built) libretro_systems = es_systems_libretro_systems( self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores() ) diff --git a/scripts/scraper/knulli_scraper.py b/scripts/scraper/knulli_scraper.py index 763c25ec4..f94f56704 100644 --- a/scripts/scraper/knulli_scraper.py +++ b/scripts/scraper/knulli_scraper.py @@ -37,18 +37,17 @@ from .batocera_scraper import ( ) from .batocera_scraper import ( _not_cores, + boards_build, configgen_cores, es_systems_by_system, es_systems_libretro_systems, extract_systems_dict, - package_names, - packages_and_programs, platform_systems, - repo_tree, + records_run, requirements_from_systems, - submodule_revision, ) from .es_emulators import attach_system_cores +from .targets.knulli_targets_scraper import Scraper as BoardTargets from .batocera_launchers import configs_root from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers from .knulli_launchers import configgen as knulli_configgen @@ -134,15 +133,11 @@ class Scraper(BaseScraper): lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path), lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path), ) - # The packages KNULLI builds come from its own tree, its batocera - # submodule and its buildroot submodule: a default no package of - # them builds (suyu) is not run. - paths = repo_tree(REPO, self.tag) - paths += repo_tree("batocera-linux/batocera.linux", revision) - paths += repo_tree( - "knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag) - ) - built = packages_and_programs(package_names(paths), launchers) + # What runs is what at least one board builds, read by the targets + # scraper from the Kconfig of KNULLI's tree and its batocera and + # buildroot submodules (boards_build). + built = boards_build(BoardTargets().build_selection(self.tag), launchers) + launchers = records_run(launchers, built) cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built) # Each system's own list (psx, es_systems.yml:640-653: libretro # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and diff --git a/scripts/scraper/targets/batocera_targets_scraper.py b/scripts/scraper/targets/batocera_targets_scraper.py index f13615ccb..ce272de03 100644 --- a/scripts/scraper/targets/batocera_targets_scraper.py +++ b/scripts/scraper/targets/batocera_targets_scraper.py @@ -3,6 +3,8 @@ Sources (batocera-linux/batocera.linux): - configs/batocera-*.board -- board definitions, each sets BR2_PACKAGE_BATOCERA_TARGET_* - package/batocera/core/batocera-system/Config.in -- select PACKAGE if CONDITION lines + - Config.in and every Kconfig file it sources -- the selects a package + declares itself (advanced_drastic selects drastic) - package/batocera/emulationstation/batocera-es-system/es_systems.yml -- emulator requireAnyOf flag mapping Sources (batocera-linux/buildroot, at the submodule commit the tag pins): @@ -19,6 +21,8 @@ import re import sys import urllib.error import urllib.request +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from datetime import datetime, timezone import yaml @@ -79,6 +83,16 @@ def _fetch(url: str, headers: dict | None = None) -> str: raise RuntimeError(f"cannot fetch {url}: {e}") from e +def _fetch_present(url: str) -> str | None: + """A file's text, or None when the server answers that it does not exist.""" + try: + return _fetch(url) + except RuntimeError as e: + if isinstance(e.__cause__, urllib.error.HTTPError) and e.__cause__.code == 404: + return None + raise + + def _fetch_json(url: str) -> list | dict: text = _fetch(url, headers=github_headers()) try: @@ -210,6 +224,7 @@ BUILDROOT_KCONFIG = ( ) _CONFIG_LINE_RE = re.compile(r"^\s*(?:menu)?config\s+(BR2_\w+)") +_SOURCE_RE = re.compile(r'^\s*source\s+"\$BR2_EXTERNAL_\w+_PATH/([^"]+)"') _OWNED_SELECT_RE = re.compile(r"^\s+select\s+(BR2_\w+)(?:\s+if\s+([^#]+))?") _BOARD_FLAG_RE = re.compile(r"^(BR2_\w+)=y\s*$", re.MULTILINE) _BOARD_INCLUDE_RE = re.compile(r"^(-?)include\s+(\S+)", re.MULTILINE) @@ -342,6 +357,43 @@ def _arch_from_flag(flag: str, board_flags: set[str] | frozenset[str] = frozense return "aarch64" +@dataclass(frozen=True) +class BuildSelection: + """What the boards of a build turn on, read once from its Kconfig. + + `targets` is each board's entry of the targets file. `emulators` holds + every es_systems name some board builds and `listed` every name + es_systems ties to a package, `symbols` every symbol some board ends up + with, and `packages` the config symbol each package folder of the tree + defines. + """ + + targets: dict[str, dict] + emulators: frozenset[str] + listed: frozenset[str] + symbols: frozenset[str] + packages: dict[str, str] + + +def _sourced(text: str) -> list[str]: + """The files a Kconfig file sources, relative to the external tree.""" + return [m.group(1) for line in text.splitlines() if (m := _SOURCE_RE.match(line))] + + +def _package_symbols(files: dict[str, str]) -> dict[str, str]: + """package folder -> the first config symbol its Config.in defines.""" + found: dict[str, str] = {} + for path, text in files.items(): + parts = path.split("/") + if len(parts) < 3 or parts[-1] != "Config.in" or "package" not in parts: + continue + config = next((m.group(1) for line in text.splitlines() + if (m := _CONFIG_LINE_RE.match(line))), None) + if config: + found.setdefault(parts[-2], config) + return found + + class Scraper(BaseTargetScraper): """Cross-references Batocera boards, Config.in, and es_systems to build target lists. @@ -357,6 +409,9 @@ class Scraper(BaseTargetScraper): ES_SYSTEMS_PATH = "package/batocera/emulationstation/batocera-es-system/es_systems.yml" SYSTEM_SYMBOL = "BR2_PACKAGE_BATOCERA_SYSTEM" BUILDROOT_REPO = "batocera-linux/buildroot" + # Path prefix -> repository of a submodule the root Config.in sources + # from: KNULLI reads Batocera's packages as batocera/package/... + SUBMODULES: dict[str, str] = {} def __init__(self, url: str = "") -> None: super().__init__(url=url or f"https://github.com/{self.REPO}") @@ -411,6 +466,45 @@ class Scraper(BaseTargetScraper): pending += [(inc, bool(dash)) for dash, inc in _BOARD_INCLUDE_RE.findall(text)] return target, flags + def _kconfig_files(self) -> dict[str, str]: + """The root Config.in and every file it sources, recursively. + + The build reads these and nothing else: a package's own Config.in + can select another package (advanced_drastic selects drastic), and + such a select sits in no system Config.in. A sourced file absent at + the pin defines nothing and is named: KNULLI 20260511 sources + package/utils/vtree/Config.in, a package its tree gained later. + """ + bases: list[tuple[str, str]] = [] + for prefix, repo in self.SUBMODULES.items(): + pin = _fetch_json(self._api(prefix.rstrip("/"))) + sha = pin.get("sha") if isinstance(pin, dict) else None + if not sha: + raise RuntimeError(f"no {prefix.rstrip('/')} submodule commit at {self.tag}") + bases.append((prefix, f"https://raw.githubusercontent.com/{repo}/{sha}/")) + + def url(path: str) -> str: + for prefix, base in bases: + if path.startswith(prefix): + return base + path.removeprefix(prefix) + return self._raw(path) + + files: dict[str, str] = {} + absent: list[str] = [] + pending = ["Config.in"] + with ThreadPoolExecutor(max_workers=8) as pool: + while pending: + found = dict(zip(pending, pool.map(lambda path: _fetch_present(url(path)), pending))) + absent += [path for path, text in found.items() if text is None] + files.update((path, text or "") for path, text in found.items()) + pending = sorted({src for text in found.values() if text + for src in _sourced(text)} - set(files)) + if "Config.in" in absent or len(files) - len(absent) < 2: + raise RuntimeError(f"Config.in at {self.tag} sources no readable Kconfig file") + for path in absent: + print(f" sourced but absent at {self.tag}: {path}", file=sys.stderr) + return files + def _buildroot_rules(self) -> list[tuple[str, str, str]]: """Buildroot's selects for the symbols the tree tests, at its pin.""" pin = _fetch_json(self._api("buildroot")) @@ -423,26 +517,32 @@ class Scraper(BaseTargetScraper): rules += _parse_owned_selects(_fetch(f"{raw}/{path}")) return rules - def fetch_targets(self) -> dict: - """Build per-board emulator availability map.""" - self.tag = self.pinned_revision(self.PLATFORM) + def build_selection(self, tag: str | None = None) -> BuildSelection: + """Each board's selection, from the Kconfig the build reads. + + `tag` is the revision to read, by default the one the platform's + YAML was scraped at: a platform scraper moving to a new release + passes its own. + """ + self.tag = tag or self.pinned_revision(self.PLATFORM) print(" fetching board list...", file=sys.stderr) boards = self._list_boards() - print(" fetching Config.in...", file=sys.stderr) - config_in_url = self._raw(self.CONFIG_IN_PATH) - config_in_text = _fetch(config_in_url) + print(" fetching Kconfig files...", file=sys.stderr) + files = self._kconfig_files() + config_in_text = files.get(self.CONFIG_IN_PATH) if not config_in_text: - raise RuntimeError(f"empty Config.in from {config_in_url}") + raise RuntimeError(f"Config.in at {self.tag} does not source {self.CONFIG_IN_PATH}") meta_rules = _parse_meta_flags(config_in_text) - rules = _parse_owned_selects(config_in_text) + rules = [rule for text in files.values() for rule in _parse_owned_selects(text)] print( - f" parsed {len(meta_rules)} meta-flag rules, {len(rules)} select lines", + f" parsed {len(meta_rules)} meta-flag rules, {len(rules)} select lines " + f"in {len(files)} Kconfig files", file=sys.stderr, ) - if not rules: - raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {config_in_url}") + if not _parse_owned_selects(config_in_text): + raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {self.CONFIG_IN_PATH}") rules += self._buildroot_rules() print(" fetching es_systems.yml...", file=sys.stderr) @@ -459,9 +559,10 @@ class Scraper(BaseTargetScraper): raise RuntimeError(f"no package->emulator mappings parsed from {es_systems_url}") images_by_flag = _parse_images(config_in_text) if not images_by_flag: - raise RuntimeError(f"no target image list parsed from {config_in_url}") + raise RuntimeError(f"no target image list parsed from {self.CONFIG_IN_PATH}") targets: dict[str, dict] = {} + symbols: set[str] = set() for board_name in sorted(boards): target_key = board_name.removeprefix(self.BOARD_PREFIX).removesuffix(".board") print(f" processing {target_key}...", file=sys.stderr) @@ -476,6 +577,7 @@ class Scraper(BaseTargetScraper): if self.SYSTEM_SYMBOL not in selected_packages: print(f" {board_name} builds no system image", file=sys.stderr) continue + symbols |= selected_packages # Map selected packages to emulator names via es_systems.yml emulators: set[str] = set() @@ -496,11 +598,22 @@ class Scraper(BaseTargetScraper): file=sys.stderr, ) + return BuildSelection( + targets=targets, + emulators=frozenset(c for entry in targets.values() for c in entry["cores"]), + listed=frozenset(n for names in package_to_emulators.values() for n in names), + symbols=frozenset(symbols), + packages=_package_symbols(files), + ) + + def fetch_targets(self) -> dict: + """Build per-board emulator availability map.""" + selection = self.build_selection() return { "platform": self.PLATFORM, "source": f"{self.url}/tree/{self.tag}", "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "targets": targets, + "targets": selection.targets, } diff --git a/scripts/scraper/targets/knulli_targets_scraper.py b/scripts/scraper/targets/knulli_targets_scraper.py index e19f72f27..16de256e7 100644 --- a/scripts/scraper/targets/knulli_targets_scraper.py +++ b/scripts/scraper/targets/knulli_targets_scraper.py @@ -9,6 +9,8 @@ the tag platforms/knulli.yml names): the root Config.in in place of Batocera's batocera-system/Config.in - package/emulationstation/knulli-es-system/es_systems.yml -- emulator requireAnyOf flag mapping + - Config.in and what it sources, Batocera's packages read from the + batocera submodule at the commit the tag pins Sources (knulli-cfw/buildroot, at the submodule commit the tag pins): - arch/Config.in, package/mesa3d/Config.in, package/libglvnd/Config.in @@ -32,6 +34,7 @@ class Scraper(BatoceraScraper): ES_SYSTEMS_PATH = "package/emulationstation/knulli-es-system/es_systems.yml" SYSTEM_SYMBOL = "BR2_PACKAGE_KNULLI_SYSTEM" BUILDROOT_REPO = "knulli-cfw/buildroot" + SUBMODULES = {"batocera/": "batocera-linux/batocera.linux"} def main() -> None: diff --git a/tests/test_elec_scrapers.py b/tests/test_elec_scrapers.py index 0ee2f1950..da854d70c 100644 --- a/tests/test_elec_scrapers.py +++ b/tests/test_elec_scrapers.py @@ -284,7 +284,10 @@ class TheExportKeepsTheirShape(unittest.TestCase): class TheCommittedListsAgree(unittest.TestCase): """A system's cores are a part of the platform's: a name only the system - carries would escape every reader of the platform list.""" + carries would escape every reader of the platform list. A system keeps + the list its ES file writes, which says whose files its entries ask for, + so the one name it may carry beyond the platform's is one no device + image builds (EmuELEC's VVVVVV port engine, KNULLI's duckstation).""" def test_system_cores_are_platform_cores(self): import yaml @@ -292,10 +295,13 @@ class TheCommittedListsAgree(unittest.TestCase): for name in ("amberelec", "emuelec", "knulli"): with self.subTest(platform=name): config = yaml.safe_load((REPO_ROOT / "platforms" / f"{name}.yml").read_text()) + targets = yaml.safe_load( + (REPO_ROOT / "platforms" / "targets" / f"{name}.yml").read_text())["targets"] + built = {c for entry in targets.values() for c in entry["cores"]} platform = {str(c) for c in config["cores"]} listed = {str(c) for s in config["systems"].values() for c in s.get("cores", [])} self.assertTrue(listed, "no system carries its cores") - self.assertEqual(listed - platform, set()) + self.assertEqual(listed - platform - (listed - built), set()) if __name__ == "__main__": diff --git a/tests/test_launcher_paths.py b/tests/test_launcher_paths.py index 55564f564..1a37a4fba 100644 --- a/tests/test_launcher_paths.py +++ b/tests/test_launcher_paths.py @@ -231,15 +231,20 @@ class APortableUserFolder(unittest.TestCase): -class ADefaultTheBuildLacksIsNotRun(unittest.TestCase): - """KNULLI's configgen-defaults.yml picks suyu for the Switch, which its - es_systems.yml does not offer and no package builds.""" +class OnlyWhatABoardBuildsIsRun(unittest.TestCase): + """KNULLI's es_systems.yml lists citron and shadps4, which its Config.in + selects only on x86_64 boards (it has none), and its + configgen-defaults.yml picks suyu, which no package builds.""" ES = textwrap.dedent("""\ switch: emulators: citron: citron: { requireAnyOf: [BR2_PACKAGE_CITRON] } + gba: + emulators: + libretro: + mgba: { requireAnyOf: [BR2_PACKAGE_LIBRETRO_MGBA] } voxatron: emulators: {} """) @@ -253,31 +258,66 @@ class ADefaultTheBuildLacksIsNotRun(unittest.TestCase): kodi: emulator: kodi core: kodi + mpv: + emulator: mpv + core: mpv """) LAUNCHERS = [{"emulator": "lexaloffle", "kind": "fixed_name", "passes_to": "exec", "search_dir": "/userdata/bios"}] - def test_only_what_a_package_or_the_user_supplies_is_run(self): + def selection(self, symbols: set[str]): + from scraper.targets.batocera_targets_scraper import BuildSelection + + return BuildSelection( + targets={}, emulators=frozenset({"mgba"}), + listed=frozenset({"citron", "mgba"}), symbols=frozenset(symbols), + packages={"kodi": "BR2_PACKAGE_KODI"}, + ) + + def test_only_what_a_board_builds_or_the_user_supplies_is_run(self): from scraper import batocera_scraper as bs - paths = ["package/batocera/emulators/citron/Config.in", "package/kodi/Config.in"] - built = bs.packages_and_programs(bs.package_names(paths), self.LAUNCHERS) + # A board selects kodi's package and buildroot's mpv; citron is + # listed by es_systems, so the symbol of a folder of that name does + # not count for it. + built = bs.boards_build( + self.selection({"BR2_PACKAGE_KODI", "BR2_PACKAGE_MPV", "BR2_PACKAGE_CITRON_X"}), + self.LAUNCHERS, + ) cores, standalone, _dual = bs.configgen_cores(self.DEFAULTS, self.ES, set(), built) - self.assertEqual(sorted(cores), ["citron", "kodi", "vox_official"]) - self.assertIn("lexaloffle", standalone) - self.assertNotIn("suyu", standalone) + self.assertEqual(sorted(cores), ["kodi", "mgba", "mpv", "vox_official"]) + self.assertEqual(sorted(standalone), ["kodi", "lexaloffle", "mpv", "vox_official"]) + # Each system keeps the list es_systems.yml writes (whose files its + # entries ask for), less a default no package builds. by_system = bs.es_systems_by_system(self.ES, self.DEFAULTS, set(), built) self.assertEqual(by_system["switch"], ["citron"]) - # Without the packages, every default counts, as before. + self.assertEqual(by_system["voxatron"], ["lexaloffle", "vox_official"]) + self.assertEqual(by_system["gba"], ["mgba"]) + + def test_a_listed_name_is_built_only_through_its_packages(self): + from scraper import batocera_scraper as bs + + built = bs.boards_build(self.selection({"BR2_PACKAGE_CITRON"}), []) + self.assertFalse(built("citron")) + self.assertTrue(built("mgba")) + self.assertFalse(built("lexaloffle")) + # Without a selection, every name counts, as before. self.assertIn("suyu", bs.configgen_cores(self.DEFAULTS, self.ES, set())[0]) - def test_the_real_knulli_list_names_no_suyu(self): + def test_the_real_lists_name_only_what_a_board_builds(self): import yaml - config = yaml.safe_load((REPO_ROOT / "platforms" / "knulli.yml").read_text()) - self.assertNotIn("suyu", config["cores"]) - self.assertNotIn("suyu", config.get("standalone_cores") or []) - + for platform, absent in (("knulli", ("suyu", "citron", "shadps4", "pcsx2")),): + config = yaml.safe_load((REPO_ROOT / "platforms" / f"{platform}.yml").read_text()) + names = set(config["cores"]) | set(config.get("standalone_cores") or []) + targets = yaml.safe_load( + (REPO_ROOT / "platforms" / "targets" / f"{platform}.yml").read_text())["targets"] + boards = {c for entry in targets.values() for c in entry["cores"]} + with self.subTest(platform=platform): + self.assertFalse(names & set(absent)) + # What a board builds and the list once named is still there, + # and what no board builds is a program the user supplies. + self.assertLessEqual(names - boards, {"kodi", "lexaloffle", "mpv", "vox_official"}) class AUserFolderInTheConfigsFolder(unittest.TestCase): @@ -289,7 +329,8 @@ class AUserFolderInTheConfigsFolder(unittest.TestCase): from common import standalone_set profile = load_emulator_profiles(str(REPO_ROOT / "emulators"))["shadps4"] - for platform in ("batocera", "knulli"): + # KNULLI's generator does the same, but no KNULLI board builds shadps4. + for platform in ("batocera",): config = load_platform_config(platform, str(REPO_ROOT / "platforms")) ((view, _standalone),) = platform_layouts("shadps4", profile, standalone_set(config)) with self.subTest(platform=platform): diff --git a/tests/test_standalone_links.py b/tests/test_standalone_links.py index 1fcd5eedd..7638dfafa 100644 --- a/tests/test_standalone_links.py +++ b/tests/test_standalone_links.py @@ -548,12 +548,12 @@ class OpenmsxOnEachPlatform(unittest.TestCase): """openMSX finds system ROMs by SHA1 in share/systemroms below its user directory and in every filepool entry. RetroBat sets OPENMSX_HOME to bios/openmsx, RetroPie links share/systemroms of its user directory to - BIOS/openmsx, Batocera and KNULLI add bios/Machines and bios/openmsx to - the filepool and list their own files in bios/openmsx.""" + BIOS/openmsx, Batocera adds bios/Machines and bios/openmsx to the + filepool and lists its own files in bios/openmsx. KNULLI's generator + does the same, but no KNULLI board builds openmsx.""" EXPECTED = { "batocera": {"share/systemroms": ("", "openmsx")}, - "knulli": {"share/systemroms": ("", "openmsx")}, "retrobat": {"": ("", "openmsx")}, "retropie": {"share/systemroms": ("", "openmsx")}, } diff --git a/tests/test_target_scrapers.py b/tests/test_target_scrapers.py index d55033cff..81c389d5d 100644 --- a/tests/test_target_scrapers.py +++ b/tests/test_target_scrapers.py @@ -97,6 +97,9 @@ class FailedRequestsStopTheScrape(unittest.TestCase): _refuse() return _Body(b"BR2_PACKAGE_BATOCERA_TARGET_X86_64=y\n" b"BR2_PACKAGE_BATOCERA_SYSTEM=y\n") + if url.startswith("https://raw.githubusercontent.com/") and url.count("/") == 6: + return _Body(b'source "$BR2_EXTERNAL_BATOCERA_PATH/' + b'package/batocera/core/batocera-system/Config.in"\n') if url.endswith("Config.in"): return _Body(b"config BR2_PACKAGE_BATOCERA_SYSTEM\n" b"\tselect BR2_PACKAGE_MGBA\n\n" @@ -262,6 +265,15 @@ class KnulliPerBoard(unittest.TestCase): "BR2_arm=y\nBR2_PACKAGE_BATOCERA_TARGET_H700=y\n" ), "configs/knulli-h700_armhf_libs.packages": "BR2_PACKAGE_SDL2=y\n", + # The root Config.in sources KNULLI's system and Batocera's packages + # from the batocera submodule, where a package selects another. + "Config.in": ( + 'source "$BR2_EXTERNAL_KNULLI_PATH/package/system/knulli-system/Config.in"\n' + 'source "$BR2_EXTERNAL_KNULLI_PATH/batocera/package/batocera/emulators/' + 'ppsspp/Config.in"\n' + '#source "$BR2_EXTERNAL_KNULLI_PATH/package/utils/off/Config.in"\n' + 'source "$BR2_EXTERNAL_KNULLI_PATH/package/utils/vtree/Config.in"\n' + ), "package/system/knulli-system/Config.in": ( "if BR2_PACKAGE_BATOCERA_TARGET_H700 || \\\n" " BR2_PACKAGE_BATOCERA_TARGET_A133\n" @@ -282,6 +294,8 @@ class KnulliPerBoard(unittest.TestCase): " mgba: {requireAnyOf: [BR2_PACKAGE_LIBRETRO_MGBA]}\n" "psp:\n emulators:\n ppsspp:\n" " ppsspp: {requireAnyOf: [BR2_PACKAGE_PPSSPP]}\n" + "pspassets:\n emulators:\n libretro:\n" + " assets: {requireAnyOf: [BR2_PACKAGE_PPSSPP_ASSETS]}\n" ), } @@ -314,6 +328,13 @@ class KnulliPerBoard(unittest.TestCase): return _Body(json.dumps([{"name": n} for n in names])) if url == f"{api}buildroot?ref={pin}": return _Body('{"sha": "c2d7985b"}') + if url == f"{api}batocera?ref={pin}": + return _Body('{"sha": "487014d9"}') + if url == ("https://raw.githubusercontent.com/batocera-linux/batocera.linux/" + "487014d9/package/batocera/emulators/ppsspp/Config.in"): + return _Body("config BR2_PACKAGE_PPSSPP\n\tselect BR2_PACKAGE_PPSSPP_ASSETS\n") + if url == f"{raw}package/utils/vtree/Config.in": + raise urllib.error.HTTPError(url, 404, "Not Found", {}, None) if url.startswith("https://raw.githubusercontent.com/knulli-cfw/buildroot/c2d7985b/"): return _Body("config BR2_aarch64\n\tselect BR2_ARCH_IS_64\n") path = url.removeprefix(raw) @@ -330,11 +351,12 @@ class KnulliPerBoard(unittest.TestCase): self.assertEqual(sorted(data["targets"]), ["h700", "r16"]) # The images name what a card's boot/knulli.board holds. self.assertEqual(data["targets"]["h700"], - {"architecture": "aarch64", "cores": ["mgba", "ppsspp"], + {"architecture": "aarch64", "cores": ["assets", "mgba", "ppsspp"], "images": ["rg35xx-plus", "rg35xx-sp"]}) self.assertEqual(data["targets"]["r16"], {"architecture": "arm", "cores": ["mgba"], "images": ["miyoo-a30"]}) - self.assertTrue(all(pin in url or "buildroot/c2d7985b" in url for url in seen), seen) + self.assertTrue(all(pin in url or "buildroot/c2d7985b" in url + or "batocera.linux/487014d9" in url for url in seen), seen) class RocknixPerDevice(unittest.TestCase):