diff --git a/platforms/knulli.yml b/platforms/knulli.yml index d2aa23488..c6923e911 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -12,55 +12,36 @@ cores: - advanced_drastic - aethersx2 - amiberry -- applewin - arduous - atari800 - azahar - beetle-saturn - bennugd -- bigpemu -- blastem - bluemsx -- boom3 - bsnes -- bsnes_hd - bstone - cannonball - cap32 -- catacombgl - cdogs - cemu - cgenius -- citra -- citron - clk - corsixth -- demul -- desmume - devilutionx - dhewm3 -- dice - dolphin -- dolphin_triforce -- dosbox - dosbox-x - dosbox_pure - dosbox_staging - dosboxx -- drastic -- duckstation -- duckstation-legacy - dxx-rebirth - easyrpg -- ecwolf - eduke32 -- eka2l1 - emuscv - etlegacy - fake08 - fallout1-ce - fallout2-ce -- fba2x - fbalpha - fbneo - fceumm @@ -68,20 +49,15 @@ cores: - flycast-xtreme - flycastvl - fmsx -- fpinball - freechaf - freeintv -- freej2me -- fsuae - fury - fuse - gambatte - gearsystem - genesisplusgx -- genesisplusgx-expanded - genesisplusgx-wide - geolith -- gong - gpsp - gsplus - gw @@ -92,15 +68,9 @@ cores: - hcl - holani - hurrican -- hypseus-singe -- ikemen -- imame4all - ioquake3 - iortcw -- jazz2-native -- kronos - lightspark -- lindbergh-loader - lowresnx - lutro - mame @@ -117,7 +87,6 @@ cores: - mesen-s - mgba - minivmac -- model2emu - moonlight - mpv - mrboom @@ -134,21 +103,14 @@ cores: - openbor7142 - openbor7530 - openjazz -- openjk -- openjkdf2 -- openlara -- openmohaa -- openmsx - opera - parallel_n64 - pce - pce_fast - pcfx -- pcsx2 - pcsx_rearmed - picodrive - play -- pocketsnes - pokemini - potator - ppsspp @@ -158,20 +120,15 @@ cores: - puae2021 - px68k - pygame -- pyxel - quasi88 - raze -- redream - reminiscence - rpcs3 -- ruffle -- ryujinx - samcoupe - same_cdi - sameduck - scummvm - sdlpop -- shadps4 - smsplus - snes9x - snes9x_next @@ -180,31 +137,21 @@ cores: - sonic2013 - sonic3-air - soniccd -- squirreljme - stella -- stella2014 - superbroswar -- supermodel -- supermodel-legacy - swanstation - taradino - tgbdual -- theforceengine - theodore - thextech - tic80 -- tr1x -- tr2x -- tsugaru - tyrian - tyrquake - uae4arm -- uqm - uzem - vb - vba-m - vecx -- vemulator - vice - vice_x128 - vice_x64 @@ -221,24 +168,18 @@ cores: - vkquake2 - vkquake3 - vox_official -- vpinball -- wasm4 - x1 - x128 - x16emu - x64 - x64dtv -- xash3d_fwgs - xemu -- xenia -- xenia-canary - xpet - xplus4 - xrick - xscpu64 - xvic - yabasanshiro -- ymir - zc210 systems: atari-400-800: @@ -4879,61 +4820,39 @@ standalone_cores: - advanced_drastic - aethersx2 - amiberry -- applewin - azahar -- bigpemu - bstone - cannonball -- catacombgl - cdogs - cemu - cgenius -- citra -- citron - clk - corsixth -- demul - devilutionx - dhewm3 - dolphin -- dolphin_triforce -- dosbox - dosbox-x - dosbox_staging - dosboxx -- drastic -- duckstation -- duckstation-legacy - dxx-rebirth - easyrpg -- ecwolf - eduke32 -- eka2l1 - etlegacy - fallout1-ce - fallout2-ce -- fba2x - flycast -- fpinball -- fsuae - fury - gsplus - gzdoom - hatari - hcl - hurrican -- hypseus-singe -- ikemen - ioquake3 - iortcw -- jazz2-native -- kodi - lexaloffle - lightspark -- lindbergh-loader - mame - melonds -- model2emu - moonlight - mpv - mupen64plus @@ -4943,71 +4862,46 @@ standalone_cores: - openbor7142 - openbor7530 - openjazz -- openjk -- openjkdf2 -- openmohaa -- openmsx -- pcsx2 - play - ppsspp - pygame -- pyxel - raze -- redream - rpcs3 -- ruffle -- ryujinx - samcoupe - scummvm - sdlpop -- shadps4 - solarus - sonic-mania - sonic2013 - sonic3-air - soniccd - stella -- supermodel -- supermodel-legacy - taradino -- theforceengine - thextech -- tr1x -- tr2x -- tsugaru - tyrian -- uqm - vice - vita3k - vkquake - vkquake2 - vkquake3 - vox_official -- vpinball - x128 - x16emu - x64 - x64dtv -- xash3d_fwgs - xemu -- xenia -- xenia-canary - xpet - xplus4 - xscpu64 - xvic - yabasanshiro -- ymir dual_mode_cores: -- applewin -- citra - dolphin - easyrpg - flycast - hatari - mame - melonds -- pcsx2 - play - ppsspp - scummvm @@ -5393,129 +5287,6 @@ libretro_systems: zc210: - zc210 launchers: -- emulator: duckstation - build: standalone - kind: names_searched - slot: NTSCU - names: - - scph101.bin - - scph1001.bin - - scph5501.bin - - scph7001.bin - - scph7501.bin - case_insensitive: true - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCU" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ - \ 589-611" -- emulator: duckstation - build: standalone - kind: names_searched - slot: PAL - names: - - scph1002.bin - - scph5502.bin - - scph5552.bin - - scph7002.bin - - scph7502.bin - - scph9002.bin - - scph102a.bin - - scph102b.bin - case_insensitive: true - search_dir: /userdata/bios - passes_to: "[BIOS] PathPAL" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ - \ 589-611" -- emulator: duckstation - build: standalone - kind: names_searched - slot: NTSCJ - names: - - scph100.bin - - scph1000.bin - - scph3000.bin - - scph3500.bin - - scph5500.bin - - scph7000.bin - - scph7003.bin - case_insensitive: true - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCJ" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ - \ 589-611" -- emulator: duckstation - build: standalone - kind: names_searched - slot: Uni - names: - - psxonpsp660.bin - - ps1_rom.bin - case_insensitive: true - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCU, PathPAL, PathNTSCJ" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ - \ 589-611" -- emulator: duckstation-legacy - build: standalone - kind: names_searched - slot: NTSCU - names: - - scph101.bin - - scph1001.bin - - scph5501.bin - - scph7001.bin - - scph7501.bin - case_insensitive: false - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCU" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" -- emulator: duckstation-legacy - build: standalone - kind: names_searched - slot: PAL - names: - - scph1002.bin - - scph5502.bin - - scph5552.bin - - scph7002.bin - - scph7502.bin - - scph9002.bin - - scph102a.bin - - scph102b.bin - case_insensitive: false - search_dir: /userdata/bios - passes_to: "[BIOS] PathPAL" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" -- emulator: duckstation-legacy - build: standalone - kind: names_searched - slot: NTSCJ - names: - - scph100.bin - - scph1000.bin - - scph3000.bin - - scph3500.bin - - scph5500.bin - - scph7000.bin - - scph7003.bin - case_insensitive: false - search_dir: /userdata/bios - passes_to: "[BIOS] PathNTSCJ" - fails_when_missing: No PSX1 BIOS found - fails_scope: emulator - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" - emulator: hatari build: standalone kind: names_searched @@ -5683,33 +5454,6 @@ launchers: passes_to: environment FLYCAST_BIOS_PATH fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/flycast/flycastGenerator.py:239" -- emulator: shadps4 - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/system/configs/shadps4 - passes_to: working directory of shadps4Generator.py - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/shadps4/shadps4Generator.py:33,\ - \ 164" -- emulator: openmsx - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/Machines - passes_to: "written: filepool add -path /userdata/bios/Machines -types system_rom\ - \ -position 1" - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/openmsx/openmsxGenerator.py:95" -- emulator: openmsx - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/openmsx - passes_to: "written: filepool add -path /userdata/bios/openmsx -types system_rom\ - \ -position 2" - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/openmsx/openmsxGenerator.py:96" - emulator: amiberry build: standalone kind: passes_path @@ -5765,15 +5509,6 @@ launchers: depends_on_option: neocd_bios source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroOptions.py:2558-2559,\ \ 2566-2569" -- emulator: pcsx2 - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/ps2 - passes_to: "inis/PCSX2.ini [Folders] Bios = ../../../bios/ps2, relative to /userdata/system/configs/PCSX2" - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/pcsx2/pcsx2Generator.py:92,\ - \ 176, 213" - emulator: aethersx2 build: standalone kind: passes_path @@ -5919,29 +5654,6 @@ launchers: fails_when_missing: false depends_on_option: "Rpcs3Generator.getFirmwareVersion() is None" source_ref: "package/system/knulli-configgen/configgen/configgen/generators/rpcs3/rpcs3Generator.py:355-358" -- emulator: ryujinx - build: standalone - kind: copies - names: - - switch/prod.keys - case_insensitive: false - search_dir: /userdata/bios - passes_to: /userdata/system/configs/Ryujinx/system/prod.keys - fails_when_missing: false - depends_on_option: "ryujinxKeys.exists()" - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/ryujinx/ryujinxGenerator.py:107-109" -- emulator: pcsx2 - build: standalone - kind: copies - names: - - patches.zip - case_insensitive: false - search_dir: /usr/share/knulli/datainit/bios/ps2 - passes_to: /userdata/bios/ps2/patches.zip - fails_when_missing: false - depends_on_option: "not pcsx2Patches.exists()" - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/pcsx2/pcsx2Generator.py:78,\ - \ 121-122" - emulator: aethersx2 build: standalone kind: copies @@ -5962,14 +5674,6 @@ launchers: passes_to: g_cfg_rom_path fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/gsplus/gsplusGenerator.py:106" -- emulator: tsugaru - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/fmtowns - passes_to: command line argument 1 - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/tsugaru/tsugaruGenerator.py:18-19" - emulator: scummvm build: standalone kind: passes_path @@ -5978,14 +5682,6 @@ launchers: passes_to: command line --extrapath fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/scummvm/scummvmGenerator.py:117-123" -- emulator: fsuae - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/amiga - passes_to: command line --kickstarts_dir - fails_when_missing: false - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/fsuae/fsuaeGenerator.py:63-69" - emulator: mame build: standalone kind: passes_path @@ -6039,16 +5735,6 @@ launchers: passes_to: command line -crosshairpath fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:180" -- emulator: fpinball - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios - passes_to: environment W_CACHE - fails_when_missing: false - depends_on_option: "not wsh57_done.exists()" - source_ref: "package/system/knulli-configgen/configgen/configgen/generators/fpinball/fpinballGenerator.py:37,\ - \ 39-43" - emulator: retroarch build: libretro kind: passes_path @@ -6142,15 +5828,6 @@ launchers: depends_on_option: not system.name == 'ti99' source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:431-432,\ \ 435" -- emulator: citron - build: standalone - kind: passes_path - names: [] - search_dir: /userdata/bios/switch - passes_to: "CitronPath::KeysDir in src/common/fs/path_util.cpp returns it, in place\ - \ of citron_path / KEYS_DIR" - fails_when_missing: false - source_ref: "batocera/package/batocera/emulators/citron/002-adjust-paths.patch:9-10" configs_root: path: /userdata/system/configs from_bios: ../system/configs diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index 8ef10db16..84a22c2b3 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -14,7 +14,7 @@ import re import sys import urllib.error import urllib.request -from collections.abc import Callable, Iterable +from collections.abc import Callable from pathlib import Path @@ -30,6 +30,8 @@ from .base_scraper import ( requirement_entry, ) from .batocera_launchers import Configgen, configs_root, extract_launchers +from .targets.batocera_targets_scraper import BuildSelection +from .targets.batocera_targets_scraper import Scraper as BoardTargets PLATFORM_NAME = "batocera" @@ -489,76 +491,51 @@ def configgen_cores( for GameCube). `dropped` names what es_systems.yml files under an emulator without being one. - A default es_systems.yml does not offer names an emulator the build may - lack: KNULLI's defaults pick suyu for the Switch, and no package of its - tree, its batocera submodule or its buildroot builds suyu. `built` says - whether the build has a standalone emulator (packages_and_programs); a - default it answers no for is not run, nor its core. + `built` says whether some board builds a name (boards_build). What it + answers no for is not run: KNULLI's es_systems lists citron and shadps4, + which no KNULLI board selects, and its defaults pick suyu, which no + package builds. A default's core follows a standalone emulator the + build runs (vox_official is lexaloffle's Voxatron build). """ data = yaml_load_upstream(defaults_text) cores, standalone = es_systems_emulators(es_systems_text) - offered = set(cores) libretro = es_systems_libretro(es_systems_text) if not cores: raise RuntimeError("es_systems.yml names no emulator") + if built is not None: + cores = {name for name in cores if built(name)} + standalone &= cores for system, cfg in data.items(): if system == "default" or not isinstance(cfg, dict): continue emulator = cfg.get("emulator", "") core = cfg.get("core", "") - if (built is not None and emulator and emulator != "libretro" - and emulator not in offered and not built(emulator)): - continue - if core: - cores.add(core) if emulator and emulator != "libretro": + if built is not None and not built(emulator): + continue standalone.add(emulator) if core and core != emulator: standalone.add(core) + elif built is not None and core and not built(core): + continue + if core: + cores.add(core) dual = (standalone & libretro) - dropped return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual) -BATOCERA_REPO = "batocera-linux/batocera.linux" +def boards_build(selection: BuildSelection, launchers: list[dict]) -> Callable[[str], bool]: + """Whether some board of a build runs an emulator. - -def package_names(paths: Iterable[str]) -> set[str]: - """The buildroot packages a tree defines: each folder holding a Config.in.""" - return { - path.rsplit("/", 2)[-2] - for path in paths - if path.endswith("/Config.in") and path.count("/") >= 1 - } - - -def repo_tree(repo: str, ref: str) -> list[str]: - """Every path of a GitHub repository at a revision.""" - url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1" - req = urllib.request.Request(url, headers=github_headers()) - with urllib.request.urlopen(req, timeout=60) as resp: - found = json.loads(resp.read().decode("utf-8")) - if found.get("truncated"): - raise RuntimeError(f"{url}: tree truncated") - return [entry["path"] for entry in found.get("tree", [])] - - -def submodule_revision(repo: str, path: str, ref: str) -> str: - """The commit a repository's submodule at `path` pins at a revision.""" - url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}" - req = urllib.request.Request(url, headers=github_headers()) - with urllib.request.urlopen(req, timeout=30) as resp: - found = json.loads(resp.read().decode("utf-8")) - if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"): - raise RuntimeError(f"{url}: {path} is not a submodule") - return str(found["sha"]) - - -def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]: - """Whether a build has a standalone emulator. - - It does when a package of that name exists, or when its launcher runs a - program the user puts in the BIOS folder (Batocera's lexaloffle runs - /userdata/bios/pico-8/pico8), which no package builds. + es_systems.yml ties each name to the packages that build it + (requireAnyOf), and a name it lists is built when a board selects one of + them: KNULLI lists citron and shadps4, and its Config.in selects them + only on x86_64 boards, which it has none of. A name only + configgen-defaults.yml gives is built when a board selects the package + of that name: the symbol its Config.in defines, else the one Buildroot + derives from the folder (BR2_PACKAGE_, package/pkg-generic.mk). A + program the launcher runs from the BIOS folder (lexaloffle, + /userdata/bios/pico-8/pico8) needs no package. """ supplied = { name.strip() @@ -567,7 +544,27 @@ def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable and str(record.get("search_dir") or "").startswith("/userdata/bios") for name in str(record.get("emulator", "")).split(",") } - return lambda emulator: emulator in packages or emulator in supplied + + def built(name: str) -> bool: + if name in selection.emulators or name in supplied: + return True + if name in selection.listed: + return False + symbol = selection.packages.get(name) or ( + "BR2_PACKAGE_" + name.upper().replace("-", "_")) + return symbol in selection.symbols + + return built + + +def records_run(launchers: list[dict], built: Callable[[str], bool]) -> list[dict]: + """The launcher records of emulators the build runs. + + configgen keeps the generators of emulators a build does not select + (KNULLI's citron, pcsx2, shadps4): what they read is never read on the + device, so the records do not describe the platform. + """ + return [r for r in launchers if any(built(n) for n in str(r["emulator"]).split(", "))] class Scraper(BaseScraper): @@ -578,20 +575,13 @@ class Scraper(BaseScraper): super().__init__(url=SOURCE_URL.format(tag=self.tag)) def _fetch_cores( - self, launchers: list[dict] | None = None + self, built: Callable[[str], bool] | None = None ) -> tuple[list[str], list[str], list[str]]: """Core names, standalone cores and dual-mode cores at the pinned tag. - A default es_systems.yml does not offer counts when a package of - Batocera's tree or its buildroot builds it, or when its launcher - runs a program from the BIOS folder. + With `built` (boards_build), a name counts when some board builds + it or its launcher runs a program from the BIOS folder. """ - built = None - if launchers is not None: - paths = repo_tree(BATOCERA_REPO, self.tag) - buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag) - paths += repo_tree("batocera-linux/buildroot", buildroot) - built = packages_and_programs(package_names(paths), launchers) return configgen_cores( self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)), self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), @@ -633,7 +623,9 @@ class Scraper(BaseScraper): launchers = extract_launchers( lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}") ) - cores, standalone, dual = self._fetch_cores(launchers) + built = boards_build(BoardTargets().build_selection(self.tag), launchers) + launchers = records_run(launchers, built) + cores, standalone, dual = self._fetch_cores(built) libretro_systems = es_systems_libretro_systems( self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores() ) diff --git a/scripts/scraper/knulli_scraper.py b/scripts/scraper/knulli_scraper.py index 763c25ec4..f94f56704 100644 --- a/scripts/scraper/knulli_scraper.py +++ b/scripts/scraper/knulli_scraper.py @@ -37,18 +37,17 @@ from .batocera_scraper import ( ) from .batocera_scraper import ( _not_cores, + boards_build, configgen_cores, es_systems_by_system, es_systems_libretro_systems, extract_systems_dict, - package_names, - packages_and_programs, platform_systems, - repo_tree, + records_run, requirements_from_systems, - submodule_revision, ) from .es_emulators import attach_system_cores +from .targets.knulli_targets_scraper import Scraper as BoardTargets from .batocera_launchers import configs_root from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers from .knulli_launchers import configgen as knulli_configgen @@ -134,15 +133,11 @@ class Scraper(BaseScraper): lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path), lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path), ) - # The packages KNULLI builds come from its own tree, its batocera - # submodule and its buildroot submodule: a default no package of - # them builds (suyu) is not run. - paths = repo_tree(REPO, self.tag) - paths += repo_tree("batocera-linux/batocera.linux", revision) - paths += repo_tree( - "knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag) - ) - built = packages_and_programs(package_names(paths), launchers) + # What runs is what at least one board builds, read by the targets + # scraper from the Kconfig of KNULLI's tree and its batocera and + # buildroot submodules (boards_build). + built = boards_build(BoardTargets().build_selection(self.tag), launchers) + launchers = records_run(launchers, built) cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built) # Each system's own list (psx, es_systems.yml:640-653: libretro # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and diff --git a/scripts/scraper/targets/batocera_targets_scraper.py b/scripts/scraper/targets/batocera_targets_scraper.py index f13615ccb..ce272de03 100644 --- a/scripts/scraper/targets/batocera_targets_scraper.py +++ b/scripts/scraper/targets/batocera_targets_scraper.py @@ -3,6 +3,8 @@ Sources (batocera-linux/batocera.linux): - configs/batocera-*.board -- board definitions, each sets BR2_PACKAGE_BATOCERA_TARGET_* - package/batocera/core/batocera-system/Config.in -- select PACKAGE if CONDITION lines + - Config.in and every Kconfig file it sources -- the selects a package + declares itself (advanced_drastic selects drastic) - package/batocera/emulationstation/batocera-es-system/es_systems.yml -- emulator requireAnyOf flag mapping Sources (batocera-linux/buildroot, at the submodule commit the tag pins): @@ -19,6 +21,8 @@ import re import sys import urllib.error import urllib.request +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from datetime import datetime, timezone import yaml @@ -79,6 +83,16 @@ def _fetch(url: str, headers: dict | None = None) -> str: raise RuntimeError(f"cannot fetch {url}: {e}") from e +def _fetch_present(url: str) -> str | None: + """A file's text, or None when the server answers that it does not exist.""" + try: + return _fetch(url) + except RuntimeError as e: + if isinstance(e.__cause__, urllib.error.HTTPError) and e.__cause__.code == 404: + return None + raise + + def _fetch_json(url: str) -> list | dict: text = _fetch(url, headers=github_headers()) try: @@ -210,6 +224,7 @@ BUILDROOT_KCONFIG = ( ) _CONFIG_LINE_RE = re.compile(r"^\s*(?:menu)?config\s+(BR2_\w+)") +_SOURCE_RE = re.compile(r'^\s*source\s+"\$BR2_EXTERNAL_\w+_PATH/([^"]+)"') _OWNED_SELECT_RE = re.compile(r"^\s+select\s+(BR2_\w+)(?:\s+if\s+([^#]+))?") _BOARD_FLAG_RE = re.compile(r"^(BR2_\w+)=y\s*$", re.MULTILINE) _BOARD_INCLUDE_RE = re.compile(r"^(-?)include\s+(\S+)", re.MULTILINE) @@ -342,6 +357,43 @@ def _arch_from_flag(flag: str, board_flags: set[str] | frozenset[str] = frozense return "aarch64" +@dataclass(frozen=True) +class BuildSelection: + """What the boards of a build turn on, read once from its Kconfig. + + `targets` is each board's entry of the targets file. `emulators` holds + every es_systems name some board builds and `listed` every name + es_systems ties to a package, `symbols` every symbol some board ends up + with, and `packages` the config symbol each package folder of the tree + defines. + """ + + targets: dict[str, dict] + emulators: frozenset[str] + listed: frozenset[str] + symbols: frozenset[str] + packages: dict[str, str] + + +def _sourced(text: str) -> list[str]: + """The files a Kconfig file sources, relative to the external tree.""" + return [m.group(1) for line in text.splitlines() if (m := _SOURCE_RE.match(line))] + + +def _package_symbols(files: dict[str, str]) -> dict[str, str]: + """package folder -> the first config symbol its Config.in defines.""" + found: dict[str, str] = {} + for path, text in files.items(): + parts = path.split("/") + if len(parts) < 3 or parts[-1] != "Config.in" or "package" not in parts: + continue + config = next((m.group(1) for line in text.splitlines() + if (m := _CONFIG_LINE_RE.match(line))), None) + if config: + found.setdefault(parts[-2], config) + return found + + class Scraper(BaseTargetScraper): """Cross-references Batocera boards, Config.in, and es_systems to build target lists. @@ -357,6 +409,9 @@ class Scraper(BaseTargetScraper): ES_SYSTEMS_PATH = "package/batocera/emulationstation/batocera-es-system/es_systems.yml" SYSTEM_SYMBOL = "BR2_PACKAGE_BATOCERA_SYSTEM" BUILDROOT_REPO = "batocera-linux/buildroot" + # Path prefix -> repository of a submodule the root Config.in sources + # from: KNULLI reads Batocera's packages as batocera/package/... + SUBMODULES: dict[str, str] = {} def __init__(self, url: str = "") -> None: super().__init__(url=url or f"https://github.com/{self.REPO}") @@ -411,6 +466,45 @@ class Scraper(BaseTargetScraper): pending += [(inc, bool(dash)) for dash, inc in _BOARD_INCLUDE_RE.findall(text)] return target, flags + def _kconfig_files(self) -> dict[str, str]: + """The root Config.in and every file it sources, recursively. + + The build reads these and nothing else: a package's own Config.in + can select another package (advanced_drastic selects drastic), and + such a select sits in no system Config.in. A sourced file absent at + the pin defines nothing and is named: KNULLI 20260511 sources + package/utils/vtree/Config.in, a package its tree gained later. + """ + bases: list[tuple[str, str]] = [] + for prefix, repo in self.SUBMODULES.items(): + pin = _fetch_json(self._api(prefix.rstrip("/"))) + sha = pin.get("sha") if isinstance(pin, dict) else None + if not sha: + raise RuntimeError(f"no {prefix.rstrip('/')} submodule commit at {self.tag}") + bases.append((prefix, f"https://raw.githubusercontent.com/{repo}/{sha}/")) + + def url(path: str) -> str: + for prefix, base in bases: + if path.startswith(prefix): + return base + path.removeprefix(prefix) + return self._raw(path) + + files: dict[str, str] = {} + absent: list[str] = [] + pending = ["Config.in"] + with ThreadPoolExecutor(max_workers=8) as pool: + while pending: + found = dict(zip(pending, pool.map(lambda path: _fetch_present(url(path)), pending))) + absent += [path for path, text in found.items() if text is None] + files.update((path, text or "") for path, text in found.items()) + pending = sorted({src for text in found.values() if text + for src in _sourced(text)} - set(files)) + if "Config.in" in absent or len(files) - len(absent) < 2: + raise RuntimeError(f"Config.in at {self.tag} sources no readable Kconfig file") + for path in absent: + print(f" sourced but absent at {self.tag}: {path}", file=sys.stderr) + return files + def _buildroot_rules(self) -> list[tuple[str, str, str]]: """Buildroot's selects for the symbols the tree tests, at its pin.""" pin = _fetch_json(self._api("buildroot")) @@ -423,26 +517,32 @@ class Scraper(BaseTargetScraper): rules += _parse_owned_selects(_fetch(f"{raw}/{path}")) return rules - def fetch_targets(self) -> dict: - """Build per-board emulator availability map.""" - self.tag = self.pinned_revision(self.PLATFORM) + def build_selection(self, tag: str | None = None) -> BuildSelection: + """Each board's selection, from the Kconfig the build reads. + + `tag` is the revision to read, by default the one the platform's + YAML was scraped at: a platform scraper moving to a new release + passes its own. + """ + self.tag = tag or self.pinned_revision(self.PLATFORM) print(" fetching board list...", file=sys.stderr) boards = self._list_boards() - print(" fetching Config.in...", file=sys.stderr) - config_in_url = self._raw(self.CONFIG_IN_PATH) - config_in_text = _fetch(config_in_url) + print(" fetching Kconfig files...", file=sys.stderr) + files = self._kconfig_files() + config_in_text = files.get(self.CONFIG_IN_PATH) if not config_in_text: - raise RuntimeError(f"empty Config.in from {config_in_url}") + raise RuntimeError(f"Config.in at {self.tag} does not source {self.CONFIG_IN_PATH}") meta_rules = _parse_meta_flags(config_in_text) - rules = _parse_owned_selects(config_in_text) + rules = [rule for text in files.values() for rule in _parse_owned_selects(text)] print( - f" parsed {len(meta_rules)} meta-flag rules, {len(rules)} select lines", + f" parsed {len(meta_rules)} meta-flag rules, {len(rules)} select lines " + f"in {len(files)} Kconfig files", file=sys.stderr, ) - if not rules: - raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {config_in_url}") + if not _parse_owned_selects(config_in_text): + raise RuntimeError(f"no 'select BR2_PACKAGE_*' lines parsed from {self.CONFIG_IN_PATH}") rules += self._buildroot_rules() print(" fetching es_systems.yml...", file=sys.stderr) @@ -459,9 +559,10 @@ class Scraper(BaseTargetScraper): raise RuntimeError(f"no package->emulator mappings parsed from {es_systems_url}") images_by_flag = _parse_images(config_in_text) if not images_by_flag: - raise RuntimeError(f"no target image list parsed from {config_in_url}") + raise RuntimeError(f"no target image list parsed from {self.CONFIG_IN_PATH}") targets: dict[str, dict] = {} + symbols: set[str] = set() for board_name in sorted(boards): target_key = board_name.removeprefix(self.BOARD_PREFIX).removesuffix(".board") print(f" processing {target_key}...", file=sys.stderr) @@ -476,6 +577,7 @@ class Scraper(BaseTargetScraper): if self.SYSTEM_SYMBOL not in selected_packages: print(f" {board_name} builds no system image", file=sys.stderr) continue + symbols |= selected_packages # Map selected packages to emulator names via es_systems.yml emulators: set[str] = set() @@ -496,11 +598,22 @@ class Scraper(BaseTargetScraper): file=sys.stderr, ) + return BuildSelection( + targets=targets, + emulators=frozenset(c for entry in targets.values() for c in entry["cores"]), + listed=frozenset(n for names in package_to_emulators.values() for n in names), + symbols=frozenset(symbols), + packages=_package_symbols(files), + ) + + def fetch_targets(self) -> dict: + """Build per-board emulator availability map.""" + selection = self.build_selection() return { "platform": self.PLATFORM, "source": f"{self.url}/tree/{self.tag}", "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "targets": targets, + "targets": selection.targets, } diff --git a/scripts/scraper/targets/knulli_targets_scraper.py b/scripts/scraper/targets/knulli_targets_scraper.py index e19f72f27..16de256e7 100644 --- a/scripts/scraper/targets/knulli_targets_scraper.py +++ b/scripts/scraper/targets/knulli_targets_scraper.py @@ -9,6 +9,8 @@ the tag platforms/knulli.yml names): the root Config.in in place of Batocera's batocera-system/Config.in - package/emulationstation/knulli-es-system/es_systems.yml -- emulator requireAnyOf flag mapping + - Config.in and what it sources, Batocera's packages read from the + batocera submodule at the commit the tag pins Sources (knulli-cfw/buildroot, at the submodule commit the tag pins): - arch/Config.in, package/mesa3d/Config.in, package/libglvnd/Config.in @@ -32,6 +34,7 @@ class Scraper(BatoceraScraper): ES_SYSTEMS_PATH = "package/emulationstation/knulli-es-system/es_systems.yml" SYSTEM_SYMBOL = "BR2_PACKAGE_KNULLI_SYSTEM" BUILDROOT_REPO = "knulli-cfw/buildroot" + SUBMODULES = {"batocera/": "batocera-linux/batocera.linux"} def main() -> None: diff --git a/tests/test_elec_scrapers.py b/tests/test_elec_scrapers.py index 0ee2f1950..da854d70c 100644 --- a/tests/test_elec_scrapers.py +++ b/tests/test_elec_scrapers.py @@ -284,7 +284,10 @@ class TheExportKeepsTheirShape(unittest.TestCase): class TheCommittedListsAgree(unittest.TestCase): """A system's cores are a part of the platform's: a name only the system - carries would escape every reader of the platform list.""" + carries would escape every reader of the platform list. A system keeps + the list its ES file writes, which says whose files its entries ask for, + so the one name it may carry beyond the platform's is one no device + image builds (EmuELEC's VVVVVV port engine, KNULLI's duckstation).""" def test_system_cores_are_platform_cores(self): import yaml @@ -292,10 +295,13 @@ class TheCommittedListsAgree(unittest.TestCase): for name in ("amberelec", "emuelec", "knulli"): with self.subTest(platform=name): config = yaml.safe_load((REPO_ROOT / "platforms" / f"{name}.yml").read_text()) + targets = yaml.safe_load( + (REPO_ROOT / "platforms" / "targets" / f"{name}.yml").read_text())["targets"] + built = {c for entry in targets.values() for c in entry["cores"]} platform = {str(c) for c in config["cores"]} listed = {str(c) for s in config["systems"].values() for c in s.get("cores", [])} self.assertTrue(listed, "no system carries its cores") - self.assertEqual(listed - platform, set()) + self.assertEqual(listed - platform - (listed - built), set()) if __name__ == "__main__": diff --git a/tests/test_launcher_paths.py b/tests/test_launcher_paths.py index 55564f564..1a37a4fba 100644 --- a/tests/test_launcher_paths.py +++ b/tests/test_launcher_paths.py @@ -231,15 +231,20 @@ class APortableUserFolder(unittest.TestCase): -class ADefaultTheBuildLacksIsNotRun(unittest.TestCase): - """KNULLI's configgen-defaults.yml picks suyu for the Switch, which its - es_systems.yml does not offer and no package builds.""" +class OnlyWhatABoardBuildsIsRun(unittest.TestCase): + """KNULLI's es_systems.yml lists citron and shadps4, which its Config.in + selects only on x86_64 boards (it has none), and its + configgen-defaults.yml picks suyu, which no package builds.""" ES = textwrap.dedent("""\ switch: emulators: citron: citron: { requireAnyOf: [BR2_PACKAGE_CITRON] } + gba: + emulators: + libretro: + mgba: { requireAnyOf: [BR2_PACKAGE_LIBRETRO_MGBA] } voxatron: emulators: {} """) @@ -253,31 +258,66 @@ class ADefaultTheBuildLacksIsNotRun(unittest.TestCase): kodi: emulator: kodi core: kodi + mpv: + emulator: mpv + core: mpv """) LAUNCHERS = [{"emulator": "lexaloffle", "kind": "fixed_name", "passes_to": "exec", "search_dir": "/userdata/bios"}] - def test_only_what_a_package_or_the_user_supplies_is_run(self): + def selection(self, symbols: set[str]): + from scraper.targets.batocera_targets_scraper import BuildSelection + + return BuildSelection( + targets={}, emulators=frozenset({"mgba"}), + listed=frozenset({"citron", "mgba"}), symbols=frozenset(symbols), + packages={"kodi": "BR2_PACKAGE_KODI"}, + ) + + def test_only_what_a_board_builds_or_the_user_supplies_is_run(self): from scraper import batocera_scraper as bs - paths = ["package/batocera/emulators/citron/Config.in", "package/kodi/Config.in"] - built = bs.packages_and_programs(bs.package_names(paths), self.LAUNCHERS) + # A board selects kodi's package and buildroot's mpv; citron is + # listed by es_systems, so the symbol of a folder of that name does + # not count for it. + built = bs.boards_build( + self.selection({"BR2_PACKAGE_KODI", "BR2_PACKAGE_MPV", "BR2_PACKAGE_CITRON_X"}), + self.LAUNCHERS, + ) cores, standalone, _dual = bs.configgen_cores(self.DEFAULTS, self.ES, set(), built) - self.assertEqual(sorted(cores), ["citron", "kodi", "vox_official"]) - self.assertIn("lexaloffle", standalone) - self.assertNotIn("suyu", standalone) + self.assertEqual(sorted(cores), ["kodi", "mgba", "mpv", "vox_official"]) + self.assertEqual(sorted(standalone), ["kodi", "lexaloffle", "mpv", "vox_official"]) + # Each system keeps the list es_systems.yml writes (whose files its + # entries ask for), less a default no package builds. by_system = bs.es_systems_by_system(self.ES, self.DEFAULTS, set(), built) self.assertEqual(by_system["switch"], ["citron"]) - # Without the packages, every default counts, as before. + self.assertEqual(by_system["voxatron"], ["lexaloffle", "vox_official"]) + self.assertEqual(by_system["gba"], ["mgba"]) + + def test_a_listed_name_is_built_only_through_its_packages(self): + from scraper import batocera_scraper as bs + + built = bs.boards_build(self.selection({"BR2_PACKAGE_CITRON"}), []) + self.assertFalse(built("citron")) + self.assertTrue(built("mgba")) + self.assertFalse(built("lexaloffle")) + # Without a selection, every name counts, as before. self.assertIn("suyu", bs.configgen_cores(self.DEFAULTS, self.ES, set())[0]) - def test_the_real_knulli_list_names_no_suyu(self): + def test_the_real_lists_name_only_what_a_board_builds(self): import yaml - config = yaml.safe_load((REPO_ROOT / "platforms" / "knulli.yml").read_text()) - self.assertNotIn("suyu", config["cores"]) - self.assertNotIn("suyu", config.get("standalone_cores") or []) - + for platform, absent in (("knulli", ("suyu", "citron", "shadps4", "pcsx2")),): + config = yaml.safe_load((REPO_ROOT / "platforms" / f"{platform}.yml").read_text()) + names = set(config["cores"]) | set(config.get("standalone_cores") or []) + targets = yaml.safe_load( + (REPO_ROOT / "platforms" / "targets" / f"{platform}.yml").read_text())["targets"] + boards = {c for entry in targets.values() for c in entry["cores"]} + with self.subTest(platform=platform): + self.assertFalse(names & set(absent)) + # What a board builds and the list once named is still there, + # and what no board builds is a program the user supplies. + self.assertLessEqual(names - boards, {"kodi", "lexaloffle", "mpv", "vox_official"}) class AUserFolderInTheConfigsFolder(unittest.TestCase): @@ -289,7 +329,8 @@ class AUserFolderInTheConfigsFolder(unittest.TestCase): from common import standalone_set profile = load_emulator_profiles(str(REPO_ROOT / "emulators"))["shadps4"] - for platform in ("batocera", "knulli"): + # KNULLI's generator does the same, but no KNULLI board builds shadps4. + for platform in ("batocera",): config = load_platform_config(platform, str(REPO_ROOT / "platforms")) ((view, _standalone),) = platform_layouts("shadps4", profile, standalone_set(config)) with self.subTest(platform=platform): diff --git a/tests/test_standalone_links.py b/tests/test_standalone_links.py index 1fcd5eedd..7638dfafa 100644 --- a/tests/test_standalone_links.py +++ b/tests/test_standalone_links.py @@ -548,12 +548,12 @@ class OpenmsxOnEachPlatform(unittest.TestCase): """openMSX finds system ROMs by SHA1 in share/systemroms below its user directory and in every filepool entry. RetroBat sets OPENMSX_HOME to bios/openmsx, RetroPie links share/systemroms of its user directory to - BIOS/openmsx, Batocera and KNULLI add bios/Machines and bios/openmsx to - the filepool and list their own files in bios/openmsx.""" + BIOS/openmsx, Batocera adds bios/Machines and bios/openmsx to the + filepool and lists its own files in bios/openmsx. KNULLI's generator + does the same, but no KNULLI board builds openmsx.""" EXPECTED = { "batocera": {"share/systemroms": ("", "openmsx")}, - "knulli": {"share/systemroms": ("", "openmsx")}, "retrobat": {"": ("", "openmsx")}, "retropie": {"share/systemroms": ("", "openmsx")}, } diff --git a/tests/test_target_scrapers.py b/tests/test_target_scrapers.py index d55033cff..81c389d5d 100644 --- a/tests/test_target_scrapers.py +++ b/tests/test_target_scrapers.py @@ -97,6 +97,9 @@ class FailedRequestsStopTheScrape(unittest.TestCase): _refuse() return _Body(b"BR2_PACKAGE_BATOCERA_TARGET_X86_64=y\n" b"BR2_PACKAGE_BATOCERA_SYSTEM=y\n") + if url.startswith("https://raw.githubusercontent.com/") and url.count("/") == 6: + return _Body(b'source "$BR2_EXTERNAL_BATOCERA_PATH/' + b'package/batocera/core/batocera-system/Config.in"\n') if url.endswith("Config.in"): return _Body(b"config BR2_PACKAGE_BATOCERA_SYSTEM\n" b"\tselect BR2_PACKAGE_MGBA\n\n" @@ -262,6 +265,15 @@ class KnulliPerBoard(unittest.TestCase): "BR2_arm=y\nBR2_PACKAGE_BATOCERA_TARGET_H700=y\n" ), "configs/knulli-h700_armhf_libs.packages": "BR2_PACKAGE_SDL2=y\n", + # The root Config.in sources KNULLI's system and Batocera's packages + # from the batocera submodule, where a package selects another. + "Config.in": ( + 'source "$BR2_EXTERNAL_KNULLI_PATH/package/system/knulli-system/Config.in"\n' + 'source "$BR2_EXTERNAL_KNULLI_PATH/batocera/package/batocera/emulators/' + 'ppsspp/Config.in"\n' + '#source "$BR2_EXTERNAL_KNULLI_PATH/package/utils/off/Config.in"\n' + 'source "$BR2_EXTERNAL_KNULLI_PATH/package/utils/vtree/Config.in"\n' + ), "package/system/knulli-system/Config.in": ( "if BR2_PACKAGE_BATOCERA_TARGET_H700 || \\\n" " BR2_PACKAGE_BATOCERA_TARGET_A133\n" @@ -282,6 +294,8 @@ class KnulliPerBoard(unittest.TestCase): " mgba: {requireAnyOf: [BR2_PACKAGE_LIBRETRO_MGBA]}\n" "psp:\n emulators:\n ppsspp:\n" " ppsspp: {requireAnyOf: [BR2_PACKAGE_PPSSPP]}\n" + "pspassets:\n emulators:\n libretro:\n" + " assets: {requireAnyOf: [BR2_PACKAGE_PPSSPP_ASSETS]}\n" ), } @@ -314,6 +328,13 @@ class KnulliPerBoard(unittest.TestCase): return _Body(json.dumps([{"name": n} for n in names])) if url == f"{api}buildroot?ref={pin}": return _Body('{"sha": "c2d7985b"}') + if url == f"{api}batocera?ref={pin}": + return _Body('{"sha": "487014d9"}') + if url == ("https://raw.githubusercontent.com/batocera-linux/batocera.linux/" + "487014d9/package/batocera/emulators/ppsspp/Config.in"): + return _Body("config BR2_PACKAGE_PPSSPP\n\tselect BR2_PACKAGE_PPSSPP_ASSETS\n") + if url == f"{raw}package/utils/vtree/Config.in": + raise urllib.error.HTTPError(url, 404, "Not Found", {}, None) if url.startswith("https://raw.githubusercontent.com/knulli-cfw/buildroot/c2d7985b/"): return _Body("config BR2_aarch64\n\tselect BR2_ARCH_IS_64\n") path = url.removeprefix(raw) @@ -330,11 +351,12 @@ class KnulliPerBoard(unittest.TestCase): self.assertEqual(sorted(data["targets"]), ["h700", "r16"]) # The images name what a card's boot/knulli.board holds. self.assertEqual(data["targets"]["h700"], - {"architecture": "aarch64", "cores": ["mgba", "ppsspp"], + {"architecture": "aarch64", "cores": ["assets", "mgba", "ppsspp"], "images": ["rg35xx-plus", "rg35xx-sp"]}) self.assertEqual(data["targets"]["r16"], {"architecture": "arm", "cores": ["mgba"], "images": ["miyoo-a30"]}) - self.assertTrue(all(pin in url or "buildroot/c2d7985b" in url for url in seen), seen) + self.assertTrue(all(pin in url or "buildroot/c2d7985b" in url + or "batocera.linux/487014d9" in url for url in seen), seen) class RocknixPerDevice(unittest.TestCase):