fix: keep the cores batocera's retroarch installs

Batocera builds RetroArch with its core downloader (configure defaults,
qb/config.params.sh:149-150), shown in the menu since it is no Lakka
build (config.def.h:743-745), and configgen sets no URL, so the compiled
one applies: buildbot linux/x86_64 or linux/armhf, none on aarch64
(config.def.h:1855-1861). The targets scraper names that list per board
(downloads, an armv7-A CPU read from arch/Config.in.arm), the builder
and the published target lists add it, and the platform names the union
as installable_cores. applewin, gong, squirreljme and vemulator come
back that way. A build option that compiles the downloader out, or a
shipped retroarch.cfg that blanks its URL, turns it off.
This commit is contained in:
Abdessamad Derraz committed 2026-11-09 21:53:15 +01:00
1 parent a7e757598f
commit 828d1527ee
9 files changed
+1037 -16

No files matched your search

File diff suppressed because it is too large. Load diff
+151
View File
@@ -5798,6 +5798,157 @@ dual_mode_cores:
- scummvm
- stella
- tic80
installable_cores:
- "2048"
- 3dengine
- DoubleCherryGB
- amiarcadia
- anarch
- applewin
- ardens
- b2
- bbkemu
- boom3_xp
- bsnes-jg
- bsnes2014_accuracy
- bsnes2014_balanced
- bsnes2014_performance
- bsnes_cplusplus98
- bsnes_hd_beta
- bsnes_mercury_accuracy
- bsnes_mercury_balanced
- bsnes_mercury_performance
- cdi2015
- chailove
- citra
- citra2018
- clownmdemu
- coolbaremu
- craft
- crocods
- desmume2015
- dingooemu
- dinothawr
- dirksimple
- dosbox_core
- dosbox_svn
- dosbox_svn_ce
- doukutsu_rs
- ep128emu_core
- fbalpha2012
- fbalpha2012_cps1
- fbalpha2012_cps2
- fbalpha2012_cps3
- fbalpha2012_neogeo
- fixgb
- fixnes
- frodo
- galaksija
- gam4980
- gametank
- gearboy
- geargrafx
- gearlynx
- geartowns
- genesis_plus_gx
- genesis_plus_gx_wide
- geolith
- gme
- gong
- hatari2014
- hbmame
- higan_sfc
- higan_sfc_balanced
- hyperscanemu
- hypseushw
- irogb
- jaxe
- jollycv
- jumpnbump
- linuxloader
- m2000
- mame2000
- mame2003
- mame2003_midway
- mame2003_plus
- mame2010
- mame2015
- mame2016
- mcsoftserve
- mednafen_gba
- mednafen_pce
- mednafen_pce_fast
- mednafen_pcfx
- mednafen_psx_hw
- mednafen_saturn
- mednafen_snes
- mednafen_supafaust
- mednafen_vb
- mesen2
- meteor
- mojozork
- mu
- mupen64plus_next
- n64emu
- native32emu
- nekop2
- nicaiemu
- noods
- nside_sfc_balanced
- nuance
- numero
- oberon
- odin
- omicron
- openlara
- panda3ds
- parallel_dc
- pcee2
- pcem
- piece
- playdiaemu
- pocketcdg
- pokket
- pokketstation
- qemu
- quicknes
- race
- radio
- remotejoy
- retro8
- romcleaner
- rust_dos
- rustynes
- sameboy
- skyemu
- snes9x2002
- snes9x2005
- snes9x2005_plus
- snes9x2010
- spmp8000emu
- squirreljme
- stella2023
- tamalibretro
- test
- thepowdertoy
- tia
- uw8
- uzdoom
- vaporspec
- vba_next
- vbam
- vemulator
- vice_xcbm2
- vice_xcbm5x0
- video_processor
- virtualxt
- vitaquake2-rogue
- vitaquake2-xatrix
- vitaquake2-zaero
- vitaquake3
- wqxemu
- xenia_edge
- yabause
libretro_systems:
"81":
- zx81
+9 -1
View File
@@ -1,6 +1,6 @@
platform: batocera
source: https://github.com/batocera-linux/batocera.linux/tree/batocera-43.1
scraped_at: '2026-11-09T19:49:31Z'
scraped_at: '2026-11-09T20:38:15Z'
targets:
a3gen2:
architecture: aarch64
@@ -790,6 +790,7 @@ targets:
- xvic
- yquake2
- zc210
downloads: linux-armhf
bcm2837:
architecture: aarch64
cores:
@@ -1075,6 +1076,7 @@ targets:
- orangepi-one
- orangepi-pc
- orangepi-plus2e
downloads: linux-armhf
h5:
architecture: aarch64
cores:
@@ -2181,6 +2183,7 @@ targets:
- xvic
- yquake2
- zc210
downloads: linux-armhf
qcs6490:
architecture: aarch64
cores:
@@ -2516,6 +2519,7 @@ targets:
- powkiddy_a13
- ps5000
- ps7000
downloads: linux-armhf
rk3288:
architecture: arm
cores:
@@ -2674,6 +2678,7 @@ targets:
images:
- miqi
- tinkerboard
downloads: linux-armhf
rk3326:
architecture: aarch64
cores:
@@ -3887,6 +3892,7 @@ targets:
- xvic
- yquake2
- zc210
downloads: linux-armhf
s905:
architecture: aarch64
cores:
@@ -5501,6 +5507,7 @@ targets:
- ymir
- yquake2
- zc210
downloads: linux-x86_64
zen3:
architecture: x86_64
cores:
@@ -5732,3 +5739,4 @@ targets:
- zc210
images:
- x86-64-v3
downloads: linux-x86_64
+8
View File
@@ -45,6 +45,14 @@
},
"uniqueItems": true
},
"installable_cores": {
"description": "Buildbot libretro cores the platform's RetroArch core downloader installs on at least one board, from the RetroArch targets entry of that board's architecture.",
"type": "array",
"items": {
"type": "string"
},
"uniqueItems": true
},
"dual_mode_cores": {
"type": "array",
"items": {
+22 -9
View File
@@ -869,11 +869,24 @@ def load_target_config(
)
)
declared = (targets[canonical] or {}).get("cores")
return _target_core_set(targets[canonical] or {}, overrides, canonical, platforms_dir)
def _target_core_set(
entry: dict, overrides: dict, target: str, platforms_dir: str
) -> set[str] | None:
"""A target's cores: what the board builds, overrides applied, and the
buildbot cores its RetroArch core downloader installs (`downloads`, a
RetroArch targets entry), which a Batocera x86_64 or armv7 board can
fetch from its menu."""
declared = entry.get("cores")
if declared is None:
return None
cores = set(str(c) for c in declared)
return apply_target_overrides(cores, overrides, canonical)
cores = apply_target_overrides({str(c) for c in declared}, overrides, target)
downloads = entry.get("downloads")
if downloads:
cores |= load_target_config("retroarch", str(downloads), platforms_dir) or set()
return cores
def apply_target_overrides(
@@ -917,12 +930,9 @@ def list_available_targets(
result = []
for tname, tdata in sorted(data.get("targets", {}).items()):
aliases = (overrides.get(tname) or {}).get("aliases", [])
declared = (tdata or {}).get("cores")
# None: the target runs every core the platform declares.
count = (
None if declared is None
else len(apply_target_overrides({str(c) for c in declared}, overrides, tname))
)
found = _target_core_set(tdata or {}, overrides, tname, platforms_dir)
count = None if found is None else len(found)
result.append(
{
"name": tname,
@@ -2533,6 +2543,7 @@ def group_identical_platforms(
cores if isinstance(cores, str) else sorted(map(str, cores or [])),
sorted(map(str, config.get("standalone_cores") or [])),
sorted(map(str, config.get("dual_mode_cores") or [])),
sorted(map(str, config.get("installable_cores") or [])),
# One archive cannot serve a filesystem that folds case and one
# that does not: Lakka keeps bios.col beside BIOS.col.
bool(config.get("case_insensitive_fs")),
@@ -3202,9 +3213,11 @@ def resolve_platform_cores(
# An emulator the platform launches standalone runs there even when
# its own list names only the cores: Batocera's es_systems.yml files
# lexaloffle and gsplus as standalone emulators and nowhere else.
# A core the platform's RetroArch installs from the buildbot runs
# there as well (installable_cores, Batocera x86_64 and armv7).
core_set = {str(c) for c in cores_config} | {
str(c) for c in config.get("standalone_cores") or []
}
} | {str(c) for c in config.get("installable_cores") or []}
index = upstream_profile_index(profiles)
platform = str(config.get("_key") or "")
+21 -2
View File
@@ -18,7 +18,7 @@ from collections.abc import Callable
from pathlib import Path
from common import yaml_load
from common import load_target_config, yaml_load
from hashing import cut_md5_owner
from safeparse import yaml_load_upstream
@@ -557,6 +557,21 @@ def boards_build(selection: BuildSelection, launchers: list[dict]) -> Callable[[
return built
def installable_cores(selection: BuildSelection, cores: list[str]) -> list[str]:
"""Buildbot cores a board's RetroArch core downloader installs, beyond
the cores the image builds.
The targets scraper names, per board, the RetroArch targets entry its
downloader reads (`downloads`): a Batocera x86_64 board installs any
core of linux-x86_64, an armv7 one any of linux-armhf.
"""
found: set[str] = set()
for entry in selection.targets.values():
if entry.get("downloads"):
found |= load_target_config("retroarch", entry["downloads"]) or set()
return sorted(found - set(cores))
def records_run(launchers: list[dict], built: Callable[[str], bool]) -> list[dict]:
"""The launcher records of emulators the build runs.
@@ -623,9 +638,11 @@ class Scraper(BaseScraper):
launchers = extract_launchers(
lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
)
built = boards_build(BoardTargets().build_selection(self.tag), launchers)
selection = BoardTargets().build_selection(self.tag)
built = boards_build(selection, launchers)
launchers = records_run(launchers, built)
cores, standalone, dual = self._fetch_cores(built)
installable = installable_cores(selection, cores)
libretro_systems = es_systems_libretro_systems(
self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
)
@@ -645,6 +662,8 @@ class Scraper(BaseScraper):
if dual:
# Launched both ways: each layout gets its files.
result["dual_mode_cores"] = dual
if installable:
result["installable_cores"] = installable
# The systems each libretro core runs for: its save folders.
result["libretro_systems"] = libretro_systems
result["launchers"] = launchers
@@ -219,6 +219,8 @@ def _parse_meta_flags(text: str) -> list[tuple[str, str]]:
# ("select BR2_PACKAGE_XROAR if BR2_PACKAGE_HAS_LIBGL") reached no board.
BUILDROOT_KCONFIG = (
"arch/Config.in",
# Each armv7-A CPU selects BR2_ARM_CPU_ARMV7A (cortex_a15_a7 included).
"arch/Config.in.arm",
"package/mesa3d/Config.in",
"package/libglvnd/Config.in",
)
@@ -330,6 +332,51 @@ def _parse_es_systems(text: str) -> dict[str, list[str]]:
return package_to_emulators
# RetroArch's core downloader fetches from DEFAULT_BUILDBOT_SERVER_URL, which
# Linux sets for x86_64, i386 and armv7 hard-float alone, and leaves empty on
# aarch64 (config.def.h:1855-1864 at Batocera's 4b74434 and at v1.22.2).
def buildbot_target(symbols: set[str] | frozenset[str]) -> str | None:
"""The RetroArch targets entry a board's core downloader installs from.
`symbols` are the board's active symbols. An armv7-A CPU selects
BR2_ARM_CPU_ARMV7A (arch/Config.in.arm) and is hard-float unless the
board asks for soft-float (BR2_ARM_EABI): Buildroot picks EABIHF for a
CPU with a VFP, which every Cortex-A has. armv6 (bcm2835), aarch64 and
riscv64 get no buildbot URL.
"""
if "BR2_x86_64" in symbols:
return "linux-x86_64"
if "BR2_i386" in symbols:
return "linux-x86"
if "BR2_ARM_CPU_ARMV7A" in symbols and "BR2_ARM_EABI" not in symbols:
return "linux-armhf"
return None
# configure options and make variables that compile the downloader out.
_NO_DOWNLOADER = ("--disable-update-cores", "--disable-online-updater", "--disable-networking",
"HAVE_UPDATE_CORES=0", "HAVE_ONLINE_UPDATER=0", "HAVE_LAKKA=1")
_BLANK_CORES_URL = re.compile(r'^\s*core_updater_buildbot_cores_url\s*=\s*""', re.MULTILINE)
def downloader_disabled(retroarch_mk: str, shipped_cfgs: dict[str, str]) -> str | None:
"""Why RetroArch's core downloader installs nothing, or None.
Compiled out by its build options, or left without a URL by a shipped
retroarch.cfg. A config that hides the menu entry alone does not stop
it: without HAVE_LAKKA, Settings > User Interface offers it back
(menu_setting.c). A config that blanks the URL for some boards only is
a case this reader does not split, and an error.
"""
for option in _NO_DOWNLOADER:
if option in retroarch_mk:
return f"retroarch.mk sets {option}"
blank = sorted(path for path, text in shipped_cfgs.items() if _BLANK_CORES_URL.search(text))
if blank:
raise RuntimeError(f"{', '.join(blank)} blank the core downloader URL: split per board")
return None
# Buildroot's architecture choice, as a board sets it (arch/Config.in).
_ARCH_SYMBOLS = {
"BR2_aarch64": "aarch64",
@@ -361,7 +408,8 @@ def _arch_from_flag(flag: str, board_flags: set[str] | frozenset[str] = frozense
class BuildSelection:
"""What the boards of a build turn on, read once from its Kconfig.
`targets` is each board's entry of the targets file. `emulators` holds
`targets` is each board's entry of the targets file, `downloads` naming
the RetroArch targets entry its core downloader installs from. `emulators` holds
every es_systems name some board builds and `listed` every name
es_systems ties to a package, `symbols` every symbol some board ends up
with, and `packages` the config symbol each package folder of the tree
@@ -412,6 +460,7 @@ class Scraper(BaseTargetScraper):
# Path prefix -> repository of a submodule the root Config.in sources
# from: KNULLI reads Batocera's packages as batocera/package/...
SUBMODULES: dict[str, str] = {}
RETROARCH_MK = "package/batocera/emulators/retroarch/retroarch/retroarch.mk"
def __init__(self, url: str = "") -> None:
super().__init__(url=url or f"https://github.com/{self.REPO}")
@@ -505,6 +554,23 @@ class Scraper(BaseTargetScraper):
print(f" sourced but absent at {self.tag}: {path}", file=sys.stderr)
return files
def _shipped_retroarch_cfgs(self) -> dict[str, str]:
"""Every retroarch.cfg the tree ships, its submodules included."""
repos = [("", self.REPO, self.tag)]
for prefix, repo in self.SUBMODULES.items():
pin = _fetch_json(self._api(prefix.rstrip("/")))
repos.append((prefix, repo, pin["sha"]))
found: dict[str, str] = {}
for prefix, repo, ref in repos:
listing = _fetch_json(f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1")
if not isinstance(listing, dict) or listing.get("truncated"):
raise RuntimeError(f"{repo} tree at {ref} is unreadable or truncated")
for entry in listing.get("tree", []):
if re.search(r"(^|/)retroarch\.cfg$", entry["path"]):
raw = f"https://raw.githubusercontent.com/{repo}/{ref}/{entry['path']}"
found[prefix + entry["path"]] = _fetch(raw)
return found
def _buildroot_rules(self) -> list[tuple[str, str, str]]:
"""Buildroot's selects for the symbols the tree tests, at its pin."""
pin = _fetch_json(self._api("buildroot"))
@@ -560,6 +626,10 @@ class Scraper(BaseTargetScraper):
images_by_flag = _parse_images(config_in_text)
if not images_by_flag:
raise RuntimeError(f"no target image list parsed from {self.CONFIG_IN_PATH}")
disabled = downloader_disabled(_fetch(self._raw(self.RETROARCH_MK)),
self._shipped_retroarch_cfgs())
if disabled:
print(f" core downloader off: {disabled}", file=sys.stderr)
targets: dict[str, dict] = {}
symbols: set[str] = set()
@@ -593,6 +663,9 @@ class Scraper(BaseTargetScraper):
images = sorted(set(images_by_flag.get(primary_flag, [])))
if images and images != [target_key]:
targets[target_key]["images"] = images
downloads = None if disabled else buildbot_target(selected_packages)
if downloads:
targets[target_key]["downloads"] = downloads
print(
f" {len(emulators)} emulators ({len(selected_packages)} packages selected)",
file=sys.stderr,
@@ -35,6 +35,7 @@ class Scraper(BatoceraScraper):
SYSTEM_SYMBOL = "BR2_PACKAGE_KNULLI_SYSTEM"
BUILDROOT_REPO = "knulli-cfw/buildroot"
SUBMODULES = {"batocera/": "batocera-linux/batocera.linux"}
RETROARCH_MK = "package/retroarch/retroarch/retroarch.mk"
def main() -> None:
+73 -3
View File
@@ -97,6 +97,10 @@ class FailedRequestsStopTheScrape(unittest.TestCase):
_refuse()
return _Body(b"BR2_PACKAGE_BATOCERA_TARGET_X86_64=y\n"
b"BR2_PACKAGE_BATOCERA_SYSTEM=y\n")
if "/git/trees/" in url:
return _Body(b'{"tree": [], "truncated": false}')
if url.endswith("retroarch.mk"):
return _Body(b"RETROARCH_CONF_OPTS = --enable-networking\n")
if url.startswith("https://raw.githubusercontent.com/") and url.count("/") == 6:
return _Body(b'source "$BR2_EXTERNAL_BATOCERA_PATH/'
b'package/batocera/core/batocera-system/Config.in"\n')
@@ -243,6 +247,60 @@ class BatoceraDerivedFlags(unittest.TestCase):
self.assertIn("BR2_PACKAGE_DUCKSTATION_LEGACY", r16)
class CoreDownloaderReach(unittest.TestCase):
"""Batocera and KNULLI build RetroArch with its core downloader, shown in
the menu, and its compiled buildbot URL serves Linux x86_64, i386 and
armv7 hard-float alone (config.def.h:743-749, 1855-1864): a board of
those runs any core of the buildbot list, an aarch64 one none."""
def test_each_architecture_reads_its_buildbot_list(self):
from scraper.targets.batocera_targets_scraper import buildbot_target # noqa: PLC0415
self.assertEqual(buildbot_target({"BR2_x86_64"}), "linux-x86_64")
self.assertEqual(buildbot_target({"BR2_arm", "BR2_ARM_CPU_ARMV7A"}), "linux-armhf")
self.assertIsNone(buildbot_target({"BR2_arm", "BR2_ARM_CPU_ARMV7A", "BR2_ARM_EABI"}))
self.assertIsNone(buildbot_target({"BR2_arm", "BR2_arm1176jzf_s"}))
self.assertIsNone(buildbot_target({"BR2_aarch64"}))
def test_a_build_or_a_blank_url_turns_it_off(self):
from scraper.targets.batocera_targets_scraper import downloader_disabled # noqa: PLC0415
self.assertIsNone(downloader_disabled("--enable-networking", {
"a/retroarch.cfg": 'menu_show_core_updater = "false"\n'}))
self.assertIn("HAVE_LAKKA=1", downloader_disabled("make HAVE_LAKKA=1", {}))
with self.assertRaises(RuntimeError):
downloader_disabled("", {"a/retroarch.cfg": 'core_updater_buildbot_cores_url = ""\n'})
def test_a_board_that_downloads_gets_the_buildbot_list(self):
from common import load_target_config # noqa: PLC0415
root = str(REPO_ROOT / "platforms")
x86 = set(load_target_config("retroarch", "linux-x86_64", root) or ())
armhf = set(load_target_config("retroarch", "linux-armhf", root) or ())
self.assertLessEqual(x86, set(load_target_config("batocera", "x86_64", root) or ()))
self.assertLessEqual(armhf, set(load_target_config("batocera", "rk3288", root) or ()))
self.assertFalse(x86 <= set(load_target_config("batocera", "bcm2711", root) or ()))
def test_the_platform_names_what_its_boards_install(self):
import yaml # noqa: PLC0415
from common import load_target_config # noqa: PLC0415
root = str(REPO_ROOT / "platforms")
for platform in ("batocera",):
config = yaml.safe_load((REPO_ROOT / "platforms" / f"{platform}.yml").read_text())
targets = yaml.safe_load(
(REPO_ROOT / "platforms" / "targets" / f"{platform}.yml").read_text())["targets"]
fetched = set()
for entry in targets.values():
if entry.get("downloads"):
fetched |= load_target_config("retroarch", entry["downloads"], root) or set()
if entry["architecture"] == "aarch64":
self.assertNotIn("downloads", entry)
with self.subTest(platform=platform):
self.assertEqual(set(config["installable_cores"]), fetched - set(config["cores"]))
class KnulliPerBoard(unittest.TestCase):
"""KNULLI builds per board on Batocera's model from its own tree:
configs/knulli-*.board, package/system/knulli-system/Config.in and
@@ -257,7 +315,7 @@ class KnulliPerBoard(unittest.TestCase):
"BR2_PACKAGE_BATOCERA_TARGET_H700=y\n"
),
"configs/knulli-r16.board": (
"include knulli-board.common\n\nBR2_arm=y\n"
"include knulli-board.common\n\nBR2_arm=y\nBR2_cortex_a7=y\n"
"BR2_PACKAGE_BATOCERA_TARGET_SUNXI_R16=y\n"
),
"configs/knulli-h700_armhf_libs.board": (
@@ -330,11 +388,21 @@ class KnulliPerBoard(unittest.TestCase):
return _Body('{"sha": "c2d7985b"}')
if url == f"{api}batocera?ref={pin}":
return _Body('{"sha": "487014d9"}')
if "/git/trees/" in url:
# A board config that hides the downloader leaves its URL.
cfg = "board/r16/retroarch.cfg" if "knulli-linux" in url else "x"
return _Body(json.dumps({"tree": [{"path": cfg}], "truncated": False}))
if url == f"{raw}board/r16/retroarch.cfg":
return _Body('menu_show_core_updater = "false"\n')
if url == f"{raw}package/retroarch/retroarch/retroarch.mk":
return _Body("RETROARCH_CONF_OPTS = --enable-networking\n")
if url == ("https://raw.githubusercontent.com/batocera-linux/batocera.linux/"
"487014d9/package/batocera/emulators/ppsspp/Config.in"):
return _Body("config BR2_PACKAGE_PPSSPP\n\tselect BR2_PACKAGE_PPSSPP_ASSETS\n")
if url == f"{raw}package/utils/vtree/Config.in":
raise urllib.error.HTTPError(url, 404, "Not Found", {}, None)
if url.endswith("/c2d7985b/arch/Config.in.arm"):
return _Body("config BR2_cortex_a7\n\tselect BR2_ARM_CPU_ARMV7A\n")
if url.startswith("https://raw.githubusercontent.com/knulli-cfw/buildroot/c2d7985b/"):
return _Body("config BR2_aarch64\n\tselect BR2_ARCH_IS_64\n")
path = url.removeprefix(raw)
@@ -353,10 +421,12 @@ class KnulliPerBoard(unittest.TestCase):
self.assertEqual(data["targets"]["h700"],
{"architecture": "aarch64", "cores": ["assets", "mgba", "ppsspp"],
"images": ["rg35xx-plus", "rg35xx-sp"]})
# An armv7 hard-float board's RetroArch installs the armhf buildbot list.
self.assertEqual(data["targets"]["r16"],
{"architecture": "arm", "cores": ["mgba"], "images": ["miyoo-a30"]})
{"architecture": "arm", "cores": ["mgba"], "images": ["miyoo-a30"],
"downloads": "linux-armhf"})
self.assertTrue(all(pin in url or "buildroot/c2d7985b" in url
or "batocera.linux/487014d9" in url for url in seen), seen)
or "487014d9" in url for url in seen), seen)
class RocknixPerDevice(unittest.TestCase):