diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 545ad2f82..bff39bebe 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -865,7 +865,6 @@ platforms: # 99-distribution.conf:320 passes --strictfilter, an argument the script # does not handle, so that path reports nothing. No release since # 20230203: the scraper reads the commit of the newest prerelease. - # Several device images, no per-device core list: no target file. amberelec: config: amberelec.yml status: active @@ -906,7 +905,8 @@ platforms: # member's md5 and report UNTESTED whatever the archive holds. Run by # EmulationStation's BIOS screen and by ee_check_bios after a failed # launch (99-emuelec.conf:261). naomi and Dinothawr are written twice in - # the dict and the later entry is the one Python keeps. + # the dict and the later entry is the one Python keeps. One image per + # device the release publishes, its cores read by emuelec_targets_scraper. emuelec: config: emuelec.yml status: active @@ -922,8 +922,8 @@ platforms: # RetroArch saves beside each game: no single place. save_directory: null schedule: monthly - target_scraper: null - target_source: null + target_scraper: emuelec_targets + target_source: https://github.com/EmuELEC/EmuELEC/blob/dev/packages/sx05re/emuelec/package.mk install: detect: # EmuELEC d4c7b2f1 scripts/image:172-176, options:4. diff --git a/platforms/targets/_overrides.yml b/platforms/targets/_overrides.yml index d07226f89..c6ed66feb 100644 --- a/platforms/targets/_overrides.yml +++ b/platforms/targets/_overrides.yml @@ -195,7 +195,7 @@ knulli: - rice - vaixterm -# EmuELEC and AmberELEC have no target file: the scrapers read these lists. +# Read by the EmuELEC and AmberELEC platform and target scrapers alike. # mpv and ffplay play the Media Player system's videos (EmuELEC # emuelecRunEmu.sh:301-303, AmberELEC runemu.py:122). gmloader runs a # GameMaker game's own Android library (emuelecRunEmu.sh:331-334). Ports diff --git a/platforms/targets/emuelec.yml b/platforms/targets/emuelec.yml new file mode 100644 index 000000000..31041341a --- /dev/null +++ b/platforms/targets/emuelec.yml @@ -0,0 +1,499 @@ +platform: emuelec +source: https://github.com/EmuELEC/EmuELEC/blob/v4.8/packages/sx05re/emuelec/package.mk +devices_source: https://github.com/EmuELEC/EmuELEC/releases/tag/v4.8 +scraped_at: '2026-11-06T10:27:34Z' +targets: + amlogic-ng: + architecture: aarch64 + cores: + - '2048' + - '81' + - AMIBERRY + - AdvanceMame + - DOSBOXSDL2 + - Duckstation + - ECWolf + - FBNEOSA + - HATARISA + - HYPSEUS + - OPENBOR + - PPSSPPSDL + - Pico-8 + - SCUMMVMSA + - STELLASA + - Solarus + - a5200 + - atari800 + - bgdi + - bk + - bluemsx + - cannonball + - cap32 + - daphne + - dinothawr + - dolphin + - dosbox_pure + - dosbox_svn + - easyrpg + - fake08 + - fbalpha2012 + - fbneo + - fceumm + - fceumm_mod + - flycast + - flycast_32b + - flycastsa + - fmsx + - freeintv + - freej2me + - fuse + - gambatte + - gearboy + - gearcoleco + - geargrafx + - gearlynx + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_cart_special + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - handy + - hatari + - ikemen + - jzintv + - mame + - mame2003_plus + - mame2010 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_saturn + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - mesen + - mesen-s + - mgba + - mrboom + - mupen64plus_32b + - mupen64plus_next + - mupen64plus_next_alt + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - opera + - parallel_n64_32b + - pcsx_rearmed + - pcsx_rearmed_32b + - picodrive + - pocketcdg + - pokemini + - potator + - prboom + - prosystem + - puae + - puae2021 + - px68k + - reminiscence + - same_cdi + - sameboy + - sameduck + - scummvm + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - stella + - swanstation + - tgbdual + - tic80 + - tyrquake + - uae4arm_32b + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xplus4 + - vice_xvic + - virtualjaguar + - x1 + - xrick + - yabasanshiro + gameforce: + architecture: aarch64 + cores: + - '2048' + - '81' + - AMIBERRY + - AdvanceMame + - DOSBOXSDL2 + - ECWolf + - FBNEOSA + - HATARISA + - HYPSEUS + - OPENBOR + - PPSSPPSDL + - Pico-8 + - SCUMMVMSA + - STELLASA + - Solarus + - a5200 + - atari800 + - bgdi + - bk + - bluemsx + - cannonball + - cap32 + - daphne + - dinothawr + - dosbox_pure + - dosbox_svn + - easyrpg + - fake08 + - fbalpha2012 + - fbneo + - fceumm + - fceumm_mod + - flycast + - flycast_32b + - flycastsa + - fmsx + - freeintv + - freej2me + - fuse + - gambatte + - gearboy + - gearcoleco + - geargrafx + - gearlynx + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_cart_special + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - handy + - hatari + - ikemen + - jzintv + - mame2003_plus + - mame2010 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - mgba + - mrboom + - mupen64plus_32b + - mupen64plus_next + - mupen64plus_next_alt + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - opera + - parallel_n64_32b + - pcsx_rearmed + - pcsx_rearmed_32b + - picodrive + - pocketcdg + - pokemini + - potator + - prboom + - prosystem + - puae + - puae2021 + - px68k + - reminiscence + - same_cdi + - sameboy + - sameduck + - scummvm + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - stella + - swanstation + - tgbdual + - tic80 + - tyrquake + - uae4arm_32b + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xplus4 + - vice_xvic + - x1 + - xrick + - yabasanshiro + odroidgoadvance: + architecture: aarch64 + cores: + - '2048' + - '81' + - AMIBERRY + - AdvanceMame + - DOSBOXSDL2 + - ECWolf + - FBNEOSA + - HATARISA + - HYPSEUS + - OPENBOR + - PPSSPPSDL + - Pico-8 + - SCUMMVMSA + - STELLASA + - Solarus + - a5200 + - atari800 + - bgdi + - bk + - bluemsx + - cannonball + - cap32 + - daphne + - dinothawr + - dosbox_pure + - dosbox_svn + - easyrpg + - fake08 + - fbalpha2012 + - fbneo + - fceumm + - fceumm_mod + - flycast + - flycast_32b + - flycastsa + - fmsx + - freeintv + - freej2me + - fuse + - gambatte + - gearboy + - gearcoleco + - geargrafx + - gearlynx + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_cart_special + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - handy + - hatari + - ikemen + - jzintv + - mame2003_plus + - mame2010 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - mgba + - mrboom + - mupen64plus_32b + - mupen64plus_next + - mupen64plus_next_alt + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - opera + - parallel_n64_32b + - pcsx_rearmed + - pcsx_rearmed_32b + - picodrive + - pocketcdg + - pokemini + - potator + - prboom + - prosystem + - puae + - puae2021 + - px68k + - reminiscence + - same_cdi + - sameboy + - sameduck + - scummvm + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - stella + - swanstation + - tgbdual + - tic80 + - tyrquake + - uae4arm_32b + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xplus4 + - vice_xvic + - x1 + - xrick + - yabasanshiro + odroidm1: + architecture: aarch64 + cores: + - '2048' + - '81' + - AMIBERRY + - AdvanceMame + - DOSBOXSDL2 + - Duckstation + - ECWolf + - FBNEOSA + - HATARISA + - HYPSEUS + - OPENBOR + - PPSSPPSDL + - Pico-8 + - SCUMMVMSA + - STELLASA + - Solarus + - a5200 + - atari800 + - bgdi + - bk + - bluemsx + - cannonball + - cap32 + - daphne + - dinothawr + - dolphin + - dosbox_pure + - dosbox_svn + - easyrpg + - fake08 + - fbalpha2012 + - fbneo + - fceumm + - fceumm_mod + - flycast + - flycast_32b + - flycastsa + - fmsx + - freeintv + - freej2me + - fuse + - gambatte + - gearboy + - gearcoleco + - geargrafx + - gearlynx + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_cart_special + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - handy + - hatari + - ikemen + - jzintv + - mame + - mame2003_plus + - mame2010 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_saturn + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - mesen + - mesen-s + - mgba + - mrboom + - mupen64plus_32b + - mupen64plus_next + - mupen64plus_next_alt + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - opera + - parallel_n64_32b + - pcsx_rearmed + - pcsx_rearmed_32b + - picodrive + - pocketcdg + - pokemini + - potator + - prboom + - prosystem + - puae + - puae2021 + - px68k + - reminiscence + - same_cdi + - sameboy + - sameduck + - scummvm + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - stella + - swanstation + - tgbdual + - tic80 + - tyrquake + - uae4arm_32b + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xplus4 + - vice_xvic + - virtualjaguar + - x1 + - xrick + - yabasanshiro diff --git a/scripts/scraper/targets/emuelec_targets_scraper.py b/scripts/scraper/targets/emuelec_targets_scraper.py new file mode 100644 index 000000000..3991c68c2 --- /dev/null +++ b/scripts/scraper/targets/emuelec_targets_scraper.py @@ -0,0 +1,662 @@ +"""Scraper for EmuELEC per-device emulator availability. + +Sources (EmuELEC/EmuELEC, at the tag platforms/emuelec.yml names, v4.8): + - the release's asset names: one `EmuELEC-.-` + image per device (Amlogic-ng, GameForce, OdroidGoAdvance, OdroidM1 at + v4.8). The tree names no device list: projects/ holds the directories + of many more devices than the release builds. + - packages/sx05re/emuelec/package.mk -- the metapackage, whose + PKG_DEPENDS_TARGET is each device's package list (lines 8-77) + - distributions/EmuELEC/options -- LIBRETRO_CORES and + LIBRETRO_S922X_CORES (lines 274-373) + - each core package's package.mk -- PKG_ARCH, which scripts/build:28-30 + reads to skip a package, and the install step naming the .so or binary + - packages/sx05re/emuelec-emulationstation/package.mk -- the cores and + emulators makeinstall_target deletes from es_systems.cfg per device + (lines 114-143) + - es_systems.json and ports.yaml, read the way the platform scraper reads + them + +A device keeps a core when a package it builds installs that core and its +es_systems.cfg still offers it. The OdroidGoAdvance image carries neither +DuckStation nor Dolphin nor the MAME core: without a target list, its pack +shipped their firmware anyway. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import datetime, timezone +from typing import NamedTuple + +import yaml + +from . import BaseTargetScraper + +PLATFORM_NAME = "emuelec" + +REPO = "EmuELEC/EmuELEC" +RAW = "https://raw.githubusercontent.com/EmuELEC/EmuELEC/{ref}/{path}" +RELEASE_API = "https://api.github.com/repos/EmuELEC/EmuELEC/releases/tags/{ref}" +META_MK = "packages/sx05re/emuelec/package.mk" +OPTIONS = "distributions/EmuELEC/options" +ES_MK = "packages/sx05re/emuelec-emulationstation/package.mk" + +try: + from ..base_scraper import fetch_bytes, github_headers + from ..batocera_scraper import _not_cores + from ..emuelec_scraper import ( + BUILDS_OF, ES_SYSTEMS_TEMPLATE, PORTS_TEMPLATE, RUNNERS, port_engines, with_ports, + ) + from ..es_emulators import SystemEmulators, es_cores +except ImportError: + from scraper.base_scraper import fetch_bytes, github_headers + from scraper.batocera_scraper import _not_cores + from scraper.emuelec_scraper import ( + BUILDS_OF, ES_SYSTEMS_TEMPLATE, PORTS_TEMPLATE, RUNNERS, port_engines, with_ports, + ) + from scraper.es_emulators import SystemEmulators, es_cores + +_ASSET_RE = re.compile(r"^EmuELEC-(?P[A-Za-z0-9-]+)\.(?P[a-z0-9_]+)-") + +# The metapackage, line for line. Each rule below transcribes these lines; +# a tag that rewrites one stops the scrape instead of reading it wrong. +_META_LINES = ( + 'PKG_DEPENDS_TARGET="toolchain ${OPENGLES} emuelec-emulationstation retroarch"', + 'PKG_DEPENDS_TARGET+=" emuelec-tools ${PKG_EMUS} ${PKG_EXPERIMENTAL}"', + 'if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || ' + '[ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then', + 'PKG_DEPENDS_TARGET+=" ${LIBRETRO_S922X_CORES}"', + 'if [ "${DEVICE}" == "OdroidGoAdvance" ] || [ "${DEVICE}" == "GameForce" ]; then', + 'PKG_DEPENDS_TARGET+=" kmscon odroidgoa-utils"', + "for discore in duckstation mesen-s virtualjaguar quicknes MC; do", + 'PKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore} | |")', + 'PKG_DEPENDS_TARGET+=" yabasanshiro"', + 'PKG_DEPENDS_TARGET+=" fbterm"', + 'if [ "${ARCH}" == "aarch64" ]; then', + "for discore in quicknes parallel-n64 pcsx_rearmed; do", + 'PKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore}| |")', + 'PKG_DEPENDS_TARGET+=" dolphinSA"', + 'if [ "${DEVICE}" == "Amlogic-old" ]; then', + "for discore in yabasanshiroSA_1_11 yabasanshiroSA_1_5 yabasanshiro same_cdi duckstation; do", + 'PKG_DEPENDS_TARGET+=" mame"', + 'if [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then', + "for discore in flycast-dojo; do", +) +_LIB32_RE = re.compile(r'PKG_DEPENDS_TARGET\+=" (swanstation [^"]*)"') +_BIG = ("Amlogic-ng", "Amlogic-no", "RK356x", "OdroidM1") +_HANDHELD = ("OdroidGoAdvance", "GameForce") + +# What emuelec-emulationstation's makeinstall_target does to es_systems.cfg. +_ES_LINES = ( + 'if [[ "${DEVICE}" != "Amlogic-ng" || "${DEVICE}" != "Amlogic-ne" || ' + '"${DEVICE}" != "Amlogic-no" ]]; then', + 'if [[ ${DEVICE} == "OdroidGoAdvance" || "${DEVICE}" == "GameForce" ]]; then', + 'remove_cores="mesen-s quicknes mame2016 mesen"', + 'remove_cores="mesen-s quicknes mame2016 mesen yabasanshiroSA yabasanshiro"', + "xmlstarlet ed -L -P -d \"/systemList/system[name='saturn']\" ${CORESFILE}", + "xmlstarlet ed -L -P -d \"/systemList/system[name='phillips-cdi']\" ${CORESFILE}", + 'sed -i "s|${discore}||g" ${CORESFILE}', + 'if [[ ${DEVICE} != "OdroidGoAdvance" ]] && [[ "${DEVICE}" != "GameForce" ]]; then', + "xmlstarlet ed -L -P -d \"/systemList/system/emulators/emulator[@name='retrorun']\" ${CORESFILE}", + "xmlstarlet ed -L -P -d \"/systemList/system/emulators/emulator[@name='Duckstation']\" ${CORESFILE}", +) + + +class Install(NamedTuple): + """One line of a package.mk that puts a core on the image.""" + + path: str + line: str + cores: tuple[str, ...] + # The ARCH test the line sits under in makeinstall_target, if any. + arch: str | None = None + + +def _lr(package: str, line: str, *cores: str, arch: str | None = None) -> Install: + return Install(f"packages/sx05re/libretro/{package}/package.mk", line, cores, arch) + + +def _sa(package: str, line: str, *cores: str) -> Install: + return Install(f"packages/sx05re/emulators/{package}/package.mk", line, cores) + + +def _lib32(package: str, line: str, *cores: str) -> Install: + return Install(f"packages/lib32/emuelec/libretro/{package}/package.mk", line, cores) + + +_TO = "${INSTALL}/usr/lib/libretro/" + +# Package -> the ES core names its install step puts on the image, with the +# line that does it (v4.8). A libretro core is the .so it copies to +# /usr/lib/libretro, the name emuelecRunEmu.sh:101-116 loads as +# ${CORE}_libretro. A standalone name is the program the runner's case for +# it starts (emuelecRunEmu.sh:164-353), installed by the line cited, or by +# the package's default make/cmake/meson install where the line cited only +# adds the launch script beside it. +PROVIDERS: dict[str, tuple[Install, ...]] = { + "2048": (_lr("2048", f"cp 2048_libretro.so {_TO}", "2048"),), + "81": (_lr("81", f"cp 81_libretro.so {_TO}", "81"),), + "a5200": (_lr("a5200", f"cp a5200_libretro.so {_TO}", "a5200"),), + "atari800": (_lr("atari800", f"cp atari800_libretro.so {_TO}", "atari800"),), + "beetle-lynx": (_lr("beetle-lynx", f"cp mednafen_lynx_libretro.so {_TO}", "mednafen_lynx"),), + "beetle-ngp": (_lr("beetle-ngp", f"cp mednafen_ngp_libretro.so {_TO}", "mednafen_ngp"),), + "beetle-pce": (_lr("beetle-pce", f"cp mednafen_pce_fast_libretro.so {_TO}", + "mednafen_pce_fast"),), + "beetle-pcfx": (_lr("beetle-pcfx", f"cp mednafen_pcfx_libretro.so {_TO}", "mednafen_pcfx"),), + # line 20: the build makes mednafen_saturn_libretro.so (Makefile TARGET_NAME). + "beetle-saturn": (_lr("beetle-saturn", "cp mednafen_saturn_*.so $INSTALL/usr/lib/libretro/", + "mednafen_saturn"),), + "beetle-supergrafx": (_lr("beetle-supergrafx", f"cp mednafen_supergrafx_libretro.so {_TO}", + "mednafen_supergrafx"),), + "beetle-vb": (_lr("beetle-vb", f"cp mednafen_vb_libretro.so {_TO}", "mednafen_vb"),), + "beetle-wswan": (_lr("beetle-wswan", f"cp mednafen_wswan_libretro.so {_TO}", + "mednafen_wswan"),), + "bk": (_lr("bk", f"cp ${{PKG_BUILD}}/bk_libretro.so {_TO}", "bk"),), + "bluemsx": (_lr("bluemsx", f"cp bluemsx_libretro.so {_TO}", "bluemsx"),), + "cannonball": (_lr("cannonball", f"cp cannonball_libretro.so {_TO}", "cannonball"),), + "cap32": (_lr("cap32", f"cp cap32_libretro.so {_TO}", "cap32"),), + "daphne": (_lr("daphne", f"cp daphne_libretro.so {_TO}", "daphne"),), + "dinothawr": (_lr("dinothawr", f"cp dinothawr_libretro.so {_TO}", "dinothawr"),), + "dosbox-pure": (_lr("dosbox-pure", f"cp dosbox_pure_libretro.so {_TO}dosbox_pure_libretro.so", + "dosbox_pure"),), + "dosbox-svn": (_lr("dosbox-svn", "cp ${PKG_BUILD}/libretro/dosbox_svn_libretro.so " + "${INSTALL}/usr/lib/libretro", "dosbox_svn"),), + "easyrpg": (_lr("easyrpg", f"cp ${{PKG_BUILD}}/.${{TARGET_NAME}}/easyrpg_libretro.so {_TO}", + "easyrpg"),), + "fake_08": (_lr("fake_08", f"cp -v ${{PKG_BUILD}}/platform/libretro/fake08_libretro.so {_TO}", + "fake08"),), + "fbalpha2012": (_lr("fbalpha2012", f"cp fbalpha2012_libretro.so {_TO}", "fbalpha2012"),), + "fbneo": (_lr("fbneo", f"cp ${{PKG_BUILD}}/src/burner/libretro/fbneo_libretro.so {_TO}", + "fbneo"),), + "fceumm": (_lr("fceumm", f"cp fceumm_libretro.so {_TO}", "fceumm"),), + "fceumm-mod": (_lr("fceumm-mod", f"cp ${{PKG_BUILD}}/fceumm_mod_libretro.* {_TO}", + "fceumm_mod"),), + # lines 47-51: flycast_32b on arm, flycast elsewhere. + "flycast": ( + _lr("flycast", f"cp flycast_libretro.so {_TO}flycast_32b_libretro.so", "flycast_32b", + arch="arm"), + _lr("flycast", f"cp flycast_libretro.so {_TO}", "flycast", arch="!arm"), + ), + "fmsx-libretro": (_lr("fmsx-libretro", f"cp fmsx_libretro.so {_TO}", "fmsx"),), + "freeintv": (_lr("freeintv", f"cp freeintv_libretro.so {_TO}", "freeintv"),), + "freej2me": (_lr("freej2me", 'cp `find . -name "freej2me_libretro.so" | xargs echo` ' + "${INSTALL}/usr/lib/libretro", "freej2me"),), + "fuse-libretro": (_lr("fuse-libretro", f"cp fuse_libretro.so {_TO}", "fuse"),), + "gambatte": (_lr("gambatte", f"cp gambatte_libretro.so {_TO}", "gambatte"),), + "gearboy": (_lr("gearboy", f"cp platforms/libretro/gearboy_libretro.so {_TO}", "gearboy"),), + "gearcoleco": (_lr("gearcoleco", f"cp platforms/libretro/gearcoleco_libretro.so {_TO}", + "gearcoleco"),), + "geargrafx": (_lr("geargrafx", f"cp platforms/libretro/geargrafx_libretro.so {_TO}", + "geargrafx"),), + "gearlynx": (_lr("gearlynx", f"cp platforms/libretro/gearlynx_libretro.so {_TO}", + "gearlynx"),), + "gearsystem": (_lr("gearsystem", f"cp platforms/libretro/gearsystem_libretro.so {_TO}", + "gearsystem"),), + "genesis-plus-gx": (_lr("genesis-plus-gx", + f"cp genesis_plus_gx_libretro.so {_TO}genesis_plus_gx_libretro.so", + "genesis_plus_gx"),), + "genesis-plus-gx-wide": (_lr("genesis-plus-gx-wide", "cp genesis_plus_gx_wide_libretro.so " + f"{_TO}genesis_plus_gx_wide_libretro.so", + "genesis_plus_gx_wide"),), + # PKG_NAME is genesis_plus_gx_cart_special (line 4). + "genesis_plus_gx_cart_special": (_lr("genesis_plus_gx_cart_special", + f"cp genesis_plus_gx_libretro.so {_TO}" + "${PKG_NAME}_libretro.so", + "genesis_plus_gx_cart_special"),), + "geolith": (_lr("geolith", f"cp geolith_libretro.so {_TO}", "geolith"),), + "gpsp": (_lr("gpsp", f"cp -v gpsp_libretro.so {_TO}", "gpsp"),), + "gw-libretro": (_lr("gw-libretro", f"cp gw_libretro.so {_TO}", "gw"),), + "handy": (_lr("handy", f"cp handy_libretro.so {_TO}", "handy"),), + "hatari": (_lr("hatari", f"cp ../hatari_libretro.so {_TO}", "hatari"),), + # line 65: the build's only .so is mame_libretro.so. + "mame": (_lr("mame", f"cp *.so {_TO}", "mame"),), + "mame2003-plus": (_lr("mame2003-plus", f"cp mame2003_plus_libretro.so {_TO}", + "mame2003_plus"),), + "mame2010": (_lr("mame2010", f"cp mame2010_libretro.so {_TO}", "mame2010"),), + "meowpc98": (_lr("meowpc98", f"cp libretro/nekop2_libretro.so {_TO}", "nekop2"),), + "mesen": (_lr("mesen", f"cp Libretro/mesen_libretro.so {_TO}", "mesen"),), + "mesen-s": (_lr("mesen-s", f"cp Libretro/mesen-s_libretro.so {_TO}", "mesen-s"),), + "mgba": (_lr("mgba", f"cp mgba_libretro.so {_TO}", "mgba"),), + "mrboom": (_lr("mrboom", f"cp mrboom_libretro.so {_TO}", "mrboom"),), + # PKG_ARCH="arm" (line 25): scripts/build skips it on aarch64. + "mupen64plus": (_lr("mupen64plus", f"cp mupen64plus_libretro.so {_TO}" + "mupen64plus_32b_libretro.so", "mupen64plus_32b"),), + "mupen64plus-nx": (_lr("mupen64plus-nx", f"cp mupen64plus_next_libretro.so {_TO}", + "mupen64plus_next"),), + "mupen64plus-nx-alt": (_lr("mupen64plus-nx-alt", f"cp mupen64plus_next_libretro.so {_TO}" + "mupen64plus_next_alt_libretro.so", "mupen64plus_next_alt"),), + "neocd_libretro": (_lr("neocd_libretro", f"cp ${{PKG_BUILD}}/neocd_libretro.so {_TO}", + "neocd"),), + "nestopia": (_lr("nestopia", f"cp libretro/nestopia_libretro.so {_TO}", "nestopia"),), + "np2kai": (_lr("np2kai", f"cp ${{PKG_BUILD}}/sdl/np2kai_libretro.so {_TO}", "np2kai"),), + "nxengine": (_lr("nxengine", f"cp nxengine_libretro.so {_TO}", "nxengine"),), + "o2em": (_lr("o2em", f"cp o2em_libretro.so {_TO}", "o2em"),), + "opera": (_lr("opera", f"cp opera_libretro.so {_TO}", "opera"),), + # lines 42-46: pcsx_rearmed_32b on arm, pcsx_rearmed elsewhere. + "pcsx_rearmed": ( + _lr("pcsx_rearmed", "cp pcsx_rearmed_libretro.so " + "${INSTALL}${INSTALLTO}/pcsx_rearmed_32b_libretro.so", "pcsx_rearmed_32b", arch="arm"), + _lr("pcsx_rearmed", "cp pcsx_rearmed_libretro.so ${INSTALL}${INSTALLTO}", + "pcsx_rearmed", arch="!arm"), + ), + "picodrive": (_lr("picodrive", f"cp ${{PKG_BUILD}}/picodrive_libretro.so {_TO}", + "picodrive"),), + "pocketcdg": (_lr("pocketcdg", f"cp pocketcdg_libretro.so {_TO}", "pocketcdg"),), + "pokemini": (_lr("pokemini", f"cp pokemini_libretro.so {_TO}", "pokemini"),), + "potator": (_lr("potator", "cp ${PKG_BUILD}/platform/libretro/potator_libretro.so " + f"{_TO}potator_libretro.so", "potator"),), + "prboom": (_lr("prboom", f"cp prboom_libretro.so {_TO}prboom_libretro.so", "prboom"),), + "prosystem": (_lr("prosystem", f"cp prosystem_libretro.so {_TO}", "prosystem"),), + "puae": (_lr("puae", f"cp puae_libretro.so {_TO}", "puae"),), + "puae2021": (_lr("puae2021", f"cp -v puae2021_libretro.so {_TO}", "puae2021"),), + "px68k": (_lr("px68k", f"cp px68k_libretro.so {_TO}", "px68k"),), + "reminiscence": (_lr("reminiscence", f"cp reminiscence_libretro.so {_TO}", "reminiscence"),), + "same_cdi": (_lr("same_cdi", f"cp -v same_cdi_libretro.so {_TO}", "same_cdi"),), + "sameboy": (_lr("sameboy", f"cp libretro/sameboy_libretro.so {_TO}", "sameboy"),), + "sameduck": (_lr("sameduck", f"cp build/bin/sameduck_libretro.so {_TO}", "sameduck"),), + "scummvm": (_lr("scummvm", 'cp "${PKG_BUILD}/backends/platform/libretro/scummvm_libretro."' + "{so,info} ${INSTALL}/usr/lib/libretro/", "scummvm"),), + "snes9x": (_lr("snes9x", f"cp libretro/snes9x_libretro.so {_TO}", "snes9x"),), + "snes9x2002": (_lr("snes9x2002", f"cp snes9x2002_libretro.so {_TO}", "snes9x2002"),), + "snes9x2005_plus": (_lr("snes9x2005_plus", f"cp snes9x2005_plus_libretro.so {_TO}", + "snes9x2005_plus"),), + "snes9x2010": (_lr("snes9x2010", f"cp snes9x2010_libretro.so {_TO}", "snes9x2010"),), + "stella": (_lr("stella", f"cp ${{PKG_BUILD}}/src/os/libretro/stella_libretro.so {_TO}", + "stella"),), + "supafaust": (_lr("supafaust", f"cp -v mednafen_supafaust_libretro.so {_TO}", + "mednafen_supafaust"),), + "swanstation": (_lr("swanstation", "cp ${PKG_BUILD}/.${TARGET_NAME}/swanstation_libretro.so " + f"{_TO}", "swanstation"),), + "tgbdual": (_lr("tgbdual", f"cp tgbdual_libretro.so {_TO}", "tgbdual"),), + "tic-80": (_lr("tic-80", "cp ${PKG_BUILD}/.${TARGET_NAME}/bin/tic80_libretro.so " + f"{_TO}tic80_libretro.so", "tic80"),), + "tyrquake": (_lr("tyrquake", f"cp tyrquake_libretro.so {_TO}", "tyrquake"),), + # PKG_ARCH="arm" (line 25): scripts/build skips it on aarch64. + "uae4arm": (_lr("uae4arm", f"cp uae4arm_libretro.so {_TO}uae4arm_32b_libretro.so", + "uae4arm_32b"),), + "uzem": (_lr("uzem", f"cp uzem_libretro.so {_TO}", "uzem"),), + "vba-next": (_lr("vba-next", f"cp vba_next_libretro.so {_TO}", "vba_next"),), + "vbam": (_lr("vbam", f"cp ../src/libretro/vbam_libretro.so {_TO}", "vbam"),), + "vecx": (_lr("vecx", f"cp vecx_libretro.so {_TO}vecx_libretro.so", "vecx"),), + # line 55: make_target builds one vice__libretro.so per emulator it + # loops over (lines 37-50), x64, x64sc, x128, xplus4 and xvic among them. + "vice": (_lr("vice", f"cp built/vice_*_libretro.so {_TO}", + "vice_x128", "vice_x64", "vice_x64sc", "vice_xplus4", "vice_xvic"),), + "virtualjaguar": (_lr("virtualjaguar", f"cp virtualjaguar_libretro.so {_TO}", + "virtualjaguar"),), + "xmil": (_lr("xmil", f"cp ${{PKG_BUILD}}/libretro/x1_libretro.so {_TO}", "x1"),), + "xrick": (_lr("xrick", f"cp xrick_libretro.so {_TO}", "xrick"),), + "yabasanshiro": (_lr("yabasanshiro", f"cp yabause/src/libretro/yabasanshiro_libretro.so {_TO}", + "yabasanshiro"),), + "lib32-flycast": (_lib32("lib32-flycast", "cp -va flycast_libretro.so " + f"{_TO}flycast_32b_libretro.so", "flycast_32b"),), + "lib32-mupen64plus": (_lib32("lib32-mupen64plus", "cp mupen64plus_libretro.so " + f"{_TO}mupen64plus_32b_libretro.so", "mupen64plus_32b"),), + "lib32-parallel-n64": (_lib32("lib32-parallel-n64", "cp parallel_n64_libretro.so " + f"{_TO}parallel_n64_32b_libretro.so", "parallel_n64_32b"),), + "lib32-pcsx_rearmed": (_lib32("lib32-pcsx_rearmed", "cp pcsx_rearmed_libretro.so " + f"{_TO}pcsx_rearmed_32b_libretro.so", "pcsx_rearmed_32b"),), + "lib32-uae4arm": (_lib32("lib32-uae4arm", "cp uae4arm_libretro.so " + f"{_TO}uae4arm_32b_libretro.so", "uae4arm_32b"),), + # Standalone emulators, by the program the runner starts. + "PPSSPPSDL": (_sa("PPSSPPSDL", 'cp `find . -name "PPSSPPSDL" | xargs echo` ' + "${INSTALL}/usr/bin/PPSSPPSDL", "PPSSPPSDL"),), + "advancemame": (_sa("advancemame", "cp -r ${PKG_BUILD}/obj/mame/linux/blend/advmame " + "${INSTALL}/usr/bin", "AdvanceMame"),), + "amiberry": (_sa("amiberry", "cp -a amiberry* ${INSTALL}/usr/bin/amiberry", "AMIBERRY"),), + "dolphinSA": (_sa("dolphinSA", "cp -rf ${PKG_BUILD}/.${TARGET_NAME}/Binaries/" + "dolphin-emu-nogui ${INSTALL}/usr/bin", "dolphin"),), + # dosbox comes from the meson install, dosbox.start from line 28. + "dosbox-staging": (_sa("dosbox-staging", "cp -a ${PKG_DIR}/scripts/* ${INSTALL}/usr/bin/", + "DOSBOXSDL2"),), + "duckstation": (_sa("duckstation", "cp -rf ${PKG_BUILD}/.${TARGET_NAME}/bin/" + "duckstation-nogui ${INSTALL}/usr/bin", "Duckstation"),), + "ecwolf": (_sa("ecwolf", "cp ${PKG_BUILD}/ecwolf ${INSTALL}/usr/bin/ecwolf", "ECWolf"),), + "fbneoSA": (_sa("fbneoSA", "cp -rf ${PKG_BUILD}/fbneo ${INSTALL}/usr/bin", "FBNEOSA"),), + "flycastsa": (_sa("flycastsa", "cp ${PKG_BUILD}/.${TARGET_NAME}/flycast " + "${INSTALL}/usr/bin/flycast", "flycastsa"),), + "hatarisa": (_sa("hatarisa", "cp src/hatari ${INSTALL}/usr/bin", "HATARISA"),), + # hypseus comes from the cmake install (PKG_TOOLCHAIN, line 13). + "hypseus-singe": (_sa("hypseus-singe", 'PKG_TOOLCHAIN="cmake"', "HYPSEUS"),), + "ikemen-go": (_sa("ikemen-go", 'cp "$PKG_BUILD/bin/Ikemen_Go_"* "$INSTALL/usr/bin/Ikemen_Go"', + "ikemen"),), + "jzintv": (_sa("jzintv", "cp bin/jzintv ${INSTALL}/usr/bin", "jzintv"),), + # The metapackage's dependencies (line 10); mupen64plus itself comes from + # mupen64plussa-ui-console's makeinstall_target. + "mupen64plussa": ( + _sa("mupen64plussa", 'PKG_DEPENDS_TARGET="toolchain mupen64plussa-core ' + "mupen64plussa-ui-console mupen64plussa-audio-sdl mupen64plussa-input-sdl " + 'mupen64plussa-rsp-hle mupen64plussa-video-rice mupen64plussa-video-glide64mk2"', + "mupen64plussa"), + Install("packages/sx05re/emulators/mupen64plussa/mupen64plussa-ui-console/package.mk", + "cp ${PKG_BUILD}/projects/unix/mupen64plus ${UBINDIR}", ()), + ), + "openbor": (_sa("openbor", 'cp `find . -name "OpenBOR.elf" | xargs echo` ' + "${INSTALL}/usr/bin/OpenBOR", "OPENBOR"),), + # scummvm comes from the make install, moved to /usr/bin at line 51. + "scummvmsa": (_sa("scummvmsa", "mv ${INSTALL}/usr/local/bin ${INSTALL}/usr/", "SCUMMVMSA"),), + # solarus-run comes from the cmake install, solarus.sh from line 22. + "solarus": (_sa("solarus", "cp ${PKG_DIR}/scripts/* ${INSTALL}/usr/bin", "Solarus"),), + "stellasa": (_sa("stellasa", "cp -rf ${PKG_BUILD}/stella ${INSTALL}/usr/bin", "STELLASA"),), + "yabasanshiroSA_1_11": (_sa("yabasanshiroSA_1_11", "cp -a ${PKG_BUILD}/src/retro_arena/" + "yabasanshiro ${INSTALL}/usr/bin", "yabasanshiro"),), + "lib32-bennugd-monolithic": (Install( + "packages/lib32/emuelec/ports/lib32-bennugd-monolithic/package.mk", + "cp ${PKG_BUILD}/projects/cmake/bgdi/bgdi ${INSTALL}/usr/bin", ("bgdi",)),), + # The metapackage's own bin/ (line 82) carries pico8.sh, which starts the + # PICO-8 binary the user copies to bios/pico-8. + "emuelec": (Install(META_MK, "cp -rf ${PKG_DIR}/bin ${INSTALL}/usr", ("Pico-8",)),), +} + +# Packages of a device list whose install step puts no ES core on the image. +NO_ES_CORE: dict[str, str] = { + "toolchain": "build toolchain", + "${OPENGLES}": "graphics driver", + "emuelec-emulationstation": "the frontend", + "retroarch": "the libretro frontend", + "lib32-retroarch": "the 32-bit libretro frontend, retroarch32", + "emuelec-tools": "system tools", + "emuelec-32bit-info": "the .info files of the 32-bit cores", + "lib32-essential": "32-bit runtime", + "lib32-libusb": "32-bit library", + "lib32-box86": "x86 translator", + "fbterm": "terminal", + "kmscon": "terminal", + "odroidgoa-utils": "device tools and retrorun, a runner", + "amiberry-lite": "amiberry-lite, which es_systems.json names nowhere", + "yabasanshiroSA_1_5": "yabasanshiro1_5, which es_systems.json names nowhere", + "lib32-droidports": "gmloader, which es_cores drops (_overrides.yml)", + "gme": "gme_libretro.so, which es_systems.json names nowhere", + "nestopiaCV": "nestopiaCV_libretro.so, which es_systems.json names nowhere", + "quasi88": "quasi88_libretro.so, which es_systems.json names nowhere", + "spring": "spring_*_libretro.so, which es_systems.json names nowhere", +} + +# ES core names no package of any device builds, by the package that would. +# The port engines are dependencies of emuelec-ports (its package.mk, +# lines 17-59), and no package of the image depends on emuelec-ports; +# flycast-dojo is in no device list. +UNBUILT: dict[str, str] = { + **dict.fromkeys( + ("CGeniusExe", "Chocolate-Doom", "GZDoom", "LZDoom", "OpenJazz", "RigelEngine", + "VVVVVV", "abuse", "bs", "bstone", "ccleste", "cdogs-sdl", "devilutionx", + "eduke32", "fallout-ce", "fallout2-ce", "fheroes2", "hcl", "hode", "hurrican", + "opentyrian", "prince", "smw", "sonic2013", "soniccd", "sonicmania", "supertux2", + "supertuxkart", "xash3d"), + "emuelec-ports", + ), + "flycast_dojo": "flycast-dojo", +} + + +def release_devices(asset_names: list[str]) -> dict[str, str]: + """{device: architecture} of each image a release publishes.""" + found: dict[str, str] = {} + for name in asset_names: + match = _ASSET_RE.match(name) + if match: + found.setdefault(match.group("device"), match.group("arch")) + if not found: + raise RuntimeError("the EmuELEC release publishes no device image") + return found + + +def _require(text: str, lines: tuple[str, ...], where: str) -> None: + present = {line.strip() for line in text.splitlines()} + missing = [line for line in lines if line not in present] + if missing: + raise RuntimeError(f"{where} no longer reads {missing[0]!r}") + + +def _quoted(text: str, name: str, where: str) -> list[str]: + """The words of `NAME="..."`; a backslash-newline inside the quotes is + removed by bash, so the value reads as one line.""" + match = re.search(rf'^\s*{name}="([^"]*)"', text, re.MULTILINE) + if match is None: + raise RuntimeError(f"{where} sets no {name}") + return match.group(1).replace("\\\n", " ").split() + + +class Metapackage(NamedTuple): + emus: list[str] + experimental: list[str] + s922x: list[str] + lib32: list[str] + + +def read_metapackage(package_mk: str, options: str) -> Metapackage: + """The lists the metapackage rules work on, with PKG_EMUS expanded.""" + _require(package_mk, _META_LINES, META_MK) + emus = _quoted(package_mk, "PKG_EMUS", META_MK) + if emus[:1] != ["${LIBRETRO_CORES}"]: + raise RuntimeError(f"{META_MK}: PKG_EMUS no longer starts with ${{LIBRETRO_CORES}}") + lib32 = _LIB32_RE.search(package_mk) + if lib32 is None: + raise RuntimeError(f"{META_MK}: no aarch64 lib32 list") + return Metapackage( + emus=_quoted(options, "LIBRETRO_CORES", OPTIONS) + emus[1:], + experimental=_quoted(package_mk, "PKG_EXPERIMENTAL", META_MK), + s922x=_quoted(options, "LIBRETRO_S922X_CORES", OPTIONS), + lib32=lib32.group(1).replace("\\\n", " ").split(), + ) + + +def _sed_first(words: str, pattern: str) -> str: + """`echo ${PKG_DEPENDS_TARGET} | sed "s|| |"`: the unquoted echo + joins the words with single spaces, sed replaces the first match only, + anywhere in the line, a word's tail included.""" + line = " ".join(words.split()) + at = line.find(pattern) + return line if at < 0 else f"{line[:at]} {line[at + len(pattern):]}" + + +def device_packages(meta: Metapackage, device: str, arch: str) -> list[str]: + """PKG_DEPENDS_TARGET of the metapackage for one device (lines 8-77).""" + depends = "toolchain ${OPENGLES} emuelec-emulationstation retroarch" + depends += " emuelec-tools " + " ".join(meta.emus) + " " + " ".join(meta.experimental) + if device in _BIG: + depends += " " + " ".join(meta.s922x) + if device in _HANDHELD: + depends += " kmscon odroidgoa-utils" + for discore in ("duckstation", "mesen-s", "virtualjaguar", "quicknes", "MC"): + depends = _sed_first(depends, f"{discore} ") + depends += " yabasanshiro" + else: + depends += " fbterm" + if arch == "aarch64": + for discore in ("quicknes", "parallel-n64", "pcsx_rearmed"): + depends = _sed_first(depends, discore) + depends += " " + " ".join(meta.lib32) + if device in _BIG: + depends += " dolphinSA" + if device == "Amlogic-old": + for discore in ("yabasanshiroSA_1_11", "yabasanshiroSA_1_5", "yabasanshiro", + "same_cdi", "duckstation"): + depends = _sed_first(depends, f"{discore} ") + if device in _BIG: + depends += " mame" + if device in ("RK356x", "OdroidM1"): + depends = _sed_first(depends, "flycast-dojo") + return depends.split() + + +def arch_builds(pkg_arch: str, arch: str) -> bool: + """scripts/build:28-30: an empty PKG_ARCH builds everywhere, `!` + excludes, otherwise the list names the arch or `any`.""" + words = pkg_arch.split() + if not words: + return True + if f"!{arch}" in words: + return False + return arch in words or "any" in words + + +def pkg_arch(package_mk: str) -> str: + match = re.search(r'^PKG_ARCH="([^"]*)"', package_mk, re.MULTILINE) + return match.group(1) if match else "" + + +def es_systems_for(es_json: str, es_mk: str, device: str) -> SystemEmulators: + """es_systems.json as es_systems.cfg reads after makeinstall_target's + deletions on one device (emuelec-emulationstation/package.mk:114-143). + + `sed "s|X||g"` matches a core written without attributes: + generate_es_systems.py gives `default="true"` to the first default core + of a system and `incompatible_extensions` to a core listing them. + """ + _require(es_mk, _ES_LINES, ES_MK) + if device in _HANDHELD: + remove_cores = {"mesen-s", "quicknes", "mame2016", "mesen"} + drop_systems: set[str] = set() + drop_emulators = {"Duckstation"} + elif device == "Amlogic-old": + remove_cores = {"mesen-s", "quicknes", "mame2016", "mesen", "yabasanshiroSA", + "yabasanshiro"} + drop_systems = {"saturn", "phillips-cdi"} + drop_emulators = {"Duckstation", "retrorun"} + else: + # Line 117 holds for every device: no name is all three at once. + remove_cores = set() + drop_systems = set() + drop_emulators = {"retrorun"} + systems: SystemEmulators = {} + for system in json.loads(es_json).get("systems", []): + if system["name"] in drop_systems: + continue + default_set = False + emulators = [] + for emulator in system.get("emulators") or []: + kept = [] + for core in emulator.get("cores") or []: + bare = "incompatible" not in core + if core.get("default", False) and not default_set: + default_set = True + bare = False + if not (bare and core["name"] in remove_cores): + kept.append(core["name"]) + if emulator["name"] not in drop_emulators: + emulators.append((emulator["name"], kept)) + systems[system["name"]] = emulators + return systems + + +def device_cores( + packages: list[str], + arch: str, + archs: dict[str, str], + systems: SystemEmulators, +) -> list[str]: + """ES core names a device both installs and offers. + + `archs` maps each package of PROVIDERS to its PKG_ARCH. + """ + unknown = sorted( + p for p in set(packages) if p not in PROVIDERS and p not in NO_ES_CORE + and p not in UNBUILT.values() + ) + if unknown: + raise RuntimeError(f"{META_MK}: no install step read for {', '.join(unknown)}") + unbuilt = sorted(set(packages) & set(UNBUILT.values())) + if unbuilt: + raise RuntimeError(f"{META_MK}: {unbuilt[0]} is built and its cores are read as absent") + installed: set[str] = set() + for package in [*set(packages), "emuelec"]: + if package not in PROVIDERS or not arch_builds(archs[package], arch): + continue + for install in PROVIDERS[package]: + if install.arch is None or (install.arch == arch) or ( + install.arch.startswith("!") and install.arch[1:] != arch + ): + installed.update(install.cores) + offered, _standalone, _dual = es_cores( + systems, RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME) + ) + return sorted(installed & set(offered)) + + +def check_names(systems: SystemEmulators) -> None: + """Every ES core name is installed by a package read or known unbuilt.""" + offered, _standalone, _dual = es_cores( + systems, RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME) + ) + provided = {core for installs in PROVIDERS.values() for i in installs for core in i.cores} + unread = sorted(set(offered) - provided - set(UNBUILT)) + if unread: + raise RuntimeError(f"es_systems.json names {', '.join(unread)}, no package read installs it") + + +def check_installs(texts: dict[str, str]) -> dict[str, str]: + """{package: PKG_ARCH}, after checking each cited install line is there.""" + archs: dict[str, str] = {} + for package, installs in PROVIDERS.items(): + for install in installs: + _require(texts[install.path], (install.line,), install.path) + archs[package] = pkg_arch(texts[installs[0].path]) + return archs + + +class Scraper(BaseTargetScraper): + """Per-device core lists from EmuELEC's metapackage.""" + + def __init__(self, url: str = "https://github.com/" + REPO) -> None: + super().__init__(url=url) + self.ref = "" + + def _raw(self, path: str) -> str: + return fetch_bytes(RAW.format(ref=self.ref, path=path)).decode("utf-8") + + def fetch_targets(self) -> dict: + self.ref = self.pinned_revision(PLATFORM_NAME) + release = json.loads( + fetch_bytes(RELEASE_API.format(ref=self.ref), headers=github_headers()) + ) + devices = release_devices([asset["name"] for asset in release.get("assets", [])]) + meta = read_metapackage(self._raw(META_MK), self._raw(OPTIONS)) + paths = sorted({i.path for installs in PROVIDERS.values() for i in installs}) + archs = check_installs({path: self._raw(path) for path in paths}) + es_json = fetch_bytes(ES_SYSTEMS_TEMPLATE.format(ref=self.ref)).decode("utf-8") + ports = port_engines(fetch_bytes(PORTS_TEMPLATE.format(ref=self.ref)).decode("utf-8")) + es_mk = self._raw(ES_MK) + check_names(with_ports(es_systems_for(es_json, es_mk, "Amlogic-ng"), ports)) + targets: dict[str, dict] = {} + for device, arch in sorted(devices.items()): + systems = with_ports(es_systems_for(es_json, es_mk, device), ports) + cores = device_cores(device_packages(meta, device, arch), arch, archs, systems) + targets[device.lower()] = {"architecture": arch, "cores": cores} + print(f" {device.lower()} ({arch}): {len(cores)} cores", file=sys.stderr) + return { + "platform": PLATFORM_NAME, + "source": f"{self.url}/blob/{self.ref}/{META_MK}", + "devices_source": release["html_url"], + "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "targets": targets, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + allow_abbrev=False, description="Scrape EmuELEC per-device cores" + ) + parser.add_argument("--dry-run", action="store_true", help="Show target summary") + parser.add_argument("--output", "-o", help="Output YAML file") + args = parser.parse_args() + + scraper = Scraper() + data = scraper.fetch_targets() + if args.dry_run: + for name, info in data["targets"].items(): + print(f" {name} ({info['architecture']}): {len(info['cores'])} cores") + return + if args.output: + scraper.write_output(data, args.output) + print(f"Written to {args.output}") + return + print(yaml.dump(data, default_flow_style=False, sort_keys=False)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_emuelec_targets.py b/tests/test_emuelec_targets.py new file mode 100644 index 000000000..c8f911433 --- /dev/null +++ b/tests/test_emuelec_targets.py @@ -0,0 +1,266 @@ +"""EmuELEC's images differ by device, and only its metapackage says how. + +packages/sx05re/emuelec/package.mk (v4.8) builds the S922X cores, Dolphin +and the MAME core only on Amlogic-ng, Amlogic-no, RK356x and OdroidM1, and +takes DuckStation out on the two handhelds with a sed that replaces the +first match only. With no target list, an OdroidGoAdvance pack carried the +firmware of cores its image never had. +""" + +from __future__ import annotations + +import json +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from scraper.targets import emuelec_targets_scraper as scraper # noqa: E402 + +# packages/sx05re/emuelec/package.mk:8-77 at v4.8, PKG_EMUS and +# PKG_EXPERIMENTAL shortened. +PACKAGE_MK = """PKG_DEPENDS_TARGET="toolchain ${OPENGLES} emuelec-emulationstation retroarch" +PKG_EXPERIMENTAL="nestopiaCV quasi88 yabasanshiroSA_1_11 yabasanshiroSA_1_5 fbneoSA" +PKG_EMUS="${LIBRETRO_CORES} advancemame duckstation pcsx_rearmed flycastsa" +PKG_DEPENDS_TARGET+=" emuelec-tools ${PKG_EMUS} ${PKG_EXPERIMENTAL}" + +if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then +\tPKG_DEPENDS_TARGET+=" ${LIBRETRO_S922X_CORES}" +fi + +if [ "${DEVICE}" == "OdroidGoAdvance" ] || [ "${DEVICE}" == "GameForce" ]; then +\tPKG_DEPENDS_TARGET+=" kmscon odroidgoa-utils" +\tfor discore in duckstation mesen-s virtualjaguar quicknes MC; do +\t\tPKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore} | |") +\tdone +\tPKG_DEPENDS_TARGET+=" yabasanshiro" +else +\tPKG_DEPENDS_TARGET+=" fbterm" +fi + +if [ "${ARCH}" == "aarch64" ]; then + for discore in quicknes parallel-n64 pcsx_rearmed; do +\t\tPKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore}| |") +\tdone + + PKG_DEPENDS_TARGET+=" swanstation \\ + lib32-essential \\ + lib32-pcsx_rearmed \\ + lib32-parallel-n64 + lib32-libusb" + + if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then + PKG_DEPENDS_TARGET+=" dolphinSA" + fi + + if [ "${DEVICE}" == "Amlogic-old" ]; then + for discore in yabasanshiroSA_1_11 yabasanshiroSA_1_5 yabasanshiro same_cdi duckstation; do + PKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore} | |") + done + fi +fi + +if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then +\tPKG_DEPENDS_TARGET+=" mame" +fi + +if [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then + for discore in flycast-dojo; do +\t\tPKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore}| |") +\tdone +fi +""" + +# distributions/EmuELEC/options:274-373 at v4.8, shortened. +OPTIONS = """LIBRETRO_CORES="mgba \\ +parallel-n64 \\ +pcsx_rearmed \\ +flycast" + +LIBRETRO_S922X_CORES="beetle-saturn \\ +mesen \\ +quicknes \\ +virtualjaguar" +""" + +# packages/sx05re/emuelec-emulationstation/package.mk:114-143 at v4.8. +ES_MK = """CORESFILE="${INSTALL}/usr/config/emulationstation/es_systems.cfg" + +if [[ "${DEVICE}" != "Amlogic-ng" || "${DEVICE}" != "Amlogic-ne" || "${DEVICE}" != "Amlogic-no" ]]; then + if [[ ${DEVICE} == "OdroidGoAdvance" || "${DEVICE}" == "GameForce" ]]; then + remove_cores="mesen-s quicknes mame2016 mesen" + elif [ "${DEVICE}" == "Amlogic-old" ]; then + remove_cores="mesen-s quicknes mame2016 mesen yabasanshiroSA yabasanshiro" + xmlstarlet ed -L -P -d "/systemList/system[name='saturn']" ${CORESFILE} + xmlstarlet ed -L -P -d "/systemList/system[name='phillips-cdi']" ${CORESFILE} + xmlstarlet ed -L -P -d "/systemList/system/emulators/emulator[@name='Duckstation']" ${CORESFILE} + fi + for discore in ${remove_cores}; do + sed -i "s|${discore}||g" ${CORESFILE} + sed -i '/^[[:space:]]*$/d' ${CORESFILE} + done +fi + +if [[ ${DEVICE} != "OdroidGoAdvance" ]] && [[ "${DEVICE}" != "GameForce" ]]; then +\txmlstarlet ed -L -P -d "/systemList/system/emulators/emulator[@name='retrorun']" ${CORESFILE} +else +\txmlstarlet ed -L -P -d "/systemList/system/emulators/emulator[@name='Duckstation']" ${CORESFILE} +fi +""" + + +def _system(name: str, *emulators: tuple[str, list[dict]]) -> dict: + return {"name": name, "emulators": [{"name": e, "cores": c} for e, c in emulators]} + + +ES_JSON = json.dumps({"systems": [ + _system("nes", ("libretro", [{"name": "mesen", "default": True}])), + _system("snes", ("libretro", [{"name": "snes9x", "default": True}, {"name": "mesen-s"}])), + _system("psx", + ("libretro", [{"name": "pcsx_rearmed", "default": True}, + {"name": "pcsx_rearmed_32b"}, {"name": "swanstation"}]), + ("Duckstation", [{"name": "Duckstation"}]), + ("retrorun", [{"name": "pcsx_rearmed_32b"}])), + _system("n64", ("libretro", [{"name": "parallel_n64_32b"}])), +]}) + +ARCHS = {"mesen": "", "duckstation": "aarch64", "pcsx_rearmed": "any", + "lib32-pcsx_rearmed": "aarch64", "swanstation": "any", + "lib32-parallel-n64": "aarch64"} + +ASSETS = [ + "EmuELEC-Amlogic-ng.aarch64-4.8-Generic.img.gz", + "EmuELEC-Amlogic-ng.aarch64-4.8.tar", + "EmuELEC-GameForce.aarch64-4.8-chi.img.gz.sha256", + "EmuELEC-OdroidGoAdvance.aarch64-4.8-odroidgo2.tar", + "EmuELEC-OdroidM1.aarch64-4.8.img.gz", +] + + +class TheMetapackageDecidesEachDevice(unittest.TestCase): + def setUp(self): + self.meta = scraper.read_metapackage(PACKAGE_MK, OPTIONS) + + def test_devices_are_the_images_the_release_publishes(self): + self.assertEqual(scraper.release_devices(ASSETS), { + "Amlogic-ng": "aarch64", "GameForce": "aarch64", + "OdroidGoAdvance": "aarch64", "OdroidM1": "aarch64", + }) + + def test_the_lists_the_rules_work_on(self): + self.assertEqual(self.meta.emus, ["mgba", "parallel-n64", "pcsx_rearmed", "flycast", + "advancemame", "duckstation", "pcsx_rearmed", + "flycastsa"]) + self.assertEqual(self.meta.lib32, ["swanstation", "lib32-essential", + "lib32-pcsx_rearmed", "lib32-parallel-n64", + "lib32-libusb"]) + + def test_the_big_devices_build_the_s922x_cores_dolphin_and_mame(self): + packages = scraper.device_packages(self.meta, "Amlogic-ng", "aarch64") + for package in ("beetle-saturn", "mesen", "virtualjaguar", "dolphinSA", "mame", + "duckstation", "fbterm"): + self.assertIn(package, packages) + # quicknes is added, then the aarch64 loop takes it out. + self.assertNotIn("quicknes", packages) + + def test_the_handhelds_lose_duckstation_and_gain_their_tools(self): + packages = scraper.device_packages(self.meta, "OdroidGoAdvance", "aarch64") + for package in ("duckstation", "mesen", "beetle-saturn", "dolphinSA", "mame", "fbterm"): + self.assertNotIn(package, packages) + self.assertEqual(packages[-1:], ["lib32-libusb"]) + self.assertIn("kmscon", packages) + self.assertIn("yabasanshiro", packages) + + def test_sed_takes_out_the_first_match_only(self): + # pcsx_rearmed is written twice, in LIBRETRO_CORES and in PKG_EMUS: + # the aarch64 loop removes the first and the 64-bit core is built. + for device in ("Amlogic-ng", "OdroidGoAdvance"): + packages = scraper.device_packages(self.meta, device, "aarch64") + self.assertEqual(packages.count("pcsx_rearmed"), 1, device) + self.assertNotIn("parallel-n64", packages, device) + + def test_a_pattern_with_a_trailing_space_misses_the_last_word(self): + self.assertEqual(scraper._sed_first("a duckstation", "duckstation "), "a duckstation") + self.assertEqual(scraper._sed_first("a duckstation b", "duckstation "), "a b") + # Without the space, the pattern takes a word's tail too. + self.assertEqual(scraper._sed_first("lib32-pcsx_rearmed", "pcsx_rearmed"), "lib32- ") + + def test_a_changed_rule_stops_the_scrape(self): + changed = PACKAGE_MK.replace("duckstation mesen-s virtualjaguar quicknes MC", + "duckstation mesen-s") + with self.assertRaises(RuntimeError): + scraper.read_metapackage(changed, OPTIONS) + + +class EachPackageBuildsWhereItsArchAllows(unittest.TestCase): + def test_pkg_arch(self): + self.assertTrue(scraper.arch_builds("", "aarch64")) + self.assertTrue(scraper.arch_builds("any", "aarch64")) + self.assertTrue(scraper.arch_builds("aarch64 arm", "aarch64")) + # mupen64plus and uae4arm say PKG_ARCH="arm": skipped on aarch64. + self.assertFalse(scraper.arch_builds("arm", "aarch64")) + self.assertFalse(scraper.arch_builds("any !aarch64", "aarch64")) + + def test_every_cited_install_line_is_read(self): + install = scraper.PROVIDERS["mgba"][0] + text = f'PKG_ARCH="any"\nmakeinstall_target() {{\n {install.line}\n}}\n' + with self.assertRaises(RuntimeError): + scraper._require(text.replace("mgba_libretro", "mgba2_libretro"), + (install.line,), install.path) + scraper._require(text, (install.line,), install.path) + + +class TheFrontendOffersWhatTheDeviceKeeps(unittest.TestCase): + def test_handhelds_drop_bare_cores_and_duckstation(self): + systems = scraper.es_systems_for(ES_JSON, ES_MK, "GameForce") + # mesen is the nes default: written with an attribute, sed misses it. + self.assertEqual(systems["nes"], [("libretro", ["mesen"])]) + self.assertEqual(systems["snes"], [("libretro", ["snes9x"])]) + self.assertEqual([e for e, _ in systems["psx"]], ["libretro", "retrorun"]) + + def test_other_devices_drop_retrorun(self): + systems = scraper.es_systems_for(ES_JSON, ES_MK, "Amlogic-ng") + self.assertEqual([e for e, _ in systems["psx"]], ["libretro", "Duckstation"]) + self.assertEqual(systems["snes"], [("libretro", ["snes9x", "mesen-s"])]) + + def test_a_device_keeps_what_it_builds_and_offers(self): + meta = scraper.read_metapackage(PACKAGE_MK, OPTIONS) + archs = {**dict.fromkeys(scraper.PROVIDERS, "any"), **ARCHS} + cores = {} + for device in ("Amlogic-ng", "OdroidGoAdvance"): + packages = [p for p in scraper.device_packages(meta, device, "aarch64") + if p in scraper.PROVIDERS or p in scraper.NO_ES_CORE] + systems = scraper.es_systems_for(ES_JSON, ES_MK, device) + cores[device] = scraper.device_cores(packages, "aarch64", archs, systems) + self.assertEqual(cores["Amlogic-ng"], ["Duckstation", "mesen", "parallel_n64_32b", + "pcsx_rearmed", "pcsx_rearmed_32b", + "swanstation"]) + self.assertEqual(cores["OdroidGoAdvance"], ["parallel_n64_32b", "pcsx_rearmed", + "pcsx_rearmed_32b", "swanstation"]) + + def test_a_package_never_read_stops_the_scrape(self): + archs = dict.fromkeys(scraper.PROVIDERS, "any") + with self.assertRaises(RuntimeError): + scraper.device_cores(["mgba", "newcore"], "aarch64", archs, {}) + + +class TheCommittedListFollowsTheRules(unittest.TestCase): + def test_the_handhelds_list_no_core_they_do_not_build(self): + import yaml # noqa: PLC0415 + + targets = yaml.safe_load((REPO_ROOT / "platforms" / "targets" / "emuelec.yml") + .read_text(encoding="utf-8"))["targets"] + self.assertEqual(set(targets), {"amlogic-ng", "gameforce", "odroidgoadvance", "odroidm1"}) + big = set(targets["amlogic-ng"]["cores"]) + for device in ("gameforce", "odroidgoadvance"): + self.assertEqual(big - set(targets[device]["cores"]), { + "Duckstation", "dolphin", "mame", "mednafen_saturn", "mesen", "mesen-s", + "virtualjaguar", + }) + self.assertEqual(targets["odroidm1"]["cores"], targets["amlogic-ng"]["cores"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/adding-a-scraper.md b/wiki/adding-a-scraper.md index 7dc29ef32..a8ea434ef 100644 --- a/wiki/adding-a-scraper.md +++ b/wiki/adding-a-scraper.md @@ -339,6 +339,7 @@ Register the target scraper in `_registry.yml`: | `recalbox_targets` | board defconfigs + `system.ini` | Buildroot packages each image enables, cores per system | | `rocknix_targets` | emulators `package.mk` + device options | Cores per SoC block and 32-bit support per device | | `amberelec_targets` | Makefile devices + metapackage `package.mk` + `es_systems.cfg` | Extra cores kept only on the devices the metapackage names | +| `emuelec_targets` | release asset names + metapackage `package.mk` + each core's install step | Packages per device, mapped to cores by the `.so` or program each installs | ## Parser modules diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 38ce5b358..8eb805706 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -56,7 +56,7 @@ Every module in `tests/`, by the area it guards. | Verification, cross-reference and profile checks | synthetic + profiles | `test_bios_layout`, `test_cache_independent_counts`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_names`, `test_profile_paths`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_data_dir_notice`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | | Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_core_data_trees`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | -| Hardware targets and systems | synthetic + target files | `test_amberelec_targets`, `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | +| Hardware targets and systems | synthetic + target files | `test_amberelec_targets`, `test_emuelec_targets`, `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_games`, `test_install_offline`, `test_install_selection`, `test_install_tui` | | Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` | | Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` | diff --git a/wiki/tools.md b/wiki/tools.md index c8911f3b5..65a175d91 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1719,6 +1719,7 @@ Located in `scripts/scraper/targets/`. Each inherits `BaseTargetScraper` and imp | `recalbox_targets_scraper` | board defconfigs + system.ini cores | board | | `rocknix_targets_scraper` | emulators package.mk + device options | device | | `amberelec_targets_scraper` | Makefile devices + metapackage extra cores + es_systems.cfg | device | +| `emuelec_targets_scraper` | release assets + metapackage package.mk + core install steps + es_systems.json | device | | `lakka_targets_scraper` | libretro_cores package.mk + project and device options | device image | | `emudeck_targets_scraper` | EmuScripts (SteamOS and Windows) | operating system | | `retropie_targets_scraper` | scriptmodules + rp_module_flags | platform flag |