From 5e90a455e43448404f01102c3828633b64dbed19 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:09:05 +0100 Subject: [PATCH 1/5] feat: add EmuELEC hardware targets The v4.8 release publishes four images (Amlogic-ng, GameForce, OdroidGoAdvance, OdroidM1, read from its asset names). The metapackage (packages/sx05re/emuelec/package.mk:8-77) builds the S922X cores, dolphinSA and mame only on Amlogic-ng, Amlogic-no, RK356x and OdroidM1, and removes duckstation on the handhelds with a sed that replaces the first match only: pcsx_rearmed, written twice, keeps its 64-bit build. Each package is mapped to its cores by the .so or program its install step copies, PKG_ARCH is applied as scripts/build:28-30 does, and es_systems.cfg is edited per device as emuelec-emulationstation/package.mk:114-143 does. Amlogic-ng and OdroidM1 list 124 cores, the handhelds 117. --- platforms/_registry.yml | 8 +- platforms/targets/_overrides.yml | 2 +- platforms/targets/emuelec.yml | 499 +++++++++++++ .../targets/emuelec_targets_scraper.py | 662 ++++++++++++++++++ tests/test_emuelec_targets.py | 266 +++++++ wiki/adding-a-scraper.md | 1 + wiki/testing-guide.md | 2 +- wiki/tools.md | 1 + 8 files changed, 1435 insertions(+), 6 deletions(-) create mode 100644 platforms/targets/emuelec.yml create mode 100644 scripts/scraper/targets/emuelec_targets_scraper.py create mode 100644 tests/test_emuelec_targets.py diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 545ad2f82..bff39bebe 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -865,7 +865,6 @@ platforms: # 99-distribution.conf:320 passes --strictfilter, an argument the script # does not handle, so that path reports nothing. No release since # 20230203: the scraper reads the commit of the newest prerelease. - # Several device images, no per-device core list: no target file. amberelec: config: amberelec.yml status: active @@ -906,7 +905,8 @@ platforms: # member's md5 and report UNTESTED whatever the archive holds. Run by # EmulationStation's BIOS screen and by ee_check_bios after a failed # launch (99-emuelec.conf:261). naomi and Dinothawr are written twice in - # the dict and the later entry is the one Python keeps. + # the dict and the later entry is the one Python keeps. One image per + # device the release publishes, its cores read by emuelec_targets_scraper. emuelec: config: emuelec.yml status: active @@ -922,8 +922,8 @@ platforms: # RetroArch saves beside each game: no single place. save_directory: null schedule: monthly - target_scraper: null - target_source: null + target_scraper: emuelec_targets + target_source: https://github.com/EmuELEC/EmuELEC/blob/dev/packages/sx05re/emuelec/package.mk install: detect: # EmuELEC d4c7b2f1 scripts/image:172-176, options:4. diff --git a/platforms/targets/_overrides.yml b/platforms/targets/_overrides.yml index d07226f89..c6ed66feb 100644 --- a/platforms/targets/_overrides.yml +++ b/platforms/targets/_overrides.yml @@ -195,7 +195,7 @@ knulli: - rice - vaixterm -# EmuELEC and AmberELEC have no target file: the scrapers read these lists. +# Read by the EmuELEC and AmberELEC platform and target scrapers alike. # mpv and ffplay play the Media Player system's videos (EmuELEC # emuelecRunEmu.sh:301-303, AmberELEC runemu.py:122). gmloader runs a # GameMaker game's own Android library (emuelecRunEmu.sh:331-334). Ports diff --git a/platforms/targets/emuelec.yml b/platforms/targets/emuelec.yml new file mode 100644 index 000000000..31041341a --- /dev/null +++ b/platforms/targets/emuelec.yml @@ -0,0 +1,499 @@ +platform: emuelec +source: https://github.com/EmuELEC/EmuELEC/blob/v4.8/packages/sx05re/emuelec/package.mk +devices_source: https://github.com/EmuELEC/EmuELEC/releases/tag/v4.8 +scraped_at: '2026-11-06T10:27:34Z' +targets: + amlogic-ng: + architecture: aarch64 + cores: + - '2048' + - '81' + - AMIBERRY + - AdvanceMame + - DOSBOXSDL2 + - Duckstation + - ECWolf + - FBNEOSA + - HATARISA + - HYPSEUS + - OPENBOR + - PPSSPPSDL + - Pico-8 + - SCUMMVMSA + - STELLASA + - Solarus + - a5200 + - atari800 + - bgdi + - bk + - bluemsx + - cannonball + - cap32 + - daphne + - dinothawr + - dolphin + - dosbox_pure + - dosbox_svn + - easyrpg + - fake08 + - fbalpha2012 + - fbneo + - fceumm + - fceumm_mod + - flycast + - flycast_32b + - flycastsa + - fmsx + - freeintv + - freej2me + - fuse + - gambatte + - gearboy + - gearcoleco + - geargrafx + - gearlynx + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_cart_special + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - handy + - hatari + - ikemen + - jzintv + - mame + - mame2003_plus + - mame2010 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_saturn + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - mesen + - mesen-s + - mgba + - mrboom + - mupen64plus_32b + - mupen64plus_next + - mupen64plus_next_alt + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - opera + - parallel_n64_32b + - pcsx_rearmed + - pcsx_rearmed_32b + - picodrive + - pocketcdg + - pokemini + - potator + - prboom + - prosystem + - puae + - puae2021 + - px68k + - reminiscence + - same_cdi + - sameboy + - sameduck + - scummvm + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - stella + - swanstation + - tgbdual + - tic80 + - tyrquake + - uae4arm_32b + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xplus4 + - vice_xvic + - virtualjaguar + - x1 + - xrick + - yabasanshiro + gameforce: + architecture: aarch64 + cores: + - '2048' + - '81' + - AMIBERRY + - AdvanceMame + - DOSBOXSDL2 + - ECWolf + - FBNEOSA + - HATARISA + - HYPSEUS + - OPENBOR + - PPSSPPSDL + - Pico-8 + - SCUMMVMSA + - STELLASA + - Solarus + - a5200 + - atari800 + - bgdi + - bk + - bluemsx + - cannonball + - cap32 + - daphne + - dinothawr + - dosbox_pure + - dosbox_svn + - easyrpg + - fake08 + - fbalpha2012 + - fbneo + - fceumm + - fceumm_mod + - flycast + - flycast_32b + - flycastsa + - fmsx + - freeintv + - freej2me + - fuse + - gambatte + - gearboy + - gearcoleco + - geargrafx + - gearlynx + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_cart_special + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - handy + - hatari + - ikemen + - jzintv + - mame2003_plus + - mame2010 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - mgba + - mrboom + - mupen64plus_32b + - mupen64plus_next + - mupen64plus_next_alt + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - opera + - parallel_n64_32b + - pcsx_rearmed + - pcsx_rearmed_32b + - picodrive + - pocketcdg + - pokemini + - potator + - prboom + - prosystem + - puae + - puae2021 + - px68k + - reminiscence + - same_cdi + - sameboy + - sameduck + - scummvm + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - stella + - swanstation + - tgbdual + - tic80 + - tyrquake + - uae4arm_32b + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xplus4 + - vice_xvic + - x1 + - xrick + - yabasanshiro + odroidgoadvance: + architecture: aarch64 + cores: + - '2048' + - '81' + - AMIBERRY + - AdvanceMame + - DOSBOXSDL2 + - ECWolf + - FBNEOSA + - HATARISA + - HYPSEUS + - OPENBOR + - PPSSPPSDL + - Pico-8 + - SCUMMVMSA + - STELLASA + - Solarus + - a5200 + - atari800 + - bgdi + - bk + - bluemsx + - cannonball + - cap32 + - daphne + - dinothawr + - dosbox_pure + - dosbox_svn + - easyrpg + - fake08 + - fbalpha2012 + - fbneo + - fceumm + - fceumm_mod + - flycast + - flycast_32b + - flycastsa + - fmsx + - freeintv + - freej2me + - fuse + - gambatte + - gearboy + - gearcoleco + - geargrafx + - gearlynx + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_cart_special + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - handy + - hatari + - ikemen + - jzintv + - mame2003_plus + - mame2010 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - mgba + - mrboom + - mupen64plus_32b + - mupen64plus_next + - mupen64plus_next_alt + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - opera + - parallel_n64_32b + - pcsx_rearmed + - pcsx_rearmed_32b + - picodrive + - pocketcdg + - pokemini + - potator + - prboom + - prosystem + - puae + - puae2021 + - px68k + - reminiscence + - same_cdi + - sameboy + - sameduck + - scummvm + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - stella + - swanstation + - tgbdual + - tic80 + - tyrquake + - uae4arm_32b + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xplus4 + - vice_xvic + - x1 + - xrick + - yabasanshiro + odroidm1: + architecture: aarch64 + cores: + - '2048' + - '81' + - AMIBERRY + - AdvanceMame + - DOSBOXSDL2 + - Duckstation + - ECWolf + - FBNEOSA + - HATARISA + - HYPSEUS + - OPENBOR + - PPSSPPSDL + - Pico-8 + - SCUMMVMSA + - STELLASA + - Solarus + - a5200 + - atari800 + - bgdi + - bk + - bluemsx + - cannonball + - cap32 + - daphne + - dinothawr + - dolphin + - dosbox_pure + - dosbox_svn + - easyrpg + - fake08 + - fbalpha2012 + - fbneo + - fceumm + - fceumm_mod + - flycast + - flycast_32b + - flycastsa + - fmsx + - freeintv + - freej2me + - fuse + - gambatte + - gearboy + - gearcoleco + - geargrafx + - gearlynx + - gearsystem + - genesis_plus_gx + - genesis_plus_gx_cart_special + - genesis_plus_gx_wide + - geolith + - gpsp + - gw + - handy + - hatari + - ikemen + - jzintv + - mame + - mame2003_plus + - mame2010 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_saturn + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - mesen + - mesen-s + - mgba + - mrboom + - mupen64plus_32b + - mupen64plus_next + - mupen64plus_next_alt + - mupen64plussa + - nekop2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - opera + - parallel_n64_32b + - pcsx_rearmed + - pcsx_rearmed_32b + - picodrive + - pocketcdg + - pokemini + - potator + - prboom + - prosystem + - puae + - puae2021 + - px68k + - reminiscence + - same_cdi + - sameboy + - sameduck + - scummvm + - snes9x + - snes9x2002 + - snes9x2005_plus + - snes9x2010 + - stella + - swanstation + - tgbdual + - tic80 + - tyrquake + - uae4arm_32b + - uzem + - vba_next + - vbam + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xplus4 + - vice_xvic + - virtualjaguar + - x1 + - xrick + - yabasanshiro diff --git a/scripts/scraper/targets/emuelec_targets_scraper.py b/scripts/scraper/targets/emuelec_targets_scraper.py new file mode 100644 index 000000000..3991c68c2 --- /dev/null +++ b/scripts/scraper/targets/emuelec_targets_scraper.py @@ -0,0 +1,662 @@ +"""Scraper for EmuELEC per-device emulator availability. + +Sources (EmuELEC/EmuELEC, at the tag platforms/emuelec.yml names, v4.8): + - the release's asset names: one `EmuELEC-.-` + image per device (Amlogic-ng, GameForce, OdroidGoAdvance, OdroidM1 at + v4.8). The tree names no device list: projects/ holds the directories + of many more devices than the release builds. + - packages/sx05re/emuelec/package.mk -- the metapackage, whose + PKG_DEPENDS_TARGET is each device's package list (lines 8-77) + - distributions/EmuELEC/options -- LIBRETRO_CORES and + LIBRETRO_S922X_CORES (lines 274-373) + - each core package's package.mk -- PKG_ARCH, which scripts/build:28-30 + reads to skip a package, and the install step naming the .so or binary + - packages/sx05re/emuelec-emulationstation/package.mk -- the cores and + emulators makeinstall_target deletes from es_systems.cfg per device + (lines 114-143) + - es_systems.json and ports.yaml, read the way the platform scraper reads + them + +A device keeps a core when a package it builds installs that core and its +es_systems.cfg still offers it. The OdroidGoAdvance image carries neither +DuckStation nor Dolphin nor the MAME core: without a target list, its pack +shipped their firmware anyway. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import datetime, timezone +from typing import NamedTuple + +import yaml + +from . import BaseTargetScraper + +PLATFORM_NAME = "emuelec" + +REPO = "EmuELEC/EmuELEC" +RAW = "https://raw.githubusercontent.com/EmuELEC/EmuELEC/{ref}/{path}" +RELEASE_API = "https://api.github.com/repos/EmuELEC/EmuELEC/releases/tags/{ref}" +META_MK = "packages/sx05re/emuelec/package.mk" +OPTIONS = "distributions/EmuELEC/options" +ES_MK = "packages/sx05re/emuelec-emulationstation/package.mk" + +try: + from ..base_scraper import fetch_bytes, github_headers + from ..batocera_scraper import _not_cores + from ..emuelec_scraper import ( + BUILDS_OF, ES_SYSTEMS_TEMPLATE, PORTS_TEMPLATE, RUNNERS, port_engines, with_ports, + ) + from ..es_emulators import SystemEmulators, es_cores +except ImportError: + from scraper.base_scraper import fetch_bytes, github_headers + from scraper.batocera_scraper import _not_cores + from scraper.emuelec_scraper import ( + BUILDS_OF, ES_SYSTEMS_TEMPLATE, PORTS_TEMPLATE, RUNNERS, port_engines, with_ports, + ) + from scraper.es_emulators import SystemEmulators, es_cores + +_ASSET_RE = re.compile(r"^EmuELEC-(?P[A-Za-z0-9-]+)\.(?P[a-z0-9_]+)-") + +# The metapackage, line for line. Each rule below transcribes these lines; +# a tag that rewrites one stops the scrape instead of reading it wrong. +_META_LINES = ( + 'PKG_DEPENDS_TARGET="toolchain ${OPENGLES} emuelec-emulationstation retroarch"', + 'PKG_DEPENDS_TARGET+=" emuelec-tools ${PKG_EMUS} ${PKG_EXPERIMENTAL}"', + 'if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || ' + '[ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then', + 'PKG_DEPENDS_TARGET+=" ${LIBRETRO_S922X_CORES}"', + 'if [ "${DEVICE}" == "OdroidGoAdvance" ] || [ "${DEVICE}" == "GameForce" ]; then', + 'PKG_DEPENDS_TARGET+=" kmscon odroidgoa-utils"', + "for discore in duckstation mesen-s virtualjaguar quicknes MC; do", + 'PKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore} | |")', + 'PKG_DEPENDS_TARGET+=" yabasanshiro"', + 'PKG_DEPENDS_TARGET+=" fbterm"', + 'if [ "${ARCH}" == "aarch64" ]; then', + "for discore in quicknes parallel-n64 pcsx_rearmed; do", + 'PKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore}| |")', + 'PKG_DEPENDS_TARGET+=" dolphinSA"', + 'if [ "${DEVICE}" == "Amlogic-old" ]; then', + "for discore in yabasanshiroSA_1_11 yabasanshiroSA_1_5 yabasanshiro same_cdi duckstation; do", + 'PKG_DEPENDS_TARGET+=" mame"', + 'if [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then', + "for discore in flycast-dojo; do", +) +_LIB32_RE = re.compile(r'PKG_DEPENDS_TARGET\+=" (swanstation [^"]*)"') +_BIG = ("Amlogic-ng", "Amlogic-no", "RK356x", "OdroidM1") +_HANDHELD = ("OdroidGoAdvance", "GameForce") + +# What emuelec-emulationstation's makeinstall_target does to es_systems.cfg. +_ES_LINES = ( + 'if [[ "${DEVICE}" != "Amlogic-ng" || "${DEVICE}" != "Amlogic-ne" || ' + '"${DEVICE}" != "Amlogic-no" ]]; then', + 'if [[ ${DEVICE} == "OdroidGoAdvance" || "${DEVICE}" == "GameForce" ]]; then', + 'remove_cores="mesen-s quicknes mame2016 mesen"', + 'remove_cores="mesen-s quicknes mame2016 mesen yabasanshiroSA yabasanshiro"', + "xmlstarlet ed -L -P -d \"/systemList/system[name='saturn']\" ${CORESFILE}", + "xmlstarlet ed -L -P -d \"/systemList/system[name='phillips-cdi']\" ${CORESFILE}", + 'sed -i "s|${discore}||g" ${CORESFILE}', + 'if [[ ${DEVICE} != "OdroidGoAdvance" ]] && [[ "${DEVICE}" != "GameForce" ]]; then', + "xmlstarlet ed -L -P -d \"/systemList/system/emulators/emulator[@name='retrorun']\" ${CORESFILE}", + "xmlstarlet ed -L -P -d \"/systemList/system/emulators/emulator[@name='Duckstation']\" ${CORESFILE}", +) + + +class Install(NamedTuple): + """One line of a package.mk that puts a core on the image.""" + + path: str + line: str + cores: tuple[str, ...] + # The ARCH test the line sits under in makeinstall_target, if any. + arch: str | None = None + + +def _lr(package: str, line: str, *cores: str, arch: str | None = None) -> Install: + return Install(f"packages/sx05re/libretro/{package}/package.mk", line, cores, arch) + + +def _sa(package: str, line: str, *cores: str) -> Install: + return Install(f"packages/sx05re/emulators/{package}/package.mk", line, cores) + + +def _lib32(package: str, line: str, *cores: str) -> Install: + return Install(f"packages/lib32/emuelec/libretro/{package}/package.mk", line, cores) + + +_TO = "${INSTALL}/usr/lib/libretro/" + +# Package -> the ES core names its install step puts on the image, with the +# line that does it (v4.8). A libretro core is the .so it copies to +# /usr/lib/libretro, the name emuelecRunEmu.sh:101-116 loads as +# ${CORE}_libretro. A standalone name is the program the runner's case for +# it starts (emuelecRunEmu.sh:164-353), installed by the line cited, or by +# the package's default make/cmake/meson install where the line cited only +# adds the launch script beside it. +PROVIDERS: dict[str, tuple[Install, ...]] = { + "2048": (_lr("2048", f"cp 2048_libretro.so {_TO}", "2048"),), + "81": (_lr("81", f"cp 81_libretro.so {_TO}", "81"),), + "a5200": (_lr("a5200", f"cp a5200_libretro.so {_TO}", "a5200"),), + "atari800": (_lr("atari800", f"cp atari800_libretro.so {_TO}", "atari800"),), + "beetle-lynx": (_lr("beetle-lynx", f"cp mednafen_lynx_libretro.so {_TO}", "mednafen_lynx"),), + "beetle-ngp": (_lr("beetle-ngp", f"cp mednafen_ngp_libretro.so {_TO}", "mednafen_ngp"),), + "beetle-pce": (_lr("beetle-pce", f"cp mednafen_pce_fast_libretro.so {_TO}", + "mednafen_pce_fast"),), + "beetle-pcfx": (_lr("beetle-pcfx", f"cp mednafen_pcfx_libretro.so {_TO}", "mednafen_pcfx"),), + # line 20: the build makes mednafen_saturn_libretro.so (Makefile TARGET_NAME). + "beetle-saturn": (_lr("beetle-saturn", "cp mednafen_saturn_*.so $INSTALL/usr/lib/libretro/", + "mednafen_saturn"),), + "beetle-supergrafx": (_lr("beetle-supergrafx", f"cp mednafen_supergrafx_libretro.so {_TO}", + "mednafen_supergrafx"),), + "beetle-vb": (_lr("beetle-vb", f"cp mednafen_vb_libretro.so {_TO}", "mednafen_vb"),), + "beetle-wswan": (_lr("beetle-wswan", f"cp mednafen_wswan_libretro.so {_TO}", + "mednafen_wswan"),), + "bk": (_lr("bk", f"cp ${{PKG_BUILD}}/bk_libretro.so {_TO}", "bk"),), + "bluemsx": (_lr("bluemsx", f"cp bluemsx_libretro.so {_TO}", "bluemsx"),), + "cannonball": (_lr("cannonball", f"cp cannonball_libretro.so {_TO}", "cannonball"),), + "cap32": (_lr("cap32", f"cp cap32_libretro.so {_TO}", "cap32"),), + "daphne": (_lr("daphne", f"cp daphne_libretro.so {_TO}", "daphne"),), + "dinothawr": (_lr("dinothawr", f"cp dinothawr_libretro.so {_TO}", "dinothawr"),), + "dosbox-pure": (_lr("dosbox-pure", f"cp dosbox_pure_libretro.so {_TO}dosbox_pure_libretro.so", + "dosbox_pure"),), + "dosbox-svn": (_lr("dosbox-svn", "cp ${PKG_BUILD}/libretro/dosbox_svn_libretro.so " + "${INSTALL}/usr/lib/libretro", "dosbox_svn"),), + "easyrpg": (_lr("easyrpg", f"cp ${{PKG_BUILD}}/.${{TARGET_NAME}}/easyrpg_libretro.so {_TO}", + "easyrpg"),), + "fake_08": (_lr("fake_08", f"cp -v ${{PKG_BUILD}}/platform/libretro/fake08_libretro.so {_TO}", + "fake08"),), + "fbalpha2012": (_lr("fbalpha2012", f"cp fbalpha2012_libretro.so {_TO}", "fbalpha2012"),), + "fbneo": (_lr("fbneo", f"cp ${{PKG_BUILD}}/src/burner/libretro/fbneo_libretro.so {_TO}", + "fbneo"),), + "fceumm": (_lr("fceumm", f"cp fceumm_libretro.so {_TO}", "fceumm"),), + "fceumm-mod": (_lr("fceumm-mod", f"cp ${{PKG_BUILD}}/fceumm_mod_libretro.* {_TO}", + "fceumm_mod"),), + # lines 47-51: flycast_32b on arm, flycast elsewhere. + "flycast": ( + _lr("flycast", f"cp flycast_libretro.so {_TO}flycast_32b_libretro.so", "flycast_32b", + arch="arm"), + _lr("flycast", f"cp flycast_libretro.so {_TO}", "flycast", arch="!arm"), + ), + "fmsx-libretro": (_lr("fmsx-libretro", f"cp fmsx_libretro.so {_TO}", "fmsx"),), + "freeintv": (_lr("freeintv", f"cp freeintv_libretro.so {_TO}", "freeintv"),), + "freej2me": (_lr("freej2me", 'cp `find . -name "freej2me_libretro.so" | xargs echo` ' + "${INSTALL}/usr/lib/libretro", "freej2me"),), + "fuse-libretro": (_lr("fuse-libretro", f"cp fuse_libretro.so {_TO}", "fuse"),), + "gambatte": (_lr("gambatte", f"cp gambatte_libretro.so {_TO}", "gambatte"),), + "gearboy": (_lr("gearboy", f"cp platforms/libretro/gearboy_libretro.so {_TO}", "gearboy"),), + "gearcoleco": (_lr("gearcoleco", f"cp platforms/libretro/gearcoleco_libretro.so {_TO}", + "gearcoleco"),), + "geargrafx": (_lr("geargrafx", f"cp platforms/libretro/geargrafx_libretro.so {_TO}", + "geargrafx"),), + "gearlynx": (_lr("gearlynx", f"cp platforms/libretro/gearlynx_libretro.so {_TO}", + "gearlynx"),), + "gearsystem": (_lr("gearsystem", f"cp platforms/libretro/gearsystem_libretro.so {_TO}", + "gearsystem"),), + "genesis-plus-gx": (_lr("genesis-plus-gx", + f"cp genesis_plus_gx_libretro.so {_TO}genesis_plus_gx_libretro.so", + "genesis_plus_gx"),), + "genesis-plus-gx-wide": (_lr("genesis-plus-gx-wide", "cp genesis_plus_gx_wide_libretro.so " + f"{_TO}genesis_plus_gx_wide_libretro.so", + "genesis_plus_gx_wide"),), + # PKG_NAME is genesis_plus_gx_cart_special (line 4). + "genesis_plus_gx_cart_special": (_lr("genesis_plus_gx_cart_special", + f"cp genesis_plus_gx_libretro.so {_TO}" + "${PKG_NAME}_libretro.so", + "genesis_plus_gx_cart_special"),), + "geolith": (_lr("geolith", f"cp geolith_libretro.so {_TO}", "geolith"),), + "gpsp": (_lr("gpsp", f"cp -v gpsp_libretro.so {_TO}", "gpsp"),), + "gw-libretro": (_lr("gw-libretro", f"cp gw_libretro.so {_TO}", "gw"),), + "handy": (_lr("handy", f"cp handy_libretro.so {_TO}", "handy"),), + "hatari": (_lr("hatari", f"cp ../hatari_libretro.so {_TO}", "hatari"),), + # line 65: the build's only .so is mame_libretro.so. + "mame": (_lr("mame", f"cp *.so {_TO}", "mame"),), + "mame2003-plus": (_lr("mame2003-plus", f"cp mame2003_plus_libretro.so {_TO}", + "mame2003_plus"),), + "mame2010": (_lr("mame2010", f"cp mame2010_libretro.so {_TO}", "mame2010"),), + "meowpc98": (_lr("meowpc98", f"cp libretro/nekop2_libretro.so {_TO}", "nekop2"),), + "mesen": (_lr("mesen", f"cp Libretro/mesen_libretro.so {_TO}", "mesen"),), + "mesen-s": (_lr("mesen-s", f"cp Libretro/mesen-s_libretro.so {_TO}", "mesen-s"),), + "mgba": (_lr("mgba", f"cp mgba_libretro.so {_TO}", "mgba"),), + "mrboom": (_lr("mrboom", f"cp mrboom_libretro.so {_TO}", "mrboom"),), + # PKG_ARCH="arm" (line 25): scripts/build skips it on aarch64. + "mupen64plus": (_lr("mupen64plus", f"cp mupen64plus_libretro.so {_TO}" + "mupen64plus_32b_libretro.so", "mupen64plus_32b"),), + "mupen64plus-nx": (_lr("mupen64plus-nx", f"cp mupen64plus_next_libretro.so {_TO}", + "mupen64plus_next"),), + "mupen64plus-nx-alt": (_lr("mupen64plus-nx-alt", f"cp mupen64plus_next_libretro.so {_TO}" + "mupen64plus_next_alt_libretro.so", "mupen64plus_next_alt"),), + "neocd_libretro": (_lr("neocd_libretro", f"cp ${{PKG_BUILD}}/neocd_libretro.so {_TO}", + "neocd"),), + "nestopia": (_lr("nestopia", f"cp libretro/nestopia_libretro.so {_TO}", "nestopia"),), + "np2kai": (_lr("np2kai", f"cp ${{PKG_BUILD}}/sdl/np2kai_libretro.so {_TO}", "np2kai"),), + "nxengine": (_lr("nxengine", f"cp nxengine_libretro.so {_TO}", "nxengine"),), + "o2em": (_lr("o2em", f"cp o2em_libretro.so {_TO}", "o2em"),), + "opera": (_lr("opera", f"cp opera_libretro.so {_TO}", "opera"),), + # lines 42-46: pcsx_rearmed_32b on arm, pcsx_rearmed elsewhere. + "pcsx_rearmed": ( + _lr("pcsx_rearmed", "cp pcsx_rearmed_libretro.so " + "${INSTALL}${INSTALLTO}/pcsx_rearmed_32b_libretro.so", "pcsx_rearmed_32b", arch="arm"), + _lr("pcsx_rearmed", "cp pcsx_rearmed_libretro.so ${INSTALL}${INSTALLTO}", + "pcsx_rearmed", arch="!arm"), + ), + "picodrive": (_lr("picodrive", f"cp ${{PKG_BUILD}}/picodrive_libretro.so {_TO}", + "picodrive"),), + "pocketcdg": (_lr("pocketcdg", f"cp pocketcdg_libretro.so {_TO}", "pocketcdg"),), + "pokemini": (_lr("pokemini", f"cp pokemini_libretro.so {_TO}", "pokemini"),), + "potator": (_lr("potator", "cp ${PKG_BUILD}/platform/libretro/potator_libretro.so " + f"{_TO}potator_libretro.so", "potator"),), + "prboom": (_lr("prboom", f"cp prboom_libretro.so {_TO}prboom_libretro.so", "prboom"),), + "prosystem": (_lr("prosystem", f"cp prosystem_libretro.so {_TO}", "prosystem"),), + "puae": (_lr("puae", f"cp puae_libretro.so {_TO}", "puae"),), + "puae2021": (_lr("puae2021", f"cp -v puae2021_libretro.so {_TO}", "puae2021"),), + "px68k": (_lr("px68k", f"cp px68k_libretro.so {_TO}", "px68k"),), + "reminiscence": (_lr("reminiscence", f"cp reminiscence_libretro.so {_TO}", "reminiscence"),), + "same_cdi": (_lr("same_cdi", f"cp -v same_cdi_libretro.so {_TO}", "same_cdi"),), + "sameboy": (_lr("sameboy", f"cp libretro/sameboy_libretro.so {_TO}", "sameboy"),), + "sameduck": (_lr("sameduck", f"cp build/bin/sameduck_libretro.so {_TO}", "sameduck"),), + "scummvm": (_lr("scummvm", 'cp "${PKG_BUILD}/backends/platform/libretro/scummvm_libretro."' + "{so,info} ${INSTALL}/usr/lib/libretro/", "scummvm"),), + "snes9x": (_lr("snes9x", f"cp libretro/snes9x_libretro.so {_TO}", "snes9x"),), + "snes9x2002": (_lr("snes9x2002", f"cp snes9x2002_libretro.so {_TO}", "snes9x2002"),), + "snes9x2005_plus": (_lr("snes9x2005_plus", f"cp snes9x2005_plus_libretro.so {_TO}", + "snes9x2005_plus"),), + "snes9x2010": (_lr("snes9x2010", f"cp snes9x2010_libretro.so {_TO}", "snes9x2010"),), + "stella": (_lr("stella", f"cp ${{PKG_BUILD}}/src/os/libretro/stella_libretro.so {_TO}", + "stella"),), + "supafaust": (_lr("supafaust", f"cp -v mednafen_supafaust_libretro.so {_TO}", + "mednafen_supafaust"),), + "swanstation": (_lr("swanstation", "cp ${PKG_BUILD}/.${TARGET_NAME}/swanstation_libretro.so " + f"{_TO}", "swanstation"),), + "tgbdual": (_lr("tgbdual", f"cp tgbdual_libretro.so {_TO}", "tgbdual"),), + "tic-80": (_lr("tic-80", "cp ${PKG_BUILD}/.${TARGET_NAME}/bin/tic80_libretro.so " + f"{_TO}tic80_libretro.so", "tic80"),), + "tyrquake": (_lr("tyrquake", f"cp tyrquake_libretro.so {_TO}", "tyrquake"),), + # PKG_ARCH="arm" (line 25): scripts/build skips it on aarch64. + "uae4arm": (_lr("uae4arm", f"cp uae4arm_libretro.so {_TO}uae4arm_32b_libretro.so", + "uae4arm_32b"),), + "uzem": (_lr("uzem", f"cp uzem_libretro.so {_TO}", "uzem"),), + "vba-next": (_lr("vba-next", f"cp vba_next_libretro.so {_TO}", "vba_next"),), + "vbam": (_lr("vbam", f"cp ../src/libretro/vbam_libretro.so {_TO}", "vbam"),), + "vecx": (_lr("vecx", f"cp vecx_libretro.so {_TO}vecx_libretro.so", "vecx"),), + # line 55: make_target builds one vice__libretro.so per emulator it + # loops over (lines 37-50), x64, x64sc, x128, xplus4 and xvic among them. + "vice": (_lr("vice", f"cp built/vice_*_libretro.so {_TO}", + "vice_x128", "vice_x64", "vice_x64sc", "vice_xplus4", "vice_xvic"),), + "virtualjaguar": (_lr("virtualjaguar", f"cp virtualjaguar_libretro.so {_TO}", + "virtualjaguar"),), + "xmil": (_lr("xmil", f"cp ${{PKG_BUILD}}/libretro/x1_libretro.so {_TO}", "x1"),), + "xrick": (_lr("xrick", f"cp xrick_libretro.so {_TO}", "xrick"),), + "yabasanshiro": (_lr("yabasanshiro", f"cp yabause/src/libretro/yabasanshiro_libretro.so {_TO}", + "yabasanshiro"),), + "lib32-flycast": (_lib32("lib32-flycast", "cp -va flycast_libretro.so " + f"{_TO}flycast_32b_libretro.so", "flycast_32b"),), + "lib32-mupen64plus": (_lib32("lib32-mupen64plus", "cp mupen64plus_libretro.so " + f"{_TO}mupen64plus_32b_libretro.so", "mupen64plus_32b"),), + "lib32-parallel-n64": (_lib32("lib32-parallel-n64", "cp parallel_n64_libretro.so " + f"{_TO}parallel_n64_32b_libretro.so", "parallel_n64_32b"),), + "lib32-pcsx_rearmed": (_lib32("lib32-pcsx_rearmed", "cp pcsx_rearmed_libretro.so " + f"{_TO}pcsx_rearmed_32b_libretro.so", "pcsx_rearmed_32b"),), + "lib32-uae4arm": (_lib32("lib32-uae4arm", "cp uae4arm_libretro.so " + f"{_TO}uae4arm_32b_libretro.so", "uae4arm_32b"),), + # Standalone emulators, by the program the runner starts. + "PPSSPPSDL": (_sa("PPSSPPSDL", 'cp `find . -name "PPSSPPSDL" | xargs echo` ' + "${INSTALL}/usr/bin/PPSSPPSDL", "PPSSPPSDL"),), + "advancemame": (_sa("advancemame", "cp -r ${PKG_BUILD}/obj/mame/linux/blend/advmame " + "${INSTALL}/usr/bin", "AdvanceMame"),), + "amiberry": (_sa("amiberry", "cp -a amiberry* ${INSTALL}/usr/bin/amiberry", "AMIBERRY"),), + "dolphinSA": (_sa("dolphinSA", "cp -rf ${PKG_BUILD}/.${TARGET_NAME}/Binaries/" + "dolphin-emu-nogui ${INSTALL}/usr/bin", "dolphin"),), + # dosbox comes from the meson install, dosbox.start from line 28. + "dosbox-staging": (_sa("dosbox-staging", "cp -a ${PKG_DIR}/scripts/* ${INSTALL}/usr/bin/", + "DOSBOXSDL2"),), + "duckstation": (_sa("duckstation", "cp -rf ${PKG_BUILD}/.${TARGET_NAME}/bin/" + "duckstation-nogui ${INSTALL}/usr/bin", "Duckstation"),), + "ecwolf": (_sa("ecwolf", "cp ${PKG_BUILD}/ecwolf ${INSTALL}/usr/bin/ecwolf", "ECWolf"),), + "fbneoSA": (_sa("fbneoSA", "cp -rf ${PKG_BUILD}/fbneo ${INSTALL}/usr/bin", "FBNEOSA"),), + "flycastsa": (_sa("flycastsa", "cp ${PKG_BUILD}/.${TARGET_NAME}/flycast " + "${INSTALL}/usr/bin/flycast", "flycastsa"),), + "hatarisa": (_sa("hatarisa", "cp src/hatari ${INSTALL}/usr/bin", "HATARISA"),), + # hypseus comes from the cmake install (PKG_TOOLCHAIN, line 13). + "hypseus-singe": (_sa("hypseus-singe", 'PKG_TOOLCHAIN="cmake"', "HYPSEUS"),), + "ikemen-go": (_sa("ikemen-go", 'cp "$PKG_BUILD/bin/Ikemen_Go_"* "$INSTALL/usr/bin/Ikemen_Go"', + "ikemen"),), + "jzintv": (_sa("jzintv", "cp bin/jzintv ${INSTALL}/usr/bin", "jzintv"),), + # The metapackage's dependencies (line 10); mupen64plus itself comes from + # mupen64plussa-ui-console's makeinstall_target. + "mupen64plussa": ( + _sa("mupen64plussa", 'PKG_DEPENDS_TARGET="toolchain mupen64plussa-core ' + "mupen64plussa-ui-console mupen64plussa-audio-sdl mupen64plussa-input-sdl " + 'mupen64plussa-rsp-hle mupen64plussa-video-rice mupen64plussa-video-glide64mk2"', + "mupen64plussa"), + Install("packages/sx05re/emulators/mupen64plussa/mupen64plussa-ui-console/package.mk", + "cp ${PKG_BUILD}/projects/unix/mupen64plus ${UBINDIR}", ()), + ), + "openbor": (_sa("openbor", 'cp `find . -name "OpenBOR.elf" | xargs echo` ' + "${INSTALL}/usr/bin/OpenBOR", "OPENBOR"),), + # scummvm comes from the make install, moved to /usr/bin at line 51. + "scummvmsa": (_sa("scummvmsa", "mv ${INSTALL}/usr/local/bin ${INSTALL}/usr/", "SCUMMVMSA"),), + # solarus-run comes from the cmake install, solarus.sh from line 22. + "solarus": (_sa("solarus", "cp ${PKG_DIR}/scripts/* ${INSTALL}/usr/bin", "Solarus"),), + "stellasa": (_sa("stellasa", "cp -rf ${PKG_BUILD}/stella ${INSTALL}/usr/bin", "STELLASA"),), + "yabasanshiroSA_1_11": (_sa("yabasanshiroSA_1_11", "cp -a ${PKG_BUILD}/src/retro_arena/" + "yabasanshiro ${INSTALL}/usr/bin", "yabasanshiro"),), + "lib32-bennugd-monolithic": (Install( + "packages/lib32/emuelec/ports/lib32-bennugd-monolithic/package.mk", + "cp ${PKG_BUILD}/projects/cmake/bgdi/bgdi ${INSTALL}/usr/bin", ("bgdi",)),), + # The metapackage's own bin/ (line 82) carries pico8.sh, which starts the + # PICO-8 binary the user copies to bios/pico-8. + "emuelec": (Install(META_MK, "cp -rf ${PKG_DIR}/bin ${INSTALL}/usr", ("Pico-8",)),), +} + +# Packages of a device list whose install step puts no ES core on the image. +NO_ES_CORE: dict[str, str] = { + "toolchain": "build toolchain", + "${OPENGLES}": "graphics driver", + "emuelec-emulationstation": "the frontend", + "retroarch": "the libretro frontend", + "lib32-retroarch": "the 32-bit libretro frontend, retroarch32", + "emuelec-tools": "system tools", + "emuelec-32bit-info": "the .info files of the 32-bit cores", + "lib32-essential": "32-bit runtime", + "lib32-libusb": "32-bit library", + "lib32-box86": "x86 translator", + "fbterm": "terminal", + "kmscon": "terminal", + "odroidgoa-utils": "device tools and retrorun, a runner", + "amiberry-lite": "amiberry-lite, which es_systems.json names nowhere", + "yabasanshiroSA_1_5": "yabasanshiro1_5, which es_systems.json names nowhere", + "lib32-droidports": "gmloader, which es_cores drops (_overrides.yml)", + "gme": "gme_libretro.so, which es_systems.json names nowhere", + "nestopiaCV": "nestopiaCV_libretro.so, which es_systems.json names nowhere", + "quasi88": "quasi88_libretro.so, which es_systems.json names nowhere", + "spring": "spring_*_libretro.so, which es_systems.json names nowhere", +} + +# ES core names no package of any device builds, by the package that would. +# The port engines are dependencies of emuelec-ports (its package.mk, +# lines 17-59), and no package of the image depends on emuelec-ports; +# flycast-dojo is in no device list. +UNBUILT: dict[str, str] = { + **dict.fromkeys( + ("CGeniusExe", "Chocolate-Doom", "GZDoom", "LZDoom", "OpenJazz", "RigelEngine", + "VVVVVV", "abuse", "bs", "bstone", "ccleste", "cdogs-sdl", "devilutionx", + "eduke32", "fallout-ce", "fallout2-ce", "fheroes2", "hcl", "hode", "hurrican", + "opentyrian", "prince", "smw", "sonic2013", "soniccd", "sonicmania", "supertux2", + "supertuxkart", "xash3d"), + "emuelec-ports", + ), + "flycast_dojo": "flycast-dojo", +} + + +def release_devices(asset_names: list[str]) -> dict[str, str]: + """{device: architecture} of each image a release publishes.""" + found: dict[str, str] = {} + for name in asset_names: + match = _ASSET_RE.match(name) + if match: + found.setdefault(match.group("device"), match.group("arch")) + if not found: + raise RuntimeError("the EmuELEC release publishes no device image") + return found + + +def _require(text: str, lines: tuple[str, ...], where: str) -> None: + present = {line.strip() for line in text.splitlines()} + missing = [line for line in lines if line not in present] + if missing: + raise RuntimeError(f"{where} no longer reads {missing[0]!r}") + + +def _quoted(text: str, name: str, where: str) -> list[str]: + """The words of `NAME="..."`; a backslash-newline inside the quotes is + removed by bash, so the value reads as one line.""" + match = re.search(rf'^\s*{name}="([^"]*)"', text, re.MULTILINE) + if match is None: + raise RuntimeError(f"{where} sets no {name}") + return match.group(1).replace("\\\n", " ").split() + + +class Metapackage(NamedTuple): + emus: list[str] + experimental: list[str] + s922x: list[str] + lib32: list[str] + + +def read_metapackage(package_mk: str, options: str) -> Metapackage: + """The lists the metapackage rules work on, with PKG_EMUS expanded.""" + _require(package_mk, _META_LINES, META_MK) + emus = _quoted(package_mk, "PKG_EMUS", META_MK) + if emus[:1] != ["${LIBRETRO_CORES}"]: + raise RuntimeError(f"{META_MK}: PKG_EMUS no longer starts with ${{LIBRETRO_CORES}}") + lib32 = _LIB32_RE.search(package_mk) + if lib32 is None: + raise RuntimeError(f"{META_MK}: no aarch64 lib32 list") + return Metapackage( + emus=_quoted(options, "LIBRETRO_CORES", OPTIONS) + emus[1:], + experimental=_quoted(package_mk, "PKG_EXPERIMENTAL", META_MK), + s922x=_quoted(options, "LIBRETRO_S922X_CORES", OPTIONS), + lib32=lib32.group(1).replace("\\\n", " ").split(), + ) + + +def _sed_first(words: str, pattern: str) -> str: + """`echo ${PKG_DEPENDS_TARGET} | sed "s|| |"`: the unquoted echo + joins the words with single spaces, sed replaces the first match only, + anywhere in the line, a word's tail included.""" + line = " ".join(words.split()) + at = line.find(pattern) + return line if at < 0 else f"{line[:at]} {line[at + len(pattern):]}" + + +def device_packages(meta: Metapackage, device: str, arch: str) -> list[str]: + """PKG_DEPENDS_TARGET of the metapackage for one device (lines 8-77).""" + depends = "toolchain ${OPENGLES} emuelec-emulationstation retroarch" + depends += " emuelec-tools " + " ".join(meta.emus) + " " + " ".join(meta.experimental) + if device in _BIG: + depends += " " + " ".join(meta.s922x) + if device in _HANDHELD: + depends += " kmscon odroidgoa-utils" + for discore in ("duckstation", "mesen-s", "virtualjaguar", "quicknes", "MC"): + depends = _sed_first(depends, f"{discore} ") + depends += " yabasanshiro" + else: + depends += " fbterm" + if arch == "aarch64": + for discore in ("quicknes", "parallel-n64", "pcsx_rearmed"): + depends = _sed_first(depends, discore) + depends += " " + " ".join(meta.lib32) + if device in _BIG: + depends += " dolphinSA" + if device == "Amlogic-old": + for discore in ("yabasanshiroSA_1_11", "yabasanshiroSA_1_5", "yabasanshiro", + "same_cdi", "duckstation"): + depends = _sed_first(depends, f"{discore} ") + if device in _BIG: + depends += " mame" + if device in ("RK356x", "OdroidM1"): + depends = _sed_first(depends, "flycast-dojo") + return depends.split() + + +def arch_builds(pkg_arch: str, arch: str) -> bool: + """scripts/build:28-30: an empty PKG_ARCH builds everywhere, `!` + excludes, otherwise the list names the arch or `any`.""" + words = pkg_arch.split() + if not words: + return True + if f"!{arch}" in words: + return False + return arch in words or "any" in words + + +def pkg_arch(package_mk: str) -> str: + match = re.search(r'^PKG_ARCH="([^"]*)"', package_mk, re.MULTILINE) + return match.group(1) if match else "" + + +def es_systems_for(es_json: str, es_mk: str, device: str) -> SystemEmulators: + """es_systems.json as es_systems.cfg reads after makeinstall_target's + deletions on one device (emuelec-emulationstation/package.mk:114-143). + + `sed "s|X||g"` matches a core written without attributes: + generate_es_systems.py gives `default="true"` to the first default core + of a system and `incompatible_extensions` to a core listing them. + """ + _require(es_mk, _ES_LINES, ES_MK) + if device in _HANDHELD: + remove_cores = {"mesen-s", "quicknes", "mame2016", "mesen"} + drop_systems: set[str] = set() + drop_emulators = {"Duckstation"} + elif device == "Amlogic-old": + remove_cores = {"mesen-s", "quicknes", "mame2016", "mesen", "yabasanshiroSA", + "yabasanshiro"} + drop_systems = {"saturn", "phillips-cdi"} + drop_emulators = {"Duckstation", "retrorun"} + else: + # Line 117 holds for every device: no name is all three at once. + remove_cores = set() + drop_systems = set() + drop_emulators = {"retrorun"} + systems: SystemEmulators = {} + for system in json.loads(es_json).get("systems", []): + if system["name"] in drop_systems: + continue + default_set = False + emulators = [] + for emulator in system.get("emulators") or []: + kept = [] + for core in emulator.get("cores") or []: + bare = "incompatible" not in core + if core.get("default", False) and not default_set: + default_set = True + bare = False + if not (bare and core["name"] in remove_cores): + kept.append(core["name"]) + if emulator["name"] not in drop_emulators: + emulators.append((emulator["name"], kept)) + systems[system["name"]] = emulators + return systems + + +def device_cores( + packages: list[str], + arch: str, + archs: dict[str, str], + systems: SystemEmulators, +) -> list[str]: + """ES core names a device both installs and offers. + + `archs` maps each package of PROVIDERS to its PKG_ARCH. + """ + unknown = sorted( + p for p in set(packages) if p not in PROVIDERS and p not in NO_ES_CORE + and p not in UNBUILT.values() + ) + if unknown: + raise RuntimeError(f"{META_MK}: no install step read for {', '.join(unknown)}") + unbuilt = sorted(set(packages) & set(UNBUILT.values())) + if unbuilt: + raise RuntimeError(f"{META_MK}: {unbuilt[0]} is built and its cores are read as absent") + installed: set[str] = set() + for package in [*set(packages), "emuelec"]: + if package not in PROVIDERS or not arch_builds(archs[package], arch): + continue + for install in PROVIDERS[package]: + if install.arch is None or (install.arch == arch) or ( + install.arch.startswith("!") and install.arch[1:] != arch + ): + installed.update(install.cores) + offered, _standalone, _dual = es_cores( + systems, RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME) + ) + return sorted(installed & set(offered)) + + +def check_names(systems: SystemEmulators) -> None: + """Every ES core name is installed by a package read or known unbuilt.""" + offered, _standalone, _dual = es_cores( + systems, RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME) + ) + provided = {core for installs in PROVIDERS.values() for i in installs for core in i.cores} + unread = sorted(set(offered) - provided - set(UNBUILT)) + if unread: + raise RuntimeError(f"es_systems.json names {', '.join(unread)}, no package read installs it") + + +def check_installs(texts: dict[str, str]) -> dict[str, str]: + """{package: PKG_ARCH}, after checking each cited install line is there.""" + archs: dict[str, str] = {} + for package, installs in PROVIDERS.items(): + for install in installs: + _require(texts[install.path], (install.line,), install.path) + archs[package] = pkg_arch(texts[installs[0].path]) + return archs + + +class Scraper(BaseTargetScraper): + """Per-device core lists from EmuELEC's metapackage.""" + + def __init__(self, url: str = "https://github.com/" + REPO) -> None: + super().__init__(url=url) + self.ref = "" + + def _raw(self, path: str) -> str: + return fetch_bytes(RAW.format(ref=self.ref, path=path)).decode("utf-8") + + def fetch_targets(self) -> dict: + self.ref = self.pinned_revision(PLATFORM_NAME) + release = json.loads( + fetch_bytes(RELEASE_API.format(ref=self.ref), headers=github_headers()) + ) + devices = release_devices([asset["name"] for asset in release.get("assets", [])]) + meta = read_metapackage(self._raw(META_MK), self._raw(OPTIONS)) + paths = sorted({i.path for installs in PROVIDERS.values() for i in installs}) + archs = check_installs({path: self._raw(path) for path in paths}) + es_json = fetch_bytes(ES_SYSTEMS_TEMPLATE.format(ref=self.ref)).decode("utf-8") + ports = port_engines(fetch_bytes(PORTS_TEMPLATE.format(ref=self.ref)).decode("utf-8")) + es_mk = self._raw(ES_MK) + check_names(with_ports(es_systems_for(es_json, es_mk, "Amlogic-ng"), ports)) + targets: dict[str, dict] = {} + for device, arch in sorted(devices.items()): + systems = with_ports(es_systems_for(es_json, es_mk, device), ports) + cores = device_cores(device_packages(meta, device, arch), arch, archs, systems) + targets[device.lower()] = {"architecture": arch, "cores": cores} + print(f" {device.lower()} ({arch}): {len(cores)} cores", file=sys.stderr) + return { + "platform": PLATFORM_NAME, + "source": f"{self.url}/blob/{self.ref}/{META_MK}", + "devices_source": release["html_url"], + "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "targets": targets, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + allow_abbrev=False, description="Scrape EmuELEC per-device cores" + ) + parser.add_argument("--dry-run", action="store_true", help="Show target summary") + parser.add_argument("--output", "-o", help="Output YAML file") + args = parser.parse_args() + + scraper = Scraper() + data = scraper.fetch_targets() + if args.dry_run: + for name, info in data["targets"].items(): + print(f" {name} ({info['architecture']}): {len(info['cores'])} cores") + return + if args.output: + scraper.write_output(data, args.output) + print(f"Written to {args.output}") + return + print(yaml.dump(data, default_flow_style=False, sort_keys=False)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_emuelec_targets.py b/tests/test_emuelec_targets.py new file mode 100644 index 000000000..c8f911433 --- /dev/null +++ b/tests/test_emuelec_targets.py @@ -0,0 +1,266 @@ +"""EmuELEC's images differ by device, and only its metapackage says how. + +packages/sx05re/emuelec/package.mk (v4.8) builds the S922X cores, Dolphin +and the MAME core only on Amlogic-ng, Amlogic-no, RK356x and OdroidM1, and +takes DuckStation out on the two handhelds with a sed that replaces the +first match only. With no target list, an OdroidGoAdvance pack carried the +firmware of cores its image never had. +""" + +from __future__ import annotations + +import json +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from scraper.targets import emuelec_targets_scraper as scraper # noqa: E402 + +# packages/sx05re/emuelec/package.mk:8-77 at v4.8, PKG_EMUS and +# PKG_EXPERIMENTAL shortened. +PACKAGE_MK = """PKG_DEPENDS_TARGET="toolchain ${OPENGLES} emuelec-emulationstation retroarch" +PKG_EXPERIMENTAL="nestopiaCV quasi88 yabasanshiroSA_1_11 yabasanshiroSA_1_5 fbneoSA" +PKG_EMUS="${LIBRETRO_CORES} advancemame duckstation pcsx_rearmed flycastsa" +PKG_DEPENDS_TARGET+=" emuelec-tools ${PKG_EMUS} ${PKG_EXPERIMENTAL}" + +if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then +\tPKG_DEPENDS_TARGET+=" ${LIBRETRO_S922X_CORES}" +fi + +if [ "${DEVICE}" == "OdroidGoAdvance" ] || [ "${DEVICE}" == "GameForce" ]; then +\tPKG_DEPENDS_TARGET+=" kmscon odroidgoa-utils" +\tfor discore in duckstation mesen-s virtualjaguar quicknes MC; do +\t\tPKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore} | |") +\tdone +\tPKG_DEPENDS_TARGET+=" yabasanshiro" +else +\tPKG_DEPENDS_TARGET+=" fbterm" +fi + +if [ "${ARCH}" == "aarch64" ]; then + for discore in quicknes parallel-n64 pcsx_rearmed; do +\t\tPKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore}| |") +\tdone + + PKG_DEPENDS_TARGET+=" swanstation \\ + lib32-essential \\ + lib32-pcsx_rearmed \\ + lib32-parallel-n64 + lib32-libusb" + + if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then + PKG_DEPENDS_TARGET+=" dolphinSA" + fi + + if [ "${DEVICE}" == "Amlogic-old" ]; then + for discore in yabasanshiroSA_1_11 yabasanshiroSA_1_5 yabasanshiro same_cdi duckstation; do + PKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore} | |") + done + fi +fi + +if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then +\tPKG_DEPENDS_TARGET+=" mame" +fi + +if [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then + for discore in flycast-dojo; do +\t\tPKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore}| |") +\tdone +fi +""" + +# distributions/EmuELEC/options:274-373 at v4.8, shortened. +OPTIONS = """LIBRETRO_CORES="mgba \\ +parallel-n64 \\ +pcsx_rearmed \\ +flycast" + +LIBRETRO_S922X_CORES="beetle-saturn \\ +mesen \\ +quicknes \\ +virtualjaguar" +""" + +# packages/sx05re/emuelec-emulationstation/package.mk:114-143 at v4.8. +ES_MK = """CORESFILE="${INSTALL}/usr/config/emulationstation/es_systems.cfg" + +if [[ "${DEVICE}" != "Amlogic-ng" || "${DEVICE}" != "Amlogic-ne" || "${DEVICE}" != "Amlogic-no" ]]; then + if [[ ${DEVICE} == "OdroidGoAdvance" || "${DEVICE}" == "GameForce" ]]; then + remove_cores="mesen-s quicknes mame2016 mesen" + elif [ "${DEVICE}" == "Amlogic-old" ]; then + remove_cores="mesen-s quicknes mame2016 mesen yabasanshiroSA yabasanshiro" + xmlstarlet ed -L -P -d "/systemList/system[name='saturn']" ${CORESFILE} + xmlstarlet ed -L -P -d "/systemList/system[name='phillips-cdi']" ${CORESFILE} + xmlstarlet ed -L -P -d "/systemList/system/emulators/emulator[@name='Duckstation']" ${CORESFILE} + fi + for discore in ${remove_cores}; do + sed -i "s|${discore}||g" ${CORESFILE} + sed -i '/^[[:space:]]*$/d' ${CORESFILE} + done +fi + +if [[ ${DEVICE} != "OdroidGoAdvance" ]] && [[ "${DEVICE}" != "GameForce" ]]; then +\txmlstarlet ed -L -P -d "/systemList/system/emulators/emulator[@name='retrorun']" ${CORESFILE} +else +\txmlstarlet ed -L -P -d "/systemList/system/emulators/emulator[@name='Duckstation']" ${CORESFILE} +fi +""" + + +def _system(name: str, *emulators: tuple[str, list[dict]]) -> dict: + return {"name": name, "emulators": [{"name": e, "cores": c} for e, c in emulators]} + + +ES_JSON = json.dumps({"systems": [ + _system("nes", ("libretro", [{"name": "mesen", "default": True}])), + _system("snes", ("libretro", [{"name": "snes9x", "default": True}, {"name": "mesen-s"}])), + _system("psx", + ("libretro", [{"name": "pcsx_rearmed", "default": True}, + {"name": "pcsx_rearmed_32b"}, {"name": "swanstation"}]), + ("Duckstation", [{"name": "Duckstation"}]), + ("retrorun", [{"name": "pcsx_rearmed_32b"}])), + _system("n64", ("libretro", [{"name": "parallel_n64_32b"}])), +]}) + +ARCHS = {"mesen": "", "duckstation": "aarch64", "pcsx_rearmed": "any", + "lib32-pcsx_rearmed": "aarch64", "swanstation": "any", + "lib32-parallel-n64": "aarch64"} + +ASSETS = [ + "EmuELEC-Amlogic-ng.aarch64-4.8-Generic.img.gz", + "EmuELEC-Amlogic-ng.aarch64-4.8.tar", + "EmuELEC-GameForce.aarch64-4.8-chi.img.gz.sha256", + "EmuELEC-OdroidGoAdvance.aarch64-4.8-odroidgo2.tar", + "EmuELEC-OdroidM1.aarch64-4.8.img.gz", +] + + +class TheMetapackageDecidesEachDevice(unittest.TestCase): + def setUp(self): + self.meta = scraper.read_metapackage(PACKAGE_MK, OPTIONS) + + def test_devices_are_the_images_the_release_publishes(self): + self.assertEqual(scraper.release_devices(ASSETS), { + "Amlogic-ng": "aarch64", "GameForce": "aarch64", + "OdroidGoAdvance": "aarch64", "OdroidM1": "aarch64", + }) + + def test_the_lists_the_rules_work_on(self): + self.assertEqual(self.meta.emus, ["mgba", "parallel-n64", "pcsx_rearmed", "flycast", + "advancemame", "duckstation", "pcsx_rearmed", + "flycastsa"]) + self.assertEqual(self.meta.lib32, ["swanstation", "lib32-essential", + "lib32-pcsx_rearmed", "lib32-parallel-n64", + "lib32-libusb"]) + + def test_the_big_devices_build_the_s922x_cores_dolphin_and_mame(self): + packages = scraper.device_packages(self.meta, "Amlogic-ng", "aarch64") + for package in ("beetle-saturn", "mesen", "virtualjaguar", "dolphinSA", "mame", + "duckstation", "fbterm"): + self.assertIn(package, packages) + # quicknes is added, then the aarch64 loop takes it out. + self.assertNotIn("quicknes", packages) + + def test_the_handhelds_lose_duckstation_and_gain_their_tools(self): + packages = scraper.device_packages(self.meta, "OdroidGoAdvance", "aarch64") + for package in ("duckstation", "mesen", "beetle-saturn", "dolphinSA", "mame", "fbterm"): + self.assertNotIn(package, packages) + self.assertEqual(packages[-1:], ["lib32-libusb"]) + self.assertIn("kmscon", packages) + self.assertIn("yabasanshiro", packages) + + def test_sed_takes_out_the_first_match_only(self): + # pcsx_rearmed is written twice, in LIBRETRO_CORES and in PKG_EMUS: + # the aarch64 loop removes the first and the 64-bit core is built. + for device in ("Amlogic-ng", "OdroidGoAdvance"): + packages = scraper.device_packages(self.meta, device, "aarch64") + self.assertEqual(packages.count("pcsx_rearmed"), 1, device) + self.assertNotIn("parallel-n64", packages, device) + + def test_a_pattern_with_a_trailing_space_misses_the_last_word(self): + self.assertEqual(scraper._sed_first("a duckstation", "duckstation "), "a duckstation") + self.assertEqual(scraper._sed_first("a duckstation b", "duckstation "), "a b") + # Without the space, the pattern takes a word's tail too. + self.assertEqual(scraper._sed_first("lib32-pcsx_rearmed", "pcsx_rearmed"), "lib32- ") + + def test_a_changed_rule_stops_the_scrape(self): + changed = PACKAGE_MK.replace("duckstation mesen-s virtualjaguar quicknes MC", + "duckstation mesen-s") + with self.assertRaises(RuntimeError): + scraper.read_metapackage(changed, OPTIONS) + + +class EachPackageBuildsWhereItsArchAllows(unittest.TestCase): + def test_pkg_arch(self): + self.assertTrue(scraper.arch_builds("", "aarch64")) + self.assertTrue(scraper.arch_builds("any", "aarch64")) + self.assertTrue(scraper.arch_builds("aarch64 arm", "aarch64")) + # mupen64plus and uae4arm say PKG_ARCH="arm": skipped on aarch64. + self.assertFalse(scraper.arch_builds("arm", "aarch64")) + self.assertFalse(scraper.arch_builds("any !aarch64", "aarch64")) + + def test_every_cited_install_line_is_read(self): + install = scraper.PROVIDERS["mgba"][0] + text = f'PKG_ARCH="any"\nmakeinstall_target() {{\n {install.line}\n}}\n' + with self.assertRaises(RuntimeError): + scraper._require(text.replace("mgba_libretro", "mgba2_libretro"), + (install.line,), install.path) + scraper._require(text, (install.line,), install.path) + + +class TheFrontendOffersWhatTheDeviceKeeps(unittest.TestCase): + def test_handhelds_drop_bare_cores_and_duckstation(self): + systems = scraper.es_systems_for(ES_JSON, ES_MK, "GameForce") + # mesen is the nes default: written with an attribute, sed misses it. + self.assertEqual(systems["nes"], [("libretro", ["mesen"])]) + self.assertEqual(systems["snes"], [("libretro", ["snes9x"])]) + self.assertEqual([e for e, _ in systems["psx"]], ["libretro", "retrorun"]) + + def test_other_devices_drop_retrorun(self): + systems = scraper.es_systems_for(ES_JSON, ES_MK, "Amlogic-ng") + self.assertEqual([e for e, _ in systems["psx"]], ["libretro", "Duckstation"]) + self.assertEqual(systems["snes"], [("libretro", ["snes9x", "mesen-s"])]) + + def test_a_device_keeps_what_it_builds_and_offers(self): + meta = scraper.read_metapackage(PACKAGE_MK, OPTIONS) + archs = {**dict.fromkeys(scraper.PROVIDERS, "any"), **ARCHS} + cores = {} + for device in ("Amlogic-ng", "OdroidGoAdvance"): + packages = [p for p in scraper.device_packages(meta, device, "aarch64") + if p in scraper.PROVIDERS or p in scraper.NO_ES_CORE] + systems = scraper.es_systems_for(ES_JSON, ES_MK, device) + cores[device] = scraper.device_cores(packages, "aarch64", archs, systems) + self.assertEqual(cores["Amlogic-ng"], ["Duckstation", "mesen", "parallel_n64_32b", + "pcsx_rearmed", "pcsx_rearmed_32b", + "swanstation"]) + self.assertEqual(cores["OdroidGoAdvance"], ["parallel_n64_32b", "pcsx_rearmed", + "pcsx_rearmed_32b", "swanstation"]) + + def test_a_package_never_read_stops_the_scrape(self): + archs = dict.fromkeys(scraper.PROVIDERS, "any") + with self.assertRaises(RuntimeError): + scraper.device_cores(["mgba", "newcore"], "aarch64", archs, {}) + + +class TheCommittedListFollowsTheRules(unittest.TestCase): + def test_the_handhelds_list_no_core_they_do_not_build(self): + import yaml # noqa: PLC0415 + + targets = yaml.safe_load((REPO_ROOT / "platforms" / "targets" / "emuelec.yml") + .read_text(encoding="utf-8"))["targets"] + self.assertEqual(set(targets), {"amlogic-ng", "gameforce", "odroidgoadvance", "odroidm1"}) + big = set(targets["amlogic-ng"]["cores"]) + for device in ("gameforce", "odroidgoadvance"): + self.assertEqual(big - set(targets[device]["cores"]), { + "Duckstation", "dolphin", "mame", "mednafen_saturn", "mesen", "mesen-s", + "virtualjaguar", + }) + self.assertEqual(targets["odroidm1"]["cores"], targets["amlogic-ng"]["cores"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/adding-a-scraper.md b/wiki/adding-a-scraper.md index 7dc29ef32..a8ea434ef 100644 --- a/wiki/adding-a-scraper.md +++ b/wiki/adding-a-scraper.md @@ -339,6 +339,7 @@ Register the target scraper in `_registry.yml`: | `recalbox_targets` | board defconfigs + `system.ini` | Buildroot packages each image enables, cores per system | | `rocknix_targets` | emulators `package.mk` + device options | Cores per SoC block and 32-bit support per device | | `amberelec_targets` | Makefile devices + metapackage `package.mk` + `es_systems.cfg` | Extra cores kept only on the devices the metapackage names | +| `emuelec_targets` | release asset names + metapackage `package.mk` + each core's install step | Packages per device, mapped to cores by the `.so` or program each installs | ## Parser modules diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 38ce5b358..8eb805706 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -56,7 +56,7 @@ Every module in `tests/`, by the area it guards. | Verification, cross-reference and profile checks | synthetic + profiles | `test_bios_layout`, `test_cache_independent_counts`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_names`, `test_profile_paths`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_data_dir_notice`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | | Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_core_data_trees`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | -| Hardware targets and systems | synthetic + target files | `test_amberelec_targets`, `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | +| Hardware targets and systems | synthetic + target files | `test_amberelec_targets`, `test_emuelec_targets`, `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_games`, `test_install_offline`, `test_install_selection`, `test_install_tui` | | Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` | | Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` | diff --git a/wiki/tools.md b/wiki/tools.md index c8911f3b5..65a175d91 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1719,6 +1719,7 @@ Located in `scripts/scraper/targets/`. Each inherits `BaseTargetScraper` and imp | `recalbox_targets_scraper` | board defconfigs + system.ini cores | board | | `rocknix_targets_scraper` | emulators package.mk + device options | device | | `amberelec_targets_scraper` | Makefile devices + metapackage extra cores + es_systems.cfg | device | +| `emuelec_targets_scraper` | release assets + metapackage package.mk + core install steps + es_systems.json | device | | `lakka_targets_scraper` | libretro_cores package.mk + project and device options | device image | | `emudeck_targets_scraper` | EmuScripts (SteamOS and Windows) | operating system | | `retropie_targets_scraper` | scriptmodules + rp_module_flags | platform flag | From 44565a07bb87342733be26ae77658f1ff71e4315 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:09:09 +0100 Subject: [PATCH 2/5] feat: read the EmuELEC device in the installer EmuELEC's scripts/image writes COREELEC_DEVICE to os-release (v4.8, line 189), as a GameForce image shows. The installer reads it the way it does for AmberELEC and applies install/targets/emuelec.json. MiSTer FPGA takes EmuELEC's place in the test of a platform without targets. --- install.ps1 | 2 +- install.py | 8 +- install.sh | 2 +- install/launchers/index.json | 16 +- install/targets/emuelec.json | 492 +++++++++++++++++++++++++++++++++++ tests/test_install.py | 24 +- wiki/installer.md | 2 +- 7 files changed, 526 insertions(+), 20 deletions(-) create mode 100644 install/targets/emuelec.json diff --git a/install.ps1 b/install.ps1 index 3b53ab0a0..28de54233 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "b532d7f12f92dc5934e210b7359e849e796abf39d9d1598eac14fe91dc8c86ac" + $defaultInstallSha256 = "57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index bd32bf94b..04a697af5 100644 --- a/install.py +++ b/install.py @@ -109,8 +109,8 @@ TARGETS_URL = f"{BASE_URL}/install/targets/{{platform}}.json" # has no hardware targets, where the run sent the user after a list that # exists nowhere. TARGET_PLATFORMS = frozenset({ - "amberelec", "batocera", "emudeck", "knulli", "lakka", "recalbox", "retroarch", "retrobat", - "retrodeck", "retropie", "rocknix", "romm", + "amberelec", "batocera", "emudeck", "emuelec", "knulli", "lakka", "recalbox", "retroarch", + "retrobat", "retrodeck", "retropie", "rocknix", "romm", }) RAW_FILE_URL = f"{BASE_URL}/{{path}}" RELEASE_URL = ( @@ -1484,9 +1484,11 @@ def _parse_bash_var(path: Path, key: str) -> str | None: # board/scripts/post-build-script.sh:264-278) and writes KNULLI_SYSTEM_ARCH # to knulli.arch (package/system/knulli-system/knulli-system.mk:98-99). # AmberELEC's scripts/image writes COREELEC_DEVICE (a21975bf, line 168), the -# DEVICE its Makefile builds (RG351P). +# DEVICE its Makefile builds (RG351P). EmuELEC's writes the same line (v4.8, +# line 189), the DEVICE of the release image (OdroidGoAdvance). _BOARD_SOURCES: dict[str, tuple[str, str]] = { "amberelec": ("os_release", "COREELEC_DEVICE"), + "emuelec": ("os_release", "COREELEC_DEVICE"), "batocera": ("file", "/usr/share/batocera/batocera.arch"), "recalbox": ("file", "/recalbox/recalbox.arch"), "rocknix": ("os_release", "HW_DEVICE"), diff --git a/install.sh b/install.sh index 8e65e5916..a03428f05 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="b532d7f12f92dc5934e210b7359e849e796abf39d9d1598eac14fe91dc8c86ac" +DEFAULT_INSTALL_SHA256="57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index 428c911fc..f6dc104a6 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1112275, - "sha256": "b532d7f12f92dc5934e210b7359e849e796abf39d9d1598eac14fe91dc8c86ac", + "size": 1112438, + "sha256": "57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1165902, - "sha256": "30b3cd30b3a75c4e2fb0784589f1b12d3ac2fb6ec345e874a2a8d58681c236c8" + "size": 1166065, + "sha256": "1c9e5737a0b32c85108f556c2799cf0ebb5ce52cbd2bf501bc24783fa3a04b9a" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1112275, - "sha256": "b532d7f12f92dc5934e210b7359e849e796abf39d9d1598eac14fe91dc8c86ac", + "size": 1112438, + "sha256": "57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1112275, - "sha256": "b532d7f12f92dc5934e210b7359e849e796abf39d9d1598eac14fe91dc8c86ac", + "size": 1112438, + "sha256": "57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4", "executable": false }, { diff --git a/install/targets/emuelec.json b/install/targets/emuelec.json new file mode 100644 index 000000000..c3363d940 --- /dev/null +++ b/install/targets/emuelec.json @@ -0,0 +1,492 @@ +{ + "amlogic-ng": [ + "2048", + "81", + "AMIBERRY", + "AdvanceMame", + "DOSBOXSDL2", + "Duckstation", + "ECWolf", + "FBNEOSA", + "HATARISA", + "HYPSEUS", + "OPENBOR", + "PPSSPPSDL", + "Pico-8", + "SCUMMVMSA", + "STELLASA", + "Solarus", + "a5200", + "atari800", + "bgdi", + "bk", + "bluemsx", + "cannonball", + "cap32", + "daphne", + "dinothawr", + "dolphin", + "dosbox_pure", + "dosbox_svn", + "easyrpg", + "fake08", + "fbalpha2012", + "fbneo", + "fceumm", + "fceumm_mod", + "flycast", + "flycast_32b", + "flycastsa", + "fmsx", + "freeintv", + "freej2me", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "geargrafx", + "gearlynx", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_cart_special", + "genesis_plus_gx_wide", + "geolith", + "gpsp", + "gw", + "handy", + "hatari", + "ikemen", + "jzintv", + "mame", + "mame2003_plus", + "mame2010", + "mednafen_lynx", + "mednafen_ngp", + "mednafen_pce_fast", + "mednafen_pcfx", + "mednafen_saturn", + "mednafen_supafaust", + "mednafen_supergrafx", + "mednafen_vb", + "mednafen_wswan", + "mesen", + "mesen-s", + "mgba", + "mrboom", + "mupen64plus_32b", + "mupen64plus_next", + "mupen64plus_next_alt", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "nxengine", + "o2em", + "opera", + "parallel_n64_32b", + "pcsx_rearmed", + "pcsx_rearmed_32b", + "picodrive", + "pocketcdg", + "pokemini", + "potator", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "reminiscence", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "stella", + "swanstation", + "tgbdual", + "tic80", + "tyrquake", + "uae4arm_32b", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_x64sc", + "vice_xplus4", + "vice_xvic", + "virtualjaguar", + "x1", + "xrick", + "yabasanshiro" + ], + "gameforce": [ + "2048", + "81", + "AMIBERRY", + "AdvanceMame", + "DOSBOXSDL2", + "ECWolf", + "FBNEOSA", + "HATARISA", + "HYPSEUS", + "OPENBOR", + "PPSSPPSDL", + "Pico-8", + "SCUMMVMSA", + "STELLASA", + "Solarus", + "a5200", + "atari800", + "bgdi", + "bk", + "bluemsx", + "cannonball", + "cap32", + "daphne", + "dinothawr", + "dosbox_pure", + "dosbox_svn", + "easyrpg", + "fake08", + "fbalpha2012", + "fbneo", + "fceumm", + "fceumm_mod", + "flycast", + "flycast_32b", + "flycastsa", + "fmsx", + "freeintv", + "freej2me", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "geargrafx", + "gearlynx", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_cart_special", + "genesis_plus_gx_wide", + "geolith", + "gpsp", + "gw", + "handy", + "hatari", + "ikemen", + "jzintv", + "mame2003_plus", + "mame2010", + "mednafen_lynx", + "mednafen_ngp", + "mednafen_pce_fast", + "mednafen_pcfx", + "mednafen_supafaust", + "mednafen_supergrafx", + "mednafen_vb", + "mednafen_wswan", + "mgba", + "mrboom", + "mupen64plus_32b", + "mupen64plus_next", + "mupen64plus_next_alt", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "nxengine", + "o2em", + "opera", + "parallel_n64_32b", + "pcsx_rearmed", + "pcsx_rearmed_32b", + "picodrive", + "pocketcdg", + "pokemini", + "potator", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "reminiscence", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "stella", + "swanstation", + "tgbdual", + "tic80", + "tyrquake", + "uae4arm_32b", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_x64sc", + "vice_xplus4", + "vice_xvic", + "x1", + "xrick", + "yabasanshiro" + ], + "odroidgoadvance": [ + "2048", + "81", + "AMIBERRY", + "AdvanceMame", + "DOSBOXSDL2", + "ECWolf", + "FBNEOSA", + "HATARISA", + "HYPSEUS", + "OPENBOR", + "PPSSPPSDL", + "Pico-8", + "SCUMMVMSA", + "STELLASA", + "Solarus", + "a5200", + "atari800", + "bgdi", + "bk", + "bluemsx", + "cannonball", + "cap32", + "daphne", + "dinothawr", + "dosbox_pure", + "dosbox_svn", + "easyrpg", + "fake08", + "fbalpha2012", + "fbneo", + "fceumm", + "fceumm_mod", + "flycast", + "flycast_32b", + "flycastsa", + "fmsx", + "freeintv", + "freej2me", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "geargrafx", + "gearlynx", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_cart_special", + "genesis_plus_gx_wide", + "geolith", + "gpsp", + "gw", + "handy", + "hatari", + "ikemen", + "jzintv", + "mame2003_plus", + "mame2010", + "mednafen_lynx", + "mednafen_ngp", + "mednafen_pce_fast", + "mednafen_pcfx", + "mednafen_supafaust", + "mednafen_supergrafx", + "mednafen_vb", + "mednafen_wswan", + "mgba", + "mrboom", + "mupen64plus_32b", + "mupen64plus_next", + "mupen64plus_next_alt", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "nxengine", + "o2em", + "opera", + "parallel_n64_32b", + "pcsx_rearmed", + "pcsx_rearmed_32b", + "picodrive", + "pocketcdg", + "pokemini", + "potator", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "reminiscence", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "stella", + "swanstation", + "tgbdual", + "tic80", + "tyrquake", + "uae4arm_32b", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_x64sc", + "vice_xplus4", + "vice_xvic", + "x1", + "xrick", + "yabasanshiro" + ], + "odroidm1": [ + "2048", + "81", + "AMIBERRY", + "AdvanceMame", + "DOSBOXSDL2", + "Duckstation", + "ECWolf", + "FBNEOSA", + "HATARISA", + "HYPSEUS", + "OPENBOR", + "PPSSPPSDL", + "Pico-8", + "SCUMMVMSA", + "STELLASA", + "Solarus", + "a5200", + "atari800", + "bgdi", + "bk", + "bluemsx", + "cannonball", + "cap32", + "daphne", + "dinothawr", + "dolphin", + "dosbox_pure", + "dosbox_svn", + "easyrpg", + "fake08", + "fbalpha2012", + "fbneo", + "fceumm", + "fceumm_mod", + "flycast", + "flycast_32b", + "flycastsa", + "fmsx", + "freeintv", + "freej2me", + "fuse", + "gambatte", + "gearboy", + "gearcoleco", + "geargrafx", + "gearlynx", + "gearsystem", + "genesis_plus_gx", + "genesis_plus_gx_cart_special", + "genesis_plus_gx_wide", + "geolith", + "gpsp", + "gw", + "handy", + "hatari", + "ikemen", + "jzintv", + "mame", + "mame2003_plus", + "mame2010", + "mednafen_lynx", + "mednafen_ngp", + "mednafen_pce_fast", + "mednafen_pcfx", + "mednafen_saturn", + "mednafen_supafaust", + "mednafen_supergrafx", + "mednafen_vb", + "mednafen_wswan", + "mesen", + "mesen-s", + "mgba", + "mrboom", + "mupen64plus_32b", + "mupen64plus_next", + "mupen64plus_next_alt", + "mupen64plussa", + "nekop2", + "neocd", + "nestopia", + "np2kai", + "nxengine", + "o2em", + "opera", + "parallel_n64_32b", + "pcsx_rearmed", + "pcsx_rearmed_32b", + "picodrive", + "pocketcdg", + "pokemini", + "potator", + "prboom", + "prosystem", + "puae", + "puae2021", + "px68k", + "reminiscence", + "same_cdi", + "sameboy", + "sameduck", + "scummvm", + "snes9x", + "snes9x2002", + "snes9x2005_plus", + "snes9x2010", + "stella", + "swanstation", + "tgbdual", + "tic80", + "tyrquake", + "uae4arm_32b", + "uzem", + "vba_next", + "vbam", + "vecx", + "vice_x128", + "vice_x64", + "vice_x64sc", + "vice_xplus4", + "vice_xvic", + "virtualjaguar", + "x1", + "xrick", + "yabasanshiro" + ] +} diff --git a/tests/test_install.py b/tests/test_install.py index 7e1659a07..252f8a4c8 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3552,7 +3552,7 @@ class ACardPreparedOnAComputerGetsItsBoard(unittest.TestCase): class APlatformWithoutTargetsSaysSoOffline(unittest.TestCase): """Offline, --target on AmberELEC asked for install/targets/amberelec.json, which existed nowhere: online the same run said it had no targets. - EmuELEC has none still.""" + MiSTer FPGA has none still.""" def test_the_list_matches_the_published_targets(self): published = {path.stem for path in (REPO_ROOT / "install" / "targets").glob("*.json")} @@ -3560,16 +3560,16 @@ class APlatformWithoutTargetsSaysSoOffline(unittest.TestCase): def test_offline_says_there_are_no_targets(self): dest = Path(_tempdir(self)) - for extra in ([], ["--manifest", str(REPO_ROOT / "install" / "emuelec.json")]): + for extra in ([], ["--manifest", str(REPO_ROOT / "install" / "misterfpga.json")]): with self.subTest(manifest=bool(extra)): proc = subprocess.run( - [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "emuelec", - "--dest", str(dest), "--target", "rk3326", "--offline", "--no-input", + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "misterfpga", + "--dest", str(dest), "--target", "de10-nano", "--offline", "--no-input", "--dry-run", *extra], capture_output=True, text=True, timeout=120, stdin=subprocess.DEVNULL) self.assertEqual(proc.returncode, 2, proc.stderr) - self.assertIn("EmuELEC has no hardware targets", proc.stderr) - self.assertNotIn("targets/emuelec.json", proc.stderr) + self.assertIn("MiSTer FPGA has no hardware targets", proc.stderr) + self.assertNotIn("targets/misterfpga.json", proc.stderr) class TheBoardTheSystemNamesIsTheTarget(unittest.TestCase): @@ -3635,6 +3635,18 @@ class TheBoardTheSystemNamesIsTheTarget(unittest.TestCase): .read_text(encoding="utf-8")) self.assertIn("rg351p", targets) + def test_emuelec_names_its_device(self): + # An OdroidGoAdvance image carries no DuckStation, Dolphin or MAME + # core, and the full list was installed: EmuELEC had no target list. + with unittest.mock.patch.object( + install, "_parse_os_release", return_value={"COREELEC_DEVICE": "OdroidGoAdvance"} + ): + self.assertEqual(install.detect_board("emuelec"), "odroidgoadvance") + targets = json.loads((REPO_ROOT / "install" / "targets" / "emuelec.json") + .read_text(encoding="utf-8")) + self.assertIn("odroidgoadvance", targets) + self.assertNotIn("Duckstation", targets["odroidgoadvance"]) + def test_lakka_names_its_device_or_its_project(self): # An RPi4 Lakka installed 5.7 GB of every board's files: its board # was in os-release and never read. diff --git a/wiki/installer.md b/wiki/installer.md index d7061b97d..6378b191a 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -968,7 +968,7 @@ what the options name, everything by default. |--------|--------| | `--platform NAME` | Install for this platform instead of the detected one. A system `--list-platforms` names under its own files (`muos`, `arkos`) stands for the file list it gets, and the run says which. ES-DE, LaunchBox and Android, which keep no BIOS folder, are answered with the run that fills each emulator's folder. Unknown names are refused with the closest ones | | `--dest PATH` | Destination directory, overriding detection. Without `--platform`, the file list is the platform detected here: at a terminal the installer asks, Enter taking the one found, and otherwise it uses the one found, RetroArch's when nothing is found, and refuses when several are. The run prints whose list it is. A folder that does not exist is named with the nearest one that does, then created, except for a system that runs from its SD card, which refuses it. When more than its last level is missing (a mistyped path), a run at a terminal asks first, and the answer defaults to no | -| `--target NAME` | Keep only the files the cores of that hardware target need, and the systems they emulate. An unknown target is refused rather than ignored, since carrying on would install everything. On Batocera, Recalbox, ROCKNIX, Lakka, KNULLI and RetroPie the board is read from the system (`batocera.arch`, `recalbox.arch`, `HW_DEVICE`, `LIBREELEC_DEVICE` or `LIBREELEC_PROJECT`, `knulli.arch`, the Pi model in `/proc/cpuinfo` where `/opt/retropie` exists) and applied when no `--target` is given, from a pack alone too when it carries the core list per board. On a computer, the board a Batocera or KNULLI card names on its boot partition (`boot/batocera.board`, `boot/knulli.board`) is applied the same way, in line mode and on the full screen. The card names its image (`anbernic-rg35xx-plus`, `x86-64-v3`), which the target list maps to its board (`h700`, `zen3`) | +| `--target NAME` | Keep only the files the cores of that hardware target need, and the systems they emulate. An unknown target is refused rather than ignored, since carrying on would install everything. On Batocera, Recalbox, ROCKNIX, Lakka, KNULLI, AmberELEC, EmuELEC and RetroPie the board is read from the system (`batocera.arch`, `recalbox.arch`, `HW_DEVICE`, `LIBREELEC_DEVICE` or `LIBREELEC_PROJECT`, `knulli.arch`, `COREELEC_DEVICE`, the Pi model in `/proc/cpuinfo` where `/opt/retropie` exists) and applied when no `--target` is given, from a pack alone too when it carries the core list per board. On a computer, the board a Batocera or KNULLI card names on its boot partition (`boot/batocera.board`, `boot/knulli.board`) is applied the same way, in line mode and on the full screen. The card names its image (`anbernic-rg35xx-plus`, `x86-64-v3`), which the target list maps to its board (`h700`, `zen3`) | | `--every-board` | Install the files of every board instead of the one the system names, for a card that moves between devices | | `--installed-cores` | Keep only what the cores already installed read (RetroArch, Lakka, RetroPie). See [Installed cores](#installed-cores) | | `--cores-dir PATH` | The folder RetroArch loads its cores from, for `--installed-cores` when it is not found. Implies `--installed-cores`. A folder that does not exist or cannot be opened is refused with the reason (code 1) | From f55e473183fa39f84d37e53f092f0f9b1853fac0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:55:13 +0100 Subject: [PATCH 3/5] chore: one clause per sentence in a comment --- scripts/scraper/targets/emuelec_targets_scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/scraper/targets/emuelec_targets_scraper.py b/scripts/scraper/targets/emuelec_targets_scraper.py index 3991c68c2..aa1faf6c1 100644 --- a/scripts/scraper/targets/emuelec_targets_scraper.py +++ b/scripts/scraper/targets/emuelec_targets_scraper.py @@ -325,7 +325,7 @@ PROVIDERS: dict[str, tuple[Install, ...]] = { "ikemen-go": (_sa("ikemen-go", 'cp "$PKG_BUILD/bin/Ikemen_Go_"* "$INSTALL/usr/bin/Ikemen_Go"', "ikemen"),), "jzintv": (_sa("jzintv", "cp bin/jzintv ${INSTALL}/usr/bin", "jzintv"),), - # The metapackage's dependencies (line 10); mupen64plus itself comes from + # The metapackage's dependencies (line 10). mupen64plus itself comes from # mupen64plussa-ui-console's makeinstall_target. "mupen64plussa": ( _sa("mupen64plussa", 'PKG_DEPENDS_TARGET="toolchain mupen64plussa-core ' From ebe9745e7f81d6a11694775f88baa768b3b96560 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:55:57 +0100 Subject: [PATCH 4/5] feat: count visits without cookies Pages, referring domains and a few actions (searches that found nothing, downloads, missing addresses, "Did this page help?") are sent to GoatCounter. Nothing is stored on the device, and nothing is sent under Do Not Track or Global Privacy Control. wiki/privacy.md says what is sent. --- tests/test_site_nav.py | 2 +- tests/test_site_privacy.py | 79 +++++++++++++++ website/src/components/Footer.astro | 26 ++++- website/src/scripts/count.ts | 152 ++++++++++++++++++++++++++++ wiki/privacy.md | 50 +++++++++ 5 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 tests/test_site_privacy.py create mode 100644 website/src/scripts/count.ts create mode 100644 wiki/privacy.md diff --git a/tests/test_site_nav.py b/tests/test_site_nav.py index fd66f61fd..f445b6765 100644 --- a/tests/test_site_nav.py +++ b/tests/test_site_nav.py @@ -427,7 +427,7 @@ START_HERE = [ "/retrobios/", "/retrobios/wiki/what-are-bios-files/", "/retrobios/which-pack/", "/retrobios/wiki/first-install/", "/retrobios/wiki/troubleshooting/", "/retrobios/wiki/faq/", "/retrobios/wiki/legal/", "/retrobios/wiki/glossary/", - "/retrobios/platforms/", + "/retrobios/wiki/privacy/", "/retrobios/platforms/", ] diff --git a/tests/test_site_privacy.py b/tests/test_site_privacy.py new file mode 100644 index 000000000..a478c836b --- /dev/null +++ b/tests/test_site_privacy.py @@ -0,0 +1,79 @@ +"""What the site counts stays what wiki/privacy.md says it counts. + +The counter is website/src/scripts/count.ts. These checks read its source: +the guarantees are in a few lines that a later edit could drop without any +page looking different. +""" +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +COUNTER = REPO_ROOT / "website" / "src" / "scripts" / "count.ts" +FOOTER = REPO_ROOT / "website" / "src" / "components" / "Footer.astro" +PRIVACY = REPO_ROOT / "wiki" / "privacy.md" + + +class TheCounterAsksNothingOfTheDevice(unittest.TestCase): + def setUp(self) -> None: + self.source = COUNTER.read_text(encoding="utf-8") + # Comments state the rules: only the code must keep them. + self.code = re.sub(r"^\s*//.*$", "", self.source, flags=re.M) + + def test_no_storage_on_the_device(self) -> None: + for api in ("document.cookie", "localStorage", "sessionStorage", "indexedDB"): + with self.subTest(api=api): + self.assertNotIn(api, self.code) + + def test_a_declined_browser_sends_nothing(self) -> None: + self.assertIn("doNotTrack === '1'", self.code) + self.assertIn("globalPrivacyControl === true", self.code) + self.assertRegex(self.code, r"const send = .*\n\tif \(!enabled\) return;") + + def test_only_the_published_site_counts(self) -> None: + self.assertIn("location.hostname === HOST", self.code) + + def test_no_query_anchor_or_device_detail_is_read(self) -> None: + for read in ("location.search", "location.hash", "location.href,", "screen.", "navigator.language"): + with self.subTest(read=read): + self.assertNotIn(read, self.code) + + def test_the_referrer_is_reduced_to_its_domain(self) -> None: + self.assertIn("from.origin", self.code) + self.assertNotIn("from.pathname", self.code) + + def test_a_search_with_personal_details_is_not_sent(self) -> None: + self.assertIn("!query.includes('@')", self.code) + self.assertIn("\\d{7,}", self.code) + self.assertIn("message.startsWith('No results for')", self.code) + + def test_one_endpoint(self) -> None: + self.assertEqual(set(re.findall(r"https://[\w.-]+", self.code)), + {"https://retrobios.goatcounter.com"}) + + +class ThePageSaysWhatIsCounted(unittest.TestCase): + def test_every_page_loads_the_counter_and_links_the_page(self) -> None: + footer = FOOTER.read_text(encoding="utf-8") + self.assertIn("import '../scripts/count.ts';", footer) + self.assertIn("wiki/privacy/", footer) + # Hidden until the script shows it: a declined browser is not asked. + self.assertRegex(footer, r"data-rb-helpful-box hidden") + + def test_the_page_names_each_thing_sent(self) -> None: + page = PRIVACY.read_text(encoding="utf-8") + for said in ("No cookies", "Do Not Track", "Global Privacy Control", + "copy button", "found nothing", "Did this page help?", + "The installer sends nothing back", "retrobios.goatcounter.com"): + with self.subTest(said=said): + self.assertIn(said, page) + + def test_the_installer_has_no_counter(self) -> None: + installer = (REPO_ROOT / "install.py").read_text(encoding="utf-8") + self.assertNotIn("goatcounter", installer.lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/website/src/components/Footer.astro b/website/src/components/Footer.astro index c70aae7ea..ba5c39970 100644 --- a/website/src/components/Footer.astro +++ b/website/src/components/Footer.astro @@ -1,7 +1,8 @@ --- // Starlight's footer (edit link, previous and next), who keeps the project // and where to report a problem, the release the packs come from, and the -// licence line. The release is read from release.json, written when the +// licence line, and "Did this page help?" (src/scripts/count.ts, shown +// only where the answer is counted). The release is read from release.json, written when the // packs were published: the site may be newer than the packs. None of it // is in the search index: "MIT for the tooling" was the excerpt of short // pages, and "bios", "release" and "problem" matched every page. @@ -24,9 +25,16 @@ const date = releaseDate(tag);
+

Maintained by Abdess on GitHub.{' '} Report a problem. + Privacy.

Packs from release {tag}{date && <>, }. @@ -38,6 +46,10 @@ const date = releaseDate(tag);

+ + diff --git a/website/src/scripts/count.ts b/website/src/scripts/count.ts new file mode 100644 index 000000000..a401fc8d6 --- /dev/null +++ b/website/src/scripts/count.ts @@ -0,0 +1,152 @@ +// What the site counts, as wiki/privacy.md states it. No cookie, no storage +// on the device, no identifier: one request per page or action to +// GoatCounter, which keeps totals. Nothing is sent when the browser asks +// not to be tracked (Do Not Track, Global Privacy Control), from a local +// build, or from a page not served at the site's address. +// +// Sent: the page path (never its query or #anchor), the domain of the site +// the reader came from, and the name of an action. An action is a click on +// a copy button, a pack or file download, a link that leaves the site, a +// site search that found nothing (the words, when they cannot hold a +// person's details), and the answer to "Did this page help?". + +const ENDPOINT = 'https://retrobios.goatcounter.com/count'; +const HOST = 'abdess.github.io'; +const BASE = import.meta.env.BASE_URL.replace(/\/?$/, '/'); + +const declined = (): boolean => { + const nav = navigator as Navigator & { globalPrivacyControl?: boolean }; + return nav.doNotTrack === '1' || nav.globalPrivacyControl === true; +}; +const enabled = location.hostname === HOST && !declined(); + +// The path under the site, "/emulators/mgba/": the base names every page. +const pagePath = (): string => { + const path = location.pathname; + return path.startsWith(BASE) ? `/${path.slice(BASE.length)}` : path; +}; + +const send = (params: Record): void => { + if (!enabled) return; + const url = `${ENDPOINT}?${new URLSearchParams({ ...params, rnd: Math.random().toString(36).slice(2) })}`; + if (!navigator.sendBeacon?.(url)) new Image().src = url; +}; + +// An action, filed under its name with the page it happened on as title. +export const countEvent = (name: string): void => { + send({ p: name.slice(0, 120), t: pagePath(), e: 'true' }); +}; + +// The site the reader came from, by domain alone: a path can carry a +// search or a private thread. +const referrer = (): string => { + try { + const from = new URL(document.referrer); + return from.hostname && from.hostname !== location.hostname ? from.origin : ''; + } catch { + return ''; + } +}; + +const countPage = (): void => { + const params: Record = { p: pagePath() }; + const from = referrer(); + if (from) params.r = from; + // GitHub Pages serves 404.html at the address asked for: the path says + // which link is broken, the title that it was one. + if (document.title.startsWith('Page not found')) params.t = '404'; + send(params); +}; + +// Clicks + +const slug = (text: string): string => text.trim().toLowerCase().replace(/[^a-z0-9.+-]+/g, '-').replace(/^-|-$/g, '').slice(0, 60); + +const linkAction = (link: HTMLAnchorElement): string | undefined => { + let url: URL; + try { + url = new URL(link.href, location.href); + } catch { + return undefined; + } + const release = /\/releases\/download\/[^/]+\/([^/?#]+)$/.exec(url.pathname); + if (url.hostname === 'github.com' && release) return `download/${decodeURIComponent(release[1])}`; + if (url.hostname === 'raw.githubusercontent.com') { + return `file/${decodeURIComponent(url.pathname.split('/').pop() ?? '')}`; + } + if (url.hostname === location.hostname) { + const get = new RegExp(`^${BASE}get/([^/]+)/`).exec(url.pathname); + return get ? `get/${get[1]}` : undefined; + } + return url.protocol.startsWith('http') ? `out/${url.hostname}` : undefined; +}; + +const onClick = (event: MouseEvent): void => { + const target = event.target instanceof Element ? event.target : null; + if (!target) return; + const copy = target.closest('.expressive-code .copy button, [data-copy]'); + if (copy) { + const frame = copy.closest('.frame'); + const title = frame?.querySelector('.title')?.textContent ?? ''; + countEvent(`copy/${slug(title) || (copy.matches('[data-copy]') ? 'picker' : 'code')}`); + return; + } + const help = target.closest('[data-rb-helpful]'); + if (help) { + countEvent(`helpful/${help.dataset.rbHelpful}`); + return; + } + const link = target.closest('a[href]'); + const action = link && linkAction(link); + if (action) countEvent(action); +}; + +// Search + +// A search that found nothing names a missing page. Its words are sent +// only when they cannot hold a person's details: short, no "@", no long +// run of digits. A hash is a file asked for, and is kept. +const isHash = (query: string): boolean => /^[0-9a-f]{8}$|^[0-9a-f]{32}$|^[0-9a-f]{40}$|^[0-9a-f]{64}$/.test(query); +const sendable = (query: string): boolean => + isHash(query) || (query.length >= 2 && query.length <= 40 && !query.includes('@') && !/\d{7,}/.test(query)); + +const watchSearch = (): void => { + const root = document.getElementById('starlight__search'); + if (!root) return; + const sent = new Set(); + let timer: number | undefined; + const settle = (): void => { + const input = root.querySelector('.pagefind-ui__search-input'); + const query = (input?.value ?? '').trim().toLowerCase().replace(/\s+/g, ' '); + // Pagefind's English message once a search has ended with nothing; + // while it runs the message reads "Searching for ...". + const message = root.querySelector('.pagefind-ui__message')?.textContent ?? ''; + if (!query || !message.startsWith('No results for') || sent.has(query) || !sendable(query)) return; + sent.add(query); + countEvent(`search-none/${query}`); + }; + new MutationObserver(() => { + window.clearTimeout(timer); + timer = window.setTimeout(settle, 1500); + }).observe(root, { childList: true, subtree: true, characterData: true }); +}; + +// "Did this page help?": the buttons answer once, then say thanks. +const wireHelpful = (): void => { + // Shown only where an answer is sent: a box that counts nothing would ask for nothing. + if (!enabled) return; + for (const box of document.querySelectorAll('[data-rb-helpful-box]')) { + box.hidden = false; + box.addEventListener('click', (event) => { + if (!(event.target instanceof Element) || !event.target.closest('[data-rb-helpful]')) return; + for (const button of box.querySelectorAll('button')) button.remove(); + const thanks = box.querySelector('[data-rb-helpful-thanks]'); + if (thanks) thanks.hidden = false; + }); + } +}; + +countPage(); +document.addEventListener('click', onClick, { capture: true }); +watchSearch(); +wireHelpful(); diff --git a/wiki/privacy.md b/wiki/privacy.md new file mode 100644 index 000000000..6f0854844 --- /dev/null +++ b/wiki/privacy.md @@ -0,0 +1,50 @@ +--- +level: start +order: 65 +terms: privacy, cookies, analytics, tracking +--- + +# What we count + +No cookies. No account. Nothing is stored on your device, and nothing +links a visit to a person. + +## On this site + +When a page opens, your browser sends one request to +[GoatCounter](https://www.goatcounter.com), an open-source counter. It +carries: + +- the page address, without the part after `?` or `#`; +- the site you came from, by its domain only (`google.com`, `reddit.com`). + +GoatCounter reads the browser name and the country from that request. It does +not keep your IP address. To tell a reload from a second visitor it keeps a +code made from the request for 8 hours, then drops it. + +These actions are counted the same way, with the page they happened on: + +- a click on a copy button, a pack download or a file link; +- a site search that found nothing, with its words, so a missing page shows up. + Words with an `@` or a long number are never sent; +- the answer to "Did this page help?". + +Not sent: your screen size, your language, a search that found results. + +The totals are kept two years and are public: +[retrobios.goatcounter.com](https://retrobios.goatcounter.com). + +If your browser sends Do Not Track or Global Privacy Control, nothing is sent, +and the "Did this page help?" question is not shown. A content blocker that +stops GoatCounter changes nothing on the site. + +## The installer + +The installer sends nothing back. It downloads files from GitHub, which sees +those requests as it sees any download. + +## Search engines + +Google and Bing tell us which searches showed the site and how often it was +clicked. They report totals, from their own records, never from your +browser. From b23840bc218a49a5339bbcf309702f76094da16b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:55:57 +0100 Subject: [PATCH 5/5] feat: add site insights report Search Console queries by page, file names searched against the collection, GoatCounter totals and GitHub traffic in one Markdown report. --- scripts/site_insights.py | 311 ++++++++++++++++++++++++++++++++++++ tests/test_site_insights.py | 67 ++++++++ wiki/testing-guide.md | 2 +- wiki/tools.md | 1 + 4 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 scripts/site_insights.py create mode 100644 tests/test_site_insights.py diff --git a/scripts/site_insights.py b/scripts/site_insights.py new file mode 100644 index 000000000..ec6949f66 --- /dev/null +++ b/scripts/site_insights.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +"""What readers look for and do not find, from totals only. + +Four sources, none of which names a person: + +- Google Search Console: the searches that showed the site, by page. +- GoatCounter: page views, the domains readers came from, and the actions + website/src/scripts/count.ts counts (searches that found nothing, + "Did this page help?", downloads, broken links). wiki/privacy.md says + what is sent. +- GitHub: repository traffic (14 days) and release download counts. +- The collection: which searched file names it holds, and which no + emulator profile declares. + +The report is Markdown on stdout. Credentials stay outside the repository: + + ~/.config/retrobios/gsc-service-account.json Search Console, read only + ~/.config/retrobios/goatcounter-api-token.txt GoatCounter, read only + GITHUB_TOKEN or gh auth token GitHub + +Usage: + python scripts/site_insights.py + python scripts/site_insights.py --days 90 --output report.md + python scripts/site_insights.py --skip gsc,github +""" + +from __future__ import annotations + +import argparse +import base64 +import datetime as dt +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import urllib.parse +import urllib.request +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +sys.path.insert(0, os.path.dirname(__file__)) +from common import github_token, load_database, load_emulator_profiles, run_cli + +REPO_ROOT = Path(__file__).resolve().parent.parent +CONFIG = Path(os.environ.get("RETROBIOS_CONFIG", Path.home() / ".config" / "retrobios")) +SITE_PROPERTY = "https://abdess.github.io/retrobios/" +GOATCOUNTER = "https://retrobios.goatcounter.com/api/v0" +REPOSITORY = "Abdess/retrobios" +SOURCES = ("gsc", "goatcounter", "github") + +# A word of a search that names a file: "scph5501.bin", "prod.keys". +FILE_TOKEN = re.compile(r"\b[\w.+-]+\.(?:bin|rom|zip|pup|keys|img|dat|bios|ic\d+|u\d+|sys|nvm|mec|" + r"rom1|j64|pce|col|sms|gb|gbc|gba|nds|cfg|chd|iso|txt|fw|dll|xml|yaml)\b", re.I) + + +@dataclass(frozen=True) +class Query: + text: str + page: str + clicks: int + impressions: int + position: float + + @property + def ctr(self) -> float: + return self.clicks / self.impressions if self.impressions else 0.0 + + +# Reading the sources + + +def _get_json(url: str, headers: dict[str, str] | None = None, body: dict | None = None) -> dict | list: + data = json.dumps(body).encode() if body is not None else None + request = urllib.request.Request(url, data=data, headers={ + "User-Agent": "retrobios-site-insights", **({"Content-Type": "application/json"} if data else {}), + **(headers or {})}) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + + +def _google_token(key_file: Path) -> str: + """An access token for a service account, signed by openssl: the JWT + grant needs RS256 and the standard library has no RSA.""" + key = json.loads(key_file.read_text(encoding="utf-8")) + now = int(time.time()) + + def b64(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + claims = {"iss": key["client_email"], "scope": "https://www.googleapis.com/auth/webmasters.readonly", + "aud": key["token_uri"], "iat": now, "exp": now + 3600} + unsigned = f"{b64(json.dumps({'alg': 'RS256', 'typ': 'JWT'}).encode())}.{b64(json.dumps(claims).encode())}" + with tempfile.TemporaryDirectory() as scratch: + pem = Path(scratch) / "key.pem" + pem.touch(mode=0o600) + pem.write_text(key["private_key"], encoding="utf-8") + signature = subprocess.run(["openssl", "dgst", "-sha256", "-sign", str(pem)], + input=unsigned.encode(), capture_output=True, check=True).stdout + body = urllib.parse.urlencode({"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "assertion": f"{unsigned}.{b64(signature)}"}).encode() + request = urllib.request.Request(key["token_uri"], data=body) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response)["access_token"] + + +def fetch_queries(start: dt.date, end: dt.date) -> list[Query]: + token = _google_token(CONFIG / "gsc-service-account.json") + url = ("https://www.googleapis.com/webmasters/v3/sites/" + f"{urllib.parse.quote(SITE_PROPERTY, safe='')}/searchAnalytics/query") + rows: list[Query] = [] + while True: + answer = _get_json(url, {"Authorization": f"Bearer {token}"}, { + "startDate": start.isoformat(), "endDate": end.isoformat(), + "dimensions": ["query", "page"], "rowLimit": 25000, "startRow": len(rows)}) + batch = answer.get("rows", []) + rows += [Query(r["keys"][0], r["keys"][1], int(r["clicks"]), int(r["impressions"]), + float(r["position"])) for r in batch] + if len(batch) < 25000: + return rows + + +def fetch_goatcounter(start: dt.date, end: dt.date) -> dict[str, list[tuple[str, int]]]: + """Paths and events with their visit counts, and the referring domains.""" + token = (CONFIG / "goatcounter-api-token.txt").read_text(encoding="utf-8").strip() + auth = {"Authorization": f"Bearer {token}"} + span = {"start": f"{start.isoformat()}T00:00:00Z", "end": f"{end.isoformat()}T23:59:59Z"} + hits: list[tuple[str, int]] = [] + exclude: list[str] = [] + while True: + params = urllib.parse.urlencode({**span, "limit": 100, "exclude_paths": ",".join(exclude)}) + answer = _get_json(f"{GOATCOUNTER}/stats/hits?{params}", auth) + for hit in answer.get("hits", []): + hits.append((hit["path"] + (" [404]" if hit.get("title") == "404" else ""), int(hit["count"]))) + exclude.append(str(hit["path_id"])) + if not answer.get("more"): + break + refs = _get_json(f"{GOATCOUNTER}/stats/toprefs?{urllib.parse.urlencode({**span, 'limit': 50})}", auth) + referrers = [(r.get("name") or "(direct)", int(r["count"])) for r in refs.get("stats", [])] + return {"hits": hits, "referrers": referrers} + + +def fetch_github() -> dict: + token = github_token() + if not token: + raise SystemExit("GitHub traffic needs a token: GITHUB_TOKEN, or gh auth login") + auth = {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"} + api = f"https://api.github.com/repos/{REPOSITORY}" + releases = _get_json(f"{api}/releases?per_page=10", auth) + return { + "views": _get_json(f"{api}/traffic/views", auth), + "clones": _get_json(f"{api}/traffic/clones", auth), + "referrers": _get_json(f"{api}/traffic/popular/referrers", auth), + "paths": _get_json(f"{api}/traffic/popular/paths", auth), + "downloads": [(release["tag_name"], asset["name"], asset["download_count"]) + for release in releases for asset in release.get("assets", [])], + } + + +# Reading the figures + + +def searched_files(queries: list[Query], held: set[str], declared: set[str]) -> list[tuple[str, int, str]]: + """File names typed in searches, by impressions, with what the project + knows of each: held in the collection, declared by a profile only, or + unknown.""" + seen: dict[str, int] = defaultdict(int) + shown: dict[str, str] = {} + for query in queries: + for token in FILE_TOKEN.findall(query.text): + key = token.casefold() + seen[key] += query.impressions + shown.setdefault(key, token) + state = {key: "held" if key in held else "declared, not held" if key in declared else "unknown" + for key in seen} + return sorted(((shown[key], seen[key], state[key]) for key in seen), key=lambda row: -row[1]) + + +def missed_clicks(queries: list[Query], least: int = 100) -> list[Query]: + """On the first page but rarely clicked: what the result says loses + the reader.""" + return sorted((q for q in queries if q.position <= 10 and q.impressions >= least and q.ctr < 0.03), + key=lambda q: -q.impressions) + + +def near_first_page(queries: list[Query], least: int = 50) -> list[Query]: + return sorted((q for q in queries if 10 < q.position <= 20 and q.impressions >= least), + key=lambda q: -q.impressions) + + +def site_actions(hits: list[tuple[str, int]]) -> dict[str, list[tuple[str, int]]]: + """Events by kind ("search-none", "helpful", "download"...), pages apart.""" + groups: dict[str, list[tuple[str, int]]] = defaultdict(list) + for path, count in hits: + if path.endswith(" [404]"): + groups["404"].append((path.removesuffix(" [404]"), count)) + elif not path.startswith("/") and "/" in path: + kind, _, name = path.partition("/") + groups[kind].append((name, count)) + else: + groups["pages"].append((path, count)) + return {kind: sorted(rows, key=lambda row: -row[1]) for kind, rows in groups.items()} + + +# The report + + +def _table(header: tuple[str, ...], rows: list[tuple], limit: int = 25) -> list[str]: + if not rows: + return ["None.", ""] + lines = ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)] + lines += ["| " + " | ".join(str(cell) for cell in row) + " |" for row in rows[:limit]] + return [*lines, ""] + + +def _page(url: str) -> str: + return url.removeprefix(SITE_PROPERTY.rstrip("/")) or "/" + + +def report(start: dt.date, end: dt.date, queries: list[Query] | None, goat: dict | None, + github: dict | None, held: set[str], declared: set[str]) -> str: + out = [f"# Site insights, {start.isoformat()} to {end.isoformat()}", ""] + if queries is not None: + clicks = sum(q.clicks for q in queries) + shown = sum(q.impressions for q in queries) + out += ["## Google searches", "", + f"{clicks:,} clicks, {shown:,} impressions, {len({q.text for q in queries}):,} searches.", "", + "### On the first page, rarely clicked", "", + *_table(("Search", "Page", "Impressions", "CTR", "Position"), + [(q.text, _page(q.page), f"{q.impressions:,}", f"{q.ctr:.1%}", f"{q.position:.1f}") + for q in missed_clicks(queries)]), + "### Just past the first page", "", + *_table(("Search", "Page", "Impressions", "Position"), + [(q.text, _page(q.page), f"{q.impressions:,}", f"{q.position:.1f}") + for q in near_first_page(queries)]), + "### File names searched", "", + *_table(("File", "Impressions", "In the project"), + [(f"`{name}`", f"{seen:,}", state) + for name, seen, state in searched_files(queries, held, declared)], limit=40)] + if goat is not None: + actions = site_actions(goat["hits"]) + named = {"search-none": "Site searches that found nothing", "404": "Broken addresses", + "helpful": "Did this page help?", "get": "Pack pages opened", + "download": "Release downloads", "file": "File downloads", "copy": "Copy buttons", + "out": "Links leaving the site", "pages": "Pages"} + out += ["## On the site", ""] + for kind, title in named.items(): + if kind in actions: + out += [f"### {title}", "", *_table(("Name", "Visits"), [(n, f"{c:,}") for n, c in actions[kind]])] + out += ["### Came from", "", *_table(("Site", "Visits"), [(n, f"{c:,}") for n, c in goat["referrers"]])] + if github is not None: + views, clones = github["views"], github["clones"] + out += ["## GitHub", "", + f"Last 14 days: {views.get('count', 0):,} views by {views.get('uniques', 0):,} visitors, " + f"{clones.get('count', 0):,} clones.", "", + "### Came from", "", + *_table(("Site", "Views", "Visitors"), + [(r["referrer"], r["count"], r["uniques"]) for r in github["referrers"]]), + "### Most viewed", "", + *_table(("Path", "Views"), [(p["path"], p["count"]) for p in github["paths"]]), + "### Release downloads", "", + *_table(("Release", "File", "Downloads"), + sorted(github["downloads"], key=lambda row: -row[2]), limit=40)] + return "\n".join(out).rstrip() + "\n" + + +def collection_names(db: dict, profiles: dict[str, dict]) -> tuple[set[str], set[str]]: + """File names the collection holds, and those a profile declares.""" + held = {name.casefold() for name in db.get("indexes", {}).get("by_name", {})} + declared: set[str] = set() + for profile in profiles.values(): + for entry in profile.get("files") or []: + if isinstance(entry, dict): + for name in [entry.get("name"), *(entry.get("aliases") or [])]: + if name: + declared.add(Path(str(name)).name.casefold()) + return held, declared + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], allow_abbrev=False) + parser.add_argument("--days", type=int, default=28, help="days to read, ending 3 days ago (default 28)") + parser.add_argument("--skip", default="", help=f"sources to leave out, among {', '.join(SOURCES)}") + parser.add_argument("--output", type=Path, help="write the report here instead of stdout") + args = parser.parse_args() + skip = {name.strip() for name in args.skip.split(",") if name.strip()} + unknown = skip - set(SOURCES) + if unknown: + parser.error(f"unknown source(s): {', '.join(sorted(unknown))}") + # Search Console data settles two to three days after the fact. + end = dt.date.today() - dt.timedelta(days=3) + start = end - dt.timedelta(days=args.days - 1) + held, declared = collection_names(load_database(str(REPO_ROOT / "database.json")), + load_emulator_profiles(str(REPO_ROOT / "emulators"))) + queries = None if "gsc" in skip else fetch_queries(start, end) + goat = None if "goatcounter" in skip else fetch_goatcounter(start, dt.date.today()) + github = None if "github" in skip else fetch_github() + text = report(start, end, queries, goat, github, held, declared) + if args.output: + args.output.write_text(text, encoding="utf-8") + print(f"Wrote {args.output}", file=sys.stderr) + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + run_cli(main) diff --git a/tests/test_site_insights.py b/tests/test_site_insights.py new file mode 100644 index 000000000..c9f9b2af9 --- /dev/null +++ b/tests/test_site_insights.py @@ -0,0 +1,67 @@ +"""The readings of scripts/site_insights.py, on synthetic figures.""" +from __future__ import annotations + +import datetime as dt +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import site_insights as si # noqa: E402 + +PAGE = "https://abdess.github.io/retrobios/emulators/mgba/" + + +class FileNamesInSearches(unittest.TestCase): + def test_each_name_carries_what_the_project_knows(self) -> None: + queries = [si.Query("gba_bios.bin download", PAGE, 1, 300, 8.0), + si.Query("where to put GBA_BIOS.BIN", PAGE, 0, 50, 9.0), + si.Query("dsp1b.rom snes", PAGE, 0, 40, 9.0), + si.Query("mystery.pup", PAGE, 0, 10, 30.0), + si.Query("gba bios", PAGE, 5, 900, 7.0)] + rows = si.searched_files(queries, held={"gba_bios.bin"}, declared={"gba_bios.bin", "dsp1b.rom"}) + self.assertEqual(rows, [("gba_bios.bin", 350, "held"), ("dsp1b.rom", 40, "declared, not held"), + ("mystery.pup", 10, "unknown")]) + + +class SearchGaps(unittest.TestCase): + def test_first_page_rarely_clicked(self) -> None: + low = si.Query("swanstation", PAGE, 3, 540, 5.6) + fine = si.Query("nethersx2 bios", PAGE, 54, 484, 4.6) + far = si.Query("xbox bios", PAGE, 0, 900, 10.7) + self.assertEqual(si.missed_clicks([low, fine, far]), [low]) + self.assertEqual(si.near_first_page([low, fine, far]), [far]) + + +class SiteActions(unittest.TestCase): + def test_events_are_grouped_by_kind_and_pages_apart(self) -> None: + hits = [("/emulators/mgba/", 40), ("search-none/kenji nx", 3), ("search-none/dsp1b.rom", 7), + ("helpful/no", 2), ("/old/link/ [404]", 5), ("download/Batocera_43.1_BIOS_Pack.zip", 9)] + actions = si.site_actions(hits) + self.assertEqual(actions["search-none"], [("dsp1b.rom", 7), ("kenji nx", 3)]) + self.assertEqual(actions["404"], [("/old/link/", 5)]) + self.assertEqual(actions["pages"], [("/emulators/mgba/", 40)]) + self.assertEqual(actions["helpful"], [("no", 2)]) + + +class TheReport(unittest.TestCase): + def test_a_skipped_source_leaves_no_section(self) -> None: + text = si.report(dt.date(2026, 10, 1), dt.date(2026, 10, 28), + [si.Query("gba_bios.bin", PAGE, 1, 300, 8.0)], None, None, + held={"gba_bios.bin"}, declared=set()) + self.assertIn("## Google searches", text) + self.assertIn("| gba_bios.bin | /emulators/mgba/ | 300 | 0.3% | 8.0 |", text) + self.assertNotIn("## On the site", text) + self.assertNotIn("## GitHub", text) + + def test_names_come_from_profiles_and_the_collection(self) -> None: + db = {"indexes": {"by_name": {"SCPH5501.BIN": ["x"]}}} + profiles = {"p": {"files": [{"name": "bios/dsp1b.rom", "aliases": ["DSP1B.bin"]}, "str"]}} + held, declared = si.collection_names(db, profiles) + self.assertEqual(held, {"scph5501.bin"}) + self.assertEqual(declared, {"dsp1b.rom", "dsp1b.bin"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 3ef512d15..58c8e7763 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -65,7 +65,7 @@ Every module in `tests/`, by the area it guards. | Pack probe | the cores a platform runs for a system, staging grown to a fixed point, runs mapped to pack and manifest entries, the line and its schema, the probe lock per run, that no page shows its lines. One class builds a real pack narrowed to PlayStation and lays it out both ways, from the archive and with `install.py --dest`, and skips without the collection | `test_pack_probe` | | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_insights`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_privacy`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` | | Games on the card | small DATs, listxml and setup sources, a fake fetcher, fixture install lists, and the committed data against its schemas | `test_arcade_parents`, `test_game_catalogs`, `test_game_headers`, `test_game_regions`, `test_games_index`, `test_romfolders_card`, `test_romfolders_es` | ## Test architecture diff --git a/wiki/tools.md b/wiki/tools.md index dab60e5b0..095c473be 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1353,6 +1353,7 @@ before a release (wiki/release-process.md). Its lines stay in | `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database | | `generate_site.py` | Generate the site (this documentation) into the Starlight project in `website/` | | `sitesources.py` | Record which cited source paths each pinned upstream tree holds, for the site's source links (`--fetch` reads a tree no cache holds) | +| `site_insights.py` | What readers look for and do not find, from totals only: searches that show the site but are rarely clicked, file names searched and whether the collection holds them, site searches that found nothing, broken addresses, "Did this page help?" answers, GitHub traffic and release downloads (`--days`, `--skip gsc,goatcounter,github`, `--output`). Reads Search Console, GoatCounter and GitHub with credentials kept outside the repository. [What the site counts](privacy.md) | | `validate_site.py` | Validate rendered metadata, heading order, image alternatives, JSON-LD, local resources, links and fragments, and the page size and search index budgets | | `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held | | `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch ` reads FBNeo at that tag, branch or commit |