feat: add knulli target scraper

KNULLI builds per board on Batocera's Kconfig model from its own tree (knulli-cfw/knulli-linux at the tag platforms/knulli.yml pins): configs/knulli-*.board, package/system/knulli-system/Config.in, knulli-es-system/es_systems.yml and the knulli-cfw/buildroot submodule. Ten boards, the armhf library builds excluded.

platforms/_registry.yml

@@ -925,7 +925,8 @@ platforms:
   # ApiSystem.cpp:764-774). Arch R (archr-linux/Arch-R dbd45d45) ships the
   # same dict as archr-systems: lines 1-870 are identical, only the prefix
   # (/storage/roms instead of /userdata) and the es_systems.cfg path differ.
-  # Many boards, no per-board core list: no target file.
+  # Boards follow Batocera's Kconfig model (configs/knulli-*.board set
+  # BR2_PACKAGE_BATOCERA_TARGET_*), read by knulli_targets_scraper.
   knulli:
     config: knulli.yml
     status: active
@@ -947,8 +948,8 @@ platforms:
       by_core: false
       by_system: true
     schedule: monthly
-    target_scraper: null
-    target_source: null
+    target_scraper: knulli_targets
+    target_source: https://github.com/knulli-cfw/knulli-linux/tree/knulli-main/configs
     install:
       detect:
       # knulli-linux 57afa662: OS_NAME in os-release

platforms/targets/_overrides.yml

@@ -172,12 +172,34 @@ rocknix:
     amd64:
       aliases: [x86_64, pc]
 
-# KNULLI reads Batocera's list above, plus its own: vaixterm is a terminal,
-# filed as the VaixTerm system's emulator (configgen-defaults.yml:669-671).
+# KNULLI's platform scraper reads Batocera's list above, plus this one. The
+# target list reads this one alone, so it repeats Batocera's names, all of
+# which KNULLI's es_systems.yml carries under the same nodes. vaixterm is a
+# terminal, filed as the VaixTerm system's emulator
+# (configgen-defaults.yml:669-671).
 knulli:
   targets:
     _default:
       remove_cores:
+        - A500
+        - "A500+"
+        - A600
+        - A1000
+        - A1200
+        - A3000
+        - A4000
+        - CD32
+        - CDTV
+        - flatpak
+        - odcommander
+        - sh
+        - steam
+        - wine-tkg
+        - wine
+        - mugen
+        - gliden64
+        - glide64mk2
+        - rice
         - vaixterm
 
 # EmuELEC and AmberELEC have no target file: the scrapers read these lists.

scripts/scraper/targets/knulli_targets_scraper.py

@@ -0,0 +1,42 @@
+"""Scraper for KNULLI per-board emulator availability.
+
+KNULLI is a Buildroot external tree on Batocera's model, read with the
+Batocera scraper's Kconfig resolution. Sources (knulli-cfw/knulli-linux, at
+the tag platforms/knulli.yml names):
+  - configs/knulli-*.board -- board definitions, each sets
+    BR2_PACKAGE_BATOCERA_TARGET_* and includes knulli-board.common
+  - package/system/knulli-system/Config.in -- the system config, sourced by
+    the root Config.in in place of Batocera's batocera-system/Config.in
+  - package/emulationstation/knulli-es-system/es_systems.yml
+    -- emulator requireAnyOf flag mapping
+Sources (knulli-cfw/buildroot, at the submodule commit the tag pins):
+  - arch/Config.in, package/mesa3d/Config.in, package/libglvnd/Config.in
+
+The *_armhf_libs boards include knulli-board.libs, which builds 32-bit
+libraries for the matching image and turns no system on: they are no target.
+"""
+
+from __future__ import annotations
+
+from .batocera_targets_scraper import Scraper as BatoceraScraper
+from .batocera_targets_scraper import main as batocera_main
+
+PLATFORM_NAME = "knulli"
+
+
+class Scraper(BatoceraScraper):
+    PLATFORM = PLATFORM_NAME
+    REPO = "knulli-cfw/knulli-linux"
+    BOARD_PREFIX = "knulli-"
+    CONFIG_IN_PATH = "package/system/knulli-system/Config.in"
+    ES_SYSTEMS_PATH = "package/emulationstation/knulli-es-system/es_systems.yml"
+    SYSTEM_SYMBOL = "BR2_PACKAGE_KNULLI_SYSTEM"
+    BUILDROOT_REPO = "knulli-cfw/buildroot"
+
+
+def main() -> None:
+    batocera_main(Scraper)
+
+
+if __name__ == "__main__":
+    main()

tests/test_target_scrapers.py

@@ -176,6 +176,153 @@ class BatoceraKconfig(unittest.TestCase):
                           "BR2_PACKAGE_RUFFLE"} & active)
 
 
+class BatoceraDerivedFlags(unittest.TestCase):
+    """A condition continued with a backslash was cut at its first line:
+    "BR2_PACKAGE_BATOCERA_TARGET_X86_ANY ||" held on every board, so GLES3
+    did too, and a GLES2 Raspberry Pi got yabasanshiro and eduke32. A block
+    whose config is not a package (KNULLI's BR2_KERNEL_LEGACY_3_10) sent
+    the regex backtracking without end."""
+
+    CONFIG_IN = (
+        "if BR2_PACKAGE_BATOCERA_TARGET_X86_64 || \\\n"
+        "   BR2_PACKAGE_BATOCERA_TARGET_H700\n"
+        "\tconfig BR2_PACKAGE_BATOCERA_GLES3\n\tbool \"gles3\"\n\tdefault y\nendif\n\n"
+        "if BR2_PACKAGE_BATOCERA_TARGET_SUNXI_R16\n"
+        "        config BR2_KERNEL_LEGACY_3_10\n        default y\nendif\n\n"
+        + "config BR2_PACKAGE_FILLER\n\tbool \"filler\"\n" * 400
+        + "config BR2_PACKAGE_BATOCERA_SYSTEM\n"
+        "\tselect BR2_PACKAGE_EDUKE32\tif BR2_PACKAGE_BATOCERA_GLES3\n"
+        "\tselect BR2_PACKAGE_DUCKSTATION_LEGACY\tif !BR2_PACKAGE_BATOCERA_GLES3\n"
+    )
+
+    def active(self, board: set[str]) -> frozenset[str]:
+        from scraper.targets.batocera_targets_scraper import (  # noqa: PLC0415
+            _active_symbols,
+            _parse_meta_flags,
+            _parse_owned_selects,
+        )
+
+        return _active_symbols(
+            board | {"BR2_PACKAGE_BATOCERA_SYSTEM"},
+            _parse_meta_flags(self.CONFIG_IN),
+            _parse_owned_selects(self.CONFIG_IN),
+        )
+
+    def test_a_continued_condition_keeps_its_later_lines(self):
+        from scraper.targets.batocera_targets_scraper import (  # noqa: PLC0415
+            _parse_meta_flags,
+        )
+
+        rules = dict(_parse_meta_flags(self.CONFIG_IN))
+        self.assertIn("BR2_PACKAGE_BATOCERA_TARGET_H700", rules["BR2_PACKAGE_BATOCERA_GLES3"])
+        self.assertEqual(rules["BR2_KERNEL_LEGACY_3_10"],
+                         "BR2_PACKAGE_BATOCERA_TARGET_SUNXI_R16")
+
+    def test_gles3_follows_the_board(self):
+        h700 = self.active({"BR2_PACKAGE_BATOCERA_TARGET_H700"})
+        self.assertIn("BR2_PACKAGE_EDUKE32", h700)
+        r16 = self.active({"BR2_PACKAGE_BATOCERA_TARGET_SUNXI_R16"})
+        self.assertNotIn("BR2_PACKAGE_BATOCERA_GLES3", r16)
+        self.assertNotIn("BR2_PACKAGE_EDUKE32", r16)
+
+    def test_a_negated_select_drops_once_the_flag_is_derived(self):
+        """GLES3 is derived on the first pass, after which `!GLES3` no
+        longer holds: accumulating kept duckstation-legacy on every GLES3
+        board."""
+        h700 = self.active({"BR2_PACKAGE_BATOCERA_TARGET_H700"})
+        self.assertNotIn("BR2_PACKAGE_DUCKSTATION_LEGACY", h700)
+        r16 = self.active({"BR2_PACKAGE_BATOCERA_TARGET_SUNXI_R16"})
+        self.assertIn("BR2_PACKAGE_DUCKSTATION_LEGACY", r16)
+
+
+class KnulliPerBoard(unittest.TestCase):
+    """KNULLI builds per board on Batocera's model from its own tree:
+    configs/knulli-*.board, package/system/knulli-system/Config.in and
+    package/emulationstation/knulli-es-system/es_systems.yml, with
+    BR2_PACKAGE_KNULLI_SYSTEM as the config every image turns on."""
+
+    FILES = {
+        "configs/knulli-board.common": "BR2_PACKAGE_KNULLI_SYSTEM=y\n",
+        "configs/knulli-board.libs": "BR2_PACKAGE_BATOCERA_SYSTEM=n\n",
+        "configs/knulli-h700.board": (
+            "include knulli-board.common\n\nBR2_aarch64=y\n"
+            "BR2_PACKAGE_BATOCERA_TARGET_H700=y\n"
+        ),
+        "configs/knulli-r16.board": (
+            "include knulli-board.common\n\nBR2_arm=y\n"
+            "BR2_PACKAGE_BATOCERA_TARGET_SUNXI_R16=y\n"
+        ),
+        "configs/knulli-h700_armhf_libs.board": (
+            "include knulli-board.libs\ninclude knulli-h700_armhf_libs.packages\n"
+            "BR2_arm=y\nBR2_PACKAGE_BATOCERA_TARGET_H700=y\n"
+        ),
+        "configs/knulli-h700_armhf_libs.packages": "BR2_PACKAGE_SDL2=y\n",
+        "package/system/knulli-system/Config.in": (
+            "if BR2_PACKAGE_BATOCERA_TARGET_H700 || \\\n"
+            "   BR2_PACKAGE_BATOCERA_TARGET_A133\n"
+            "\tconfig BR2_PACKAGE_BATOCERA_GLES3\n\tdefault y\nendif\n\n"
+            "config BR2_PACKAGE_KNULLI_SYSTEM\n"
+            "\tselect BR2_PACKAGE_LIBRETRO_MGBA\n"
+            "\tselect BR2_PACKAGE_PPSSPP\tif BR2_PACKAGE_BATOCERA_GLES3\n"
+        ),
+        "package/emulationstation/knulli-es-system/es_systems.yml": (
+            "gba:\n  emulators:\n    libretro:\n"
+            "      mgba: {requireAnyOf: [BR2_PACKAGE_LIBRETRO_MGBA]}\n"
+            "psp:\n  emulators:\n    ppsspp:\n"
+            "      ppsspp: {requireAnyOf: [BR2_PACKAGE_PPSSPP]}\n"
+        ),
+    }
+
+    def test_each_board_gets_its_own_cores(self):
+        from scripts.scraper.targets import knulli_targets_scraper  # noqa: PLC0415
+
+        pin = knulli_targets_scraper.Scraper.pinned_revision("knulli")
+        raw = f"https://raw.githubusercontent.com/knulli-cfw/knulli-linux/{pin}/"
+        api = "https://api.github.com/repos/knulli-cfw/knulli-linux/contents/"
+        seen: list[str] = []
+
+        class _Body:
+            def __init__(self, body: str):
+                self.body = body.encode()
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *exc):
+                return False
+
+            def read(self):
+                return self.body
+
+        def urlopen(request, *_a, **_k):
+            url = request.full_url
+            seen.append(url)
+            if url == f"{api}configs?ref={pin}":
+                names = [p.removeprefix("configs/") for p in self.FILES if p.startswith("configs/")]
+                return _Body(json.dumps([{"name": n} for n in names]))
+            if url == f"{api}buildroot?ref={pin}":
+                return _Body('{"sha": "c2d7985b"}')
+            if url.startswith("https://raw.githubusercontent.com/knulli-cfw/buildroot/c2d7985b/"):
+                return _Body("config BR2_aarch64\n\tselect BR2_ARCH_IS_64\n")
+            path = url.removeprefix(raw)
+            if path in self.FILES:
+                return _Body(self.FILES[path])
+            _refuse()
+
+        with mock.patch.object(batocera_targets_scraper.urllib.request, "urlopen", urlopen):
+            data = knulli_targets_scraper.Scraper().fetch_targets()
+
+        self.assertEqual(data["platform"], "knulli")
+        self.assertEqual(data["source"],
+                         f"https://github.com/knulli-cfw/knulli-linux/tree/{pin}")
+        self.assertEqual(sorted(data["targets"]), ["h700", "r16"])
+        self.assertEqual(data["targets"]["h700"],
+                         {"architecture": "aarch64", "cores": ["mgba", "ppsspp"]})
+        self.assertEqual(data["targets"]["r16"],
+                         {"architecture": "arm", "cores": ["mgba"]})
+        self.assertTrue(all(pin in url or "buildroot/c2d7985b" in url for url in seen), seen)
+
+
 class RocknixPerDevice(unittest.TestCase):
     """ROCKNIX had no target list: an H700 handheld got the PS2, GameCube
     and PS3 firmware of emulators its image does not carry."""

wiki/adding-a-scraper.md

@@ -332,6 +332,7 @@ Register the target scraper in `_registry.yml`:
 |---------|--------|----------|
 | `retroarch_targets` | libretro buildbot nightly | Scrape directory listings for each target arch |
 | `batocera_targets` | Config.in + es_systems.yml | Cross-reference kernel config with system definitions |
+| `knulli_targets` | KNULLI's own Config.in + es_systems.yml | The Batocera scraper's Kconfig resolution against KNULLI's tree |
 | `emudeck_targets` | EmuScripts + RetroArch cores | GitHub API for script availability per OS |
 | `retropie_targets` | scriptmodules + rp_module_flags | Parse Bash scriptmodules for platform flags |
 | `lakka_targets` | Lakka-LibreELEC `package.mk` and project options | Core list minus each project and device's exclusions |

wiki/tools.md

@@ -1648,6 +1648,7 @@ Located in `scripts/scraper/targets/`. Each inherits `BaseTargetScraper` and imp
 |---------|--------|----------------|
 | `retroarch_targets_scraper` | libretro buildbot nightly listings | buildbot architecture |
 | `batocera_targets_scraper` | Config.in + es_systems.yml + Buildroot Kconfig | board |
+| `knulli_targets_scraper` | the same, read from KNULLI's tree and Buildroot fork | board |
 | `recalbox_targets_scraper` | board defconfigs + system.ini cores | board |
 | `rocknix_targets_scraper` | emulators package.mk + device options | device |
 | `lakka_targets_scraper` | libretro_cores package.mk + project and device options | device image |