fix: list the libretro cores rocknix installs

ROCKNIX listed retroarch, which stood for every libretro profile (351
resolved), while its devices build 124 libretro packages. The platform
list now names the cores those packages install on at least one device.

The targets scraper read a package's name as its core: vice-lr installs
vice_x128, vice_x64, vice_xpet, vice_xplus4 and vice_xvic, which reached
no device. It now reads each package's makeinstall_target, the file
copied to ${INSTALL}/usr/lib/libretro under its target name, through
TARGET_ARCH and ENABLE_32BIT (pcsx_rearmed-lr, desmume-lr). 155
profiles resolve, each built by a device, and no device loses one.

platforms/rocknix.yml

@@ -6,66 +6,191 @@ base_destination: bios
 hash_type: md5
 verification_mode: md5
 cores:
-- retroarch
+- "81"
+- DoubleCherryGB
+- a5200
 - amiberry
+- arduous
 - ares
 - ares-sa
 - armsx2
 - armsx2-sa
+- atari800
 - azahar
 - azahar-sa
+- b2
+- beetle_gba
+- beetle_lynx
+- beetle_ngp
+- beetle_pce
+- beetle_pce_fast
+- beetle_pcfx
+- beetle_psx
+- beetle_saturn
+- beetle_supafaust
+- beetle_supergrafx
+- beetle_vb
+- beetle_wswan
 - bigpemu
 - bigpemu-sa
+- bk
+- bluemsx
+- boom3
+- boom3_xp
+- bsnes
+- bsnes2014_accuracy
+- bsnes2014_balanced
+- bsnes2014_performance
+- bsnes_hd_beta
+- bsnes_mercury_accuracy
+- bsnes_mercury_balanced
+- bsnes_mercury_performance
+- cap32
 - cemu
 - cemu-sa
+- crocods
 - daedalusx64
 - daedalusx64-sa
+- daphne
+- desmume
 - dolphin
 - dolphin-sa
+- dosbox_core
+- dosbox_pure
 - drastic
 - drastic-sa
 - dsperate
 - dsperate-sa
 - duckstation
 - duckstation-sa
+- easyrpg
+- ecwolf
+- emuscv
+- fake08
+- fbalpha2012
+- fbalpha2019
+- fbneo
+- fceumm
 - flycast
 - flycast-sa
+- flycast2021
+- fmsx
+- freeintv
+- freej2me
+- fuse
+- gambatte
+- gearboy
+- gearcoleco
+- geargrafx
+- gearlynx
+- gearsystem
+- genesis_plus_gx
+- genesis_plus_gx_wide
+- geolith
 - gopher64
 - gopher64-sa
+- gpsp
+- gw
 - gzdoom
 - gzdoom-sa
+- handy
 - hatari
 - hatarisa
 - heroic
 - hypseus-singe
+- jaxe
+- kronos
+- mame
+- mame2003_plus
+- mame2010
+- mame2015
 - mednafen
 - melonds
 - melonds-sa
+- melondsds
+- mesen
+- mesen-s
+- mgba
+- minivmac
+- mojozork
+- mu
 - mupen64plus
 - mupen64plus-sa
+- mupen64plus_next
 - nanoboyadvance
 - nanoboyadvance-sa
+- neocd
+- nestopia
+- np2kai
+- o2em
 - openbor
+- opera
+- parallel_n64
+- pcsx_rearmed
+- pcsx_rearmed32
 - pico-8
+- picodrive
+- pokemini
+- potator
 - ppsspp
 - ppsspp-sa
+- prboom
+- prosystem
+- puae
+- puae2021
+- px68k
+- quasi88
+- quicknes
+- race
 - rpcs3
 - rpcs3-sa
+- same_cdi
+- sameboy
+- sameduck
 - scummvm
 - scummvm-sa
 - skyemu
 - skyemu-sa
+- smsplus
+- snes9x
+- snes9x2002
+- snes9x2005_plus
+- snes9x2010
 - steam
+- stella
 - supermodel
 - supermodel-sa
+- supersnes9x
+- swanstation
+- tgbdual
+- theodore
+- tic80
 - touchhle
 - touchhle-sa
+- tyrquake
+- uae4arm
+- uzem
+- vba_next
+- vbam
+- vecx
 - vice
 - vice-sa
+- vice_x128
+- vice_x64
+- vice_xpet
+- vice_xplus4
+- vice_xvic
+- vircon32
+- virtualjaguar
 - vita3k
 - vita3k-sa
+- vitaquake2
+- vitaquake3
+- wasm4
+- x1
 - xemu
 - xemu-sa
+- yabadabadoo
 - yabasanshiro
 - yabasanshiro-sa
 standalone_cores:

scripts/scraper/rocknix_scraper.py

@@ -190,6 +190,25 @@ class Scraper(BaseScraper):
         self.tag = None if url else fetch_github_latest_version(GITHUB_REPO)
         super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.tag))
 
+    def _built_cores(self, standalone: list[str]) -> tuple[list[str], list[str]]:
+        """(cores, standalone cores) at least one device image carries.
+
+        RetroArch runs the libretro cores the packages of its emulators
+        metapackage install per device
+        (projects/ROCKNIX/packages/virtual/emulators/package.mk), read by
+        the targets scraper, where every libretro profile counted before. A
+        standalone package no device carries is not run either.
+        """
+        try:
+            from .targets.rocknix_targets_scraper import Scraper as DeviceTargets
+        except ImportError:
+            from scraper.targets.rocknix_targets_scraper import Scraper as DeviceTargets
+        builds = DeviceTargets().device_builds(self.tag or launcher_revision(self.url))
+        libretro = sorted({core for _arch, lr, _sa in builds.values() for core in lr})
+        carried = set(standalone_emulators(sorted({p for *_rest, sa in builds.values() for p in sa})))
+        standalone = [name for name in standalone if name in carried]
+        return sorted({*libretro, *standalone}), standalone
+
     def _fetch_standalone(self) -> list[str]:
         """The packages of projects/ROCKNIX/packages/emulators/standalone."""
         import json
@@ -281,7 +300,7 @@ class Scraper(BaseScraper):
             )
             entry["files"].append(requirement_entry(req))
 
-        standalone = self._fetch_standalone()
+        cores, standalone = self._built_cores(self._fetch_standalone())
         return {
             "platform": "ROCKNIX",
             "version": self.tag or fetch_github_latest_version(GITHUB_REPO),
@@ -290,9 +309,7 @@ class Scraper(BaseScraper):
             "base_destination": "bios",
             "hash_type": "md5",
             "verification_mode": "md5",
-            # RetroArch with every libretro core, plus the standalone
-            # emulators ROCKNIX packages beside it.
-            "cores": ["retroarch", *standalone],
+            "cores": cores,
             "standalone_cores": standalone,
             # Launched both ways only where package.mk also adds the
             # emulator as a RetroArch core.

scripts/scraper/targets/rocknix_targets_scraper.py

@@ -5,6 +5,8 @@ Sources (ROCKNIX/distribution, at the release tag platforms/rocknix.yml names):
     image carries, those aarch64 adds, and a `case "${DEVICE}"` block per SoC
   - projects/ROCKNIX/devices/<DEVICE>/options -- ENABLE_32BIT, which the 32-bit
     cores of a device depend on (distributions/ROCKNIX/options defaults it on)
+  - each libretro package's package.mk -- the cores its makeinstall_target
+    copies to ${INSTALL}/usr/lib/libretro (vice-lr installs ten of them)
 
 An H700 or RK3326 handheld carries neither Dolphin nor AetherSX2: without a
 target list, its pack shipped the PS2, GameCube and PS3 firmware anyway.
@@ -13,6 +15,7 @@ target list, its pack shipped the PS2, GameCube and PS3 firmware anyway.
 from __future__ import annotations
 
 import argparse
+import json
 import re
 import sys
 from datetime import datetime, timezone
@@ -89,18 +92,90 @@ def parse_device_cores(
     return libretro, standalone
 
 
-def core_names(libretro: list[str], standalone: list[str]) -> list[str]:
-    """Package names as the profiles' cores: lists spell them.
+_INSTALL_BODY_RE = re.compile(r"^makeinstall_target\(\)\s*\{\n(.*?)^\}", re.M | re.S)
+_COPY_RE = re.compile(r"\bcp\s+(.*?)\s+\$\{?INSTALL\}?/usr/lib/libretro(?:/(\S*))?")
+_CORE_FILE_RE = re.compile(r"([\w+.-]+)_libretro\.(?:so\b|\{[^}]*\bso\b[^}]*\})")
+_TEST_RE = re.compile(r'\[\s*"\$\{(\w+)\}"\s*=\s*"([^"]*)"\s*\]')
 
-    A libretro package is the core's name with `-lr` (`beetle-psx-lr`,
-    `neocd_lr`), spelled with underscores as the buildbot names it. A
-    standalone package is kept under both spellings, as the platform
-    scraper keeps it: `vice-sa` is the only name vice-standalone answers
-    to, `vice` that of the libretro profile.
+
+def installed_cores(package_mk: str, arch: str, with_32bit: bool) -> list[str]:
+    """The cores a libretro package installs on a device.
+
+    The name is that of the file makeinstall_target copies to
+    ${INSTALL}/usr/lib/libretro ($INSTALL in some), renamed when the copy names its target:
+    beetle-psx-lr copies mednafen_psx_libretro.so as beetle_psx_libretro.so,
+    vice-lr ten EMUTYPE builds. pcsx_rearmed-lr picks its file by
+    TARGET_ARCH and adds the 32-bit build when ENABLE_32BIT is on, desmume-lr
+    copies the 32-bit build on aarch64 (ARCH). A test on another variable,
+    or a package that installs no core, is an error.
     """
-    names = {re.sub(r"[-_]lr$", "", package).replace("-", "_") for package in libretro}
-    names.update(standalone_emulators(standalone))
-    return sorted(names)
+    body = _INSTALL_BODY_RE.search(package_mk)
+    if body is None:
+        raise RuntimeError("package.mk has no makeinstall_target")
+    # config/options:36 sets TARGET_ARCH="${ARCH}".
+    values = {"ARCH": arch, "TARGET_ARCH": arch, "ENABLE_32BIT": "true" if with_32bit else "false"}
+
+    def holds(condition: str) -> bool:
+        tests = _TEST_RE.findall(condition)
+        if not tests or re.sub(_TEST_RE, "", condition).strip(" ;&") not in ("", "then"):
+            raise RuntimeError(f"makeinstall_target tests {condition.strip()!r}")
+        for variable, value in tests:
+            if variable not in values:
+                raise RuntimeError(f"makeinstall_target tests ${{{variable}}}")
+            if values[variable] != value:
+                return False
+        return True
+
+    found: list[str] = []
+    stack: list[bool] = []
+    in_case = False
+    for raw in body.group(1).splitlines():
+        line = raw.strip()
+        if re.match(r"case \$\{TARGET_ARCH\} in$", line):
+            in_case = True
+            stack.append(False)
+            continue
+        if line == "esac":
+            in_case = False
+            stack.pop()
+            continue
+        if in_case and (arm := re.match(r"([\w|*]+)\)\s*(.*)$", line)):
+            stack[-1] = arch in arm.group(1).split("|") or arm.group(1) == "*"
+            line = arm.group(2)
+        if line.startswith("if "):
+            stack.append(holds(line[3:]))
+            continue
+        if line == "else":
+            stack[-1] = not stack[-1]
+            continue
+        if line == "fi":
+            stack.pop()
+            continue
+        if not all(stack):
+            continue
+        for sources, target in _COPY_RE.findall(line):
+            if named := _CORE_FILE_RE.search(target):
+                found.append(named.group(1))
+                continue
+            for source in sources.split():
+                if not source.startswith("-") and (named := _CORE_FILE_RE.search(source.rsplit("/", 1)[-1])):
+                    found.append(named.group(1))
+    if not found:
+        raise RuntimeError("makeinstall_target installs no libretro core")
+    return sorted(set(found))
+
+
+def package_paths(tree: list[str]) -> dict[str, str]:
+    """package name -> its package.mk, a ROCKNIX project package first.
+
+    The build takes projects/ROCKNIX/packages/<...>/<name> over
+    packages/<...>/<name> when both exist.
+    """
+    found: dict[str, str] = {}
+    for path in sorted(tree, key=lambda p: not p.startswith("projects/ROCKNIX/")):
+        if path.endswith("/package.mk"):
+            found.setdefault(path.rsplit("/", 2)[-2], path)
+    return found
 
 
 # A package that builds nothing itself and pulls other cores in. idtech-lr
@@ -149,13 +224,40 @@ class Scraper(BaseTargetScraper):
             raise RuntimeError(f"no device in {DEVICES_DIR} at {self.tag}")
         return sorted(devices)
 
-    def fetch_targets(self) -> dict:
-        self.tag = self.pinned_revision(PLATFORM_NAME)
+    def _tree(self) -> list[str]:
+        listing = json.loads(fetch_bytes(
+            f"https://api.github.com/repos/ROCKNIX/distribution/git/trees/{self.tag}?recursive=1",
+            headers=github_headers(),
+        ))
+        if listing.get("truncated"):
+            raise RuntimeError(f"ROCKNIX tree at {self.tag} is truncated")
+        return [entry["path"] for entry in listing.get("tree", [])]
+
+    def device_builds(self, tag: str | None = None) -> dict[str, tuple[str, list[str], list[str]]]:
+        """{device: (architecture, libretro cores, standalone packages)}.
+
+        Read at `tag`, by default the revision platforms/rocknix.yml was
+        scraped at: metapackages resolved to the packages they pull in, and
+        each libretro package to the cores it installs on the device.
+        """
+        self.tag = tag or self.pinned_revision(PLATFORM_NAME)
         package_mk = self._raw(EMULATORS_MK)
         if "LIBRETRO_CORES" not in package_mk:
             raise RuntimeError(f"no LIBRETRO_CORES in {EMULATORS_MK} at {self.tag}")
+        paths = package_paths(self._tree())
+        texts: dict[str, str] = {}
 
-        targets: dict[str, dict] = {}
+        def cores_of(package: str, arch: str, with_32bit: bool) -> list[str]:
+            if package not in paths:
+                raise RuntimeError(f"no package.mk for {package} at {self.tag}")
+            if package not in texts:
+                texts[package] = self._raw(paths[package])
+            try:
+                return installed_cores(texts[package], arch, with_32bit)
+            except RuntimeError as e:
+                raise RuntimeError(f"{paths[package]}: {e}") from e
+
+        builds: dict[str, tuple[str, list[str], list[str]]] = {}
         for device in self._devices():
             options = self._raw(f"{DEVICES_DIR}/{device}/options")
             arch = "x86_64" if device == "AMD64" else "aarch64"
@@ -165,9 +267,19 @@ class Scraper(BaseTargetScraper):
                 if meta in libretro:
                     libretro.remove(meta)
                     libretro += metapackage_cores(self._raw(path), arch)
-            cores = core_names(libretro, standalone)
-            targets[device.lower()] = {"architecture": arch, "cores": cores}
-            print(f"  {device.lower()} ({arch}): {len(cores)} cores", file=sys.stderr)
+            cores = sorted({c for package in libretro for c in cores_of(package, arch, with_32bit)})
+            builds[device.lower()] = (arch, cores, standalone)
+        return builds
+
+    def fetch_targets(self) -> dict:
+        targets: dict[str, dict] = {}
+        for device, (arch, libretro, standalone) in self.device_builds().items():
+            # A standalone package is kept under both spellings, as the
+            # platform scraper keeps it: vice-sa is the only name
+            # vice-standalone answers to, vice that of the libretro profile.
+            cores = sorted({*libretro, *standalone_emulators(standalone)})
+            targets[device] = {"architecture": arch, "cores": cores}
+            print(f"  {device} ({arch}): {len(cores)} cores", file=sys.stderr)
 
         return {
             "platform": PLATFORM_NAME,

tests/test_launchers_rocknix.py

@@ -619,8 +619,10 @@ class PlatformRecords(unittest.TestCase):
         cls.config = yaml.safe_load((REPO_ROOT / "platforms" / "rocknix.yml").read_text())
 
     def test_every_emulator_is_one_rocknix_packages(self):
+        # RetroArch itself runs the libretro cores the list names.
         cores = set(self.config["cores"])
-        self.assertEqual(sorted({r["emulator"] for r in self.config["launchers"]} - cores), [])
+        self.assertEqual(sorted({r["emulator"] for r in self.config["launchers"]} - cores),
+                         ["retroarch"])
 
     def test_every_reference_is_in_the_rocknix_project(self):
         for record in self.config["launchers"]:

tests/test_scraper_contract.py

@@ -480,7 +480,10 @@ class RocknixRunsItsStandaloneEmulators(unittest.TestCase):
         root = Path(__file__).resolve().parents[1]
         platform = yaml.safe_load((root / "platforms" / "rocknix.yml").read_text())
         self.assertIn(f"/distribution/{platform['version']}/", platform["source"])
-        self.assertIn("retroarch", platform["cores"])
+        # RetroArch runs the libretro cores some device image installs, not
+        # every libretro profile.
+        self.assertNotIn("retroarch", platform["cores"])
+        self.assertIn("beetle_psx", platform["cores"])
         registry = yaml.safe_load((root / "platforms" / "_registry.yml").read_text())
         self.assertNotIn("cores", registry["platforms"]["rocknix"])
 

tests/test_target_scrapers.py

@@ -393,35 +393,60 @@ class RocknixPerDevice(unittest.TestCase):
     )
 
     def test_each_device_gets_its_own_cores(self):
-        from scraper.targets.rocknix_targets_scraper import (  # noqa: PLC0415
-            core_names,
-            parse_device_cores,
-        )
+        from scraper.targets.rocknix_targets_scraper import parse_device_cores  # noqa: PLC0415
 
-        h700 = core_names(*parse_device_cores(self.PACKAGE_MK, "H700", "aarch64", True))
-        self.assertTrue({"drastic", "gpsp", "duckstation", "beetle_psx", "hatari",
-                         "81", "neocd"} <= set(h700))
-        self.assertFalse({"aethersx2", "dolphin"} & set(h700))
-        no32 = core_names(*parse_device_cores(self.PACKAGE_MK, "H700", "aarch64", False))
-        self.assertNotIn("gpsp", no32)
-        amd64 = core_names(*parse_device_cores(self.PACKAGE_MK, "AMD64", "x86_64", False))
-        self.assertIn("dolphin", amd64)
-        self.assertNotIn("drastic", amd64)
+        libretro, standalone = parse_device_cores(self.PACKAGE_MK, "H700", "aarch64", True)
+        self.assertTrue({"gpsp-lr", "duckstation-lr", "beetle-psx-lr", "81-lr",
+                         "neocd_lr"} <= set(libretro))
+        self.assertTrue({"drastic-sa", "hatarisa"} <= set(standalone))
+        self.assertFalse({"aethersx2-sa", "dolphin-sa"} & set(standalone))
+        libretro, _ = parse_device_cores(self.PACKAGE_MK, "H700", "aarch64", False)
+        self.assertNotIn("gpsp-lr", libretro)
+        _, standalone = parse_device_cores(self.PACKAGE_MK, "AMD64", "x86_64", False)
+        self.assertIn("dolphin-sa", standalone)
+        self.assertNotIn("drastic-sa", standalone)
+
+    def test_a_package_names_the_cores_it_installs(self):
+        """vice-lr builds ten EMUTYPEs: read as the package name, vice_x64
+        and its siblings reached no ROCKNIX device."""
+        from scraper.targets.rocknix_targets_scraper import installed_cores  # noqa: PLC0415
+
+        vice = ("makeinstall_target() {\n  mkdir -p ${INSTALL}/usr/lib/libretro\n"
+                "    cp -a built/vice_x128_libretro.so ${INSTALL}/usr/lib/libretro\n"
+                "    cp -a built/vice_x64_libretro.so ${INSTALL}/usr/lib/libretro\n}\n")
+        self.assertEqual(installed_cores(vice, "aarch64", True), ["vice_x128", "vice_x64"])
+        renamed = ("makeinstall_target() {\n"
+                   "    cp -a mednafen_psx_libretro.so ${INSTALL}/usr/lib/libretro/beetle_psx_libretro.so\n"
+                   "    cp -a fake08_libretro.{so,info} $INSTALL/usr/lib/libretro\n}\n")
+        self.assertEqual(installed_cores(renamed, "aarch64", True), ["beetle_psx", "fake08"])
+        pcsx = ("makeinstall_target() {\n"
+                "    case ${TARGET_ARCH} in\n"
+                "      aarch64|x86_64) cp -a ../pcsx_rearmed_libretro.so ${INSTALL}/usr/lib/libretro ;;\n"
+                "      arm) cp -a ../pcsx_rearmed_libretro.so ${INSTALL}/usr/lib/libretro/pcsx_rearmed32_libretro.so ;;\n"
+                "    esac\n"
+                '    if [ "${TARGET_ARCH}" = "aarch64" ] && [ "${ENABLE_32BIT}" = "true" ]; then\n'
+                "      cp -a x/usr/lib/libretro/pcsx_rearmed32_libretro.so ${INSTALL}/usr/lib/libretro\n"
+                "    fi\n}\n")
+        self.assertEqual(installed_cores(pcsx, "aarch64", True), ["pcsx_rearmed", "pcsx_rearmed32"])
+        self.assertEqual(installed_cores(pcsx, "aarch64", False), ["pcsx_rearmed"])
+        self.assertEqual(installed_cores(pcsx, "x86_64", True), ["pcsx_rearmed"])
+        with self.assertRaises(RuntimeError):
+            installed_cores('makeinstall_target() {\n  if [ "${DEVICE}" = "H700" ]; then\n'
+                            "    cp a_libretro.so ${INSTALL}/usr/lib/libretro\n  fi\n}\n",
+                            "aarch64", True)
+        with self.assertRaises(RuntimeError):
+            installed_cores("makeinstall_target() {\n  :\n}\n", "aarch64", True)
 
     def test_a_standalone_package_keeps_both_spellings(self):
         """vice-standalone answers to vice-sa only, as scummvm-standalone to
         scummvm-sa: stripped to vice, every targeted pack left them out
         while the untargeted one, read from the platform scraper, kept them."""
         from scraper.rocknix_scraper import standalone_emulators  # noqa: PLC0415
-        from scraper.targets.rocknix_targets_scraper import (  # noqa: PLC0415
-            core_names,
-            parse_device_cores,
-        )
+        from scraper.targets.rocknix_targets_scraper import parse_device_cores  # noqa: PLC0415
 
-        libretro, standalone = parse_device_cores(self.PACKAGE_MK, "SM8550", "aarch64", False)
-        names = set(core_names(libretro, standalone))
+        _, standalone = parse_device_cores(self.PACKAGE_MK, "SM8550", "aarch64", False)
+        names = set(standalone_emulators(standalone))
         self.assertTrue({"aethersx2-sa", "aethersx2", "dolphin-sa", "dolphin"} <= names)
-        self.assertTrue(set(standalone_emulators(standalone)) <= names)
 
     def test_a_metapackage_names_the_cores_it_installs(self):
         from scraper.targets.rocknix_targets_scraper import metapackage_cores  # noqa: PLC0415