libretro/scripts/scraper/targets/retropie_targets_scraper.py

"""Scraper for RetroPie libretro core availability per platform.

Source: https://github.com/RetroPie/RetroPie-Setup/tree/master/scriptmodules/libretrocores
Parses rp_module_id and rp_module_flags from each scriptmodule to determine
which platforms each core supports.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import urllib.error
import urllib.request
from datetime import datetime, timezone

import yaml

from . import BaseTargetScraper

PLATFORM_NAME = "retropie"

GITHUB_API_URL = (
    "https://api.github.com/repos/RetroPie/RetroPie-Setup/contents"
    "/scriptmodules/libretrocores"
)
RAW_BASE_URL = (
    "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/master"
    "/scriptmodules/libretrocores/"
)

# Platform flag sets: flags that the platform possesses
PLATFORM_FLAGS: dict[str, set[str]] = {
    "rpi1": {"arm", "armv6", "rpi", "gles"},
    "rpi2": {"arm", "armv7", "neon", "rpi", "gles"},
    "rpi3": {"arm", "armv8", "neon", "rpi", "gles"},
    "rpi4": {"arm", "armv8", "neon", "rpi", "gles", "gles3", "gles31"},
    "rpi5": {"arm", "armv8", "neon", "rpi", "gles", "gles3", "gles31"},
    "x86": {"x86"},
    "x86_64": {"x86"},
}

ARCH_MAP: dict[str, str] = {
    "rpi1": "armv6",
    "rpi2": "armv7",
    "rpi3": "armv7",
    "rpi4": "aarch64",
    "rpi5": "aarch64",
    "x86": "x86",
    "x86_64": "x86_64",
}

# Flags that are build directives, not platform restrictions
_BUILD_FLAGS = {"nodistcc"}

_MODULE_ID_RE = re.compile(r'rp_module_id\s*=\s*["\']([^"\']+)["\']')
_MODULE_FLAGS_RE = re.compile(r'rp_module_flags\s*=\s*["\']([^"\']*)["\']')


def _fetch(url: str, accept: str = "text/plain") -> str | None:
    try:
        req = urllib.request.Request(
            url,
            headers={"User-Agent": "retrobios-scraper/1.0", "Accept": accept},
        )
        with urllib.request.urlopen(req, timeout=30) as resp:
            return resp.read().decode("utf-8")
    except urllib.error.URLError as e:
        print(f"  skip {url}: {e}", file=sys.stderr)
        return None


def _is_available(flags_str: str, platform: str) -> bool:
    """Return True if the core is available on the given platform."""
    platform_has = PLATFORM_FLAGS.get(platform, set())
    tokens = flags_str.split() if flags_str.strip() else []

    for token in tokens:
        if token in _BUILD_FLAGS:
            continue
        if token.startswith("!"):
            # Exclusion: if platform has this flag, core is excluded
            flag = token[1:]
            if flag in platform_has:
                return False
        else:
            # Requirement: platform must have this flag
            if token not in platform_has:
                return False

    return True


def _parse_module(content: str) -> tuple[str | None, str]:
    """Return (module_id, flags_string) from a scriptmodule file."""
    id_match = _MODULE_ID_RE.search(content)
    flags_match = _MODULE_FLAGS_RE.search(content)
    module_id = id_match.group(1) if id_match else None
    flags = flags_match.group(1) if flags_match else ""
    return module_id, flags


class Scraper(BaseTargetScraper):
    """Fetches RetroPie libretro core availability by parsing scriptmodules."""

    def __init__(self, url: str = GITHUB_API_URL):
        super().__init__(url=url)

    def _list_scriptmodules(self) -> list[str]:
        """Return list of .sh filenames from the libretrocores directory."""
        raw = _fetch(self.url, accept="application/vnd.github+json")
        if raw is None:
            return []
        try:
            entries = json.loads(raw)
        except json.JSONDecodeError as e:
            print(f"  JSON parse error: {e}", file=sys.stderr)
            return []
        return [e["name"] for e in entries if e.get("name", "").endswith(".sh")]

    def _fetch_module(self, filename: str) -> str | None:
        return _fetch(f"{RAW_BASE_URL}{filename}")

    def fetch_targets(self) -> dict:
        print("  listing RetroPie scriptmodules...", file=sys.stderr)
        filenames = self._list_scriptmodules()
        if not filenames:
            print("  warning: no scriptmodules found", file=sys.stderr)

        # {platform: [core_id, ...]}
        platform_cores: dict[str, list[str]] = {p: [] for p in PLATFORM_FLAGS}

        for filename in filenames:
            content = self._fetch_module(filename)
            if content is None:
                continue
            module_id, flags = _parse_module(content)
            if not module_id:
                print(f"  warning: no rp_module_id in {filename}", file=sys.stderr)
                continue
            # Normalize: strip lr- prefix and convert hyphens to underscores
            # to match emulator profile keys (lr-beetle-psx -> beetle_psx)
            core_name = module_id
            if core_name.startswith("lr-"):
                core_name = core_name[3:]
            core_name = core_name.replace("-", "_")
            for platform in PLATFORM_FLAGS:
                if _is_available(flags, platform):
                    platform_cores[platform].append(core_name)

        print(f"  parsed {len(filenames)} scriptmodules", file=sys.stderr)

        targets: dict[str, dict] = {}
        for platform, arch in ARCH_MAP.items():
            cores = sorted(platform_cores.get(platform, []))
            targets[platform] = {
                "architecture": arch,
                "cores": cores,
            }

        return {
            "platform": "retropie",
            "source": self.url,
            "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
            "targets": targets,
        }


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Scrape RetroPie libretro core targets from scriptmodules"
    )
    parser.add_argument("--dry-run", action="store_true", help="Show target summary")
    parser.add_argument("--output", "-o", help="Output YAML file")
    args = parser.parse_args()

    scraper = Scraper()
    data = scraper.fetch_targets()

    if args.dry_run:
        for name, info in data["targets"].items():
            print(f"  {name} ({info['architecture']}): {len(info['cores'])} cores")
        return

    if args.output:
        scraper.write_output(data, args.output)
        print(f"Written to {args.output}")
        return

    print(yaml.dump(data, default_flow_style=False, sort_keys=False))


if __name__ == "__main__":
    main()