libretro/scripts/scraper/knulli_scraper.py

#!/usr/bin/env python3
"""Scraper for KNULLI's knulli-systems.

Source: package/system/knulli-scripts/scripts/knulli-systems
Format: Python source, Batocera's `systems` dict, parsed by batocera_scraper
Hash: MD5

`msx1` and `msx2+` are written twice in the dict. Python keeps the later
value of a repeated key when the script runs, and so does the parser.

Verification (knulli-systems at the release tag):
- checkBios (lines 867-927): md5sum() of the file at /userdata/<file>
  against the declared md5, an empty md5 meaning presence only. For a
  zippedFile entry, checkInsideZip() (lines 814-845) matches the member name
  with casefold() and hashes it. A system or file carrying `emulator` and
  `core` is checked only when es_systems.cfg offers that pair (coreExists,
  lines 847-865).
- Run by EmulationStation's BIOS screen (knulli-cfw/batocera-emulationstation
  10834d00, es-app/src/ApiSystem.cpp:764-774) and by knulli-support:69.

es_systems.yml and configgen-defaults.yml have Batocera's shape, so the cores
are read with Batocera's code. The configgen generators do not: the hatari,
melonds and gsplus generators and libretroOptions.py are older forms than the
ones batocera_launchers reads, so no launcher layer is written.
"""

from __future__ import annotations

import re
import urllib.error
import urllib.request

from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
from .batocera_scraper import (
    SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
)
from .batocera_scraper import (
    _not_cores,
    configgen_cores,
    es_systems_by_system,
    es_systems_libretro_systems,
    extract_systems_dict,
    platform_systems,
    requirements_from_systems,
)
from .es_emulators import attach_system_cores

PLATFORM_NAME = "knulli"

REPO = "knulli-cfw/knulli-linux"
_RAW = "https://raw.githubusercontent.com/knulli-cfw/knulli-linux/{ref}/"
SOURCE_TEMPLATE = _RAW + "package/system/knulli-scripts/scripts/knulli-systems"
CONFIGGEN_DEFAULTS_TEMPLATE = (
    _RAW + "package/system/knulli-configgen/configs/configgen-defaults.yml"
)
ES_SYSTEMS_TEMPLATE = _RAW + "package/emulationstation/knulli-es-system/es_systems.yml"

# Batocera's names, plus the keys KNULLI writes that Batocera does not,
# filed under the slug the other scrapers already give them. msu-md holds the
# Mega-CD BIOS files Batocera files under megadrive-msu.
SYSTEM_SLUG_MAP = {
    **BATOCERA_SLUGS,
    "o2em": "magnavox-odyssey2",
    "msu-md": "sega-mega-cd",
    "xbox": "microsoft-xbox",
}


class Scraper(BaseScraper):
    """Scraper for KNULLI's knulli-systems."""

    def __init__(self, url: str | None = None) -> None:
        self.tag = "" if url else fetch_github_latest_version(REPO)
        super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.tag))

    def _fetch_text(self, url: str) -> str:
        req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
        try:
            with urllib.request.urlopen(req, timeout=30) as resp:
                return resp.read().decode("utf-8")
        except urllib.error.URLError as e:
            raise ConnectionError(f"Failed to fetch {url}: {e}") from e

    def validate_format(self, raw_data: str) -> bool:
        has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
        return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data

    def fetch_requirements(self) -> list[BiosRequirement]:
        raw = self._fetch_raw()
        if not self.validate_format(raw):
            raise ValueError("knulli-systems format validation failed")
        return requirements_from_systems(extract_systems_dict(raw), SYSTEM_SLUG_MAP)

    def generate_platform_yaml(self) -> dict:
        systems = platform_systems(
            self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
        )
        # es_systems.yml files machine selectors and plugins under an
        # emulator as Batocera's does: Batocera's list of them applies, with
        # KNULLI's own.
        defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag))
        es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))
        dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME)
        cores, standalone, dual = configgen_cores(defaults, es_systems, dropped)
        # Each system's own list (psx, es_systems.yml:640-653: libretro
        # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and
        # duckstation-legacy).
        attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped))
        result = {
            "platform": "KNULLI",
            "version": self.tag,
            "homepage": "https://knulli.org",
            "source": self.url,
            "base_destination": "bios",
            "hash_type": "md5",
            "verification_mode": "md5",
            "cores": cores,
            "systems": systems,
        }
        if standalone:
            result["standalone_cores"] = standalone
        if dual:
            result["dual_mode_cores"] = dual
        # configgen hands a libretro core /userdata/saves/<system>, as
        # Batocera's does: the systems each core runs for are its save folders.
        result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)
        return result


def main() -> None:
    from .base_scraper import scraper_cli

    scraper_cli(Scraper, "Scrape KNULLI BIOS requirements")


if __name__ == "__main__":
    main()