#!/usr/bin/env python3 """Scraper for KNULLI's knulli-systems. Source: package/system/knulli-scripts/scripts/knulli-systems Format: Python source, Batocera's `systems` dict, parsed by batocera_scraper Hash: MD5 `msx1` and `msx2+` are written twice in the dict. Python keeps the later value of a repeated key when the script runs, and so does the parser. Verification (knulli-systems at the release tag): - checkBios (lines 867-927): md5sum() of the file at /userdata/ against the declared md5, an empty md5 meaning presence only. For a zippedFile entry, checkInsideZip() (lines 814-845) matches the member name with casefold() and hashes it. A system or file carrying `emulator` and `core` is checked only when es_systems.cfg offers that pair (coreExists, lines 847-865). - Run by EmulationStation's BIOS screen (knulli-cfw/batocera-emulationstation 10834d00, es-app/src/ApiSystem.cpp:764-774) and by knulli-support:69. es_systems.yml and configgen-defaults.yml have Batocera's shape, so the cores are read with Batocera's code. The configgen generators do not: the hatari, melonds and gsplus generators and libretroOptions.py are older forms than the ones batocera_launchers reads, so no launcher layer is written. """ from __future__ import annotations import re import urllib.error import urllib.request from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version from .batocera_scraper import ( SYSTEM_SLUG_MAP as BATOCERA_SLUGS, ) from .batocera_scraper import ( _not_cores, configgen_cores, es_systems_by_system, es_systems_libretro_systems, extract_systems_dict, platform_systems, requirements_from_systems, ) from .es_emulators import attach_system_cores PLATFORM_NAME = "knulli" REPO = "knulli-cfw/knulli-linux" _RAW = "https://raw.githubusercontent.com/knulli-cfw/knulli-linux/{ref}/" SOURCE_TEMPLATE = _RAW + "package/system/knulli-scripts/scripts/knulli-systems" CONFIGGEN_DEFAULTS_TEMPLATE = ( _RAW + "package/system/knulli-configgen/configs/configgen-defaults.yml" ) ES_SYSTEMS_TEMPLATE = _RAW + "package/emulationstation/knulli-es-system/es_systems.yml" # Batocera's names, plus the keys KNULLI writes that Batocera does not, # filed under the slug the other scrapers already give them. msu-md holds the # Mega-CD BIOS files Batocera files under megadrive-msu. SYSTEM_SLUG_MAP = { **BATOCERA_SLUGS, "o2em": "magnavox-odyssey2", "msu-md": "sega-mega-cd", "xbox": "microsoft-xbox", } class Scraper(BaseScraper): """Scraper for KNULLI's knulli-systems.""" def __init__(self, url: str | None = None) -> None: self.tag = "" if url else fetch_github_latest_version(REPO) super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.tag)) def _fetch_text(self, url: str) -> str: req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"}) try: with urllib.request.urlopen(req, timeout=30) as resp: return resp.read().decode("utf-8") except urllib.error.URLError as e: raise ConnectionError(f"Failed to fetch {url}: {e}") from e def validate_format(self, raw_data: str) -> bool: has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data def fetch_requirements(self) -> list[BiosRequirement]: raw = self._fetch_raw() if not self.validate_format(raw): raise ValueError("knulli-systems format validation failed") return requirements_from_systems(extract_systems_dict(raw), SYSTEM_SLUG_MAP) def generate_platform_yaml(self) -> dict: systems = platform_systems( self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP ) # es_systems.yml files machine selectors and plugins under an # emulator as Batocera's does: Batocera's list of them applies, with # KNULLI's own. defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag)) es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag)) dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME) cores, standalone, dual = configgen_cores(defaults, es_systems, dropped) # Each system's own list (psx, es_systems.yml:640-653: libretro # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and # duckstation-legacy). attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped)) result = { "platform": "KNULLI", "version": self.tag, "homepage": "https://knulli.org", "source": self.url, "base_destination": "bios", "hash_type": "md5", "verification_mode": "md5", "cores": cores, "systems": systems, } if standalone: result["standalone_cores"] = standalone if dual: result["dual_mode_cores"] = dual # configgen hands a libretro core /userdata/saves/, as # Batocera's does: the systems each core runs for are its save folders. result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped) return result def main() -> None: from .base_scraper import scraper_cli scraper_cli(Scraper, "Scrape KNULLI BIOS requirements") if __name__ == "__main__": main()