feat: add knulli, amberelec and emuelec platforms

Each OS publishes Batocera's systems dict (knulli-systems, and
batocera-systems at AmberELEC's newest prerelease commit and EmuELEC v4.8),
read with the shared parser. Cores come from each OS's own ES list.
AmberELEC and EmuELEC declare neogeo/ and ports/ beside bios/ from
/storage/roms, so their packs extract there and core_destination puts
core files under bios/.
This commit is contained in:
Abdessamad Derraz committed 2026-10-22 03:42:20 +02:00
1 parent 2fc62bd249
commit 2bdac466a9
18 files changed
+12933 -24

No files matched your search

+82 -11
View File
@@ -313,17 +313,6 @@ platforms:
target_source: https://github.com/batocera-linux/batocera.linux
install:
detect:
# Knulli, a Batocera derivative (knulli-cfw/knulli-linux 57afa662):
# OS_NAME in os-release (board/scripts/post-build-script.sh:233-236),
# and knulli.version (package/system/knulli-system/knulli-system.mk:98-102).
- os: linux
method: os_release
match: [OS_NAME=knulli]
bios_path: /userdata/bios
- os: linux
method: file_exists
file: /usr/share/knulli/knulli.version
bios_path: /userdata/bios
# batocera.linux 38845725, package/batocera/core/batocera-system/
# batocera-system.mk:21-22 writes batocera.arch and batocera.version.
- os: linux
@@ -571,6 +560,88 @@ platforms:
method: os_release
match: [OS_NAME=rocknix, ID=rocknix]
bios_path: /storage/roms/bios
# AmberELEC's checker (batocera-systems) md5s each file under /storage/roms,
# where its list also names neogeo/neogeo.zip and neogeo/aes.zip: the pack
# is extracted there, not into bios/. EmulationStation's BIOS screen runs
# it (ApiSystem.cpp:603-613 at e3846751). The launch-failure check in
# 99-distribution.conf:320 passes --strictfilter, an argument the script
# does not handle, so that path reports nothing. No release since
# 20230203: the scraper reads the commit of the newest prerelease.
# Several device images, no per-device core list: no target file.
amberelec:
config: amberelec.yml
status: active
extract_to: /storage/roms/
logo: https://avatars.githubusercontent.com/u/102820909
scraper: amberelec
source_url: https://github.com/AmberELEC/AmberELEC/blob/dev/packages/amberelec/sources/scripts/batocera-systems
source_format: python_dict
hash_type: md5
schedule: monthly
target_scraper: null
target_source: null
install:
detect:
- os: linux
method: os_release
match: [NAME=AmberELEC]
bios_path: /storage/roms
# EmuELEC's checker md5s each file under /storage/roms and never reads
# zippedFile: its FM Towns entries hash the whole archive against a
# member's md5 and report UNTESTED whatever the archive holds. Run by
# EmulationStation's BIOS screen and by ee_check_bios after a failed
# launch (99-emuelec.conf:261). naomi and Dinothawr are written twice in
# the dict and the later entry is the one Python keeps.
emuelec:
config: emuelec.yml
status: active
extract_to: /storage/roms/
logo: https://avatars.githubusercontent.com/u/57574835
scraper: emuelec
source_url: https://github.com/EmuELEC/EmuELEC/blob/master/packages/sx05re/emuelec/bin/batocera/batocera-systems
source_format: python_dict
hash_type: md5
schedule: monthly
target_scraper: null
target_source: null
install:
detect:
- os: linux
method: os_release
match: [ID=emuelec]
bios_path: /storage/roms
# KNULLI's knulli-systems is its own fork of Batocera's list, checked the
# same way (md5, checkInsideZip, emulator/core gates) by its
# EmulationStation (knulli-cfw/batocera-emulationstation 10834d00,
# ApiSystem.cpp:764-774). Arch R (archr-linux/Arch-R dbd45d45) ships the
# same dict as archr-systems: lines 1-870 are identical, only the prefix
# (/storage/roms instead of /userdata) and the es_systems.cfg path differ.
# Many boards, no per-board core list: no target file.
knulli:
config: knulli.yml
status: active
extract_to: /userdata/bios/
logo: https://avatars.githubusercontent.com/u/166268572
scraper: knulli
source_url: https://github.com/knulli-cfw/knulli-linux/blob/knulli-main/package/system/knulli-scripts/scripts/knulli-systems
source_format: python_dict
hash_type: md5
schedule: monthly
target_scraper: null
target_source: null
install:
detect:
# knulli-linux 57afa662: OS_NAME in os-release
# (board/scripts/post-build-script.sh:233-236) and knulli.version
# (package/system/knulli-system/knulli-system.mk:98-102).
- os: linux
method: os_release
match: [OS_NAME=knulli]
bios_path: /userdata/bios
- os: linux
method: file_exists
file: /usr/share/knulli/knulli.version
bios_path: /userdata/bios
retropie:
config: retropie.yml
status: archived
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+4820
View File
File diff suppressed because it is too large. Load diff
+5
View File
@@ -66,6 +66,11 @@
"type": "string",
"description": "Base directory for BIOS files on this platform"
},
"core_destination": {
"type": "string",
"minLength": 1,
"description": "With an empty base_destination, the directory core files go under (AmberELEC and EmuELEC declare bios/, neogeo/ and ports/ from /storage/roms, and their cores read bios/)"
},
"hash_type": {
"type": "string",
"enum": [
+5 -1
View File
@@ -2842,10 +2842,14 @@ def declared_root(config: dict) -> str:
Only a platform whose base_destination is explicitly empty does that
(RetroDECK writes bios/ in front of each file), and core files go under
the same directory. A config that does not state a base says nothing,
and its common folder is a system's, not the BIOS root.
and its common folder is a system's, not the BIOS root. A platform whose
list also names folders beside the BIOS one states it as
core_destination: EmuELEC's ports/ entries are a quarter of its list.
"""
if config.get("base_destination", None) != "":
return ""
if config.get("core_destination"):
return str(config["core_destination"])
roots = [
d.split("/", 1)[0]
for system in config.get("systems", {}).values()
+46
View File
@@ -0,0 +1,46 @@
"""Exporter for AmberELEC's batocera-systems.
Batocera's script shape, so only the systems dict is rewritten. Two things
differ. Paths are relative to /storage/roms, where the list also names
neogeo/ beside bios/, and our destinations start there too. No entry carries
an emulator or core gate: coreExists() lets an ungated item through for
every build, and an addition is written the same way.
"""
from __future__ import annotations
from .base_exporter import BaseExporter
from .baseline import NativeFile
from .batocera_exporter import Exporter as BatoceraExporter
SOURCE_URL = (
"https://raw.githubusercontent.com/AmberELEC/AmberELEC/dev"
"/packages/amberelec/sources/scripts/batocera-systems"
)
class Exporter(BatoceraExporter):
"""Write AmberELEC's batocera-systems, corrected."""
PATH_PREFIX = ""
@classmethod
def writable(cls, fe: NativeFile, require: str = "") -> bool:
# The list gates nothing on a runner, so an addition needs none.
return BaseExporter.writable.__func__(cls, fe, require)
def _addition_keys(self, fe: NativeFile) -> list[tuple[str, str | None]] | None:
return []
@staticmethod
def platform_name() -> str:
return "amberelec"
@staticmethod
def file_modes() -> dict[str, int]:
# 100755 at a21975bf (git/trees).
return {"batocera-systems": 0o755}
@staticmethod
def native_sources() -> dict[str, str]:
return {"batocera-systems": SOURCE_URL}
+42 -11
View File
@@ -35,6 +35,27 @@ def _commented(text: str, position: int) -> bool:
return "#" in _STRING.sub("", text[line_start:position])
def _closing_line(text: str, start: int) -> int | None:
"""Index of the line holding the brace that closes `systems = {`.
Read from the parsed module, never executed. None when the script does
not parse or names no such assignment at line `start`.
"""
try:
tree = ast.parse(text)
except SyntaxError:
return None
for node in tree.body:
if (
isinstance(node, ast.Assign)
and node.lineno - 1 == start
and isinstance(node.value, ast.Dict)
and node.end_lineno is not None
):
return node.end_lineno - 1
return None
def _note_key(text: str) -> tuple[str, str]:
"""(file, zippedFile) of the item a line holds, ("", "") for none."""
path = re.search(r'"file"\s*:\s*"([^"]+)"', text)
@@ -55,6 +76,11 @@ class Exporter(BaseExporter):
# (duckstation-sa), which its checker does not know.
STANDALONE_RUNNERS = True
# What the script's paths add in front of a destination. Batocera's
# paths are relative to /userdata and our destinations to its bios/;
# AmberELEC's and EmuELEC's destinations already start at /storage/roms.
PATH_PREFIX = "bios/"
def __init__(self) -> None:
super().__init__()
# The emulator and core each system entry of the original is gated
@@ -200,8 +226,7 @@ class Exporter(BaseExporter):
alt = fe.native("alt_md5", "")
if alt:
item.append(f'"altmd5": "{alt}"')
declared = fe.native("native_path", "")
path = str(declared) if declared else f"bios/{fe.destination}"
path = self.declared_path(fe)
item.append(f'"file": "{path}"')
zipped = fe.native("zipped_file", "")
if zipped:
@@ -209,6 +234,12 @@ class Exporter(BaseExporter):
item.extend(self._runner_keys(originals.get(path)) if runner is None else runner)
return "{ " + ", ".join(item) + " }"
@classmethod
def declared_path(cls, fe: NativeFile) -> str:
"""The `file` the script names for an entry."""
declared = fe.native("native_path", "")
return str(declared) if declared else f"{cls.PATH_PREFIX}{fe.destination}"
@staticmethod
def _own_entry(system: NativeSystem, before: dict | None) -> dict:
"""The maintainer's entry for this system, out of the parsed pair."""
@@ -396,6 +427,9 @@ class Exporter(BaseExporter):
(i for i in range(start + 1, len(lines)) if lines[i].startswith("}")),
None,
)
if end is None:
# EmuELEC closes the dict with an indented brace.
end = _closing_line(original, start)
if end is None:
raise ValueError("batocera-systems: the systems dict is not closed")
return lines[: start + 1], lines[start + 1 : end], lines[end:]
@@ -520,14 +554,12 @@ class Exporter(BaseExporter):
content = produced[self.native_filename()]
issues: list[str] = []
block = content.split("\nsystems = {", 1)
if len(block) != 2:
return ["no systems dict in the output"]
end = block[1].find("\n}")
if end < 0:
return ["the systems dict is not closed"]
try:
exported = ast.literal_eval("{" + block[1][:end] + "\n}")
_head, body, _tail = self._split_original(content)
except ValueError as exc:
return [str(exc)]
try:
exported = ast.literal_eval("{" + "\n".join(body) + "\n}")
except (SyntaxError, ValueError) as exc:
return [f"the systems dict does not parse: {exc}"]
@@ -541,8 +573,7 @@ class Exporter(BaseExporter):
continue
declared = {bios.get("file", "") for bios in entry.get("biosFiles", [])}
for fe in files:
path = str(fe.native("native_path", "")) or f"bios/{fe.destination}"
if path not in declared:
if self.declared_path(fe) not in declared:
issues.append(f"absent: {system.native_id}/{fe.name}")
for native_id, entry in exported.items():
+32
View File
@@ -0,0 +1,32 @@
"""Exporter for EmuELEC's batocera-systems.
AmberELEC's case: paths relative to /storage/roms with ports/ and neogeo/
beside bios/, and no runner gate, since EmuELEC's checkBios never reads one.
The dict closes on an indented brace.
"""
from __future__ import annotations
from .amberelec_exporter import Exporter as AmberELECExporter
SOURCE_URL = (
"https://raw.githubusercontent.com/EmuELEC/EmuELEC/master"
"/packages/sx05re/emuelec/bin/batocera/batocera-systems"
)
class Exporter(AmberELECExporter):
"""Write EmuELEC's batocera-systems, corrected."""
@staticmethod
def platform_name() -> str:
return "emuelec"
@staticmethod
def file_modes() -> dict[str, int]:
# 100755 at v4.8 (git/trees).
return {"batocera-systems": 0o755}
@staticmethod
def native_sources() -> dict[str, str]:
return {"batocera-systems": SOURCE_URL}
+31
View File
@@ -0,0 +1,31 @@
"""Exporter for KNULLI's knulli-systems.
A fork of Batocera's script, checked the same way: es_systems names a
libretro core under `libretro` and a standalone by itself, and entries carry
Batocera's emulator and core gates.
"""
from __future__ import annotations
from .batocera_exporter import Exporter as BatoceraExporter
SOURCE_URL = (
"https://raw.githubusercontent.com/knulli-cfw/knulli-linux/knulli-main"
"/package/system/knulli-scripts/scripts/knulli-systems"
)
class Exporter(BatoceraExporter):
"""Write KNULLI's knulli-systems, corrected."""
@staticmethod
def platform_name() -> str:
return "knulli"
@staticmethod
def native_filename() -> str:
return "knulli-systems"
@staticmethod
def native_sources() -> dict[str, str]:
return {"knulli-systems": SOURCE_URL}
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
"""Scraper for AmberELEC's batocera-systems.
Source: packages/amberelec/sources/scripts/batocera-systems
Format: Python source, Batocera's `systems` dict, parsed by batocera_scraper
Hash: MD5
AmberELEC has published no release since 20230203. Its builds come from the
dev branch as prereleases in AmberELEC/AmberELEC-prerelease, each naming the
commit it was built from. The scraper reads the list at the commit of the
newest one. That list accepts every md5 the 20230203 one does for the files
it keeps (syscard3.pce 38179df8, BS-X.bin 96cf17bf) and adds
neogeo/aes.zip.
Verification (batocera-systems at that commit):
- checkBios (lines 583-643): md5sum() of the file at /storage/roms/<file>
against the declared md5, an empty md5 meaning presence only. For a
zippedFile entry, checkInsideZip() (lines 530-561) matches the member name
with casefold() and hashes it.
- Run with no argument by EmulationStation's BIOS screen
(AmberELEC/emulationstation e3846751, es-app/src/ApiSystem.cpp:603-613),
which also passes `--filter <system>`. ee_check_bios
(packages/amberelec/profile.d/99-distribution.conf:320) passes
`--strictfilter`, which the script's argument switch (lines 665-687) does
not handle: that call prints nothing and exits 0.
"""
from __future__ import annotations
import json
import re
import urllib.error
import urllib.request
from .base_scraper import BaseScraper, BiosRequirement, github_headers
from .batocera_scraper import (
SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
)
from .batocera_scraper import (
extract_systems_dict,
platform_systems,
requirements_from_systems,
)
from .es_emulators import es_cores, xml_emulators
PLATFORM_NAME = "amberelec"
REPO = "AmberELEC/AmberELEC"
PRERELEASE_REPO = "AmberELEC/AmberELEC-prerelease"
_RAW = "https://raw.githubusercontent.com/AmberELEC/AmberELEC/{ref}/"
SOURCE_TEMPLATE = _RAW + "packages/amberelec/sources/scripts/batocera-systems"
ES_SYSTEMS_TEMPLATE = _RAW + "packages/ui/emulationstation/config/es_systems.cfg"
# Batocera's names, plus the keys AmberELEC writes that Batocera does not,
# filed under the slug the other scrapers already give them.
SYSTEM_SLUG_MAP = {
**BATOCERA_SLUGS,
"atari7800": "atari-7800",
"atarilynx": "atari-lynx",
"n64": "nintendo-64",
"o2em": "magnavox-odyssey2",
}
# runemu.py: `retroarch` and `retrorun` load the libretro core es_systems.cfg
# names. `mame` lists MAME machines and runs them on the libretro MAME core
# (lines 233-261 set self.core = 'mame', line 355 builds the RetroArch command).
RUNNERS: dict[str, str | None] = {"retroarch": None, "retrorun": None, "mame": "mame"}
# runemu.py:113, PPSSPPSDL runs /usr/bin/ppsspp.sh, installed by the
# ppssppsa package (packages/games/emulators/ppssppsa).
RENAMED = {"PPSSPPSDL": "ppsspp"}
_PRERELEASE_COMMIT = re.compile(r"AmberELEC/AmberELEC@([0-9a-f]{40})")
def prerelease_commit(body: str) -> str:
"""The commit a prerelease's notes say it was built from."""
found = _PRERELEASE_COMMIT.search(body or "")
if found is None:
raise RuntimeError("the newest AmberELEC prerelease names no commit")
return found.group(1)
def fetch_newest_prerelease() -> tuple[str, str]:
"""(tag, commit) of the newest AmberELEC prerelease."""
url = f"https://api.github.com/repos/{PRERELEASE_REPO}/releases?per_page=1"
try:
req = urllib.request.Request(url, headers=github_headers())
with urllib.request.urlopen(req, timeout=15) as resp:
releases = json.loads(resp.read())
except (urllib.error.URLError, json.JSONDecodeError) as e:
raise RuntimeError(f"cannot list the releases of {PRERELEASE_REPO}: {e}") from e
if not releases or not releases[0].get("tag_name"):
raise RuntimeError(f"{PRERELEASE_REPO} lists no release")
return releases[0]["tag_name"], prerelease_commit(releases[0].get("body", ""))
class Scraper(BaseScraper):
"""Scraper for AmberELEC's batocera-systems."""
def __init__(self, url: str | None = None) -> None:
if url:
self.tag, self.commit = "", ""
else:
self.tag, self.commit = fetch_newest_prerelease()
super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.commit))
def _fetch_text(self, url: str) -> str:
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
try:
with urllib.request.urlopen(req, timeout=30) as resp:
return resp.read().decode("utf-8")
except urllib.error.URLError as e:
raise ConnectionError(f"Failed to fetch {url}: {e}") from e
def validate_format(self, raw_data: str) -> bool:
has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data
def fetch_requirements(self) -> list[BiosRequirement]:
raw = self._fetch_raw()
if not self.validate_format(raw):
raise ValueError("batocera-systems format validation failed")
# Paths are relative to /storage/roms (`prefix`, lines 667-675): the
# list names neogeo/neogeo.zip beside bios/.
return requirements_from_systems(extract_systems_dict(raw), SYSTEM_SLUG_MAP, strip="")
def generate_platform_yaml(self) -> dict:
systems = platform_systems(
self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
)
cores, standalone, dual = es_cores(
xml_emulators(self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.commit))),
RUNNERS,
RENAMED,
)
return {
"platform": "AmberELEC",
"version": self.tag,
"homepage": "https://amberelec.org",
"source": self.url,
"base_destination": "",
# RetroArch's system_directory is ~/roms/bios
# (packages/games/tools/retroarch/sources/retroarch.cfg:611).
"core_destination": "bios",
"hash_type": "md5",
"verification_mode": "md5",
"cores": cores,
"standalone_cores": standalone,
"dual_mode_cores": dual,
"systems": systems,
}
def main() -> None:
from .base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape AmberELEC BIOS requirements")
if __name__ == "__main__":
main()
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""Scraper for EmuELEC's batocera-systems.
Source: packages/sx05re/emuelec/bin/batocera/batocera-systems
Format: Python source, Batocera's `systems` dict, parsed by batocera_scraper
Hash: MD5
`naomi` and `Dinothawr` are written twice in the dict. Python keeps the later
value of a repeated key when the script runs, and so does the parser: naomi
is the entry at line 461 (naomi_boot.bin and naomi.zip, line 444 naming
naomi.zip alone), Dinothawr the one at line 776 (dinothawr.game, line 494
naming level_3-3.tmx).
Verification (batocera-systems at the release tag):
- checkBios (lines 817-833): md5sum() of the file at /storage/roms/<file>
against the declared md5, an empty md5 meaning presence only. It never
reads zippedFile: the FM Towns entries that name a member hash the whole
archive against the member's md5. There is no emulator or core gate.
- Run with no argument by EmulationStation's BIOS screen
(EmuELEC/emuelec-emulationstation 09356d0d, es-app/src/ApiSystem.cpp:
645-655, `--filter <system>` for one system), and with `--strictfilter
<platform>` by ee_check_bios after a failed launch
(packages/sx05re/emuelec/profile.d/99-emuelec.conf:261, called from
emuelecRunEmu.sh:605).
"""
from __future__ import annotations
import re
import urllib.error
import urllib.request
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
from .batocera_scraper import (
SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
)
from .batocera_scraper import (
extract_systems_dict,
platform_systems,
requirements_from_systems,
)
from .es_emulators import es_cores, json_emulators
PLATFORM_NAME = "emuelec"
REPO = "EmuELEC/EmuELEC"
_RAW = "https://raw.githubusercontent.com/EmuELEC/EmuELEC/{ref}/"
SOURCE_TEMPLATE = _RAW + "packages/sx05re/emuelec/bin/batocera/batocera-systems"
ES_SYSTEMS_TEMPLATE = _RAW + "packages/sx05re/emuelec-emulationstation/config/es_systems.json"
# Batocera's names, plus the keys EmuELEC writes that Batocera does not:
# console keys under the slug the other scrapers give them, and each port
# under the system of the profile of the engine its name gives
# (devilutionx, prboom, opentyrian, cannonball, nxengine, tyrquake,
# reminiscence, xrick, vvvvvv, eduke32, sonic2013, sonic-mania, dinothawr).
SYSTEM_SLUG_MAP = {
**BATOCERA_SLUGS,
"atari7800": "atari-7800",
"atarilynx": "atari-lynx",
"n64": "nintendo-64",
"o2em": "magnavox-odyssey2",
"Diablo": "diablo",
"Hellfire": "diablo",
"Dinothawr": "dinothawr",
"Quake": "quake",
"VVVVVV": "vvvvvv",
"DOOM1": "doom",
"DOOM2": "doom",
"OpenTyrian": "tyrian",
"Cannonball": "sega-outrun",
"CaveStory": "cave-story",
"REminiscence": "flashback",
"RickDangerous": "xrick",
"eduke32": "duke3d",
"sonic1": "sonic-1-2013",
"sonic2": "sonic-2-2013",
"sonicmania": "sonic-mania",
}
# es_systems.json names libretro cores under `libretro`, and `retrorun`
# loads the libretro core it lists.
RUNNERS: dict[str, str | None] = {"libretro": None, "retrorun": None}
# packages/sx05re/emulators/PPSSPPSDL builds PPSSPP's SDL front end.
RENAMED = {"PPSSPPSDL": "ppsspp"}
class Scraper(BaseScraper):
"""Scraper for EmuELEC's batocera-systems."""
def __init__(self, url: str | None = None) -> None:
self.tag = "" if url else fetch_github_latest_version(REPO)
super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.tag))
def _fetch_text(self, url: str) -> str:
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
try:
with urllib.request.urlopen(req, timeout=30) as resp:
return resp.read().decode("utf-8")
except urllib.error.URLError as e:
raise ConnectionError(f"Failed to fetch {url}: {e}") from e
def validate_format(self, raw_data: str) -> bool:
has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data
def fetch_requirements(self) -> list[BiosRequirement]:
raw = self._fetch_raw()
if not self.validate_format(raw):
raise ValueError("batocera-systems format validation failed")
# Paths are relative to /storage/roms (`prefix`, lines 859-865): the
# list names neogeo/ and ports/ beside bios/.
return requirements_from_systems(extract_systems_dict(raw), SYSTEM_SLUG_MAP, strip="")
def generate_platform_yaml(self) -> dict:
systems = platform_systems(
self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
)
cores, standalone, dual = es_cores(
json_emulators(self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))),
RUNNERS,
RENAMED,
)
return {
"platform": "EmuELEC",
"version": self.tag.removeprefix("v"),
"homepage": "https://emuelec.org",
"source": self.url,
"base_destination": "",
# RetroArch's system_directory is /storage/roms/bios
# (packages/sx05re/libretro_base/retroarch/package.mk:117).
"core_destination": "bios",
"hash_type": "md5",
"verification_mode": "md5",
"cores": cores,
"standalone_cores": standalone,
"dual_mode_cores": dual,
"systems": systems,
}
def main() -> None:
from .base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape EmuELEC BIOS requirements")
if __name__ == "__main__":
main()
+88
View File
@@ -0,0 +1,88 @@
"""The emulators an EmulationStation system list offers, by profile names.
AmberELEC writes its list as es_systems.cfg XML, EmuELEC as an es_systems.json
it turns into that XML at build time. Both name, per system, the emulators a
user may pick and the cores under each. Some emulators are libretro front
ends (RetroArch, retrorun) whose cores are libretro core names. Every other
emulator is a standalone program named by the OS (`HATARISA`, `PPSSPPSDL`).
"""
from __future__ import annotations
import json
import re
from safeparse import parse_untrusted_xml
def xml_emulators(text: str) -> dict[str, set[str]]:
"""Emulator -> cores, from an es_systems.cfg."""
root = parse_untrusted_xml(text, "es_systems.cfg")
pairs: dict[str, set[str]] = {}
for emulator in root.iter("emulator"):
name = emulator.get("name")
if name:
pairs.setdefault(name, set()).update(
core.text.strip() for core in emulator.iter("core") if core.text
)
if not pairs:
raise ValueError("es_systems.cfg names no emulator")
return pairs
def json_emulators(text: str) -> dict[str, set[str]]:
"""Emulator -> cores, from an es_systems.json."""
pairs: dict[str, set[str]] = {}
for system in json.loads(text).get("systems", []):
for emulator in system.get("emulators") or []:
pairs.setdefault(emulator["name"], set()).update(
core["name"] for core in emulator.get("cores") or []
)
if not pairs:
raise ValueError("es_systems.json names no emulator")
return pairs
def standalone_names(emulator: str) -> list[str]:
"""The names a profile may know a standalone emulator by.
The OS's own spelling, its lowercase, and the lowercase without the
`sa` suffix the ELEC family appends to a standalone build of an emulator
that also exists as a core (`HATARISA`, `scummvmsa`, `yabasanshiroSA`).
"""
lower = emulator.lower()
names = [emulator, lower]
stripped = re.sub(r"-?sa$", "", lower)
if stripped:
names.append(stripped)
return list(dict.fromkeys(names))
def es_cores(
pairs: dict[str, set[str]],
runners: dict[str, str | None],
renamed: dict[str, str] | None = None,
) -> tuple[list[str], list[str], list[str]]:
"""(cores, standalone_cores, dual_mode_cores) of an ES system list.
`runners` maps each emulator that runs libretro cores to None when the
cores it lists are libretro core names, or to the one core it always
loads (AmberELEC's `mame` lists MAME machines and loads the MAME core).
`renamed` gives the emulator a standalone name stands for when the
suffix rule cannot (`PPSSPPSDL` is PPSSPP), each one read in the OS's
launch code.
"""
renamed = renamed or {}
libretro: set[str] = set()
for runner, fixed in runners.items():
if runner in pairs:
libretro.update([fixed] if fixed else pairs[runner])
standalone: set[str] = set()
for emulator in pairs:
if emulator in runners:
continue
standalone.update(standalone_names(emulator))
if emulator in renamed:
standalone.add(renamed[emulator])
dual = {name for name in standalone if name in libretro}
return sorted(libretro | standalone), sorted(standalone), sorted(dual)
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python3
"""Scraper for KNULLI's knulli-systems.
Source: package/system/knulli-scripts/scripts/knulli-systems
Format: Python source, Batocera's `systems` dict, parsed by batocera_scraper
Hash: MD5
`msx1` and `msx2+` are written twice in the dict. Python keeps the later
value of a repeated key when the script runs, and so does the parser.
Verification (knulli-systems at the release tag):
- checkBios (lines 867-927): md5sum() of the file at /userdata/<file>
against the declared md5, an empty md5 meaning presence only. For a
zippedFile entry, checkInsideZip() (lines 814-845) matches the member name
with casefold() and hashes it. A system or file carrying `emulator` and
`core` is checked only when es_systems.cfg offers that pair (coreExists,
lines 847-865).
- Run by EmulationStation's BIOS screen (knulli-cfw/batocera-emulationstation
10834d00, es-app/src/ApiSystem.cpp:764-774) and by knulli-support:69.
es_systems.yml and configgen-defaults.yml have Batocera's shape, so the cores
are read with Batocera's code. The configgen generators do not: the hatari,
melonds and gsplus generators and libretroOptions.py are older forms than the
ones batocera_launchers reads, so no launcher layer is written.
"""
from __future__ import annotations
import re
import urllib.error
import urllib.request
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
from .batocera_scraper import (
SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
)
from .batocera_scraper import (
_not_cores,
configgen_cores,
extract_systems_dict,
platform_systems,
requirements_from_systems,
)
PLATFORM_NAME = "knulli"
REPO = "knulli-cfw/knulli-linux"
_RAW = "https://raw.githubusercontent.com/knulli-cfw/knulli-linux/{ref}/"
SOURCE_TEMPLATE = _RAW + "package/system/knulli-scripts/scripts/knulli-systems"
CONFIGGEN_DEFAULTS_TEMPLATE = (
_RAW + "package/system/knulli-configgen/configs/configgen-defaults.yml"
)
ES_SYSTEMS_TEMPLATE = _RAW + "package/emulationstation/knulli-es-system/es_systems.yml"
# Batocera's names, plus the keys KNULLI writes that Batocera does not,
# filed under the slug the other scrapers already give them. msu-md holds the
# Mega-CD BIOS files Batocera files under megadrive-msu.
SYSTEM_SLUG_MAP = {
**BATOCERA_SLUGS,
"o2em": "magnavox-odyssey2",
"msu-md": "sega-mega-cd",
"xbox": "microsoft-xbox",
}
class Scraper(BaseScraper):
"""Scraper for KNULLI's knulli-systems."""
def __init__(self, url: str | None = None) -> None:
self.tag = "" if url else fetch_github_latest_version(REPO)
super().__init__(url=url or SOURCE_TEMPLATE.format(ref=self.tag))
def _fetch_text(self, url: str) -> str:
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
try:
with urllib.request.urlopen(req, timeout=30) as resp:
return resp.read().decode("utf-8")
except urllib.error.URLError as e:
raise ConnectionError(f"Failed to fetch {url}: {e}") from e
def validate_format(self, raw_data: str) -> bool:
has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data
def fetch_requirements(self) -> list[BiosRequirement]:
raw = self._fetch_raw()
if not self.validate_format(raw):
raise ValueError("knulli-systems format validation failed")
return requirements_from_systems(extract_systems_dict(raw), SYSTEM_SLUG_MAP)
def generate_platform_yaml(self) -> dict:
systems = platform_systems(
self.fetch_requirements(), extract_systems_dict(self._fetch_raw()), SYSTEM_SLUG_MAP
)
# es_systems.yml files machine selectors and plugins under an
# emulator as Batocera's does: Batocera's list of them applies.
cores, standalone, dual = configgen_cores(
self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag)),
self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag)),
_not_cores("batocera"),
)
result = {
"platform": "KNULLI",
"version": self.tag,
"homepage": "https://knulli.org",
"source": self.url,
"base_destination": "bios",
"hash_type": "md5",
"verification_mode": "md5",
"cores": cores,
"systems": systems,
}
if standalone:
result["standalone_cores"] = standalone
if dual:
result["dual_mode_cores"] = dual
return result
def main() -> None:
from .base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape KNULLI BIOS requirements")
if __name__ == "__main__":
main()
+208
View File
@@ -0,0 +1,208 @@
"""AmberELEC, EmuELEC and KNULLI publish Batocera's `systems` dict.
Each scraper reads its list with batocera_scraper's parser, so a dict written
with a key twice keeps the later value as Python does, paths keep the folder
the OS's checker prefixes, and the cores come from the OS's own
EmulationStation list. Trimmed copies of the three lists stand in for the
upstream files.
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
from scraper import amberelec_scraper, emuelec_scraper, knulli_scraper # noqa: E402
from scraper.es_emulators import es_cores, json_emulators, xml_emulators # noqa: E402
CHECKER = '''
def checkBios(systems, prefix):
pass
'''
AMBERELEC = '''#!/usr/bin/env python
systems = {
"atarilynx": {"name": "Lynx", "biosFiles": [{"md5": "fcd403db69f54290b51035d82f835e7b", "file": "bios/lynxboot.img"}]},
# Neo Geo
"neogeo": {"name": "NeoGeo", "biosFiles": [{"md5": "", "file": "neogeo/neogeo.zip"},
{"md5": "", "file": "neogeo/aes.zip"}]},
}
''' + CHECKER
EMUELEC = '''#!/usr/bin/env python
systems = {
"naomi": { "name": "Naomi", "biosFiles": [ { "md5": "e63d892cdb8b532351dc7020bb60b6f4", "file": "bios/dc/naomi.zip" } ] },
"fmtowns": { "name": "Fujistu FM-Towns", "biosFiles": [
{ "md5": "34847786d7de94b5d1133c956ab1d75d", "zippedFile": "fmt_dic.rom", "file": "bios/fmtowns.zip"} ] },
"Diablo": { "name": "Diablo 1", "biosFiles": [ { "md5": "", "file": "ports/diablo/diabdat.mpq" } ] },
"naomi": { "name": "Naomi", "biosFiles": [ { "md5": "3bffafac42a7767d8dcecf771f5552ba", "file": "bios/dc/naomi_boot.bin" },
{ "md5": "e63d892cdb8b532351dc7020bb60b6f4", "file": "bios/dc/naomi.zip" } ] },
}
''' + CHECKER
KNULLI = '''#!/usr/bin/env python
systems = {
"msx1": { "name": "MSX1", "biosFiles": [ { "md5": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "file": "bios/MSX.ROM" } ] },
"msu-md": { "name": "MSU-MD", "biosFiles": [ { "md5": "e66fa1dc5820d254611fdcdba0662372", "file": "bios/bios_CD_E.bin" } ] },
"msx1": { "name": "MSX1", "emulator": "libretro", "core": "bluemsx", "biosFiles": [
{ "md5": "", "file": "bios/Machines/Shared Roms/MSX.rom" } ] },
}
''' + CHECKER
AMBERELEC_ES = """<?xml version="1.0" encoding="UTF-8"?>
<systemList>
<system><name>advision</name><emulators>
<emulator name="mame"><cores><core default="true">advision</core></cores></emulator>
</emulators></system>
<system><name>atarist</name><emulators>
<emulator name="retroarch"><cores><core>hatari</core></cores></emulator>
<emulator name="HATARISA"><cores><core>HATARISA</core></cores></emulator>
</emulators></system>
<system><name>psp</name><emulators>
<emulator name="retroarch"><cores><core>ppsspp</core></cores></emulator>
<emulator name="PPSSPPSDL"><cores><core>PPSSPPSDL</core></cores></emulator>
</emulators></system>
<system><name>dreamcast</name><emulators>
<emulator name="retrorun"><cores><core>flycast</core></cores></emulator>
</emulators></system>
</systemList>
"""
EMUELEC_ES = """{"systems": [
{"name": "imageviewer", "command": "ignore"},
{"name": "saturn", "emulators": [
{"name": "libretro", "cores": [{"name": "yabasanshiro"}]},
{"name": "yabasanshiro", "cores": [{"name": "yabasanshiroSA"}]}]},
{"name": "atari2600", "emulators": [{"name": "STELLASA", "cores": [{"name": "STELLASA"}]}]}
]}"""
def _scraper(module, raw: str):
scraper = module.Scraper(url="https://example.test/list")
scraper._raw_data = raw
return scraper
class TheListsReadAsPythonReadsThem(unittest.TestCase):
def test_amberelec_keeps_paths_beside_bios(self):
reqs = _scraper(amberelec_scraper, AMBERELEC).fetch_requirements()
self.assertEqual(
[(r.system, r.destination, r.native_id) for r in reqs],
[("atari-lynx", "bios/lynxboot.img", "atarilynx"),
("snk-neogeo", "neogeo/neogeo.zip", "neogeo"),
("snk-neogeo", "neogeo/aes.zip", "neogeo")],
)
def test_emuelec_keeps_the_later_of_a_repeated_key(self):
reqs = _scraper(emuelec_scraper, EMUELEC).fetch_requirements()
naomi = [r.destination for r in reqs if r.native_id == "naomi"]
self.assertEqual(naomi, ["bios/dc/naomi_boot.bin", "bios/dc/naomi.zip"])
self.assertEqual(len(reqs), 4)
def test_emuelec_files_a_port_under_its_engine_system(self):
reqs = _scraper(emuelec_scraper, EMUELEC).fetch_requirements()
diablo = next(r for r in reqs if r.native_id == "Diablo")
self.assertEqual((diablo.system, diablo.destination),
("diablo", "ports/diablo/diabdat.mpq"))
def test_emuelec_carries_the_member_its_list_names(self):
reqs = _scraper(emuelec_scraper, EMUELEC).fetch_requirements()
towns = next(r for r in reqs if r.native_id == "fmtowns")
self.assertEqual(towns.zipped_file, "fmt_dic.rom")
def test_knulli_strips_bios_and_keeps_the_later_key(self):
reqs = _scraper(knulli_scraper, KNULLI).fetch_requirements()
self.assertEqual(
[(r.system, r.destination, r.native_path) for r in reqs],
[("microsoft-msx", "Machines/Shared Roms/MSX.rom", "bios/Machines/Shared Roms/MSX.rom"),
("sega-mega-cd", "bios_CD_E.bin", "bios/bios_CD_E.bin")],
)
def test_a_list_without_its_checker_is_refused(self):
for module, raw in ((amberelec_scraper, AMBERELEC), (emuelec_scraper, EMUELEC),
(knulli_scraper, KNULLI)):
with self.subTest(scraper=module.__name__), self.assertRaises(ValueError):
_scraper(module, raw.replace("def checkBios", "def other")).fetch_requirements()
class TheCoresComeFromTheOsList(unittest.TestCase):
def test_amberelec_runners_and_standalone_names(self):
cores, standalone, dual = es_cores(
xml_emulators(AMBERELEC_ES), amberelec_scraper.RUNNERS, amberelec_scraper.RENAMED
)
# mame lists MAME machines and loads the MAME core: advision is no core.
self.assertIn("mame", cores)
self.assertNotIn("advision", cores)
self.assertNotIn("mame", standalone)
self.assertIn("flycast", cores)
self.assertTrue({"HATARISA", "hatari", "PPSSPPSDL", "ppsspp"} <= set(standalone))
self.assertEqual(dual, ["hatari", "ppsspp"])
def test_emuelec_runners_and_standalone_names(self):
cores, standalone, dual = es_cores(
json_emulators(EMUELEC_ES), emuelec_scraper.RUNNERS, emuelec_scraper.RENAMED
)
self.assertIn("stella", standalone)
self.assertNotIn("yabasanshiroSA", cores)
self.assertEqual(dual, ["yabasanshiro"])
def test_the_newest_prerelease_names_its_commit(self):
body = "This is a prerelease based on the commit: AmberELEC/AmberELEC@" + "a" * 40 + "."
self.assertEqual(amberelec_scraper.prerelease_commit(body), "a" * 40)
with self.assertRaises(RuntimeError):
amberelec_scraper.prerelease_commit("no commit here")
class CoreFilesGoUnderTheStatedFolder(unittest.TestCase):
"""EmuELEC's ports/ entries are a quarter of its list, so the most common
root of its destinations fell under the 90% rule and core files went to
/storage/roms itself, where no core reads them."""
def test_core_destination_wins_over_the_guess(self):
from common import declared_root
files = [{"destination": f"bios/f{i}"} for i in range(3)]
files.append({"destination": "ports/diablo/diabdat.mpq"})
config = {"base_destination": "", "systems": {"s": {"files": files}}}
self.assertEqual(declared_root(config), "")
self.assertEqual(declared_root({**config, "core_destination": "bios"}), "bios")
self.assertEqual(declared_root({**config, "base_destination": "bios",
"core_destination": "x"}), "")
def test_the_platforms_state_it(self):
import yaml
for name in ("amberelec", "emuelec"):
with self.subTest(platform=name):
config = yaml.safe_load((REPO_ROOT / "platforms" / f"{name}.yml").read_text())
self.assertEqual((config["base_destination"], config["core_destination"]),
("", "bios"))
class TheExportKeepsTheirShape(unittest.TestCase):
def test_an_indented_closing_brace_is_found(self):
from exporter.emuelec_exporter import Exporter
head, body, tail = Exporter._split_original(EMUELEC)
self.assertEqual(head[-1], "systems = {")
self.assertEqual(tail[0], " }")
self.assertIn("Diablo", "\n".join(body))
def test_paths_start_where_the_list_starts(self):
from exporter.amberelec_exporter import Exporter as Amber
from exporter.baseline import NativeFile
from exporter.knulli_exporter import Exporter as Knulli
fe = NativeFile("neogeo.zip", "neogeo/neogeo.zip", "neogeo")
self.assertEqual(Amber.declared_path(fe), "neogeo/neogeo.zip")
fe = NativeFile("bios_CD_E.bin", "bios_CD_E.bin", "msu-md")
self.assertEqual(Knulli.declared_path(fe), "bios/bios_CD_E.bin")
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -586,7 +586,7 @@ class ExporterRegistry(unittest.TestCase):
def test_an_exporter_that_patches_code_says_it_needs_the_original(self):
"""Regenerating a script from data alone drops the code around it."""
for name in ("batocera", "rocknix", "emudeck", "retrodeck", "bizhawk",
"misterfpga"):
"misterfpga", "knulli", "amberelec", "emuelec"):
with self.subTest(platform=name):
self.assertTrue(discover_exporters()[name].needs_original())
+1
View File
@@ -32,6 +32,7 @@ EMULATORS = REPO_ROOT / "emulators"
PLATFORM_WORDS = (
"retroarch", "batocera", "recalbox", "retrobat", "emudeck", "lakka",
"retrodeck", "romm", "rocknix", "retropie", "bizhawk", "mister",
"knulli", "amberelec", "emuelec",
)
# Launcher sources: configgen and emulatorLauncher generators, EmuDeck and
+6
View File
@@ -558,6 +558,9 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase):
"scraper.emudeck_scraper",
"scraper.misterfpga_scraper",
"scraper.retropie_scraper",
"scraper.amberelec_scraper",
"scraper.emuelec_scraper",
"scraper.knulli_scraper",
):
module = importlib.import_module(name)
with self.subTest(scraper=name), mock.patch(
@@ -839,8 +842,11 @@ class OneNativeIdOneSystem(unittest.TestCase):
# A platform system no profile names gets no core extra and answers no
# --system. Named here: Batocera's family folds Naomi and Atomiswave into
# one system the profiles keep apart, and MiSTer cores nothing profiles.
# EmuELEC lists Duke Nukem 2's data for RigelEngine, which no profile
# describes yet.
UNMATCHED = {
"*": {"sega-dreamcast-arcade"},
"emuelec": {"RigelEngine"},
"misterfpga": {"gameking", "interact", "marty", "studio-ii", "superacan",
"videobrain"},
}