feat: scrape the rom folders of each setup

This commit is contained in:
Abdessamad Derraz committed 2026-11-03 17:37:33 +01:00
1 parent 249b189ec9
commit 1aa36258b2
102 files changed
+49969

No files matched your search

@@ -0,0 +1,3 @@
snes:
emulator: libretro
core: pocketsnes
@@ -0,0 +1 @@
configgen-defaults-bcm2835.yml
@@ -0,0 +1,9 @@
default:
emulator:
core:
snes:
emulator: libretro
core: snes9x
mame:
emulator: libretro
core: mame078plus
+25
View File
@@ -0,0 +1,25 @@
snes:
name: Super Nintendo
hardware: console
extensions: [smc, SFC, zip]
emulators:
libretro:
snes9x: { requireAnyOf: [X] }
pocketsnes: { requireAnyOf: [Y] }
mame:
name: MAME
hardware: arcade
extensions: [zip, 7z]
emulators:
libretro:
mame078plus: { requireAnyOf: [Z] }
nogame:
name: Has no emulator
extensions: [txt]
elsewhere:
name: Outside the roms root
path: /userdata/library
extensions: [lnk]
emulators:
flatpak:
flatpak: { requireAnyOf: [W] }
@@ -0,0 +1,4 @@
SYSTEM_NAME="3do"
SYSTEM_HARDWARE="console"
SYSTEM_PATH="/storage/roms/3do"
SYSTEM_EXTENSION=".iso .bin .chd"
@@ -0,0 +1,4 @@
SYSTEM_NAME="psx"
SYSTEM_HARDWARE="console"
SYSTEM_PATH="/storage/roms/psx"
SYSTEM_EXTENSION=".cue .CUE .chd"
@@ -0,0 +1,3 @@
SYSTEM_NAME="tools"
SYSTEM_PATH="/storage/.config/modules"
SYSTEM_EXTENSION=".sh"
+23
View File
@@ -0,0 +1,23 @@
PKG_NAME="emulators"
makeinstall_target() {
add_system_dir /storage/roms/bios
add_emu_core 3do retroarch opera true
add_es_system 3do
case ${DEVICE} in
RK3326)
add_emu_core psx retroarch pcsx_rearmed true
add_emu_core psx retroarch swanstation false
;;
*)
add_emu_core psx retroarch swanstation true
add_emu_core psx retroarch pcsx_rearmed false
;;
esac
add_es_system psx
add_emu_core tools ignore ignore true
add_es_system tools
}
+9
View File
@@ -0,0 +1,9 @@
<?xml version="1.0"?>
<systemList>
<system>
<name>model2</name>
<path>%ROMPATH%/model2/roms</path>
<extension>.zip .ZIP</extension>
<command label="Model 2 (Standalone)">%EMULATOR_MODEL-2% %BASENAME% & exit</command>
</system>
</systemList>
+16
View File
@@ -0,0 +1,16 @@
<?xml version="1.0"?>
<systemList>
<system>
<name>psx</name>
<path>%ROMPATH%/psx</path>
<extension>.cue .CUE .chd .CHD</extension>
<command label="Beetle PSX">%EMULATOR_RETROARCH% -L %CORE_RETROARCH%/mednafen_psx_libretro.so %ROM%</command>
<command label="DuckStation (Standalone)">%EMULATOR_DUCKSTATION% -batch %ROM%</command>
</system>
<system>
<name>model2</name>
<path>%ROMPATH%/model2</path>
<extension>.zip</extension>
<command label="MAME">%EMULATOR_RETROARCH% -L %CORE_RETROARCH%/mame_libretro.so %ROM%</command>
</system>
</systemList>
+5
View File
@@ -0,0 +1,5 @@
ESDE_setDefaultEmulators(){
mkdir -p "$gamelists"
ESDE_setEmu 'DuckStation (Standalone)' psx
ESDE_setEmu 'Missing Label' model2
}
+31
View File
@@ -0,0 +1,31 @@
{
"systems": [
{
"name": "psx",
"path": "/storage/roms/psx",
"extensions": [".cue", ".CUE"],
"emulators": [
{"name": "libretro", "cores": [{"name": "pcsx_rearmed"}, {"name": "duckstation", "default": true}]}
]
},
{
"name": "imageviewer",
"path": "/storage/roms/screenshots",
"platform": "ignore",
"command": "ignore",
"extensions": [".png"]
},
{
"name": "pico8",
"path": "/storage/roms/pico-8",
"extensions": [".p8"],
"emulators": [{"name": "Pico-8", "cores": [{"name": "Pico-8", "default": true}]}]
},
{
"name": "pico8",
"path": "/storage/roms/pico",
"extensions": [".md"],
"emulators": [{"name": "libretro", "cores": [{"name": "genesis_plus_gx", "default": true}]}]
}
]
}
@@ -0,0 +1,12 @@
[system]
name = "mame"
roms.folder = "%ROOT%/mame"
[properties]
type = arcade
[core.0]
priority = 1
emulator = "libretro"
core = "mame2003_plus"
extensions = ".zip!"
@@ -0,0 +1,25 @@
[system]
name = "psx"
roms.folder = "%ROOT%/psx"
[properties]
type = console
[core.0]
priority = 1
emulator = "libretro"
core = "pcsx_rearmed"
extensions = ".cue .chd"
[core.1]
priority = 1
emulator = "libretro"
core = "swanstation"
extensions = ".cue .pbp"
[core.2]
priority = 2
priority.rpi1 = 0
emulator = "libretro"
core = "mednafen_psx"
extensions = ".cue"
+8
View File
@@ -0,0 +1,8 @@
psx_exts=".cue .chd"
psx_fullname="PlayStation"
nes_exts=".nes .zip"
nes_fullname="Nintendo Entertainment System"
vectrex_exts=".vec"
vectrex_fullname="Vectrex"
@@ -0,0 +1,7 @@
rp_module_id="pcsx2"
rp_module_section="exp"
rp_module_flags="!all x86"
function configure_pcsx2() {
addEmulator 1 "$md_id" "psx" "pcsx2 %ROM%"
}
@@ -0,0 +1,9 @@
rp_module_id="lr-fceumm"
rp_module_section="main"
function configure_lr-fceumm() {
local system
for system in nes fds; do
addEmulator 1 "$md_id" "$system" "$md_inst/fceumm_libretro.so"
done
}
@@ -0,0 +1,7 @@
rp_module_id="lr-pcsx-rearmed"
rp_module_section="main"
rp_module_flags="!x86"
function configure_lr-pcsx-rearmed() {
addEmulator 1 "$md_id" "psx" "$md_inst/pcsx_rearmed_libretro.so"
}
@@ -0,0 +1,6 @@
rp_module_id="lr-swanstation"
rp_module_section="opt"
function configure_lr-swanstation() {
addEmulator 1 "$md_id" "psx" "$md_inst/swanstation_libretro.so"
}
+47
View File
@@ -0,0 +1,47 @@
<?xml version="1.0"?>
<systemList>
<system>
<name>psx</name>
<path>/storage/roms/psx</path>
<extension>.cue .CUE .chd</extension>
<command>runemu.py %ROM%</command>
<platform>psx</platform>
<emulators>
<emulator name="retroarch">
<cores>
<core>pcsx_rearmed</core>
<core default="true">duckstation</core>
</cores>
</emulator>
</emulators>
</system>
<!-- <system><name>commented</name><path>/storage/roms/commented</path> --no-gui </system> -->
<system>
<name>advision</name>
<path>/storage/roms/advision/</path>
<extension>.bin .zip</extension>
<command>retroarch -L /cores/mess_libretro.so %ROM% 2>&1 > /dev/null</command>
<platform>advision</platform>
</system>
<system>
<name>tools</name>
<path>/storage/.config/modules</path>
<extension>.sh</extension>
<command>ignore</command>
<platform>ignore</platform>
</system>
<system>
<name>mame</name>
<path>/storage/roms/mame</path>
<extension>.zip</extension>
<command>runemu.py %ROM%</command>
<platform>arcade</platform>
<emulators>
<emulator name="retroarch">
<cores>
<core>mame2003_plus</core>
</cores>
</emulator>
</emulators>
</system>
</systemList>
+47
View File
@@ -0,0 +1,47 @@
<?xml version="1.0"?>
<systemList>
<system>
<name>psx</name>
<path>/storage/roms/psx</path>
<extension>.cue .CUE .chd</extension>
<command>runemu.py %ROM%</command>
<platform>psx</platform>
<emulators>
<emulator name="retroarch">
<cores>
<core>pcsx_rearmed</core>
<core default="true">swanstation</core>
</cores>
</emulator>
</emulators>
</system>
<!-- <system><name>commented</name><path>/storage/roms/commented</path> --no-gui </system> -->
<system>
<name>advision</name>
<path>/storage/roms/advision/</path>
<extension>.bin .zip</extension>
<command>retroarch -L /cores/mess_libretro.so %ROM% 2>&1 > /dev/null</command>
<platform>advision</platform>
</system>
<system>
<name>tools</name>
<path>/storage/.config/modules</path>
<extension>.sh</extension>
<command>ignore</command>
<platform>ignore</platform>
</system>
<system>
<name>mame</name>
<path>/storage/roms/mame</path>
<extension>.zip</extension>
<command>runemu.py %ROM%</command>
<platform>arcade</platform>
<emulators>
<emulator name="retroarch">
<cores>
<core>mame2003_plus</core>
</cores>
</emulator>
</emulators>
</system>
</systemList>
+64
View File
@@ -0,0 +1,64 @@
{
"onion": {
"static/packages/Emu/Nintendo - GBA (gpSP)/Emu/GBA/config.json": "{\n \"label\": \"GBA\",\n \"launch\": \"launch.sh\",\n \"rompath\": \"../../Roms/GBA\",\n \"extlist\": \"gba|GBA|zip|7z\"\n}\n",
"static/packages/Emu/Nintendo - GBA (gpSP)/Emu/GBA/launch.sh": "#!/bin/sh\ndefault_core=gpsp\ncore=\"$default_core\"\nretroarch -L .retroarch/cores/${default_core}_libretro.so \"$1\"\n",
"static/packages/RApp/Nintendo - GBA (VBA-M)/RApp/GBA_VBAM/config.json": "{\n \"label\": \"VBA-M\",\n \"rompath\": \"../../Roms/GBA\",\n \"extlist\": \"gba\"\n}\n",
"static/packages/RApp/Nintendo - GBA (VBA-M)/RApp/GBA_VBAM/launch.sh": "retroarch -L .retroarch/cores/vbam_libretro.so \"$1\"\n",
"static/packages/RApp/NEC - PC-98 (Neko Project II)/RApp/NP2/config.json": "{\n \"rompath\": \"/mnt/SDCARD/Roms/PCNINETYEIGHT\",\n \"extlist\": \"d98|hdi\"\n}\n",
"static/packages/RApp/NEC - PC-98 (Neko Project II)/RApp/NP2/launch.sh": "retroarch -L .retroarch/cores/nekop2_libretro.so \"$1\"\n",
"static/packages/App/Clock/App/Clock/config.json": "{\"label\": \"Clock\"}\n"
},
"spruce": {
"Emu/PS/config.json": "{\n \"label\": \"PlayStation\",\n \"alternativeFolderNames\": [\"psx\", \"PS1\"],\n \"extlist\": \"bin|cue|CHD\",\n \"default_emulator\": \"pcsx_rearmed\",\n \"ignoreList\": [\"neogeo.zip\"]\n}\n",
"Emu/-CUSTOM-SYSTEM-/config.json": "{\"label\": \"template\"}\n",
"Emu/PS/launch.sh": "echo\n"
},
"crossmix": {
"Emus/PS/config.json": "{\n \"rompath\": \"../../Roms/PS\",\n \"extlist\": \"cue|chd\",\n \"launchlist\": [\n {\"name\": \"--- libretro ---\", \"launch\": \"\"},\n {\"name\": \"PCSX ReARMed\", \"launch\": \"pcsx.sh\"}\n ]\n}\n",
"Emus/PS/pcsx.sh": "RA_CORE=\"$HOME/cores/pcsx_rearmed_libretro.so\"\n",
"Emus/_BootLogo/config.json": "{\"rompath\": \"../../Apps/x\"}\n"
},
"koriki": {
"base/.simplemenu/section_groups/apps and games.ini": "[CONSOLES]\nexecs = none\n\n[PS]\nexecs = /mnt/SDCARD/.simplemenu/launchers/pcsx_libretro.sh,/mnt/SDCARD/.simplemenu/launchers/pcsx_standalone.sh\nromDirs = /mnt/SDCARD/Roms/PS/\nromExts = .cue,.CHD,.pbp\n",
"base/.simplemenu/launchers/pcsx_libretro.sh": "retroarch -L /mnt/SDCARD/RetroArch/.retroarch/cores/pcsx_rearmed_libretro.so \"$1\"\n"
},
"allium": {
"consoles.toml": "[[consoles]]\nname = \"Arcade\"\ncores = [\n \"mame2003_plus\",\n \"fbneo\",\n]\npatterns = [\"ARCADE\", \"mame\"]\n\n[[consoles]]\nname = \"Game Boy Advance\"\ncores = [\"gpsp\"]\npatterns = [\"GBA\"]\nextensions = [\"gba\"]\n\n[[consoles]]\nname = \"Native\"\ncores = [\"native\"]\nextensions = [\"pak\"]\n",
"cores.toml": "[cores.native]\npath = \"/mnt/SDCARD/.allium/cores/pak/launch.sh\"\nname = \"Native\"\n\n[cores.\"dosbox_pure_0.9.7\"]\nretroarch = \"dosbox_pure_0.9.7\"\n\n[cores.mame2003_plus]\nretroarch = \"mame2003_plus\"\nname = \"MAME 2003+\"\n\n[cores.gpsp]\nretroarch = \"gpsp\"\n"
},
"minui": {
"skeleton/BASE/Roms/Sony PlayStation (PS)/.keep": "",
"skeleton/SYSTEM/rg35xx/paks/Emus/PS.pak/launch.sh": "#!/bin/sh\nEMU_EXE=pcsx_rearmed\nminarch.elf \"$CORES_PATH/${EMU_EXE}_libretro.so\" \"$1\"\n",
"skeleton/SYSTEM/miyoomini/paks/Emus/PS.pak/launch.sh": "#!/bin/sh\nEMU_EXE=pcsx_rearmed\nminarch.elf \"$CORES_PATH/${EMU_EXE}_libretro.so\" \"$1\"\n",
"skeleton/SYSTEM/tg5040/paks/Emus/PS.pak/launch.sh": "#!/bin/sh\nEMU_EXE=swanstation\nminarch.elf \"$CORES_PATH/${EMU_EXE}_libretro.so\" \"$1\"\n",
"skeleton/EXTRAS/Emus/miyoomini/A2600.pak/launch.sh": "#!/bin/sh\nEMU_EXE=stella2014\nminarch.elf \"$CORES_PATH/${EMU_EXE}_libretro.so\" \"$1\"\n"
},
"plumos": {
"systems.json": "{\n \"systems\": [\n {\n \"id\": \"nes\",\n \"enabled\": true,\n \"directory_aliases\": [\n {\"name\": \"nes\", \"priority\": 20},\n {\"name\": \"FC\", \"priority\": 10, \"shared\": true}\n ],\n \"extensions\": [\"nes\", \"zip\"],\n \"default_launch_profile\": \"retroarch:quicknes\"\n },\n {\n \"id\": \"fds\",\n \"enabled\": false,\n \"directory_aliases\": [\n {\"name\": \"FC\", \"priority\": 10, \"shared\": true},\n {\"name\": \"fds\", \"priority\": 20}\n ],\n \"extensions\": [\"fds\"],\n \"default_launch_profile\": \"standalone:fceux\"\n }\n ]\n}\n"
},
"muos": {
"core.json": "{\n \"Sony PlayStation\": {\n \"catalogue\": \"Sony PlayStation\",\n \"default\": \"psx rearmed\",\n \"friendly\": [\"ps\", \"psx\", \"sonyplaystation\"],\n \"libretro\": {\"psx rearmed\": {\"core\": \"pcsx_rearmed_libretro.so\"}}\n },\n \"Nintendo DS\": {\n \"catalogue\": \"Nintendo DS\",\n \"default\": \"drastic\",\n \"friendly\": [\"nds\", \"nintendods\"],\n \"libretro\": {}\n }\n}\n"
},
"gmenu2x": {
"board/miyoo/roms/GB/a.gb": "",
"board/miyoo/roms/GBA/b.gba": "",
"board/miyoo/roms/autorun.inf": ""
},
"gpsp": {
"miyoo/opkg_assets/gpsp.lnk": "title=gpSP\nexec=/mnt/emus/gpsp/gpsp\nselectordir=/mnt/roms/GBA\nselectorfilter=.gba,.zip\n"
},
"dist": {
"games/SNES/boot.rom": "",
"games/mame/.keep": "",
"games/readme.txt": ""
},
"romm": {
"index.ts": "const _EJS_CORES_MAP: Record<string, string[]> = {\n \"3do\": [\"opera\"],\n arcade: [\n \"mame2003_plus\",\n \"fbneo\",\n ],\n};\n\nconst _EJS_NIGHTLY_CORES_MAP: Record<string, string[]> = {\n \"3ds\": [\"azahar\"],\n};\n",
"known_bios_files.json": "{\"3do:panafz1.bin\": {}, \"64dd:IPL.n64\": {}}\n"
},
"bizhawk": {
"PathEntryCollection.cs": "\t\tprivate static readonly string COMBINED_SYSIDS_GB = string.Join(\"_\", VSystemID.Raw.GB, VSystemID.Raw.GBC, VSystemID.Raw.SGB);\n\t\t\tCommonEntriesFor(VSystemID.Raw.Sega32X, basePath: Path.Combine(\".\", \"32X\")),\n\t\t\tCommonEntriesFor(COMBINED_SYSIDS_GB, basePath: Path.Combine(\".\", \"Gameboy\")),\n\t\t\tCommonEntriesFor(VSystemID.Raw.NES, basePath: Path.Combine(\".\", \"NES\")),\n",
"RomLoader.cs": "\t\t\tpublic static readonly IReadOnlyCollection<string> GB = new[] { \"gb\", \"gbc\", \"sgb\" };\n\t\t\tpublic static readonly IReadOnlyCollection<string> NES = new[] { \"nes\", \"fds\", /*\"x\"*/ \"unf\" };\n",
"Config.cs": "\t\tpublic static readonly IReadOnlyList<(string[] AppliesTo, string[] CoreNames)> CorePickerUIData = new List<(string[], string[])>\n\t\t{\n\t\t\t([ VSystemID.Raw.GB, VSystemID.Raw.GBC ],\n\t\t\t\t[ CoreNames.Gambatte, CoreNames.Sameboy ]),\n\t\t\t([ VSystemID.Raw.NES ],\n\t\t\t\t[ CoreNames.QuickNes, CoreNames.NesHawk ]),\n\t\t};\n"
}
}
+291
View File
@@ -0,0 +1,291 @@
"""Rom folder parsers of the card OSes and the menu-less frontends, offline.
The parsers run on fixture text through a fake Fetcher, so nothing here
reaches the network. The committed setup files are checked against the
schema and their own declared sources.
"""
from __future__ import annotations
import json
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
from romfolders import split_refs # noqa: E402
from safeparse import yaml_load # noqa: E402
from scraper.romfolders import SourceError, load_sources, scrape_setup # noqa: E402
from scraper.romfolders.__main__ import dump_setup # noqa: E402
from scraper.romfolders.card import ( # noqa: E402
libretro_core,
parse_allium,
parse_bizhawk,
parse_crossmix,
parse_gmenu2x,
parse_koriki,
parse_minui,
parse_mister,
parse_muos,
parse_onion,
parse_plumos,
parse_romm,
parse_spruce,
parse_toml_subset,
ref,
)
FIXTURES = json.loads(
(ROOT / "tests" / "fixtures" / "romfolders_card" / "files.json").read_text(encoding="utf-8")
)
SETUPS = (
"onion", "allium", "spruceos", "miyoocfw", "koriki", "crossmix-os", "minui",
"myminui", "nextui", "plumos-pixel2", "plumos-mf", "muos", "misterfpga", "romm",
"bizhawk", "lakka", "retroarch",
)
class FakeFetcher:
"""Serves fixture files by source key, the Fetcher interface parsers use."""
def __init__(self, files: dict[str, dict[str, str]]) -> None:
self.files = files
def text(self, key: str, path: str) -> str:
try:
return self.files[key][path]
except KeyError as exc:
raise SourceError(f"{key}:{path} does not exist") from exc
def optional_text(self, key: str, path: str) -> str | None:
return self.files.get(key, {}).get(path)
def tree(self, key: str, prefix: str = "") -> list[str]:
return sorted(p for p in self.files.get(key, {}) if p.startswith(prefix))
def spec(source: str, **params: object) -> dict:
return {"params": {"source": source, **params}}
def by_folder(rows: list[dict]) -> dict[str, dict]:
return {row["folder"]: row for row in rows}
class LaunchScriptTests(unittest.TestCase):
def test_core_named_through_a_shell_variable(self) -> None:
script = "default_core=gpsp\nretroarch -L cores/${default_core}_libretro.so\n"
self.assertEqual(libretro_core(script), "gpsp")
def test_core_named_by_emu_exe(self) -> None:
self.assertEqual(libretro_core("EMU_EXE=fceumm\nminarch.elf x\n"), "fceumm")
def test_standalone_script_names_no_core(self) -> None:
self.assertIsNone(libretro_core("./drastic \"$1\"\n"))
def test_space_in_a_cited_path_is_escaped(self) -> None:
part = ref("os", "Emu/Commodore - Amiga/config.json", 3)
self.assertEqual(split_refs(part), [("os", "Emu/Commodore%20-%20Amiga/config.json", "3")])
class ParserTests(unittest.TestCase):
def setUp(self) -> None:
self.fetcher = FakeFetcher(FIXTURES)
def test_onion_emu_package_is_the_default_over_retroarch_section(self) -> None:
rows = by_folder(parse_onion({"params": {"source": "onion"}}, self.fetcher))
self.assertEqual(sorted(rows), ["GBA", "PCNINETYEIGHT"])
self.assertEqual(rows["GBA"]["core"], "gpsp")
self.assertEqual(rows["GBA"]["extensions"], [".gba", ".zip", ".7z"])
self.assertIn("Nintendo - GBA (VBA-M)", rows["GBA"]["note"])
self.assertNotIn("core", rows["PCNINETYEIGHT"])
def test_spruce_reads_default_and_alternative_folders(self) -> None:
rows = by_folder(parse_spruce(spec("spruce", skip=["-CUSTOM-SYSTEM-"]), self.fetcher))
self.assertEqual(list(rows), ["PS"])
self.assertEqual(rows["PS"]["aliases"], ["psx", "PS1"])
self.assertEqual(rows["PS"]["core"], "pcsx_rearmed")
self.assertEqual(rows["PS"]["extensions"], [".bin", ".cue", ".chd"])
self.assertIn("neogeo.zip", rows["PS"]["note"])
def test_crossmix_skips_separators_and_underscore_folders(self) -> None:
rows = by_folder(parse_crossmix(spec("crossmix"), self.fetcher))
self.assertEqual(list(rows), ["PS"])
self.assertEqual(rows["PS"]["core"], "pcsx_rearmed")
def test_koriki_first_launcher_wins(self) -> None:
params = {"source": "koriki", "sections": "base/.simplemenu/section_groups/apps and games.ini",
"launchers": "base/.simplemenu/launchers", "roms_prefix": "/mnt/SDCARD/Roms/",
"skip": ["CONSOLES"]}
rows = by_folder(parse_koriki({"params": params}, self.fetcher))
self.assertEqual(rows["PS"]["core"], "pcsx_rearmed")
self.assertEqual(rows["PS"]["extensions"], [".cue", ".chd", ".pbp"])
self.assertEqual(rows["PS"]["source_ref"],
"koriki base/.simplemenu/section_groups/apps%20and%20games.ini:4")
def test_allium_folder_is_the_first_pattern(self) -> None:
rows = by_folder(parse_allium(spec("allium", consoles="consoles.toml", cores="cores.toml"),
self.fetcher))
self.assertEqual(sorted(rows), ["ARCADE", "GBA"])
self.assertEqual(rows["ARCADE"]["core"], "mame2003_plus")
self.assertEqual(rows["ARCADE"]["aliases"], ["mame"])
self.assertEqual(rows["ARCADE"]["source_ref"], "allium consoles.toml:1")
def test_toml_subset_matches_tomllib(self) -> None:
try:
import tomllib
except ImportError:
self.skipTest("tomllib needs Python 3.11")
def clean(node: object) -> object:
if isinstance(node, dict):
return {k: clean(v) for k, v in node.items() if k != "__line__"}
if isinstance(node, list):
return [clean(v) for v in node]
return node
for name in ("consoles.toml", "cores.toml"):
text = FIXTURES["allium"][name]
self.assertEqual(clean(parse_toml_subset(text)), tomllib.loads(text))
def test_toml_outside_the_subset_is_refused(self) -> None:
with self.assertRaises(SourceError):
parse_toml_subset("when = 1979-05-27T07:32:00Z\n")
def test_minui_takes_the_core_most_devices_run(self) -> None:
rows = by_folder(parse_minui(spec("minui"), self.fetcher))
ps = rows["Sony PlayStation (PS)"]
self.assertEqual(ps["name"], "PS")
self.assertEqual(ps["core"], "pcsx_rearmed")
self.assertEqual(ps["board_defaults"]["tg5040"]["core"], "swanstation")
self.assertNotIn("rg35xx", ps["board_defaults"])
self.assertEqual(rows["A2600"]["core"], "stella2014")
self.assertNotIn("extensions", ps)
def test_plumos_shared_folder_goes_to_the_first_system(self) -> None:
rows = by_folder(parse_plumos(spec("plumos", path="systems.json"), self.fetcher))
self.assertEqual(rows["FC"]["name"], "nes")
self.assertEqual(rows["FC"]["core"], "quicknes")
self.assertEqual(rows["fds"]["emulator"], "standalone")
self.assertIn("FC", rows["fds"]["aliases"])
self.assertIn("disabled by default", rows["fds"]["note"])
def test_muos_folder_is_the_friendly_name_of_the_catalogue(self) -> None:
rows = by_folder(parse_muos(spec("muos", path="core.json"), self.fetcher))
self.assertEqual(rows["sonyplaystation"]["core"], "pcsx_rearmed")
self.assertEqual(rows["sonyplaystation"]["emulator"], "retroarch")
self.assertEqual(rows["nintendods"]["core"], "drastic")
self.assertEqual(rows["nintendods"]["emulator"], "external")
def test_gmenu2x_link_and_shipped_folders(self) -> None:
params = {"source": "gmenu2x", "shipped_roms": "board/miyoo/roms",
"link_roms_prefix": "/mnt/roms",
"links": [{"source": "gpsp", "path": "miyoo/opkg_assets/gpsp.lnk", "core": "gpsp"}]}
rows = by_folder(parse_gmenu2x({"params": params}, self.fetcher))
self.assertEqual(sorted(rows), ["GB", "GBA"])
self.assertEqual(rows["GBA"]["extensions"], [".gba", ".zip"])
self.assertEqual(rows["GBA"]["source_ref"], "gpsp miyoo/opkg_assets/gpsp.lnk:3")
self.assertNotIn("core", rows["GB"])
def test_mister_rows_are_the_shipped_core_folders(self) -> None:
rows = by_folder(parse_mister(spec("dist", arcade=["mame"], arcade_note="mra"), self.fetcher))
self.assertEqual(sorted(rows), ["SNES", "mame"])
self.assertTrue(rows["mame"]["arcade"])
self.assertIsNone(rows["SNES"]["arcade"])
def test_romm_reads_multiline_arrays_and_bios_only_slugs(self) -> None:
rows = by_folder(parse_romm(spec("romm", cores_map="index.ts",
bios_fixture="known_bios_files.json"), self.fetcher))
self.assertEqual(sorted(rows), ["3do", "3ds", "64dd", "arcade"])
self.assertEqual(rows["arcade"]["core"], "mame2003_plus")
self.assertIn("nightly", rows["3ds"]["note"])
self.assertNotIn("core", rows["64dd"])
def test_bizhawk_combined_systems_and_extension_lists(self) -> None:
rows = by_folder(parse_bizhawk(spec("bizhawk", paths="PathEntryCollection.cs",
rom_loader="RomLoader.cs", config="Config.cs"),
self.fetcher))
self.assertEqual(rows["Gameboy"]["name"], "GB")
self.assertEqual(rows["Gameboy"]["aliases"], ["GBC", "SGB"])
self.assertEqual(rows["Gameboy"]["extensions"], [".gb", ".gbc", ".sgb"])
self.assertEqual(rows["NES"]["extensions"], [".nes", ".fds", ".unf"])
self.assertIn("QuickNes", rows["NES"]["note"])
self.assertEqual(rows["32X"]["extensions"], [])
self.assertNotIn("core", rows["NES"])
class ScrapeTests(unittest.TestCase):
def test_two_scrapes_write_the_same_bytes(self) -> None:
setup = {
"name": "Fixture", "platform": "retroarch", "format": "plumos",
"params": {"source": "plumos", "path": "systems.json"},
"sources": {"plumos": {"repo": "https://github.com/a/b", "commit": "0" * 40}},
"roms_root": {"path": "/r", "from_bios": "../r", "source_ref": "plumos systems.json:1"},
"case_sensitive": None,
}
first = dump_setup(scrape_setup("fixture", setup, FakeFetcher(FIXTURES)))
second = dump_setup(scrape_setup("fixture", setup, FakeFetcher(FIXTURES)))
self.assertEqual(first, second)
self.assertEqual(yaml_load(first)["folders"][0]["folder"], "FC")
def test_an_empty_read_fails_unless_allowed(self) -> None:
setup = {"format": "mister", "params": {"source": "none", "arcade_note": "x"}}
with self.assertRaises(SourceError):
scrape_setup("empty", setup, FakeFetcher({}))
class CommittedSetupTests(unittest.TestCase):
def setUp(self) -> None:
self.sources = load_sources(ROOT / "platforms" / "romfolders" / "_sources")
def documents(self) -> dict[str, dict]:
out = {}
for setup in SETUPS:
path = ROOT / "platforms" / "romfolders" / f"{setup}.yml"
with path.open(encoding="utf-8") as handle:
out[setup] = yaml_load(handle)
return out
def test_every_owned_setup_has_sources_and_output(self) -> None:
for setup in SETUPS:
self.assertIn(setup, self.sources, setup)
for setup, doc in self.documents().items():
self.assertEqual(doc["setup"], setup)
self.assertEqual(doc["platform"], self.sources[setup]["platform"])
def test_every_ref_names_a_declared_source(self) -> None:
for setup, doc in self.documents().items():
declared = set(doc["sources"])
refs = [doc["roms_root"]["source_ref"]]
refs += [doc["case_ref"]] if doc.get("case_ref") else []
for block in ("skip", "hidden", "media"):
refs += [r["source_ref"] for r in doc.get(block, [])]
refs += [row["source_ref"] for row in doc["folders"]]
refs += [b["source_ref"] for row in doc["folders"]
for b in (row.get("board_defaults") or {}).values()]
for source_ref in refs:
for key, _, _ in split_refs(source_ref):
self.assertIn(key, declared, f"{setup}: {source_ref}")
def test_committed_setups_validate_against_the_schema(self) -> None:
try:
from jsonschema import Draft202012Validator
except ImportError:
self.skipTest("jsonschema not installed")
schema = json.loads((ROOT / "schemas" / "romfolders.schema.json").read_text(encoding="utf-8"))
validator = Draft202012Validator(schema)
for setup, doc in self.documents().items():
errors = [e.message for e in validator.iter_errors(doc)]
self.assertEqual(errors, [], setup)
def test_rows_with_folders_are_not_empty(self) -> None:
for setup, doc in self.documents().items():
if self.sources[setup].get("allow_empty"):
self.assertEqual(doc["folders"], [], setup)
else:
self.assertGreater(len(doc["folders"]), 0, setup)
if __name__ == "__main__":
unittest.main()
+330
View File
@@ -0,0 +1,330 @@
"""EmulationStation-family rom folder parsers, on fixtures and on the
committed setup files.
The parsers run offline against tests/fixtures/romfolders/es/ through a
fetcher that reads files instead of a forge. The committed files under
platforms/romfolders/ are checked for shape: schema, every source_ref
naming a declared source, and a second dump giving the same bytes.
"""
from __future__ import annotations
import json
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
from romfolders import setup_paths, split_refs
from scraper.romfolders import (
Fetcher,
SourceError,
load_formats,
load_sources,
scrape_setup,
)
from scraper.romfolders.__main__ import dump_setup
from scraper.romfolders.es import _module_valid
FIXTURES = ROOT / "tests" / "fixtures" / "romfolders" / "es"
ROMFOLDERS = ROOT / "platforms" / "romfolders"
RECALBOX_PENDING = (
"Recalbox source not fetched yet: run python -m scripts.scraper.romfolders --setup recalbox"
)
ES_SETUPS = (
"batocera", "knulli", "rocknix", "jelos", "kolos", "unofficialos", "archr",
"amberelec", "chaos", "pan4elec", "emuelec", "recalbox", "retrobat", "arkos",
"darkos", "retropie", "emudeck", "emudeck-windows", "retrodeck",
)
class FixtureFetcher(Fetcher):
"""Reads `<source key>/<path>` under one fixture directory."""
def __init__(self, base: Path) -> None:
super().__init__({"os": {"repo": "https://github.com/x/y", "commit": "0" * 40}})
self.base = base
def optional_text(self, key: str, path: str) -> str | None:
target = self.base / path
return target.read_text(encoding="utf-8") if target.is_file() else None
def text(self, key: str, path: str) -> str:
found = self.optional_text(key, path)
if found is None:
raise SourceError(f"{path} does not exist")
return found
def tree(self, key: str, prefix: str = "") -> list[str]:
return sorted(
p.relative_to(self.base).as_posix() for p in self.base.rglob("*")
if p.is_file() and p.relative_to(self.base).as_posix().startswith(prefix)
)
def entries(self, key: str, directory: str) -> list[str]:
return sorted(p.relative_to(self.base).as_posix() for p in (self.base / directory).iterdir())
def scrape(fmt: str, base: str, params: dict) -> dict:
spec = {"name": "Fixture", "platform": "", "format": fmt, "params": params}
return scrape_setup("fixture", spec, FixtureFetcher(FIXTURES / base))
def by_folder(doc: dict) -> dict[str, dict]:
return {row["folder"]: row for row in doc["folders"]}
class BatoceraFormat(unittest.TestCase):
def setUp(self) -> None:
self.doc = scrape("batocera-es", "batocera", {
"es_systems": "es_systems.yml", "configgen_dir": "configs"})
self.rows = by_folder(self.doc)
def test_systems_without_emulator_or_outside_root_are_left_out(self) -> None:
self.assertEqual(sorted(self.rows), ["mame", "snes"])
def test_extensions_lowercased_with_dot(self) -> None:
self.assertEqual(self.rows["snes"]["extensions"], [".sfc", ".smc", ".zip"])
def test_default_and_board_override(self) -> None:
snes = self.rows["snes"]
self.assertEqual((snes["core"], snes["emulator"]), ("snes9x", "libretro"))
self.assertEqual(snes["board_defaults"]["bcm2835"]["core"], "pocketsnes")
# A symlinked board file is followed to its target.
self.assertEqual(snes["board_defaults"]["bcm2836"]["core"], "pocketsnes")
self.assertIn("configgen-defaults-bcm2835.yml",
snes["board_defaults"]["bcm2836"]["source_ref"])
def test_arcade_flag_from_hardware(self) -> None:
self.assertTrue(self.rows["mame"]["arcade"])
self.assertNotIn("arcade", self.rows["snes"])
def test_source_ref_names_the_lines(self) -> None:
self.assertEqual(self.rows["snes"]["source_ref"],
"os es_systems.yml:1; os configs/configgen-defaults.yml:4")
class EsXmlFormat(unittest.TestCase):
def setUp(self) -> None:
self.params = {"file": "es_systems.cfg", "path_prefixes": ["/storage/roms/"], "default": "flagged"}
self.rows = by_folder(scrape("es-xml", "xml", self.params))
def test_ignored_commented_and_outside_systems_are_left_out(self) -> None:
self.assertEqual(sorted(self.rows), ["advision", "mame", "psx"])
def test_flagged_default_wins(self) -> None:
self.assertEqual(self.rows["psx"]["core"], "swanstation")
def test_first_rule_takes_the_first_core(self) -> None:
rows = by_folder(scrape("es-xml", "xml", {**self.params, "default": "first"}))
self.assertEqual(rows["psx"]["core"], "pcsx_rearmed")
def test_core_read_from_command_when_no_emulator_is_listed(self) -> None:
self.assertEqual((self.rows["advision"]["core"], self.rows["advision"]["emulator"]),
("mess", "retroarch"))
def test_arcade_flag_from_platform(self) -> None:
self.assertTrue(self.rows["mame"]["arcade"])
def test_board_file_gives_board_default(self) -> None:
# es_systems.board.cfg flags duckstation where the main file flags swanstation.
rows = by_folder(scrape("es-xml", "xml", {**self.params, "boards": {"rk3326": "es_systems.board.cfg"}}))
self.assertEqual(rows["psx"]["board_defaults"]["rk3326"]["core"], "duckstation")
self.assertNotIn("board_defaults", rows["mame"])
def test_source_ref_points_at_the_name_line(self) -> None:
self.assertEqual(self.rows["psx"]["source_ref"], "os es_systems.cfg:4")
self.assertEqual(self.rows["advision"]["source_ref"], "os es_systems.cfg:20")
class EsJsonFormat(unittest.TestCase):
def test_flagged_default_and_ignored_system(self) -> None:
rows = by_folder(scrape("es-json", "json", {
"file": "es_systems.json", "path_prefixes": ["/storage/roms/"]}))
self.assertEqual(sorted(rows), ["pico", "pico-8", "psx"])
self.assertEqual(rows["psx"]["core"], "duckstation")
self.assertEqual(rows["psx"]["extensions"], [".cue"])
def test_a_name_given_twice_names_neither_folder(self) -> None:
rows = by_folder(scrape("es-json", "json", {
"file": "es_systems.json", "path_prefixes": ["/storage/roms/"]}))
for folder, line in (("pico-8", 19), ("pico", 25)):
with self.subTest(folder=folder):
self.assertNotIn("name", rows[folder])
self.assertIn("2 systems pico8", rows[folder]["note"])
self.assertEqual(rows[folder]["source_ref"], f"os es_systems.json:{line}")
self.assertEqual(rows["pico"]["core"], "genesis_plus_gx")
class ElecConfFormat(unittest.TestCase):
def setUp(self) -> None:
self.rows = by_folder(scrape("elec-conf", "elec", {
"package": "package.mk", "conf_dir": "config/emulators",
"path_prefixes": ["/storage/roms/"]}))
def test_folders_from_conf_paths(self) -> None:
self.assertEqual(sorted(self.rows), ["3do", "psx"])
def test_star_branch_is_the_default_and_devices_are_boards(self) -> None:
psx = self.rows["psx"]
self.assertEqual(psx["core"], "swanstation")
self.assertEqual(psx["board_defaults"], {
"DEVICE=RK3326": {"core": "pcsx_rearmed", "emulator": "retroarch",
"source_ref": "os package.mk:11"}})
def test_unconditional_default(self) -> None:
self.assertEqual(self.rows["3do"]["core"], "opera")
self.assertEqual(self.rows["3do"]["source_ref"],
"os config/emulators/3do.conf:3,4; os package.mk:7; os package.mk:6")
class EsdeFormat(unittest.TestCase):
def setUp(self) -> None:
self.rows = by_folder(scrape("esde", "esde", {
"systems": {"source": "os", "path": "es_systems.xml"},
"custom": [{"source": "os", "path": "custom_systems.xml"}],
"alternatives": {"source": "os", "path": "setup.sh", "function": "ESDE_setDefaultEmulators"},
}))
def test_alternative_emulator_label_picks_the_command(self) -> None:
psx = self.rows["psx"]
self.assertEqual((psx["core"], psx["emulator"]), ("duckstation", "duckstation"))
self.assertIn("os setup.sh:3", psx["source_ref"])
def test_custom_system_replaces_the_bundled_one(self) -> None:
self.assertNotIn("model2", self.rows)
model2 = self.rows["model2/roms"]
self.assertEqual(model2["core"], "model-2")
self.assertIn("names no command", model2["note"])
def test_first_command_without_alternative(self) -> None:
rows = by_folder(scrape("esde", "esde", {"systems": {"source": "os", "path": "es_systems.xml"}}))
self.assertEqual((rows["psx"]["core"], rows["psx"]["emulator"]), ("mednafen_psx", "retroarch"))
class RecalboxFormat(unittest.TestCase):
def setUp(self) -> None:
self.rows = by_folder(scrape("recalbox", "recalbox", {"systems_dir": "systems"}))
def test_tie_goes_to_the_last_core(self) -> None:
self.assertEqual(self.rows["psx"]["core"], "swanstation")
def test_board_priority(self) -> None:
self.assertEqual(self.rows["psx"]["board_defaults"]["rpi1"]["core"], "mednafen_psx")
def test_unopened_archive_extension(self) -> None:
mame = self.rows["mame"]
self.assertEqual(mame["extensions"], [".zip"])
self.assertTrue(mame["arcade"])
self.assertIn("!", mame["note"])
class RetroPieFormat(unittest.TestCase):
def setUp(self) -> None:
self.rows = by_folder(scrape("retropie", "retropie", {
"platforms_cfg": "platforms.cfg", "module_dirs": ["emulators", "libretrocores"],
"install_sections": ["core", "main", "opt", "exp"],
"platform_flags": ["native", "64bit", "gl", "vulkan", "x11", "x86"]}))
def test_folders_and_loop_systems(self) -> None:
self.assertEqual(sorted(self.rows), ["nes", "psx", "vectrex"])
self.assertEqual(self.rows["nes"]["core"], "lr-fceumm")
self.assertEqual(self.rows["nes"]["emulator"], "retroarch")
def test_flags_and_section_order(self) -> None:
# lr-pcsx-rearmed is excluded on x86, opt comes before exp.
self.assertEqual(self.rows["psx"]["core"], "lr-swanstation")
self.assertNotIn("core", self.rows["vectrex"])
def test_module_flags_read_in_order(self) -> None:
x86 = {"x86", "64bit"}
self.assertFalse(_module_valid(["!all"], x86))
self.assertTrue(_module_valid(["!all", "x86"], x86))
self.assertFalse(_module_valid(["!x86"], x86))
self.assertTrue(_module_valid(["!arm"], x86))
class Determinism(unittest.TestCase):
def test_two_scrapes_dump_the_same_bytes(self) -> None:
params = {"file": "es_systems.cfg", "path_prefixes": ["/storage/roms/"]}
first = dump_setup(scrape("es-xml", "xml", params))
second = dump_setup(scrape("es-xml", "xml", params))
self.assertEqual(first, second)
class CommittedSetups(unittest.TestCase):
"""The setup files this parser family writes."""
@classmethod
def setUpClass(cls) -> None:
cls.sources = load_sources(ROMFOLDERS / "_sources")
cls.paths = {p.stem: p for p in setup_paths(ROMFOLDERS)}
def written(self, setup: str) -> Path:
"""The setup file, or a skip for a source not fetched yet."""
if setup == "recalbox" and setup not in self.paths:
self.skipTest(RECALBOX_PENDING)
return self.paths[setup]
def test_every_es_setup_is_declared_and_written(self) -> None:
formats = load_formats()
for setup in ES_SETUPS:
with self.subTest(setup=setup):
self.assertIn(setup, self.sources)
self.assertIn(self.sources[setup]["format"], formats)
self.assertEqual(self.written(setup).stem, setup)
def test_schema(self) -> None:
try:
from jsonschema import Draft202012Validator
except ImportError:
self.skipTest("jsonschema not installed")
from safeparse import yaml_load
schema = json.loads((ROOT / "schemas" / "romfolders.schema.json").read_text())
validator = Draft202012Validator(schema)
for setup in ES_SETUPS:
with self.subTest(setup=setup):
doc = yaml_load(self.written(setup).read_text(encoding="utf-8"))
errors = [e.message for e in validator.iter_errors(doc)]
self.assertEqual(errors, [])
def test_every_source_ref_names_a_declared_source(self) -> None:
from safeparse import yaml_load
for setup in ES_SETUPS:
with self.subTest(setup=setup):
doc = yaml_load(self.written(setup).read_text(encoding="utf-8"))
if setup not in self.paths:
continue
refs = [doc["roms_root"]["source_ref"]]
for key in ("skip", "hidden", "media"):
refs += [rule["source_ref"] for rule in doc.get(key, [])]
for row in doc["folders"]:
refs.append(row["source_ref"])
refs += [b["source_ref"] for b in (row.get("board_defaults") or {}).values()]
for ref in refs:
for key, _path, _lines in split_refs(ref):
with self.subTest(setup=setup, ref=ref):
self.assertIn(key, doc["sources"])
def test_committed_file_is_its_own_dump(self) -> None:
from safeparse import yaml_load
for setup in ES_SETUPS:
with self.subTest(setup=setup):
text = self.written(setup).read_text(encoding="utf-8")
self.assertEqual(dump_setup(yaml_load(text)), text)
def test_elec_family_skips_its_bios_folder(self) -> None:
from safeparse import yaml_load
for setup in ("rocknix", "jelos", "kolos", "unofficialos", "archr",
"amberelec", "chaos", "pan4elec", "emuelec", "arkos", "darkos"):
doc = yaml_load(self.paths[setup].read_text(encoding="utf-8"))
with self.subTest(setup=setup):
self.assertEqual(doc["roms_root"]["from_bios"], "..")
self.assertIn("bios", {rule.get("name") for rule in doc.get("skip", [])})
self.assertNotIn("bios", {row["folder"] for row in doc["folders"]})
if __name__ == "__main__":
unittest.main()