kmoody/libretro
1
0
Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-04-13 12:22:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add RetroDECK Platform Support (#36)

Browse Source 
* Add files via upload

* Add files via upload

* Update _registry.yml
This commit is contained in:
monster-penguin
2026-03-19 09:10:37 -07:00
committed by GitHub
parent 4992af07e7
commit 1fcb948a00
3 changed files with 1079 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
platforms/_registry.yml
View File

@@ -73,6 +73,20 @@ platforms:
cores: all_libretro
schedule: weekly
retrodeck:
config: retrodeck.yml
status: active
logo: "https://raw.githubusercontent.com/RetroDECK/RetroDECK/main/res/icon.svg"
scraper: retrodeck
source_url: "https://github.com/RetroDECK/components"
source_format: github_component_manifests
hash_type: md5
schedule: monthly
emulators: [duckstation, pcsx2, dolphin, rpcs3, ppsspp, cemu, xemu, vita3k, melonds, xroar]
# RetroDECK/components = separate repo containing per-component build recipes
# Each component/<name>/component_manifest.json declares BIOS requirements
# Scraper enumerates top-level dirs via GitHub API, fetches each manifest directly
retropie:
config: retropie.yml
status: archived # Last release: v4.8 (March 2022) - no update in 4 years

486
platforms/retrodeck.yml Normal file
View File

@@ -0,0 +1,486 @@
platform: RetroDECK
version: ''
homepage: "https://retrodeck.net"
source: "https://github.com/RetroDECK/components"
base_destination: bios
hash_type: md5
verification_mode: md5
systems:
sony-playstation:
files:
- name: psxonpsp660.bin
destination: psxonpsp660.bin
required: true
md5: c53ca5908936d412331790f4426c6c33
- name: scph5500.bin
destination: scph5500.bin
required: true
md5: 8dd7d5296a650fac7319bce665a6a53c
- name: scph5501.bin
destination: scph5501.bin
required: true
md5: 490f666e1afb15b7362b406ed1cea246
- name: scph5502.bin
destination: scph5502.bin
required: true
md5: 32736f17079d0b2b7024407c39bd3050
- name: scph7001.bin
destination: scph7001.bin
required: true
md5: 1e68c231d0896b7eadcad1d7d8e76129
- name: scph7002.bin
destination: scph7002.bin
required: true
md5: b9d9a0286c33dc6b7237bb13cd46fdee
- name: scph7003.bin
destination: scph7003.bin
required: true
md5: 490f666e1afb15b7362b406ed1cea246
- name: scph7502.bin
destination: scph7502.bin
required: true
md5: b9d9a0286c33dc6b7237bb13cd46fdee
- name: "scph9002(7502).bin"
destination: "scph9002(7502).bin"
required: true
md5: b9d9a0286c33dc6b7237bb13cd46fdee
- name: ps1_rom.bin
destination: ps1_rom.bin
required: true
md5: 81bbe60ba7a3d1cea1d48c14cbcc647b
- name: scph1000.bin
destination: scph1000.bin
required: true
md5: 239665b1a3dade1b5a52c06338011044
- name: scph1001.bin
destination: scph1001.bin
required: true
md5: 924e392ed05558ffdb115408c263dccf
- name: scph1002.bin
destination: scph1002.bin
required: true
md5: 54847e693405ffeb0359c6287434cbef
- name: scph100.bin
destination: scph100.bin
required: true
md5: 8abc1b549a4a80954addc48ef02c4521
- name: scph101.bin
destination: scph101.bin
required: true
md5: 6e3735ff4c7dc899ee98981385f6f3d0
- name: scph102A.bin
destination: scph102A.bin
required: true
md5: b10f5e0e3d9eb60e5159690680b1e774
- name: scph102B.bin
destination: scph102B.bin
required: true
md5: de93caec13d1a141a40a79f5c86168d6
- name: scph102C.bin
destination: scph102C.bin
required: true
md5: de93caec13d1a141a40a79f5c86168d6
- name: scph3000.bin
destination: scph3000.bin
required: true
md5: 849515939161e62f6b866f6853006780
- name: scph3500.bin
destination: scph3500.bin
required: true
md5: cba733ceeff5aef5c32254f1d617fa62
- name: scph5000.bin
destination: scph5000.bin
required: true
md5: eb201d2d98251a598af467d4347bb62f
nintendo-ds:
files:
- name: bios7.bin
destination: bios7.bin
required: true
md5: df692a80a5b1bc90728bc3dfc76cd948
- name: bios9.bin
destination: bios9.bin
required: true
md5: a392174eb3e572fed6447e956bde4b25
- name: firmware.bin
destination: firmware.bin
required: true
md5: e45033d9b0fa6b0de071292bba7c9d13
- name: biosdsi7.bin
destination: biosdsi7.bin
required: true
sha256: 2946281e730e71f7cafdb125f5cb60fed944ca5d610ee1e082c441b602b5f4e2
- name: biosdsi9.bin
destination: biosdsi9.bin
required: true
sha256: 47538922a8e8a8e79b922ff1203863ef5c40d9c54656a8d2c89c56ece52029ce
- name: dsifirmware.bin
destination: dsifirmware.bin
required: true
sha256: 11a150e3729bdde3ae8f5e7fc8be67d8bfbc548a1d2e523da58aa826ca0ffa99
sony-playstation-2:
files:
- name: ps2-0100j-20000117.bin
destination: ps2-0100j-20000117.bin
required: true
md5: acf4730ceb38ac9d8c7d8e21f2614600
- name: ps2-0101j-20000217.bin
destination: ps2-0101j-20000217.bin
required: true
md5: b1459d7446c69e3e97e6ace3ae23dd1c
- name: ps2-0120j-20001027-185015.bin
destination: ps2-0120j-20001027-185015.bin
required: true
md5: f63bc530bd7ad7c026fcd6f7bd0d9525
- name: ps2-0120j-20001027-191435.bin
destination: ps2-0120j-20001027-191435.bin
required: true
md5: cee06bd68c333fc5768244eae77e4495
- name: ps2-0150j-20010118.bin
destination: ps2-0150j-20010118.bin
required: true
md5: 815ac991d8bc3b364696bead3457de7d
- name: ps2-0160j-20010427.bin
destination: ps2-0160j-20010427.bin
required: true
md5: ab55cceea548303c22c72570cfd4dd71
- name: ps2-0170j-20030206.bin
destination: ps2-0170j-20030206.bin
required: true
md5: 312ad4816c232a9606e56f946bc0678a
- name: ps2-0200j-20040614.bin
destination: ps2-0200j-20040614.bin
required: true
md5: 0eee5d1c779aa50e94edd168b4ebf42e
- name: ps2-0210j-20040917.bin
destination: ps2-0210j-20040917.bin
required: true
md5: 1ad977bb539fc9448a08ab276a836bbc
- name: ps2-0110a-20000727.bin
destination: ps2-0110a-20000727.bin
required: true
md5: a20c97c02210f16678ca3010127caf36
- name: ps2-0120a-20000902.bin
destination: ps2-0120a-20000902.bin
required: true
md5: 8db2fbbac7413bf3e7154c1e0715e565
- name: ps2-0150a-20001228.bin
destination: ps2-0150a-20001228.bin
required: true
md5: 8accc3c49ac45f5ae2c5db0adc854633
- name: ps2-0160a-20010427.bin
destination: ps2-0160a-20010427.bin
required: true
md5: b107b5710042abe887c0f6175f6e94bb
- name: ps2-0160a-20011004.bin
destination: ps2-0160a-20011004.bin
required: true
md5: 7200a03d51cacc4c14fcdfdbc4898431
- name: ps2-0160a-20020207.bin
destination: ps2-0160a-20020207.bin
required: true
md5: d5ce2c7d119f563ce04bc04dbc3a323e
- name: ps2-0170a-20030325.bin
destination: ps2-0170a-20030325.bin
required: true
md5: 8aa12ce243210128c5074552d3b86251
- name: ps2-0190a-20030623.bin
destination: ps2-0190a-20030623.bin
required: true
md5: 35461cecaa51712b300b2d6798825048
- name: ps2-0200a-20040614.bin
destination: ps2-0200a-20040614.bin
required: true
md5: d333558cc14561c1fdc334c75d5f37b7
- name: ps2-0220a-20050620.bin
destination: ps2-0220a-20050620.bin
required: true
md5: 929a14baca1776b00869f983aa6e14d2
- name: ps2-0220a-20060210.bin
destination: ps2-0220a-20060210.bin
required: true
md5: cb801b7920a7d536ba07b6534d2433ca
- name: ps2-0220a-20060905.bin
destination: ps2-0220a-20060905.bin
required: true
md5: 40c11c063b3b9409aa5e4058e984e30c
- name: ps2-0230a-20080220.bin
destination: ps2-0230a-20080220.bin
required: true
md5: 21038400dc633070a78ad53090c53017
- name: ps2-0120e-20000902.bin
destination: ps2-0120e-20000902.bin
required: true
md5: b7fa11e87d51752a98b38e3e691cbf17
- name: ps2-0150e-20001228.bin
destination: ps2-0150e-20001228.bin
required: true
md5: 838544f12de9b0abc90811279ee223c8
- name: ps2-0160e-20010704.bin
destination: ps2-0160e-20010704.bin
required: true
md5: 491209dd815ceee9de02dbbc408c06d6
- name: ps2-0160e-20011004.bin
destination: ps2-0160e-20011004.bin
required: true
md5: 8359638e857c8bc18c3c18ac17d9cc3c
- name: ps2-0160e-20020319.bin
destination: ps2-0160e-20020319.bin
required: true
md5: 0d2228e6fd4fb639c9c39d077a9ec10c
- name: ps2-0170e-20030227.bin
destination: ps2-0170e-20030227.bin
required: true
md5: 6e69920fa6eef8522a1d688a11e41bc6
- name: ps2-0190e-20030623.bin
destination: ps2-0190e-20030623.bin
required: true
md5: bd6415094e1ce9e05daabe85de807666
- name: ps2-0200e-20040614.bin
destination: ps2-0200e-20040614.bin
required: true
md5: dc752f160044f2ed5fc1f4964db2a095
- name: ps2-0220e-20050620.bin
destination: ps2-0220e-20050620.bin
required: true
md5: 573f7d4a430c32b3cc0fd0c41e104bbd
- name: ps2-0220e-20060210.bin
destination: ps2-0220e-20060210.bin
required: true
md5: af60e6d1a939019d55e5b330d24b1c25
- name: ps2-0220e-20060905.bin
destination: ps2-0220e-20060905.bin
required: true
md5: 80bbb237a6af9c611df43b16b930b683
- name: ps2-0230e-20080220.bin
destination: ps2-0230e-20080220.bin
required: true
md5: dc69f0643a3030aaa4797501b483d6c4
- name: ps2-0160h-20010730.bin
destination: ps2-0160h-20010730.bin
required: true
md5: 352d2ff9b3f68be7e6fa7e6dd8389346
- name: ps2-0160h-20020426.bin
destination: ps2-0160h-20020426.bin
required: true
md5: 315a4003535dfda689752cb25f24785c
- name: ps2-0190h-20030623.bin
destination: ps2-0190h-20030623.bin
required: true
md5: 07b562a3f0c4b9a55834df9bbc9bd0c3
- name: ps2-0200h-20040614.bin
destination: ps2-0200h-20040614.bin
required: true
md5: 3e3e030c0f600442fa05b94f87a1e238
- name: ps2-0220h-20060905.bin
destination: ps2-0220h-20060905.bin
required: true
md5: c37bce95d32b2be480f87dd32704e664
- name: ps2-0220h-20060210.bin
destination: ps2-0220h-20060210.bin
required: true
md5: 549a66d0c698635ca9fa3ab012da7129
- name: ps2-0190c-20030623.bin
destination: ps2-0190c-20030623.bin
required: true
md5: 1b6e631b536247756287b916f9396872
- name: ps2-0190r-20030623.bin
destination: ps2-0190r-20030623.bin
required: true
md5: 0c13357c01a25a886db2356bbe73d9f0
pico8:
files:
- name: pico8
destination: pico-8/pico8
required: true
- name: pico8.dat
destination: pico-8/pico8.dat
required: true
- name: pico8_dyn
destination: pico-8/pico8_dyn
required: true
sony-psp:
files:
- name: ppge_atlas.zim
destination: ppge_atlas.zim
required: false
md5: 866855cc330b9b95cc69135fb7b41d38
xbox:
files:
- name: mcpx_1.0.bin
destination: mcpx_1.0.bin
required: true
md5: d49c52a4102f6df7bcf8d0617ac475ed
- name: Complex.bin
destination: Complex.bin
required: true
- name: Complex_4627v1.03.bin
destination: Complex_4627v1.03.bin
required: true
- name: Complex_4627.bin
destination: Complex_4627.bin
required: true
dragon32:
files:
- name: d32.rom
destination: d32.rom
required: true
md5: d35177f73cf303c5565aa13ef8ca5251,3420b96031078a4ef408cad7bf21a33f
- name: d64rom1.rom
destination: d64rom1.rom
required: true
md5: 5f0bee59710e55f5880e74890912ed78,6ab639e65c6e8fd832cb0d8ad4da1b60
- name: d64tano.rom
destination: d64tano.rom
required: true
md5: be9bc86ee5eb401d0a40d0377f65fefa
- name: d64tano2.rom
destination: d64tano2.rom
required: true
md5: fd91edce7be5e7c2d88e46b76956a8aa
- name: d200rom1.rom
destination: d200rom1.rom
required: true
md5: be9bc86ee5eb401d0a40d0377f65fefa
- name: d200rom2.rom
destination: d200rom2.rom
required: true
md5: fd91edce7be5e7c2d88e46b76956a8aa
- name: ddos10.rom
destination: ddos10.rom
required: true
md5: 1c965da49b6c5459b8353630aa1482e7
- name: ddos11c.rom
destination: ddos11c.rom
required: true
md5: d8429af1a12f7438a4bf88a5b934cb3a
- name: ddos12a.rom
destination: ddos12a.rom
required: true
md5: 55e2535dbbed7f1a26b5f263d7c72c63
- name: ddos40.rom
destination: ddos40.rom
required: true
md5: 9ddc388632cd3c376b164ba5cfc64329
- name: ddos42.rom
destination: ddos42.rom
required: true
md5: c956a854cbc4b9d1e69c000f78368668
- name: deltados.rom
destination: deltados.rom
required: true
md5: 024eac3db20f1b5cf98c30a0e4743201
- name: dplus48.rom
destination: dplus48.rom
required: true
md5: ee6f24d893a52b8efea9f787855456b5
- name: dplus49b.rom
destination: dplus49b.rom
required: true
md5: 56f1b97314e4ca82491c465bb887059e
- name: dplus50.rom
destination: dplus50.rom
required: true
md5: 35de5d28da507ebb213a26e04241d940
- name: sdose6.rom
destination: sdose6.rom
required: true
md5: 9d85e6b7133f915c021156f4b9cdb512
- name: sdose8.rom
destination: sdose8.rom
required: true
md5: 167f409b7a4b992faabb784b061ab4c6
- name: cp400bas.rom
destination: cp400bas.rom
required: true
md5: f73da4d73d6db5cdb8b3cb6a50415e38
trs80coco:
files:
- name: cp400extbas.rom
destination: cp400extbas.rom
required: true
md5: 091581001577b4a83ccfd511829de0f1
- name: cp400dsk.rom
destination: cp400dsk.rom
required: true
md5: 16d3ab9bc935f0d5651ca3f0e3030846
- name: color64extbas.rom
destination: color64extbas.rom
required: true
md5: 0d9264ffa95ba493f2b5b0d488a49e13
- name: xroarbios.rom
destination: xroarbios.rom
required: true
md5: 28dc97df470fb8660ef61b81dfd34f4a
- name: bas10.rom
destination: bas10.rom
required: true
md5: a74f3d95b395dad7cdca19d560eeea74
- name: bas11.rom
destination: bas11.rom
required: true
md5: c73fb4bff9621c5ab17f6220b20db82f
- name: bas12.rom
destination: bas12.rom
required: true
md5: c933316c7d939532a13648850c1c2aa6
- name: bas13.rom
destination: bas13.rom
required: true
md5: c2fc43556eb6b7b25bdf5955bd9df825
- name: bas14.rom
destination: bas14.rom
required: true
md5: ac33e16f677b4db52548d426174b1aaa
- name: coco3.rom
destination: coco3.rom
required: true
md5: 7233c6c429f3ce1c7392f28a933e0b6f
- name: extbas10.rom
destination: extbas10.rom
required: true
md5: fda72f415afe99b36f953bb9bc1253da
- name: extbas11.rom
destination: extbas11.rom
required: true
md5: 21070aa0496142b886c562bf76d7c113
- name: disk10.rom
destination: disk10.rom
required: true
md5: a64b3ef9efcc066b18d35b134068d1cc
- name: disk11.rom
destination: disk11.rom
required: true
md5: 8cab28f4b7311b8df63c07bb3b59bfd5
- name: fd502.rom
destination: fd502.rom
required: true
md5: 8cab28f4b7311b8df63c07bb3b59bfd5
- name: fd502ds.rom
destination: fd502ds.rom
required: true
md5: b2d43757dc6851d866021ff6c4f59205
- name: coco3p.rom
destination: coco3p.rom
required: true
md5: 4ae57e5a8e7494e5485446fefedb580b
- name: CCNPATCH.cas
destination: CCNPATCH.cas
required: true
md5: 2c395ad58a842711931679b554483a90
- name: CCNPATCH.wav
destination: CCNPATCH.wav
required: true
md5: a17397fb5408647a11d23bab959d1f97
- name: DBPATCH.cas
destination: DBPATCH.cas
required: true
md5: 65e1aab5fc5cba1b7374d9dddec25d62
- name: DBPATCH.wav
destination: DBPATCH.wav
required: true
md5: 6a07aeee664d81047672e6ff3541a12a
- name: disk.rom
destination: disk.rom
required: true
md5: 5bfcb1ae090159c8dea542b5a7c0840f

579
scripts/scraper/retrodeck_scraper.py Normal file
View File

@@ -0,0 +1,579 @@
#!/usr/bin/env python3
"""Scraper for RetroDECK BIOS requirements.
Source: https://github.com/RetroDECK/components
Format: component_manifest.json committed at <component>/component_manifest.json
Hash: MD5 primary, SHA256 for some entries (melonDS DSi BIOS)
RetroDECK verification logic:
- MD5 or SHA256 checked against expected value per file
- MD5 may be a list of multiple accepted hashes (xroar ROM variants) — joined
as comma-separated string per retrobios convention
- Files may declare paths via $bios_path, $saves_path, or $roms_path tokens
- $saves_path entries (GameCube memory card directories) are excluded —
these are directory placeholders, not BIOS files
- $roms_path entries are included with a roms/ prefix in destination,
consistent with Batocera's saves/ destination convention
- Entries with no hash are emitted without an md5 field (existence-only),
which is valid per the platform schema (e.g. pico-8 executables)
Component structure:
RetroDECK/components (GitHub, main branch)
├── <component>/component_manifest.json <- fetched directly via raw URL
├── archive_later/ <- skipped
└── archive_old/ <- skipped
BIOS may appear in three locations within a manifest:
- top-level 'bios' key (melonDS, xemu, xroar, pico-8)
- preset_actions.bios (duckstation, dolphin, pcsx2, ppsspp)
- cores.bios (not yet seen in practice, kept for safety)
ppsspp quirk: preset_actions.bios is a bare dict, not a list.
Adding to watch.yml (maintainer step):
from scraper.retrodeck_scraper import Scraper as RDS
config = RDS().generate_platform_yaml()
with open('platforms/retrodeck.yml', 'w') as f:
yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
print(f'RetroDECK: {len(config["systems"])} systems, version={config["version"]}')
"""
from __future__ import annotations
import json
import os
import re
import sys
import urllib.request
import urllib.error
from pathlib import Path
try:
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
except ImportError:
from base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
PLATFORM_NAME = "retrodeck"
COMPONENTS_REPO = "RetroDECK/components"
COMPONENTS_BRANCH = "main"
COMPONENTS_API_URL = (
f"https://api.github.com/repos/{COMPONENTS_REPO}"
f"/git/trees/{COMPONENTS_BRANCH}?recursive=0"
)
RAW_BASE_URL = (
f"https://raw.githubusercontent.com/{COMPONENTS_REPO}"
f"/{COMPONENTS_BRANCH}"
)
# Top-level directories to ignore when enumerating components
SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"}
# Default local path for --manifests-dir (standard flatpak install)
DEFAULT_LOCAL_MANIFESTS = (
"/var/lib/flatpak/app/net.retrodeck.retrodeck"
"/current/active/files/retrodeck/components"
)
# RetroDECK system ID -> retrobios slug.
# IDs absent from this map pass through unchanged (maintainer decides on slug).
# IDs mapped to None are skipped entirely (no retrobios equivalent).
SYSTEM_SLUG_MAP: dict[str, str | None] = {
# Nintendo
"nes": "nintendo-nes",
"snes": "nintendo-snes",
"snesna": "nintendo-snes",
"n64": "nintendo-64",
"n64dd": "nintendo-64dd",
"gc": "nintendo-gamecube",
"wii": "wii", # no retrobios slug yet — passes through
"wiiu": "nintendo-wii-u",
"switch": "nintendo-switch",
"gb": "nintendo-gb",
"gbc": "nintendo-gbc",
"gba": "nintendo-gba",
"nds": "nintendo-ds",
"3ds": "nintendo-3ds",
"n3ds": "nintendo-3ds", # azahar uses n3ds
"fds": "nintendo-fds",
"sgb": "nintendo-sgb",
"virtualboy": "nintendo-virtual-boy",
# Sony
"psx": "sony-playstation",
"ps2": "sony-playstation-2",
"ps3": "sony-playstation-3",
"psp": "sony-psp",
"psvita": "sony-psvita",
# Sega
"megadrive": "sega-mega-drive",
"genesis": "sega-mega-drive",
"megacd": "sega-mega-cd",
"megacdjp": "sega-mega-cd",
"segacd": "sega-mega-cd",
"saturn": "sega-saturn",
"saturnjp": "sega-saturn",
"dreamcast": "sega-dreamcast",
"naomi": "sega-dreamcast-arcade",
"naomi2": "sega-dreamcast-arcade",
"atomiswave": "sega-dreamcast-arcade",
"sega32x": "sega32x",
"sega32xjp": "sega32x",
"sega32xna": "sega32x",
"gamegear": "sega-game-gear",
"mastersystem": "sega-master-system",
# NEC
"tg16": "nec-pc-engine",
"tg-cd": "nec-pc-engine",
"pcengine": "nec-pc-engine",
"pcenginecd": "nec-pc-engine",
"pcfx": "nec-pc-fx",
# SNK
"neogeo": "snk-neogeo",
"neogeocd": "snk-neogeo-cd",
"neogeocdjp": "snk-neogeo-cd",
# Atari
"atari2600": "atari2600", # no retrobios slug yet — passes through
"atari800": "atari-400-800",
"atari5200": "atari-5200",
"atari7800": "atari-7800",
"atarilynx": "atari-lynx",
"atarist": "atari-st",
"atarijaguar": "jaguar",
# Panasonic / Philips
"3do": "panasonic-3do",
"cdimono1": "cdi",
"cdtv": "amigacdtv",
# Microsoft
"xbox": "xbox",
# Commodore
"amiga": "commodore-amiga",
"amigacd32": "amigacd32",
"c64": "commodore-c64",
# Tandy / Dragon
"coco": "trs80coco",
"dragon32": "dragon32",
"tanodragon": "dragon32", # Tano Dragon is a Dragon 32 clone
# Other
"colecovision": "coleco-colecovision",
"intellivision": "mattel-intellivision",
"o2em": "magnavox-odyssey2",
"msx": "microsoft-msx",
"msx2": "microsoft-msx",
"fmtowns": "fmtowns",
"scummvm": "scummvm",
"dos": "dos",
# Explicitly skipped — no retrobios equivalent
"mess": None,
}
# Matches all saves_path typo variants seen in the wild:
# $saves_path, $saves_paths_path, $saves_paths_paths_path, etc.
_SAVES_PATH_RE = re.compile(r"^\$saves_\w+/")
def _fetch_bytes(url: str, token: str | None = None) -> bytes:
headers = {"User-Agent": "retrobios-scraper/1.0"}
if token:
headers["Authorization"] = f"token {token}"
req = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(req, timeout=30) as resp:
return resp.read()
except urllib.error.URLError as e:
raise ConnectionError(f"Failed to fetch {url}: {e}") from e
def _fetch_json(url: str, token: str | None = None) -> dict | list:
return json.loads(_fetch_bytes(url, token).decode("utf-8"))
def _resolve_destination(raw_path: str, filename: str) -> str | None:
"""Resolve a RetroDECK path token to a retrobios destination string.
Returns None if the entry should be excluded ($saves_path variants).
$bios_path -> strip prefix; destination is bios-relative.
$roms_path -> preserve roms/ prefix (Batocera saves/ convention).
Bare directory paths get filename appended.
"""
if _SAVES_PATH_RE.match(raw_path):
return None
if raw_path.startswith("$bios_path/"):
remainder = raw_path[len("$bios_path/"):].strip("/")
if not remainder or remainder == filename:
return filename
# Subdirectory path — append filename if path looks like a directory
if not remainder.endswith(tuple(".bin .rom .zip .img .bin ".split())):
return remainder.rstrip("/") + "/" + filename
return remainder
if raw_path.startswith("$roms_path/"):
remainder = raw_path[len("$roms_path/"):].strip("/")
base = ("roms/" + remainder) if remainder else "roms"
return base.rstrip("/") + "/" + filename
# No recognised token — treat as bios-relative
remainder = raw_path.strip("/")
if not remainder:
return filename
return remainder.rstrip("/") + "/" + filename
def _normalise_md5(raw: str | list) -> str:
"""Return a comma-separated MD5 string.
xroar declares a list of accepted hashes for ROM variants;
retrobios platform schema accepts comma-separated MD5 strings.
"""
if isinstance(raw, list):
return ",".join(str(h).strip().lower() for h in raw if h)
return str(raw).strip().lower() if raw else ""
def _coerce_bios_to_list(val: object) -> list:
"""Ensure a bios value is always a list of dicts.
ppsspp declares preset_actions.bios as a bare dict, not a list.
"""
if isinstance(val, list):
return val
if isinstance(val, dict):
return [val]
return []
def _parse_required(raw: object) -> bool:
"""Coerce RetroDECK required field to bool.
Values seen: 'Required', 'Optional', 'At least one BIOS file required',
'Optional, for boot logo', True, False, absent (None).
Absent is treated as required.
"""
if isinstance(raw, bool):
return raw
if raw is None:
return True
return str(raw).strip().lower() not in ("optional", "false", "no", "0")
def _parse_manifest(data: dict) -> list[BiosRequirement]:
"""Parse one component_manifest.json into BiosRequirement objects."""
requirements: list[BiosRequirement] = []
seen: set[tuple[str, str]] = set()
for _component_key, component_val in data.items():
if not isinstance(component_val, dict):
continue
# Component-level system fallback (may be a list for multi-system components)
comp_system = component_val.get("system", "")
if isinstance(comp_system, list):
comp_system = comp_system[0] if comp_system else ""
comp_system = str(comp_system).strip().lower()
# Collect bios entries from all known locations
bios_sources: list[list] = []
if "bios" in component_val:
bios_sources.append(_coerce_bios_to_list(component_val["bios"]))
pa = component_val.get("preset_actions", {})
if isinstance(pa, dict) and "bios" in pa:
bios_sources.append(_coerce_bios_to_list(pa["bios"]))
cores = component_val.get("cores", {})
if isinstance(cores, dict) and "bios" in cores:
bios_sources.append(_coerce_bios_to_list(cores["bios"]))
if not bios_sources:
continue
for bios_list in bios_sources:
for entry in bios_list:
if not isinstance(entry, dict):
continue
filename = str(entry.get("filename", "")).strip()
if not filename:
continue
# System slug — entry-level preferred, component-level fallback
entry_system = entry.get("system", comp_system)
if isinstance(entry_system, list):
entry_system = entry_system[0] if entry_system else comp_system
entry_system = str(entry_system).strip().lower()
if entry_system in SYSTEM_SLUG_MAP:
slug = SYSTEM_SLUG_MAP[entry_system]
if slug is None:
continue # explicitly skipped (e.g. mess)
else:
slug = entry_system # unknown — pass through
# Destination resolution
paths_raw = entry.get("paths")
if paths_raw is None:
destination = filename
elif isinstance(paths_raw, list):
destination = None
for p in paths_raw:
resolved = _resolve_destination(str(p), filename)
if resolved is not None:
destination = resolved
break
if destination is None:
continue # all paths were saves_path variants — skip
else:
destination = _resolve_destination(str(paths_raw), filename)
if destination is None:
continue # saves_path — skip
# Hash fields
md5_val: str | None = None
sha256_val: str | None = None
raw_md5 = entry.get("md5")
if raw_md5:
md5_val = _normalise_md5(raw_md5) or None
raw_sha256 = entry.get("sha256")
if raw_sha256:
sha256_val = str(raw_sha256).strip().lower() or None
required = _parse_required(entry.get("required"))
dedup_key = (slug, filename.lower())
if dedup_key in seen:
continue
seen.add(dedup_key)
req = BiosRequirement(
name=filename,
system=slug,
md5=md5_val,
sha1=None,
destination=destination,
required=required,
)
req._sha256 = sha256_val # type: ignore[attr-defined]
requirements.append(req)
return requirements
class Scraper(BaseScraper):
"""Scraper for RetroDECK component_manifest.json files.
Two modes:
remote (default): fetches manifests directly from RetroDECK/components
via GitHub raw URLs, enumerating components via the
GitHub API tree endpoint
local: reads manifests from a directory on disk
(--manifests-dir or pass manifests_dir= to __init__)
"""
def __init__(
self,
manifests_dir: str | None = None,
github_token: str | None = None,
):
super().__init__()
self.manifests_dir = manifests_dir
self.github_token = github_token or os.environ.get("GITHUB_TOKEN")
self._release_version: str | None = None
# ── Remote ───────────────────────────────────────────────────────────────
def _list_component_dirs(self) -> list[str]:
"""Return top-level component directory names from the GitHub API."""
tree = _fetch_json(COMPONENTS_API_URL, self.github_token)
return [
item["path"]
for item in tree.get("tree", [])
if item["type"] == "tree" and item["path"] not in SKIP_DIRS
]
def _fetch_remote_manifests(self) -> list[dict]:
component_dirs = self._list_component_dirs()
manifests: list[dict] = []
for component in sorted(component_dirs):
url = f"{RAW_BASE_URL}/{component}/component_manifest.json"
print(f" Fetching {component}/component_manifest.json ...", file=sys.stderr)
try:
raw = _fetch_bytes(url, self.github_token)
manifests.append(json.loads(raw.decode("utf-8")))
except ConnectionError:
pass # component has no manifest — skip silently
except json.JSONDecodeError as e:
print(f" WARNING: parse error in {component}: {e}", file=sys.stderr)
return manifests
# ── Local ─────────────────────────────────────────────────────────────────
def _fetch_local_manifests(self) -> list[dict]:
root = Path(self.manifests_dir)
if not root.is_dir():
raise FileNotFoundError(f"Manifests directory not found: {root}")
manifests: list[dict] = []
# Only scan top-level component directories; skip archive and hidden dirs
for component_dir in sorted(root.iterdir()):
if not component_dir.is_dir():
continue
if component_dir.name in SKIP_DIRS or component_dir.name.startswith("."):
continue
manifest_path = component_dir / "component_manifest.json"
if not manifest_path.exists():
continue
try:
with open(manifest_path) as f:
manifests.append(json.load(f))
except (json.JSONDecodeError, OSError) as e:
print(f" WARNING: Could not parse {manifest_path}: {e}", file=sys.stderr)
return manifests
# ── BaseScraper interface ─────────────────────────────────────────────────
def fetch_requirements(self) -> list[BiosRequirement]:
manifests = (
self._fetch_local_manifests()
if self.manifests_dir
else self._fetch_remote_manifests()
)
requirements: list[BiosRequirement] = []
seen: set[tuple[str, str]] = set()
for manifest in manifests:
for req in _parse_manifest(manifest):
key = (req.system, req.name.lower())
if key not in seen:
seen.add(key)
requirements.append(req)
return requirements
def validate_format(self, raw_data: str) -> bool:
try:
return isinstance(json.loads(raw_data), dict)
except json.JSONDecodeError:
return False
def generate_platform_yaml(self) -> dict:
requirements = self.fetch_requirements()
systems: dict[str, dict] = {}
for req in requirements:
systems.setdefault(req.system, {"files": []})
entry: dict = {
"name": req.name,
"destination": req.destination,
"required": req.required,
}
if req.md5:
entry["md5"] = req.md5
sha256 = getattr(req, "_sha256", None)
if sha256 and not req.md5:
entry["sha256"] = sha256
systems[req.system]["files"].append(entry)
version = self._release_version or ""
if not version:
try:
version = fetch_github_latest_version(COMPONENTS_REPO) or ""
except (ConnectionError, OSError):
pass
return {
"platform": "RetroDECK",
"version": version,
"homepage": "https://retrodeck.net",
"source": f"https://github.com/{COMPONENTS_REPO}",
"base_destination": "bios",
"hash_type": "md5",
"verification_mode": "md5",
"systems": systems,
}
def main() -> None:
import argparse
parser = argparse.ArgumentParser(
description="Scrape RetroDECK component_manifest.json BIOS requirements"
)
parser.add_argument(
"--manifests-dir", metavar="DIR",
help=(
"Read manifests from a local directory instead of fetching from GitHub. "
f"Live install path: {DEFAULT_LOCAL_MANIFESTS}"
),
)
parser.add_argument(
"--token", metavar="TOKEN",
help="GitHub personal access token (or set GITHUB_TOKEN env var)",
)
parser.add_argument(
"--dry-run", action="store_true",
help="Print per-system summary without generating output",
)
parser.add_argument(
"--output", "-o", metavar="FILE",
help="Write generated platform YAML to FILE",
)
parser.add_argument(
"--json", action="store_true",
help="Print platform config as JSON (for debugging)",
)
args = parser.parse_args()
scraper = Scraper(manifests_dir=args.manifests_dir, github_token=args.token)
try:
reqs = scraper.fetch_requirements()
except (ConnectionError, FileNotFoundError) as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
if args.dry_run:
by_system: dict[str, list] = {}
for r in reqs:
by_system.setdefault(r.system, []).append(r)
for system, files in sorted(by_system.items()):
req_c = sum(1 for f in files if f.required)
opt_c = len(files) - req_c
print(f" {system}: {req_c} required, {opt_c} optional")
print(f"\nTotal: {len(reqs)} entries across {len(by_system)} systems")
return
config = scraper.generate_platform_yaml()
if args.json:
print(json.dumps(config, indent=2))
return
if args.output:
try:
import yaml
except ImportError:
print("Error: PyYAML required (pip install pyyaml)", file=sys.stderr)
sys.exit(1)
def _str_representer(dumper, data):
if any(c in data for c in "()[]{}:#"):
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')
return dumper.represent_scalar("tag:yaml.org,2002:str", data)
yaml.add_representer(str, _str_representer)
with open(args.output, "w") as f:
yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
total = sum(len(s["files"]) for s in config["systems"].values())
print(
f"Written {total} entries across "
f"{len(config['systems'])} systems to {args.output}"
)
return
systems = len(set(r.system for r in reqs))
print(f"Scraped {len(reqs)} entries across {systems} systems. Use --dry-run, --json, or --output FILE.")
if __name__ == "__main__":
main()