mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-10 21:43:23 -05:00
feat: add misterfpga platform support
This commit is contained in:
Abdessamad Derraz committed
1 parent
d88e87dc34
commit
8663ec17b7
29 files changed
+1631
-281
No files matched your search
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scraper for MiSTer FPGA BIOS requirements.
|
||||
|
||||
Source: ajgowans/BiosDB_MiSTer, branch db, file bios_db.json.zip
|
||||
Format: MiSTer Downloader database (JSON inside a ZIP)
|
||||
Hash: MD5
|
||||
|
||||
MiSTer verification logic (MiSTer-devel/Downloader_MiSTer):
|
||||
- file_system.py:705-712, hash_file() is a plain MD5 of the file bytes
|
||||
- jobs/process_db_index_worker.py:216, an installed file is up to date when
|
||||
its MD5 equals the description hash
|
||||
- jobs/process_db_index_worker.py:242, integrity checks compare MD5 only;
|
||||
the declared size is logged on failure but never gates the result
|
||||
- jobs/fetch_file_worker.py:61-65, a freshly downloaded file is rejected on
|
||||
MD5 mismatch
|
||||
|
||||
The database carries no required/optional flag: every entry is installed
|
||||
unless the user sets a tag filter, which defaults to empty (config.py:161).
|
||||
Entries reached through the archives section (UniBIOS) are extracted from a
|
||||
remote ZIP and carry the hash of the extracted member, not of the archive.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
try:
|
||||
from .base_scraper import BaseScraper, BiosRequirement, _read_limited
|
||||
except ImportError:
|
||||
from base_scraper import BaseScraper, BiosRequirement, _read_limited
|
||||
|
||||
PLATFORM_NAME = "misterfpga"
|
||||
|
||||
SOURCE_URL = (
|
||||
"https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/db/bios_db.json.zip"
|
||||
)
|
||||
|
||||
DB_MEMBER = "bios_db.json"
|
||||
|
||||
# Files live under games/ on the SD card; base_destination carries that prefix
|
||||
DB_ROOT = "games/"
|
||||
|
||||
# MiSTer core folder -> (retrobios system ID, display name, core repository)
|
||||
SYSTEM_MAP = {
|
||||
"3DO": ("3do", "3DO Interactive Multiplayer", "3DO_MiSTer"),
|
||||
"Apple-IIgs": ("apple-iigs", "Apple IIGS", "Apple-IIgs_MiSTer"),
|
||||
"Astrocade": ("bally-astrocade", "Bally Astrocade", "Astrocade_MiSTer"),
|
||||
"AtariLynx": ("atari-lynx", "Atari Lynx", "AtariLynx_MiSTer"),
|
||||
"CD-i": ("philips-cdi", "Philips CD-i", "CDi_MiSTer"),
|
||||
"COCO3": ("tandy-coco", "Tandy Color Computer 3", "CoCo3_MiSTer"),
|
||||
"Casio_PV-2000": ("casio-pv2000", "Casio PV-2000", "Casio_PV-2000_MiSTer"),
|
||||
"CreatiVision": ("creativision", "VTech CreatiVision", "CreatiVision_MiSTer"),
|
||||
"GAMEBOY": ("nintendo-gb", "Game Boy / Game Boy Color", "Gameboy_MiSTer"),
|
||||
"GBA": ("nintendo-gba", "Game Boy Advance", "GBA_MiSTer"),
|
||||
"Gamate": ("gamate", "Bit Corporation Gamate", "Gamate_MiSTer"),
|
||||
"GameCom": ("tiger-game-com", "Tiger Game.com", ""),
|
||||
"Intellivision": ("mattel-intellivision", "Mattel Intellivision", "Intv_MiSTer"),
|
||||
"Interact": ("interact", "Interact Home Computer", "Interact_MiSTer"),
|
||||
"Jaguar": ("atari-jaguar", "Atari Jaguar", "Jaguar_MiSTer"),
|
||||
"MSX1": ("microsoft-msx", "MSX1", "MSX1_MiSTer"),
|
||||
"MegaCD": ("sega-mega-cd", "Sega CD / Mega-CD", "MegaCD_MiSTer"),
|
||||
"N64": ("nintendo-64", "Nintendo 64", "N64_MiSTer"),
|
||||
"NEOGEO": ("snk-neogeo", "Neo Geo AES / MVS", "NeoGeo_MiSTer"),
|
||||
"NES": ("nintendo-nes", "NES / Famicom Disk System", "NES_MiSTer"),
|
||||
"NeoGeo-CD": ("snk-neogeo-cd", "Neo Geo CD", "NeoGeo_MiSTer"),
|
||||
"NeoGeoPocket": ("snk-neogeo-pocket", "Neo Geo Pocket", ""),
|
||||
"PC8801": ("nec-pc-88", "NEC PC-8801", "PC88_MiSTer"),
|
||||
"PSX": ("sony-playstation", "Sony PlayStation", "PSX_MiSTer"),
|
||||
"PocketChallengeV2": (
|
||||
"pocket-challenge-v2",
|
||||
"Benesse Pocket Challenge V2",
|
||||
"WonderSwan_MiSTer",
|
||||
),
|
||||
"PokemonMini": ("nintendo-pokemon-mini", "Pokemon Mini", "PokemonMini_MiSTer"),
|
||||
"SCV": ("epoch-scv", "Epoch Super Cassette Vision", "SuperCassetteVision_MiSTer"),
|
||||
"SGB": ("nintendo-sgb", "Super Game Boy", "SGB_MiSTer"),
|
||||
"SNES": ("nintendo-snes", "Super NES / Super Famicom", "SNES_MiSTer"),
|
||||
"Saturn": ("sega-saturn", "Sega Saturn", "Saturn_MiSTer"),
|
||||
"TGFX16-CD": ("nec-pc-engine-cd", "PC Engine CD / TurboGrafx-CD", "TurboGrafx16_MiSTer"),
|
||||
"TI-99_4A": ("ti99", "Texas Instruments TI-99/4A", "TI-99_4A_MiSTer"),
|
||||
"WonderSwan": ("wonderswan", "Bandai WonderSwan", "WonderSwan_MiSTer"),
|
||||
"WonderSwanColor": (
|
||||
"wonderswan-color",
|
||||
"Bandai WonderSwan Color",
|
||||
"WonderSwan_MiSTer",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def parse_db(raw: str) -> dict[str, dict]:
|
||||
"""Return every installable file entry of the database, keyed by path.
|
||||
|
||||
Entries listed under archives are inlined: the Downloader extracts them
|
||||
from a remote ZIP, and their hash describes the extracted member.
|
||||
"""
|
||||
data = json.loads(raw)
|
||||
entries: dict[str, dict] = dict(data.get("files", {}))
|
||||
for archive in data.get("archives", {}).values():
|
||||
inline = archive.get("summary_inline", {}).get("files", {})
|
||||
entries.update(inline)
|
||||
if not entries:
|
||||
raise ValueError("no file entries found in bios_db.json")
|
||||
return entries
|
||||
|
||||
|
||||
def system_of(path: str) -> str:
|
||||
"""Return the core folder a database path belongs to."""
|
||||
remainder = path[len(DB_ROOT):] if path.startswith(DB_ROOT) else path
|
||||
return remainder.split("/", 1)[0]
|
||||
|
||||
|
||||
class Scraper(BaseScraper):
|
||||
"""Scraper for the MiSTer BiosDB database."""
|
||||
|
||||
def __init__(self, url: str = SOURCE_URL):
|
||||
super().__init__(url=url)
|
||||
|
||||
def _fetch_raw(self) -> str:
|
||||
"""Fetch bios_db.json.zip and return the JSON it contains."""
|
||||
if self._raw_data is not None:
|
||||
return self._raw_data
|
||||
req = urllib.request.Request(
|
||||
self.url, headers={"User-Agent": "retrobios-scraper/1.0"}
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
payload = _read_limited(resp)
|
||||
except urllib.error.URLError as e:
|
||||
raise ConnectionError(f"Failed to fetch {self.url}: {e}") from e
|
||||
with zipfile.ZipFile(io.BytesIO(payload)) as archive:
|
||||
self._raw_data = archive.read(DB_MEMBER).decode("utf-8")
|
||||
return self._raw_data
|
||||
|
||||
def validate_format(self, raw_data: str) -> bool:
|
||||
"""Validate the BiosDB database format."""
|
||||
try:
|
||||
data = json.loads(raw_data)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
if data.get("db_id") != "bios_db":
|
||||
return False
|
||||
files = data.get("files")
|
||||
if not isinstance(files, dict) or not files:
|
||||
return False
|
||||
return all(
|
||||
isinstance(entry.get("hash"), str) and isinstance(entry.get("size"), int)
|
||||
for entry in files.values()
|
||||
)
|
||||
|
||||
def fetch_requirements(self) -> list[BiosRequirement]:
|
||||
"""Parse bios_db.json and return BIOS requirements."""
|
||||
raw = self._fetch_raw()
|
||||
if not self.validate_format(raw):
|
||||
raise ValueError("bios_db.json format validation failed")
|
||||
|
||||
requirements = []
|
||||
for path, entry in sorted(parse_db(raw).items()):
|
||||
folder = system_of(path)
|
||||
system, _, _ = SYSTEM_MAP.get(folder, (folder.lower(), folder, ""))
|
||||
destination = path[len(DB_ROOT):] if path.startswith(DB_ROOT) else path
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=destination.rsplit("/", 1)[-1],
|
||||
system=system,
|
||||
md5=entry["hash"],
|
||||
size=entry.get("size"),
|
||||
destination=destination,
|
||||
required=True,
|
||||
native_id=folder,
|
||||
)
|
||||
)
|
||||
return requirements
|
||||
|
||||
def generate_platform_yaml(self) -> dict:
|
||||
"""Build the MiSTer FPGA platform configuration."""
|
||||
data = json.loads(self._fetch_raw())
|
||||
systems: dict[str, dict] = {}
|
||||
|
||||
for req in self.fetch_requirements():
|
||||
_, display, repo = SYSTEM_MAP.get(
|
||||
req.native_id or "", (req.system, req.system, "")
|
||||
)
|
||||
entry = systems.setdefault(
|
||||
req.system,
|
||||
{
|
||||
"files": [],
|
||||
"native_id": req.native_id,
|
||||
"name": display,
|
||||
"docs": f"https://github.com/MiSTer-devel/{repo}" if repo else "",
|
||||
},
|
||||
)
|
||||
file_entry: dict = {
|
||||
"name": req.name,
|
||||
"destination": req.destination,
|
||||
"required": req.required,
|
||||
"md5": req.md5,
|
||||
}
|
||||
if req.size is not None:
|
||||
file_entry["size"] = req.size
|
||||
entry["files"].append(file_entry)
|
||||
|
||||
for entry in systems.values():
|
||||
if not entry["docs"]:
|
||||
del entry["docs"]
|
||||
|
||||
timestamp = data.get("timestamp")
|
||||
released = (
|
||||
datetime.fromtimestamp(timestamp, timezone.utc).strftime("%Y-%m-%d")
|
||||
if isinstance(timestamp, (int, float))
|
||||
else ""
|
||||
)
|
||||
|
||||
return {
|
||||
"platform": "MiSTer FPGA",
|
||||
"version": released,
|
||||
"homepage": "https://misterfpga.org",
|
||||
"source": SOURCE_URL,
|
||||
"base_destination": "games",
|
||||
"hash_type": "md5",
|
||||
"verification_mode": "md5",
|
||||
"cores": [],
|
||||
"systems": dict(sorted(systems.items())),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Entry point."""
|
||||
try:
|
||||
from .base_scraper import scraper_cli
|
||||
except ImportError:
|
||||
from base_scraper import scraper_cli
|
||||
|
||||
scraper_cli(Scraper, "Scrape MiSTer FPGA BIOS requirements")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user