mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-18 17:33:30 -05:00
Abdessamad Derraz
ae3dc63bc3
RomM's verify_file_hashes compares the size, then md5, sha1 or crc. We compared md5 alone, so a fixture entry with one mistyped hash was left out of the pack RomM would accept it from. The rule is declared in romm.yml and read by verify, the builder and the manifest.
238 lines
7.4 KiB
Python
238 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Scraper for RomM BIOS requirements.
|
|
|
|
Platform contributed by @PixNyb (#37).
|
|
|
|
Source: https://github.com/rommapp/romm
|
|
Format: JSON fixture mapping "slug:filename" to {size, crc, md5, sha1}
|
|
Hash: SHA1 primary (all four hashes available per entry)
|
|
|
|
RomM stores known BIOS hashes in known_bios_files.json. At startup, the
|
|
fixture is loaded into Redis. When scanning or uploading firmware, RomM
|
|
verifies: file size must match AND at least one hash (MD5, SHA1, CRC32)
|
|
must match (firmware.py:verify_file_hashes).
|
|
|
|
RomM hashes files as opaque blobs (no ZIP content inspection). Arcade
|
|
BIOS ZIPs are matched by their container hash, which varies by MAME
|
|
version and ZIP tool. This is a known limitation (rommapp/romm#2888).
|
|
|
|
Folder structure: {library}/bios/{platform_slug}/{filename} (flat).
|
|
Slugs are IGDB-style platform identifiers.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
|
|
try:
|
|
from .base_scraper import (
|
|
BaseScraper,
|
|
BiosRequirement,
|
|
fetch_github_latest_version,
|
|
requirement_entry,
|
|
)
|
|
except ImportError:
|
|
from base_scraper import (
|
|
BaseScraper,
|
|
BiosRequirement,
|
|
fetch_github_latest_version,
|
|
requirement_entry,
|
|
)
|
|
|
|
PLATFORM_NAME = "romm"
|
|
|
|
GITHUB_REPO = "rommapp/romm"
|
|
|
|
SOURCE_URL = (
|
|
"https://raw.githubusercontent.com/rommapp/romm/{tag}"
|
|
"/backend/models/fixtures/known_bios_files.json"
|
|
)
|
|
|
|
# IGDB slug -> retrobios system ID
|
|
SLUG_MAP: dict[str, str] = {
|
|
"3do": "3do",
|
|
"64dd": "nintendo-64dd",
|
|
"acpc": "amstrad-cpc",
|
|
"amiga": "commodore-amiga",
|
|
"arcade": "arcade",
|
|
"atari-st": "atari-st",
|
|
"atari5200": "atari-5200",
|
|
"atari7800": "atari-7800",
|
|
"atari8bit": "atari-400-800",
|
|
"colecovision": "coleco-colecovision",
|
|
"dc": "sega-dreamcast",
|
|
"doom": "doom",
|
|
"enterprise": "enterprise-64-128",
|
|
"fairchild-channel-f": "fairchild-channel-f",
|
|
"fds": "nintendo-fds",
|
|
"gamegear": "sega-game-gear",
|
|
"gb": "nintendo-gb",
|
|
"gba": "nintendo-gba",
|
|
"gbc": "nintendo-gbc",
|
|
"genesis": "sega-mega-drive",
|
|
"intellivision": "mattel-intellivision",
|
|
"j2me": "j2me",
|
|
"lynx": "atari-lynx",
|
|
"mac": "apple-macintosh-ii",
|
|
"msx": "microsoft-msx",
|
|
"msx2": "microsoft-msx",
|
|
"nds": "nintendo-ds",
|
|
"neo-geo-cd": "snk-neogeo-cd",
|
|
"nes": "nintendo-nes",
|
|
"ngc": "nintendo-gamecube",
|
|
"odyssey-2-slash-videopac-g7000": "magnavox-odyssey2",
|
|
"pc-9800-series": "nec-pc-98",
|
|
"pc-fx": "nec-pc-fx",
|
|
"pokemon-mini": "nintendo-pokemon-mini",
|
|
"ps2": "sony-playstation-2",
|
|
"psp": "sony-psp",
|
|
"psx": "sony-playstation",
|
|
"satellaview": "nintendo-satellaview",
|
|
"saturn": "sega-saturn",
|
|
"scummvm": "scummvm",
|
|
"segacd": "sega-mega-cd",
|
|
"sharp-x68000": "sharp-x68000",
|
|
"sms": "sega-master-system",
|
|
"snes": "nintendo-snes",
|
|
"sufami-turbo": "nintendo-sufami-turbo",
|
|
"super-gb": "nintendo-sgb",
|
|
"tg16": "nec-pc-engine",
|
|
"turbografx-cd": "nec-pc-engine-cd",
|
|
"tvc": "videoton-tvc",
|
|
"videopac-g7400": "philips-videopac",
|
|
"wolfenstein": "wolfenstein-3d",
|
|
"x1": "sharp-x1",
|
|
"xbox": "microsoft-xbox",
|
|
"zxs": "sinclair-zx-spectrum",
|
|
}
|
|
|
|
|
|
# Slugs whose player runs the same core as another slug and reads firmware
|
|
# from its own folder. RomM keys the fixture by tg16 only, but its player maps
|
|
# both tg16 and turbografx-cd to mednafen_pce (frontend/src/utils/index.ts:540-541)
|
|
# and lists firmware per platform folder (backend/handler/filesystem/
|
|
# firmware_handler.py get_firmware_fs_structure), so a CD library without the
|
|
# system cards under its own slug cannot boot a disc.
|
|
FIRMWARE_MIRRORS: dict[str, tuple[str, ...]] = {
|
|
"tg16": ("turbografx-cd",),
|
|
}
|
|
|
|
|
|
class Scraper(BaseScraper):
|
|
"""Scraper for RomM known_bios_files.json."""
|
|
|
|
def __init__(self) -> None:
|
|
self.tag = fetch_github_latest_version(GITHUB_REPO)
|
|
super().__init__(url=SOURCE_URL.format(tag=self.tag))
|
|
self._parsed: dict | None = None
|
|
|
|
def _parse_json(self) -> dict:
|
|
if self._parsed is not None:
|
|
return self._parsed
|
|
|
|
raw = self._fetch_raw()
|
|
try:
|
|
self._parsed = json.loads(raw)
|
|
except json.JSONDecodeError as e:
|
|
raise ValueError(f"Failed to parse JSON: {e}") from e
|
|
return self._parsed
|
|
|
|
def fetch_requirements(self) -> list[BiosRequirement]:
|
|
"""Parse known_bios_files.json and return BIOS requirements."""
|
|
raw = self._fetch_raw()
|
|
|
|
if not self.validate_format(raw):
|
|
raise ValueError("known_bios_files.json format validation failed")
|
|
|
|
data = self._parse_json()
|
|
requirements = []
|
|
|
|
for key, entry in data.items():
|
|
if ":" not in key:
|
|
continue
|
|
|
|
igdb_slug, filename = key.split(":", 1)
|
|
system = SLUG_MAP.get(igdb_slug)
|
|
if not system:
|
|
print(f"Warning: unmapped IGDB slug '{igdb_slug}'", file=sys.stderr)
|
|
continue
|
|
|
|
sha1 = (entry.get("sha1") or "").strip() or None
|
|
md5 = (entry.get("md5") or "").strip() or None
|
|
crc32 = (entry.get("crc") or "").strip() or None
|
|
size = int(entry["size"]) if entry.get("size") else None
|
|
|
|
for slug in (igdb_slug, *FIRMWARE_MIRRORS.get(igdb_slug, ())):
|
|
requirements.append(
|
|
BiosRequirement(
|
|
name=filename,
|
|
system=SLUG_MAP[slug],
|
|
sha1=sha1,
|
|
md5=md5,
|
|
crc32=crc32,
|
|
size=size,
|
|
destination=f"{slug}/{filename}",
|
|
required=True,
|
|
native_id=slug,
|
|
mirror_of=None if slug == igdb_slug else igdb_slug,
|
|
)
|
|
)
|
|
|
|
return requirements
|
|
|
|
def validate_format(self, raw_data: str) -> bool:
|
|
"""Validate that raw_data is a JSON dict with slug:filename keys."""
|
|
try:
|
|
data = json.loads(raw_data)
|
|
except (json.JSONDecodeError, TypeError):
|
|
return False
|
|
|
|
if not isinstance(data, dict):
|
|
return False
|
|
|
|
for key in list(data.keys())[:5]:
|
|
if ":" not in key:
|
|
return False
|
|
if not isinstance(data[key], dict):
|
|
return False
|
|
if "md5" not in data[key] and "sha1" not in data[key]:
|
|
return False
|
|
|
|
return len(data) > 0
|
|
|
|
def generate_platform_yaml(self) -> dict:
|
|
"""Generate a platform YAML config dict from scraped data."""
|
|
requirements = self.fetch_requirements()
|
|
|
|
systems: dict[str, dict] = {}
|
|
for req in requirements:
|
|
if req.system not in systems:
|
|
systems[req.system] = {"files": []}
|
|
|
|
systems[req.system]["files"].append(requirement_entry(req))
|
|
|
|
return {
|
|
"inherits": "emulatorjs",
|
|
"platform": "RomM",
|
|
"version": self.tag,
|
|
"homepage": "https://romm.app",
|
|
"source": self.url,
|
|
"base_destination": "bios",
|
|
"hash_type": "sha1",
|
|
"verification_mode": "md5",
|
|
# Firmware.verify_file_hashes: size, then md5, sha1 or crc.
|
|
"hash_rule": "size_and_any_hash",
|
|
"systems": systems,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
from scripts.scraper.base_scraper import scraper_cli
|
|
|
|
scraper_cli(Scraper, "Scrape RomM BIOS requirements")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|