mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-16 00:13:25 -05:00
feat: make the native export round-trip a platform file
This commit is contained in:
Abdessamad Derraz committed
1 parent
7b93285e1d
commit
691ccbfca7
65 files changed
+18417
-2115
No files matched your search
+105
-132
@@ -1,160 +1,133 @@
|
||||
"""Exporter for RomM known_bios_files.json format.
|
||||
"""Exporter for RomM's known_bios_files.json.
|
||||
|
||||
Produces JSON matching the exact format of
|
||||
rommapp/romm/backend/models/fixtures/known_bios_files.json:
|
||||
- Keys are "igdb_slug:filename"
|
||||
- Values contain size, crc, md5, sha1 (all optional but at least one hash)
|
||||
- Hashes are lowercase hex strings
|
||||
- Size is an integer
|
||||
Keys are "<igdb slug>:<filename>". RomM verifies a firmware file with
|
||||
`file_size_bytes == int(entry.get("size", 0))` and then one hash among md5,
|
||||
sha1 and crc, so an entry without a size can never match and an entry
|
||||
without a hash can never match either. Neither is written.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from pathlib import Path
|
||||
|
||||
from .base_exporter import BaseExporter
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
# retrobios slug -> IGDB slug (reverse of scraper SLUG_MAP)
|
||||
_REVERSE_SLUG: dict[str, str] = {
|
||||
"3do": "3do",
|
||||
"nintendo-64dd": "64dd",
|
||||
"amstrad-cpc": "acpc",
|
||||
"commodore-amiga": "amiga",
|
||||
"arcade": "arcade",
|
||||
"atari-st": "atari-st",
|
||||
"atari-5200": "atari5200",
|
||||
"atari-7800": "atari7800",
|
||||
"atari-400-800": "atari8bit",
|
||||
"coleco-colecovision": "colecovision",
|
||||
"sega-dreamcast": "dc",
|
||||
"doom": "doom",
|
||||
"enterprise-64-128": "enterprise",
|
||||
"fairchild-channel-f": "fairchild-channel-f",
|
||||
"nintendo-fds": "fds",
|
||||
"sega-game-gear": "gamegear",
|
||||
"nintendo-gb": "gb",
|
||||
"nintendo-gba": "gba",
|
||||
"nintendo-gbc": "gbc",
|
||||
"sega-mega-drive": "genesis",
|
||||
"mattel-intellivision": "intellivision",
|
||||
"j2me": "j2me",
|
||||
"atari-lynx": "lynx",
|
||||
"apple-macintosh-ii": "mac",
|
||||
"microsoft-msx": "msx",
|
||||
"nintendo-ds": "nds",
|
||||
"snk-neogeo-cd": "neo-geo-cd",
|
||||
"nintendo-nes": "nes",
|
||||
"nintendo-gamecube": "ngc",
|
||||
"magnavox-odyssey2": "odyssey-2-slash-videopac-g7000",
|
||||
"nec-pc-98": "pc-9800-series",
|
||||
"nec-pc-fx": "pc-fx",
|
||||
"nintendo-pokemon-mini": "pokemon-mini",
|
||||
"sony-playstation-2": "ps2",
|
||||
"sony-psp": "psp",
|
||||
"sony-playstation": "psx",
|
||||
"nintendo-satellaview": "satellaview",
|
||||
"sega-saturn": "saturn",
|
||||
"scummvm": "scummvm",
|
||||
"sega-mega-cd": "segacd",
|
||||
"sharp-x68000": "sharp-x68000",
|
||||
"sega-master-system": "sms",
|
||||
"nintendo-snes": "snes",
|
||||
"nintendo-sufami-turbo": "sufami-turbo",
|
||||
"nintendo-sgb": "super-gb",
|
||||
"nec-pc-engine": "tg16",
|
||||
"videoton-tvc": "tvc",
|
||||
"philips-videopac": "videopac-g7400",
|
||||
"wolfenstein-3d": "wolfenstein",
|
||||
"sharp-x1": "x1",
|
||||
"microsoft-xbox": "xbox",
|
||||
"sinclair-zx-spectrum": "zxs",
|
||||
}
|
||||
from scraper.romm_scraper import SLUG_MAP
|
||||
|
||||
from .base_exporter import BaseExporter
|
||||
from .baseline import NativeFile, NativeSystem, Report
|
||||
|
||||
SOURCE_URL = (
|
||||
"https://raw.githubusercontent.com/rommapp/romm/master/backend/models"
|
||||
"/fixtures/known_bios_files.json"
|
||||
)
|
||||
|
||||
|
||||
class Exporter(BaseExporter):
|
||||
"""Export truth data to RomM known_bios_files.json format."""
|
||||
"""Write RomM's known_bios_files.json, corrected."""
|
||||
|
||||
@staticmethod
|
||||
def platform_name() -> str:
|
||||
return "romm"
|
||||
|
||||
def export(
|
||||
self,
|
||||
truth_data: dict,
|
||||
output_path: str,
|
||||
scraped_data: dict | None = None,
|
||||
) -> None:
|
||||
native_map: dict[str, str] = {}
|
||||
if scraped_data:
|
||||
for sys_id, sys_data in scraped_data.get("systems", {}).items():
|
||||
nid = sys_data.get("native_id")
|
||||
if nid:
|
||||
native_map[sys_id] = nid
|
||||
@staticmethod
|
||||
def native_filename() -> str:
|
||||
return "known_bios_files.json"
|
||||
|
||||
output: OrderedDict[str, dict] = OrderedDict()
|
||||
@staticmethod
|
||||
def carries() -> frozenset[str]:
|
||||
return frozenset({"size", "crc32", "md5", "sha1"})
|
||||
|
||||
systems = truth_data.get("systems", {})
|
||||
for sys_id in sorted(systems):
|
||||
sys_data = systems[sys_id]
|
||||
files = sys_data.get("files", [])
|
||||
if not files:
|
||||
continue
|
||||
@staticmethod
|
||||
def native_sources() -> dict[str, str]:
|
||||
return {"known_bios_files.json": SOURCE_URL}
|
||||
|
||||
igdb_slug = native_map.get(sys_id, _REVERSE_SLUG.get(sys_id, sys_id))
|
||||
|
||||
for fe in files:
|
||||
name = fe.get("name", "")
|
||||
if name.startswith("_") or self._is_pattern(name):
|
||||
continue
|
||||
|
||||
key = f"{igdb_slug}:{name}"
|
||||
|
||||
entry: OrderedDict[str, object] = OrderedDict()
|
||||
|
||||
size = fe.get("size")
|
||||
if size is not None:
|
||||
entry["size"] = int(size)
|
||||
|
||||
crc = fe.get("crc32", "")
|
||||
if crc:
|
||||
entry["crc"] = str(crc).strip().lower()
|
||||
|
||||
md5 = fe.get("md5", "")
|
||||
if isinstance(md5, list):
|
||||
md5 = md5[0] if md5 else ""
|
||||
if md5:
|
||||
entry["md5"] = str(md5).strip().lower()
|
||||
|
||||
sha1 = fe.get("sha1", "")
|
||||
if isinstance(sha1, list):
|
||||
sha1 = sha1[0] if sha1 else ""
|
||||
if sha1:
|
||||
entry["sha1"] = str(sha1).strip().lower()
|
||||
|
||||
output[key] = entry
|
||||
|
||||
Path(output_path).write_text(
|
||||
json.dumps(output, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
@staticmethod
|
||||
def _verifiable(fe: NativeFile) -> bool:
|
||||
return bool(fe.size()) and any(
|
||||
fe.hash(h) for h in ("md5", "sha1", "crc32")
|
||||
)
|
||||
|
||||
def validate(self, truth_data: dict, output_path: str) -> list[str]:
|
||||
data = json.loads(Path(output_path).read_text(encoding="utf-8"))
|
||||
@staticmethod
|
||||
def _known_platform(native_id: str) -> bool:
|
||||
"""RomM keys by IGDB platform slug, and only looks up its own.
|
||||
|
||||
exported_names: set[str] = set()
|
||||
for key in data:
|
||||
if ":" in key:
|
||||
_, filename = key.split(":", 1)
|
||||
exported_names.add(filename)
|
||||
A key spelled with one of our slugs (capcom-cps3, snk-neogeo-mvs)
|
||||
matches nothing on their side, so it is reported rather than
|
||||
written.
|
||||
"""
|
||||
return native_id in SLUG_MAP
|
||||
|
||||
@classmethod
|
||||
def writable(cls, fe: NativeFile, require: str = "") -> bool:
|
||||
"""What RomM already ships stays; the conditions gate additions.
|
||||
|
||||
An entry of theirs that could never verify is still theirs, and the
|
||||
round trip is not the place to decide otherwise.
|
||||
"""
|
||||
if fe.platform is not None:
|
||||
return True
|
||||
return cls._verifiable(fe) and cls._known_platform(fe.native_system)
|
||||
|
||||
def render(
|
||||
self,
|
||||
systems: dict[str, NativeSystem],
|
||||
report: Report,
|
||||
originals: dict[str, str],
|
||||
scraped: dict | None = None,
|
||||
) -> dict[str, str]:
|
||||
output: OrderedDict[str, dict] = OrderedDict()
|
||||
|
||||
for system in sorted(systems.values(), key=lambda s: s.native_id):
|
||||
for fe in sorted(system.files, key=lambda f: f.name):
|
||||
if not self.writable(fe):
|
||||
continue
|
||||
entry: OrderedDict[str, str] = OrderedDict()
|
||||
# The fixture states every value as a string, size included.
|
||||
entry["size"] = str(fe.size())
|
||||
crc = fe.hash("crc32")
|
||||
if crc:
|
||||
entry["crc"] = crc
|
||||
md5 = fe.hash("md5")
|
||||
if md5:
|
||||
entry["md5"] = md5
|
||||
sha1 = fe.hash("sha1")
|
||||
if sha1:
|
||||
entry["sha1"] = sha1
|
||||
output[f"{system.native_id}:{fe.name}"] = entry
|
||||
|
||||
text = json.dumps(output, indent=2, ensure_ascii=False) + "\n"
|
||||
return {self.native_filename(): text}
|
||||
|
||||
def validate(
|
||||
self,
|
||||
systems: dict[str, NativeSystem],
|
||||
produced: dict[str, str],
|
||||
) -> list[str]:
|
||||
try:
|
||||
data = json.loads(produced[self.native_filename()])
|
||||
except json.JSONDecodeError as exc:
|
||||
return [f"the JSON does not parse: {exc}"]
|
||||
|
||||
issues: list[str] = []
|
||||
for sys_data in truth_data.get("systems", {}).values():
|
||||
for fe in sys_data.get("files", []):
|
||||
name = fe.get("name", "")
|
||||
if name.startswith("_") or self._is_pattern(name):
|
||||
for key, entry in data.items():
|
||||
slug = key.split(":", 1)[0] if ":" in key else ""
|
||||
if not slug:
|
||||
issues.append(f"key without a platform slug: {key}")
|
||||
elif not self._known_platform(slug):
|
||||
issues.append(f"platform slug RomM does not know: {slug}")
|
||||
if not entry.get("size"):
|
||||
issues.append(f"entry without a size, never verifiable: {key}")
|
||||
if not any(entry.get(h) for h in ("md5", "sha1", "crc")):
|
||||
issues.append(f"entry without a hash, never verifiable: {key}")
|
||||
|
||||
for system in systems.values():
|
||||
for fe in system.files:
|
||||
if not self.writable(fe):
|
||||
continue
|
||||
if name not in exported_names:
|
||||
issues.append(f"missing: {name}")
|
||||
if f"{system.native_id}:{fe.name}" not in data:
|
||||
issues.append(f"absent: {system.native_id}/{fe.name}")
|
||||
return issues
|
||||
Reference in new issue
Block a user