mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-21 03:09:08 -05:00
Abdessamad Derraz
4231902a6c
Profiles, the wiki, comments, docstrings, command output and schema descriptions carried em-dashes, clause-joining semicolons, unicode arrows, box-drawing banners and check-mark glyphs. Each sentence was rewritten rather than having its character swapped. Hashes, paths, source_ref values and every cited file:line are unchanged, which a comparison of each parsed profile against the previous revision confirms. validate_pr comments now say PASS and FAIL in words, and source_ref values in a profile's structured blocks render as code. test_prose_style fails if any of these comes back, in tracked files or in the generated site.
213 lines
8.5 KiB
Python
213 lines
8.5 KiB
Python
"""Exporter for RomM's known_bios_files.json.
|
|
|
|
Keys are "<igdb slug>:<filename>". RomM verifies a firmware file with
|
|
`file_size_bytes == int(entry.get("size", 0))` and then one hash among md5,
|
|
sha1 and crc, so an entry without a size can never match and an entry
|
|
without a hash can never match either. Neither is written.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from collections import OrderedDict
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from scraper.romm_scraper import SLUG_MAP
|
|
|
|
from .base_exporter import BaseExporter
|
|
from .baseline import NativeFile, NativeSystem, Report, _hash_values
|
|
|
|
SOURCE_URL = (
|
|
"https://raw.githubusercontent.com/rommapp/romm/master/backend/models"
|
|
"/fixtures/known_bios_files.json"
|
|
)
|
|
|
|
|
|
class Exporter(BaseExporter):
|
|
"""Write RomM's known_bios_files.json, corrected."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
# (system, filename) keys the platform itself declares, and the
|
|
# addition that claims each other key first: both set by render.
|
|
self._platform_keys: set[tuple[str, str]] = set()
|
|
self._first_addition: dict[tuple[str, str], int] = {}
|
|
|
|
@staticmethod
|
|
def platform_name() -> str:
|
|
return "romm"
|
|
|
|
@staticmethod
|
|
def native_filename() -> str:
|
|
return "known_bios_files.json"
|
|
|
|
@staticmethod
|
|
def carries() -> frozenset[str]:
|
|
return frozenset({"size", "crc32", "md5", "sha1"})
|
|
|
|
@staticmethod
|
|
def native_sources() -> dict[str, str]:
|
|
return {"known_bios_files.json": SOURCE_URL}
|
|
|
|
@staticmethod
|
|
def _verifiable(fe: NativeFile) -> bool:
|
|
return bool(fe.size()) and any(
|
|
fe.hash(h) for h in ("md5", "sha1", "crc32")
|
|
)
|
|
|
|
@staticmethod
|
|
def _known_platform(native_id: str) -> bool:
|
|
"""RomM keys by IGDB platform slug, and only looks up its own.
|
|
|
|
A key spelled with one of our slugs (capcom-cps3, snk-neogeo-mvs)
|
|
matches nothing on their side, so it is reported rather than
|
|
written.
|
|
"""
|
|
return native_id in SLUG_MAP
|
|
|
|
def writable(self, fe: NativeFile, require: str = "") -> bool: # noqa: ARG002
|
|
"""What RomM already ships stays. The conditions gate additions.
|
|
|
|
An entry of theirs that could never verify is still theirs, and the
|
|
round trip is not the place to decide otherwise. An addition whose
|
|
slug:filename key the platform already holds cannot be written: the
|
|
fixture is a dict, and the second write replaced RomM's Dreamcast
|
|
boot.bin with fbneo's 480-byte file of the same name.
|
|
"""
|
|
if fe.platform is not None:
|
|
return True
|
|
key = (fe.native_system, fe.name)
|
|
if key in self._platform_keys:
|
|
return False
|
|
if not (self._verifiable(fe) and self._known_platform(fe.native_system)):
|
|
return False
|
|
# Two additions under one key: the second would overwrite the first
|
|
# in the dict and both were counted landed. The first one in render
|
|
# order is written, the other is not.
|
|
return self._first_addition.get(key, id(fe)) == id(fe)
|
|
|
|
def render(
|
|
self,
|
|
systems: dict[str, NativeSystem],
|
|
report: Report,
|
|
originals: dict[str, str],
|
|
scraped: dict | None = None,
|
|
) -> dict[str, str]:
|
|
# RomM's own fixture is the base: its key order stays, and a key our
|
|
# transcription lacks (added after the scrape, or filtered from the
|
|
# model) is kept. Rebuilt from the model in our order, three hash
|
|
# corrections came out as a diff of the whole file.
|
|
original = originals.get(self.native_filename(), "")
|
|
output: OrderedDict[str, OrderedDict] = (
|
|
json.loads(original, object_pairs_hook=OrderedDict)
|
|
if original else OrderedDict()
|
|
)
|
|
self._platform_keys = {
|
|
(system.native_id, fe.name)
|
|
for system in systems.values()
|
|
for fe in system.files
|
|
if fe.platform is not None
|
|
}
|
|
self._first_addition = {}
|
|
self.normalized: list[str] = []
|
|
for system in sorted(systems.values(), key=lambda s: s.native_id):
|
|
for fe in sorted(system.files, key=lambda f: f.name):
|
|
key = (fe.native_system, fe.name)
|
|
if (
|
|
fe.platform is None
|
|
and key not in self._platform_keys
|
|
and self._verifiable(fe)
|
|
and self._known_platform(fe.native_system)
|
|
):
|
|
self._first_addition.setdefault(key, id(fe))
|
|
|
|
for system in sorted(systems.values(), key=lambda s: s.native_id):
|
|
for fe in sorted(system.files, key=lambda f: f.name):
|
|
if not self.writable(fe):
|
|
continue
|
|
key = f"{system.native_id}:{fe.name}"
|
|
# Their entry keeps its fields and their order, ours fill in.
|
|
entry: OrderedDict[str, str] = OrderedDict(output.get(key, {}))
|
|
# The fixture states every value as a string, size included.
|
|
# Where the model settles no size, RomM's own value stands.
|
|
# str(None) wrote "None", and RomM's int() on it raised.
|
|
size = fe.size()
|
|
if size is None and fe.platform is not None:
|
|
size = fe.platform.get("size")
|
|
if size is not None:
|
|
self._write(entry, "size", str(int(size)), fe, "size")
|
|
for field, hash_name in (("crc", "crc32"), ("md5", "md5"), ("sha1", "sha1")):
|
|
value = fe.record_hash(hash_name)
|
|
if value:
|
|
self._write(entry, field, value, fe, hash_name)
|
|
elif self._contradicted(fe, hash_name):
|
|
# RomM accepts any one hash: a value the truth
|
|
# rejects, left beside the corrected md5, still
|
|
# passes the dump it describes.
|
|
entry.pop(field, None)
|
|
output[key] = entry
|
|
|
|
text = json.dumps(output, indent=2, ensure_ascii=False) + "\n"
|
|
return {self.native_filename(): text}
|
|
|
|
@staticmethod
|
|
def _contradicted(fe: NativeFile, name: str) -> bool:
|
|
"""A hash the platform states and the truth, taking the entry over,
|
|
does not keep."""
|
|
return bool(_hash_values(fe.platform or {}, name)) and not fe.hashes(name)
|
|
|
|
def fills(self, fe: NativeFile, field_name: str) -> bool:
|
|
return self.states(fe, field_name) and self.record_fills(fe, field_name)
|
|
|
|
def _write(
|
|
self, entry: dict, field: str, value: str, fe: NativeFile, name: str
|
|
) -> None:
|
|
"""Set one field, noting a value the model reads as RomM's own.
|
|
|
|
RomM compares strings as stored: PLUTONIA.WAD's sha1 starts with a
|
|
space and never matches. The model normalises it away, so no
|
|
correction names it, yet the written file changes it.
|
|
"""
|
|
theirs = entry.get(field)
|
|
if (
|
|
theirs is not None
|
|
and str(theirs) != value
|
|
and str(theirs).strip().lower() == value
|
|
and name not in fe.corrections
|
|
):
|
|
self.normalized.append(f"{fe.native_system}/{fe.name} {name}")
|
|
entry[field] = value
|
|
|
|
def validate(
|
|
self,
|
|
systems: dict[str, NativeSystem],
|
|
produced: dict[str, str],
|
|
) -> list[str]:
|
|
try:
|
|
data = json.loads(produced[self.native_filename()])
|
|
except json.JSONDecodeError as exc:
|
|
return [f"the JSON does not parse: {exc}"]
|
|
|
|
issues: list[str] = []
|
|
for key, entry in data.items():
|
|
slug = key.split(":", 1)[0] if ":" in key else ""
|
|
if not slug:
|
|
issues.append(f"key without a platform slug: {key}")
|
|
elif not self._known_platform(slug):
|
|
issues.append(f"platform slug RomM does not know: {slug}")
|
|
if not str(entry.get("size", "")).isdigit():
|
|
issues.append(f"entry without a size, never verifiable: {key}")
|
|
if not any(entry.get(h) for h in ("md5", "sha1", "crc")):
|
|
issues.append(f"entry without a hash, never verifiable: {key}")
|
|
|
|
for system in systems.values():
|
|
for fe in system.files:
|
|
if not self.writable(fe):
|
|
continue
|
|
if f"{system.native_id}:{fe.name}" not in data:
|
|
issues.append(f"absent: {system.native_id}/{fe.name}")
|
|
return issues
|