mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-20 18:59:07 -05:00
Abdessamad Derraz
4231902a6c
Profiles, the wiki, comments, docstrings, command output and schema descriptions carried em-dashes, clause-joining semicolons, unicode arrows, box-drawing banners and check-mark glyphs. Each sentence was rewritten rather than having its character swapped. Hashes, paths, source_ref values and every cited file:line are unchanged, which a comparison of each parsed profile against the previous revision confirms. validate_pr comments now say PASS and FAIL in words, and source_ref values in a profile's structured blocks render as code. test_prose_style fails if any of these comes back, in tracked files or in the generated site.
379 lines
15 KiB
Python
379 lines
15 KiB
Python
"""Exporter for RetroDECK's component manifests.
|
|
|
|
RetroDECK has no single BIOS file. Each component carries its own
|
|
component_manifest.json, and the BIOS list sits inside it next to the
|
|
component's name, description and presets, at one of three keys. Only that
|
|
list is rewritten, in the component's own file, so everything else the
|
|
manifest drives is left alone.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections import OrderedDict
|
|
|
|
from .base_exporter import BaseExporter
|
|
from .baseline import NativeFile, NativeSystem, Report
|
|
|
|
COMPONENTS_REPO = "RetroDECK/components"
|
|
COMPONENTS_BRANCH = "main"
|
|
RAW_BASE = f"https://raw.githubusercontent.com/{COMPONENTS_REPO}/{COMPONENTS_BRANCH}"
|
|
MANIFEST = "component_manifest.json"
|
|
# Labels that only say yes or no, so a corrected requirement can be written
|
|
# over them. Every other label is a sentence ("At least one BIOS file
|
|
# required", "Required for some Japanese games.") and is the platform's own.
|
|
PLAIN_LABELS = ("Required", "Optional")
|
|
|
|
|
|
def _systems_of(entry: dict) -> list[str]:
|
|
"""The systems one entry serves: neogeo.zip is declared once for three."""
|
|
declared = entry.get("system")
|
|
if isinstance(declared, list):
|
|
return [str(s) for s in declared]
|
|
return [str(declared)] if declared else []
|
|
|
|
|
|
def _md5_set(value: object) -> set[str]:
|
|
values = value if isinstance(value, list) else str(value or "").split(",")
|
|
return {str(m).strip().lower() for m in values if str(m).strip()}
|
|
|
|
|
|
def _paths_key(entry: dict) -> tuple[str, ...]:
|
|
"""Where an entry says the file goes: no paths is the BIOS root."""
|
|
paths = entry.get("paths")
|
|
if isinstance(paths, list):
|
|
return tuple(sorted(str(p) for p in paths))
|
|
return (str(paths),) if paths else ()
|
|
|
|
|
|
def _matching_keys(
|
|
entry: dict, by_key: dict[tuple[str, str], OrderedDict]
|
|
) -> list[tuple[str, str]]:
|
|
"""Our keys this platform entry answers to.
|
|
|
|
An entry without a system matches ours only when one system alone
|
|
declares the name.
|
|
"""
|
|
name = str(entry.get("filename", ""))
|
|
systems = _systems_of(entry)
|
|
if systems:
|
|
return [(name, s) for s in systems if (name, s) in by_key]
|
|
named = [k for k in by_key if k[0] == name]
|
|
return named if len(named) == 1 else []
|
|
|
|
|
|
class Exporter(BaseExporter):
|
|
"""Write RetroDECK's component manifests, corrected."""
|
|
|
|
@staticmethod
|
|
def platform_name() -> str:
|
|
return "retrodeck"
|
|
|
|
@staticmethod
|
|
def native_filename() -> str:
|
|
return MANIFEST
|
|
|
|
@staticmethod
|
|
def carries() -> frozenset[str]:
|
|
return frozenset({"md5", "sha256", "required"})
|
|
|
|
@staticmethod
|
|
def needs_original() -> bool:
|
|
# A manifest is mostly presets and launch configuration, so rebuilding
|
|
# one from BIOS data alone would throw the component away.
|
|
return True
|
|
|
|
@classmethod
|
|
def writable(cls, fe: NativeFile, require: str = "") -> bool:
|
|
"""An addition lands only in a manifest it can be placed in."""
|
|
if fe.platform is None and not fe.native("component", ""):
|
|
return False
|
|
return super().writable(fe, require)
|
|
|
|
def states(self, fe: NativeFile, field_name: str) -> bool:
|
|
if id(fe) in getattr(self, "_appended", ()):
|
|
return False
|
|
if field_name == "required":
|
|
label = str(fe.native("required_label", ""))
|
|
return not label or label in PLAIN_LABELS
|
|
return super().states(fe, field_name)
|
|
|
|
@staticmethod
|
|
def _place_additions(systems: dict[str, NativeSystem]) -> None:
|
|
"""Give each addition the component its system already lives in.
|
|
|
|
A component is one emulator's manifest, and only the platform's
|
|
entries name it. A system whose entries all sit in one component
|
|
takes the addition there. One spread over several, or with none of
|
|
its own, cannot say which manifest is meant, and the addition is
|
|
left out and counted as such.
|
|
"""
|
|
for system in systems.values():
|
|
components = {
|
|
str(fe.native("component", ""))
|
|
for fe in system.files
|
|
if fe.platform is not None and fe.native("component", "")
|
|
}
|
|
if len(components) != 1:
|
|
continue
|
|
component = components.pop()
|
|
for fe in system.files:
|
|
if fe.platform is None and fe.truth is not None:
|
|
fe.truth = {**fe.truth, "component": component}
|
|
|
|
@staticmethod
|
|
def component_url(component: str) -> str:
|
|
return f"{RAW_BASE}/{component}/{MANIFEST}"
|
|
|
|
def components(self, systems: dict[str, NativeSystem]) -> list[str]:
|
|
"""Components the corrected data touches."""
|
|
found: set[str] = set()
|
|
for system in systems.values():
|
|
for fe in system.files:
|
|
component = str(fe.native("component", ""))
|
|
if component:
|
|
found.add(component)
|
|
return sorted(found)
|
|
|
|
@staticmethod
|
|
def _entry(fe: NativeFile) -> OrderedDict:
|
|
entry: OrderedDict[str, object] = OrderedDict()
|
|
entry["filename"] = fe.name
|
|
# Several accepted dumps stay a JSON array, as RetroDECK writes them:
|
|
# a comma-joined string reads back as one value that is no md5.
|
|
md5 = fe.hashes("md5")
|
|
if md5:
|
|
entry["md5"] = md5 if len(md5) > 1 else md5[0]
|
|
sha256 = fe.hash("sha256")
|
|
if sha256:
|
|
entry["sha256"] = sha256
|
|
entry["system"] = fe.native_system
|
|
description = fe.native("description", "")
|
|
if description:
|
|
entry["description"] = str(description)
|
|
# RetroDECK words the requirement in prose ("Required", "At least one
|
|
# BIOS file required"), so the platform's own wording is kept and a
|
|
# boolean is only rendered when there is none to keep.
|
|
label = str(fe.native("required_label", ""))
|
|
if label and label not in PLAIN_LABELS:
|
|
entry["required"] = label
|
|
elif fe.required:
|
|
entry["required"] = "Required"
|
|
elif label:
|
|
entry["required"] = "Optional"
|
|
destination = fe.destination
|
|
if destination and destination not in (fe.name, f"bios/{fe.name}"):
|
|
directory = destination.rsplit("/", 1)[0]
|
|
entry["paths"] = "$bios_path/" + directory.removeprefix("bios/")
|
|
return entry
|
|
|
|
def _by_component(
|
|
self, systems: dict[str, NativeSystem]
|
|
) -> dict[str, list[NativeFile]]:
|
|
grouped: dict[str, list[NativeFile]] = {}
|
|
for system in systems.values():
|
|
for fe in system.files:
|
|
component = str(fe.native("component", ""))
|
|
if component:
|
|
grouped.setdefault(component, []).append(fe)
|
|
return grouped
|
|
|
|
@staticmethod
|
|
def _bios_holder(component_value: dict) -> tuple[dict, str] | None:
|
|
"""Where in a manifest the BIOS list lives, if it has one."""
|
|
if "bios" in component_value:
|
|
return component_value, "bios"
|
|
for key in ("preset_actions", "cores"):
|
|
nested = component_value.get(key)
|
|
if isinstance(nested, dict) and "bios" in nested:
|
|
return nested, "bios"
|
|
return None
|
|
|
|
@classmethod
|
|
def _merge(
|
|
cls,
|
|
existing: object,
|
|
ours: list[OrderedDict],
|
|
additions: list[OrderedDict] | None = None,
|
|
appended: set[tuple[str, str]] | None = None,
|
|
) -> list[OrderedDict]:
|
|
"""Correct the component's own list, never replace it.
|
|
|
|
Assigning our entries wholesale dropped every file RetroDECK declares
|
|
that our model does not carry -- 177 of them across the components.
|
|
An entry the platform declares is kept, its fields corrected where the
|
|
truth has something to say and left alone where it does not, and what
|
|
the platform does not declare is appended.
|
|
"""
|
|
# Keyed by name AND system: the retroarch manifest declares
|
|
# ATARIOSB.ROM for atari5200 and atari800 with different md5 lists,
|
|
# and a name-only key wrote the first of ours over both.
|
|
by_key: OrderedDict[tuple[str, str], OrderedDict] = OrderedDict()
|
|
for entry in ours:
|
|
key = (str(entry.get("filename", "")), str(entry.get("system", "")))
|
|
if key[0] and key not in by_key:
|
|
by_key[key] = entry
|
|
|
|
merged: list[OrderedDict] = []
|
|
corrected: set[tuple[str, str]] = set()
|
|
entries = [e for e in existing if isinstance(e, dict)] if isinstance(existing, list) else []
|
|
# A name and system declared several times are several revisions (the
|
|
# 64DD IPL USA and Dev): one entry of ours corrects the revision it
|
|
# shares a hash with, never all of them.
|
|
seen_keys: dict[tuple[str, str], int] = {}
|
|
for entry in entries:
|
|
for system in _systems_of(entry):
|
|
key = (str(entry.get("filename", "")), system)
|
|
seen_keys[key] = seen_keys.get(key, 0) + 1
|
|
# A hash no revision shares is a revision the platform does not
|
|
# know: appended, not counted corrected against entries kept as
|
|
# they were (the 64DD IPL declared twice, both contradicted).
|
|
for entry in entries:
|
|
keys = _matching_keys(entry, by_key)
|
|
other_revision = (
|
|
keys
|
|
and seen_keys.get(keys[0], 0) > 1
|
|
and not _md5_set(entry.get("md5")) & _md5_set(by_key[keys[0]].get("md5"))
|
|
)
|
|
if other_revision or not keys:
|
|
merged.append(OrderedDict(entry))
|
|
continue
|
|
declared = entry.get("system")
|
|
combined = OrderedDict(entry)
|
|
# Only what the format says it corrects: the maintainer's
|
|
# description and search paths are theirs (neogeo.zip lost its
|
|
# four roms directories to a path that does not exist).
|
|
combined.update(
|
|
(field, value) for field, value in by_key[keys[0]].items()
|
|
if field in cls.carries() and not (field == "system" and declared)
|
|
)
|
|
# A hash of theirs that ours no longer carries is one the truth
|
|
# contradicts: kept, it went on rejecting the dump the emulator
|
|
# loads while the summary counted it cleared.
|
|
for field in ("md5", "sha256"):
|
|
if field in cls.carries() and field not in by_key[keys[0]]:
|
|
combined.pop(field, None)
|
|
if _md5_set(entry.get("md5")) == _md5_set(combined.get("md5")):
|
|
combined["md5"] = entry.get("md5")
|
|
elif isinstance(entry.get("md5"), list) and isinstance(combined.get("md5"), str):
|
|
combined["md5"] = [combined["md5"]]
|
|
merged.append(combined)
|
|
corrected.update(keys)
|
|
|
|
uncorrected = [key for key in by_key if key not in corrected]
|
|
merged.extend(by_key[key] for key in uncorrected)
|
|
if appended is not None:
|
|
appended.update(uncorrected)
|
|
# A file the platform does not declare is appended unless the list
|
|
# already holds that name for that system at that path. The key
|
|
# above is name and system only, and folded melonDS's SkyEmu/
|
|
# firmware.bin into the root firmware.bin the platform declares,
|
|
# while the report counted it added.
|
|
held = {
|
|
(str(e.get("filename", "")), system, _paths_key(e))
|
|
for e in merged
|
|
for system in (_systems_of(e) or [""])
|
|
}
|
|
for entry in additions or []:
|
|
key = (str(entry.get("filename", "")), str(entry.get("system", "")), _paths_key(entry))
|
|
if key[0] and key not in held:
|
|
held.add(key)
|
|
merged.append(entry)
|
|
return merged
|
|
|
|
def render(
|
|
self,
|
|
systems: dict[str, NativeSystem],
|
|
report: Report,
|
|
originals: dict[str, str],
|
|
scraped: dict | None = None,
|
|
) -> dict[str, str]:
|
|
self._place_additions(systems)
|
|
self._appended: set[int] = set()
|
|
grouped = self._by_component(systems)
|
|
produced: dict[str, str] = {}
|
|
|
|
for component, files in sorted(grouped.items()):
|
|
path = f"{component}/{MANIFEST}"
|
|
original = originals.get(path)
|
|
if not original:
|
|
continue
|
|
try:
|
|
manifest = json.loads(original, object_pairs_hook=OrderedDict)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
entries = [self._entry(fe) for fe in files if fe.platform is not None]
|
|
additions = [self._entry(fe) for fe in files if fe.platform is None]
|
|
appended: set[tuple[str, str]] = set()
|
|
for component_value in manifest.values():
|
|
if not isinstance(component_value, dict):
|
|
continue
|
|
holder = self._bios_holder(component_value)
|
|
if holder is None:
|
|
component_value["bios"] = self._merge(
|
|
None, entries, additions, appended
|
|
)
|
|
else:
|
|
container, key = holder
|
|
container[key] = self._merge(
|
|
container.get(key), entries, additions, appended
|
|
)
|
|
break
|
|
# A declared entry whose revision the platform does not hold is
|
|
# written beside theirs: a new line, not a corrected value.
|
|
self._appended.update(
|
|
id(fe) for fe in files
|
|
if fe.platform is not None
|
|
and (str(fe.name), str(fe.native_system)) in appended
|
|
)
|
|
|
|
produced[path] = json.dumps(manifest, indent=2, ensure_ascii=False) + "\n"
|
|
|
|
return produced
|
|
|
|
def validate(
|
|
self,
|
|
systems: dict[str, NativeSystem],
|
|
produced: dict[str, str],
|
|
) -> list[str]:
|
|
issues: list[str] = []
|
|
grouped = self._by_component(systems)
|
|
|
|
for component, files in grouped.items():
|
|
path = f"{component}/{MANIFEST}"
|
|
if path not in produced:
|
|
issues.append(f"manifest not written: {path}")
|
|
continue
|
|
try:
|
|
manifest = json.loads(produced[path])
|
|
except json.JSONDecodeError as exc:
|
|
issues.append(f"{path} does not parse: {exc}")
|
|
continue
|
|
|
|
declared: set[str] = set()
|
|
placed: set[tuple[str, tuple[str, ...]]] = set()
|
|
for component_value in manifest.values():
|
|
if not isinstance(component_value, dict):
|
|
continue
|
|
holder = self._bios_holder(component_value)
|
|
if holder is None:
|
|
continue
|
|
container, key = holder
|
|
for entry in container[key]:
|
|
declared.add(entry.get("filename", ""))
|
|
placed.add((entry.get("filename", ""), _paths_key(entry)))
|
|
if not component_value.get("name") and not component_value.get(
|
|
"system"
|
|
):
|
|
issues.append(f"{path}: the component lost its identity")
|
|
|
|
for fe in files:
|
|
if fe.name not in declared:
|
|
issues.append(f"absent from {path}: {fe.name}")
|
|
elif fe.platform is None and (
|
|
fe.name, _paths_key(self._entry(fe))
|
|
) not in placed:
|
|
issues.append(f"absent from {path}: {fe.destination or fe.name}")
|
|
return issues
|