mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-21 11:19:11 -05:00
Abdessamad Derraz
540374b40d
The removal candidates looked for a reading in the profiles only, so a name the platform's own launch code opens, links or copies was listed as read by no core: Batocera's DuckStation generator walks 22 names (duckstationGenerator.py:501-515). Every name a launcher record names is now a reading on that platform. Recalbox's unexplained entries drop from 20 to 13, RetroBat's from 24 to 20, EmuDeck's from 25 to 22.
448 lines
20 KiB
Python
448 lines
20 KiB
Python
"""The corrected platform files are published with every change named.
|
|
|
|
export_native printed five examples per kind of change and nothing else
|
|
recorded the rest: a platform maintainer had no page to fetch the corrected
|
|
file from and no list to check the diff against.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
|
|
|
import export_native # noqa: E402
|
|
from siteexports import generate_exports_page, load_export_reports # noqa: E402
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def _report(platform: str, files: list[str], **changes: object) -> dict:
|
|
report = {
|
|
"platform": platform,
|
|
"originals": {f: f"https://example.org/{platform}/{f}" for f in files},
|
|
"files": files,
|
|
"summary": "3 kept, 1 added, 7 hashes corrected, 0 requirements corrected",
|
|
"hashes_corrected": [], "hashes_filled": [], "sizes_corrected": [],
|
|
"sizes_removed": [], "hashes_cleared": [], "requirements_corrected": 0,
|
|
"withheld": [], "invalid": [],
|
|
}
|
|
report.update(changes)
|
|
return report
|
|
|
|
|
|
class ExportReportIsWrittenAndRead(unittest.TestCase):
|
|
def test_report_round_trips_through_the_page_loader(self):
|
|
with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as scratch:
|
|
report = _report("recalbox", ["es_bios.xml"])
|
|
export_native.write_report(Path(scratch) / "recalbox.json", report)
|
|
(Path(scratch) / "recalbox").mkdir()
|
|
self.assertEqual(load_export_reports(scratch), [report])
|
|
|
|
def test_a_report_goes_into_a_directory_not_yet_made(self):
|
|
"""RetroPie with nothing left to correct writes no file, so its
|
|
output directory was never made and the report crashed --all."""
|
|
with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as scratch:
|
|
target = Path(scratch) / "fresh" / "retropie.json"
|
|
export_native.write_report(target, _report("retropie", []))
|
|
self.assertTrue(target.exists())
|
|
|
|
def test_export_keeps_every_change_not_the_printed_five(self):
|
|
source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8")
|
|
body = source[source.index("def export_platform("):source.index("def write_report(")]
|
|
call = body[body.index("write_report("):]
|
|
for key, value in (
|
|
("hashes_corrected", "applied"), ("hashes_filled", "filled"),
|
|
("sizes_corrected", "resized"), ("sizes_removed", "unsized"),
|
|
("hashes_cleared", "cleared"), ("invalid", "issues"),
|
|
):
|
|
self.assertIn(f'"{key}": {value},', call)
|
|
|
|
def test_default_output_is_the_published_directory(self):
|
|
source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8")
|
|
self.assertIn('"--output-dir", default="exports"', source)
|
|
|
|
|
|
class CorrectedFilesPage(unittest.TestCase):
|
|
def setUp(self):
|
|
corrected = [f"sys/file{i}.bin md5" for i in range(12)]
|
|
self.reports = [
|
|
_report("recalbox", ["es_bios.xml"], hashes_corrected=corrected),
|
|
_report("retropie", ["scriptmodules/a.sh", "scriptmodules/b.sh"],
|
|
invalid=["bash -n failed on a.sh"],
|
|
withheld=["a size the format has no field for"]),
|
|
]
|
|
self.page = generate_exports_page(self.reports, {"recalbox": "Recalbox"})
|
|
|
|
def test_every_platform_has_a_row_and_a_section(self):
|
|
self.assertIn("| [Recalbox](#recalbox) | [`es_bios.xml`]", self.page)
|
|
self.assertIn("## Recalbox { #recalbox }", self.page)
|
|
self.assertIn("## retropie { #retropie }", self.page)
|
|
|
|
def test_single_file_is_downloadable_raw(self):
|
|
self.assertIn(
|
|
"(https://raw.githubusercontent.com/Abdess/retrobios/main/exports/recalbox/es_bios.xml)",
|
|
self.page,
|
|
)
|
|
|
|
def test_several_files_link_to_their_directory(self):
|
|
self.assertIn("tree/main/exports/retropie", self.page)
|
|
self.assertIn('??? note "The 2 original files it is based on"', self.page)
|
|
|
|
def test_every_change_is_listed(self):
|
|
self.assertIn('??? note "Hashes updated to the emulator source (12)"', self.page)
|
|
for i in range(12):
|
|
self.assertIn(f" - `sys/file{i}.bin md5`", self.page)
|
|
|
|
def test_a_failing_file_is_said_to_fail(self):
|
|
self.assertIn(" - bash -n failed on a.sh", self.page)
|
|
self.assertIn(" - a size the format has no field for", self.page)
|
|
|
|
def test_nothing_published_says_how_to_build(self):
|
|
self.assertIn("export_native.py --all --fetch", generate_exports_page([], {}))
|
|
|
|
def test_site_has_the_page_in_its_nav(self):
|
|
source = (ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8")
|
|
self.assertIn('{"Updated platform files": "exports.md"}', source)
|
|
workflow = (ROOT / ".github" / "workflows" / "deploy-site.yml").read_text(encoding="utf-8")
|
|
self.assertIn('"exports/**"', workflow)
|
|
|
|
|
|
class SummariesReadAsSentences(unittest.TestCase):
|
|
"""The page shows each run's summary. RetroPie's joined its parts with
|
|
semicolons and read fs-uae's "(e.g, kick13.rom)" as two file names."""
|
|
|
|
def test_an_abbreviation_is_not_a_file_name(self):
|
|
from exporter.retropie_exporter import Exporter as RetroPieExporter
|
|
|
|
names = RetroPieExporter._listed("BIOS files (e.g, kick13.rom) i.e. Kickstart")
|
|
self.assertEqual(names, {"kick13.rom"})
|
|
|
|
def test_retropie_outcome_has_no_semicolon(self):
|
|
from exporter.retropie_exporter import Exporter as RetroPieExporter
|
|
|
|
exporter = RetroPieExporter()
|
|
exporter._skipped = {"whose help text lists no files to add to": ["fs-uae", "amiberry"]}
|
|
text = exporter.outcome({}, {"a.sh": ""})
|
|
self.assertNotIn(";", text)
|
|
self.assertIn("Left as is: amiberry, fs-uae, whose help text lists no files to add to", text)
|
|
|
|
|
|
class ThePageAddressesMaintainersAsPeers(unittest.TestCase):
|
|
"""The page offers the platforms' own maintainers a file to take or
|
|
leave. "Their file", "corrected" and "of theirs" read as a verdict on
|
|
their work, which the page is not."""
|
|
|
|
def test_the_change_sentence_is_built_from_the_numbers(self):
|
|
from siteexports import change_text
|
|
|
|
report = _report("batocera", ["batocera-systems"],
|
|
hashes_corrected=["a md5", "b md5"], requirements_corrected=3)
|
|
report["counts"] = {"kept": 692, "added": 253, "not_added": 10, "not_kept": 0}
|
|
self.assertEqual(
|
|
change_text(report),
|
|
"692 entries kept, 253 files added, 2 hashes updated, "
|
|
"3 required flags aligned with the code",
|
|
)
|
|
|
|
def test_no_wording_that_judges_the_original(self):
|
|
report = _report("batocera", ["batocera-systems"], hashes_corrected=["a md5"])
|
|
report["counts"] = {"kept": 1, "added": 0, "not_added": 4, "not_kept": 1}
|
|
page = generate_exports_page([report], {}).lower()
|
|
for word in ("their file", "theirs", "corrected", "wrong", "drift", "left out"):
|
|
self.assertNotIn(word, page)
|
|
|
|
|
|
UPSTREAM_CACHE = ROOT / ".cache" / "upstream-native"
|
|
_DAT_ROM = re.compile(r"^\trom \( name (\"[^\"]+\"|\S+) (.*) \)$")
|
|
_DAT_FIELDS = {"size": "size", "crc": "crc32", "md5": "md5", "sha1": "sha1"}
|
|
_HASH_KINDS = {
|
|
"hash_corrected", "hash_filled", "hash_cleared", "size_corrected", "size_removed",
|
|
}
|
|
|
|
|
|
def _norm(value: object) -> str | None:
|
|
"""A value as the frontend reads it: a list split on commas, each value
|
|
stripped, so the layout of a wrapped attribute is not a change."""
|
|
if value in (None, ""):
|
|
return None
|
|
return ",".join(v.strip() for v in str(value).casefold().split(","))
|
|
|
|
|
|
def _parse_dat(text: str) -> dict[tuple[str, str], dict[str, str]]:
|
|
"""Each rom line of a System.dat, keyed by its section and rom name."""
|
|
roms: dict[tuple[str, str], dict[str, str]] = {}
|
|
section = ""
|
|
for line in text.splitlines():
|
|
if line.startswith('\tcomment "'):
|
|
section = line.split('"')[1]
|
|
continue
|
|
match = _DAT_ROM.match(line)
|
|
if not match:
|
|
continue
|
|
tokens = match.group(2).split()
|
|
fields = {
|
|
_DAT_FIELDS[tokens[i]]: tokens[i + 1]
|
|
for i in range(0, len(tokens) - 1, 2) if tokens[i] in _DAT_FIELDS
|
|
}
|
|
roms.setdefault((section, match.group(1).strip('"')), fields)
|
|
return roms
|
|
|
|
|
|
def _parse_es_bios(text: str) -> dict[tuple[str, str], dict[str, str]]:
|
|
"""Each bios element of an es_bios.xml, keyed by system and path."""
|
|
from common import parse_untrusted_xml
|
|
|
|
root = parse_untrusted_xml(text)
|
|
elements: dict[tuple[str, str], dict[str, str]] = {}
|
|
for system in root.iter("system"):
|
|
for bios in system.iter("bios"):
|
|
elements.setdefault((system.get("platform", ""), bios.get("path", "")), {
|
|
"md5": bios.get("md5", ""),
|
|
"required": "false" if bios.get("mandatory") == "false" else "true",
|
|
})
|
|
return elements
|
|
|
|
|
|
class ReportedChangesAreTheDiff(unittest.TestCase):
|
|
"""The report gives each change its value before and after. Read back
|
|
from the original and the updated file, the two must agree entry by
|
|
entry, or a maintainer checks the diff against a list that differs."""
|
|
|
|
def _run(self, platform: str, filename: str) -> tuple[list[dict], str, str, dict]:
|
|
from common import load_platform_config, native_layer
|
|
from exporter import discover_exporters
|
|
from exporter.baseline import build_native_model
|
|
|
|
original_path = UPSTREAM_CACHE / platform / filename
|
|
if not original_path.is_file():
|
|
self.skipTest(f"no cached original for {platform}")
|
|
config = load_platform_config(platform, str(ROOT / "platforms"))
|
|
truth = export_native.current_truth(platform, config, str(ROOT / "platforms"))
|
|
scraped = native_layer(config)
|
|
systems, report = build_native_model(truth, scraped)
|
|
exporter = discover_exporters()[platform]()
|
|
original = original_path.read_text(encoding="utf-8")
|
|
originals = {
|
|
p.name: p.read_text(encoding="utf-8")
|
|
for p in (UPSTREAM_CACHE / platform).iterdir() if p.is_file()
|
|
}
|
|
produced = exporter.render(systems, report, originals, scraped)[filename]
|
|
changes = export_native.landed_changes(exporter, systems).records
|
|
return changes, original, produced, {"exporter": exporter, "systems": systems}
|
|
|
|
@staticmethod
|
|
def _diff(before: dict, after: dict) -> tuple[set, set]:
|
|
changed = {
|
|
(key, name, _norm(before[key].get(name)), _norm(after[key].get(name)))
|
|
for key in before.keys() & after.keys()
|
|
for name in before[key].keys() | after[key].keys()
|
|
if _norm(before[key].get(name)) != _norm(after[key].get(name))
|
|
}
|
|
return changed, set(after) - set(before)
|
|
|
|
def test_system_dat(self):
|
|
changes, original, produced, ctx = self._run("retroarch", "System.dat")
|
|
exporter = ctx["exporter"]
|
|
rom_name = {
|
|
(fe.native_system, fe.name, fe.destination): exporter._rom_name(fe)
|
|
for system in ctx["systems"].values() for fe in system.files
|
|
}
|
|
changed, added = self._diff(_parse_dat(original), _parse_dat(produced))
|
|
reported = {
|
|
((c["system"], rom_name[(c["system"], c["name"], c["destination"])]),
|
|
c["field"], _norm(c["old"]), _norm(c["new"]))
|
|
for c in changes if c["kind"] in _HASH_KINDS
|
|
}
|
|
self.assertTrue(changed, "nothing changed, the comparison proves nothing")
|
|
self.assertEqual(reported, changed)
|
|
reported_added = {
|
|
(c["system"], rom_name[(c["system"], c["name"], c["destination"])])
|
|
for c in changes if c["kind"] == "added"
|
|
}
|
|
self.assertEqual(reported_added, added)
|
|
|
|
def test_recalbox(self):
|
|
changes, original, produced, ctx = self._run("recalbox", "es_bios.xml")
|
|
exporter = ctx["exporter"]
|
|
path_of = {
|
|
(fe.native_system, fe.name, fe.destination): exporter._path(fe)
|
|
for system in ctx["systems"].values() for fe in system.files
|
|
}
|
|
changed, added = self._diff(_parse_es_bios(original), _parse_es_bios(produced))
|
|
reported = {
|
|
((c["system"], path_of[(c["system"], c["name"], c["destination"])]),
|
|
c["field"], _norm(c["old"]), _norm(c["new"]))
|
|
for c in changes
|
|
if c["kind"] in _HASH_KINDS or c["kind"] == "requirement_corrected"
|
|
}
|
|
self.assertTrue(changed, "nothing changed, the comparison proves nothing")
|
|
self.assertEqual(reported, changed)
|
|
reported_added = {
|
|
(c["system"], path_of[(c["system"], c["name"], c["destination"])])
|
|
for c in changes if c["kind"] == "added"
|
|
}
|
|
self.assertEqual(reported_added, added)
|
|
|
|
|
|
class ThePageShowsEachValueBeforeAndAfter(unittest.TestCase):
|
|
"""A list of names gave a maintainer nothing to check a correction
|
|
against: each line now says what the value was and what it becomes."""
|
|
|
|
def setUp(self):
|
|
def change(kind: str, field: str | None, old: object, new: object) -> dict:
|
|
return {"kind": kind, "system": "psx", "name": "scph5501.bin",
|
|
"destination": "scph5501.bin", "field": field,
|
|
"old": old, "new": new}
|
|
|
|
self.report = _report(
|
|
"recalbox", ["es_bios.xml"],
|
|
hashes_corrected=["psx/scph5501.bin md5"], requirements_corrected=1,
|
|
changes=[
|
|
change("hash_corrected", "md5", "5626" + "0" * 28, "c523" + "1" * 28),
|
|
change("requirement_corrected", "required", True, False),
|
|
change("hash_filled", "sha1", None, "f" * 40),
|
|
],
|
|
)
|
|
self.page = generate_exports_page([self.report], {})
|
|
|
|
def test_a_corrected_hash_shows_both_values(self):
|
|
self.assertIn(
|
|
f" - `psx/scph5501.bin` md5 `5626{'0' * 28}` becomes `c523{'1' * 28}`",
|
|
self.page,
|
|
)
|
|
|
|
def test_a_requirement_names_the_entry_and_its_value(self):
|
|
self.assertIn('??? note "Required flags aligned with the code (1)"', self.page)
|
|
self.assertIn(" - `psx/scph5501.bin` becomes optional", self.page)
|
|
|
|
def test_a_filled_hash_shows_its_value(self):
|
|
self.assertIn(f" - `psx/scph5501.bin` sha1 `{'f' * 40}` added", self.page)
|
|
|
|
def test_no_arrow_glyph(self):
|
|
for glyph in (chr(0x2192), "->", "=>"):
|
|
self.assertNotIn(glyph, self.page)
|
|
|
|
def test_the_report_carries_every_change(self):
|
|
source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8")
|
|
body = source[source.index("def export_platform("):source.index("def write_report(")]
|
|
self.assertIn('"changes": changes.records,', body)
|
|
|
|
|
|
class RemovalCandidatesNeedAReadingNoCoreAnswers(unittest.TestCase):
|
|
"""An entry the truth does not pair can still be read: by an alias, by its
|
|
content, under a directory a core walks. Of 2776 such entries, 1189
|
|
were read that way. Only what no core answers is listed, and nothing is
|
|
removed."""
|
|
|
|
PROFILES = {
|
|
"corea": {
|
|
"type": "libretro",
|
|
"systems": ["sys-one"],
|
|
"files": [
|
|
{"name": "main.rom", "aliases": ["Alt.ROM"]},
|
|
{"name": "other.bin", "md5": "0123456789abcdef0123456789abcdef"},
|
|
{"name": "rtp/2000/", "path": "rtp/2000/", "type": "directory"},
|
|
{"name": "set.zip", "category": "bios_zip"},
|
|
],
|
|
},
|
|
}
|
|
|
|
def _scraped(self, names: list[dict], cores: list[str]) -> dict:
|
|
return {
|
|
"_key": "fake",
|
|
"cores": cores,
|
|
"systems": {"sys-one": {"files": names}},
|
|
}
|
|
|
|
def _candidates(self, files: list[dict], cores: list[str] | None = None) -> list[dict]:
|
|
from exporter.baseline import build_native_model
|
|
from exporter.candidates import removal_candidates
|
|
|
|
scraped = self._scraped(files, cores or ["corea"])
|
|
systems, _ = build_native_model({"systems": {}}, scraped)
|
|
found = removal_candidates(systems, scraped, self.PROFILES)
|
|
kept = [f.name for s in systems.values() for f in s.files]
|
|
self.assertEqual(kept, [f["name"] for f in files])
|
|
return found
|
|
|
|
def test_a_name_the_platform_launcher_opens_is_a_reading(self):
|
|
"""Batocera's DuckStation generator walks 22 names (bios_lists,
|
|
duckstationGenerator.py:501-515): a name it opens is read on
|
|
Batocera whatever the profiles call it."""
|
|
from exporter.baseline import build_native_model
|
|
from exporter.candidates import removal_candidates
|
|
|
|
scraped = self._scraped([{"name": "scph7502.bin"}], ["corea"])
|
|
scraped["launchers"] = [{
|
|
"emulator": "corea", "build": "libretro", "kind": "names_searched",
|
|
"names": ["scph1001.bin", "scph7502.bin"], "source_ref": "gen.py:1-2",
|
|
}]
|
|
systems, _ = build_native_model({"systems": {}}, scraped)
|
|
self.assertEqual(removal_candidates(systems, scraped, self.PROFILES), [])
|
|
|
|
def test_an_alias_is_a_reading(self):
|
|
self.assertEqual(self._candidates([{"name": "alt.rom"}]), [])
|
|
|
|
def test_a_hash_is_a_reading(self):
|
|
found = self._candidates([
|
|
{"name": "renamed.bin", "md5": "0123456789ABCDEF0123456789ABCDEF"},
|
|
])
|
|
self.assertEqual(found, [])
|
|
|
|
def test_a_directory_entry_covers_its_members(self):
|
|
found = self._candidates([
|
|
{"name": "x.png", "destination": "easyrpg/rtp/2000/x.png"},
|
|
])
|
|
self.assertEqual(found, [])
|
|
|
|
def test_an_entry_nothing_answers_is_a_candidate(self):
|
|
found = self._candidates([{"name": "nobody.rom"}])
|
|
self.assertEqual(len(found), 1)
|
|
self.assertEqual(found[0]["verdict"], "candidate")
|
|
self.assertEqual(found[0]["cores_examined"], ["corea"])
|
|
self.assertEqual(found[0]["unresolved_cores"], [])
|
|
|
|
def test_an_unprofiled_core_leaves_it_undetermined(self):
|
|
found = self._candidates([{"name": "nobody.rom"}], ["corea", "mystery-sa"])
|
|
self.assertEqual(found[0]["verdict"], "undetermined")
|
|
self.assertEqual(found[0]["unresolved_cores"], ["mystery-sa"])
|
|
|
|
def test_a_romset_under_a_romset_core_is_undetermined(self):
|
|
found = self._candidates([{"name": "device.zip"}])
|
|
self.assertEqual(found[0]["verdict"], "undetermined")
|
|
self.assertIn("corea", found[0]["reason"])
|
|
|
|
def test_unresolved_names_skip_the_all_libretro_markers(self):
|
|
from common import unresolved_platform_cores
|
|
|
|
config = {"cores": ["retroarch", "all_libretro", "corea", "nope"]}
|
|
self.assertEqual(unresolved_platform_cores(config, self.PROFILES), {"nope"})
|
|
self.assertEqual(unresolved_platform_cores({"cores": "all_libretro"}, {}), set())
|
|
|
|
def test_the_page_gives_the_count_and_each_entry(self):
|
|
report = _report("retroarch", ["System.dat"], removal_candidates=[
|
|
{"system": "Enterprise", "name": "zt18hfnt.rom",
|
|
"destination": "zt18hfnt.rom", "verdict": "candidate",
|
|
"reason": "no core the platform runs opens it",
|
|
"cores_examined": [], "unresolved_cores": []},
|
|
])
|
|
page = generate_exports_page([report], {})
|
|
self.assertIn("1 entry of the original is not named, by name, path or hash", page)
|
|
self.assertIn("- `Enterprise` `zt18hfnt.rom`", page)
|
|
self.assertNotIn("no entry the platform declares is removed", page)
|
|
|
|
def test_the_report_carries_the_list(self):
|
|
source = (ROOT / "scripts" / "export_native.py").read_text(encoding="utf-8")
|
|
body = source[source.index("def export_platform("):source.index("def write_report(")]
|
|
self.assertIn('"removal_candidates": candidates,', body)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|