kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-23 20:29:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge audit fixes (export)

Browse Source 
# Conflicts:
#	exports/recalbox.json
#	exports/retropie.json
This commit is contained in:
Abdessamad Derraz committed 2026-10-21 05:53:05 +02:00
2 parents ce10323f7c 28435230a7
commit a1a9da59af
13 files changed
+524 -12325
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

exports/recalbox.json
+34 -63
View File
@@ -7,23 +7,14 @@
"files": [
"es_bios.xml"
],
"summary": "357 kept, 87 added, 4 hashes corrected, 129 requirements corrected, 527 additions left out, 103 entries found by content not listed",
"summary": "357 kept, 87 added, 2 hashes corrected, 128 requirements corrected, 528 additions left out, 102 entries found by content not listed",
"counts": {
"kept": 357,
"added": 87,
"not_added": 527,
"not_added": 528,
"not_kept": 0
},
"changes": [
{
"kind": "requirement_corrected",
"system": "amiga600",
"name": "kick33180.A500",
"destination": "amiga/bios/kick33180.A500",
"field": "required",
"old": true,
"new": false
},
{
"kind": "requirement_corrected",
"system": "amiga600",
@@ -102,18 +93,6 @@
"required": false
}
},
{
"kind": "added",
"system": "amiga600",
"name": "boot-data.zip",
"destination": "whdboot/boot-data.zip",
"field": null,
"old": null,
"new": {
"md5": "9e638f3d531eb7cd82594f49a4cf03b7",
"required": false
}
},
{
"kind": "added",
"system": "amiga600",
@@ -198,6 +177,18 @@
"required": false
}
},
{
"kind": "added",
"system": "amiga600",
"name": "amiga-os-310.rom",
"destination": "amiga-os-310.rom",
"field": null,
"old": null,
"new": {
"md5": "730888fb1bd9a3606d51f772ed136528",
"required": false
}
},
{
"kind": "added",
"system": "amiga600",
@@ -318,15 +309,6 @@
"required": false
}
},
{
"kind": "requirement_corrected",
"system": "amiga1200",
"name": "kick39106.A1200",
"destination": "amiga/bios/kick39106.A1200",
"field": "required",
"old": true,
"new": false
},
{
"kind": "requirement_corrected",
"system": "amiga1200",
@@ -729,6 +711,15 @@
"old": true,
"new": false
},
{
"kind": "requirement_corrected",
"system": "pc98",
"name": "bios.rom",
"destination": "np2kai/bios.rom",
"field": "required",
"old": true,
"new": false
},
{
"kind": "requirement_corrected",
"system": "pc98",
@@ -795,6 +786,15 @@
"old": true,
"new": false
},
{
"kind": "requirement_corrected",
"system": "msx1,msx2",
"name": "FMPAC.ROM",
"destination": "msx/FMPAC.ROM",
"field": "required",
"old": true,
"new": false
},
{
"kind": "requirement_corrected",
"system": "msx1,msx2",
@@ -1941,15 +1941,6 @@
"old": true,
"new": false
},
{
"kind": "hash_corrected",
"system": "neogeocd",
"name": "neocdz.zip",
"destination": "neocdz.zip",
"field": "md5",
"old": "f39572af7584cb5b3f70ae8cc848aba2,389dbeb0cb857c130414747462cf9d71,c85b8132799f1b2ad3754a97947809d2,b9ab07a79997fc044ca164fc72cd5680",
"new": "b31a246b69493c35194fe0bac120dc37"
},
{
"kind": "requirement_corrected",
"system": "neogeocd",
@@ -1959,15 +1950,6 @@
"old": true,
"new": false
},
{
"kind": "hash_corrected",
"system": "neogeocd",
"name": "neocd.zip",
"destination": "geolith/neocd.zip",
"field": "md5",
"old": "3711783a59ef875c31076512e3f9f0e5",
"new": "006a881532f2caffc3fe9fb5593f2ed5"
},
{
"kind": "requirement_corrected",
"system": "neogeocd",
@@ -2223,15 +2205,6 @@
"required": false
}
},
{
"kind": "requirement_corrected",
"system": "cdi",
"name": "cdibios.zip",
"destination": "cdi/cdibios.zip",
"field": "required",
"old": false,
"new": true
},
{
"kind": "added",
"system": "scummvm",
@@ -2258,8 +2231,6 @@
}
],
"hashes_corrected": [
"neogeocd/neocdz.zip md5",
"neogeocd/neocd.zip md5",
"ti994a/TI-994A.ctg md5",
"ti994a/ti-disk.ctg md5"
],
@@ -2267,9 +2238,9 @@
"sizes_corrected": [],
"sizes_removed": [],
"hashes_cleared": [],
"requirements_corrected": 129,
"requirements_corrected": 128,
"withheld": [],
"found_by_content": 103,
"found_by_content": 102,
"removal_candidates": [
{
"system": "pc98",
exports/recalbox/es_bios.xml
+7 -8
View File
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<biosList xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="es_bios.xsd">
<system fullname="Amiga 600 (ECS/OCS)" platform="amiga600">
<bios path="amiga/bios/kick33180.A500" md5="85ad74194e87c08904327de1a9443b7a" core="amiberry" mandatory="false" />
<bios path="amiga/bios/kick33180.A500" md5="85ad74194e87c08904327de1a9443b7a" core="amiberry" />
<bios path="amiga/bios/kick34005.A500" md5="82a21c1890cae844b3df741f2762d48d" core="amiberry,libretro/amiberry,libretro/uae,libretro/uae4arm,libretro/uae4all" />
<bios path="amiga/bios/kick37175.A500" md5="dc10d7bdd1b6f450773dfb558477c230" core="amiberry,libretro/uae" mandatory="false" />
<bios path="amiga/bios/kick40063.A600" md5="e40a5dfb3d017ba8779faba30cbd1c8e" mandatory="false" core="amiberry,libretro/amiberry,libretro/uae,libretro/uae4arm" />
@@ -12,7 +12,6 @@
<bios path="whdboot/WHDLoad" md5="6670a9571f235ec31b8bbd48fc18f428" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/JST" md5="2a3a61bbd7c351751506e22a4354b76a" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/AmiQuit" md5="36ab26d3b855e634fdda062c052cd52b" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/boot-data.zip" md5="9e638f3d531eb7cd82594f49a4cf03b7" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/game-data/whdload_db.json" md5="14d797815aa9bbc79fe284b244ea75b1" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/save-data/Kickstarts/kick33180.A500.RTB" md5="c993feb1eeddec1d2e687aa514231028" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/save-data/Kickstarts/kick34005.A500.RTB" md5="7eb13d974af0992d672b67c317a7e0a1" core="libretro/amiberry" mandatory="false" />
@@ -32,7 +31,7 @@
<bios path="uae_data/drive_snatch_LOUD.wav" md5="40cdf89f35b68cfefd4df5fb0256e02f" core="libretro/puae,libretro/puae2021" mandatory="false" />
</system>
<system fullname="Amiga 1200 (AGA)" platform="amiga1200">
<bios path="amiga/bios/kick39106.A1200" md5="b7cc148386aa631136f510cd29e42fc3" core="amiberry" mandatory="false" />
<bios path="amiga/bios/kick39106.A1200" md5="b7cc148386aa631136f510cd29e42fc3" core="amiberry" />
<bios path="amiga/bios/kick40068.A1200" md5="646773759326fbac3b2311fd8c8793ee" core="amiberry,libretro/amiberry,libretro/uae,libretro/uae4arm" />
<bios path="amiga/bios/kick40068.A4000" md5="9bdedde6a4f33555b4a270c8ca53297d" core="amiberry,libretro/amiberry,libretro/uae" note="This bios is required only for A4000 emulation" mandatory="false" />
</system>
@@ -146,7 +145,7 @@
<bios path="np2kai/bios.rom" md5="052f5f6010877522f78803498a625727,0c052b1f1bfd0ece11286d2ff45d296f,
36293e23fe0c86cb115d59b653c13876,3af0ae018c5710eec6e2891064814138,
50274bb5dcb707e4450011b09accffcb,c70ee9df11794bd5cc8aadb3721b4a03,
cd237e16e7e77c06bb58540e9e9fca68,e246140dec5124c5e404869a84caefce" hashMatchMandatory="false" core="libretro/np2kai" />
cd237e16e7e77c06bb58540e9e9fca68,e246140dec5124c5e404869a84caefce" hashMatchMandatory="false" core="libretro/np2kai" mandatory="false" />
<bios path="np2kai/font.bmp" md5="0c8624e9ced7cca8769e5c7b0dd4279b,7da1e5b7c482d4108d22a5b09631d967" hashMatchMandatory="false" core="libretro/np2kai" mandatory="false" />
<bios path="np2kai/font.rom" md5="38d32748ae49d1815b0614970849fd40,fd0b856899aec843cbe69d2940df547f,
4133b0be0d470920da60b9ed28d2614f,693fd1da3239d4bbeafc77d211718fc5,
@@ -205,7 +204,7 @@
<bios path="msx/CYRILLIC.FNT" md5="85b38e4128bbc300e675f55b278683a8" core="libretro/fmsx" mandatory="false" />
<bios path="msx/DISK.ROM" md5="80dcd1ad1a4cf65d64b7ba10504e8190" core="libretro/fmsx" mandatory="false" />
<bios path="msx/FMPAC16.ROM" md5="af8537262df8df267072f359399a7635" core="libretro/fmsx" />
<bios path="msx/FMPAC.ROM" md5="6f69cc8b5ed761b03afd78000dfb0e19" core="libretro/fmsx" />
<bios path="msx/FMPAC.ROM" md5="6f69cc8b5ed761b03afd78000dfb0e19" core="libretro/fmsx" mandatory="false" />
<bios path="msx/ITALIC.FNT" md5="c83e50e9f33b8dd893c414691822740d" core="libretro/fmsx" mandatory="false" />
<bios path="msx/KANJI.ROM" md5="febe8782b466d7c3b16de6d104826b34" core="libretro/fmsx" mandatory="false" />
<bios path="msx/MSX2EXT.ROM" md5="2183c2aff17cf4297bdb496de78c2e8a" core="libretro/fmsx" />
@@ -615,7 +614,7 @@
b4519df27a352c2a6e42e06d31330d91,bf00272e7150b31156a8995d60bf185d,
ca66fd463ef7dcab492a6de8ce5f45eb,f4a125538dfd7a8044e025fc5188fb88,
a3b2c9a69c28ad0bd4ea07877b744bbe,00dad01abdbf8ea9e79ad2fe11bdb182" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
<bios path="neocdz.zip" md5="b31a246b69493c35194fe0bac120dc37" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
<bios path="neocdz.zip" md5="f39572af7584cb5b3f70ae8cc848aba2,389dbeb0cb857c130414747462cf9d71,c85b8132799f1b2ad3754a97947809d2,b9ab07a79997fc044ca164fc72cd5680" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
<bios path="neocd/neocd.bin" md5="f39572af7584cb5b3f70ae8cc848aba2" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="CDZ BIOS (MAME)" />
<bios path="neocd/uni-bioscd.rom" md5="08ca8b2dba6662e8024f9e789711c6fc" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Universe BIOS CD 3.3" />
<bios path="neocd/top-sp1.bin" md5="122aee210324c72e8a11116e6ef9c0d0" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Top Loader BIOS (MAME)" />
@@ -628,7 +627,7 @@
<bios path="neocd/neocd_f.rom" md5="8834880c33164ccbe6476b559f3e37de" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Front Loader BIOS" />
<bios path="neocd/000-lo.lo" md5="fc7599f3f871578fe9a0453662d1c966" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="ZOOM Rom" />
<bios path="neocd/ng-lo.rom" md5="e255264d85d5765013b1b2fa8109dd53" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="ZOOM Rom" />
<bios path="geolith/neocd.zip" md5="006a881532f2caffc3fe9fb5593f2ed5" core="libretro/geolith" mandatory="false" hashMatchMandatory="false" note="Neo-Geo CD Bios" />
<bios path="geolith/neocd.zip" md5="3711783a59ef875c31076512e3f9f0e5" core="libretro/geolith" mandatory="false" hashMatchMandatory="false" note="Neo-Geo CD Bios" />
<bios path="geolith/neocdz.zip" md5="724bc64e63d073cf81f986adfb1f1b78" core="libretro/geolith" mandatory="true" hashMatchMandatory="false" note="Neo-Geo CDZ Bios" />
</system>
<system fullname="Texas Instrument TI-99/4A" platform="ti994a">
@@ -688,7 +687,7 @@
</system>
<system fullname="Philips CD-I" platform="cdi">
<bios path="cdi/cdimono1.zip" md5="ce590c4787460161359f2b680136cb60,bbe6636ac11d3744f0710e8e820187e7,1a7c3c98c46555ff5fc93ee3feae4b47" core="libretro/cdi2015,libretro/same_cdi" mandatory="true" hashMatchMandatory="false" />
<bios path="cdi/cdibios.zip" md5="9fd6b277cd877d51426427a33d1d19bb" core="libretro/same_cdi" hashMatchMandatory="false" />
<bios path="cdi/cdibios.zip" md5="9fd6b277cd877d51426427a33d1d19bb" core="libretro/same_cdi" mandatory="false" hashMatchMandatory="false" />
<bios path="cdi/cdimono2.zip" md5="2ac8cd5bb23287e022ca17d616949255" core="libretro/same_cdi" mandatory="false" hashMatchMandatory="false" />
</system>
<system fullname="Sega Naomi 2" platform="naomi2">
exports/retropie.json
+2 -12224
View File
File diff suppressed because it is too large. Load diff
scripts/export_native.py
+18 -13
View File
@@ -509,7 +509,7 @@ def _added_fields(exporter: object, entry: object) -> dict:
if "size" in carried and entry.size() is not None:
written["size"] = entry.size()
if "required" in carried:
written["required"] = entry.required
written["required"] = exporter.required_value(entry)
return written
@@ -535,20 +535,25 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
"added", entry, None, None, _added_fields(exporter, entry)
))
continue
# Asked of the exporter: a format that names each entry's cores
# reads every entry of those cores, not only the paired one.
if (
entry.platform.get("required") is not None
and exporter.states(entry, "required")
and exporter.required_value(entry) != bool(entry.platform["required"])
):
records.append(_change(
"requirement_corrected", entry, "required",
bool(entry.platform["required"]), exporter.required_value(entry),
))
for field_name in entry.corrections:
if not exporter.states(entry, field_name):
if field_name == "required" or not exporter.states(entry, field_name):
continue
if field_name == "required":
records.append(_change(
"requirement_corrected", entry, "required",
bool(entry.platform.get("required")), entry.required,
))
else:
records.append(_change(
"hash_corrected", entry, field_name,
_platform_value(entry, field_name),
exporter.written(entry, field_name),
))
records.append(_change(
"hash_corrected", entry, field_name,
_platform_value(entry, field_name),
exporter.written(entry, field_name),
))
records.extend(
_change("hash_filled", entry, field_name, None,
exporter.written(entry, field_name))
scripts/exporter/base_exporter.py
+8
View File
@@ -200,6 +200,14 @@ class BaseExporter(ABC):
return self.single_value(fe, field_name)
return ",".join(fe.hashes(field_name))
def required_value(self, fe: NativeFile) -> bool:
"""The requirement the render writes for an entry.
The model's flag is true when any core needs the file. A format
that names the cores of each entry narrows it to those cores.
"""
return fe.required
def states(self, fe: NativeFile, field_name: str) -> bool:
"""Whether the written file carries this entry's corrected field.
scripts/exporter/baseline.py
+49 -3
View File
@@ -61,6 +61,20 @@ def checked_hashes(entry: dict | None) -> dict | None:
return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds}
def comparable(entry: dict | None) -> dict:
"""The entry with only the hashes the truth can be compared with.
Recalbox hashes a .zip by its members (Bios.cpp:64-68 calls
Zip::Md5Composite, which feeds each member's bytes in name order into
one md5), never by the container's bytes. Its md5 on an archive is that
composite, a digest no profile declares: read as a container md5 it was
replaced by geolith's, which Recalbox never computes.
"""
if not entry or entry.get("zip_digest") != "composite":
return entry or {}
return {k: v for k, v in entry.items() if k != "md5"}
def is_label(entry: dict) -> bool:
"""A name only content-scanning cores give: no core opens that file.
@@ -104,6 +118,14 @@ class NativeFile:
# For an addition, each core that wants it as (runs standalone, name the
# platform gives it), empty when no core the platform runs is named.
runners: list[tuple[bool, str]] = field(default_factory=list)
# The platform reads this file's md5 as a composite of its members
# (Recalbox on a .zip): the truth's md5 describes the container and
# can be neither compared with it nor written in its place.
composite_md5: bool = False
# Every truth entry of the system that answers to this platform entry's
# name, whichever one the pairing took: the element names its cores, and
# each core's own entry for the file says whether that core needs it.
namesakes: list[dict] = field(default_factory=list)
@property
def origin(self) -> str:
@@ -129,7 +151,9 @@ class NativeFile:
truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and
reading it as a contradiction emptied the md5 the frontend checks.
"""
truth, plat = _field_sets(self._checked_truth), _field_sets(self.platform)
truth, plat = _field_sets(self._checked_truth), _field_sets(comparable(self.platform))
if self.composite_md5:
truth["md5"] = set()
if not any(truth.values()):
return "platform"
if not any(plat.values()):
@@ -152,6 +176,8 @@ class NativeFile:
the divergence is recorded, never silently merged: an emulator that
rejects a file will reject it whatever the platform declares.
"""
if field_name == "md5" and self.composite_md5:
return _hash_values(self.platform or {}, field_name)
truth_values = _hash_values(self._checked_truth, field_name)
platform_values = _hash_values(self.platform or {}, field_name)
source = self._identity_source()
@@ -186,6 +212,9 @@ class NativeFile:
chosen among those the truth accepts. A field only the truth names
is written when it names one value, which is then that dump's.
"""
if field_name == "md5" and self.composite_md5:
values = _hash_values(self.platform or {}, field_name)
return values[0] if values else ""
source = self._identity_source()
truth_values = _hash_values(self._checked_truth, field_name)
platform_values = _hash_values(self.platform or {}, field_name)
@@ -365,11 +394,13 @@ def by_hash(truth_entry: dict, platform_entry: dict) -> bool:
ours = {v for f in HASH_FIELDS for v in _hash_values(truth_entry, f)}
if not ours:
return False
return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(platform_entry, f)})
theirs = comparable(platform_entry)
return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(theirs, f)})
def _contradicts(truth_entry: dict, platform_entry: dict) -> bool:
"""A hash both entries declare, without a value in common."""
platform_entry = comparable(platform_entry)
return any(
(ours := set(_hash_values(truth_entry, f)))
and (theirs := set(_hash_values(platform_entry, f)))
@@ -445,6 +476,7 @@ def build_native_model(
systems: dict[str, NativeSystem] = {}
scraped_systems = (scraped or {}).get("systems", {})
composite_zips = (scraped or {}).get("zip_digest") == "composite"
# Pass 1: the platform's own file, grouped as the platform groups it.
for sys_key, sys_data in scraped_systems.items():
@@ -467,6 +499,7 @@ def build_native_model(
destination=destination,
native_system=native_id,
platform=file_entry,
composite_md5=file_entry.get("zip_digest") == "composite",
)
)
report.files_kept += 1
@@ -534,6 +567,16 @@ def build_native_model(
if candidate.truth is None
]
pairs = pair_entries(truth_files, [c.platform or {} for c in candidates])
for candidate in (
c
for native_id in target_ids
for c in systems.get(native_id, NativeSystem(native_id)).files
if c.platform is not None
):
own = str(candidate.name).casefold()
candidate.namesakes.extend(
entry for entry in truth_files if own in _names(entry)
)
for t_index, truth_entry in enumerate(truth_files):
name = str(truth_entry["name"])
@@ -545,7 +588,9 @@ def build_native_model(
checked = checked_hashes(truth_entry) or {}
for field_name in HASH_FIELDS:
ours = set(_hash_values(checked, field_name))
theirs = set(_hash_values(matched.platform or {}, field_name))
if field_name == "md5" and matched.composite_md5:
continue
theirs = set(_hash_values(comparable(matched.platform), field_name))
if ours and theirs and not ours & theirs:
matched.corrections.append(field_name)
report.hashes_corrected.append(
@@ -602,6 +647,7 @@ def build_native_model(
native_system=native_id,
truth=truth_entry,
foreign_system=native_id not in platform_ids,
composite_md5=composite_zips and name.lower().endswith(".zip"),
)
)
report.files_added += 1
scripts/exporter/misterfpga_exporter.py
+22 -2
View File
@@ -113,6 +113,7 @@ class Exporter(BaseExporter):
database = json.loads(original, object_pairs_hook=OrderedDict)
indexed = self._by_path(systems)
self._written = set()
self._unsized: list[str] = []
for path, entry in database.get("files", {}).items():
fe = indexed.get(path)
@@ -124,9 +125,17 @@ class Exporter(BaseExporter):
# entry without one, a new size beside the old hash describes
# no file, and the entry is left as MiSTer wrote it.
continue
size = fe.size()
if md5 != entry.get("hash") and entry.get("size") and not size:
# Downloader refuses an entry without an integer size
# (db_entity.py:187) and compares it after each fetch
# (fetch_file_worker.py:127). The old size beside the new
# hash describes the dump the truth rejects, so the entry
# stays as MiSTer wrote it.
self._unsized.append(path)
continue
self._written.add(id(fe))
entry["hash"] = md5
size = fe.size()
if size:
entry["size"] = size
@@ -143,6 +152,14 @@ class Exporter(BaseExporter):
produced[_ARCHIVE] = archive if unchanged else rebuild_archive(archive, _DB_NAME, text)
return produced
def withheld(self) -> list[str]:
"""Each hash correction left out for want of the new dump's size."""
return [
f"{path} keeps its hash: the truth states no size for the new "
"dump, and the Downloader checks every entry by size"
for path in getattr(self, "_unsized", [])
]
def validate(
self,
systems: dict[str, NativeSystem],
@@ -169,6 +186,9 @@ class Exporter(BaseExporter):
for path, fe in indexed.items():
declared = files.get(path)
md5 = fe.hash("md5")
if declared is not None and md5 and declared.get("hash") != md5:
if (
declared is not None and md5 and declared.get("hash") != md5
and path not in getattr(self, "_unsized", [])
):
issues.append(f"hash not applied: {path}")
return issues
scripts/exporter/recalbox_exporter.py
+103 -3
View File
@@ -20,15 +20,17 @@ from xml.sax.saxutils import quoteattr
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from common import parse_untrusted_xml
from common import load_emulator_profiles, parse_untrusted_xml
from .base_exporter import BaseExporter
from .baseline import NativeFile, NativeSystem, Report
from .baseline import NativeFile, NativeSystem, Report, by_destination, by_name
SOURCE_URL = (
"https://gitlab.com/recalbox/recalbox/-/raw/master/board/recalbox/fsoverlay"
"/recalbox/share_init/system/.emulationstation/es_bios.xml"
)
# Resolved from the file: the profiles say which names each core answers to.
EMULATORS_DIR = Path(__file__).resolve().parents[2] / "emulators"
SCHEMA_URL = (
"https://gitlab.com/recalbox/recalbox/-/raw/master/board/recalbox/fsoverlay"
"/recalbox/share_init/system/.emulationstation/es_bios.xsd"
@@ -149,12 +151,82 @@ class Exporter(BaseExporter):
return path
return fe.destination or fe.name
def _profile_keys(self) -> dict[str, str]:
"""Each name a core answers to, as Recalbox writes it, to its profile."""
index = getattr(self, "_keys", None)
if index is None:
index = {}
for key, profile in load_emulator_profiles(str(EMULATORS_DIR)).items():
for name in (key, *(str(c) for c in profile.get("cores") or [])):
index.setdefault(name.replace("-", "_").lower(), key)
self._keys = index
return index
def required_value(self, fe: NativeFile) -> bool:
"""Whether the cores the element names need the file.
The element lists its cores, and each core's own entry for the file,
found by name, says whether that core needs it. The md5 pairing
takes one entry only: saturn/saturn_bios.bin names yabause,
yabasanshiro and kronos, which require it, and its md5 paired it
with the beetle_saturn entry. The merged flag is true when any core
needs the file: cdi/cdibios.zip names same_cdi, which declares no
such set, and became mandatory because MAME requires it. Where none
of the element's cores declares the file, the platform's value
stands.
"""
entries = [fe.truth] if fe.truth else []
entries += [
e for e in fe.namesakes
if e is not fe.truth and self._same_file(e, fe.platform or {})
]
if not entries:
return fe.required
index = self._profile_keys()
named = {
index.get(core.rsplit("/", 1)[-1].replace("-", "_").lower(), "")
for core in fe.cores()
} - {""}
readers: set[str] = set()
requirers: set[str] = set()
for entry in entries:
cores = set(entry.get("_cores") or ()) & named
readers |= cores
required_by = entry.get("_required_by")
if required_by is None:
if entry.get("required") is not False:
requirers |= cores
else:
requirers |= cores & set(required_by)
if not readers:
if fe.platform is not None and fe.platform.get("required") is not None:
return bool(fe.platform["required"])
return fe.required
return bool(requirers)
@staticmethod
def _same_file(entry: dict, declared: dict) -> bool:
"""Whether a truth entry of that name describes this element's file.
Its path or its name in the same directory does. A name under an
unrelated directory does too: kronos reads kronos/saturn_bios.bin,
and Recalbox files the same BIOS under saturn/. A name in another
directory of the same tree does not: bluemsx reads
Machines/Shared Roms/KANJI.rom, which is not Recalbox's
Machines/MSX2+ - European/KANJI.rom.
"""
if by_destination(entry, declared) or by_name(entry, declared):
return True
theirs = str(declared.get("destination") or declared.get("name", "")).split("/")[:-1]
ours = str(entry.get("path") or entry.get("name", "")).split("/")[:-1]
return not {d.casefold() for d in theirs} & {d.casefold() for d in ours}
def _bios_element(self, fe: NativeFile, native_id: str) -> str:
attrs = [f"path={quoteattr(self._path(fe))}"]
attrs.append(f'md5={quoteattr(",".join(fe.hashes("md5")))}')
attrs.append(f'core={quoteattr(",".join(fe.cores()))}')
if not fe.required:
if not self.required_value(fe):
attrs.append('mandatory="false"')
elif fe.native("mandatory_declared", None) is True:
attrs.append('mandatory="true"')
@@ -184,6 +256,19 @@ class Exporter(BaseExporter):
return False
return super().states(fe, field_name)
def lands(self, fe: NativeFile) -> bool:
"""An element of theirs the render cannot rewrite is still in the
file, written back as Recalbox wrote it."""
return self.writable(fe) or id(fe) in getattr(self, "_held_whole", {})
def withheld(self) -> list[str]:
"""Each element kept as written because the truth names no md5."""
return [
f"{label} keeps its md5: the truth accepts another dump and "
"names no md5 for it, which es_bios.xsd requires"
for label in sorted(getattr(self, "_held_whole", {}).values())
]
@classmethod
def writable(cls, fe: NativeFile, require: str = "") -> bool:
"""es_bios.xsd makes md5 and core required: without them, no element.
@@ -234,6 +319,21 @@ class Exporter(BaseExporter):
for s in root.iter("system")
]
# Elements of theirs written back as they stand, by (platform, path).
held = {
(platform, " ".join(child.get("path", "").split()))
for platform, _, children in theirs
for child in children
if isinstance(child, dict)
}
self._held_whole = {
id(fe): f"{system.native_id}/{self._path(fe)}"
for system in systems.values()
for fe in system.files
if fe.platform is not None
and not self.writable(fe)
and (system.native_id, " ".join(self._path(fe).split())) in held
}
ours: dict[str, tuple[NativeSystem, dict[str, tuple[NativeFile, str]]]] = {}
for system in systems.values():
writable = [fe for fe in system.files if self.writable(fe)]
scripts/exporter/retropie_exporter.py
+15 -1
View File
@@ -316,6 +316,7 @@ class Exporter(BaseExporter):
str(a).lower() for a in (fe.native("aliases", []) or [])
)
self._skipped: dict[str, list[str]] = {}
self._named: set[int] = set()
produced: dict[str, str] = {}
def skip(reason: str, module: str) -> None:
@@ -360,7 +361,8 @@ class Exporter(BaseExporter):
| {str(a).lower() for a in (fe.native("aliases", []) or [])}) & listed
and (slot := self._slot_of(fe)) is not None
}
missing = self._in_search_order(self._one_per_slot(candidates, filled))
proposed = self._one_per_slot(candidates, filled)
missing = self._in_search_order(proposed)
if not missing:
continue
@@ -383,9 +385,21 @@ class Exporter(BaseExporter):
f'rp_module_help="{new_help}"',
1,
)
self._named.update(id(fe) for fe in proposed)
return produced
def lands(self, fe: NativeFile) -> bool:
"""An addition lands only where a help sentence now names it.
Every file a profiled core reads is writable in principle, yet the
render appends names to a handful of sentences: counting the rest
listed 1360 additions for one changed scriptmodule.
"""
if fe.platform is not None:
return self.writable(fe)
return id(fe) in getattr(self, "_named", set())
def outcome(
self,
systems: dict[str, NativeSystem],
scripts/exporter/romm_exporter.py
+35 -3
View File
@@ -26,6 +26,10 @@ SOURCE_URL = (
)
# Fixture field, model field, in the order RomM writes them.
_FIELDS = (("size", "size"), ("crc", "crc32"), ("md5", "md5"), ("sha1", "sha1"))
class Exporter(BaseExporter):
"""Write RomM's known_bios_files.json, corrected."""
@@ -35,6 +39,8 @@ class Exporter(BaseExporter):
# addition that claims each other key first: both set by render.
self._platform_keys: set[tuple[str, str]] = set()
self._first_addition: dict[tuple[str, str], int] = {}
# Entries of theirs the truth takes over without a size: left whole.
self._unsized: dict[int, str] = {}
@staticmethod
def platform_name() -> str:
@@ -112,6 +118,7 @@ class Exporter(BaseExporter):
if fe.platform is not None
}
self._first_addition = {}
self._unsized = {}
self.normalized: list[str] = []
self.normalized_values: list[tuple[NativeFile, str, str, str]] = []
for system in sorted(systems.values(), key=lambda s: s.native_id):
@@ -133,11 +140,22 @@ class Exporter(BaseExporter):
# Their entry keeps its fields and their order, ours fill in.
entry: OrderedDict[str, str] = OrderedDict(output.get(key, {}))
# The fixture states every value as a string, size included.
# Where the model settles no size, RomM's own value stands.
# str(None) wrote "None", and RomM's int() on it raised.
size = fe.size()
if size is None and fe.platform is not None:
size = fe.platform.get("size")
if size is None and fe.platform is not None and fe.platform.get("size"):
# The truth took the entry over without a size: RomM's
# size describes the dump the truth rejects, and RomM
# compares the size before any hash, so the new hashes
# beside it never verify. The entry stays as written.
self._unsized[id(fe)] = key
if key not in output:
output[key] = OrderedDict(
(field, value)
for field, name in _FIELDS
if (value := ",".join(_hash_values(fe.platform, name))
if name != "size" else str(fe.platform["size"]))
)
continue
if size is not None:
self._write(entry, "size", str(int(size)), fe, "size")
for field, hash_name in (("crc", "crc32"), ("md5", "md5"), ("sha1", "sha1")):
@@ -160,6 +178,20 @@ class Exporter(BaseExporter):
does not keep."""
return bool(_hash_values(fe.platform or {}, name)) and not fe.hashes(name)
def states(self, fe: NativeFile, field_name: str) -> bool:
"""An entry left whole carries none of the truth's values."""
if id(fe) in self._unsized:
return False
return super().states(fe, field_name)
def withheld(self) -> list[str]:
"""Each entry kept whole for want of the new dump's size."""
return [
f"{key} keeps its values: the truth states no size for the new "
"dump, and RomM compares the size before any hash"
for key in sorted(self._unsized.values())
]
def written(self, fe: NativeFile, field_name: str) -> str:
"""A record holds one value per field, the one record_hash names."""
return fe.record_hash(field_name)
tests/test_export_counts.py
+2 -1
View File
@@ -913,7 +913,8 @@ class TheSummaryCountsWhatTheFileCarries(unittest.TestCase):
return NativeFile(
"scph5501.bin", "scph5501.bin", "psx",
platform={"size": 524288, "crc32": "deadbeef", "md5": A, "sha1": "1" * 40},
truth={"md5": B},
# With a size of its own: without one, RomM keeps its entry whole.
truth={"md5": B, "size": 524288},
)
def test_romm_drops_the_hashes_it_counts_cleared(self):
tests/test_export_report.py
+31 -1
View File
@@ -7,7 +7,6 @@ file from and no list to check the diff against.
from __future__ import annotations
import json
import re
import sys
import tempfile
@@ -289,6 +288,37 @@ class ReportedChangesAreTheDiff(unittest.TestCase):
}
self.assertEqual(reported_added, added)
def test_retropie(self):
"""Each addition is a name a help sentence gains, and no other: the
report counted every file a profiled core reads as added."""
from common import load_platform_config, native_layer
from exporter import discover_exporters
from exporter.baseline import build_native_model
archive = UPSTREAM_CACHE / "retropie" / "RetroPie-Setup.tar.gz"
if not archive.is_file():
self.skipTest("no cached original for retropie")
config = load_platform_config("retropie", str(ROOT / "platforms"))
truth = export_native.current_truth("retropie", config, str(ROOT / "platforms"))
scraped = native_layer(config)
systems, report = build_native_model(truth, scraped)
exporter = discover_exporters()["retropie"]()
originals = exporter.unpack(archive.read_bytes())
produced = exporter.render(systems, report, originals, scraped)
self.assertTrue(produced, "nothing changed, the comparison proves nothing")
gained: set[str] = set()
for relative, text in produced.items():
before = exporter.modules({relative: originals[relative]})[relative][1]
after = exporter.modules({relative: text})[relative][1]
gained |= exporter._listed(after) - exporter._listed(before)
changes = export_native.landed_changes(exporter, systems).records
reported = {c["name"].lower() for c in changes if c["kind"] == "added"}
self.assertEqual(reported, gained)
self.assertEqual(
sum(c["kind"] == "added" for c in changes), len(gained),
"one name counted under several entries",
)
class ThePageShowsEachValueBeforeAndAfter(unittest.TestCase):
"""A list of names gave a maintainer nothing to check a correction
tests/test_exporters.py
+198 -3
View File
@@ -43,6 +43,7 @@ from export_native import ( # noqa: E402
SOURCES_INDEX,
collect_originals,
fetch,
landed_changes,
pinned_base,
source_key,
)
@@ -1084,6 +1085,167 @@ class RecalboxExport(unittest.TestCase):
}
self.assertEqual(blocks, {pentagon: ["3" * 32], scorpion: [new]})
COMPOSITE_SCRAPED = {
"zip_digest": "composite",
"systems": {"snk-neogeo-cd": {"native_id": "neogeocd", "files": [{
"name": "neocdz.zip", "destination": "neocdz.zip", "md5": "a" * 32,
"native_system": "neogeocd", "core": "libretro/fbneo",
"required": True, "zip_digest": "composite",
}]}},
}
def test_a_composite_md5_is_not_replaced_by_a_container_md5(self):
"""Recalbox hashes a .zip by its members (Bios.cpp:64-68,
Zip::Md5Composite): the truth's container md5 is another digest."""
truth = {"systems": {"snk-neogeo-cd": {"files": [
{"name": "neocdz.zip", "md5": "b" * 32, "required": True, "_cores": ["geolith"]},
{"name": "boot-data.zip", "md5": "c" * 32, "required": False, "_cores": ["geolith"]},
]}}}
systems, report = build_native_model(truth, self.COMPOSITE_SCRAPED)
exporter = discover_exporters()["recalbox"]()
produced = exporter.render(systems, report, {}, None)["es_bios.xml"]
md5_of = {b.get("path"): b.get("md5") for b in ET.fromstring(produced).iter("bios")}
self.assertEqual(md5_of, {"neocdz.zip": "a" * 32})
self.assertEqual(landed_changes(exporter, systems).records, [])
def test_a_container_md5_never_pairs_with_a_composite(self):
"""neocd.bin's md5 equals the composite of a neocdz.zip: by hash the
two would be read as one renamed file."""
truth = {"systems": {"snk-neogeo-cd": {"files": [
{"name": "neocd.bin", "md5": "a" * 32, "required": True, "_cores": ["neocd"]},
]}}}
systems, _ = build_native_model(truth, self.COMPOSITE_SCRAPED)
declared = next(fe for s in systems.values() for fe in s.files if fe.platform)
self.assertIsNone(declared.truth)
def test_the_live_export_keeps_every_zip_md5(self):
"""No element of es_bios.xml names a .zip under a new md5: none can
be stated as the composite Recalbox computes."""
path = REPO_ROOT / "exports" / "recalbox" / "es_bios.xml"
original = UPSTREAM_CACHE / "recalbox" / "es_bios.xml"
if not original.is_file():
self.skipTest("no cached original for recalbox")
def zips(text: str) -> dict[tuple[str, str], str]:
return {
(s.get("platform"), b.get("path")): ",".join(sorted(
v.strip().lower() for v in b.get("md5", "").split(",")
))
for s in ET.fromstring(text).iter("system")
for b in s.iter("bios")
if b.get("path", "").lower().endswith(".zip")
}
before = zips(original.read_text(encoding="utf-8"))
after = zips(path.read_text(encoding="utf-8"))
self.assertEqual({k: v for k, v in after.items() if k in before}, before)
self.assertEqual(set(after) - set(before), set())
def test_mandatory_follows_the_cores_the_element_names(self):
"""cdi/cdibios.zip names libretro/same_cdi, which does not read the
set: MAME requiring it says nothing about that element."""
scraped = {"systems": {"philips-cdi": {"native_id": "cdi", "files": [
{"name": "cdibios.zip", "destination": "cdi/cdibios.zip", "md5": "a" * 32,
"native_system": "cdi", "core": "libretro/same_cdi", "required": False},
{"name": "cdimono1.zip", "destination": "cdi/cdimono1.zip", "md5": "b" * 32,
"native_system": "cdi", "core": "libretro/same_cdi", "required": False},
]}}}
truth = {"systems": {"philips-cdi": {"files": [
{"name": "cdibios.zip", "required": True, "_cores": ["mame0278"],
"_required_by": ["mame0278"]},
{"name": "cdimono1.zip", "required": True, "_cores": ["same_cdi", "mame0278"],
"_required_by": ["same_cdi"]},
]}}}
systems, report = build_native_model(truth, scraped)
exporter = discover_exporters()["recalbox"]()
produced = exporter.render(systems, report, {}, None)["es_bios.xml"]
mandatory = {
b.get("path"): b.get("mandatory") for b in ET.fromstring(produced).iter("bios")
}
self.assertEqual(mandatory, {"cdi/cdibios.zip": "false", "cdi/cdimono1.zip": None})
changes = landed_changes(exporter, systems).records
self.assertEqual(
[(c["name"], c["old"], c["new"]) for c in changes],
[("cdimono1.zip", False, True)],
)
def test_mandatory_reads_the_named_cores_not_the_hash_pairing(self):
"""saturn/saturn_bios.bin names yabause, yabasanshiro and kronos,
which require it. Its md5 paired it with the beetle_saturn entry,
which does not, and the element went out mandatory="false"."""
md5 = "af5828fdff51384f99b3c4926be27762"
scraped = {"systems": {"sega-saturn": {"native_id": "saturn", "files": [{
"name": "saturn_bios.bin", "destination": "saturn/saturn_bios.bin",
"md5": md5, "native_system": "saturn", "required": True,
"core": "libretro/yabause,libretro/kronos",
}]}}}
truth = {"systems": {"sega-saturn": {"files": [
{"name": "saturn_bios.bin", "path": "saturn_bios.bin", "md5": md5,
"required": True, "_cores": ["beetle_saturn", "kronos", "yabause"],
"_required_by": ["beetle_saturn"]},
{"name": "saturn_bios.bin", "path": "kronos/saturn_bios.bin",
"required": True, "_cores": ["kronos", "yabause"],
"_required_by": ["kronos", "yabause"]},
]}}}
systems, report = build_native_model(truth, scraped)
exporter = discover_exporters()["recalbox"]()
declared = next(fe for s in systems.values() for fe in s.files if fe.platform)
self.assertEqual(declared.truth["_required_by"], ["beetle_saturn"])
produced = exporter.render(systems, report, {}, None)["es_bios.xml"]
element = next(
b for b in ET.fromstring(produced).iter("bios")
if b.get("path") == "saturn/saturn_bios.bin"
)
self.assertNotEqual(element.get("mandatory"), "false")
self.assertEqual(
[c for c in landed_changes(exporter, systems).records
if c["kind"] == "requirement_corrected"],
[],
)
def test_a_namesake_elsewhere_in_the_same_tree_is_another_file(self):
"""bluemsx reads Machines/Shared Roms/KANJI.rom, not Recalbox's
Machines/MSX2+ - European/KANJI.rom."""
scraped = {"systems": {"msx": {"native_id": "msx2", "files": [{
"name": "KANJI.rom", "destination": "Machines/MSX2+ - European/KANJI.rom",
"md5": "a" * 32, "native_system": "msx2", "required": True,
"core": "libretro/bluemsx",
}]}}}
truth = {"systems": {"msx": {"files": [{
"name": "KANJI.rom", "path": "Machines/Shared Roms/KANJI.rom",
"md5": "b" * 32, "required": False, "_cores": ["bluemsx"], "_required_by": [],
}]}}}
systems, _ = build_native_model(truth, scraped)
exporter = discover_exporters()["recalbox"]()
declared = next(fe for s in systems.values() for fe in s.files if fe.platform)
self.assertTrue(exporter.required_value(declared))
def test_an_element_kept_as_written_is_not_counted_lost(self):
"""The truth takes the entry on sha1 and size and names no md5: the
element is written back whole, and the run counted it a platform
entry the format cannot keep."""
scraped = {"systems": {"x": {"native_id": "x", "files": [{
"name": "a.bin", "destination": "x/a.bin", "md5": "a" * 32, "size": 100,
"native_system": "x", "core": "libretro/foo", "required": True,
}]}}}
truth = {"systems": {"x": {"files": [{
"name": "a.bin", "path": "x/a.bin", "sha1": "b" * 40, "size": 200,
"required": True, "_cores": ["foo"],
}]}}}
element = f' <bios path="x/a.bin" md5="{"a" * 32}" core="libretro/foo" />'
original = (
'<?xml version="1.0" encoding="UTF-8"?>\n<biosList>\n'
f' <system fullname="X" platform="x">\n{element}\n </system>\n</biosList>\n'
)
systems, report = build_native_model(truth, scraped)
exporter = discover_exporters()["recalbox"]()
produced = exporter.render(systems, report, {"es_bios.xml": original}, None)
self.assertIn(element, produced["es_bios.xml"])
fe = next(fe for s in systems.values() for fe in s.files)
self.assertTrue(exporter.lands(fe))
self.assertEqual(landed_changes(exporter, systems).records, [])
self.assertEqual(len(exporter.withheld()), 1)
def test_a_truncated_export_is_reported(self):
exporter, systems, _ = self._render()
empty = {
@@ -1424,9 +1586,17 @@ class RommSizeIsNeverNone(unittest.TestCase):
produced = exporter.render(systems, report, {}, None)
return exporter, systems, produced, json.loads(produced["known_bios_files.json"])
def test_the_platform_size_stands_when_none_is_settled(self):
_, _, _, data = self._export(524288)
self.assertEqual(data["psx:scph5501.bin"]["size"], "524288")
def test_an_entry_the_truth_takes_without_a_size_stays_whole(self):
"""RomM compares the size before any hash: the new md5 beside the
old size never verified, and the report said the size was removed."""
from export_native import landed_changes # noqa: PLC0415
exporter, systems, _, data = self._export(524288)
self.assertEqual(
data["psx:scph5501.bin"], {"size": "524288", "md5": "a" * 32}
)
self.assertEqual(landed_changes(exporter, systems).records, [])
self.assertEqual(len(exporter.withheld()), 1)
def test_no_size_is_written_rather_than_none(self):
exporter, systems, produced, data = self._export(None)
@@ -1585,6 +1755,31 @@ class MisterDatabaseComesBackAsWritten(unittest.TestCase):
export_native._write_produced(Path(tmp), {"a.zip": b"PK\x03\x04\xff"}, {})
self.assertEqual((Path(tmp) / "a.zip").read_bytes(), b"PK\x03\x04\xff")
def test_a_new_hash_without_a_size_leaves_the_entry_whole(self):
"""The Downloader checks each entry by size (fetch_file_worker.py:127):
the old size beside the new hash described no file, and the report
announced a size removal the database did not carry."""
from export_native import landed_changes # noqa: PLC0415
database = {"db_id": "bios_db", "files": {
"games/X/boot.rom": {"hash": "a" * 32, "size": 100, "url": "u"},
}}
original = json.dumps(database)
scraped = {"systems": {"x": {"files": [{
"name": "boot.rom", "destination": "X/boot.rom",
"md5": "a" * 32, "size": 100,
}]}}}
truth = {"systems": {"x": {"files": [{
"name": "boot.rom", "path": "X/boot.rom", "md5": "b" * 32, "_cores": ["x"],
}]}}}
exporter = discover_exporters()["misterfpga"]()
systems, report = build_native_model(truth, scraped)
produced = exporter.render(systems, report, {"bios_db.json": original}, scraped)
self.assertEqual(produced["bios_db.json"], original)
self.assertEqual(landed_changes(exporter, systems).records, [])
self.assertEqual(len(exporter.withheld()), 1)
self.assertEqual(exporter.validate(systems, produced), [])
def test_an_empty_truth_hands_back_the_database(self):
archive = UPSTREAM_CACHE / "misterfpga" / "bios_db.json.zip"
if not archive.exists():