kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-20 18:59:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor: pair truth and platform entries one way

Browse Source 
diff_truth matched by name with a rank, export_native by path, guarded name and hash: on current data each named files the other did not. Both now read one matcher, which also accepts aliases and sha256.
This commit is contained in:
Abdessamad Derraz committed 2026-10-18 20:48:29 +02:00
1 parent 8a053d590e
commit fc47ac9f1f
3 files changed
+156 -225
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/exporter/baseline.py
+81 -63
View File
@@ -267,6 +267,81 @@ def _match_key(entry: dict) -> tuple[str, str]:
return dest.casefold(), str(entry.get("name", "")).casefold()
def _names(entry: dict) -> set[str]:
"""The names an entry answers to: its own and the aliases it declares."""
return {
str(n).casefold()
for n in (entry.get("name", ""), *(entry.get("aliases") or []))
if n
}
def by_destination(truth_entry: dict, platform_entry: dict) -> bool:
"""The platform's destination ends in the path the truth declares.
Dolphin writes dolphin-emu/Sys/GC/USA/IPL.bin for the truth's
GC/USA/IPL.bin. A bare name is no path: dc/boot.bin ends in /boot.bin,
and fbneo's 480-byte boot.bin is not that 2 MB Dreamcast BIOS.
"""
t_dest = _match_key(truth_entry)[0]
theirs = _match_key(platform_entry)[0]
if not t_dest:
return False
return theirs == t_dest or ("/" in t_dest and theirs.endswith("/" + t_dest))
def by_name(truth_entry: dict, platform_entry: dict) -> bool:
"""One of the truth's names, in the same directory, at the same size."""
t_dest = _match_key(truth_entry)[0]
theirs_dest, theirs = _match_key(platform_entry)
if not theirs or theirs not in _names(truth_entry):
return False
# Two declared directories that differ name two files: the truth's
# BeebFile/BIOS.rom is not Recalbox's np2kai/bios.rom.
if "/" in t_dest and "/" in theirs_dest and not theirs_dest.endswith(
"/" + t_dest.rsplit("/", 1)[0].rsplit("/", 1)[-1] + "/" + theirs
):
return False
# A shared name with two declared sizes is two files: fbneo's 480-byte
# boot.bin is not RomM's 2 MB Dreamcast boot.bin.
t_size = truth_entry.get("size")
p_size = platform_entry.get("size")
return not (isinstance(t_size, int) and isinstance(p_size, int) and t_size != p_size)
def by_hash(truth_entry: dict, platform_entry: dict) -> bool:
"""A value on any of the four hash fields in common: a renamed file."""
ours = {v for f in HASH_FIELDS for v in _hash_values(truth_entry, f)}
if not ours:
return False
return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(platform_entry, f)})
def pair_truth_entry(
truth_entry: dict, candidates: list[dict], claimed: set[int]
) -> int | None:
"""Which platform entry a truth entry describes, by index, or None.
The one rule export_native and diff_truth read. Each test is tried
across every candidate before the next: with three IPL.bin under one
system, separated only by their path, a first-match-wins scan attached
the truth to whichever came first. An entry never answers for two.
"""
for test in (by_destination, by_name, by_hash):
for index, candidate in enumerate(candidates):
if index not in claimed and test(truth_entry, candidate):
return index
return None
def platform_declares(truth_entry: dict, candidates: list[dict]) -> bool:
"""A file another truth entry already claimed is still the platform's
file, not one it lacks: scph101.bin, declared by two PSX cores."""
return any(
by_destination(truth_entry, c) or by_name(truth_entry, c) for c in candidates
)
def build_native_model(
truth: dict,
scraped: dict | None,
@@ -363,13 +438,6 @@ def build_native_model(
if not name or name.startswith("_") or _is_placeholder(name):
continue
t_dest, t_name = _match_key(truth_entry)
t_hashes = {
value
for field_name in HASH_FIELDS
for value in _hash_values(truth_entry, field_name)
}
candidates = [
candidate
for native_id in target_ids
@@ -378,60 +446,10 @@ def build_native_model(
).files
if candidate.truth is None
]
def by_destination(candidate: NativeFile, t_dest: str = t_dest) -> bool:
# Dolphin writes dolphin-emu/Sys/GC/USA/IPL.bin for the truth's
# GC/USA/IPL.bin: a destination ending in the path is the file.
# A bare name is no path: dc/boot.bin ends in /boot.bin, and
# fbneo's 480-byte boot.bin is not that 2 MB Dreamcast BIOS.
# Names go through by_name and its size guard.
theirs = _match_key(candidate.platform or {})[0]
if not t_dest:
return False
return theirs == t_dest or ("/" in t_dest and theirs.endswith("/" + t_dest))
def by_name(
candidate: NativeFile,
t_dest: str = t_dest,
t_name: str = t_name,
truth_entry: dict = truth_entry,
) -> bool:
theirs_dest, theirs = _match_key(candidate.platform or {})
if not t_name or theirs != t_name:
return False
# Two declared directories that differ name two files: the
# truth's BeebFile/BIOS.rom is not Recalbox's np2kai/bios.rom.
if "/" in t_dest and "/" in theirs_dest and not theirs_dest.endswith(
"/" + t_dest.rsplit("/", 1)[0].rsplit("/", 1)[-1] + "/" + t_name
):
return False
# A shared name with two declared sizes is two files: fbneo's
# 480-byte boot.bin is not RomM's 2 MB Dreamcast boot.bin.
t_size = truth_entry.get("size")
p_size = (candidate.platform or {}).get("size")
return not (
isinstance(t_size, int) and isinstance(p_size, int) and t_size != p_size
)
def by_hash(candidate: NativeFile, t_hashes: set = t_hashes) -> bool:
if not t_hashes:
return False
theirs = {
value
for field_name in HASH_FIELDS
for value in _hash_values(candidate.platform or {}, field_name)
}
return bool(t_hashes & theirs)
# Tried in order across every candidate, not per candidate: with
# three IPL.bin under one system, separated only by their path, a
# first-match-wins scan would attach the truth to whichever came
# first and correct the wrong region's file.
matched: NativeFile | None = None
for test in (by_destination, by_name, by_hash):
matched = next((c for c in candidates if test(c)), None)
if matched is not None:
break
index = pair_truth_entry(
truth_entry, [c.platform or {} for c in candidates], set()
)
matched = candidates[index] if index is not None else None
if matched is not None:
matched.truth = truth_entry
@@ -462,12 +480,12 @@ def build_native_model(
# platform's file, not a new one: scph101.bin, declared by two
# PSX cores, was written into RetroDECK's manifest a second time.
declared = [
candidate
candidate.platform
for native_id in target_ids
for candidate in systems.get(native_id, NativeSystem(native_id)).files
if candidate.platform is not None
]
if any(by_destination(c) or by_name(c) for c in declared):
if platform_declares(truth_entry, declared):
continue
# The truth knows a file the platform does not declare.
scripts/truth.py
+33 -106
View File
@@ -576,52 +576,6 @@ def _hash_values(entry: dict, field: str) -> set[str]:
return set(parse_md5_list(raw)) if isinstance(raw, (str, list)) else set()
def _match_renames(
unmatched_truth: list[dict], unmatched_scraped: dict
) -> tuple[set[int], set]:
"""Pair files a platform renamed with the truth entry they came from.
A platform is free to call a file whatever it likes -- Batocera ships
the same ROM as ROM1 -- so a name that matches nothing is not yet a
gap. When an unmatched file on either side carries the same hash, it
is one file under two names, and counting it as both missing and extra
would report a gap that does not exist.
Returns the positions in unmatched_truth and the scraped keys that pair
up: a name does not identify a truth entry, two can share it.
"""
# Hash-based fallback: detect platform renames (e.g. Batocera ROM → ROM1)
# If an unmatched scraped file shares a hash with an unmatched truth file,
# it's the same file under a different name: a platform rename, not a gap.
rename_matched_truth: set[int] = set()
rename_matched_scraped: set = set()
if unmatched_truth and unmatched_scraped:
# Build hash → truth file index for unmatched truth files
truth_hash_index: dict[str, int] = {}
for position, fe in enumerate(unmatched_truth):
for h in ("sha1", "md5", "crc32"):
for val in _hash_values(fe, h):
truth_hash_index[val] = position
for s_key, s_entry in unmatched_scraped.items():
position = next(
(
truth_hash_index[val]
for h in ("sha1", "md5", "crc32")
for val in sorted(_hash_values(s_entry, h))
if val in truth_hash_index
),
None,
)
if position is not None:
# Rename detected. Count as matched
rename_matched_truth.add(position)
rename_matched_scraped.add(s_key)
return rename_matched_truth, rename_matched_scraped
def _hash_set(entry: dict) -> set[str]:
return {value for h in ("sha1", "md5", "crc32") for value in _hash_values(entry, h)}
@@ -630,22 +584,6 @@ def _path_tail(value: object) -> str:
return str(value or "").replace("\\", "/").casefold()
def _pair_rank(truth_entry: dict, scraped_entry: dict) -> tuple[bool, bool, bool, bool]:
"""How well a same-named truth entry describes a scraped one.
Exact path suffix, then same directory, then primary name over alias,
then a shared hash.
"""
destination = _path_tail(scraped_entry.get("destination"))
t_path = _path_tail(truth_entry.get("path"))
return (
bool(t_path) and destination.endswith(t_path),
"/" in t_path and t_path.rsplit("/", 1)[0] == destination.rpartition("/")[0],
truth_entry["name"].lower() == scraped_entry["name"].lower(),
bool(_hash_set(scraped_entry) & _hash_set(truth_entry)),
)
def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None:
"""The first hash both sides declare without a value in common.
@@ -672,17 +610,19 @@ def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None:
def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
"""Compare files between truth and scraped for a single system.
A name can stand for several files of one system (Dolphin's IPL.bin
under GC/USA, GC/EUR and GC/JAP), so each scraped entry is paired with
the same-named truth entry at its destination first, and an entry
never answers for two.
Entries are paired by pair_truth_entry, the rule export_native applies:
destination, then name or alias in the same directory at the same size,
then any hash in common. Two rules had diverged, and diff_truth flagged
a hash the export did not correct while the export corrected hashes the
diff never named. An entry never answers for two.
"""
truth_files = truth_sys.get("files", [])
truth_index: dict[str, list[int]] = {}
for position, fe in enumerate(truth_files):
for key in [fe["name"], *fe.get("aliases", [])]:
truth_index.setdefault(key.lower(), []).append(position)
from exporter.baseline import _is_placeholder, pair_truth_entry, platform_declares
truth_files = [
fe for fe in truth_sys.get("files", [])
if fe.get("name") and not fe["name"].startswith("_")
and not _is_placeholder(fe["name"])
]
scraped_files = scraped_sys.get("files", [])
missing: list[dict] = []
@@ -691,22 +631,21 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
extra_phantom: list[dict] = []
extra_unprofiled: list[dict] = []
matched: set[int] = set()
unmatched_scraped: dict[int, dict] = {}
for s_position, s_entry in enumerate(scraped_files):
candidates = [
p for p in truth_index.get(s_entry["name"].lower(), []) if p not in matched
]
if not candidates:
if s_entry["name"].lower() not in truth_index:
unmatched_scraped[s_position] = s_entry
claimed: set[int] = set()
for t_entry in truth_files:
s_position = pair_truth_entry(t_entry, scraped_files, claimed)
if s_position is None:
if not platform_declares(t_entry, scraped_files):
missing.append(
{
"name": t_entry["name"],
"cores": list(t_entry.get("_cores", [])),
"source_refs": list(t_entry.get("_source_refs", [])),
}
)
continue
# The first best candidate wins, as max() keeps the first maximum.
ranked = [(_pair_rank(truth_files[p], s_entry), p) for p in candidates]
t_position = max(ranked, key=lambda pair: pair[0])[1]
matched.add(t_position)
t_entry = truth_files[t_position]
claimed.add(s_position)
s_entry = scraped_files[s_position]
mismatch = _hash_mismatch(t_entry, s_entry)
if mismatch:
hash_mismatch.append(mismatch)
@@ -721,25 +660,13 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
}
)
# Collect unmatched files from both sides
unmatched_truth = [
fe for position, fe in enumerate(truth_files) if position not in matched
]
rename_matched_truth, rename_matched_scraped = _match_renames(
unmatched_truth, unmatched_scraped
)
# Truth files not matched (by name, alias, or hash) -> missing
for position, fe in enumerate(unmatched_truth):
if position not in rename_matched_truth:
missing.append(
{
"name": fe["name"],
"cores": list(fe.get("_cores", [])),
"source_refs": list(fe.get("_source_refs", [])),
}
)
# A scraped entry no truth entry describes, by path, name or hash.
unmatched_scraped = {
position: s_entry
for position, s_entry in enumerate(scraped_files)
if position not in claimed
and not any(platform_declares(t, [s_entry]) for t in truth_files)
}
# Scraped files not in truth -> extra
coverage = truth_sys.get("_coverage", {})
@@ -749,7 +676,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
seen_extra: set[tuple[str, str]] = set()
for s_key, s_entry in unmatched_scraped.items():
key = (s_entry["name"].lower(), _path_tail(s_entry.get("destination")))
if s_key in rename_matched_scraped or key in seen_extra:
if key in seen_extra:
continue
seen_extra.add(key)
entry = {"name": s_entry["name"]}