mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-21 03:09:08 -05:00
feat: mark files a core finds by content
DuckStation identifies its BIOS by MD5 in a folder it scans (bios.cpp:208), so its entries' names are labels. The export wrote them into Batocera's and ROCKNIX's lists as files to hold: a user with only scph5501.bin read 104 missing BIOS where DuckStation boots. Labels also lent their md5 to a name another core opens, and paired by name they replaced System.dat's scph5000.bin hashes. found_by: name | content, on a profile or a file and keyed by build like load_from. The truth records which cores open each name (_named_by). A label pairs with a platform entry by hash only, is never added, merges with an opened name only on a shared hash, and is counted in the export report rather than listed as missing.
This commit is contained in:
1 parent
eaf6d1859d
commit
e5174a07bf
8 files changed
+182
-6
No files matched your search
@@ -612,6 +612,10 @@ def export_platform(
|
||||
withheld = exporter.withheld()
|
||||
if withheld:
|
||||
summary += f", {_count(len(withheld), 'correction')} withheld"
|
||||
if report.labels:
|
||||
# A content-scanning core's names are labels: a list checked by path
|
||||
# cannot hold them.
|
||||
summary += f", {_count(report.labels, 'entry', 'entries')} found by content not listed"
|
||||
messages.append(summary)
|
||||
messages.extend(f"withheld: {reason}" for reason in withheld)
|
||||
for correction in applied[:5]:
|
||||
@@ -653,6 +657,7 @@ def export_platform(
|
||||
"hashes_cleared": cleared,
|
||||
"requirements_corrected": requirements,
|
||||
"withheld": list(withheld),
|
||||
"found_by_content": report.labels,
|
||||
"invalid": issues,
|
||||
})
|
||||
return True, messages
|
||||
|
||||
@@ -61,6 +61,15 @@ def checked_hashes(entry: dict | None) -> dict | None:
|
||||
return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds}
|
||||
|
||||
|
||||
def is_label(entry: dict) -> bool:
|
||||
"""A name only content-scanning cores give: no core opens that file.
|
||||
|
||||
DuckStation identifies its BIOS by MD5 in a folder it scans, so
|
||||
scph1001_v20.bin names nothing a platform could check by path.
|
||||
"""
|
||||
return bool(entry.get("_cores")) and "_named_by" in entry and not entry["_named_by"]
|
||||
|
||||
|
||||
def _is_placeholder(name: str) -> bool:
|
||||
return "<" in name or ">" in name or "*" in name
|
||||
|
||||
@@ -282,6 +291,8 @@ class Report:
|
||||
files_added: int = 0
|
||||
hashes_corrected: list[str] = field(default_factory=list)
|
||||
required_corrected: list[str] = field(default_factory=list)
|
||||
# Entries only content-scanning cores name: never written as files.
|
||||
labels: int = 0
|
||||
|
||||
@property
|
||||
def corrections(self) -> int:
|
||||
@@ -394,7 +405,8 @@ def pair_entries(truth_entries: list[dict], candidates: list[dict]) -> dict[int,
|
||||
claimed: set[int] = set()
|
||||
for test in _PAIRING_TESTS:
|
||||
for t_index, truth_entry in enumerate(truth_entries):
|
||||
if t_index in pairs:
|
||||
# A label answers for no name or path, only for its content.
|
||||
if t_index in pairs or (test is not by_hash and is_label(truth_entry)):
|
||||
continue
|
||||
for index, candidate in enumerate(candidates):
|
||||
if index not in claimed and test(truth_entry, candidate):
|
||||
@@ -549,6 +561,10 @@ def build_native_model(
|
||||
)
|
||||
continue
|
||||
|
||||
if is_label(truth_entry):
|
||||
report.labels += 1
|
||||
continue
|
||||
|
||||
# A file another core's entry already claimed is still the
|
||||
# platform's file, not a new one: scph101.bin, declared by two
|
||||
# PSX cores, was written into RetroDECK's manifest a second time.
|
||||
|
||||
+24
-5
@@ -16,7 +16,7 @@ from common import (
|
||||
standalone_set as platform_standalone_set,
|
||||
)
|
||||
from hashing import parse_md5_list
|
||||
from validation import filter_files_by_mode, read_from_system_dir
|
||||
from validation import filter_files_by_mode, opens_by_name, read_from_system_dir
|
||||
|
||||
|
||||
|
||||
@@ -177,7 +177,15 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non
|
||||
def other_path(f: dict) -> str:
|
||||
return str(f.get("path") or "").casefold()
|
||||
|
||||
same_name = [f for f in files if f["name"].lower() == name_lower]
|
||||
# A label and a name a core opens are one file only on a shared hash:
|
||||
# pcsx_rearmed's hashless scph1001.bin took the md5 of DuckStation's
|
||||
# label of that name, another dump.
|
||||
labelled = not file_entry.get("_opens_by_name", True)
|
||||
same_name = [
|
||||
f for f in files
|
||||
if f["name"].lower() == name_lower
|
||||
and (labelled == (not f.get("_named_by", True)) or _shares_hash(f, file_entry))
|
||||
]
|
||||
return (
|
||||
next((f for f in same_name if path and other_path(f) == path), None)
|
||||
# Another core's file of the same name is the same file unless their
|
||||
@@ -201,6 +209,13 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non
|
||||
)
|
||||
|
||||
|
||||
def _shares_hash(a: dict, b: dict) -> bool:
|
||||
return any(
|
||||
_declared(a, field) & _declared(b, field)
|
||||
for field in ("sha1", "md5", "sha256", "crc32")
|
||||
)
|
||||
|
||||
|
||||
def _declared(entry: dict, field: str) -> set[str]:
|
||||
"""The values an entry declares for a hash field, one or a list."""
|
||||
value = entry.get(field)
|
||||
@@ -221,6 +236,8 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None:
|
||||
"""Add one core's declaration to the entry that already stands for it."""
|
||||
revision = emu_name in existing.get("_cores", ())
|
||||
existing["_cores"] = existing.get("_cores", set()) | {emu_name}
|
||||
if file_entry.get("_opens_by_name", True):
|
||||
existing["_named_by"] = existing.get("_named_by", set()) | {emu_name}
|
||||
sr = file_entry.get("source_ref")
|
||||
if sr is not None:
|
||||
sr_key = _serialize_source_ref(sr)
|
||||
@@ -317,6 +334,7 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
|
||||
entry["crc32"] = crc.removeprefix("0x").lower()
|
||||
entry["_cores"] = {emu_name}
|
||||
entry["_required_by"] = {emu_name} if file_entry.get("required") else set()
|
||||
entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set()
|
||||
sr = file_entry.get("source_ref")
|
||||
entry["_source_refs"] = {_serialize_source_ref(sr)} if sr is not None else set()
|
||||
if db:
|
||||
@@ -545,7 +563,7 @@ def generate_platform_truth(
|
||||
cores_profiled.add(emu_name)
|
||||
|
||||
filtered = _archives_not_members([
|
||||
fe
|
||||
{**fe, "_opens_by_name": opens_by_name(fe, profile, standalone)}
|
||||
for view, standalone in platform_layouts(emu_name, profile, standalone_set)
|
||||
for fe in _system_dir_files(view, standalone)
|
||||
])
|
||||
@@ -637,6 +655,7 @@ def generate_platform_truth(
|
||||
for fe in sys_data.get("files", []):
|
||||
fe["_cores"] = sorted(fe.get("_cores", set()))
|
||||
fe["_required_by"] = sorted(fe.get("_required_by", set()))
|
||||
fe["_named_by"] = sorted(fe.get("_named_by", set()))
|
||||
fe["_source_refs"] = sorted(fe.get("_source_refs", set()))
|
||||
# Add per-system coverage
|
||||
cov = system_cores.get(sys_id, {})
|
||||
@@ -717,7 +736,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
correct while the export corrected hashes the diff never named. An entry
|
||||
never answers for two.
|
||||
"""
|
||||
from exporter.baseline import _is_placeholder, pair_entries, platform_declares
|
||||
from exporter.baseline import _is_placeholder, is_label, pair_entries, platform_declares
|
||||
|
||||
truth_files = [
|
||||
fe for fe in truth_sys.get("files", [])
|
||||
@@ -737,7 +756,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
for t_index, t_entry in enumerate(truth_files):
|
||||
s_position = pairs.get(t_index)
|
||||
if s_position is None:
|
||||
if not platform_declares(t_entry, scraped_files):
|
||||
if not is_label(t_entry) and not platform_declares(t_entry, scraped_files):
|
||||
missing.append(
|
||||
{
|
||||
"name": t_entry["name"],
|
||||
|
||||
@@ -512,6 +512,23 @@ def declaration_settles(
|
||||
return False
|
||||
|
||||
|
||||
def opens_by_name(file_entry: dict, profile: dict, standalone: bool = False) -> bool:
|
||||
"""Whether the code opens this entry by its name.
|
||||
|
||||
A core that scans a folder and identifies firmware by content
|
||||
(DuckStation by MD5, bios.cpp:208) gives its entries names that are
|
||||
labels. `found_by` says which, on the file or the profile, keyed by
|
||||
build when the two builds differ.
|
||||
"""
|
||||
for source in (file_entry, profile):
|
||||
value = source.get("found_by")
|
||||
if isinstance(value, dict):
|
||||
value = value.get("standalone" if standalone else "libretro")
|
||||
if value:
|
||||
return value != "content"
|
||||
return True
|
||||
|
||||
|
||||
def filter_files_by_mode(files: list[dict], standalone: bool) -> list[dict]:
|
||||
"""Filter file entries by libretro/standalone mode."""
|
||||
result = []
|
||||
|
||||
Reference in new issue
Block a user