feat: mark files a core finds by content

DuckStation identifies its BIOS by MD5 in a folder it scans
(bios.cpp:208), so its entries' names are labels. The export wrote them
into Batocera's and ROCKNIX's lists as files to hold: a user with only
scph5501.bin read 104 missing BIOS where DuckStation boots. Labels also
lent their md5 to a name another core opens, and paired by name they
replaced System.dat's scph5000.bin hashes.

found_by: name | content, on a profile or a file and keyed by build like
load_from. The truth records which cores open each name (_named_by). A
label pairs with a platform entry by hash only, is never added, merges
with an opened name only on a shared hash, and is counted in the export
report rather than listed as missing.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 16:32:22 +02:00
1 parent eaf6d1859d
commit e5174a07bf
8 files changed
+182 -6

No files matched your search

+5
View File
@@ -612,6 +612,10 @@ def export_platform(
withheld = exporter.withheld()
if withheld:
summary += f", {_count(len(withheld), 'correction')} withheld"
if report.labels:
# A content-scanning core's names are labels: a list checked by path
# cannot hold them.
summary += f", {_count(report.labels, 'entry', 'entries')} found by content not listed"
messages.append(summary)
messages.extend(f"withheld: {reason}" for reason in withheld)
for correction in applied[:5]:
@@ -653,6 +657,7 @@ def export_platform(
"hashes_cleared": cleared,
"requirements_corrected": requirements,
"withheld": list(withheld),
"found_by_content": report.labels,
"invalid": issues,
})
return True, messages
+17 -1
View File
@@ -61,6 +61,15 @@ def checked_hashes(entry: dict | None) -> dict | None:
return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds}
def is_label(entry: dict) -> bool:
"""A name only content-scanning cores give: no core opens that file.
DuckStation identifies its BIOS by MD5 in a folder it scans, so
scph1001_v20.bin names nothing a platform could check by path.
"""
return bool(entry.get("_cores")) and "_named_by" in entry and not entry["_named_by"]
def _is_placeholder(name: str) -> bool:
return "<" in name or ">" in name or "*" in name
@@ -282,6 +291,8 @@ class Report:
files_added: int = 0
hashes_corrected: list[str] = field(default_factory=list)
required_corrected: list[str] = field(default_factory=list)
# Entries only content-scanning cores name: never written as files.
labels: int = 0
@property
def corrections(self) -> int:
@@ -394,7 +405,8 @@ def pair_entries(truth_entries: list[dict], candidates: list[dict]) -> dict[int,
claimed: set[int] = set()
for test in _PAIRING_TESTS:
for t_index, truth_entry in enumerate(truth_entries):
if t_index in pairs:
# A label answers for no name or path, only for its content.
if t_index in pairs or (test is not by_hash and is_label(truth_entry)):
continue
for index, candidate in enumerate(candidates):
if index not in claimed and test(truth_entry, candidate):
@@ -549,6 +561,10 @@ def build_native_model(
)
continue
if is_label(truth_entry):
report.labels += 1
continue
# A file another core's entry already claimed is still the
# platform's file, not a new one: scph101.bin, declared by two
# PSX cores, was written into RetroDECK's manifest a second time.
+24 -5
View File
@@ -16,7 +16,7 @@ from common import (
standalone_set as platform_standalone_set,
)
from hashing import parse_md5_list
from validation import filter_files_by_mode, read_from_system_dir
from validation import filter_files_by_mode, opens_by_name, read_from_system_dir
@@ -177,7 +177,15 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non
def other_path(f: dict) -> str:
return str(f.get("path") or "").casefold()
same_name = [f for f in files if f["name"].lower() == name_lower]
# A label and a name a core opens are one file only on a shared hash:
# pcsx_rearmed's hashless scph1001.bin took the md5 of DuckStation's
# label of that name, another dump.
labelled = not file_entry.get("_opens_by_name", True)
same_name = [
f for f in files
if f["name"].lower() == name_lower
and (labelled == (not f.get("_named_by", True)) or _shares_hash(f, file_entry))
]
return (
next((f for f in same_name if path and other_path(f) == path), None)
# Another core's file of the same name is the same file unless their
@@ -201,6 +209,13 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non
)
def _shares_hash(a: dict, b: dict) -> bool:
return any(
_declared(a, field) & _declared(b, field)
for field in ("sha1", "md5", "sha256", "crc32")
)
def _declared(entry: dict, field: str) -> set[str]:
"""The values an entry declares for a hash field, one or a list."""
value = entry.get(field)
@@ -221,6 +236,8 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None:
"""Add one core's declaration to the entry that already stands for it."""
revision = emu_name in existing.get("_cores", ())
existing["_cores"] = existing.get("_cores", set()) | {emu_name}
if file_entry.get("_opens_by_name", True):
existing["_named_by"] = existing.get("_named_by", set()) | {emu_name}
sr = file_entry.get("source_ref")
if sr is not None:
sr_key = _serialize_source_ref(sr)
@@ -317,6 +334,7 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
entry["crc32"] = crc.removeprefix("0x").lower()
entry["_cores"] = {emu_name}
entry["_required_by"] = {emu_name} if file_entry.get("required") else set()
entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set()
sr = file_entry.get("source_ref")
entry["_source_refs"] = {_serialize_source_ref(sr)} if sr is not None else set()
if db:
@@ -545,7 +563,7 @@ def generate_platform_truth(
cores_profiled.add(emu_name)
filtered = _archives_not_members([
fe
{**fe, "_opens_by_name": opens_by_name(fe, profile, standalone)}
for view, standalone in platform_layouts(emu_name, profile, standalone_set)
for fe in _system_dir_files(view, standalone)
])
@@ -637,6 +655,7 @@ def generate_platform_truth(
for fe in sys_data.get("files", []):
fe["_cores"] = sorted(fe.get("_cores", set()))
fe["_required_by"] = sorted(fe.get("_required_by", set()))
fe["_named_by"] = sorted(fe.get("_named_by", set()))
fe["_source_refs"] = sorted(fe.get("_source_refs", set()))
# Add per-system coverage
cov = system_cores.get(sys_id, {})
@@ -717,7 +736,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
correct while the export corrected hashes the diff never named. An entry
never answers for two.
"""
from exporter.baseline import _is_placeholder, pair_entries, platform_declares
from exporter.baseline import _is_placeholder, is_label, pair_entries, platform_declares
truth_files = [
fe for fe in truth_sys.get("files", [])
@@ -737,7 +756,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
for t_index, t_entry in enumerate(truth_files):
s_position = pairs.get(t_index)
if s_position is None:
if not platform_declares(t_entry, scraped_files):
if not is_label(t_entry) and not platform_declares(t_entry, scraped_files):
missing.append(
{
"name": t_entry["name"],
+17
View File
@@ -512,6 +512,23 @@ def declaration_settles(
return False
def opens_by_name(file_entry: dict, profile: dict, standalone: bool = False) -> bool:
"""Whether the code opens this entry by its name.
A core that scans a folder and identifies firmware by content
(DuckStation by MD5, bios.cpp:208) gives its entries names that are
labels. `found_by` says which, on the file or the profile, keyed by
build when the two builds differ.
"""
for source in (file_entry, profile):
value = source.get("found_by")
if isinstance(value, dict):
value = value.get("standalone" if standalone else "libretro")
if value:
return value != "content"
return True
def filter_files_by_mode(files: list[dict], standalone: bool) -> list[dict]:
"""Filter file entries by libretro/standalone mode."""
result = []