fix: count data directory readers in truth

LRPS2 reads pcsx2/resources/GameIndex.yaml through its data directory
and declares no file entry for it, so the truth entry other cores build
named armsx2 and yaps2 alone and hash_conflict missed LRPS2's copy. A
core whose data directory holds a file at an entry's path now folds in
as a reader with the cached copy's hashes, unless the profile lists what
it reads under that tree (yaps2, pcee2).
This commit is contained in:
Abdessamad Derraz committed 2026-10-21 11:07:40 +02:00
1 parent 9a0501d9d3
commit db0e10b687
5 files changed
+201 -3

No files matched your search

+2 -1
View File
@@ -375,7 +375,8 @@ def current_truth(platform: str, config: dict, platforms_dir: str) -> dict:
"""The truth model generate_truth would write now, as YAML reads it back."""
registry, profiles, db = _truth_sources(platforms_dir)
model = generate_platform_truth(
platform, config, registry.get(platform, {}), profiles, db=db
platform, config, registry.get(platform, {}), profiles, db=db,
data_registry=load_data_dir_registry(platforms_dir),
)
return yaml_load(yaml.dump(model, default_flow_style=False, sort_keys=False,
allow_unicode=True)) or {}
+3
View File
@@ -16,6 +16,7 @@ sys.path.insert(0, os.path.dirname(__file__))
from common import (
platform_arg,
list_registered_platforms,
load_data_dir_registry,
load_database,
load_emulator_profiles,
load_platform_config,
@@ -77,6 +78,7 @@ def main(argv: list[str] | None = None) -> None:
registry = (yaml_load(f) or {}).get("platforms", {})
profiles = load_emulator_profiles(args.emulators_dir)
data_registry = load_data_dir_registry(args.platforms_dir)
# Load database (optional)
# The default may be absent, but a --db that was named must exist.
@@ -132,6 +134,7 @@ def main(argv: list[str] | None = None) -> None:
profiles,
db=db,
target_cores=target_cores,
data_registry=data_registry,
)
out_path = os.path.join(output_dir, f"{name}.yml")
+97 -1
View File
@@ -7,15 +7,18 @@ and diffs truth against scraped platform data to find divergences.
from __future__ import annotations
import copy
import os
import sys
from pathlib import Path
from common import (
_norm_system_id,
load_data_dir_registry,
platform_layouts,
resolve_platform_cores,
standalone_set as platform_standalone_set,
)
from hashing import parse_md5_list
from hashing import compute_hashes, parse_md5_list
from validation import (
filter_files_by_mode,
installed_with_emulator,
@@ -508,6 +511,90 @@ def _declaration_verdict(entry: dict, declaration: dict, db: dict | None) -> str
return "undecided"
_REPO_PLATFORMS = Path(__file__).resolve().parent.parent / "platforms"
def _fold_data_directory_readers(
systems: dict,
profiles: dict,
cores: list[str],
standalone_set: set[str],
platform_name: str,
data_registry: dict,
) -> None:
"""Count a core that reads a data directory among the readers of its files.
LRPS2 reads pcsx2/resources/GameIndex.yaml from its data directory and
declares no file entry for it, so the entry other cores build for that
path named armsx2 and yaps2 alone: RetroArch's System.dat line, which
describes LRPS2's copy, was paired with armsx2's revision and nothing
said the cores disagree. The copy in the directory's cache is what the
reader reads, so its hashes fold in as that core's declaration and a
different value marks hash_conflict. Only entries other cores already
build are touched: a directory's other files are not BIOS entries.
"""
by_path: dict[str, list[dict]] = {}
for sys_data in systems.values():
for entry in sys_data.get("files", []):
by_path.setdefault(entry.get("path") or entry.get("name", ""), []).append(entry)
if not by_path:
return
for emu_name in cores:
for view, standalone in platform_layouts(
emu_name, profiles[emu_name], standalone_set
):
trees = [
tree for tree in filter_files_by_mode(
[t for t in view.get("data_directories") or [] if isinstance(t, dict)],
standalone,
)
if tree.get("ref") in data_registry
]
for tree in trees:
cache = data_registry[tree["ref"]].get("local_cache", "")
if not cache or not os.path.isdir(cache):
continue
# Below the build's own folder, as the emulator pack lays the
# tree out: PPSSPP's assets sit under PPSSPP/, not at the root.
structure = (profiles[emu_name].get("pack_structure") or {}).get(
"standalone" if standalone else "libretro", ""
)
prefix = "/".join(
part.strip("/") for part in (structure, str(tree.get("destination") or ""))
if part and part.strip("/")
)
prefix = f"{prefix}/" if prefix else ""
if prefix and any(
str(f.get("path") or "").startswith(prefix)
for f in view.get("files") or [] if isinstance(f, dict)
):
# The profile names what it reads in the tree: yaps2
# lists its resources, pcee2 lists patches.zip alone
# (its build compiles GameIndex.yaml and the shaders in).
continue
installed = platform_name in (tree.get("installed_on") or [])
for path, entries in by_path.items():
if not path.startswith(prefix):
continue
copy_path = os.path.join(cache, path[len(prefix):])
if not os.path.isfile(copy_path):
continue
held = compute_hashes(copy_path, frozenset({"sha1", "md5", "crc32"}))
declaration = {
"name": os.path.basename(path),
"required": bool(tree.get("required", False)),
"source_ref": tree.get("source_ref"),
**held,
"_opens_by_name": True,
"_brought": None if installed else (
f"{emu_name}:{'standalone' if standalone else 'libretro'}"
),
}
for entry in entries:
if emu_name not in entry.get("_cores", set()):
_fold_into(entry, dict(declaration), emu_name)
def generate_platform_truth(
platform_name: str,
config: dict,
@@ -515,6 +602,7 @@ def generate_platform_truth(
profiles: dict[str, dict],
db: dict | None = None,
target_cores: set[str] | None = None,
data_registry: dict | None = None,
) -> dict:
"""Generate ground-truth system data for a platform from emulator profiles.
@@ -526,6 +614,8 @@ def generate_platform_truth(
profiles: all loaded emulator profiles
db: optional database for hash enrichment
target_cores: optional hardware target core filter
data_registry: the data directory registry (_data_dirs.yml), read
from the repository's platforms/ when not given
Returns a dict with platform metadata, systems, and per-file details
including which cores reference each file.
@@ -702,6 +792,12 @@ def generate_platform_truth(
)
sys_cov["unprofiled"].add(emu_name)
_fold_data_directory_readers(
systems, profiles, sorted(cores_profiled), standalone_set, platform_name,
load_data_dir_registry(str(_REPO_PLATFORMS)) if data_registry is None
else data_registry,
)
for sys_data in systems.values():
_require_by_slot(sys_data.get("files", []))