mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-25 13:19:03 -05:00
fix: count data directory readers in truth
LRPS2 reads pcsx2/resources/GameIndex.yaml through its data directory and declares no file entry for it, so the truth entry other cores build named armsx2 and yaps2 alone and hash_conflict missed LRPS2's copy. A core whose data directory holds a file at an entry's path now folds in as a reader with the cached copy's hashes, unless the profile lists what it reads under that tree (yaps2, pcee2).
This commit is contained in:
1 parent
9a0501d9d3
commit
db0e10b687
5 files changed
+201
-3
No files matched your search
@@ -375,7 +375,8 @@ def current_truth(platform: str, config: dict, platforms_dir: str) -> dict:
|
||||
"""The truth model generate_truth would write now, as YAML reads it back."""
|
||||
registry, profiles, db = _truth_sources(platforms_dir)
|
||||
model = generate_platform_truth(
|
||||
platform, config, registry.get(platform, {}), profiles, db=db
|
||||
platform, config, registry.get(platform, {}), profiles, db=db,
|
||||
data_registry=load_data_dir_registry(platforms_dir),
|
||||
)
|
||||
return yaml_load(yaml.dump(model, default_flow_style=False, sort_keys=False,
|
||||
allow_unicode=True)) or {}
|
||||
|
||||
@@ -16,6 +16,7 @@ sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import (
|
||||
platform_arg,
|
||||
list_registered_platforms,
|
||||
load_data_dir_registry,
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
@@ -77,6 +78,7 @@ def main(argv: list[str] | None = None) -> None:
|
||||
registry = (yaml_load(f) or {}).get("platforms", {})
|
||||
|
||||
profiles = load_emulator_profiles(args.emulators_dir)
|
||||
data_registry = load_data_dir_registry(args.platforms_dir)
|
||||
|
||||
# Load database (optional)
|
||||
# The default may be absent, but a --db that was named must exist.
|
||||
@@ -132,6 +134,7 @@ def main(argv: list[str] | None = None) -> None:
|
||||
profiles,
|
||||
db=db,
|
||||
target_cores=target_cores,
|
||||
data_registry=data_registry,
|
||||
)
|
||||
|
||||
out_path = os.path.join(output_dir, f"{name}.yml")
|
||||
|
||||
+97
-1
@@ -7,15 +7,18 @@ and diffs truth against scraped platform data to find divergences.
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from common import (
|
||||
_norm_system_id,
|
||||
load_data_dir_registry,
|
||||
platform_layouts,
|
||||
resolve_platform_cores,
|
||||
standalone_set as platform_standalone_set,
|
||||
)
|
||||
from hashing import parse_md5_list
|
||||
from hashing import compute_hashes, parse_md5_list
|
||||
from validation import (
|
||||
filter_files_by_mode,
|
||||
installed_with_emulator,
|
||||
@@ -508,6 +511,90 @@ def _declaration_verdict(entry: dict, declaration: dict, db: dict | None) -> str
|
||||
return "undecided"
|
||||
|
||||
|
||||
_REPO_PLATFORMS = Path(__file__).resolve().parent.parent / "platforms"
|
||||
|
||||
|
||||
def _fold_data_directory_readers(
|
||||
systems: dict,
|
||||
profiles: dict,
|
||||
cores: list[str],
|
||||
standalone_set: set[str],
|
||||
platform_name: str,
|
||||
data_registry: dict,
|
||||
) -> None:
|
||||
"""Count a core that reads a data directory among the readers of its files.
|
||||
|
||||
LRPS2 reads pcsx2/resources/GameIndex.yaml from its data directory and
|
||||
declares no file entry for it, so the entry other cores build for that
|
||||
path named armsx2 and yaps2 alone: RetroArch's System.dat line, which
|
||||
describes LRPS2's copy, was paired with armsx2's revision and nothing
|
||||
said the cores disagree. The copy in the directory's cache is what the
|
||||
reader reads, so its hashes fold in as that core's declaration and a
|
||||
different value marks hash_conflict. Only entries other cores already
|
||||
build are touched: a directory's other files are not BIOS entries.
|
||||
"""
|
||||
by_path: dict[str, list[dict]] = {}
|
||||
for sys_data in systems.values():
|
||||
for entry in sys_data.get("files", []):
|
||||
by_path.setdefault(entry.get("path") or entry.get("name", ""), []).append(entry)
|
||||
if not by_path:
|
||||
return
|
||||
for emu_name in cores:
|
||||
for view, standalone in platform_layouts(
|
||||
emu_name, profiles[emu_name], standalone_set
|
||||
):
|
||||
trees = [
|
||||
tree for tree in filter_files_by_mode(
|
||||
[t for t in view.get("data_directories") or [] if isinstance(t, dict)],
|
||||
standalone,
|
||||
)
|
||||
if tree.get("ref") in data_registry
|
||||
]
|
||||
for tree in trees:
|
||||
cache = data_registry[tree["ref"]].get("local_cache", "")
|
||||
if not cache or not os.path.isdir(cache):
|
||||
continue
|
||||
# Below the build's own folder, as the emulator pack lays the
|
||||
# tree out: PPSSPP's assets sit under PPSSPP/, not at the root.
|
||||
structure = (profiles[emu_name].get("pack_structure") or {}).get(
|
||||
"standalone" if standalone else "libretro", ""
|
||||
)
|
||||
prefix = "/".join(
|
||||
part.strip("/") for part in (structure, str(tree.get("destination") or ""))
|
||||
if part and part.strip("/")
|
||||
)
|
||||
prefix = f"{prefix}/" if prefix else ""
|
||||
if prefix and any(
|
||||
str(f.get("path") or "").startswith(prefix)
|
||||
for f in view.get("files") or [] if isinstance(f, dict)
|
||||
):
|
||||
# The profile names what it reads in the tree: yaps2
|
||||
# lists its resources, pcee2 lists patches.zip alone
|
||||
# (its build compiles GameIndex.yaml and the shaders in).
|
||||
continue
|
||||
installed = platform_name in (tree.get("installed_on") or [])
|
||||
for path, entries in by_path.items():
|
||||
if not path.startswith(prefix):
|
||||
continue
|
||||
copy_path = os.path.join(cache, path[len(prefix):])
|
||||
if not os.path.isfile(copy_path):
|
||||
continue
|
||||
held = compute_hashes(copy_path, frozenset({"sha1", "md5", "crc32"}))
|
||||
declaration = {
|
||||
"name": os.path.basename(path),
|
||||
"required": bool(tree.get("required", False)),
|
||||
"source_ref": tree.get("source_ref"),
|
||||
**held,
|
||||
"_opens_by_name": True,
|
||||
"_brought": None if installed else (
|
||||
f"{emu_name}:{'standalone' if standalone else 'libretro'}"
|
||||
),
|
||||
}
|
||||
for entry in entries:
|
||||
if emu_name not in entry.get("_cores", set()):
|
||||
_fold_into(entry, dict(declaration), emu_name)
|
||||
|
||||
|
||||
def generate_platform_truth(
|
||||
platform_name: str,
|
||||
config: dict,
|
||||
@@ -515,6 +602,7 @@ def generate_platform_truth(
|
||||
profiles: dict[str, dict],
|
||||
db: dict | None = None,
|
||||
target_cores: set[str] | None = None,
|
||||
data_registry: dict | None = None,
|
||||
) -> dict:
|
||||
"""Generate ground-truth system data for a platform from emulator profiles.
|
||||
|
||||
@@ -526,6 +614,8 @@ def generate_platform_truth(
|
||||
profiles: all loaded emulator profiles
|
||||
db: optional database for hash enrichment
|
||||
target_cores: optional hardware target core filter
|
||||
data_registry: the data directory registry (_data_dirs.yml), read
|
||||
from the repository's platforms/ when not given
|
||||
|
||||
Returns a dict with platform metadata, systems, and per-file details
|
||||
including which cores reference each file.
|
||||
@@ -702,6 +792,12 @@ def generate_platform_truth(
|
||||
)
|
||||
sys_cov["unprofiled"].add(emu_name)
|
||||
|
||||
_fold_data_directory_readers(
|
||||
systems, profiles, sorted(cores_profiled), standalone_set, platform_name,
|
||||
load_data_dir_registry(str(_REPO_PLATFORMS)) if data_registry is None
|
||||
else data_registry,
|
||||
)
|
||||
|
||||
for sys_data in systems.values():
|
||||
_require_by_slot(sys_data.get("files", []))
|
||||
|
||||
|
||||
Reference in new issue
Block a user