fix: count data directory readers in truth

LRPS2 reads pcsx2/resources/GameIndex.yaml through its data directory
and declares no file entry for it, so the truth entry other cores build
named armsx2 and yaps2 alone and hash_conflict missed LRPS2's copy. A
core whose data directory holds a file at an entry's path now folds in
as a reader with the cached copy's hashes, unless the profile lists what
it reads under that tree (yaps2, pcee2).

scripts/export_native.py

@@ -375,7 +375,8 @@ def current_truth(platform: str, config: dict, platforms_dir: str) -> dict:
     """The truth model generate_truth would write now, as YAML reads it back."""
     registry, profiles, db = _truth_sources(platforms_dir)
     model = generate_platform_truth(
-        platform, config, registry.get(platform, {}), profiles, db=db
+        platform, config, registry.get(platform, {}), profiles, db=db,
+        data_registry=load_data_dir_registry(platforms_dir),
     )
     return yaml_load(yaml.dump(model, default_flow_style=False, sort_keys=False,
                                allow_unicode=True)) or {}

scripts/generate_truth.py

@@ -16,6 +16,7 @@ sys.path.insert(0, os.path.dirname(__file__))
 from common import (
     platform_arg,
     list_registered_platforms,
+    load_data_dir_registry,
     load_database,
     load_emulator_profiles,
     load_platform_config,
@@ -77,6 +78,7 @@ def main(argv: list[str] | None = None) -> None:
         registry = (yaml_load(f) or {}).get("platforms", {})
 
     profiles = load_emulator_profiles(args.emulators_dir)
+    data_registry = load_data_dir_registry(args.platforms_dir)
 
     # Load database (optional)
     # The default may be absent, but a --db that was named must exist.
@@ -132,6 +134,7 @@ def main(argv: list[str] | None = None) -> None:
             profiles,
             db=db,
             target_cores=target_cores,
+            data_registry=data_registry,
         )
 
         out_path = os.path.join(output_dir, f"{name}.yml")

scripts/truth.py

@@ -7,15 +7,18 @@ and diffs truth against scraped platform data to find divergences.
 from __future__ import annotations
 
 import copy
+import os
 import sys
+from pathlib import Path
 
 from common import (
     _norm_system_id,
+    load_data_dir_registry,
     platform_layouts,
     resolve_platform_cores,
     standalone_set as platform_standalone_set,
 )
-from hashing import parse_md5_list
+from hashing import compute_hashes, parse_md5_list
 from validation import (
     filter_files_by_mode,
     installed_with_emulator,
@@ -508,6 +511,90 @@ def _declaration_verdict(entry: dict, declaration: dict, db: dict | None) -> str
     return "undecided"
 
 
+_REPO_PLATFORMS = Path(__file__).resolve().parent.parent / "platforms"
+
+
+def _fold_data_directory_readers(
+    systems: dict,
+    profiles: dict,
+    cores: list[str],
+    standalone_set: set[str],
+    platform_name: str,
+    data_registry: dict,
+) -> None:
+    """Count a core that reads a data directory among the readers of its files.
+
+    LRPS2 reads pcsx2/resources/GameIndex.yaml from its data directory and
+    declares no file entry for it, so the entry other cores build for that
+    path named armsx2 and yaps2 alone: RetroArch's System.dat line, which
+    describes LRPS2's copy, was paired with armsx2's revision and nothing
+    said the cores disagree. The copy in the directory's cache is what the
+    reader reads, so its hashes fold in as that core's declaration and a
+    different value marks hash_conflict. Only entries other cores already
+    build are touched: a directory's other files are not BIOS entries.
+    """
+    by_path: dict[str, list[dict]] = {}
+    for sys_data in systems.values():
+        for entry in sys_data.get("files", []):
+            by_path.setdefault(entry.get("path") or entry.get("name", ""), []).append(entry)
+    if not by_path:
+        return
+    for emu_name in cores:
+        for view, standalone in platform_layouts(
+            emu_name, profiles[emu_name], standalone_set
+        ):
+            trees = [
+                tree for tree in filter_files_by_mode(
+                    [t for t in view.get("data_directories") or [] if isinstance(t, dict)],
+                    standalone,
+                )
+                if tree.get("ref") in data_registry
+            ]
+            for tree in trees:
+                cache = data_registry[tree["ref"]].get("local_cache", "")
+                if not cache or not os.path.isdir(cache):
+                    continue
+                # Below the build's own folder, as the emulator pack lays the
+                # tree out: PPSSPP's assets sit under PPSSPP/, not at the root.
+                structure = (profiles[emu_name].get("pack_structure") or {}).get(
+                    "standalone" if standalone else "libretro", ""
+                )
+                prefix = "/".join(
+                    part.strip("/") for part in (structure, str(tree.get("destination") or ""))
+                    if part and part.strip("/")
+                )
+                prefix = f"{prefix}/" if prefix else ""
+                if prefix and any(
+                    str(f.get("path") or "").startswith(prefix)
+                    for f in view.get("files") or [] if isinstance(f, dict)
+                ):
+                    # The profile names what it reads in the tree: yaps2
+                    # lists its resources, pcee2 lists patches.zip alone
+                    # (its build compiles GameIndex.yaml and the shaders in).
+                    continue
+                installed = platform_name in (tree.get("installed_on") or [])
+                for path, entries in by_path.items():
+                    if not path.startswith(prefix):
+                        continue
+                    copy_path = os.path.join(cache, path[len(prefix):])
+                    if not os.path.isfile(copy_path):
+                        continue
+                    held = compute_hashes(copy_path, frozenset({"sha1", "md5", "crc32"}))
+                    declaration = {
+                        "name": os.path.basename(path),
+                        "required": bool(tree.get("required", False)),
+                        "source_ref": tree.get("source_ref"),
+                        **held,
+                        "_opens_by_name": True,
+                        "_brought": None if installed else (
+                            f"{emu_name}:{'standalone' if standalone else 'libretro'}"
+                        ),
+                    }
+                    for entry in entries:
+                        if emu_name not in entry.get("_cores", set()):
+                            _fold_into(entry, dict(declaration), emu_name)
+
+
 def generate_platform_truth(
     platform_name: str,
     config: dict,
@@ -515,6 +602,7 @@ def generate_platform_truth(
     profiles: dict[str, dict],
     db: dict | None = None,
     target_cores: set[str] | None = None,
+    data_registry: dict | None = None,
 ) -> dict:
     """Generate ground-truth system data for a platform from emulator profiles.
 
@@ -526,6 +614,8 @@ def generate_platform_truth(
         profiles: all loaded emulator profiles
         db: optional database for hash enrichment
         target_cores: optional hardware target core filter
+        data_registry: the data directory registry (_data_dirs.yml), read
+                from the repository's platforms/ when not given
 
     Returns a dict with platform metadata, systems, and per-file details
     including which cores reference each file.
@@ -702,6 +792,12 @@ def generate_platform_truth(
             )
             sys_cov["unprofiled"].add(emu_name)
 
+    _fold_data_directory_readers(
+        systems, profiles, sorted(cores_profiled), standalone_set, platform_name,
+        load_data_dir_registry(str(_REPO_PLATFORMS)) if data_registry is None
+        else data_registry,
+    )
+
     for sys_data in systems.values():
         _require_by_slot(sys_data.get("files", []))
 

tests/test_truth_data_dir_readers.py

@@ -0,0 +1,98 @@
+"""A core that reads a file through its data directory is one of its readers.
+
+LRPS2 reads pcsx2/resources/GameIndex.yaml from the tree its data directory
+reference names and declares no file entry for it. The truth entry other
+cores build for that path named armsx2 and yaps2 alone, so the export paired
+RetroArch's System.dat line, which describes LRPS2's copy, with armsx2's
+revision, and hash_conflict did not see LRPS2's value.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+sys.path.insert(0, str(REPO_ROOT / "scripts" / "exporter"))
+
+from exporter.baseline import checked_hashes  # noqa: E402
+from truth import generate_platform_truth  # noqa: E402
+
+GAME_INDEX = "pcsx2/resources/GameIndex.yaml"
+DECLARED = b"the revision armsx2 declares"
+TREE_COPY = b"the revision the tree holds"
+
+
+def _profile(name: str, **fields) -> dict:
+    return {"emulator": name, "type": "libretro", "systems": ["sony-playstation-2"], **fields}
+
+
+class DataDirectoryReaders(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
+        self.addCleanup(self._tmp.cleanup)
+        root = Path(self._tmp.name)
+        for tree, content in (("lrps2", TREE_COPY), ("resources", TREE_COPY)):
+            relative = GAME_INDEX if tree == "lrps2" else "GameIndex.yaml"
+            copy = root / tree / relative
+            copy.parent.mkdir(parents=True)
+            copy.write_bytes(content)
+        self.registry = {
+            "lrps2": {"local_cache": str(root / "lrps2")},
+            "resources": {"local_cache": str(root / "resources")},
+        }
+        self.profiles = {
+            "armsx2": _profile("armsx2", files=[{
+                "name": "GameIndex.yaml", "path": GAME_INDEX, "system": "sony-playstation-2",
+                "sha1": hashlib.sha1(DECLARED).hexdigest(),
+            }]),
+            "lrps2": _profile("lrps2", data_directories=[
+                {"ref": "lrps2", "destination": ""},
+            ], files=[{"name": "<bios>.bin", "path": "pcsx2/bios/<bios>.bin"}]),
+            # Names what it reads in the tree, patches.zip alone: the tree
+            # adds no reader of GameIndex.yaml (pcee2 compiles it in).
+            "listed": _profile("listed", data_directories=[
+                {"ref": "resources", "destination": "pcsx2/resources"},
+            ], files=[{"name": "patches.zip", "path": "pcsx2/resources/patches.zip"}]),
+        }
+        self.config = {"cores": ["armsx2", "lrps2", "listed"],
+                       "systems": {"sony-playstation-2": {"files": []}}}
+
+    def _entry(self, registry: dict) -> dict:
+        model = generate_platform_truth(
+            "plat", self.config, {}, self.profiles, data_registry=registry,
+        )
+        return next(
+            f for f in model["systems"]["sony-playstation-2"]["files"]
+            if f["name"] == "GameIndex.yaml"
+        )
+
+    def test_the_reader_counts_and_its_copy_conflicts(self):
+        entry = self._entry(self.registry)
+        self.assertEqual(entry["_cores"], ["armsx2", "lrps2"])
+        self.assertIn("sha1", entry["hash_conflict"])
+        # The kept value describes one revision among those the cores read:
+        # it pairs a line, it never corrects one.
+        self.assertNotIn("sha1", checked_hashes(entry))
+
+    def test_without_the_cache_nothing_is_folded(self):
+        entry = self._entry({})
+        self.assertEqual(entry["_cores"], ["armsx2"])
+        self.assertNotIn("hash_conflict", entry)
+
+    def test_the_same_copy_adds_a_reader_and_no_conflict(self):
+        tree_copy = Path(self.registry["lrps2"]["local_cache"]) / GAME_INDEX
+        tree_copy.write_bytes(DECLARED)
+        entry = self._entry(self.registry)
+        self.assertEqual(entry["_cores"], ["armsx2", "lrps2"])
+        self.assertNotIn("hash_conflict", entry)
+
+
+if __name__ == "__main__":
+    os.environ.setdefault("RETROBIOS_OFFLINE", "1")
+    unittest.main()

wiki/testing-guide.md

@@ -44,7 +44,7 @@ Every module in `tests/`, by the area it guards.
 | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
 | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
 | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` |
-| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` |
+| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
 | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` |
 | Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |