diff --git a/scripts/export_native.py b/scripts/export_native.py index fd2f0a71..dbdd2ee1 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -375,7 +375,8 @@ def current_truth(platform: str, config: dict, platforms_dir: str) -> dict: """The truth model generate_truth would write now, as YAML reads it back.""" registry, profiles, db = _truth_sources(platforms_dir) model = generate_platform_truth( - platform, config, registry.get(platform, {}), profiles, db=db + platform, config, registry.get(platform, {}), profiles, db=db, + data_registry=load_data_dir_registry(platforms_dir), ) return yaml_load(yaml.dump(model, default_flow_style=False, sort_keys=False, allow_unicode=True)) or {} diff --git a/scripts/generate_truth.py b/scripts/generate_truth.py index f2b9ce62..37e2728f 100644 --- a/scripts/generate_truth.py +++ b/scripts/generate_truth.py @@ -16,6 +16,7 @@ sys.path.insert(0, os.path.dirname(__file__)) from common import ( platform_arg, list_registered_platforms, + load_data_dir_registry, load_database, load_emulator_profiles, load_platform_config, @@ -77,6 +78,7 @@ def main(argv: list[str] | None = None) -> None: registry = (yaml_load(f) or {}).get("platforms", {}) profiles = load_emulator_profiles(args.emulators_dir) + data_registry = load_data_dir_registry(args.platforms_dir) # Load database (optional) # The default may be absent, but a --db that was named must exist. @@ -132,6 +134,7 @@ def main(argv: list[str] | None = None) -> None: profiles, db=db, target_cores=target_cores, + data_registry=data_registry, ) out_path = os.path.join(output_dir, f"{name}.yml") diff --git a/scripts/truth.py b/scripts/truth.py index d9739ca3..3488136d 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -7,15 +7,18 @@ and diffs truth against scraped platform data to find divergences. from __future__ import annotations import copy +import os import sys +from pathlib import Path from common import ( _norm_system_id, + load_data_dir_registry, platform_layouts, resolve_platform_cores, standalone_set as platform_standalone_set, ) -from hashing import parse_md5_list +from hashing import compute_hashes, parse_md5_list from validation import ( filter_files_by_mode, installed_with_emulator, @@ -508,6 +511,90 @@ def _declaration_verdict(entry: dict, declaration: dict, db: dict | None) -> str return "undecided" +_REPO_PLATFORMS = Path(__file__).resolve().parent.parent / "platforms" + + +def _fold_data_directory_readers( + systems: dict, + profiles: dict, + cores: list[str], + standalone_set: set[str], + platform_name: str, + data_registry: dict, +) -> None: + """Count a core that reads a data directory among the readers of its files. + + LRPS2 reads pcsx2/resources/GameIndex.yaml from its data directory and + declares no file entry for it, so the entry other cores build for that + path named armsx2 and yaps2 alone: RetroArch's System.dat line, which + describes LRPS2's copy, was paired with armsx2's revision and nothing + said the cores disagree. The copy in the directory's cache is what the + reader reads, so its hashes fold in as that core's declaration and a + different value marks hash_conflict. Only entries other cores already + build are touched: a directory's other files are not BIOS entries. + """ + by_path: dict[str, list[dict]] = {} + for sys_data in systems.values(): + for entry in sys_data.get("files", []): + by_path.setdefault(entry.get("path") or entry.get("name", ""), []).append(entry) + if not by_path: + return + for emu_name in cores: + for view, standalone in platform_layouts( + emu_name, profiles[emu_name], standalone_set + ): + trees = [ + tree for tree in filter_files_by_mode( + [t for t in view.get("data_directories") or [] if isinstance(t, dict)], + standalone, + ) + if tree.get("ref") in data_registry + ] + for tree in trees: + cache = data_registry[tree["ref"]].get("local_cache", "") + if not cache or not os.path.isdir(cache): + continue + # Below the build's own folder, as the emulator pack lays the + # tree out: PPSSPP's assets sit under PPSSPP/, not at the root. + structure = (profiles[emu_name].get("pack_structure") or {}).get( + "standalone" if standalone else "libretro", "" + ) + prefix = "/".join( + part.strip("/") for part in (structure, str(tree.get("destination") or "")) + if part and part.strip("/") + ) + prefix = f"{prefix}/" if prefix else "" + if prefix and any( + str(f.get("path") or "").startswith(prefix) + for f in view.get("files") or [] if isinstance(f, dict) + ): + # The profile names what it reads in the tree: yaps2 + # lists its resources, pcee2 lists patches.zip alone + # (its build compiles GameIndex.yaml and the shaders in). + continue + installed = platform_name in (tree.get("installed_on") or []) + for path, entries in by_path.items(): + if not path.startswith(prefix): + continue + copy_path = os.path.join(cache, path[len(prefix):]) + if not os.path.isfile(copy_path): + continue + held = compute_hashes(copy_path, frozenset({"sha1", "md5", "crc32"})) + declaration = { + "name": os.path.basename(path), + "required": bool(tree.get("required", False)), + "source_ref": tree.get("source_ref"), + **held, + "_opens_by_name": True, + "_brought": None if installed else ( + f"{emu_name}:{'standalone' if standalone else 'libretro'}" + ), + } + for entry in entries: + if emu_name not in entry.get("_cores", set()): + _fold_into(entry, dict(declaration), emu_name) + + def generate_platform_truth( platform_name: str, config: dict, @@ -515,6 +602,7 @@ def generate_platform_truth( profiles: dict[str, dict], db: dict | None = None, target_cores: set[str] | None = None, + data_registry: dict | None = None, ) -> dict: """Generate ground-truth system data for a platform from emulator profiles. @@ -526,6 +614,8 @@ def generate_platform_truth( profiles: all loaded emulator profiles db: optional database for hash enrichment target_cores: optional hardware target core filter + data_registry: the data directory registry (_data_dirs.yml), read + from the repository's platforms/ when not given Returns a dict with platform metadata, systems, and per-file details including which cores reference each file. @@ -702,6 +792,12 @@ def generate_platform_truth( ) sys_cov["unprofiled"].add(emu_name) + _fold_data_directory_readers( + systems, profiles, sorted(cores_profiled), standalone_set, platform_name, + load_data_dir_registry(str(_REPO_PLATFORMS)) if data_registry is None + else data_registry, + ) + for sys_data in systems.values(): _require_by_slot(sys_data.get("files", [])) diff --git a/tests/test_truth_data_dir_readers.py b/tests/test_truth_data_dir_readers.py new file mode 100644 index 00000000..f5897883 --- /dev/null +++ b/tests/test_truth_data_dir_readers.py @@ -0,0 +1,98 @@ +"""A core that reads a file through its data directory is one of its readers. + +LRPS2 reads pcsx2/resources/GameIndex.yaml from the tree its data directory +reference names and declares no file entry for it. The truth entry other +cores build for that path named armsx2 and yaps2 alone, so the export paired +RetroArch's System.dat line, which describes LRPS2's copy, with armsx2's +revision, and hash_conflict did not see LRPS2's value. +""" + +from __future__ import annotations + +import hashlib +import os +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) +sys.path.insert(0, str(REPO_ROOT / "scripts" / "exporter")) + +from exporter.baseline import checked_hashes # noqa: E402 +from truth import generate_platform_truth # noqa: E402 + +GAME_INDEX = "pcsx2/resources/GameIndex.yaml" +DECLARED = b"the revision armsx2 declares" +TREE_COPY = b"the revision the tree holds" + + +def _profile(name: str, **fields) -> dict: + return {"emulator": name, "type": "libretro", "systems": ["sony-playstation-2"], **fields} + + +class DataDirectoryReaders(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") + self.addCleanup(self._tmp.cleanup) + root = Path(self._tmp.name) + for tree, content in (("lrps2", TREE_COPY), ("resources", TREE_COPY)): + relative = GAME_INDEX if tree == "lrps2" else "GameIndex.yaml" + copy = root / tree / relative + copy.parent.mkdir(parents=True) + copy.write_bytes(content) + self.registry = { + "lrps2": {"local_cache": str(root / "lrps2")}, + "resources": {"local_cache": str(root / "resources")}, + } + self.profiles = { + "armsx2": _profile("armsx2", files=[{ + "name": "GameIndex.yaml", "path": GAME_INDEX, "system": "sony-playstation-2", + "sha1": hashlib.sha1(DECLARED).hexdigest(), + }]), + "lrps2": _profile("lrps2", data_directories=[ + {"ref": "lrps2", "destination": ""}, + ], files=[{"name": ".bin", "path": "pcsx2/bios/.bin"}]), + # Names what it reads in the tree, patches.zip alone: the tree + # adds no reader of GameIndex.yaml (pcee2 compiles it in). + "listed": _profile("listed", data_directories=[ + {"ref": "resources", "destination": "pcsx2/resources"}, + ], files=[{"name": "patches.zip", "path": "pcsx2/resources/patches.zip"}]), + } + self.config = {"cores": ["armsx2", "lrps2", "listed"], + "systems": {"sony-playstation-2": {"files": []}}} + + def _entry(self, registry: dict) -> dict: + model = generate_platform_truth( + "plat", self.config, {}, self.profiles, data_registry=registry, + ) + return next( + f for f in model["systems"]["sony-playstation-2"]["files"] + if f["name"] == "GameIndex.yaml" + ) + + def test_the_reader_counts_and_its_copy_conflicts(self): + entry = self._entry(self.registry) + self.assertEqual(entry["_cores"], ["armsx2", "lrps2"]) + self.assertIn("sha1", entry["hash_conflict"]) + # The kept value describes one revision among those the cores read: + # it pairs a line, it never corrects one. + self.assertNotIn("sha1", checked_hashes(entry)) + + def test_without_the_cache_nothing_is_folded(self): + entry = self._entry({}) + self.assertEqual(entry["_cores"], ["armsx2"]) + self.assertNotIn("hash_conflict", entry) + + def test_the_same_copy_adds_a_reader_and_no_conflict(self): + tree_copy = Path(self.registry["lrps2"]["local_cache"]) / GAME_INDEX + tree_copy.write_bytes(DECLARED) + entry = self._entry(self.registry) + self.assertEqual(entry["_cores"], ["armsx2", "lrps2"]) + self.assertNotIn("hash_conflict", entry) + + +if __name__ == "__main__": + os.environ.setdefault("RETROBIOS_OFFLINE", "1") + unittest.main() diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index ef2392a8..bf11ba88 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -44,7 +44,7 @@ Every module in `tests/`, by the area it guards. | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` | -| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` | +| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | | Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |