diff --git a/emulators/lrps2.yml b/emulators/lrps2.yml index 4f6e6d36..566ce784 100644 --- a/emulators/lrps2.yml +++ b/emulators/lrps2.yml @@ -5,6 +5,7 @@ emulator: LRPS2 type: libretro core_classification: community_fork +bios_mode: agnostic source: "https://github.com/libretro/ps2" upstream: "https://github.com/PCSX2/pcsx2" profiled_date: "2026-03-25" diff --git a/emulators/melonds.yml b/emulators/melonds.yml index 93c9d7d5..f831a241 100644 --- a/emulators/melonds.yml +++ b/emulators/melonds.yml @@ -77,6 +77,7 @@ files: source_ref: "src/SPI.cpp:197-211, src/frontend/Util_ROM.cpp:201-217" - name: dsi_nand.bin + agnostic: true system: nintendo-dsi description: "DSi NAND dump" required: true diff --git a/emulators/pcsx2.yml b/emulators/pcsx2.yml index 466a4711..50238781 100644 --- a/emulators/pcsx2.yml +++ b/emulators/pcsx2.yml @@ -1,6 +1,7 @@ emulator: PCSX2 type: standalone core_classification: official_port +bios_mode: agnostic source: "https://github.com/PCSX2/pcsx2" upstream: "https://github.com/PCSX2/pcsx2" cores: diff --git a/scripts/common.py b/scripts/common.py index 9040d939..65cfe3a3 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -546,6 +546,25 @@ def resolve_local_file( if fn.casefold() in basename_targets: return os.path.join(root, fn), "data_dir" + # Agnostic fallback: for filename-agnostic files, find any DB file + # matching the system path prefix and size criteria + if file_entry.get("agnostic"): + agnostic_prefix = file_entry.get("agnostic_path_prefix", "") + min_size = file_entry.get("min_size", 0) + max_size = file_entry.get("max_size", float("inf")) + exact_size = file_entry.get("size") + if exact_size and not min_size: + min_size = exact_size + max_size = exact_size + if agnostic_prefix: + for _sha1, entry in files_db.items(): + path = entry.get("path", "") + if not path.startswith(agnostic_prefix): + continue + size = entry.get("size", 0) + if min_size <= size <= max_size and os.path.exists(path): + return path, "agnostic_fallback" + return None, "not_found" diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 8b861959..81b1ed57 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -422,6 +422,91 @@ def _collect_emulator_extras( "source_emulator": profile.get("emulator", emu_name), }) + # Third pass: agnostic scan — for filename-agnostic cores, include all + # DB files matching the system path prefix and size criteria. + files_db = db.get("files", {}) + for emu_name, profile in sorted(profiles.items()): + if profile.get("type") in ("launcher", "alias"): + continue + if emu_name not in relevant: + continue + is_profile_agnostic = profile.get("bios_mode") == "agnostic" + if not is_profile_agnostic: + if not any(f.get("agnostic") for f in profile.get("files", [])): + continue + + for f in profile.get("files", []): + if not is_profile_agnostic and not f.get("agnostic"): + continue + fname = f.get("name", "") + if not fname: + continue + + # Derive path prefix from the representative file in the DB + path_prefix = None + sha1_list = by_name.get(fname, []) + for sha1 in sha1_list: + entry = files_db.get(sha1, {}) + path = entry.get("path", "") + if path: + parts = path.rsplit("/", 1) + if len(parts) == 2: + path_prefix = parts[0] + "/" + break + + if not path_prefix: + # Fallback: try other files in the profile for the same system + for other_f in profile.get("files", []): + if other_f is f: + continue + other_name = other_f.get("name", "") + for sha1 in by_name.get(other_name, []): + entry = files_db.get(sha1, {}) + path = entry.get("path", "") + if path: + parts = path.rsplit("/", 1) + if len(parts) == 2: + path_prefix = parts[0] + "/" + break + if path_prefix: + break + + if not path_prefix: + continue + + # Size criteria from the file entry + min_size = f.get("min_size", 0) + max_size = f.get("max_size", float("inf")) + exact_size = f.get("size") + if exact_size and not min_size: + min_size = exact_size + max_size = exact_size + + # Scan DB for all files under this prefix matching size + for sha1, entry in files_db.items(): + path = entry.get("path", "") + if not path.startswith(path_prefix): + continue + size = entry.get("size", 0) + if not (min_size <= size <= max_size): + continue + scan_name = entry.get("name", "") + if not scan_name: + continue + dest = scan_name + full_dest = f"{base_dest}/{dest}" if base_dest else dest + if full_dest in seen_dests: + continue + seen_dests.add(full_dest) + extras.append({ + "name": scan_name, + "destination": dest, + "required": False, + "hle_fallback": False, + "source_emulator": profile.get("emulator", emu_name), + "agnostic_scan": True, + }) + return extras @@ -621,6 +706,24 @@ def _build_readme(platform_name: str, platform_display: str, return header + guide + footer +def _build_agnostic_rename_readme( + destination: str, original: str, alternatives: list[str], +) -> str: + """Build a README explaining an agnostic file rename.""" + lines = [ + "This file was renamed for compatibility:", + f" {destination} <- {original}", + "", + ] + if alternatives: + lines.append("All variants included in this pack:") + for alt in sorted(alternatives): + lines.append(f" {alt}") + lines.append("") + lines.append(f"To use a different variant, rename it to: {destination}") + return "\n".join(lines) + "\n" + + def generate_pack( platform_name: str, platforms_dir: str, @@ -788,10 +891,71 @@ def generate_pack( continue if status == "not_found": - if not already_packed: - missing_files.append(file_entry["name"]) - file_status[dedup_key] = "missing" - continue + # Agnostic fallback: if an agnostic core covers this system, + # find any matching file in the DB + by_name = db.get("indexes", {}).get("by_name", {}) + files_db = db.get("files", {}) + agnostic_path = None + agnostic_resolved = False + if emu_profiles: + for _emu_key, _emu_prof in emu_profiles.items(): + if _emu_prof.get("bios_mode") != "agnostic": + continue + if sys_id not in set(_emu_prof.get("systems", [])): + continue + for _ef in _emu_prof.get("files", []): + ef_name = _ef.get("name", "") + for _sha1 in by_name.get(ef_name, []): + _entry = files_db.get(_sha1, {}) + _path = _entry.get("path", "") + if _path: + _prefix = _path.rsplit("/", 1)[0] + "/" + _min = _ef.get("min_size", 0) + _max = _ef.get("max_size", float("inf")) + if _ef.get("size") and not _min: + _min = _ef["size"] + _max = _ef["size"] + for _s, _e in files_db.items(): + if _e.get("path", "").startswith(_prefix): + if _min <= _e.get("size", 0) <= _max: + if os.path.exists(_e["path"]): + local_path = _e["path"] + agnostic_path = _prefix + agnostic_resolved = True + break + break + if agnostic_resolved: + break + if agnostic_resolved: + break + + if agnostic_resolved and local_path: + # Write rename README + original_name = os.path.basename(local_path) + dest_name = file_entry.get("name", "") + if original_name != dest_name and agnostic_path: + alt_names = [] + for _s, _e in files_db.items(): + _p = _e.get("path", "") + if _p.startswith(agnostic_path): + _n = _e.get("name", "") + if _n and _n != original_name: + alt_names.append(_n) + readme_text = _build_agnostic_rename_readme( + dest_name, original_name, alt_names, + ) + readme_name = f"RENAMED_{dest_name}.txt" + readme_full = f"{base_dest}/{readme_name}" if base_dest else readme_name + if readme_full not in seen_destinations: + zf.writestr(readme_full, readme_text) + seen_destinations.add(readme_full) + status = "agnostic_fallback" + # Fall through to normal packing below + else: + if not already_packed: + missing_files.append(file_entry["name"]) + file_status[dedup_key] = "missing" + continue if status == "hash_mismatch" and verification_mode != "existence": zf_name = file_entry.get("zipped_file") diff --git a/scripts/verify.py b/scripts/verify.py index 3ba9e454..38666ba5 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -291,6 +291,10 @@ def find_undeclared_files( if emu_name not in relevant: continue + # Skip agnostic profiles entirely (filename-agnostic BIOS detection) + if profile.get("bios_mode") == "agnostic": + continue + # Check if this profile is standalone: match profile name or any cores: alias is_standalone = emu_name in standalone_set or bool( standalone_set & {str(c) for c in profile.get("cores", [])} @@ -317,6 +321,10 @@ def find_undeclared_files( if load_from and load_from != "system_dir": continue + # Skip agnostic files (filename-agnostic, handled by agnostic scan) + if f.get("agnostic"): + continue + archive = f.get("archive") # Skip files declared by the platform (by name or archive) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 79135c3a..d8f09a8c 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -170,6 +170,7 @@ class TestE2E(unittest.TestCase): "md5": info["md5"], "name": name, "crc32": info.get("crc32", ""), + "size": len(info["data"]), } by_md5[info["md5"]] = sha1 by_name.setdefault(name, []).append(sha1) @@ -510,6 +511,33 @@ class TestE2E(unittest.TestCase): with open(os.path.join(self.emulators_dir, "test_renamed.yml"), "w") as fh: yaml.dump(emu_renamed, fh) + # Agnostic profile (bios_mode: agnostic) — skipped by find_undeclared_files + emu_agnostic = { + "emulator": "TestAgnostic", + "type": "standalone", + "bios_mode": "agnostic", + "systems": ["console-a"], + "files": [ + {"name": "correct_hash.bin", "required": True, + "min_size": 1, "max_size": 999999}, + ], + } + with open(os.path.join(self.emulators_dir, "test_agnostic.yml"), "w") as fh: + yaml.dump(emu_agnostic, fh) + + # Mixed profile with per-file agnostic + emu_mixed_agnostic = { + "emulator": "TestMixedAgnostic", + "type": "libretro", + "systems": ["console-a"], + "files": [ + {"name": "undeclared_req.bin", "required": True}, + {"name": "agnostic_file.bin", "required": True, "agnostic": True}, + ], + } + with open(os.path.join(self.emulators_dir, "test_mixed_agnostic.yml"), "w") as fh: + yaml.dump(emu_mixed_agnostic, fh) + # --------------------------------------------------------------- # THE TEST -one method per feature area, all using same fixtures # --------------------------------------------------------------- @@ -3411,6 +3439,66 @@ class TestE2E(unittest.TestCase): self.assertEqual(result["summary"]["total_missing"], 0) self.assertEqual(result["summary"]["systems_compared"], 1) + def test_179_agnostic_profile_skipped_in_undeclared(self): + """bios_mode: agnostic profiles are skipped entirely by find_undeclared_files.""" + config = load_platform_config("test_existence", self.platforms_dir) + profiles = load_emulator_profiles(self.emulators_dir) + undeclared = find_undeclared_files(config, self.emulators_dir, self.db, profiles) + emulators = {u["emulator"] for u in undeclared} + # TestAgnostic should NOT appear in undeclared (bios_mode: agnostic) + self.assertNotIn("TestAgnostic", emulators) + + def test_180_agnostic_file_skipped_in_undeclared(self): + """Files with agnostic: true are skipped, others in same profile are not.""" + config = load_platform_config("test_existence", self.platforms_dir) + profiles = load_emulator_profiles(self.emulators_dir) + undeclared = find_undeclared_files(config, self.emulators_dir, self.db, profiles) + names = {u["name"] for u in undeclared} + # agnostic_file.bin should NOT be in undeclared (agnostic: true) + self.assertNotIn("agnostic_file.bin", names) + # undeclared_req.bin should still be in undeclared (not agnostic) + self.assertIn("undeclared_req.bin", names) + + def test_181_agnostic_extras_scan(self): + """Agnostic profiles add all matching DB files as extras.""" + from generate_pack import _collect_emulator_extras + config = load_platform_config("test_existence", self.platforms_dir) + profiles = load_emulator_profiles(self.emulators_dir) + extras = _collect_emulator_extras( + config, self.emulators_dir, self.db, set(), "system", profiles, + ) + agnostic_extras = [e for e in extras if e.get("source_emulator") == "TestAgnostic"] + # Agnostic scan should find files in the same directory as correct_hash.bin + self.assertTrue(len(agnostic_extras) > 0, "Agnostic scan should produce extras") + # All agnostic extras should have agnostic_scan flag + for e in agnostic_extras: + self.assertTrue(e.get("agnostic_scan", False)) + + def test_182_agnostic_rename_readme(self): + """_build_agnostic_rename_readme generates correct text.""" + from generate_pack import _build_agnostic_rename_readme + result = _build_agnostic_rename_readme( + "dsi_nand.bin", "DSi_Nand_AUS.bin", + ["DSi_Nand_EUR.bin", "DSi_Nand_USA.bin"], + ) + self.assertIn("dsi_nand.bin <- DSi_Nand_AUS.bin", result) + self.assertIn("DSi_Nand_EUR.bin", result) + self.assertIn("DSi_Nand_USA.bin", result) + self.assertIn("rename it to: dsi_nand.bin", result) + + def test_183_agnostic_resolve_fallback(self): + """resolve_local_file with agnostic fallback finds a system file.""" + file_entry = { + "name": "nonexistent_agnostic.bin", + "agnostic": True, + "min_size": 1, + "max_size": 999999, + "agnostic_path_prefix": self.bios_dir + "/", + } + path, status = resolve_local_file(file_entry, self.db) + self.assertIsNotNone(path) + self.assertEqual(status, "agnostic_fallback") + if __name__ == "__main__": unittest.main()