mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-04-13 04:12:33 -05:00
feat: agnostic bios mode for filename-agnostic emulators
bios_mode: agnostic (profile) and agnostic: true (file) for emulators that accept any valid BIOS without specific filename. find_undeclared_files skips agnostic entries, pack extras scan includes all matching DB files by path prefix + size criteria, resolve_local_file has agnostic fallback with rename README. applied to pcsx2, lrps2 (bios_mode), melonds dsi_nand (file).
This commit is contained in:
Abdessamad Derraz
@@ -5,6 +5,7 @@
|
||||
emulator: LRPS2
|
||||
type: libretro
|
||||
core_classification: community_fork
|
||||
bios_mode: agnostic
|
||||
source: "https://github.com/libretro/ps2"
|
||||
upstream: "https://github.com/PCSX2/pcsx2"
|
||||
profiled_date: "2026-03-25"
|
||||
|
||||
@@ -77,6 +77,7 @@ files:
|
||||
source_ref: "src/SPI.cpp:197-211, src/frontend/Util_ROM.cpp:201-217"
|
||||
|
||||
- name: dsi_nand.bin
|
||||
agnostic: true
|
||||
system: nintendo-dsi
|
||||
description: "DSi NAND dump"
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
emulator: PCSX2
|
||||
type: standalone
|
||||
core_classification: official_port
|
||||
bios_mode: agnostic
|
||||
source: "https://github.com/PCSX2/pcsx2"
|
||||
upstream: "https://github.com/PCSX2/pcsx2"
|
||||
cores:
|
||||
|
||||
@@ -546,6 +546,25 @@ def resolve_local_file(
|
||||
if fn.casefold() in basename_targets:
|
||||
return os.path.join(root, fn), "data_dir"
|
||||
|
||||
# Agnostic fallback: for filename-agnostic files, find any DB file
|
||||
# matching the system path prefix and size criteria
|
||||
if file_entry.get("agnostic"):
|
||||
agnostic_prefix = file_entry.get("agnostic_path_prefix", "")
|
||||
min_size = file_entry.get("min_size", 0)
|
||||
max_size = file_entry.get("max_size", float("inf"))
|
||||
exact_size = file_entry.get("size")
|
||||
if exact_size and not min_size:
|
||||
min_size = exact_size
|
||||
max_size = exact_size
|
||||
if agnostic_prefix:
|
||||
for _sha1, entry in files_db.items():
|
||||
path = entry.get("path", "")
|
||||
if not path.startswith(agnostic_prefix):
|
||||
continue
|
||||
size = entry.get("size", 0)
|
||||
if min_size <= size <= max_size and os.path.exists(path):
|
||||
return path, "agnostic_fallback"
|
||||
|
||||
return None, "not_found"
|
||||
|
||||
|
||||
|
||||
@@ -422,6 +422,91 @@ def _collect_emulator_extras(
|
||||
"source_emulator": profile.get("emulator", emu_name),
|
||||
})
|
||||
|
||||
# Third pass: agnostic scan — for filename-agnostic cores, include all
|
||||
# DB files matching the system path prefix and size criteria.
|
||||
files_db = db.get("files", {})
|
||||
for emu_name, profile in sorted(profiles.items()):
|
||||
if profile.get("type") in ("launcher", "alias"):
|
||||
continue
|
||||
if emu_name not in relevant:
|
||||
continue
|
||||
is_profile_agnostic = profile.get("bios_mode") == "agnostic"
|
||||
if not is_profile_agnostic:
|
||||
if not any(f.get("agnostic") for f in profile.get("files", [])):
|
||||
continue
|
||||
|
||||
for f in profile.get("files", []):
|
||||
if not is_profile_agnostic and not f.get("agnostic"):
|
||||
continue
|
||||
fname = f.get("name", "")
|
||||
if not fname:
|
||||
continue
|
||||
|
||||
# Derive path prefix from the representative file in the DB
|
||||
path_prefix = None
|
||||
sha1_list = by_name.get(fname, [])
|
||||
for sha1 in sha1_list:
|
||||
entry = files_db.get(sha1, {})
|
||||
path = entry.get("path", "")
|
||||
if path:
|
||||
parts = path.rsplit("/", 1)
|
||||
if len(parts) == 2:
|
||||
path_prefix = parts[0] + "/"
|
||||
break
|
||||
|
||||
if not path_prefix:
|
||||
# Fallback: try other files in the profile for the same system
|
||||
for other_f in profile.get("files", []):
|
||||
if other_f is f:
|
||||
continue
|
||||
other_name = other_f.get("name", "")
|
||||
for sha1 in by_name.get(other_name, []):
|
||||
entry = files_db.get(sha1, {})
|
||||
path = entry.get("path", "")
|
||||
if path:
|
||||
parts = path.rsplit("/", 1)
|
||||
if len(parts) == 2:
|
||||
path_prefix = parts[0] + "/"
|
||||
break
|
||||
if path_prefix:
|
||||
break
|
||||
|
||||
if not path_prefix:
|
||||
continue
|
||||
|
||||
# Size criteria from the file entry
|
||||
min_size = f.get("min_size", 0)
|
||||
max_size = f.get("max_size", float("inf"))
|
||||
exact_size = f.get("size")
|
||||
if exact_size and not min_size:
|
||||
min_size = exact_size
|
||||
max_size = exact_size
|
||||
|
||||
# Scan DB for all files under this prefix matching size
|
||||
for sha1, entry in files_db.items():
|
||||
path = entry.get("path", "")
|
||||
if not path.startswith(path_prefix):
|
||||
continue
|
||||
size = entry.get("size", 0)
|
||||
if not (min_size <= size <= max_size):
|
||||
continue
|
||||
scan_name = entry.get("name", "")
|
||||
if not scan_name:
|
||||
continue
|
||||
dest = scan_name
|
||||
full_dest = f"{base_dest}/{dest}" if base_dest else dest
|
||||
if full_dest in seen_dests:
|
||||
continue
|
||||
seen_dests.add(full_dest)
|
||||
extras.append({
|
||||
"name": scan_name,
|
||||
"destination": dest,
|
||||
"required": False,
|
||||
"hle_fallback": False,
|
||||
"source_emulator": profile.get("emulator", emu_name),
|
||||
"agnostic_scan": True,
|
||||
})
|
||||
|
||||
return extras
|
||||
|
||||
|
||||
@@ -621,6 +706,24 @@ def _build_readme(platform_name: str, platform_display: str,
|
||||
return header + guide + footer
|
||||
|
||||
|
||||
def _build_agnostic_rename_readme(
|
||||
destination: str, original: str, alternatives: list[str],
|
||||
) -> str:
|
||||
"""Build a README explaining an agnostic file rename."""
|
||||
lines = [
|
||||
"This file was renamed for compatibility:",
|
||||
f" {destination} <- {original}",
|
||||
"",
|
||||
]
|
||||
if alternatives:
|
||||
lines.append("All variants included in this pack:")
|
||||
for alt in sorted(alternatives):
|
||||
lines.append(f" {alt}")
|
||||
lines.append("")
|
||||
lines.append(f"To use a different variant, rename it to: {destination}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def generate_pack(
|
||||
platform_name: str,
|
||||
platforms_dir: str,
|
||||
@@ -788,10 +891,71 @@ def generate_pack(
|
||||
continue
|
||||
|
||||
if status == "not_found":
|
||||
if not already_packed:
|
||||
missing_files.append(file_entry["name"])
|
||||
file_status[dedup_key] = "missing"
|
||||
continue
|
||||
# Agnostic fallback: if an agnostic core covers this system,
|
||||
# find any matching file in the DB
|
||||
by_name = db.get("indexes", {}).get("by_name", {})
|
||||
files_db = db.get("files", {})
|
||||
agnostic_path = None
|
||||
agnostic_resolved = False
|
||||
if emu_profiles:
|
||||
for _emu_key, _emu_prof in emu_profiles.items():
|
||||
if _emu_prof.get("bios_mode") != "agnostic":
|
||||
continue
|
||||
if sys_id not in set(_emu_prof.get("systems", [])):
|
||||
continue
|
||||
for _ef in _emu_prof.get("files", []):
|
||||
ef_name = _ef.get("name", "")
|
||||
for _sha1 in by_name.get(ef_name, []):
|
||||
_entry = files_db.get(_sha1, {})
|
||||
_path = _entry.get("path", "")
|
||||
if _path:
|
||||
_prefix = _path.rsplit("/", 1)[0] + "/"
|
||||
_min = _ef.get("min_size", 0)
|
||||
_max = _ef.get("max_size", float("inf"))
|
||||
if _ef.get("size") and not _min:
|
||||
_min = _ef["size"]
|
||||
_max = _ef["size"]
|
||||
for _s, _e in files_db.items():
|
||||
if _e.get("path", "").startswith(_prefix):
|
||||
if _min <= _e.get("size", 0) <= _max:
|
||||
if os.path.exists(_e["path"]):
|
||||
local_path = _e["path"]
|
||||
agnostic_path = _prefix
|
||||
agnostic_resolved = True
|
||||
break
|
||||
break
|
||||
if agnostic_resolved:
|
||||
break
|
||||
if agnostic_resolved:
|
||||
break
|
||||
|
||||
if agnostic_resolved and local_path:
|
||||
# Write rename README
|
||||
original_name = os.path.basename(local_path)
|
||||
dest_name = file_entry.get("name", "")
|
||||
if original_name != dest_name and agnostic_path:
|
||||
alt_names = []
|
||||
for _s, _e in files_db.items():
|
||||
_p = _e.get("path", "")
|
||||
if _p.startswith(agnostic_path):
|
||||
_n = _e.get("name", "")
|
||||
if _n and _n != original_name:
|
||||
alt_names.append(_n)
|
||||
readme_text = _build_agnostic_rename_readme(
|
||||
dest_name, original_name, alt_names,
|
||||
)
|
||||
readme_name = f"RENAMED_{dest_name}.txt"
|
||||
readme_full = f"{base_dest}/{readme_name}" if base_dest else readme_name
|
||||
if readme_full not in seen_destinations:
|
||||
zf.writestr(readme_full, readme_text)
|
||||
seen_destinations.add(readme_full)
|
||||
status = "agnostic_fallback"
|
||||
# Fall through to normal packing below
|
||||
else:
|
||||
if not already_packed:
|
||||
missing_files.append(file_entry["name"])
|
||||
file_status[dedup_key] = "missing"
|
||||
continue
|
||||
|
||||
if status == "hash_mismatch" and verification_mode != "existence":
|
||||
zf_name = file_entry.get("zipped_file")
|
||||
|
||||
@@ -291,6 +291,10 @@ def find_undeclared_files(
|
||||
if emu_name not in relevant:
|
||||
continue
|
||||
|
||||
# Skip agnostic profiles entirely (filename-agnostic BIOS detection)
|
||||
if profile.get("bios_mode") == "agnostic":
|
||||
continue
|
||||
|
||||
# Check if this profile is standalone: match profile name or any cores: alias
|
||||
is_standalone = emu_name in standalone_set or bool(
|
||||
standalone_set & {str(c) for c in profile.get("cores", [])}
|
||||
@@ -317,6 +321,10 @@ def find_undeclared_files(
|
||||
if load_from and load_from != "system_dir":
|
||||
continue
|
||||
|
||||
# Skip agnostic files (filename-agnostic, handled by agnostic scan)
|
||||
if f.get("agnostic"):
|
||||
continue
|
||||
|
||||
archive = f.get("archive")
|
||||
|
||||
# Skip files declared by the platform (by name or archive)
|
||||
|
||||
@@ -170,6 +170,7 @@ class TestE2E(unittest.TestCase):
|
||||
"md5": info["md5"],
|
||||
"name": name,
|
||||
"crc32": info.get("crc32", ""),
|
||||
"size": len(info["data"]),
|
||||
}
|
||||
by_md5[info["md5"]] = sha1
|
||||
by_name.setdefault(name, []).append(sha1)
|
||||
@@ -510,6 +511,33 @@ class TestE2E(unittest.TestCase):
|
||||
with open(os.path.join(self.emulators_dir, "test_renamed.yml"), "w") as fh:
|
||||
yaml.dump(emu_renamed, fh)
|
||||
|
||||
# Agnostic profile (bios_mode: agnostic) — skipped by find_undeclared_files
|
||||
emu_agnostic = {
|
||||
"emulator": "TestAgnostic",
|
||||
"type": "standalone",
|
||||
"bios_mode": "agnostic",
|
||||
"systems": ["console-a"],
|
||||
"files": [
|
||||
{"name": "correct_hash.bin", "required": True,
|
||||
"min_size": 1, "max_size": 999999},
|
||||
],
|
||||
}
|
||||
with open(os.path.join(self.emulators_dir, "test_agnostic.yml"), "w") as fh:
|
||||
yaml.dump(emu_agnostic, fh)
|
||||
|
||||
# Mixed profile with per-file agnostic
|
||||
emu_mixed_agnostic = {
|
||||
"emulator": "TestMixedAgnostic",
|
||||
"type": "libretro",
|
||||
"systems": ["console-a"],
|
||||
"files": [
|
||||
{"name": "undeclared_req.bin", "required": True},
|
||||
{"name": "agnostic_file.bin", "required": True, "agnostic": True},
|
||||
],
|
||||
}
|
||||
with open(os.path.join(self.emulators_dir, "test_mixed_agnostic.yml"), "w") as fh:
|
||||
yaml.dump(emu_mixed_agnostic, fh)
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# THE TEST -one method per feature area, all using same fixtures
|
||||
# ---------------------------------------------------------------
|
||||
@@ -3411,6 +3439,66 @@ class TestE2E(unittest.TestCase):
|
||||
self.assertEqual(result["summary"]["total_missing"], 0)
|
||||
self.assertEqual(result["summary"]["systems_compared"], 1)
|
||||
|
||||
def test_179_agnostic_profile_skipped_in_undeclared(self):
|
||||
"""bios_mode: agnostic profiles are skipped entirely by find_undeclared_files."""
|
||||
config = load_platform_config("test_existence", self.platforms_dir)
|
||||
profiles = load_emulator_profiles(self.emulators_dir)
|
||||
undeclared = find_undeclared_files(config, self.emulators_dir, self.db, profiles)
|
||||
emulators = {u["emulator"] for u in undeclared}
|
||||
# TestAgnostic should NOT appear in undeclared (bios_mode: agnostic)
|
||||
self.assertNotIn("TestAgnostic", emulators)
|
||||
|
||||
def test_180_agnostic_file_skipped_in_undeclared(self):
|
||||
"""Files with agnostic: true are skipped, others in same profile are not."""
|
||||
config = load_platform_config("test_existence", self.platforms_dir)
|
||||
profiles = load_emulator_profiles(self.emulators_dir)
|
||||
undeclared = find_undeclared_files(config, self.emulators_dir, self.db, profiles)
|
||||
names = {u["name"] for u in undeclared}
|
||||
# agnostic_file.bin should NOT be in undeclared (agnostic: true)
|
||||
self.assertNotIn("agnostic_file.bin", names)
|
||||
# undeclared_req.bin should still be in undeclared (not agnostic)
|
||||
self.assertIn("undeclared_req.bin", names)
|
||||
|
||||
def test_181_agnostic_extras_scan(self):
|
||||
"""Agnostic profiles add all matching DB files as extras."""
|
||||
from generate_pack import _collect_emulator_extras
|
||||
config = load_platform_config("test_existence", self.platforms_dir)
|
||||
profiles = load_emulator_profiles(self.emulators_dir)
|
||||
extras = _collect_emulator_extras(
|
||||
config, self.emulators_dir, self.db, set(), "system", profiles,
|
||||
)
|
||||
agnostic_extras = [e for e in extras if e.get("source_emulator") == "TestAgnostic"]
|
||||
# Agnostic scan should find files in the same directory as correct_hash.bin
|
||||
self.assertTrue(len(agnostic_extras) > 0, "Agnostic scan should produce extras")
|
||||
# All agnostic extras should have agnostic_scan flag
|
||||
for e in agnostic_extras:
|
||||
self.assertTrue(e.get("agnostic_scan", False))
|
||||
|
||||
def test_182_agnostic_rename_readme(self):
|
||||
"""_build_agnostic_rename_readme generates correct text."""
|
||||
from generate_pack import _build_agnostic_rename_readme
|
||||
result = _build_agnostic_rename_readme(
|
||||
"dsi_nand.bin", "DSi_Nand_AUS.bin",
|
||||
["DSi_Nand_EUR.bin", "DSi_Nand_USA.bin"],
|
||||
)
|
||||
self.assertIn("dsi_nand.bin <- DSi_Nand_AUS.bin", result)
|
||||
self.assertIn("DSi_Nand_EUR.bin", result)
|
||||
self.assertIn("DSi_Nand_USA.bin", result)
|
||||
self.assertIn("rename it to: dsi_nand.bin", result)
|
||||
|
||||
def test_183_agnostic_resolve_fallback(self):
|
||||
"""resolve_local_file with agnostic fallback finds a system file."""
|
||||
file_entry = {
|
||||
"name": "nonexistent_agnostic.bin",
|
||||
"agnostic": True,
|
||||
"min_size": 1,
|
||||
"max_size": 999999,
|
||||
"agnostic_path_prefix": self.bios_dir + "/",
|
||||
}
|
||||
path, status = resolve_local_file(file_entry, self.db)
|
||||
self.assertIsNotNone(path)
|
||||
self.assertEqual(status, "agnostic_fallback")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user