kmoody/libretro
1
0
Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-04-15 21:32:32 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: dedup edge cases — preserve non-ZIP different-name files

Browse Source 
non-ZIP files with different names but same content (64DD_IPL_US.n64
vs IPL_USA.n64) are now preserved — each name may be needed by a
different emulator. only same-name duplicates and MAME ZIP clones
are removed. added empty directory cleanup post-dedup.
This commit is contained in:
Abdessamad Derraz
2026-03-24 21:39:25 +01:00
parent fb1007496d
commit 85308edd73
1 changed files with 26 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
scripts/dedup.py
View File

@@ -106,16 +106,20 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
name_paths.sort(key=path_priority) name_paths.sort(key=path_priority)
true_dupes_to_remove.extend(name_paths[1:]) true_dupes_to_remove.extend(name_paths[1:])
# MAME device clones: different filenames, same content, in MAME dirs # Different filenames, same content — need special handling
unique_names = sorted(by_name.keys()) unique_names = sorted(by_name.keys())
if len(unique_names) > 1: if len(unique_names) > 1:
# Check if these are all MAME ZIPs # Check if these are all in MAME/Arcade dirs AND all ZIPs
all_mame = all( all_mame_zip = (
any(_is_mame_dir(p) for p in name_paths) all(
for name_paths in by_name.values() any(_is_mame_dir(p) for p in name_paths)
for name_paths in by_name.values()
)
and all(n.endswith(".zip") for n in unique_names)
) )
if all_mame and all(n.endswith(".zip") for n in unique_names): if all_mame_zip:
# Pick canonical (shortest name) and record clones # MAME device clones: different ZIP names, same ROM content
# Keep one canonical, remove clones, record in clone map
canonical_name = min(unique_names, key=len) canonical_name = min(unique_names, key=len)
clone_names = sorted(n for n in unique_names if n != canonical_name) clone_names = sorted(n for n in unique_names if n != canonical_name)
if clone_names: if clone_names:
@@ -124,11 +128,14 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
"clones": clone_names, "clones": clone_names,
"total_copies": sum(len(by_name[n]) for n in clone_names), "total_copies": sum(len(by_name[n]) for n in clone_names),
} }
# Remove all clone copies (keep one per unique name for now,
# or remove all clones and rely on pack-time assembly)
for clone_name in clone_names: for clone_name in clone_names:
for p in by_name[clone_name]: for p in by_name[clone_name]:
true_dupes_to_remove.append(p) true_dupes_to_remove.append(p)
else:
# Non-MAME different names (e.g., 64DD_IPL_US.n64 vs IPL_USA.n64)
# Keep ALL — each name may be needed by a different emulator
# Only remove true duplicates (same name in multiple dirs)
pass
if not true_dupes_to_remove: if not true_dupes_to_remove:
continue continue
@@ -181,9 +188,19 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
if alias_names: if alias_names:
print(f" MAME clones: {alias_names}") print(f" MAME clones: {alias_names}")
# Clean up empty directories
if not dry_run:
empty_cleaned = 0
for root, dirs, files in os.walk(bios_dir, topdown=False):
if not files and not dirs and root != bios_dir:
os.rmdir(root)
empty_cleaned += 1
prefix = "Would remove" if dry_run else "Removed" prefix = "Would remove" if dry_run else "Removed"
print(f"\n{prefix}: {total_removed} files") print(f"\n{prefix}: {total_removed} files")
print(f"Space {'to save' if dry_run else 'saved'}: {total_saved / 1024 / 1024:.1f} MB") print(f"Space {'to save' if dry_run else 'saved'}: {total_saved / 1024 / 1024:.1f} MB")
if not dry_run and empty_cleaned:
print(f"Cleaned {empty_cleaned} empty directories")
# Write MAME clone mapping # Write MAME clone mapping
if mame_clones: if mame_clones: