From 837ea786b0f1d0ec7b72bcf8c50a17539e51f289 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Tue, 11 Aug 2026 18:12:35 +0200
Subject: [PATCH] refactor: separate the dedup decision from the deletion

deduplicate reached complexity 50 by mixing the question of which
copies may go with the work of removing them. The decision is now its
own function: same name in two directories is a true duplicate,
different names only collapse as MAME device clones, and anywhere else
each name may be the one some emulator looks for.

Verified against the real collection: both versions plan the same 99
removals.
---
 scripts/dedup.py | 97 +++++++++++++++++++++++++++++-------------------
 1 file changed, 58 insertions(+), 39 deletions(-)

diff --git a/scripts/dedup.py b/scripts/dedup.py
index e4582691..a23e88e9 100644
--- a/scripts/dedup.py
+++ b/scripts/dedup.py
@@ -77,6 +77,63 @@ def scan_duplicates(bios_dir: str) -> dict[str, list[str]]:
     return sha1_to_paths
 
 
+def _plan_group_removals(
+    sha1: str, paths: list[str], mame_clones: dict
+) -> list[str]:
+    """Decide which copies of one content hash may go.
+
+    Two files with the same content are not automatically interchangeable.
+    Same name in different directories is a true duplicate. Different names
+    are only collapsible when they are MAME device clones, which is why the
+    check is narrowed to zips under a MAME directory: anywhere else each
+    name may be the one some emulator looks for.
+
+    Records clone groups in *mame_clones* and returns the paths to remove.
+    """
+    true_dupes_to_remove: list[str] = []
+    # Separate by filename -same name = true duplicate, different name = clone
+    by_name: dict[str, list[str]] = defaultdict(list)
+    for p in paths:
+        by_name[os.path.basename(p)].append(p)
+
+    # True duplicates: same filename in multiple directories
+    true_dupes_to_remove = []
+    for name, name_paths in by_name.items():
+        if len(name_paths) > 1:
+            name_paths.sort(key=path_priority)
+            true_dupes_to_remove.extend(name_paths[1:])
+
+    # Different filenames, same content -need special handling
+    unique_names = sorted(by_name.keys())
+    if len(unique_names) > 1:
+        # Check if these are all in MAME/Arcade dirs AND all ZIPs
+        all_mame_zip = all(
+            any(_is_mame_dir(p) for p in name_paths)
+            for name_paths in by_name.values()
+        ) and all(n.endswith(".zip") for n in unique_names)
+        if all_mame_zip:
+            # MAME device clones: different ZIP names, same ROM content
+            # Keep one canonical, remove clones, record in clone map
+            canonical_name = min(unique_names, key=len)
+            clone_names = sorted(n for n in unique_names if n != canonical_name)
+            if clone_names:
+                mame_clones[canonical_name] = {
+                    "sha1": sha1,
+                    "clones": clone_names,
+                    "total_copies": sum(len(by_name[n]) for n in clone_names),
+                }
+                for clone_name in clone_names:
+                    for p in by_name[clone_name]:
+                        true_dupes_to_remove.append(p)
+        else:
+            # Non-MAME different names (e.g., 64DD_IPL_US.n64 vs IPL_USA.n64)
+            # Keep ALL -each name may be needed by a different emulator
+            # Only remove true duplicates (same name in multiple dirs)
+            pass
+
+    return true_dupes_to_remove
+
+
 def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
     """Remove true duplicates, map MAME device clones.
 
@@ -95,45 +152,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
         if len(paths) <= 1:
             continue
 
-        # Separate by filename -same name = true duplicate, different name = clone
-        by_name: dict[str, list[str]] = defaultdict(list)
-        for p in paths:
-            by_name[os.path.basename(p)].append(p)
-
-        # True duplicates: same filename in multiple directories
-        true_dupes_to_remove = []
-        for name, name_paths in by_name.items():
-            if len(name_paths) > 1:
-                name_paths.sort(key=path_priority)
-                true_dupes_to_remove.extend(name_paths[1:])
-
-        # Different filenames, same content -need special handling
-        unique_names = sorted(by_name.keys())
-        if len(unique_names) > 1:
-            # Check if these are all in MAME/Arcade dirs AND all ZIPs
-            all_mame_zip = all(
-                any(_is_mame_dir(p) for p in name_paths)
-                for name_paths in by_name.values()
-            ) and all(n.endswith(".zip") for n in unique_names)
-            if all_mame_zip:
-                # MAME device clones: different ZIP names, same ROM content
-                # Keep one canonical, remove clones, record in clone map
-                canonical_name = min(unique_names, key=len)
-                clone_names = sorted(n for n in unique_names if n != canonical_name)
-                if clone_names:
-                    mame_clones[canonical_name] = {
-                        "sha1": sha1,
-                        "clones": clone_names,
-                        "total_copies": sum(len(by_name[n]) for n in clone_names),
-                    }
-                    for clone_name in clone_names:
-                        for p in by_name[clone_name]:
-                            true_dupes_to_remove.append(p)
-            else:
-                # Non-MAME different names (e.g., 64DD_IPL_US.n64 vs IPL_USA.n64)
-                # Keep ALL -each name may be needed by a different emulator
-                # Only remove true duplicates (same name in multiple dirs)
-                pass
+        true_dupes_to_remove = _plan_group_removals(sha1, paths, mame_clones)
 
         if not true_dupes_to_remove:
             continue