fix: account for every profile entry in gaps

scripts/cross_reference.py

@@ -19,6 +19,7 @@ import os
 import sys
 import zipfile
 import zlib
+from collections import Counter
 from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
@@ -668,6 +669,32 @@ def slot_key(f: dict, standalone: bool = False) -> tuple:
     )
 
 
+# Why an entry is left out of the comparison, in the order they are printed.
+SET_ASIDE = {
+    "other_versions": "other versions of a slot",
+    "not_read": "not read on this platform",
+    "outside_scope": "outside the pack",
+}
+
+
+def profile_breakdown(data: dict) -> str:
+    """Where each entry of a profile went, adding up to its file count.
+
+    The heading said 261 files, 7 declared and 238 undeclared, and the 16
+    others were nowhere: the versions a slot accepts, the entries the
+    platform never opens and the ones no pack carries.
+    """
+    parts = [f"{data['declared_files']} declared by platforms",
+             f"{data['undeclared_files']} undeclared"]
+    if data["unsourceable_files"]:
+        parts.append(f"{data['unsourceable_files']} unsourceable")
+    aside = data["set_aside"]
+    detail = [f"{aside[key]} {label}" for key, label in SET_ASIDE.items() if aside.get(key)]
+    if detail:
+        parts.append(f"{sum(aside.values())} set aside ({', '.join(detail)})")
+    return ", ".join(parts)
+
+
 def _cross_reference_profile(
     emu_name: str,
     profile: dict,
@@ -727,6 +754,19 @@ def _cross_reference_profile(
     for f, is_standalone in layout_files:
         versions.setdefault(slot_key(f, is_standalone), []).append(f)
 
+    # What became of each entry, so the heading adds up: a file a platform
+    # declares, one it does not, or one set aside and why. An entry read by
+    # two builds keeps the first answer that compares it. Each layout lists
+    # the profile's entries in their order, so a position names one entry.
+    position = {id(f): i for view, _ in layouts for i, f in enumerate(view.get("files", []))}
+    fates: dict[int, str] = {}
+
+    def fate(f: dict, what: str) -> None:
+        key = position[id(f)]
+        current = fates.get(key)
+        if current is None or (current in SET_ASIDE and what not in SET_ASIDE):
+            fates[key] = what
+
     for f, is_standalone in layout_files:
         fname = f.get("name", "")
         # The destination the running build reads, as find_undeclared_files
@@ -745,13 +785,21 @@ def _cross_reference_profile(
         if len(layouts) > 1:
             # One file read at one path by both builds counts once.
             seen_key = (effective_path.casefold(), seen_key)
-        if not fname or seen_key in seen_files or unread_on(f, platform):
+        if not fname:
+            fate(f, "outside_scope")
+            continue
+        if seen_key in seen_files:
+            fate(f, "other_versions")
+            continue
+        if unread_on(f, platform):
+            fate(f, "not_read")
             continue
 
         # Collect unsourceable files separately (documented, not a gap)
         unsourceable_reason = f.get("unsourceable", "")
         if unsourceable_reason:
             seen_files.add(seen_key)
+            fate(f, "unsourceable")
             unsourceable_list.append({
                 "name": fname,
                 "required": f.get("required", False),
@@ -761,6 +809,7 @@ def _cross_reference_profile(
             continue
 
         if outside_gap_scope(f, is_standalone):
+            fate(f, "outside_scope")
             continue
 
         archive = f.get("archive")
@@ -776,6 +825,7 @@ def _cross_reference_profile(
 
         if in_platform:
             seen_files.add(seen_key)
+            fate(f, "declared")
             covered.append({
                 "name": fname,
                 "path": effective_path,
@@ -785,6 +835,7 @@ def _cross_reference_profile(
             continue
 
         seen_files.add(seen_key)
+        fate(f, "undeclared")
 
         if archive:
             archive_key = (
@@ -856,6 +907,7 @@ def _cross_reference_profile(
     for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]):
         gaps.append(ag)
 
+    tally = Counter(fates.values())
     report[emu_name] = {
         "emulator": profile.get("emulator", emu_name),
         "systems": systems,
@@ -869,6 +921,11 @@ def _cross_reference_profile(
         "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"),
         "gap_details": gaps,
         "unsourceable": unsourceable_list,
+        # Entries, not rows: an archive's members are one gap row.
+        "declared_files": tally["declared"],
+        "undeclared_files": tally["undeclared"],
+        "unsourceable_files": tally["unsourceable"],
+        "set_aside": {key: tally[key] for key in SET_ASIDE},
     }
 
 
@@ -969,11 +1026,7 @@ def print_report(
 
         systems = ", ".join(data["systems"])
         print(f"\n{data['emulator']}" + (f" ({systems})" if systems else ""))
-        print(
-            f"  {data['total_files']} files in profile, "
-            f"{data['platform_covered']} declared by platforms, "
-            f"{gaps} undeclared"
-        )
+        print(f"  {data['total_files']} files in profile: {profile_breakdown(data)}")
         print(f"  Gaps: {status}")
 
         for g in data["gap_details"]:

tests/test_tool_messages.py

@@ -97,6 +97,39 @@ class AMistypedDataKeyGetsTheClosestName(unittest.TestCase):
                       proc.stderr)
         self.assertNotIn("ppsspp-assets", proc.stderr)
 
+class TheGapHeadingAddsUp(unittest.TestCase):
+    """Amiberry read 261 files in profile, 7 declared and 238 undeclared:
+    16 entries were set aside without a word."""
+
+    def test_every_entry_is_counted_once(self):
+        import cross_reference as xref
+
+        files = [
+            {"name": "kick.rom", "required": True},
+            {"name": "game.grp", "size": 10},
+            {"name": "game.grp", "size": 20},
+            {"name": "a.bin", "archive": "set.zip"},
+            {"name": "b.bin", "archive": "set.zip"},
+            {"name": "picked.bin", "path": None},
+            {"name": "gone.bin", "unsourceable": "never distributed"},
+        ]
+        profiles = {"emu": {"emulator": "Emu", "type": "libretro", "systems": ["s"],
+                            "files": files}}
+        report = xref.cross_reference(profiles, {}, {"files": {}, "indexes": {}},
+                                      all_declared={"kick.rom": [(None, "kick.rom")]})
+        data = report["emu"]
+        self.assertEqual(
+            xref.profile_breakdown(data),
+            "1 declared by platforms, 3 undeclared, 1 unsourceable, "
+            "2 set aside (1 other versions of a slot, 1 outside the pack)",
+        )
+        self.assertEqual(
+            data["declared_files"] + data["undeclared_files"] + data["unsourceable_files"]
+            + sum(data["set_aside"].values()),
+            data["total_files"],
+        )
+
+
 class TheTruthDiffShowsWhereAndAsksNothing(unittest.TestCase):
     """The markdown wrote `**Remove** hiscore.dat (phantom)` beside six
     `**Add** hiscore.dat`, none with a path, for files profiled cores read."""