feat: list files only an emulator install provides

A file installed with its emulator is already on the user's machine and the packs leave it out. bundled_only.py lists the collected copies nothing else needs: no platform declares them, no entry of any build needs them, no install manifest ships them.

scripts/bundled_only.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Collected files that only an emulator's own install provides.
+
+A file installed with the emulator is already on the user's machine, and the
+packs leave it out. A copy in bios/ that nothing else asks for is then dead
+weight: no platform declares it, no profile entry of any build needs it from
+the collection, and no install manifest ships it. This lists those copies, so
+they can be removed, and fails under --strict while one remains.
+
+Usage:
+    python scripts/bundled_only.py
+    python scripts/bundled_only.py --strict
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from common import (
+    expand_directory_entries,
+    list_registered_platforms,
+    load_database,
+    load_emulator_profiles,
+    load_platform_config,
+)
+from cross_reference import FileConsumers
+
+
+def _entries(profile: dict, db: dict) -> list[dict]:
+    files = [f for f in profile.get("files") or [] if isinstance(f, dict)]
+    return expand_directory_entries(files, db, False) + [
+        f for f in expand_directory_entries(files, db, True) if f not in files
+    ]
+
+
+def shipped_by_manifests(install_dir: Path) -> set[str]:
+    """Every sha1 an install manifest places, alternatives included."""
+    shipped: set[str] = set()
+    for path in sorted(install_dir.glob("*.json")):
+        manifest = json.loads(path.read_text(encoding="utf-8"))
+        for entry in manifest.get("files") or []:
+            shipped.add(str(entry.get("sha1") or ""))
+            shipped.update(str(a.get("sha1") or "") for a in entry.get("alternatives") or [])
+    return shipped - {""}
+
+
+def bundled_only(db: dict, profiles: dict, platforms_dir: str, install_dir: Path) -> list[dict]:
+    """Collected files a bundled entry resolves to and nothing else needs."""
+    consumers = FileConsumers(db)
+    needed: set[str] = set(shipped_by_manifests(install_dir))
+    for name in list_registered_platforms(platforms_dir, include_archived=True):
+        config = load_platform_config(name, platforms_dir)
+        for system in config.get("systems", {}).values():
+            for entry in system.get("files", []):
+                sha1 = consumers.sha1_of(entry, entry.get("destination", ""))
+                if sha1:
+                    needed.add(sha1)
+    bundled: dict[str, set[str]] = {}
+    for emulator, profile in profiles.items():
+        if profile.get("type") == "alias":
+            continue
+        for entry in _entries(profile, db):
+            sha1 = consumers.sha1_of(entry, entry.get("path") or "", owner=emulator)
+            if not sha1:
+                continue
+            if entry.get("bundled"):
+                bundled.setdefault(sha1, set()).add(emulator)
+            else:
+                needed.add(sha1)
+    files = db.get("files", {})
+    return [
+        {"sha1": sha1, "path": files[sha1]["path"], "size": files[sha1].get("size", 0),
+         "emulators": sorted(emulators)}
+        for sha1, emulators in sorted(bundled.items(), key=lambda kv: files[kv[0]]["path"])
+        if sha1 not in needed and sha1 in files
+    ]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
+    parser.add_argument("--db", default="database.json", help="database to read (default: %(default)s)")
+    parser.add_argument("--emulators-dir", default="emulators", help="profiles (default: %(default)s)")
+    parser.add_argument("--platforms-dir", default="platforms", help="platform lists (default: %(default)s)")
+    parser.add_argument("--install-dir", default="install", type=Path,
+                        help="install manifests, whose files are shipped (default: %(default)s)")
+    parser.add_argument("--json", action="store_true", help="print the list as JSON")
+    parser.add_argument("--strict", action="store_true", help="exit 1 while a file is listed")
+    args = parser.parse_args()
+
+    rows = bundled_only(
+        load_database(args.db), load_emulator_profiles(args.emulators_dir),
+        args.platforms_dir, args.install_dir,
+    )
+    if args.json:
+        print(json.dumps(rows, indent=2))
+    else:
+        for row in rows:
+            print(f"{row['path']}  ({', '.join(row['emulators'])})")
+        total = sum(row["size"] for row in rows)
+        print(f"{len(rows)} collected files only an emulator install provides, "
+              f"{total / 1024**2:.1f} MB")
+    return 1 if args.strict and rows else 0
+
+
+if __name__ == "__main__":
+    from common import run_cli
+
+    run_cli(main)

tests/test_audit_regressions.py

@@ -5627,5 +5627,36 @@ class TheReleaseBuildsEveryServedPackInOneRun(unittest.TestCase):
         self.assertNotIn("generate_pack.py --platform retropie", page)
 
 
+class WhatOnlyAnInstallProvidesIsListed(unittest.TestCase):
+    """A file installed with its emulator is already on the user's machine.
+    A copy nothing else asks for is dead weight in the collection."""
+
+    def test_only_the_bundled_only_copy_is_listed(self):
+        import tempfile
+
+        import bundled_only
+
+        with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp, \
+                mock.patch.object(bundled_only, "list_registered_platforms", return_value=[]):
+            def entry(sha1, name):
+                path = Path(tmp, name)
+                path.write_bytes(name.encode())
+                return {"path": str(path), "name": name, "sha1": sha1,
+                        "md5": sha1[:32], "size": path.stat().st_size}
+
+            db = {"files": {"a" * 40: entry("a" * 40, "icon.png"),
+                            "b" * 40: entry("b" * 40, "bios.rom")},
+                  "indexes": {"by_name": {"icon.png": ["a" * 40], "bios.rom": ["b" * 40]},
+                              "by_md5": {}, "by_crc32": {}, "by_sha256": {},
+                              "by_path_suffix": {}}}
+            profiles = {
+                "app": {"files": [{"name": "icon.png", "sha1": "a" * 40, "bundled": True},
+                                  {"name": "bios.rom", "sha1": "b" * 40, "bundled": True}]},
+                "core": {"files": [{"name": "bios.rom", "sha1": "b" * 40}]},
+            }
+            rows = bundled_only.bundled_only(db, profiles, tmp, Path(tmp))
+            self.assertEqual([row["path"] for row in rows], [str(Path(tmp, "icon.png"))])
+
+
 if __name__ == "__main__":
     unittest.main()

wiki/tools.md

@@ -610,6 +610,7 @@ Further options, by script:
 | `migrate.py` | `--dry-run`, `--include-branches` also reads BIOS files from every remote branch |
 | `provenance_report.py` | `--provenance-dir`, `--write-derivable PATH` (with `--derive`, and the pipeline writes `derivable.json`) |
 | `refresh_stale.py` | `--freshness-arg FLAG` passes a flag through to `check_freshness.py`, repeatable |
+| `bundled_only.py` | `--strict` (exit 1 while a collected file only an emulator install provides remains), `--json`, `--install-dir` |
 | `release_record.py` | `--install-dir` (manifests the archives are held against), `--output` |
 | `release_notes.py` | `--whats-new FILE` (required, appended as written), `--record`, `--platforms-dir`, `--output` |
 | `restore_large_files.py` | `--cache` (downloaded assets), `--gitignore`, `--db` |