diff --git a/scripts/bundled_only.py b/scripts/bundled_only.py new file mode 100644 index 00000000..c6576104 --- /dev/null +++ b/scripts/bundled_only.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Collected files that only an emulator's own install provides. + +A file installed with the emulator is already on the user's machine, and the +packs leave it out. A copy in bios/ that nothing else asks for is then dead +weight: no platform declares it, no profile entry of any build needs it from +the collection, and no install manifest ships it. This lists those copies, so +they can be removed, and fails under --strict while one remains. + +Usage: + python scripts/bundled_only.py + python scripts/bundled_only.py --strict +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from common import ( + expand_directory_entries, + list_registered_platforms, + load_database, + load_emulator_profiles, + load_platform_config, +) +from cross_reference import FileConsumers + + +def _entries(profile: dict, db: dict) -> list[dict]: + files = [f for f in profile.get("files") or [] if isinstance(f, dict)] + return expand_directory_entries(files, db, False) + [ + f for f in expand_directory_entries(files, db, True) if f not in files + ] + + +def shipped_by_manifests(install_dir: Path) -> set[str]: + """Every sha1 an install manifest places, alternatives included.""" + shipped: set[str] = set() + for path in sorted(install_dir.glob("*.json")): + manifest = json.loads(path.read_text(encoding="utf-8")) + for entry in manifest.get("files") or []: + shipped.add(str(entry.get("sha1") or "")) + shipped.update(str(a.get("sha1") or "") for a in entry.get("alternatives") or []) + return shipped - {""} + + +def bundled_only(db: dict, profiles: dict, platforms_dir: str, install_dir: Path) -> list[dict]: + """Collected files a bundled entry resolves to and nothing else needs.""" + consumers = FileConsumers(db) + needed: set[str] = set(shipped_by_manifests(install_dir)) + for name in list_registered_platforms(platforms_dir, include_archived=True): + config = load_platform_config(name, platforms_dir) + for system in config.get("systems", {}).values(): + for entry in system.get("files", []): + sha1 = consumers.sha1_of(entry, entry.get("destination", "")) + if sha1: + needed.add(sha1) + bundled: dict[str, set[str]] = {} + for emulator, profile in profiles.items(): + if profile.get("type") == "alias": + continue + for entry in _entries(profile, db): + sha1 = consumers.sha1_of(entry, entry.get("path") or "", owner=emulator) + if not sha1: + continue + if entry.get("bundled"): + bundled.setdefault(sha1, set()).add(emulator) + else: + needed.add(sha1) + files = db.get("files", {}) + return [ + {"sha1": sha1, "path": files[sha1]["path"], "size": files[sha1].get("size", 0), + "emulators": sorted(emulators)} + for sha1, emulators in sorted(bundled.items(), key=lambda kv: files[kv[0]]["path"]) + if sha1 not in needed and sha1 in files + ] + + +def main() -> int: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0]) + parser.add_argument("--db", default="database.json", help="database to read (default: %(default)s)") + parser.add_argument("--emulators-dir", default="emulators", help="profiles (default: %(default)s)") + parser.add_argument("--platforms-dir", default="platforms", help="platform lists (default: %(default)s)") + parser.add_argument("--install-dir", default="install", type=Path, + help="install manifests, whose files are shipped (default: %(default)s)") + parser.add_argument("--json", action="store_true", help="print the list as JSON") + parser.add_argument("--strict", action="store_true", help="exit 1 while a file is listed") + args = parser.parse_args() + + rows = bundled_only( + load_database(args.db), load_emulator_profiles(args.emulators_dir), + args.platforms_dir, args.install_dir, + ) + if args.json: + print(json.dumps(rows, indent=2)) + else: + for row in rows: + print(f"{row['path']} ({', '.join(row['emulators'])})") + total = sum(row["size"] for row in rows) + print(f"{len(rows)} collected files only an emulator install provides, " + f"{total / 1024**2:.1f} MB") + return 1 if args.strict and rows else 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index b992864f..553ad5b6 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5627,5 +5627,36 @@ class TheReleaseBuildsEveryServedPackInOneRun(unittest.TestCase): self.assertNotIn("generate_pack.py --platform retropie", page) +class WhatOnlyAnInstallProvidesIsListed(unittest.TestCase): + """A file installed with its emulator is already on the user's machine. + A copy nothing else asks for is dead weight in the collection.""" + + def test_only_the_bundled_only_copy_is_listed(self): + import tempfile + + import bundled_only + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp, \ + mock.patch.object(bundled_only, "list_registered_platforms", return_value=[]): + def entry(sha1, name): + path = Path(tmp, name) + path.write_bytes(name.encode()) + return {"path": str(path), "name": name, "sha1": sha1, + "md5": sha1[:32], "size": path.stat().st_size} + + db = {"files": {"a" * 40: entry("a" * 40, "icon.png"), + "b" * 40: entry("b" * 40, "bios.rom")}, + "indexes": {"by_name": {"icon.png": ["a" * 40], "bios.rom": ["b" * 40]}, + "by_md5": {}, "by_crc32": {}, "by_sha256": {}, + "by_path_suffix": {}}} + profiles = { + "app": {"files": [{"name": "icon.png", "sha1": "a" * 40, "bundled": True}, + {"name": "bios.rom", "sha1": "b" * 40, "bundled": True}]}, + "core": {"files": [{"name": "bios.rom", "sha1": "b" * 40}]}, + } + rows = bundled_only.bundled_only(db, profiles, tmp, Path(tmp)) + self.assertEqual([row["path"] for row in rows], [str(Path(tmp, "icon.png"))]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/tools.md b/wiki/tools.md index 45a4c1cf..816ae606 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -610,6 +610,7 @@ Further options, by script: | `migrate.py` | `--dry-run`, `--include-branches` also reads BIOS files from every remote branch | | `provenance_report.py` | `--provenance-dir`, `--write-derivable PATH` (with `--derive`, and the pipeline writes `derivable.json`) | | `refresh_stale.py` | `--freshness-arg FLAG` passes a flag through to `check_freshness.py`, repeatable | +| `bundled_only.py` | `--strict` (exit 1 while a collected file only an emulator install provides remains), `--json`, `--install-dir` | | `release_record.py` | `--install-dir` (manifests the archives are held against), `--output` | | `release_notes.py` | `--whats-new FILE` (required, appended as written), `--record`, `--platforms-dir`, `--output` | | `restore_large_files.py` | `--cache` (downloaded assets), `--gitignore`, `--db` |