From 5763ce731dc570c27c3b7b500f6c502ca2984992 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sat, 28 Mar 2026 18:02:54 +0100 Subject: [PATCH] feat: add --manifest mode to generate_pack --- scripts/generate_pack.py | 261 +++++++++++++++++++++++++++++++++++++++ tests/test_e2e.py | 86 +++++++++++++ 2 files changed, 347 insertions(+) diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 1f1c4824..6fc34aee 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -1338,6 +1338,8 @@ def main(): help="Hash(es) to look up or pack (comma-separated)") parser.add_argument("--from-md5-file", help="File with hashes (one per line)") + parser.add_argument("--manifest", action="store_true", + help="Output JSON manifests instead of ZIP packs") args = parser.parse_args() if args.list: @@ -1403,6 +1405,12 @@ def main(): parser.error("--target requires --platform or --all") if args.target and has_emulator: parser.error("--target is incompatible with --emulator") + if args.manifest and not (has_platform or has_all): + parser.error("--manifest requires --platform or --all") + if args.manifest and has_emulator: + parser.error("--manifest is incompatible with --emulator") + if args.manifest and args.split: + parser.error("--manifest is incompatible with --split") # Hash lookup / pack mode if has_from_md5: @@ -1506,6 +1514,29 @@ def main(): groups = group_identical_platforms(platforms, args.platforms_dir, target_cores_cache if args.target else None) + # Manifest mode: JSON output instead of ZIP + if args.manifest: + registry_path = os.path.join(args.platforms_dir, "_registry.yml") + os.makedirs(args.output_dir, exist_ok=True) + for group_platforms, representative in groups: + print(f"\nGenerating manifest for {representative}...") + try: + tc = target_cores_cache.get(representative) if args.target else None + manifest = generate_manifest( + representative, args.platforms_dir, db, args.bios_dir, + registry_path, emulators_dir=args.emulators_dir, + zip_contents=zip_contents, emu_profiles=emu_profiles, + target_cores=tc, + ) + out_path = os.path.join(args.output_dir, f"{representative}.json") + with open(out_path, "w") as f: + json.dump(manifest, f, indent=2) + print(f" {out_path}: {manifest['total_files']} files, " + f"{manifest['total_size']} bytes") + except (FileNotFoundError, OSError, yaml.YAMLError) as e: + print(f" ERROR: {e}") + return + for group_platforms, representative in groups: variants = [p for p in group_platforms if p != representative] if variants: @@ -1567,6 +1598,236 @@ def main(): sys.exit(1) +# --------------------------------------------------------------------------- +# Manifest generation (JSON inventory for install.py) +# --------------------------------------------------------------------------- + +_GITIGNORE_ENTRIES: set[str] | None = None + + +def _load_gitignore_entries(repo_root: str) -> set[str]: + """Load gitignored paths (large files) from .gitignore at repo root.""" + global _GITIGNORE_ENTRIES + if _GITIGNORE_ENTRIES is not None: + return _GITIGNORE_ENTRIES + gitignore = os.path.join(repo_root, ".gitignore") + entries: set[str] = set() + if os.path.exists(gitignore): + with open(gitignore) as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + entries.add(line) + _GITIGNORE_ENTRIES = entries + return entries + + +def _is_large_file(local_path: str, repo_root: str) -> bool: + """Check if a file is a large file (>50MB or in .gitignore).""" + if local_path and os.path.exists(local_path): + if os.path.getsize(local_path) > 50_000_000: + return True + gitignore = _load_gitignore_entries(repo_root) + # Check if the path relative to repo root is in .gitignore + try: + rel = os.path.relpath(local_path, repo_root) + except ValueError: + rel = "" + return rel in gitignore + + +def _get_repo_path(sha1: str, db: dict) -> str: + """Get the repo path for a file by SHA1 lookup.""" + entry = db.get("files", {}).get(sha1, {}) + return entry.get("path", "") + + +def generate_manifest( + platform_name: str, + platforms_dir: str, + db: dict, + bios_dir: str, + registry_path: str, + emulators_dir: str = "emulators", + zip_contents: dict | None = None, + emu_profiles: dict | None = None, + target_cores: set[str] | None = None, +) -> dict: + """Generate a JSON manifest for a platform (same resolution as generate_pack). + + Returns a dict ready for JSON serialization with file inventory, + install hints, and download metadata. + """ + config = load_platform_config(platform_name, platforms_dir) + if zip_contents is None: + zip_contents = {} + if emu_profiles is None: + emu_profiles = load_emulator_profiles(emulators_dir) + + platform_display = config.get("platform", platform_name) + base_dest = config.get("base_destination", "") + case_insensitive = config.get("case_insensitive_fs", False) + + # Load registry for install metadata + registry: dict = {} + if os.path.exists(registry_path): + with open(registry_path) as f: + registry = yaml.safe_load(f) or {} + plat_registry = registry.get("platforms", {}).get(platform_name, {}) + install_section = plat_registry.get("install", {}) + detect = install_section.get("detect", []) + standalone_copies = install_section.get("standalone_copies", []) + + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # Filter systems by target + from common import resolve_platform_cores + plat_cores = resolve_platform_cores(config, emu_profiles) if target_cores else None + pack_systems = filter_systems_by_target( + config.get("systems", {}), + emu_profiles, + target_cores, + platform_cores=plat_cores, + ) + + seen_destinations: set[str] = set() + seen_lower: set[str] = set() + seen_parents: set[str] = set() + manifest_files: list[dict] = [] + total_size = 0 + + # Phase 1: baseline files + for sys_id, system in sorted(pack_systems.items()): + for file_entry in system.get("files", []): + dest = _sanitize_path(file_entry.get("destination", file_entry["name"])) + if not dest: + continue + full_dest = f"{base_dest}/{dest}" if base_dest else dest + + dedup_key = full_dest + if dedup_key in seen_destinations: + continue + if case_insensitive and dedup_key.lower() in seen_lower: + continue + if _has_path_conflict(full_dest, seen_destinations, seen_parents): + continue + + storage = file_entry.get("storage", "embedded") + if storage == "user_provided": + continue + + local_path, status = resolve_file(file_entry, db, bios_dir, zip_contents) + if status in ("not_found", "external"): + continue + + # Get SHA1 and size + sha1 = file_entry.get("sha1", "") + file_size = 0 + if local_path and os.path.exists(local_path): + file_size = os.path.getsize(local_path) + if not sha1: + hashes = compute_hashes(local_path) + sha1 = hashes["sha1"] + + repo_path = _get_repo_path(sha1, db) if sha1 else "" + + entry: dict = { + "dest": full_dest, + "sha1": sha1, + "size": file_size, + "repo_path": repo_path, + "cores": None, + } + + if _is_large_file(local_path or "", repo_root): + entry["storage"] = "release" + entry["release_asset"] = os.path.basename(local_path) if local_path else file_entry["name"] + + manifest_files.append(entry) + total_size += file_size + seen_destinations.add(dedup_key) + _register_path(dedup_key, seen_destinations, seen_parents) + if case_insensitive: + seen_lower.add(dedup_key.lower()) + + # Phase 2: core complement (emulator extras) + core_files = _collect_emulator_extras( + config, emulators_dir, db, + seen_destinations, base_dest, emu_profiles, target_cores=target_cores, + ) + for fe in core_files: + dest = _sanitize_path(fe.get("destination", fe["name"])) + if not dest: + continue + if base_dest: + full_dest = f"{base_dest}/{dest}" + elif "/" not in dest: + full_dest = f"bios/{dest}" + else: + full_dest = dest + + if full_dest in seen_destinations: + continue + if case_insensitive and full_dest.lower() in seen_lower: + continue + if _has_path_conflict(full_dest, seen_destinations, seen_parents): + continue + + local_path, status = resolve_file(fe, db, bios_dir, zip_contents) + if status in ("not_found", "external", "user_provided"): + continue + + sha1 = "" + file_size = 0 + if local_path and os.path.exists(local_path): + file_size = os.path.getsize(local_path) + hashes = compute_hashes(local_path) + sha1 = hashes["sha1"] + + repo_path = _get_repo_path(sha1, db) if sha1 else "" + source_emu = fe.get("source_emulator", "") + + entry = { + "dest": full_dest, + "sha1": sha1, + "size": file_size, + "repo_path": repo_path, + "cores": [source_emu] if source_emu else [], + } + + if _is_large_file(local_path or "", repo_root): + entry["storage"] = "release" + entry["release_asset"] = os.path.basename(local_path) if local_path else fe["name"] + + manifest_files.append(entry) + total_size += file_size + seen_destinations.add(full_dest) + _register_path(full_dest, seen_destinations, seen_parents) + if case_insensitive: + seen_lower.add(full_dest.lower()) + + # No phase 3 (data directories) — skipped for manifest + + now = __import__("datetime").datetime.now( + __import__("datetime").timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + + result: dict = { + "manifest_version": 1, + "platform": platform_name, + "display_name": platform_display, + "version": "1.0", + "generated": now, + "base_destination": base_dest, + "detect": detect, + "standalone_copies": standalone_copies, + "total_files": len(manifest_files), + "total_size": total_size, + "files": manifest_files, + } + return result + + # --------------------------------------------------------------------------- # Post-generation pack verification + manifest + SHA256SUMS # --------------------------------------------------------------------------- diff --git a/tests/test_e2e.py b/tests/test_e2e.py index a5f26008..84fcd354 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -2437,6 +2437,92 @@ class TestE2E(unittest.TestCase): "standalone_copies", registry["platforms"]["emudeck"]["install"], ) + def test_91_generate_manifest(self): + """generate_manifest returns valid manifest dict with expected fields.""" + from generate_pack import generate_manifest + + # Create a minimal registry file for the test + registry_path = os.path.join(self.platforms_dir, "_test_registry.yml") + registry_data = { + "platforms": { + "test_existence": { + "install": { + "detect": [{"os": "linux", "method": "path_exists", + "path": "/test/bios"}], + }, + }, + }, + } + with open(registry_path, "w") as fh: + yaml.dump(registry_data, fh) + + manifest = generate_manifest( + "test_existence", self.platforms_dir, self.db, self.bios_dir, + registry_path, emulators_dir=self.emulators_dir, + ) + + self.assertEqual(manifest["manifest_version"], 1) + self.assertEqual(manifest["platform"], "test_existence") + self.assertEqual(manifest["display_name"], "TestExistence") + self.assertIn("generated", manifest) + self.assertIn("files", manifest) + self.assertIsInstance(manifest["files"], list) + self.assertEqual(manifest["total_files"], len(manifest["files"])) + self.assertGreater(len(manifest["files"]), 0) + self.assertEqual(manifest["base_destination"], "system") + self.assertEqual(manifest["detect"], registry_data["platforms"]["test_existence"]["install"]["detect"]) + + for f in manifest["files"]: + self.assertIn("dest", f) + self.assertIn("sha1", f) + self.assertIn("size", f) + self.assertIn("repo_path", f) + self.assertIn("cores", f) + self.assertIsInstance(f["size"], int) + self.assertGreater(len(f["sha1"]), 0) + + def test_92_manifest_matches_zip(self): + """Manifest file destinations match ZIP contents (excluding metadata).""" + from generate_pack import generate_manifest, generate_pack + + registry_path = os.path.join(self.platforms_dir, "_test_registry.yml") + registry_data = { + "platforms": { + "test_existence": { + "install": {"detect": []}, + }, + }, + } + with open(registry_path, "w") as fh: + yaml.dump(registry_data, fh) + + # Generate ZIP + output_dir = os.path.join(self.root, "pack_manifest_cmp") + os.makedirs(output_dir, exist_ok=True) + zip_path = generate_pack( + "test_existence", self.platforms_dir, self.db, self.bios_dir, + output_dir, emulators_dir=self.emulators_dir, + ) + self.assertIsNotNone(zip_path) + + # Get ZIP file destinations (exclude metadata) + with zipfile.ZipFile(zip_path) as zf: + zip_names = { + n for n in zf.namelist() + if not n.startswith("INSTRUCTIONS_") + and n != "manifest.json" + and n != "README.txt" + } + + # Generate manifest + manifest = generate_manifest( + "test_existence", self.platforms_dir, self.db, self.bios_dir, + registry_path, emulators_dir=self.emulators_dir, + ) + manifest_dests = {f["dest"] for f in manifest["files"]} + + self.assertEqual(manifest_dests, zip_names) + if __name__ == "__main__": unittest.main()