From 24e9b820ae804002031671da854cfc08e01cbf70 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:55:42 +0200 Subject: [PATCH] feat: make packs reproducible byte for byte Two builds of the same pack from the same inputs produced different archives. Of 67 members, 65 were already identical: only README.txt and manifest.json differed, both stamped with the wall clock by writestr and the second carrying a generated timestamp. Generated members now use the epoch the archive rebuilder already applies, and the timestamp comes from the database snapshot the pack was built from, so the same data yields the same bytes. Install manifests skipped archived platforms, which is why RetroPie had none; archived means upstream is no longer scraped, not that the packs stopped shipping. A target-filtered manifest also had no record of its filter beyond the filename, so it carries one the way a region-filtered manifest already does. --- schemas/install-manifest.schema.json | 216 ++++++++++++++++++++++----- scripts/generate_pack.py | 120 +++++++++++---- tests/test_deterministic_zip.py | 97 ++++++++++++ 3 files changed, 370 insertions(+), 63 deletions(-) diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index 9da97398..0e9ff2ef 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -4,37 +4,136 @@ "title": "RetroBIOS installer manifest", "type": "object", "additionalProperties": false, - "required": ["manifest_version", "source", "regions", "platform", "display_name", "version", "generated", "base_destination", "detect", "standalone_copies", "total_files", "total_size", "total_omitted", "omitted_files", "files"], + "required": [ + "manifest_version", + "source", + "regions", + "platform", + "display_name", + "version", + "generated", + "base_destination", + "detect", + "standalone_copies", + "total_files", + "total_size", + "total_omitted", + "omitted_files", + "files" + ], "properties": { - "manifest_version": { "const": 2 }, - "source": { "enum": ["full", "platform", "truth"] }, - "regions": { "type": "array", "items": { "type": "string" }, "uniqueItems": true }, - "platform": { "type": "string", "minLength": 1 }, - "display_name": { "type": "string", "minLength": 1 }, - "version": { "type": "string" }, - "generated": { "type": "string", "format": "date-time" }, - "base_destination": { "type": "string" }, - "detect": { "type": "array", "items": { "type": "object" } }, - "standalone_copies": { "type": "array", "items": { "type": "object" } }, - "total_files": { "type": "integer", "minimum": 0 }, - "total_size": { "type": "integer", "minimum": 0 }, - "total_omitted": { "type": "integer", "minimum": 0 }, + "manifest_version": { + "const": 2 + }, + "source": { + "enum": [ + "full", + "platform", + "truth" + ] + }, + "regions": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "target": { + "type": "string", + "minLength": 1, + "description": "Hardware target this manifest was filtered for; absent when unfiltered" + }, + "platform": { + "type": "string", + "minLength": 1 + }, + "display_name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string" + }, + "generated": { + "type": "string", + "format": "date-time" + }, + "base_destination": { + "type": "string" + }, + "detect": { + "type": "array", + "items": { + "type": "object" + } + }, + "standalone_copies": { + "type": "array", + "items": { + "type": "object" + } + }, + "total_files": { + "type": "integer", + "minimum": 0 + }, + "total_size": { + "type": "integer", + "minimum": 0 + }, + "total_omitted": { + "type": "integer", + "minimum": 0 + }, "omitted_files": { "type": "array", "items": { "type": "object", "additionalProperties": false, - "required": ["dest", "name", "system", "required", "reason", "cores"], + "required": [ + "dest", + "name", + "system", + "required", + "reason", + "cores" + ], "properties": { - "dest": { "type": "string", "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" }, - "name": { "type": "string", "minLength": 1 }, - "system": { "type": "string" }, - "required": { "type": "boolean" }, - "reason": { "enum": ["hash_mismatch", "not_found", "external", "user_provided"] }, + "dest": { + "type": "string", + "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "system": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "reason": { + "enum": [ + "hash_mismatch", + "not_found", + "external", + "user_provided" + ] + }, "cores": { "oneOf": [ - { "type": "null" }, - { "type": "array", "items": { "type": "string" }, "uniqueItems": true } + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } ] } } @@ -45,27 +144,74 @@ "items": { "type": "object", "additionalProperties": false, - "required": ["dest", "sha1", "sha256", "size", "repo_path", "cores"], + "required": [ + "dest", + "sha1", + "sha256", + "size", + "repo_path", + "cores" + ], "properties": { - "dest": { "type": "string", "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" }, - "sha1": { "type": "string", "pattern": "^[a-f0-9]{40}$" }, - "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, - "size": { "type": "integer", "minimum": 0 }, - "repo_path": { "type": "string" }, + "dest": { + "type": "string", + "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" + }, + "sha1": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "repo_path": { + "type": "string" + }, "cores": { "oneOf": [ - { "type": "null" }, - { "type": "array", "items": { "type": "string" }, "uniqueItems": true } + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } ] }, - "storage": { "const": "release" }, - "release_asset": { "type": "string", "pattern": "^[^/\\\\]+$" } + "storage": { + "const": "release" + }, + "release_asset": { + "type": "string", + "pattern": "^[^/\\\\]+$" + } }, "oneOf": [ - { "required": ["release_asset"] }, { - "properties": { "repo_path": { "type": "string", "pattern": "^bios/" } }, - "not": { "required": ["release_asset"] } + "required": [ + "release_asset" + ] + }, + { + "properties": { + "repo_path": { + "type": "string", + "pattern": "^bios/" + } + }, + "not": { + "required": [ + "release_asset" + ] + } } ] } diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 9d7ba056..74de145a 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -27,9 +27,8 @@ from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from common import ( - ArtifactLockBusy, - MANUFACTURER_PREFIXES, artifact_lock, + ArtifactLockBusy, build_target_cores_cache, build_zip_contents_index, check_inside_zip, @@ -46,14 +45,17 @@ from common import ( load_database, load_emulator_profiles, load_platform_config, + MANUFACTURER_PREFIXES, parse_md5_list, require_yaml, resolution_is_hash_exact, resolve_local_file, safe_extract_zip, + yaml_load, ) import region as region_mod -from deterministic_zip import rebuild_zip_deterministic +from deterministic_zip import _FIXED_DATE_TIME, rebuild_zip_deterministic +from nativemode import hash_mismatch_excludes_file from validation import ( _build_validation_index, check_file_validation, @@ -66,7 +68,6 @@ DEFAULT_PLATFORMS_DIR = "platforms" DEFAULT_DB_FILE = "database.json" DEFAULT_OUTPUT_DIR = "dist" DEFAULT_BIOS_DIR = "bios" -MAX_ENTRY_SIZE = 512 * 1024 * 1024 # 512MB # CLI-wide network policy. Public helpers can override it explicitly, while # every CLI mode inherits --offline (including large-file release fallback). @@ -75,6 +76,35 @@ _OFFLINE = False _HEX_RE = re.compile(r"\b([0-9a-fA-F]{8,40})\b") +def _write_generated_member(zf: zipfile.ZipFile, arcname: str, text: str) -> None: + """Write a pack member the builder composes rather than copies. + + READMEs, instruction notes and the manifest are produced at build time, + so writestr would stamp them with the wall clock and give the same pack + a different hash on every run. Pinning them to the epoch the archive + rebuilder already uses makes a pack a function of its inputs alone. + """ + info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o100644 << 16 + zf.writestr(info, text) + + +def _build_timestamp(db: dict | None = None) -> str: + """Timestamp for generated artifacts. + + Reads the database snapshot the artifact was built from, so rebuilding + the same data twice yields the same value. Falls back to the clock only + when no database is at hand. + """ + stamp = (db or {}).get("generated_at") + if isinstance(stamp, str) and stamp: + return stamp + from datetime import datetime, timezone + + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + def _detect_hash_type(h: str) -> str: n = len(h) if n == 40: @@ -222,6 +252,17 @@ def _find_candidate_satisfying_both( return None +def _target_tag(target_name: str) -> str: + """Filename tag for a hardware target. + + Without it a targeted pack takes the same name as the full one and + overwrites it. + """ + return "".join( + part.title() for part in re.split(r"[-_\s]+", target_name.strip()) if part + ) + + def _sanitize_path(raw: str) -> str: """Strip path traversal components from a relative path.""" raw = raw.replace("\\", "/") @@ -1120,6 +1161,7 @@ def generate_pack( source: str = "full", flatten: bool = True, regions: list[str] | None = None, + target_name: str | None = None, offline: bool | None = None, ) -> str | None: """Generate a ZIP pack for a platform. @@ -1139,6 +1181,7 @@ def generate_pack( req_tag = "_Required" if required_only else "" source_tag = {"platform": "_Platform", "truth": "_Truth"}.get(source, "") region_tag_str = f"_{region_mod.region_tag(regions)}" if regions else "" + target_tag = f"_{_target_tag(target_name)}" if target_name else "" sys_tag = "" if system_filter: @@ -1153,7 +1196,7 @@ def generate_pack( display_parts.append("_".join(p.title() for p in parts if p)) sys_tag = "_" + "_".join(display_parts) - zip_name = f"{platform_display.replace(' ', '_')}{version_tag}{source_tag}{region_tag_str}{req_tag}_BIOS_Pack{sys_tag}.zip" + zip_name = f"{platform_display.replace(' ', '_')}{version_tag}{source_tag}{region_tag_str}{target_tag}{req_tag}_BIOS_Pack{sys_tag}.zip" zip_path = os.path.join(output_dir, zip_name) os.makedirs(output_dir, exist_ok=True) @@ -1364,7 +1407,8 @@ def generate_pack( instr_path = ( f"{base_dest}/{instr_name}" if base_dest else instr_name ) - zf.writestr( + _write_generated_member( + zf, _flat(instr_path, base_dest, flatten), f"File needed: {file_entry['name']}\n\n{instructions}\n", ) @@ -1472,7 +1516,11 @@ def generate_pack( else readme_name ) if readme_full not in seen_destinations: - zf.writestr(_flat(readme_full, base_dest, flatten), readme_text) + _write_generated_member( + zf, + _flat(readme_full, base_dest, flatten), + readme_text, + ) seen_destinations.add(readme_full) status = "agnostic_fallback" # Fall through to normal packing below @@ -1749,7 +1797,7 @@ def generate_pack( _pack_registry: dict = {} if _registry_path.exists(): with open(_registry_path) as _rf: - _pack_registry = (yaml.safe_load(_rf) or {}).get("platforms", {}) + _pack_registry = (yaml_load(_rf) or {}).get("platforms", {}) readme_text = _build_readme( platform_name, platform_display, base_dest, total_files, num_systems, source=source, @@ -1757,7 +1805,7 @@ def generate_pack( regions=regions, fallback_systems=region_fallbacks, ) - zf.writestr("README.txt", readme_text) + _write_generated_member(zf, "README.txt", readme_text) files_ok = sum(1 for s in file_status.values() if s == "ok") files_untested = sum(1 for s in file_status.values() if s == "untested") @@ -2123,7 +2171,9 @@ def generate_emulator_pack( seen_lower.add(dest.lower()) instr = fe.get("instructions", "Please provide this file manually.") instr_name = f"INSTRUCTIONS_{fe['name']}.txt" - zf.writestr(instr_name, f"File needed: {fe['name']}\n\n{instr}\n") + _write_generated_member( + zf, instr_name, f"File needed: {fe['name']}\n\n{instr}\n" + ) total_files += 1 continue @@ -2321,6 +2371,7 @@ def generate_split_packs( required_only: bool = False, source: str = "full", regions: list[str] | None = None, + target_name: str | None = None, offline: bool | None = None, ) -> list[str]: """Generate split packs (one ZIP per system or manufacturer).""" @@ -2387,6 +2438,7 @@ def generate_split_packs( precomputed_extras=group_extras, source=source, regions=regions, + target_name=target_name, offline=offline, ) if zip_path: @@ -2394,8 +2446,9 @@ def generate_split_packs( ver_tag = f"_{version.replace(' ', '')}" if version else "" req_tag = "_Required" if required_only else "" rgn_tag = f"_{region_mod.region_tag(regions)}" if regions else "" + tgt_tag = f"_{_target_tag(target_name)}" if target_name else "" safe_group = group_name.replace(" ", "_") - new_name = f"{platform_display.replace(' ', '_')}{ver_tag}{source_tag}{rgn_tag}{req_tag}_{safe_group}_BIOS_Pack.zip" + new_name = f"{platform_display.replace(' ', '_')}{ver_tag}{source_tag}{rgn_tag}{tgt_tag}{req_tag}_{safe_group}_BIOS_Pack.zip" new_path = os.path.join(split_dir, new_name) if new_path != zip_path: os.rename(zip_path, new_path) @@ -2565,7 +2618,7 @@ def generate_target_manifests(targets_dir: str, output_dir: str) -> None: if yml_file.name.startswith("_"): continue with open(yml_file) as f: - data = yaml.safe_load(f) or {} + data = yaml_load(f) or {} targets = data.get("targets", {}) if not isinstance(targets, dict): raise ValueError(f"{yml_file}: targets must be a mapping") @@ -2679,7 +2732,7 @@ def _run_manifest_mode( registry: dict = {} if os.path.exists(registry_path): with open(registry_path) as _rf: - registry = yaml.safe_load(_rf) or {} + registry = yaml_load(_rf) or {} if args.all_variants: variants = [ @@ -2707,6 +2760,7 @@ def _run_manifest_mode( target_cores=tc, source=source, regions=getattr(args, "regions", None), + target_name=args.target, offline=args.offline, ) source_suffix = {"platform": "_platform", "truth": "_truth"}.get(source, "") @@ -2715,10 +2769,13 @@ def _run_manifest_mode( region_suffix = ( f"_{region_mod.region_tag(rgn).lower()}" if rgn else "" ) + target_suffix = ( + f"_{_target_tag(args.target).lower()}" if args.target else "" + ) out_path = os.path.join( args.output_dir, f"{representative}{source_suffix}{region_suffix}" - f"{req_suffix}.json", + f"{target_suffix}{req_suffix}.json", ) _write_manifest_if_changed(out_path, manifest) print( @@ -2732,7 +2789,7 @@ def _run_manifest_mode( alias_path = os.path.join( args.output_dir, f"{alias_plat}{source_suffix}{region_suffix}" - f"{req_suffix}.json", + f"{target_suffix}{req_suffix}.json", ) alias_manifest = dict(manifest) alias_manifest["platform"] = alias_plat @@ -3056,6 +3113,7 @@ def _run_platform_packs( required_only=required_only, source=source, regions=getattr(args, "regions", None), + target_name=args.target, offline=args.offline, ) print(f" Split into {len(zip_paths)} packs") @@ -3076,6 +3134,7 @@ def _run_platform_packs( system_filter=system_filter, source=source, regions=getattr(args, "regions", None), + target_name=args.target, offline=args.offline, ) if not args.split and zip_path and aliases: @@ -3363,8 +3422,13 @@ def main(): # Platform mode if args.all: + # Install manifests cover every registered platform. "Archived" means + # upstream is no longer scraped on a schedule, not that the platform + # lost its users: its packs still ship, so the installer still needs + # its file list. platforms = list_registered_platforms( - args.platforms_dir, include_archived=args.include_archived + args.platforms_dir, + include_archived=args.include_archived or args.manifest, ) elif args.platform: platforms = [args.platform] @@ -3473,6 +3537,7 @@ def generate_manifest( target_cores: set[str] | None = None, source: str = "full", regions: list[str] | None = None, + target_name: str | None = None, offline: bool | None = None, ) -> dict: """Generate a JSON manifest for a platform (same resolution as generate_pack). @@ -3495,7 +3560,7 @@ def generate_manifest( registry: dict = {} if os.path.exists(registry_path): with open(registry_path) as f: - registry = yaml.safe_load(f) or {} + registry = yaml_load(f) or {} plat_registry = registry.get("platforms", {}).get(platform_name, {}) install_section = plat_registry.get("install", {}) detect = install_section.get("detect", []) @@ -3762,16 +3827,17 @@ def generate_manifest( # No phase 3 (data directories) -skipped for manifest - now = ( - __import__("datetime") - .datetime.now(__import__("datetime").timezone.utc) - .strftime("%Y-%m-%dT%H:%M:%SZ") - ) + now = _build_timestamp(db) result: dict = { "manifest_version": 2, "source": source, "regions": list(regions or []), + # A target-filtered manifest is only distinguishable from the full one + # by its filename, so it records the filter the way regions does. + # Absent on an unfiltered manifest rather than empty, to leave the + # twelve default files byte-identical. + **({"target": target_name} if target_name else {}), "platform": platform_name, "display_name": platform_display, "version": "1.0", @@ -3825,9 +3891,7 @@ def verify_pack( "schema_version": 1, "version": 1, "generator": "retrobios generate_pack.py", - "generated": __import__("datetime") - .datetime.now(__import__("datetime").timezone.utc) - .strftime("%Y-%m-%dT%H:%M:%SZ"), + "generated": _build_timestamp(db), "files": [], } errors = [] @@ -3996,7 +4060,7 @@ def inject_manifest(zip_path: str, manifest: dict) -> None: # which would leave the manifest uncompressed and give the same pack # two different byte layouts depending on the path taken here. with zipfile.ZipFile(zip_path, "a", zipfile.ZIP_DEFLATED) as zf: - zf.writestr("manifest.json", manifest_json) + _write_generated_member(zf, "manifest.json", manifest_json) else: # Rebuild to replace existing manifest import tempfile as _tempfile @@ -4014,7 +4078,7 @@ def inject_manifest(zip_path: str, manifest: dict) -> None: if item.filename == "manifest.json": continue dst.writestr(item, src.read(item.filename)) - dst.writestr("manifest.json", manifest_json) + _write_generated_member(dst, "manifest.json", manifest_json) os.replace(tmp_path, zip_path) except (OSError, zipfile.BadZipFile): os.unlink(tmp_path) @@ -4187,7 +4251,7 @@ def _intentional_hash_exclusion( external-download failure and a packable alternative stay conformance errors in both modes. """ - if not entries or verification_mode == "existence": + if not entries or not hash_mismatch_excludes_file(verification_mode): return False archive_index = zip_contents if zip_contents is not None else {} for entry in entries: diff --git a/tests/test_deterministic_zip.py b/tests/test_deterministic_zip.py index 5fc54b86..34d7684d 100644 --- a/tests/test_deterministic_zip.py +++ b/tests/test_deterministic_zip.py @@ -2,7 +2,9 @@ from __future__ import annotations +import hashlib import tempfile +import time import unittest import zipfile from pathlib import Path @@ -152,5 +154,100 @@ class TestVerifyDeterminism(unittest.TestCase): self.assertEqual(original, rebuilt) +class TestPackDeterminism(unittest.TestCase): + """A pack is a function of its inputs, not of the clock it was built on.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.bios = self.tmp / "bios" + self.platforms = self.tmp / "platforms" + self.bios.mkdir() + self.platforms.mkdir() + + payload = b"DETERMINISM_PAYLOAD" + (self.bios / "boot.rom").write_bytes(payload) + sha1 = hashlib.sha1(payload).hexdigest() + self.db = { + "generated_at": "2026-01-02T03:04:05Z", + "files": { + sha1: { + "path": str(self.bios / "boot.rom"), + "name": "boot.rom", + "size": len(payload), + "sha1": sha1, + "md5": hashlib.md5(payload).hexdigest(), + }, + }, + "indexes": { + "by_md5": {hashlib.md5(payload).hexdigest(): sha1}, + "by_name": {"boot.rom": [sha1]}, + "by_crc32": {}, + "by_sha256": {}, + "by_path_suffix": {}, + }, + } + (self.platforms / "detplat.yml").write_text( + "platform: DetPlat\n" + "verification_mode: existence\n" + "base_destination: system\n" + "systems:\n" + " test-sys:\n" + " files:\n" + " - name: boot.rom\n" + " destination: boot.rom\n" + " required: true\n" + ) + + def tearDown(self): + self._tmp.cleanup() + + def _build(self, out_name: str) -> Path: + from scripts.generate_pack import generate_pack + + out = self.tmp / out_name + out.mkdir() + path = generate_pack( + "detplat", str(self.platforms), self.db, str(self.bios), str(out), + offline=True, + ) + self.assertIsNotNone(path) + return Path(path) + + def test_two_builds_are_byte_identical(self): + first = self._build("out1") + time.sleep(2.1) # ZIP mtimes are 2s-granular: cross a whole tick + second = self._build("out2") + self.assertEqual( + hashlib.sha256(first.read_bytes()).hexdigest(), + hashlib.sha256(second.read_bytes()).hexdigest(), + ) + + def test_generated_members_carry_the_fixed_epoch(self): + pack = self._build("out1") + with zipfile.ZipFile(pack) as zf: + generated = [ + i for i in zf.infolist() + if i.filename.rsplit("/", 1)[-1].startswith( + ("README.txt", "INSTRUCTIONS_", "RENAMED_") + ) + ] + self.assertTrue(generated, "pack should carry a generated README") + for info in generated: + self.assertEqual(info.date_time, _FIXED_DATE_TIME) + + def test_injected_manifest_is_pinned_to_the_database(self): + from scripts.generate_pack import inject_manifest, verify_pack + + pack = self._build("out1") + ok, manifest = verify_pack(str(pack), self.db) + self.assertTrue(ok, manifest.get("errors")) + self.assertEqual(manifest["generated"], self.db["generated_at"]) + inject_manifest(str(pack), manifest) + with zipfile.ZipFile(pack) as zf: + info = zf.getinfo("manifest.json") + self.assertEqual(info.date_time, _FIXED_DATE_TIME) + + if __name__ == "__main__": unittest.main()