diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index ca6124d4..71ef1df4 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -1178,6 +1178,10 @@ def generate_pack( continue if _has_path_conflict(full_dest, seen_destinations, seen_parents): + # Content ships under the conflicting shape (file vs dir, + # e.g. SGB1.sfc): count it so pack totals match verify.py, + # which resolves each declaration independently + file_status.setdefault(dedup_key, "ok") continue storage = file_entry.get("storage", "embedded") @@ -2458,6 +2462,9 @@ def _run_verify_packs(args): """Extract each pack and verify file paths + hashes.""" import shutil + with open(args.db) as f: + verify_db = json.load(f) + platforms = list_registered_platforms(args.platforms_dir) if args.platform: platforms = [args.platform] @@ -2532,6 +2539,19 @@ def _run_verify_packs(args): fp = os.path.join(parent, e) break if not os.path.exists(fp): + # File-vs-directory conflict: upstream declares both + # SGB1.sfc and SGB1.sfc/program.rom; the builder can + # only ship one shape, the other was skipped + ancestor = os.path.dirname(fp) + conflicted = False + while len(ancestor) > len(extract_dir): + if os.path.isfile(ancestor): + conflicted = True + break + ancestor = os.path.dirname(ancestor) + if conflicted or os.path.isdir(fp): + ok += 1 + continue missing.append(f"{sys_id}: {dest}") continue if mode == "existence": @@ -2582,6 +2602,11 @@ def _run_verify_packs(args): ) if collision: ok += 1 + elif not _repo_satisfies_declaration([fe], verify_db, "md5"): + # No repo file matches the declared hash: shipped file + # is the best effort, the gap is a data issue reported + # by verify.py, not a pack error + ok += 1 else: hash_fail.append(f"{sys_id}: {dest}") @@ -3691,6 +3716,14 @@ def verify_pack_against_platform( baseline_checked = 0 baseline_present = 0 decl_by_member: dict[str, list[dict]] = {} + # Mirror the builder's path-conflict logic: a declaration whose path + # collides file-vs-directory with a packed member was skipped by the + # builder (upstream declares e.g. both SGB1.sfc and SGB1.sfc/program.rom) + zip_parents: set[str] = set() + for n in zip_set: + parts = n.split("/") + for i in range(1, len(parts)): + zip_parents.add("/".join(parts[:i])) for sys_id, system in config.get("systems", {}).items(): for fe in system.get("files", []): dest = fe.get("destination", fe.get("name", "")) @@ -3703,6 +3736,10 @@ def verify_pack_against_platform( member = expected elif expected.lower() in zip_lower: member = zip_lower[expected.lower()] + elif _has_path_conflict(expected, zip_set, zip_parents): + # Skipped by the builder for the same reason: not an error + baseline_present += 1 + continue else: errors.append(f"baseline missing: {expected}") continue diff --git a/scripts/pipeline.py b/scripts/pipeline.py index e16e32d4..96a114a2 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -313,6 +313,23 @@ def main(): # Step 4: Generate packs pack_output = "" if not args.skip_packs: + # Purge stale packs: leftover ZIPs from previous builds would be + # picked up by pack verification and shipped in releases. + out_dir = Path(args.output_dir) + if out_dir.is_dir(): + stale = [ + p for p in out_dir.iterdir() + if p.is_file() and ( + p.suffix == ".zip" + or ".zip." in p.name + or p.name == "SHA256SUMS.txt" + ) + ] + for p in stale: + p.unlink() + if stale: + print(f"Purged {len(stale)} stale pack file(s) from {out_dir}/") + pack_cmd = [ sys.executable, "scripts/generate_pack.py", diff --git a/scripts/verify.py b/scripts/verify.py index 9fcf2363..b9c13103 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -477,6 +477,11 @@ def find_undeclared_files( path_base, by_name, by_path_suffix, data_names, by_name_lower, ) + if not in_repo: + # Hash fallback: the repo may hold the content under a + # different filename (exos21.rom vs exos21.bin) + _lp, _st = resolve_local_file(f, db, dest_hint=dest) + in_repo = _st not in ("not_found",) and _lp is not None checks = _parse_validation(f.get("validation")) undeclared.append( diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 21e1d976..34f6f786 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -4885,6 +4885,81 @@ struct BurnDriver BurnDrvneogeo = { self.assertIsNone(local) self.assertEqual(status, "not_found") + def test_219_batocera_stable_tag_selection(self): + """Stable tag picker takes the highest batocera-N(.M) tag.""" + from scripts.scraper.batocera_scraper import pick_stable_tag + + tags = [ + "uboot-rockpro64", "batocera-pre-br2-external", + "batocera-41", "batocera-42", "batocera-43", "batocera-43.1", + "master_20180518", + ] + self.assertEqual(pick_stable_tag(tags), "batocera-43.1") + self.assertEqual(pick_stable_tag(["batocera-9", "batocera-10"]), "batocera-10") + self.assertIsNone(pick_stable_tag(["uboot-x", "random"])) + + def test_218_resolve_by_crc32_and_size(self): + """crc-only profile entries resolve by crc32 confirmed by size.""" + import tempfile + import zlib + + from common import resolve_local_file + + with tempfile.TemporaryDirectory() as tmpdir: + data = b"exos rom bytes" * 100 + path = os.path.join(tmpdir, "exos21.rom") + with open(path, "wb") as fh: + fh.write(data) + crc = f"{zlib.crc32(data) & 0xffffffff:08x}" + db = { + "files": { + "sha1e": {"name": "exos21.rom", "path": path, "size": len(data)}, + }, + "indexes": { + "by_md5": {}, + "by_name": {"exos21.rom": ["sha1e"]}, + "by_crc32": {crc: "sha1e"}, + "by_path_suffix": {}, + }, + } + # Different extension in the profile, crc32+size identify it + local, status = resolve_local_file( + {"name": "exos21.bin", "crc32": crc, "size": len(data)}, db + ) + self.assertEqual(local, path) + self.assertEqual(status, "exact") + # Wrong size rejects the crc match + local, status = resolve_local_file( + {"name": "exos21.bin", "crc32": crc, "size": 1}, db + ) + self.assertIsNone(local) + + def test_217_resolve_name_casefold_fallback(self): + """Name resolution falls back to a case-insensitive match.""" + import tempfile + + from common import resolve_local_file + + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, "VEC_Minestorm.vec") + with open(path, "wb") as fh: + fh.write(b"minestorm rom") + db = { + "files": {"sha1v": {"name": "VEC_Minestorm.vec", "path": path}}, + "indexes": { + "by_md5": {}, + "by_name": {"VEC_Minestorm.vec": ["sha1v"]}, + "by_crc32": {}, + "by_path_suffix": {}, + }, + } + # Upstream declares a different casing, no hash + local, status = resolve_local_file( + {"name": "VEC_MineStorm.vec"}, db + ) + self.assertEqual(local, path) + self.assertEqual(status, "exact") + def test_215_check_member_hash_inside_zip(self): """zipped_file entries verify the ROM inside the ZIP, not the ZIP.""" import hashlib as hl