feat: pipeline compares split parts to the pack

Step 5 compared every narrowed pack with verify and never the --split parts with the full pack, so a part writing a path the full pack settles passed every check. check_split_packs builds each platform's parts in memory and compares their union with the full pack, path for path and byte for byte.
This commit is contained in:
Abdessamad Derraz committed 2026-11-06 08:27:48 +01:00
1 parent 3e2a5279a1
commit 04aad25683
4 files changed
+142 -1

No files matched your search

+88
View File
@@ -55,6 +55,7 @@ import subprocess
import sys
import tempfile
import time
from collections.abc import Callable
from pathlib import Path
# The build scripts need CPython 3.9, which builds the same database,
@@ -810,6 +811,92 @@ def check_narrowed_packs(
return all_ok
def split_differences(
full: object, parts: dict[str, object], same_bytes: Callable[[str, str], bool],
) -> list[str]:
"""Where the --split parts of a pack, together, are not the full pack.
A path in a part and not in the full pack, a path of the full pack in no
part, and a path a part fills with other bytes than the full pack.
"""
full_members = {m for m in full.members if not _PACK_NOTES.search(m)}
union = {
m for record in parts.values() for m in record.members
if not _PACK_NOTES.search(m)
}
lines = [f"in a part, not in the full pack: {m}" for m in sorted(union - full_members)]
lines += [f"in the full pack, in no part: {m}" for m in sorted(full_members - union)]
lines += sorted({
f"other bytes in {name}: {member}"
for name, record in parts.items()
for member, source in record.sources.items()
if member in full.sources and not same_bytes(source, full.sources[member])
})
return lines
def check_split_packs(
emulators_dir: str = "emulators", platforms_dir: str = "platforms",
db_path: str = "database.json", include_archived: bool = False,
) -> bool:
"""Build each platform's --split parts in memory and compare them with
the full pack: the parts must add up to it, path for path and byte for
byte. Each part is a pack of its own and used to settle its paths alone:
RetroDECK's parts carried bios/SGB1.sfc as a file and as a directory,
RetroBat's 21 openmsx/ twins of openMSX/ files. Offline, as every
comparison of this step.
"""
import generate_pack # noqa: PLC0415
import packresolve # noqa: PLC0415
from common import ( # noqa: PLC0415
build_zip_contents_index,
group_identical_platforms,
list_registered_platforms,
load_data_dir_registry,
load_database,
load_emulator_profiles,
)
print(f"\n--- {step_title(STEP_LABELS['consistency'])}, split packs ---")
db = load_database(db_path)
zip_contents = build_zip_contents_index(db)
profiles = load_emulator_profiles(emulators_dir)
registry = load_data_dir_registry(platforms_dir)
groups = group_identical_platforms(
list_registered_platforms(platforms_dir, include_archived=include_archived),
platforms_dir,
)
was_offline = packresolve._OFFLINE
packresolve.set_offline(True)
all_ok = True
try:
for _members, platform in groups:
args = dict(
emulators_dir=emulators_dir, zip_contents=zip_contents,
data_registry=registry, emu_profiles=profiles, offline=True,
)
full = generate_pack.PackRecord()
parts: dict[str, generate_pack.PackRecord] = {}
with contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()):
generate_pack.generate_pack(
platform, platforms_dir, db, "bios", "", record=full, **args,
)
generate_pack.generate_split_packs(
platform, platforms_dir, db, "bios", "", records=parts, **args,
)
lines = split_differences(full, parts, generate_pack._same_bytes)
if lines:
all_ok = False
print(f" {platform} --split: {len(lines)} differences")
print("\n".join(f" {line}" for line in lines[:20]))
finally:
packresolve.set_offline(was_offline)
print(f" {len(groups)} split platform packs compared, "
f"{'all equal' if all_ok else 'differences above'}")
return all_ok
class _Skipped:
"""A step that did not run. Truthy, so it never fails the pipeline, and
distinct, so the summary does not report it as work that was done."""
@@ -1373,6 +1460,7 @@ def main() -> None:
ok = check_narrowed_packs(
include_archived=args.include_archived, full=args.full_narrowings,
) and ok
ok = check_split_packs(include_archived=args.include_archived) and ok
print(
f"--- {step_title(STEP_LABELS['consistency'])}, counts and paths: "
f"{'OK' if ok else 'FAILED'} ---"