diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 244c19aa5..cfb63107b 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -55,6 +55,7 @@ import subprocess import sys import tempfile import time +from collections.abc import Callable from pathlib import Path # The build scripts need CPython 3.9, which builds the same database, @@ -810,6 +811,92 @@ def check_narrowed_packs( return all_ok +def split_differences( + full: object, parts: dict[str, object], same_bytes: Callable[[str, str], bool], +) -> list[str]: + """Where the --split parts of a pack, together, are not the full pack. + + A path in a part and not in the full pack, a path of the full pack in no + part, and a path a part fills with other bytes than the full pack. + """ + full_members = {m for m in full.members if not _PACK_NOTES.search(m)} + union = { + m for record in parts.values() for m in record.members + if not _PACK_NOTES.search(m) + } + lines = [f"in a part, not in the full pack: {m}" for m in sorted(union - full_members)] + lines += [f"in the full pack, in no part: {m}" for m in sorted(full_members - union)] + lines += sorted({ + f"other bytes in {name}: {member}" + for name, record in parts.items() + for member, source in record.sources.items() + if member in full.sources and not same_bytes(source, full.sources[member]) + }) + return lines + + +def check_split_packs( + emulators_dir: str = "emulators", platforms_dir: str = "platforms", + db_path: str = "database.json", include_archived: bool = False, +) -> bool: + """Build each platform's --split parts in memory and compare them with + the full pack: the parts must add up to it, path for path and byte for + byte. Each part is a pack of its own and used to settle its paths alone: + RetroDECK's parts carried bios/SGB1.sfc as a file and as a directory, + RetroBat's 21 openmsx/ twins of openMSX/ files. Offline, as every + comparison of this step. + """ + import generate_pack # noqa: PLC0415 + import packresolve # noqa: PLC0415 + from common import ( # noqa: PLC0415 + build_zip_contents_index, + group_identical_platforms, + list_registered_platforms, + load_data_dir_registry, + load_database, + load_emulator_profiles, + ) + + print(f"\n--- {step_title(STEP_LABELS['consistency'])}, split packs ---") + db = load_database(db_path) + zip_contents = build_zip_contents_index(db) + profiles = load_emulator_profiles(emulators_dir) + registry = load_data_dir_registry(platforms_dir) + groups = group_identical_platforms( + list_registered_platforms(platforms_dir, include_archived=include_archived), + platforms_dir, + ) + was_offline = packresolve._OFFLINE + packresolve.set_offline(True) + all_ok = True + try: + for _members, platform in groups: + args = dict( + emulators_dir=emulators_dir, zip_contents=zip_contents, + data_registry=registry, emu_profiles=profiles, offline=True, + ) + full = generate_pack.PackRecord() + parts: dict[str, generate_pack.PackRecord] = {} + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + generate_pack.generate_pack( + platform, platforms_dir, db, "bios", "", record=full, **args, + ) + generate_pack.generate_split_packs( + platform, platforms_dir, db, "bios", "", records=parts, **args, + ) + lines = split_differences(full, parts, generate_pack._same_bytes) + if lines: + all_ok = False + print(f" {platform} --split: {len(lines)} differences") + print("\n".join(f" {line}" for line in lines[:20])) + finally: + packresolve.set_offline(was_offline) + print(f" {len(groups)} split platform packs compared, " + f"{'all equal' if all_ok else 'differences above'}") + return all_ok + + class _Skipped: """A step that did not run. Truthy, so it never fails the pipeline, and distinct, so the summary does not report it as work that was done.""" @@ -1373,6 +1460,7 @@ def main() -> None: ok = check_narrowed_packs( include_archived=args.include_archived, full=args.full_narrowings, ) and ok + ok = check_split_packs(include_archived=args.include_archived) and ok print( f"--- {step_title(STEP_LABELS['consistency'])}, counts and paths: " f"{'OK' if ok else 'FAILED'} ---" diff --git a/tests/test_split_extras.py b/tests/test_split_extras.py index e64fff4a3..9fa7c4d1a 100644 --- a/tests/test_split_extras.py +++ b/tests/test_split_extras.py @@ -195,5 +195,48 @@ class SplitPartsHoldWhatTheFullPackHolds(unittest.TestCase): self._assert_parts_add_up(full, parts) +class PipelineComparesTheSplitParts(unittest.TestCase): + """Step 5 compared the --region, --target, --system, --required-only and + --no-game-data packs with verify, and nothing compared the --split parts + with the full pack: RetroDECK's SGB1.sfc file/directory clash and + RetroBat's 21 openmsx/openMSX twins passed every check.""" + + def setUp(self): + import pipeline + + self.pipeline = pipeline + + def _record(self, sources: dict[str, str]) -> PackRecord: + record = PackRecord() + record.members = [*sources, "README.txt"] + record.sources = dict(sources) + return record + + def test_equal_parts_report_nothing(self): + full = self._record({"a.bin": "x", "b.bin": "y"}) + parts = {"A": self._record({"a.bin": "x"}), "B": self._record({"b.bin": "y"})} + self.assertEqual(self.pipeline.split_differences(full, parts, str.__eq__), []) + + def test_each_difference_is_named(self): + full = self._record({"SGB1.sfc": "x", "kept.bin": "k", "lost.bin": "l"}) + parts = { + "Snes": self._record({"SGB1.sfc": "x", "kept.bin": "other"}), + "Other": self._record({"SGB1.sfc/program.rom": "p"}), + } + self.assertEqual( + self.pipeline.split_differences(full, parts, str.__eq__), + [ + "in a part, not in the full pack: SGB1.sfc/program.rom", + "in the full pack, in no part: lost.bin", + "other bytes in Snes: kept.bin", + ], + ) + + def test_step_five_runs_the_comparison(self): + source = (REPO_ROOT / "scripts" / "pipeline.py").read_text() + step = source[source.index("# Step 5: Consistency check"):source.index("# Step 6:")] + self.assertIn("check_split_packs(", step) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md index c17cc07ac..78c3fe16a 100644 --- a/wiki/advanced-usage.md +++ b/wiki/advanced-usage.md @@ -63,6 +63,10 @@ Group the split ZIPs by manufacturer (Sony, Nintendo, Sega, etc.): python scripts/generate_pack.py --platform retroarch --split --group-by manufacturer ``` +The parts together hold exactly the files of the full pack, so every part +can be extracted into the same folder. Core files of a system the platform +does not list go in an `Other_Cores` part. + ### System-specific packs Extract only the files for a single system within a platform: diff --git a/wiki/tools.md b/wiki/tools.md index 78dfb21b1..8dbf84bae 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -52,7 +52,7 @@ Pipeline steps: | 4c | Generate target manifests | `--skip-packs` | | 4d | Collected copies only an emulator install provides (`bundled_only.py --strict`, gates the run) | - | | 4e | Games index: what the installer reads for the games on a card (`generate_games_index.py`, fails under its coverage floor) | - (reads the committed install lists) | -| 5 | Consistency check: counts, paths per archive, every emulator and system pack, the narrowed platform packs | `--skip-packs`, or no full pack built (`--source platform` or `truth`) | +| 5 | Consistency check: counts, paths per archive, every emulator and system pack, the narrowed platform packs, the `--split` parts | `--skip-packs`, or no full pack built (`--source platform` or `truth`) | | 6 | Pack integrity (extract + hash) | `--skip-packs`, or no full pack built | | 7 | Generate README and CONTRIBUTING, then the installer page's tables and run samples (`generate_installer_doc.py`) | `--skip-docs` | | 8 | Generate site | `--skip-docs` | @@ -110,6 +110,12 @@ alone, run `python scripts/check_buildbot_system.py`. `--source` and `--one-per-slot` are not compared: verify has no report of a platform-only, truth-only or one-per-slot pack. A test fails when a new narrowing reaches the pack name without being compared or exempted. +- **The `--split` parts add up to the full pack.** Step 5 builds the parts of + every platform in memory and compares their union with the full pack, path + for path and byte for byte: a part is built against the full pack's own + placement, so a path the full pack settles (a file/directory clash, a case + twin, a path two systems declare) is not written again in a part. The 17 + platform packs took about four and a half minutes on 2026-11-06. - **Generators write through `write_if_changed()`.** A new timestamp alone rewrites nothing, so a diff on a generated file always means content. - **One run per output directory.** Pack generation takes a lock on