mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-16 14:47:49 -06:00
feat: pipeline compares split parts to the pack
Step 5 compared every narrowed pack with verify and never the --split parts with the full pack, so a part writing a path the full pack settles passed every check. check_split_packs builds each platform's parts in memory and compares their union with the full pack, path for path and byte for byte.
This commit is contained in:
1 parent
3e2a5279a1
commit
04aad25683
4 files changed
+142
-1
No files matched your search
@@ -55,6 +55,7 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
# The build scripts need CPython 3.9, which builds the same database,
|
||||
@@ -810,6 +811,92 @@ def check_narrowed_packs(
|
||||
return all_ok
|
||||
|
||||
|
||||
def split_differences(
|
||||
full: object, parts: dict[str, object], same_bytes: Callable[[str, str], bool],
|
||||
) -> list[str]:
|
||||
"""Where the --split parts of a pack, together, are not the full pack.
|
||||
|
||||
A path in a part and not in the full pack, a path of the full pack in no
|
||||
part, and a path a part fills with other bytes than the full pack.
|
||||
"""
|
||||
full_members = {m for m in full.members if not _PACK_NOTES.search(m)}
|
||||
union = {
|
||||
m for record in parts.values() for m in record.members
|
||||
if not _PACK_NOTES.search(m)
|
||||
}
|
||||
lines = [f"in a part, not in the full pack: {m}" for m in sorted(union - full_members)]
|
||||
lines += [f"in the full pack, in no part: {m}" for m in sorted(full_members - union)]
|
||||
lines += sorted({
|
||||
f"other bytes in {name}: {member}"
|
||||
for name, record in parts.items()
|
||||
for member, source in record.sources.items()
|
||||
if member in full.sources and not same_bytes(source, full.sources[member])
|
||||
})
|
||||
return lines
|
||||
|
||||
|
||||
def check_split_packs(
|
||||
emulators_dir: str = "emulators", platforms_dir: str = "platforms",
|
||||
db_path: str = "database.json", include_archived: bool = False,
|
||||
) -> bool:
|
||||
"""Build each platform's --split parts in memory and compare them with
|
||||
the full pack: the parts must add up to it, path for path and byte for
|
||||
byte. Each part is a pack of its own and used to settle its paths alone:
|
||||
RetroDECK's parts carried bios/SGB1.sfc as a file and as a directory,
|
||||
RetroBat's 21 openmsx/ twins of openMSX/ files. Offline, as every
|
||||
comparison of this step.
|
||||
"""
|
||||
import generate_pack # noqa: PLC0415
|
||||
import packresolve # noqa: PLC0415
|
||||
from common import ( # noqa: PLC0415
|
||||
build_zip_contents_index,
|
||||
group_identical_platforms,
|
||||
list_registered_platforms,
|
||||
load_data_dir_registry,
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
)
|
||||
|
||||
print(f"\n--- {step_title(STEP_LABELS['consistency'])}, split packs ---")
|
||||
db = load_database(db_path)
|
||||
zip_contents = build_zip_contents_index(db)
|
||||
profiles = load_emulator_profiles(emulators_dir)
|
||||
registry = load_data_dir_registry(platforms_dir)
|
||||
groups = group_identical_platforms(
|
||||
list_registered_platforms(platforms_dir, include_archived=include_archived),
|
||||
platforms_dir,
|
||||
)
|
||||
was_offline = packresolve._OFFLINE
|
||||
packresolve.set_offline(True)
|
||||
all_ok = True
|
||||
try:
|
||||
for _members, platform in groups:
|
||||
args = dict(
|
||||
emulators_dir=emulators_dir, zip_contents=zip_contents,
|
||||
data_registry=registry, emu_profiles=profiles, offline=True,
|
||||
)
|
||||
full = generate_pack.PackRecord()
|
||||
parts: dict[str, generate_pack.PackRecord] = {}
|
||||
with contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(io.StringIO()):
|
||||
generate_pack.generate_pack(
|
||||
platform, platforms_dir, db, "bios", "", record=full, **args,
|
||||
)
|
||||
generate_pack.generate_split_packs(
|
||||
platform, platforms_dir, db, "bios", "", records=parts, **args,
|
||||
)
|
||||
lines = split_differences(full, parts, generate_pack._same_bytes)
|
||||
if lines:
|
||||
all_ok = False
|
||||
print(f" {platform} --split: {len(lines)} differences")
|
||||
print("\n".join(f" {line}" for line in lines[:20]))
|
||||
finally:
|
||||
packresolve.set_offline(was_offline)
|
||||
print(f" {len(groups)} split platform packs compared, "
|
||||
f"{'all equal' if all_ok else 'differences above'}")
|
||||
return all_ok
|
||||
|
||||
|
||||
class _Skipped:
|
||||
"""A step that did not run. Truthy, so it never fails the pipeline, and
|
||||
distinct, so the summary does not report it as work that was done."""
|
||||
@@ -1373,6 +1460,7 @@ def main() -> None:
|
||||
ok = check_narrowed_packs(
|
||||
include_archived=args.include_archived, full=args.full_narrowings,
|
||||
) and ok
|
||||
ok = check_split_packs(include_archived=args.include_archived) and ok
|
||||
print(
|
||||
f"--- {step_title(STEP_LABELS['consistency'])}, counts and paths: "
|
||||
f"{'OK' if ok else 'FAILED'} ---"
|
||||
|
||||
Reference in new issue
Block a user