diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 2c2f23ede..244c19aa5 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -562,6 +562,7 @@ def check_emulator_packs( comparison holds them together. No archive is written and no file read. """ import generate_pack # noqa: PLC0415 + import packresolve # noqa: PLC0415 import verify # noqa: PLC0415 from common import ( # noqa: PLC0415 ProfileSelectionError, @@ -580,16 +581,23 @@ def check_emulator_packs( def compare(names: list[str], standalone: bool, label: str, **narrowing: object) -> bool: record = generate_pack.PackRecord() - with contextlib.redirect_stdout(io.StringIO()), \ - contextlib.redirect_stderr(io.StringIO()): - generate_pack.generate_emulator_pack( - names, emulators_dir, db, "bios", "", standalone, zip_contents, - offline=True, platforms_dir=platforms_dir, record=record, **narrowing, - ) - report = verify.verify_emulator( - names, emulators_dir, db, standalone, platforms_dir=platforms_dir, - **narrowing, - ) + was_offline = packresolve._OFFLINE + # The pack is built offline: verify counts a release asset the cache + # lacks the same way, missing, as the narrowed-pack step does. + packresolve.set_offline(True) + try: + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + generate_pack.generate_emulator_pack( + names, emulators_dir, db, "bios", "", standalone, zip_contents, + offline=True, platforms_dir=platforms_dir, record=record, **narrowing, + ) + report = verify.verify_emulator( + names, emulators_dir, db, standalone, platforms_dir=platforms_dir, + **narrowing, + ) + finally: + packresolve.set_offline(was_offline) lines = _set_difference_lines( label, set(report["pack_destinations"]), set(record.files) ) diff --git a/scripts/verify.py b/scripts/verify.py index f47c7f011..9fbdfa9a3 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -2058,7 +2058,7 @@ def verify_emulator( it and every member it carries are optional, as the builder leaves them. """ from packextras import expand_agnostic_entries - from packresolve import resolve_file + from packresolve import pending_release_asset, resolve_file, run_is_offline from packpaths import ( _archive_destination, _register_path, @@ -2084,6 +2084,16 @@ def verify_emulator( validation_index = _build_validation_index(selected_profiles) data_registry = load_data_dir_registry(platforms_dir) + def pending(entry: dict, local_path: str | None, status: str) -> tuple[str | None, str]: + # Read from disk and the cache, as _resolve_for_platform does: a + # release asset the cache lacks is fetched by an online build + # (generate_emulator_pack resolves with offline=offline). + if status in ("not_found", "hash_mismatch") and not run_is_offline(): + asset = pending_release_asset(entry, db) + if asset: + return asset, RELEASE_ASSET_PENDING + return local_path, status + details = [] file_status: dict[str, str] = {} file_severity: dict[str, str] = {} @@ -2223,14 +2233,15 @@ def verify_emulator( archive = file_entry.get("archive") if archive and archive not in seen_archives: seen_archives.add(archive) - local_path, resolve_status = resolve_file( - archive_entry(archive, files), + own_entry = archive_entry(archive, files) + local_path, resolve_status = pending(own_entry, *resolve_file( + own_entry, db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry=data_registry, offline=True, - ) + )) required = any( f.get("archive") == archive and f.get("required", True) for f in files @@ -2254,6 +2265,8 @@ def verify_emulator( "required": required, "path": local_path, } + if resolve_status == RELEASE_ASSET_PENDING: + result["release_asset"] = local_path else: result = { "name": archive, @@ -2314,7 +2327,7 @@ def verify_emulator( dest_hint = file_entry.get("path", "") # The emulator pack's resolution, release assets included. - local_path, resolve_status = resolve_file( + local_path, resolve_status = pending(file_entry, *resolve_file( {**with_archive_members(file_entry, files), "source_profile": emu_name}, db, DEFAULT_BIOS_DIR, @@ -2322,7 +2335,7 @@ def verify_emulator( dest_hint=dest_hint, data_dir_registry=data_registry, offline=True, - ) + )) if resolve_status in ("not_found", "user_provided", "external"): local_path = None name = file_entry.get("name", "") @@ -2332,6 +2345,15 @@ def verify_emulator( stored = _storage_verdict(file_entry, emulator_pack=True) if stored is not None: result = stored + elif resolve_status == RELEASE_ASSET_PENDING: + result = { + "name": name, + "status": Status.OK, + "required": required, + "path": local_path, + "release_asset": local_path, + "reason": "release asset, fetched when the pack is built", + } elif not local_path: result = {"name": name, "status": Status.MISSING, "required": required} # An entry the profile documents as unsourceable is absent by @@ -2582,6 +2604,10 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None: if unsourceable: parts.append(f"{unsourceable} unsourceable") print(f"{label}: {', '.join(parts)} [{mode}]") + fetched = sorted({d["name"] for d in result["details"] if d.get("release_asset")}) + if fetched: + shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "") + print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}") seen = set() for d in result["details"]: @@ -2712,8 +2738,6 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser "--destinations describes the full pack and is incompatible with --system"), (args.destinations and not (args.platform or args.all), "--destinations requires --platform or --all"), - (args.offline and not (args.platform or args.all), - "--offline requires --platform or --all"), # Game data is categorized against a platform's list and its cores. (getattr(args, "no_game_data", False) and not (args.platform or args.all), "--no-game-data requires --platform or --all"), diff --git a/tests/test_verify_release_asset.py b/tests/test_verify_release_asset.py index 58150b375..55afa91f0 100644 --- a/tests/test_verify_release_asset.py +++ b/tests/test_verify_release_asset.py @@ -29,7 +29,7 @@ import common # noqa: E402 import generate_db # noqa: E402 import generate_pack as builder # noqa: E402 import packresolve # noqa: E402 -from verify import Status, verify_platform # noqa: E402 +from verify import Status, verify_emulator, verify_platform # noqa: E402 PAYLOAD = b"a firmware image the repository does not commit" SHA1 = hashlib.sha1(PAYLOAD).hexdigest() @@ -176,6 +176,65 @@ class CoreExtraReleaseAssetNotCached(ReleaseAssetNotCached): test_the_online_build_packs_it = None +class EmulatorReleaseAssetNotCached(ReleaseAssetNotCached): + """verify --emulator against generate_emulator_pack: the emulator pack + resolves with offline=offline and fetched the asset online, while verify + forced offline resolution and called it missing.""" + + def setUp(self): + super().setUp() + self.profile = { + "emulator": "Big Core", "type": "libretro", "systems": ["sys"], + "files": [{"name": "big.bin", "required": True, "sha1": SHA1}], + } + Path("emulators/bigcore.yml").write_text(yaml.dump(self.profile)) + common._emulator_profiles_cache.clear() + + def _report(self) -> dict: + common._emulator_profiles_cache.clear() + return verify_emulator(["bigcore"], "emulators", self.db, platforms_dir="platforms") + + def _entry(self) -> dict: + return next(d for d in self._report()["details"] if d["name"] == "big.bin") + + def test_counted_as_fetched_when_the_pack_is_built(self): + entry = self._entry() + self.assertEqual(entry["status"], Status.OK) + self.assertEqual(entry["release_asset"], REGISTERED[0]) + + def test_its_path_is_a_pack_destination(self): + self.assertIn("big.bin", self._report()["pack_destinations"]) + + def test_offline_counts_it_missing(self): + packresolve.set_offline(True) + self.assertEqual(self._entry()["status"], Status.MISSING) + self.assertNotIn("big.bin", self._report()["pack_destinations"]) + + def test_an_asset_with_other_content_is_not_counted(self): + self.profile["files"][0]["sha1"] = "0" * 40 + Path("emulators/bigcore.yml").write_text(yaml.dump(self.profile)) + self.assertEqual(self._entry()["status"], Status.MISSING) + + def test_an_archive_asset_is_counted_too(self): + self.profile["files"] = [{"name": "inner.rom", "archive": "big.bin"}] + Path("emulators/bigcore.yml").write_text(yaml.dump(self.profile)) + self.assertIn("big.bin", self._report()["pack_destinations"]) + + def test_the_online_build_packs_it(self): + cache = Path("cache/big.bin") + cache.parent.mkdir() + cache.write_bytes(PAYLOAD) + record = builder.PackRecord() + with mock.patch.object( + packresolve, "fetch_large_file", return_value=str(cache) + ): + builder.generate_emulator_pack( + ["bigcore"], "emulators", self.db, "bios", "out", + offline=False, platforms_dir="platforms", record=record, + ) + self.assertEqual(set(record.files), set(self._report()["pack_destinations"])) + + class OfflinePipelineVerifiesOffline(unittest.TestCase): """The offline pipeline builds packs without release assets: its verify step has to count them the same way, or step 5 compares two answers.""" diff --git a/wiki/tools.md b/wiki/tools.md index dab60e5b0..78dfb21b1 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -154,7 +154,7 @@ python scripts/verify.py --platform recalbox --region us # regional priority l python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named python scripts/verify.py --emulator mame --required-only # the pack built with --required-only python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds -python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build +python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build (any mode, --emulator included) python scripts/verify.py --list-platforms # list all platforms python scripts/verify.py --list-emulators # list all emulators python scripts/verify.py --list-systems # list all systems