mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-25 21:29:05 -05:00
fix: count an uncached release asset as fetched
verify resolved offline, so a release asset absent from bios/ and the cache read as missing while an online build fetched and packed it. verify now reads the database record of the registered path and counts such a declaration present, listed as fetched at build time. --offline keeps the old count, and the offline pipeline passes it to its verify step.
This commit is contained in:
1 parent
b4e5e42c97
commit
9a0501d9d3
6 files changed
+251
-4
No files matched your search
@@ -139,6 +139,70 @@ def _verified_asset(
|
||||
return None
|
||||
|
||||
|
||||
def run_is_offline(offline: bool | None = None) -> bool:
|
||||
"""Whether this run fetches nothing: --offline, or RETROBIOS_OFFLINE."""
|
||||
return bool(
|
||||
(_OFFLINE if offline is None else offline)
|
||||
or os.environ.get("RETROBIOS_OFFLINE")
|
||||
)
|
||||
|
||||
|
||||
_RECORDS_BY_PATH: list[tuple[dict, int, dict[str, dict]]] = []
|
||||
|
||||
|
||||
def _records_by_path(db: dict) -> dict[str, dict]:
|
||||
"""The database records by path, built once per database."""
|
||||
files = db.get("files", {})
|
||||
if _RECORDS_BY_PATH:
|
||||
cached, size, index = _RECORDS_BY_PATH[0]
|
||||
if cached is files and size == len(files):
|
||||
return index
|
||||
index = {record.get("path"): record for record in files.values()}
|
||||
_RECORDS_BY_PATH[:] = [(files, len(files), index)]
|
||||
return index
|
||||
|
||||
|
||||
def pending_release_asset(
|
||||
file_entry: dict, db: dict, registered: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""The release asset an online build fetches for an entry, if one does.
|
||||
|
||||
The registered path whose database record carries what the entry
|
||||
declares, read without a download: verify never fetches, and a release
|
||||
asset the cache lacks was reported missing while an online build packed
|
||||
it. ``registered`` is the .gitignore registry (load_registered_paths).
|
||||
"""
|
||||
from largefiles import asset_candidates, asset_names, load_registered_paths
|
||||
|
||||
name = file_entry.get("name", "")
|
||||
if not name or name.endswith("/") or file_entry.get("type") == "directory":
|
||||
return None
|
||||
declared = _declared_hashes(file_entry)
|
||||
if file_entry.get("unsourceable") and not any(declared.values()):
|
||||
return None
|
||||
if file_entry.get("zipped_file") and not any(declared.values()):
|
||||
return None
|
||||
if registered is None:
|
||||
registered = load_registered_paths()
|
||||
assets = set(asset_candidates(name, registered))
|
||||
if not assets:
|
||||
return None
|
||||
records = _records_by_path(db)
|
||||
declared_size = file_entry.get("size")
|
||||
for path, asset in sorted(asset_names(registered).items()):
|
||||
record = records.get(path)
|
||||
if asset not in assets or record is None:
|
||||
continue
|
||||
if isinstance(declared_size, int) and record.get("size") != declared_size:
|
||||
continue
|
||||
if all(
|
||||
not values or str(record.get(field, "")).lower() in values
|
||||
for field, values in declared.items()
|
||||
):
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
# Digest length in hex digits -> kind. 29 is the MD5 Batocera writes with
|
||||
# its last three digits cut, matched by prefix as the resolver does.
|
||||
_HASH_KINDS = {8: "crc32", 29: "md5", 32: "md5", 40: "sha1", 64: "sha256"}
|
||||
|
||||
@@ -873,6 +873,10 @@ def main() -> None:
|
||||
verify_cmd.append("--include-archived")
|
||||
if args.target:
|
||||
verify_cmd.extend(["--target", args.target])
|
||||
if args.offline:
|
||||
# The packs of step 4 fetch no release asset: a missing one is
|
||||
# missing on both sides.
|
||||
verify_cmd.append("--offline")
|
||||
ok, verify_output = run(verify_cmd, STEP_LABELS["verify"])
|
||||
results["verify"] = ok
|
||||
all_ok = all_ok and ok
|
||||
|
||||
+50
-3
@@ -149,6 +149,11 @@ class Status:
|
||||
MISSING = "missing"
|
||||
|
||||
|
||||
# A declaration whose file is a release asset the cache does not hold yet:
|
||||
# an online build fetches it, verify reads the database record instead.
|
||||
RELEASE_ASSET_PENDING = "release_asset_pending"
|
||||
|
||||
|
||||
class Severity:
|
||||
CRITICAL = "critical" # required file missing or bad hash (Recalbox RED)
|
||||
WARNING = "warning" # optional missing or hash mismatch (Recalbox YELLOW)
|
||||
@@ -1075,7 +1080,7 @@ def _resolve_for_platform(
|
||||
disk and the large-file cache: verify never downloads. A frontend that
|
||||
never reads the bytes takes the filename-free core's own image.
|
||||
"""
|
||||
from packresolve import resolve_declaration
|
||||
from packresolve import pending_release_asset, resolve_declaration, run_is_offline
|
||||
|
||||
override = slot_overrides.get(
|
||||
sanitize_pack_path(file_entry.get("destination", file_entry.get("name", "")))
|
||||
@@ -1084,6 +1089,12 @@ def _resolve_for_platform(
|
||||
file_entry, db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry,
|
||||
override, offline=True,
|
||||
)
|
||||
if status in ("not_found", "hash_mismatch") and not run_is_offline():
|
||||
# Not in the cache, and the release holds a copy that answers the
|
||||
# declaration: an online build fetches and packs it (resolve_file).
|
||||
asset = pending_release_asset(file_entry, db)
|
||||
if asset:
|
||||
return asset, RELEASE_ASSET_PENDING
|
||||
if status == "not_found" and not reads_file_contents(mode):
|
||||
found = agnostic_substitute(file_entry, sys_id, db, platform_profiles)
|
||||
if found:
|
||||
@@ -1234,7 +1245,23 @@ def verify_platform(
|
||||
file_entry.get("destination", file_entry.get("name", ""))
|
||||
)
|
||||
stored = _storage_verdict(file_entry, emulator_pack=False)
|
||||
if stored is not None:
|
||||
if resolve_status == RELEASE_ASSET_PENDING:
|
||||
# Counted as an online build counts it, which fetches the
|
||||
# asset. Under --offline the builder cannot, and neither
|
||||
# does this report (_resolve_for_platform).
|
||||
result = {
|
||||
"name": file_entry.get("name", ""),
|
||||
"required": file_entry.get("required", True),
|
||||
"status": Status.OK,
|
||||
"release_asset": local_path,
|
||||
"reason": "release asset, fetched when the pack is built",
|
||||
}
|
||||
packed_order.append((
|
||||
sys_id, f"{base_dest}/{destination}" if base_dest else destination, None,
|
||||
))
|
||||
shipped_keys.add(_dest_key(file_entry, sys_id))
|
||||
local_path = None
|
||||
elif stored is not None:
|
||||
result = stored
|
||||
local_path = None
|
||||
# The builder keeps the slot: instructions for a user_provided
|
||||
@@ -1744,6 +1771,12 @@ def print_platform_result(
|
||||
if missing:
|
||||
parts.append(f"{missing} missing")
|
||||
print(f"{label}: {', '.join(parts)} [{mode}]")
|
||||
fetched = sorted({d["name"] for d in result["details"] if d.get("release_asset")})
|
||||
if fetched:
|
||||
# Counted present: an online build downloads them. --offline counts
|
||||
# them missing, as an offline build leaves them out.
|
||||
shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "")
|
||||
print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}")
|
||||
fallbacks = result.get("region_fallbacks") or []
|
||||
if fallbacks:
|
||||
from packreadme import _group_label
|
||||
@@ -2440,6 +2473,7 @@ def _refuse_listing_narrowings(
|
||||
("--strict", args.strict),
|
||||
("--verbose", args.verbose),
|
||||
("--destinations", args.destinations),
|
||||
("--offline", args.offline),
|
||||
):
|
||||
if on:
|
||||
parser.error(f"{flag} is incompatible with {listing}")
|
||||
@@ -2457,6 +2491,8 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
|
||||
"--target is incompatible with --emulator and --system"),
|
||||
(args.destinations and not (args.platform or args.all),
|
||||
"--destinations requires --platform or --all"),
|
||||
(args.offline and not (args.platform or args.all),
|
||||
"--offline requires --platform or --all"),
|
||||
(args.destinations and getattr(args, "region", None),
|
||||
"--destinations describes the full pack and is incompatible with --region"),
|
||||
):
|
||||
@@ -2536,6 +2572,11 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
help="Show emulator ground truth details",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="JSON output")
|
||||
parser.add_argument(
|
||||
"--offline", action="store_true",
|
||||
help="Report as an offline build packs: a release asset the cache "
|
||||
"lacks is missing. Without it, it counts as fetched at build time",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--destinations", metavar="FILE",
|
||||
help="With --platform or --all, also write the paths each pack holds "
|
||||
@@ -2552,6 +2593,10 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
work_from_repo_root(args, parser, ["db", "platforms_dir", "emulators_dir"])
|
||||
if args.platform:
|
||||
require_platform(args.platform, args.platforms_dir)
|
||||
if args.offline:
|
||||
import packresolve
|
||||
|
||||
packresolve.set_offline(True)
|
||||
|
||||
requested_regions: list[str] = []
|
||||
if getattr(args, "region", None):
|
||||
@@ -2696,7 +2741,9 @@ def _json_details(details: list[dict], verbose: bool) -> list[dict]:
|
||||
used to be accepted beside --json and dropped, the output byte for byte
|
||||
the same, so a script asking for the passing entries got nothing.
|
||||
"""
|
||||
return list(details) if verbose else [d for d in details if d["status"] != Status.OK]
|
||||
return list(details) if verbose else [
|
||||
d for d in details if d["status"] != Status.OK or d.get("release_asset")
|
||||
]
|
||||
|
||||
|
||||
def _exit_if_strict(strict: bool, results: list[dict]) -> None:
|
||||
|
||||
Reference in new issue
Block a user