chore: merge pack and resolver fixes

This commit is contained in:
Abdessamad Derraz committed 2026-11-07 00:06:15 +01:00
commit 70f530516d
66 files changed
+1294 -786

No files matched your search

+2 -2
View File
@@ -25,8 +25,8 @@ def accepts(entry: dict, path: Path, mode: str) -> bool:
member = names.get(entry["zipped_file"].casefold())
return bool(member) and hashlib.new(digest, archive.read(member)).hexdigest() in values
actual = hashlib.new(digest, path.read_bytes()).hexdigest()
if entry.get("zip_digest") != "composite" and any(actual.startswith(v) for v in values):
return True # a truncated md5 (Batocera) compares by its prefix
if entry.get("zip_digest") != "composite" and actual in values:
return True # whole digests, as batocera-systems:1088 compares them
if digest == DIGESTS["md5"] and path.suffix.lower() == ".zip": # Recalbox: members in name order
with zipfile.ZipFile(path) as archive:
names = sorted(name for name in archive.namelist() if not name.endswith("/"))
+61 -15
View File
@@ -1509,6 +1509,15 @@ def resolve_local_file(
names_to_try.append(hint_base)
md5_list = parse_md5_list(file_entry.get("md5"))
files_db = db.get("files", {})
by_md5 = db.get("indexes", {}).get("by_md5", {})
# A value cut short stands for the one held md5 it starts, and only
# then (cut_md5_owner): the dump it names is still name-guarded below.
cut_owners = {
cut: owner for cut in md5_list
if len(cut) < 32 and (owner := cut_md5_owner(cut, by_md5))
}
md5_list = [cut_owners.get(value, value) for value in md5_list]
sha1_candidates = [
str(value).strip().lower()
for value in (sha1 if isinstance(sha1, list) else [sha1] if sha1 else [])
@@ -1536,8 +1545,6 @@ def resolve_local_file(
has_strong_hash = bool(
sha1_candidates or sha256_candidates or md5_list or crc_candidates
)
files_db = db.get("files", {})
by_md5 = db.get("indexes", {}).get("by_md5", {})
by_name = db.get("indexes", {}).get("by_name", {})
by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
@@ -1558,7 +1565,7 @@ def resolve_local_file(
# With zipped_file the MD5 identifies the member, not the container.
if md5_list and not zipped_file:
actual_md5 = str(entry.get("md5", "")).lower()
if not any(actual_md5.startswith(expected) for expected in md5_list):
if actual_md5 not in md5_list:
return None
statuses.append("md5_exact")
if crc_candidates:
@@ -1642,20 +1649,13 @@ def resolve_local_file(
sha1_match = by_md5.get(md5_candidate)
if sha1_match and sha1_match in files_db:
path = files_db[sha1_match]["path"]
# Full MD5 (32 chars) is a strong identifier: trust it
# without name guard. Truncated MD5 still needs name check
# to avoid cross-contamination.
# A full md5 as declared is a strong identifier: trusted
# without name guard. One completed from a cut value still
# needs the name check to avoid cross-contamination.
status = _record_match_status(sha1_match)
if status and os.path.exists(path):
if len(md5_candidate) >= 32 or _md5_name_ok(path):
if md5_candidate not in cut_owners.values() or _md5_name_ok(path):
return path, status
if len(md5_candidate) < 32:
for db_md5, db_sha1 in by_md5.items():
if db_md5.startswith(md5_candidate) and db_sha1 in files_db:
path = files_db[db_sha1]["path"]
status = _record_match_status(db_sha1)
if status and os.path.exists(path) and _md5_name_ok(path):
return path, status
# 2b. Path suffix lookup is useful for same-named regional files, but it
# is identity evidence only when no content hash was declared. A stale
@@ -1776,9 +1776,15 @@ def resolve_local_file(
# none of them answers. melonDS Nightly's dsi_nand.bin is read at
# dsi/nand.bin: with the DSi image absent from the checkout, the
# basename let the Wii nand.bin outrank the entry's own DSi variant.
# A name whose held file no declaration proves, while declarations
# of that name prove another dump beside it, means that dump: the
# DSi devkit boot ROMs filed as dsi_bios7.bin went to RetroBat.
by_proven_name = db.get("indexes", {}).get("by_proven_name", {})
for tier in (own_names, names_to_try[len(own_names):]):
for try_name in tier:
for match_sha1 in by_name.get(try_name, []):
for match_sha1 in (
by_proven_name.get(try_name) or by_name.get(try_name, [])
):
if match_sha1 in files_db and _size_ok(match_sha1):
path = files_db[match_sha1]["path"]
if os.path.exists(path) and path not in candidates:
@@ -2830,6 +2836,45 @@ def preferred_profile(index: dict[str, set[str]], name: str) -> str | None:
return name if name in claimants else sorted(claimants)[0]
def without_profile_blind(
platforms: list[str],
flags: list[str],
platforms_dir: str,
profiles: dict[str, dict],
target_cores: dict[str, set[str] | None] | None,
is_all: bool,
) -> list[str]:
"""The platforms a narrowing read from profiles can apply to.
Regions, search order and game data are read from the profiles of the
cores a platform runs. MiSTer and BizHawk declare `cores: []`: a
--region, --one-per-slot or --no-game-data pack of theirs withdrew
nothing while its name and README said it had. A named platform is
refused (exit 1). Under --all it is left out with a notice. generate_pack
and verify both read this.
"""
if not flags:
return platforms
blind = [
name for name in platforms
if not resolve_platform_cores(
load_platform_config(name, platforms_dir),
profiles,
target_cores=(target_cores or {}).get(name),
)
]
for name in blind:
message = (
f"{name} runs no core a profile describes, and {', '.join(flags)} "
"reads the profiles"
)
if not is_all:
print(f"ERROR: {message}: drop {', '.join(flags)}", file=sys.stderr)
sys.exit(1)
print(f" {message}: left out")
return [name for name in platforms if name not in blind]
def resolve_platform_cores(
config: dict,
profiles: dict[str, dict],
@@ -3635,6 +3680,7 @@ from hashing import ( # noqa: E402,F401
md5sum,
md5_composite,
parse_md5_list,
cut_md5_owner,
_ALL_ALGORITHMS,
_md5_composite_cache,
)
+1 -4
View File
@@ -317,10 +317,7 @@ def _record_carries(sha1: str, record: dict, declared: dict[str, set[str]]) -> b
**{field: str(record.get(field) or "").lower() for field in ("md5", "sha256", "crc32")},
}
return any(
actual[field] and (
actual[field] in values
or (field == "md5" and any(len(v) < 32 and actual[field].startswith(v) for v in values))
)
actual[field] and actual[field] in values
for field, values in declared.items()
)
+4 -32
View File
@@ -22,11 +22,6 @@ from common import _norm_system_id, resolve_platform_cores, unresolved_platform_
from exporter.baseline import HASH_FIELDS, NativeSystem, _hash_values
from launchers import _names as launcher_names
# Batocera cuts some md5 to 29 characters: the prefix still names the dump.
_FULL_MD5 = 32
_CUT_MD5 = 29
def _tails(path: str) -> list[str]:
"""Every trailing run of segments of a path, longest first."""
parts = [p for p in path.casefold().strip("/").split("/") if p]
@@ -48,22 +43,12 @@ def _declared_hashes(entry: dict) -> dict[str, set[str]]:
return {f: values for f, values in found.items() if values}
def _same(declared: str, accepted: str) -> bool:
"""Equal, or a cut md5 that starts the accepted one."""
return declared == accepted or (
_CUT_MD5 <= len(declared) < _FULL_MD5 and accepted.startswith(declared)
)
def _accepts(accepted: set[str], declared: dict[str, set[str]]) -> bool:
"""True when the hashes a profile entry accepts, the members of its
archive included, hold one the platform entry declares."""
for values in declared.values():
if values & accepted:
return True
cut = [d for d in values if _CUT_MD5 <= len(d) < _FULL_MD5]
if cut and any(a.startswith(d) for d in cut for a in accepted):
return True
return False
@@ -72,7 +57,7 @@ def _contradicts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
the profile entry describes another dump under the same name."""
for field_name, values in declared.items():
own = set(_hash_values(profile_entry, field_name))
if own and not any(_same(d, a) for d in values for a in own):
if own and not set(values) & set(own):
return True
return False
@@ -158,10 +143,7 @@ class ReadEvidence:
hashes = {
v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
}
if hashes & self.hashes or any(
len(v) < _FULL_MD5 and any(h.startswith(v) for h in self.hashes)
for v in hashes if len(v) >= _CUT_MD5
):
if hashes & self.hashes:
return "hash"
full = "/" + (tails[0] if tails else "")
if any("/" + d in full for d in self.directories):
@@ -188,7 +170,7 @@ class ProofIndex:
def __init__(self, readers: list[ReadEvidence]) -> None:
self._index: dict[str, dict[str, set[int]]] = {
kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member")
kind: {} for kind in ("tail", "launched", "name", "hash", "dir", "tree", "member")
}
for position, evidence in enumerate(readers):
for kind, keys in (
@@ -199,11 +181,6 @@ class ProofIndex:
):
for key in keys:
self._index[kind].setdefault(key, set()).add(position)
# A cut md5 names any hash it is the start of.
for value in evidence.hashes:
for length in range(_CUT_MD5, _FULL_MD5):
if len(value) >= length:
self._index["cut"].setdefault(value[:length], set()).add(position)
def _hits(self, kind: str, keys: Iterable[str]) -> set[int]:
table = self._index[kind]
@@ -224,7 +201,6 @@ class ProofIndex:
hashes = {
v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
}
cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5}
# "/" + d in "/" + path: d is a run of whole segments ending in "/",
# so it is a run of the path's segments before the last one.
parts = tails[0].split("/") if tails else []
@@ -236,7 +212,7 @@ class ProofIndex:
"path": self._hits("tail", tails[:-1]),
"launcher": self._hits("launched", names | last),
"name": self._hits("name", names | last),
"hash": self._hits("hash", hashes) | self._hits("cut", cut),
"hash": self._hits("hash", hashes),
"directory": self._hits("dir", runs),
"data_directory": self._hits("tree", runs)
| self._index["member"].get(tails[0] if tails else "", set()),
@@ -463,10 +439,6 @@ class EntryCategories:
for value in values:
for position, f in by_hash_index.get(value, ()):
found[position] = f
if _CUT_MD5 <= len(value) < _FULL_MD5:
for accepted, hits in by_hash_index.items():
if accepted.startswith(value):
found.update(hits)
if found:
return [cited(found[p]) for p in sorted(found)]
named = {position: f for n in names for position, f in by_name_index.get(n, ())}
+73
View File
@@ -318,6 +318,76 @@ def shadowed_alias_index(
return {alias: sorted(sha1s) for alias, sha1s in sorted(index.items())}
def _declarations(platforms_dir: Path, emulators_dir: Path) -> list[dict]:
"""Every file entry the registered platforms and the profiles declare."""
sources = [
platforms_dir / f"{name}.yml"
for name in list_registered_platforms(str(platforms_dir))
] if platforms_dir.is_dir() else []
entries: list[dict] = []
for source in sources:
with open(source) as f:
config = yaml_load(f) or {}
for system in (config.get("systems") or {}).values():
entries.extend(system.get("files") or [])
if emulators_dir.is_dir():
for source in source_yaml_paths(emulators_dir):
with open(source) as f:
entries.extend((yaml_load(f) or {}).get("files") or [])
return entries
def proven_name_index(
files: dict, indexes: dict, declarations: list[dict]
) -> dict[str, list[str]]:
"""Names whose held primary no declaration proves, mapped to the primaries
of the same directory that declarations of that name prove by hash.
A hashless entry takes a name's file by name alone. The collection files
the DSi Panda devkit boot ROMs as dsi_bios7.bin and dsi_bios9.bin, with
the eMMC key-Y zeroed, while every hash declared under those names
designates a retail dump beside them: RetroBat and dsperate were served
the devkit ROMs. Another revision kept in .variants/ is not a primary,
and a dump filed elsewhere is another machine's file.
"""
by_hash = {
"sha1": {sha1: sha1 for sha1 in files},
"md5": indexes.get("by_md5", {}),
"crc32": indexes.get("by_crc32", {}),
"sha256": indexes.get("by_sha256", {}),
}
proven: dict[str, set[str]] = {}
for entry in declarations:
# A zipped_file hash is the member's, not the file the name designates.
if not isinstance(entry, dict) or entry.get("zipped_file"):
continue
name = str(entry.get("name") or "").rsplit("/", 1)[-1]
if not name:
continue
for kind, index in by_hash.items():
for value in _as_list(entry.get(kind)):
held = index.get(value)
if held in files:
proven.setdefault(name, set()).add(held)
result: dict[str, list[str]] = {}
for name, designated in sorted(proven.items()):
primaries = {
sha1 for sha1 in indexes.get("by_name", {}).get(name, [])
if sha1 in files and "/.variants/" not in files[sha1]["path"]
}
if not primaries or primaries & designated:
continue
homes = {os.path.dirname(files[sha1]["path"]) for sha1 in primaries}
beside = [
sha1 for sha1 in designated
if "/.variants/" not in files[sha1]["path"]
and os.path.dirname(files[sha1]["path"]) in homes
]
if beside:
result[name] = sorted(beside, key=lambda sha1: files[sha1]["path"])
return result
def load_cache(cache_path: str) -> dict:
"""Load cache file if it exists."""
try:
@@ -476,6 +546,9 @@ def _generate(args: argparse.Namespace) -> int:
indexes = build_indexes(files, aliases)
indexes["by_shadowed_alias"] = shadowed_alias_index(shadowed, indexes["by_name"])
indexes["by_proven_name"] = proven_name_index(
files, indexes, _declarations(Path("platforms"), Path("emulators"))
)
total_size = sum(entry["size"] for entry in files.values())
database = {
+70 -13
View File
@@ -76,6 +76,7 @@ from common import (
load_database,
load_emulator_profiles,
load_platform_config,
without_profile_blind,
MANUFACTURER_PREFIXES,
require_yaml,
ProfileSelectionError,
@@ -373,13 +374,25 @@ def _platform_pack_stem(
rep_cfg = load_platform_config(representative, platforms_dir)
version = rep_cfg.get("version", rep_cfg.get("dat_version", ""))
version_tag = f"_{_name_part(str(version))}" if version else ""
return platform_pack_prefix(group_platforms, representative, platforms_dir) + version_tag
def platform_pack_prefix(
group_platforms: list[str], representative: str, platforms_dir: str
) -> str:
"""Platform part of a pack name without the version.
The site reads a pack's version back by removing this prefix: a version
may hold an underscore (AmberELEC prerelease-20261019_1835).
"""
if len(group_platforms) <= 1:
return _name_part(rep_cfg.get("platform", representative), "_") + version_tag
rep_cfg = load_platform_config(representative, platforms_dir)
return _name_part(rep_cfg.get("platform", representative), "_")
names = [
load_platform_config(p, platforms_dir).get("platform", p)
for p in group_platforms
]
return "_".join(_name_part(n) for n in names) + version_tag
return "_".join(_name_part(n) for n in names)
def platform_list_version(config: dict, platform: str) -> str:
@@ -685,6 +698,9 @@ def _select_variants(
one_per_slot: bool,
required_only: bool = False,
undeclared: list[dict] | None = None,
zip_contents: dict | None = None,
data_registry: dict | None = None,
slot_overrides: dict[str, str] | None = None,
) -> tuple[set, list, list]:
"""Which regional and slot alternatives this pack leaves out.
@@ -714,6 +730,9 @@ def _select_variants(
required_only=required_only,
held_out=held,
undeclared=undeclared,
zip_contents=zip_contents,
data_registry=data_registry,
slot_overrides=slot_overrides,
)
if regions:
region_index = region_mod.platform_region_index(
@@ -968,7 +987,8 @@ def generate_pack(
region_drops, region_fallbacks, slot_undecidable = _select_variants(
config, pack_systems, emulators_dir, db, base_dest, emu_profiles,
target_cores, source, regions, one_per_slot, required_only,
undeclared=undeclared,
undeclared=undeclared, zip_contents=zip_contents,
data_registry=data_registry, slot_overrides=slot_overrides,
)
with (
@@ -1646,6 +1666,9 @@ def generate_emulator_pack(
) -> str | None:
"""Generate a ZIP pack for specific emulator profiles.
Returns the path, "" when no file is needed (verify --emulator reports
0/0 OK), None on failure.
`pack_name` is the caller's own file name: the system pack names its
archive after the system, and joining 23 SNES emulator names went past
the 255 bytes a file name may hold. `system_ids` keeps only the entries
@@ -1680,8 +1703,14 @@ def generate_emulator_pack(
if record is None:
os.makedirs(output_dir, exist_ok=True)
data_registry = load_data_dir_registry(
platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms")
)
region_drops: set[str] = (
emulator_region_drops(selected, standalone, regions, required_only, db)
emulator_region_drops(
selected, standalone, regions, required_only, db,
zip_contents=zip_contents, data_registry=data_registry,
)
if regions
else set()
)
@@ -1712,9 +1741,6 @@ def generate_emulator_pack(
missing_files.append(name)
data_dir_notices: list[str] = []
data_registry = load_data_dir_registry(
platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms")
)
with (
contextlib.nullcontext(record) if record is not None
@@ -1984,10 +2010,16 @@ def generate_emulator_pack(
)
# Remove empty ZIP (no files packed and no missing = nothing to ship)
if record is None and total_files + data_files == 0 and not (
nothing_needed = total_files + data_files == 0 and not (
missing_files or rejected_files or instruction_files or unsourceable_files
):
)
if record is None and nothing_needed:
os.unlink(zip_path)
# What verify --emulator reports as 0/0 OK: no file is needed, so
# no pack is written and the run succeeds. Printed with the path,
# the run named a file it had just removed and exited 1.
print(f" {', '.join(display_names)}: no file needed, no pack written")
return ""
# Report. The members of data trees are counted apart: verify --emulator
# counts the profile's files, and folding 3500 tree members into the
@@ -2044,7 +2076,10 @@ def generate_system_pack(
offline: bool | None = None,
platforms_dir: str | None = None,
) -> str | None:
"""Generate a ZIP pack for all emulators supporting given system IDs."""
"""Generate a ZIP pack for all emulators supporting given system IDs.
Returns the path, "" when no file is needed, None on failure.
"""
profiles = load_emulator_profiles(emulators_dir)
matching = profiles_for_systems(profiles, system_ids, standalone)
@@ -2108,7 +2143,7 @@ def generate_system_pack(
final_dir=output_dir,
)
if not result:
return None
return result
new_path = os.path.join(output_dir, final_name)
os.replace(result, new_path)
return new_path
@@ -3722,7 +3757,7 @@ def _main(held: contextlib.ExitStack) -> None:
offline=args.offline,
platforms_dir=args.platforms_dir,
)
if not built:
if built is None:
sys.exit(1)
return
@@ -3743,7 +3778,7 @@ def _main(held: contextlib.ExitStack) -> None:
offline=args.offline,
platforms_dir=args.platforms_dir,
)
if not built:
if built is None:
sys.exit(1)
return
@@ -3787,6 +3822,20 @@ def _main(held: contextlib.ExitStack) -> None:
platforms = _platforms_with_systems(
args, platforms, system_filter, emu_profiles, target_cores_cache
)
platforms = without_profile_blind(
platforms,
[
flag for flag, given in (
("--region", getattr(args, "regions", None)),
("--one-per-slot", getattr(args, "one_per_slot", False)),
("--no-game-data", getattr(args, "no_game_data", False)),
) if given
],
args.platforms_dir,
emu_profiles,
target_cores_cache if args.target else None,
args.all,
)
data_registry = load_data_dir_registry(args.platforms_dir)
# The output directory is claimed before the data directory refresh: a
@@ -4365,6 +4414,9 @@ def _manifest_selection(
required_only: bool,
regions: list[str] | None,
undeclared: list[dict] | None = None,
zip_contents: dict | None = None,
data_registry: dict | None = None,
slot_overrides: dict[str, str] | None = None,
) -> tuple[set[str], SelectionIndex, set[str]]:
"""Destinations a region list removes, the groups the installer replays,
and the destinations the builder holds.
@@ -4388,6 +4440,9 @@ def _manifest_selection(
extras_out=extras,
held_out=held,
undeclared=undeclared,
zip_contents=zip_contents,
data_registry=data_registry,
slot_overrides=slot_overrides,
)
region_index = region_mod.platform_region_index(
config, emu_profiles, target_cores
@@ -4667,6 +4722,8 @@ def generate_manifest(
region_drops, selection, builder_held = _manifest_selection(
config, pack_systems, emulators_dir, db, base_dest, emu_profiles,
target_cores, source, required_only, regions, undeclared=undeclared,
zip_contents=zip_contents, data_registry=data_registry,
slot_overrides=slot_overrides,
)
# Phase 1: baseline files
+25 -5
View File
@@ -608,11 +608,26 @@ def _release_date(tag: str) -> str:
return f"{moment.day} {moment.strftime('%B %Y')}"
def _pack_version(pack_name: str) -> str:
def _pack_version(pack_name: str, platforms: list[str]) -> str:
"""The platform version a pack was built for, from its name
(Batocera_43.1_BIOS_Pack.zip is 43.1)."""
(Batocera_43.1_BIOS_Pack.zip is 43.1).
Read past the platform part the builder writes, not at the last
underscore: AmberELEC_prerelease-20261019_1835 read as 1835.
"""
from generate_pack import platform_pack_prefix # noqa: PLC0415
stem = pack_name.removesuffix("_BIOS_Pack.zip")
return stem.rsplit("_", 1)[1] if "_" in stem else ""
prefix = platform_pack_prefix(
platforms, platforms[0], str(siteinputs.current().platforms)
) + "_"
return stem[len(prefix):] if stem.startswith(prefix) else ""
def _pack_platforms(platform_name: str, record: dict, pack_name: str) -> list[str]:
"""The platforms a release pack was built for, the one asked first."""
listed = list(record.get("packs", {}).get(pack_name, {}).get("platforms") or [])
return listed or [platform_name]
def release_line(platform_name: str, record: dict) -> list[str]:
@@ -631,7 +646,9 @@ def release_line(platform_name: str, record: dict) -> list[str]:
]
config = load_platform_config(platform_name, str(siteinputs.current().platforms))
page_version = str(config.get("version") or "")
pack_version = _pack_version(pack_name)
pack_version = _pack_version(
pack_name, _pack_platforms(platform_name, record, pack_name)
)
if page_version and pack_version and page_version.lstrip("v") != pack_version.lstrip("v"):
display = config.get("platform", platform_name)
lines.append(f"This pack was built for {display} {pack_version}. The command "
@@ -658,7 +675,10 @@ def setup_version(platform_name: str, record: dict | None = None) -> str:
except FileNotFoundError:
return ""
pack_name = release_record.pack_name_for(platform_name, record or {})
version = (_pack_version(pack_name) if pack_name else "") or str(config.get("version") or "")
version = (
_pack_version(pack_name, _pack_platforms(platform_name, record or {}, pack_name))
if pack_name else ""
) or str(config.get("version") or "")
if not version:
return ""
return f"{config.get('platform', platform_name)} {version.removeprefix('v')}"
+15
View File
@@ -11,6 +11,7 @@ import zipfile
import zlib
from pathlib import Path
from typing import BinaryIO
from collections.abc import Iterable
_ALL_ALGORITHMS = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"})
@@ -130,6 +131,20 @@ def md5_composite_of(source: str | Path | BinaryIO) -> str:
h.update(zf.read(name))
return h.hexdigest()
def cut_md5_owner(cut: str, md5s: Iterable[str]) -> str | None:
"""The one md5 a value cut short starts, None when none or several do.
The single rule for a cut md5, read by the scraper, the resolver and
--from-md5. It names a dump, it never verifies one: Batocera compares
the whole string (batocera-systems:1088), so every verdict compares
exact digests.
"""
if not cut or len(cut) >= 32:
return None
found = {md5 for md5 in md5s if md5.startswith(cut)}
return found.pop() if len(found) == 1 else None
def parse_hash_list(raw: str | list | int | None) -> list[str]:
"""Normalize any hash field into its accepted values, lowercase.
+25 -15
View File
@@ -250,23 +250,33 @@ def _pin_verified(
passed off as the one verified. None when the name moved in between.
"""
revision = os.path.join(dest_dir, VERIFIED_STORE, snapshot[0]["sha1"])
os.makedirs(revision, exist_ok=True)
final = os.path.join(revision, name)
if os.path.isfile(final):
return final
scratch = os.path.join(revision, f".{os.getpid()}.{os.urandom(4).hex()}.tmp")
try:
os.link(path, scratch)
except FileNotFoundError:
return None
try:
if os.stat(scratch).st_ino != snapshot[2]:
return None
os.replace(scratch, final)
return final
finally:
# A prune removes an empty revision directory: one that went between
# makedirs and link is made again, once.
for _attempt in range(2):
os.makedirs(revision, exist_ok=True)
# The link must hold the inode just verified. An older link left by a
# copy that lost its cache name has one link left, and the prune of
# this very download removed it after it was returned.
with contextlib.suppress(FileNotFoundError):
os.unlink(scratch)
if os.stat(final).st_ino == snapshot[2]:
return final
scratch = os.path.join(revision, f".{os.getpid()}.{os.urandom(4).hex()}.tmp")
try:
os.link(path, scratch)
except FileNotFoundError:
if not os.path.exists(path):
return None
continue
try:
if os.stat(scratch).st_ino != snapshot[2]:
return None
os.replace(scratch, final)
return final
finally:
with contextlib.suppress(FileNotFoundError):
os.unlink(scratch)
return None
def _prune_verified(dest_dir: str) -> None:
+47 -13
View File
@@ -1222,8 +1222,14 @@ def _collect_emulator_extras(
)
continue
if claimants is not None:
# Keyed as extra_region_key keys the extra that wins the path: a
# losing claim kept the bare path, and on RetroDECK (declared
# root bios/) verify --region listed Rustation's scph1002b.bin
# as packed while the builder had withdrawn bios/scph1002b.bin.
claimants[(u.get("emulator", ""), u.get("name", ""), u.get("path") or "")] = (
sanitize_pack_path(placements[0][0])
prefixed_destination(
sanitize_pack_path(placements[0][0]), declared_root(config)
)
)
for dest, full_dest in placements:
candidates = [
@@ -1633,17 +1639,26 @@ def report_owned_by_systems(report: list[dict], system_ids: list[str]) -> list[d
]
def _ships(config: dict, file_entry: dict, db: dict) -> bool:
def _ships(
config: dict,
file_entry: dict,
db: dict,
zip_contents: dict | None,
data_registry: dict | None,
override: str | None,
) -> bool:
"""Whether the builder packs this declaration: resolved, and not refused
by a hash frontend. A dump whose md5 Recalbox would reject is left out
of the pack, so it cannot win a regional slot over a file that ships.
Resolved with the builder's own inputs: without the data directories,
AmberELEC's bluemsx config.ini read as missing while the pack held it.
"""
from packresolve import resolve_declaration
from validation import baseline_verdict
# The builder's resolution: no destination hint, release assets included.
path, status = resolve_declaration(
file_entry, db, "bios", None, None, None, offline=True,
file_entry, db, "bios", zip_contents, data_registry, override, offline=True,
)
if not path:
return False
@@ -1668,6 +1683,9 @@ def platform_region_groups(
extras_out: list[dict] | None = None,
held_out: set[str] | None = None,
undeclared: list[dict] | None = None,
zip_contents: dict | None = None,
data_registry: dict | None = None,
slot_overrides: dict[str, str] | None = None,
) -> tuple[dict[str, list[tuple[str, str]]], dict[tuple[str, str, str], str]]:
"""Group a platform's pack candidates the way region filtering reads them.
@@ -1682,7 +1700,9 @@ def platform_region_groups(
two answer differently on the same request. `extras_out` receives the
core extras that were grouped, for the install manifest. ``undeclared``
is the gap report when the caller holds it (undeclared_report, with the
same target_cores and include_all).
same target_cores and include_all). ``zip_contents``, ``data_registry``
and ``slot_overrides`` (keyed by full pack destination) are the ones the
builder resolves with, so a held declaration is the one the pack ships.
"""
groups: dict[str, list[tuple[str, str]]] = {}
for sys_id, system in systems.items():
@@ -1695,8 +1715,10 @@ def platform_region_groups(
)
if dest:
members.append((dest, file_entry.get("name", "")))
full_dest = f"{base_dest}/{dest}" if base_dest else dest
if held_out is not None and db is not None and _ships(
config, file_entry, db
config, file_entry, db, zip_contents, data_registry,
(slot_overrides or {}).get(full_dest),
):
held_out.add(dest)
@@ -1799,6 +1821,8 @@ def emulator_region_drops(
regions: list[str],
required_only: bool = False,
db: dict | None = None,
zip_contents: dict | None = None,
data_registry: dict | None = None,
) -> set[str]:
"""Destinations an emulator-mode `--region` withdraws.
@@ -1810,7 +1834,8 @@ def emulator_region_drops(
With `db`, a requested-region file the collection lacks never displaces
a held one, as in platform mode: a held scph5501.bin was withdrawn for a
Japanese image nobody had collected, and the pack shipped no PlayStation
BIOS at all.
BIOS at all. ``zip_contents`` and ``data_registry`` are the ones the
emulator pack resolves with.
"""
import region as region_mod
from packpaths import _resolve_destination
@@ -1832,27 +1857,36 @@ def emulator_region_drops(
(dest, fe.get("name", ""))
)
if held is not None and _emulator_ships(
fe, emu_name, profile, db, dest, validation_index
fe, emu_name, profile, db, validation_index,
zip_contents, data_registry,
):
held.add(dest)
return region_mod.resolve_region_drops(region_groups, region_index, regions, held)
def _emulator_ships(
file_entry: dict, emu_name: str, profile: dict, db: dict, dest: str,
validation_index: dict,
file_entry: dict, emu_name: str, profile: dict, db: dict,
validation_index: dict, zip_contents: dict | None, data_registry: dict | None,
) -> bool:
"""Whether the emulator pack ships this entry: held, and not refused by
the core's own check, the test _ships applies on the platform side.
Resolved as generate_emulator_pack resolves: without the data
directories, dolphin-mmjr2's nwc24msg.cfg read as missing while the pack
held it.
"""
from packresolve import resolve_file
from validation import validated_choice, with_archive_members
local, _status = resolve_local_file(
local, status = resolve_file(
{**with_archive_members(file_entry, profile.get("files", [])),
"source_profile": emu_name},
db, dest_hint=dest,
db, "bios", zip_contents,
dest_hint=file_entry.get("path", ""),
data_dir_registry=data_registry,
offline=True,
)
if not local:
if status in ("not_found", "user_provided") or not local:
return False
_chosen, disagreement = validated_choice(
file_entry, local, db, validation_index, "bios", None, emulators={emu_name},
+6 -5
View File
@@ -14,6 +14,7 @@ import urllib.request
from pathlib import Path
from largefiles import fetch_large_file
from hashing import cut_md5_owner
from common import (
declared_hash_verdict,
resolution_is_hash_exact,
@@ -203,8 +204,8 @@ def pending_release_asset(
return None
# Digest length in hex digits -> kind. 29 is the MD5 Batocera writes with
# its last three digits cut, matched by prefix as the resolver does.
# Digest length in hex digits -> kind. A 29-digit MD5 is a value cut short,
# looked up as the resolver looks one up (cut_md5_owner).
_HASH_KINDS = {8: "crc32", 29: "md5", 32: "md5", 40: "sha1", 64: "sha256"}
_HEX_DIGITS = re.compile(r"[0-9a-f]+")
_WORD = re.compile(r"[0-9A-Za-z]+")
@@ -289,9 +290,9 @@ def sha1_for_hash(hash_type: str, hash_val: str, db: dict) -> str | None:
elif hash_type == "md5":
by_md5 = indexes.get("by_md5", {})
sha1 = by_md5.get(hash_val)
if sha1 is None and len(hash_val) < 32:
prefixed = {v for k, v in by_md5.items() if k.startswith(hash_val)}
sha1 = prefixed.pop() if len(prefixed) == 1 else None
if sha1 is None:
owner = cut_md5_owner(hash_val, by_md5)
sha1 = by_md5.get(owner) if owner else None
else:
sha1 = next(
(k for k, e in files_db.items() if e.get("sha256") == hash_val), None
+8
View File
@@ -30,6 +30,7 @@ from packextras import platform_region_groups
from packpaths import _flat
from nativemode import reads_file_contents
import region as region_mod
import slots
from packresolve import resolve_declaration, resolve_file
from exporter.candidates import BIOS, EntryCategories
from gamedata import without_game_data
@@ -706,6 +707,7 @@ def verify_pack_against_platform(
config, emu_profiles, target_cores
)
held: set[str] = set()
held_zip_contents = build_zip_contents_index(db) if db is not None else {}
region_groups, _extra_dests = platform_region_groups(
config,
expected_systems,
@@ -718,6 +720,12 @@ def verify_pack_against_platform(
include_all=(source == "truth"),
required_only=required_only,
held_out=held,
zip_contents=held_zip_contents,
data_registry=data_registry,
slot_overrides=slots.pack_overrides(
config, emu_profiles, db, held_zip_contents, data_registry,
target_cores=target_cores, source=source,
) if db is not None else None,
)
region_drops = region_mod.resolve_region_drops(
region_groups, region_index, regions, held
+11 -8
View File
@@ -29,14 +29,15 @@ from common import copy_file_atomic, load_database
def gitignored_paths(gitignore: str) -> set[str]:
"""Paths the repository keeps out of git, as written in .gitignore."""
try:
with open(gitignore) as f:
return {
line.strip() for line in f if line.strip().startswith("bios/")
}
except FileNotFoundError:
return set()
"""Paths the repository keeps out of git, as written in .gitignore.
A missing file raises: read as empty, it left nothing to restore and
nothing unsatisfied, and the run exited 0 with every asset absent.
"""
with open(gitignore) as f:
return {
line.strip() for line in f if line.strip().startswith("bios/")
}
def index_cache(cache_dir: str) -> dict[str, str]:
@@ -144,6 +145,8 @@ def main() -> None:
help="the .gitignore listing release-asset files (default: %(default)s)",
)
args = parser.parse_args()
if not os.path.isfile(args.gitignore):
parser.error(f"no file at {args.gitignore}: it lists the release assets")
_restored, unsatisfied = restore(args.cache, args.db, args.gitignore)
if unsatisfied:
raise SystemExit(1)
+6 -5
View File
@@ -18,6 +18,7 @@ from pathlib import Path
from common import yaml_load
from hashing import cut_md5_owner
from safeparse import yaml_load_upstream
from .base_scraper import (
@@ -312,15 +313,15 @@ def _load_md5_index() -> dict[str, str]:
def _resolve_truncated_md5(md5: str, md5_index: dict[str, str]) -> str:
"""Resolve a truncated MD5 to its full 32-char version via prefix match."""
"""Resolve a truncated MD5 to its full 32-char version (cut_md5_owner)."""
if not md5 or len(md5) == 32:
return md5
if not _MD5_RE.match(md5):
return md5
matches = [k for k in md5_index if k.startswith(md5)]
if len(matches) == 1:
print(f" fixed truncated md5: {md5} -> {matches[0]}", file=sys.stderr)
return matches[0]
owner = cut_md5_owner(md5, md5_index)
if owner:
print(f" fixed truncated md5: {md5} -> {owner}", file=sys.stderr)
return owner
return md5
+28
View File
@@ -20,6 +20,7 @@ feeds the report this module prints.
from __future__ import annotations
import os
import zlib
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
@@ -892,10 +893,37 @@ def _accepted_by_all(path: str, entries: list[dict], db: dict) -> bool:
for field_name in ("md5", "sha1", "sha256", "crc32"):
declared = _declared_hashes(entry, field_name)
if declared and str(record.get(field_name, "")).lower() not in declared:
if field_name == "crc32" and _headerless_crc32(path, record, entry) in declared:
continue
return False
return True
COPIER_HEADER = 512
def _headerless_crc32(path: str, record: dict, entry: dict) -> str | None:
"""The crc32 of a file past its 512-byte copier header, when the entry
declares both the file's size and the size without the header.
Geargrafx drops the header when the file is 512 bytes over the expected
size and computes the crc32 on the rest (src/media.cpp:754-773). The
profile lists both sizes. EmuELEC's headered syscard3.pce (crc32
64f78e3c) is the 6d9a73ef image the profile names.
"""
sizes = entry.get("size")
declared = sizes if isinstance(sizes, list) else [sizes]
size = record.get("size")
if not isinstance(size, int) or not {size, size - COPIER_HEADER} <= set(declared):
return None
try:
with open(path, "rb") as f:
f.seek(COPIER_HEADER)
return f"{zlib.crc32(f.read()):08x}"
except OSError:
return None
def _same_file_family(paths: list[str]) -> bool:
"""Whether the paths are one file and its own pinned variants.
+3 -5
View File
@@ -857,8 +857,8 @@ def frontend_digest_matches(
resolve_local_file calls a file `hash_mismatch` when any declared hash
disagrees, but a frontend compares one digest. verify_entry_md5 reads it
too: Batocera compares a truncated md5 by prefix and Recalbox hashes a
ZIP by its members. Under SIZE_AND_ANY_HASH (RomM) the declared size must
too: Recalbox hashes a ZIP by its members. Batocera compares the whole
md5 string (batocera-systems:1088), so a value cut short accepts nothing. Under SIZE_AND_ANY_HASH (RomM) the declared size must
match and then any one of md5, sha1 and crc32 does: a mistyped hash next
to a right one does not withhold the file, a wrong size does.
"""
@@ -918,9 +918,7 @@ def digest_verdict(
# Recalbox hashes a .zip by its members only (Bios.cpp:64-68), so the
# container's own md5 answers nothing there.
composite_only = file_entry.get("zip_digest") == "composite"
if not composite_only and any(
actual == d or (len(d) < 32 and actual.startswith(d)) for d in declared
):
if not composite_only and actual in declared:
return True
member_digest = composite()
return bool(member_digest) and member_digest.lower() in declared
+29 -5
View File
@@ -67,6 +67,7 @@ from common import (
load_database,
load_emulator_profiles,
load_platform_config,
without_profile_blind,
resolve_system_arg,
print_platform_listing,
select_systems,
@@ -1189,12 +1190,13 @@ def verify_platform(
else load_emulator_profiles(emulators_dir)
)
base_dest = config.get("base_destination", "")
pack_overrides = slots.pack_overrides(
config, profiles, db, zip_contents, data_dir_registry,
target_cores=target_cores,
)
slot_overrides = {
(key[len(base_dest) + 1:] if base_dest and key.startswith(f"{base_dest}/") else key): path
for key, path in slots.pack_overrides(
config, profiles, db, zip_contents, data_dir_registry,
target_cores=target_cores,
).items()
for key, path in pack_overrides.items()
}
# Build HLE + validation indexes from emulator profiles
@@ -1258,6 +1260,9 @@ def verify_platform(
target_cores=target_cores,
required_only=required_only,
held_out=held,
zip_contents=zip_contents,
data_registry=data_dir_registry,
slot_overrides=pack_overrides,
)
region_index = region_mod.platform_region_index(config, profiles, target_cores)
region_drops = region_mod.resolve_region_drops(
@@ -2138,7 +2143,10 @@ def verify_emulator(
if regions:
from packextras import emulator_region_drops
region_drops = emulator_region_drops(selected, standalone, regions, required_only, db=db)
region_drops = emulator_region_drops(
selected, standalone, regions, required_only, db=db,
zip_contents=zip_contents, data_registry=data_registry,
)
# The emulator pack walks profiles in name order and keeps the first
# file at a path, folding case: a later profile's different file there
@@ -2998,6 +3006,22 @@ def _main(held: contextlib.ExitStack) -> None:
print(f"ERROR: {reason}", file=sys.stderr)
sys.exit(1)
# generate_pack refuses these on a platform whose cores no profile
# describes, and the report answers the same request the same way.
platforms = without_profile_blind(
platforms,
[
flag for flag, given in (
("--region", requested_regions),
("--no-game-data", getattr(args, "no_game_data", False)),
) if given
],
args.platforms_dir,
emu_profiles,
target_cores_cache if args.target else None,
args.all,
)
# Group identical platforms (same function as generate_pack)
groups = group_identical_platforms(
platforms, args.platforms_dir, target_cores_cache if args.target else None