mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-13 13:33:52 -06:00
chore: merge pack and resolver fixes
This commit is contained in:
commit
70f530516d
66 files changed
+1294
-786
No files matched your search
@@ -25,8 +25,8 @@ def accepts(entry: dict, path: Path, mode: str) -> bool:
|
||||
member = names.get(entry["zipped_file"].casefold())
|
||||
return bool(member) and hashlib.new(digest, archive.read(member)).hexdigest() in values
|
||||
actual = hashlib.new(digest, path.read_bytes()).hexdigest()
|
||||
if entry.get("zip_digest") != "composite" and any(actual.startswith(v) for v in values):
|
||||
return True # a truncated md5 (Batocera) compares by its prefix
|
||||
if entry.get("zip_digest") != "composite" and actual in values:
|
||||
return True # whole digests, as batocera-systems:1088 compares them
|
||||
if digest == DIGESTS["md5"] and path.suffix.lower() == ".zip": # Recalbox: members in name order
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
names = sorted(name for name in archive.namelist() if not name.endswith("/"))
|
||||
|
||||
+61
-15
@@ -1509,6 +1509,15 @@ def resolve_local_file(
|
||||
names_to_try.append(hint_base)
|
||||
|
||||
md5_list = parse_md5_list(file_entry.get("md5"))
|
||||
files_db = db.get("files", {})
|
||||
by_md5 = db.get("indexes", {}).get("by_md5", {})
|
||||
# A value cut short stands for the one held md5 it starts, and only
|
||||
# then (cut_md5_owner): the dump it names is still name-guarded below.
|
||||
cut_owners = {
|
||||
cut: owner for cut in md5_list
|
||||
if len(cut) < 32 and (owner := cut_md5_owner(cut, by_md5))
|
||||
}
|
||||
md5_list = [cut_owners.get(value, value) for value in md5_list]
|
||||
sha1_candidates = [
|
||||
str(value).strip().lower()
|
||||
for value in (sha1 if isinstance(sha1, list) else [sha1] if sha1 else [])
|
||||
@@ -1536,8 +1545,6 @@ def resolve_local_file(
|
||||
has_strong_hash = bool(
|
||||
sha1_candidates or sha256_candidates or md5_list or crc_candidates
|
||||
)
|
||||
files_db = db.get("files", {})
|
||||
by_md5 = db.get("indexes", {}).get("by_md5", {})
|
||||
by_name = db.get("indexes", {}).get("by_name", {})
|
||||
by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
|
||||
|
||||
@@ -1558,7 +1565,7 @@ def resolve_local_file(
|
||||
# With zipped_file the MD5 identifies the member, not the container.
|
||||
if md5_list and not zipped_file:
|
||||
actual_md5 = str(entry.get("md5", "")).lower()
|
||||
if not any(actual_md5.startswith(expected) for expected in md5_list):
|
||||
if actual_md5 not in md5_list:
|
||||
return None
|
||||
statuses.append("md5_exact")
|
||||
if crc_candidates:
|
||||
@@ -1642,20 +1649,13 @@ def resolve_local_file(
|
||||
sha1_match = by_md5.get(md5_candidate)
|
||||
if sha1_match and sha1_match in files_db:
|
||||
path = files_db[sha1_match]["path"]
|
||||
# Full MD5 (32 chars) is a strong identifier: trust it
|
||||
# without name guard. Truncated MD5 still needs name check
|
||||
# to avoid cross-contamination.
|
||||
# A full md5 as declared is a strong identifier: trusted
|
||||
# without name guard. One completed from a cut value still
|
||||
# needs the name check to avoid cross-contamination.
|
||||
status = _record_match_status(sha1_match)
|
||||
if status and os.path.exists(path):
|
||||
if len(md5_candidate) >= 32 or _md5_name_ok(path):
|
||||
if md5_candidate not in cut_owners.values() or _md5_name_ok(path):
|
||||
return path, status
|
||||
if len(md5_candidate) < 32:
|
||||
for db_md5, db_sha1 in by_md5.items():
|
||||
if db_md5.startswith(md5_candidate) and db_sha1 in files_db:
|
||||
path = files_db[db_sha1]["path"]
|
||||
status = _record_match_status(db_sha1)
|
||||
if status and os.path.exists(path) and _md5_name_ok(path):
|
||||
return path, status
|
||||
|
||||
# 2b. Path suffix lookup is useful for same-named regional files, but it
|
||||
# is identity evidence only when no content hash was declared. A stale
|
||||
@@ -1776,9 +1776,15 @@ def resolve_local_file(
|
||||
# none of them answers. melonDS Nightly's dsi_nand.bin is read at
|
||||
# dsi/nand.bin: with the DSi image absent from the checkout, the
|
||||
# basename let the Wii nand.bin outrank the entry's own DSi variant.
|
||||
# A name whose held file no declaration proves, while declarations
|
||||
# of that name prove another dump beside it, means that dump: the
|
||||
# DSi devkit boot ROMs filed as dsi_bios7.bin went to RetroBat.
|
||||
by_proven_name = db.get("indexes", {}).get("by_proven_name", {})
|
||||
for tier in (own_names, names_to_try[len(own_names):]):
|
||||
for try_name in tier:
|
||||
for match_sha1 in by_name.get(try_name, []):
|
||||
for match_sha1 in (
|
||||
by_proven_name.get(try_name) or by_name.get(try_name, [])
|
||||
):
|
||||
if match_sha1 in files_db and _size_ok(match_sha1):
|
||||
path = files_db[match_sha1]["path"]
|
||||
if os.path.exists(path) and path not in candidates:
|
||||
@@ -2830,6 +2836,45 @@ def preferred_profile(index: dict[str, set[str]], name: str) -> str | None:
|
||||
return name if name in claimants else sorted(claimants)[0]
|
||||
|
||||
|
||||
def without_profile_blind(
|
||||
platforms: list[str],
|
||||
flags: list[str],
|
||||
platforms_dir: str,
|
||||
profiles: dict[str, dict],
|
||||
target_cores: dict[str, set[str] | None] | None,
|
||||
is_all: bool,
|
||||
) -> list[str]:
|
||||
"""The platforms a narrowing read from profiles can apply to.
|
||||
|
||||
Regions, search order and game data are read from the profiles of the
|
||||
cores a platform runs. MiSTer and BizHawk declare `cores: []`: a
|
||||
--region, --one-per-slot or --no-game-data pack of theirs withdrew
|
||||
nothing while its name and README said it had. A named platform is
|
||||
refused (exit 1). Under --all it is left out with a notice. generate_pack
|
||||
and verify both read this.
|
||||
"""
|
||||
if not flags:
|
||||
return platforms
|
||||
blind = [
|
||||
name for name in platforms
|
||||
if not resolve_platform_cores(
|
||||
load_platform_config(name, platforms_dir),
|
||||
profiles,
|
||||
target_cores=(target_cores or {}).get(name),
|
||||
)
|
||||
]
|
||||
for name in blind:
|
||||
message = (
|
||||
f"{name} runs no core a profile describes, and {', '.join(flags)} "
|
||||
"reads the profiles"
|
||||
)
|
||||
if not is_all:
|
||||
print(f"ERROR: {message}: drop {', '.join(flags)}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f" {message}: left out")
|
||||
return [name for name in platforms if name not in blind]
|
||||
|
||||
|
||||
def resolve_platform_cores(
|
||||
config: dict,
|
||||
profiles: dict[str, dict],
|
||||
@@ -3635,6 +3680,7 @@ from hashing import ( # noqa: E402,F401
|
||||
md5sum,
|
||||
md5_composite,
|
||||
parse_md5_list,
|
||||
cut_md5_owner,
|
||||
_ALL_ALGORITHMS,
|
||||
_md5_composite_cache,
|
||||
)
|
||||
|
||||
@@ -317,10 +317,7 @@ def _record_carries(sha1: str, record: dict, declared: dict[str, set[str]]) -> b
|
||||
**{field: str(record.get(field) or "").lower() for field in ("md5", "sha256", "crc32")},
|
||||
}
|
||||
return any(
|
||||
actual[field] and (
|
||||
actual[field] in values
|
||||
or (field == "md5" and any(len(v) < 32 and actual[field].startswith(v) for v in values))
|
||||
)
|
||||
actual[field] and actual[field] in values
|
||||
for field, values in declared.items()
|
||||
)
|
||||
|
||||
|
||||
@@ -22,11 +22,6 @@ from common import _norm_system_id, resolve_platform_cores, unresolved_platform_
|
||||
from exporter.baseline import HASH_FIELDS, NativeSystem, _hash_values
|
||||
from launchers import _names as launcher_names
|
||||
|
||||
# Batocera cuts some md5 to 29 characters: the prefix still names the dump.
|
||||
_FULL_MD5 = 32
|
||||
_CUT_MD5 = 29
|
||||
|
||||
|
||||
def _tails(path: str) -> list[str]:
|
||||
"""Every trailing run of segments of a path, longest first."""
|
||||
parts = [p for p in path.casefold().strip("/").split("/") if p]
|
||||
@@ -48,22 +43,12 @@ def _declared_hashes(entry: dict) -> dict[str, set[str]]:
|
||||
return {f: values for f, values in found.items() if values}
|
||||
|
||||
|
||||
def _same(declared: str, accepted: str) -> bool:
|
||||
"""Equal, or a cut md5 that starts the accepted one."""
|
||||
return declared == accepted or (
|
||||
_CUT_MD5 <= len(declared) < _FULL_MD5 and accepted.startswith(declared)
|
||||
)
|
||||
|
||||
|
||||
def _accepts(accepted: set[str], declared: dict[str, set[str]]) -> bool:
|
||||
"""True when the hashes a profile entry accepts, the members of its
|
||||
archive included, hold one the platform entry declares."""
|
||||
for values in declared.values():
|
||||
if values & accepted:
|
||||
return True
|
||||
cut = [d for d in values if _CUT_MD5 <= len(d) < _FULL_MD5]
|
||||
if cut and any(a.startswith(d) for d in cut for a in accepted):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -72,7 +57,7 @@ def _contradicts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
|
||||
the profile entry describes another dump under the same name."""
|
||||
for field_name, values in declared.items():
|
||||
own = set(_hash_values(profile_entry, field_name))
|
||||
if own and not any(_same(d, a) for d in values for a in own):
|
||||
if own and not set(values) & set(own):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -158,10 +143,7 @@ class ReadEvidence:
|
||||
hashes = {
|
||||
v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
|
||||
}
|
||||
if hashes & self.hashes or any(
|
||||
len(v) < _FULL_MD5 and any(h.startswith(v) for h in self.hashes)
|
||||
for v in hashes if len(v) >= _CUT_MD5
|
||||
):
|
||||
if hashes & self.hashes:
|
||||
return "hash"
|
||||
full = "/" + (tails[0] if tails else "")
|
||||
if any("/" + d in full for d in self.directories):
|
||||
@@ -188,7 +170,7 @@ class ProofIndex:
|
||||
|
||||
def __init__(self, readers: list[ReadEvidence]) -> None:
|
||||
self._index: dict[str, dict[str, set[int]]] = {
|
||||
kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member")
|
||||
kind: {} for kind in ("tail", "launched", "name", "hash", "dir", "tree", "member")
|
||||
}
|
||||
for position, evidence in enumerate(readers):
|
||||
for kind, keys in (
|
||||
@@ -199,11 +181,6 @@ class ProofIndex:
|
||||
):
|
||||
for key in keys:
|
||||
self._index[kind].setdefault(key, set()).add(position)
|
||||
# A cut md5 names any hash it is the start of.
|
||||
for value in evidence.hashes:
|
||||
for length in range(_CUT_MD5, _FULL_MD5):
|
||||
if len(value) >= length:
|
||||
self._index["cut"].setdefault(value[:length], set()).add(position)
|
||||
|
||||
def _hits(self, kind: str, keys: Iterable[str]) -> set[int]:
|
||||
table = self._index[kind]
|
||||
@@ -224,7 +201,6 @@ class ProofIndex:
|
||||
hashes = {
|
||||
v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
|
||||
}
|
||||
cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5}
|
||||
# "/" + d in "/" + path: d is a run of whole segments ending in "/",
|
||||
# so it is a run of the path's segments before the last one.
|
||||
parts = tails[0].split("/") if tails else []
|
||||
@@ -236,7 +212,7 @@ class ProofIndex:
|
||||
"path": self._hits("tail", tails[:-1]),
|
||||
"launcher": self._hits("launched", names | last),
|
||||
"name": self._hits("name", names | last),
|
||||
"hash": self._hits("hash", hashes) | self._hits("cut", cut),
|
||||
"hash": self._hits("hash", hashes),
|
||||
"directory": self._hits("dir", runs),
|
||||
"data_directory": self._hits("tree", runs)
|
||||
| self._index["member"].get(tails[0] if tails else "", set()),
|
||||
@@ -463,10 +439,6 @@ class EntryCategories:
|
||||
for value in values:
|
||||
for position, f in by_hash_index.get(value, ()):
|
||||
found[position] = f
|
||||
if _CUT_MD5 <= len(value) < _FULL_MD5:
|
||||
for accepted, hits in by_hash_index.items():
|
||||
if accepted.startswith(value):
|
||||
found.update(hits)
|
||||
if found:
|
||||
return [cited(found[p]) for p in sorted(found)]
|
||||
named = {position: f for n in names for position, f in by_name_index.get(n, ())}
|
||||
|
||||
@@ -318,6 +318,76 @@ def shadowed_alias_index(
|
||||
return {alias: sorted(sha1s) for alias, sha1s in sorted(index.items())}
|
||||
|
||||
|
||||
def _declarations(platforms_dir: Path, emulators_dir: Path) -> list[dict]:
|
||||
"""Every file entry the registered platforms and the profiles declare."""
|
||||
sources = [
|
||||
platforms_dir / f"{name}.yml"
|
||||
for name in list_registered_platforms(str(platforms_dir))
|
||||
] if platforms_dir.is_dir() else []
|
||||
entries: list[dict] = []
|
||||
for source in sources:
|
||||
with open(source) as f:
|
||||
config = yaml_load(f) or {}
|
||||
for system in (config.get("systems") or {}).values():
|
||||
entries.extend(system.get("files") or [])
|
||||
if emulators_dir.is_dir():
|
||||
for source in source_yaml_paths(emulators_dir):
|
||||
with open(source) as f:
|
||||
entries.extend((yaml_load(f) or {}).get("files") or [])
|
||||
return entries
|
||||
|
||||
|
||||
def proven_name_index(
|
||||
files: dict, indexes: dict, declarations: list[dict]
|
||||
) -> dict[str, list[str]]:
|
||||
"""Names whose held primary no declaration proves, mapped to the primaries
|
||||
of the same directory that declarations of that name prove by hash.
|
||||
|
||||
A hashless entry takes a name's file by name alone. The collection files
|
||||
the DSi Panda devkit boot ROMs as dsi_bios7.bin and dsi_bios9.bin, with
|
||||
the eMMC key-Y zeroed, while every hash declared under those names
|
||||
designates a retail dump beside them: RetroBat and dsperate were served
|
||||
the devkit ROMs. Another revision kept in .variants/ is not a primary,
|
||||
and a dump filed elsewhere is another machine's file.
|
||||
"""
|
||||
by_hash = {
|
||||
"sha1": {sha1: sha1 for sha1 in files},
|
||||
"md5": indexes.get("by_md5", {}),
|
||||
"crc32": indexes.get("by_crc32", {}),
|
||||
"sha256": indexes.get("by_sha256", {}),
|
||||
}
|
||||
proven: dict[str, set[str]] = {}
|
||||
for entry in declarations:
|
||||
# A zipped_file hash is the member's, not the file the name designates.
|
||||
if not isinstance(entry, dict) or entry.get("zipped_file"):
|
||||
continue
|
||||
name = str(entry.get("name") or "").rsplit("/", 1)[-1]
|
||||
if not name:
|
||||
continue
|
||||
for kind, index in by_hash.items():
|
||||
for value in _as_list(entry.get(kind)):
|
||||
held = index.get(value)
|
||||
if held in files:
|
||||
proven.setdefault(name, set()).add(held)
|
||||
result: dict[str, list[str]] = {}
|
||||
for name, designated in sorted(proven.items()):
|
||||
primaries = {
|
||||
sha1 for sha1 in indexes.get("by_name", {}).get(name, [])
|
||||
if sha1 in files and "/.variants/" not in files[sha1]["path"]
|
||||
}
|
||||
if not primaries or primaries & designated:
|
||||
continue
|
||||
homes = {os.path.dirname(files[sha1]["path"]) for sha1 in primaries}
|
||||
beside = [
|
||||
sha1 for sha1 in designated
|
||||
if "/.variants/" not in files[sha1]["path"]
|
||||
and os.path.dirname(files[sha1]["path"]) in homes
|
||||
]
|
||||
if beside:
|
||||
result[name] = sorted(beside, key=lambda sha1: files[sha1]["path"])
|
||||
return result
|
||||
|
||||
|
||||
def load_cache(cache_path: str) -> dict:
|
||||
"""Load cache file if it exists."""
|
||||
try:
|
||||
@@ -476,6 +546,9 @@ def _generate(args: argparse.Namespace) -> int:
|
||||
|
||||
indexes = build_indexes(files, aliases)
|
||||
indexes["by_shadowed_alias"] = shadowed_alias_index(shadowed, indexes["by_name"])
|
||||
indexes["by_proven_name"] = proven_name_index(
|
||||
files, indexes, _declarations(Path("platforms"), Path("emulators"))
|
||||
)
|
||||
total_size = sum(entry["size"] for entry in files.values())
|
||||
|
||||
database = {
|
||||
|
||||
+70
-13
@@ -76,6 +76,7 @@ from common import (
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
without_profile_blind,
|
||||
MANUFACTURER_PREFIXES,
|
||||
require_yaml,
|
||||
ProfileSelectionError,
|
||||
@@ -373,13 +374,25 @@ def _platform_pack_stem(
|
||||
rep_cfg = load_platform_config(representative, platforms_dir)
|
||||
version = rep_cfg.get("version", rep_cfg.get("dat_version", ""))
|
||||
version_tag = f"_{_name_part(str(version))}" if version else ""
|
||||
return platform_pack_prefix(group_platforms, representative, platforms_dir) + version_tag
|
||||
|
||||
|
||||
def platform_pack_prefix(
|
||||
group_platforms: list[str], representative: str, platforms_dir: str
|
||||
) -> str:
|
||||
"""Platform part of a pack name without the version.
|
||||
|
||||
The site reads a pack's version back by removing this prefix: a version
|
||||
may hold an underscore (AmberELEC prerelease-20261019_1835).
|
||||
"""
|
||||
if len(group_platforms) <= 1:
|
||||
return _name_part(rep_cfg.get("platform", representative), "_") + version_tag
|
||||
rep_cfg = load_platform_config(representative, platforms_dir)
|
||||
return _name_part(rep_cfg.get("platform", representative), "_")
|
||||
names = [
|
||||
load_platform_config(p, platforms_dir).get("platform", p)
|
||||
for p in group_platforms
|
||||
]
|
||||
return "_".join(_name_part(n) for n in names) + version_tag
|
||||
return "_".join(_name_part(n) for n in names)
|
||||
|
||||
|
||||
def platform_list_version(config: dict, platform: str) -> str:
|
||||
@@ -685,6 +698,9 @@ def _select_variants(
|
||||
one_per_slot: bool,
|
||||
required_only: bool = False,
|
||||
undeclared: list[dict] | None = None,
|
||||
zip_contents: dict | None = None,
|
||||
data_registry: dict | None = None,
|
||||
slot_overrides: dict[str, str] | None = None,
|
||||
) -> tuple[set, list, list]:
|
||||
"""Which regional and slot alternatives this pack leaves out.
|
||||
|
||||
@@ -714,6 +730,9 @@ def _select_variants(
|
||||
required_only=required_only,
|
||||
held_out=held,
|
||||
undeclared=undeclared,
|
||||
zip_contents=zip_contents,
|
||||
data_registry=data_registry,
|
||||
slot_overrides=slot_overrides,
|
||||
)
|
||||
if regions:
|
||||
region_index = region_mod.platform_region_index(
|
||||
@@ -968,7 +987,8 @@ def generate_pack(
|
||||
region_drops, region_fallbacks, slot_undecidable = _select_variants(
|
||||
config, pack_systems, emulators_dir, db, base_dest, emu_profiles,
|
||||
target_cores, source, regions, one_per_slot, required_only,
|
||||
undeclared=undeclared,
|
||||
undeclared=undeclared, zip_contents=zip_contents,
|
||||
data_registry=data_registry, slot_overrides=slot_overrides,
|
||||
)
|
||||
|
||||
with (
|
||||
@@ -1646,6 +1666,9 @@ def generate_emulator_pack(
|
||||
) -> str | None:
|
||||
"""Generate a ZIP pack for specific emulator profiles.
|
||||
|
||||
Returns the path, "" when no file is needed (verify --emulator reports
|
||||
0/0 OK), None on failure.
|
||||
|
||||
`pack_name` is the caller's own file name: the system pack names its
|
||||
archive after the system, and joining 23 SNES emulator names went past
|
||||
the 255 bytes a file name may hold. `system_ids` keeps only the entries
|
||||
@@ -1680,8 +1703,14 @@ def generate_emulator_pack(
|
||||
if record is None:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
data_registry = load_data_dir_registry(
|
||||
platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms")
|
||||
)
|
||||
region_drops: set[str] = (
|
||||
emulator_region_drops(selected, standalone, regions, required_only, db)
|
||||
emulator_region_drops(
|
||||
selected, standalone, regions, required_only, db,
|
||||
zip_contents=zip_contents, data_registry=data_registry,
|
||||
)
|
||||
if regions
|
||||
else set()
|
||||
)
|
||||
@@ -1712,9 +1741,6 @@ def generate_emulator_pack(
|
||||
missing_files.append(name)
|
||||
|
||||
data_dir_notices: list[str] = []
|
||||
data_registry = load_data_dir_registry(
|
||||
platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms")
|
||||
)
|
||||
|
||||
with (
|
||||
contextlib.nullcontext(record) if record is not None
|
||||
@@ -1984,10 +2010,16 @@ def generate_emulator_pack(
|
||||
)
|
||||
|
||||
# Remove empty ZIP (no files packed and no missing = nothing to ship)
|
||||
if record is None and total_files + data_files == 0 and not (
|
||||
nothing_needed = total_files + data_files == 0 and not (
|
||||
missing_files or rejected_files or instruction_files or unsourceable_files
|
||||
):
|
||||
)
|
||||
if record is None and nothing_needed:
|
||||
os.unlink(zip_path)
|
||||
# What verify --emulator reports as 0/0 OK: no file is needed, so
|
||||
# no pack is written and the run succeeds. Printed with the path,
|
||||
# the run named a file it had just removed and exited 1.
|
||||
print(f" {', '.join(display_names)}: no file needed, no pack written")
|
||||
return ""
|
||||
|
||||
# Report. The members of data trees are counted apart: verify --emulator
|
||||
# counts the profile's files, and folding 3500 tree members into the
|
||||
@@ -2044,7 +2076,10 @@ def generate_system_pack(
|
||||
offline: bool | None = None,
|
||||
platforms_dir: str | None = None,
|
||||
) -> str | None:
|
||||
"""Generate a ZIP pack for all emulators supporting given system IDs."""
|
||||
"""Generate a ZIP pack for all emulators supporting given system IDs.
|
||||
|
||||
Returns the path, "" when no file is needed, None on failure.
|
||||
"""
|
||||
profiles = load_emulator_profiles(emulators_dir)
|
||||
matching = profiles_for_systems(profiles, system_ids, standalone)
|
||||
|
||||
@@ -2108,7 +2143,7 @@ def generate_system_pack(
|
||||
final_dir=output_dir,
|
||||
)
|
||||
if not result:
|
||||
return None
|
||||
return result
|
||||
new_path = os.path.join(output_dir, final_name)
|
||||
os.replace(result, new_path)
|
||||
return new_path
|
||||
@@ -3722,7 +3757,7 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
offline=args.offline,
|
||||
platforms_dir=args.platforms_dir,
|
||||
)
|
||||
if not built:
|
||||
if built is None:
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
@@ -3743,7 +3778,7 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
offline=args.offline,
|
||||
platforms_dir=args.platforms_dir,
|
||||
)
|
||||
if not built:
|
||||
if built is None:
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
@@ -3787,6 +3822,20 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
platforms = _platforms_with_systems(
|
||||
args, platforms, system_filter, emu_profiles, target_cores_cache
|
||||
)
|
||||
platforms = without_profile_blind(
|
||||
platforms,
|
||||
[
|
||||
flag for flag, given in (
|
||||
("--region", getattr(args, "regions", None)),
|
||||
("--one-per-slot", getattr(args, "one_per_slot", False)),
|
||||
("--no-game-data", getattr(args, "no_game_data", False)),
|
||||
) if given
|
||||
],
|
||||
args.platforms_dir,
|
||||
emu_profiles,
|
||||
target_cores_cache if args.target else None,
|
||||
args.all,
|
||||
)
|
||||
|
||||
data_registry = load_data_dir_registry(args.platforms_dir)
|
||||
# The output directory is claimed before the data directory refresh: a
|
||||
@@ -4365,6 +4414,9 @@ def _manifest_selection(
|
||||
required_only: bool,
|
||||
regions: list[str] | None,
|
||||
undeclared: list[dict] | None = None,
|
||||
zip_contents: dict | None = None,
|
||||
data_registry: dict | None = None,
|
||||
slot_overrides: dict[str, str] | None = None,
|
||||
) -> tuple[set[str], SelectionIndex, set[str]]:
|
||||
"""Destinations a region list removes, the groups the installer replays,
|
||||
and the destinations the builder holds.
|
||||
@@ -4388,6 +4440,9 @@ def _manifest_selection(
|
||||
extras_out=extras,
|
||||
held_out=held,
|
||||
undeclared=undeclared,
|
||||
zip_contents=zip_contents,
|
||||
data_registry=data_registry,
|
||||
slot_overrides=slot_overrides,
|
||||
)
|
||||
region_index = region_mod.platform_region_index(
|
||||
config, emu_profiles, target_cores
|
||||
@@ -4667,6 +4722,8 @@ def generate_manifest(
|
||||
region_drops, selection, builder_held = _manifest_selection(
|
||||
config, pack_systems, emulators_dir, db, base_dest, emu_profiles,
|
||||
target_cores, source, required_only, regions, undeclared=undeclared,
|
||||
zip_contents=zip_contents, data_registry=data_registry,
|
||||
slot_overrides=slot_overrides,
|
||||
)
|
||||
|
||||
# Phase 1: baseline files
|
||||
|
||||
@@ -608,11 +608,26 @@ def _release_date(tag: str) -> str:
|
||||
return f"{moment.day} {moment.strftime('%B %Y')}"
|
||||
|
||||
|
||||
def _pack_version(pack_name: str) -> str:
|
||||
def _pack_version(pack_name: str, platforms: list[str]) -> str:
|
||||
"""The platform version a pack was built for, from its name
|
||||
(Batocera_43.1_BIOS_Pack.zip is 43.1)."""
|
||||
(Batocera_43.1_BIOS_Pack.zip is 43.1).
|
||||
|
||||
Read past the platform part the builder writes, not at the last
|
||||
underscore: AmberELEC_prerelease-20261019_1835 read as 1835.
|
||||
"""
|
||||
from generate_pack import platform_pack_prefix # noqa: PLC0415
|
||||
|
||||
stem = pack_name.removesuffix("_BIOS_Pack.zip")
|
||||
return stem.rsplit("_", 1)[1] if "_" in stem else ""
|
||||
prefix = platform_pack_prefix(
|
||||
platforms, platforms[0], str(siteinputs.current().platforms)
|
||||
) + "_"
|
||||
return stem[len(prefix):] if stem.startswith(prefix) else ""
|
||||
|
||||
|
||||
def _pack_platforms(platform_name: str, record: dict, pack_name: str) -> list[str]:
|
||||
"""The platforms a release pack was built for, the one asked first."""
|
||||
listed = list(record.get("packs", {}).get(pack_name, {}).get("platforms") or [])
|
||||
return listed or [platform_name]
|
||||
|
||||
|
||||
def release_line(platform_name: str, record: dict) -> list[str]:
|
||||
@@ -631,7 +646,9 @@ def release_line(platform_name: str, record: dict) -> list[str]:
|
||||
]
|
||||
config = load_platform_config(platform_name, str(siteinputs.current().platforms))
|
||||
page_version = str(config.get("version") or "")
|
||||
pack_version = _pack_version(pack_name)
|
||||
pack_version = _pack_version(
|
||||
pack_name, _pack_platforms(platform_name, record, pack_name)
|
||||
)
|
||||
if page_version and pack_version and page_version.lstrip("v") != pack_version.lstrip("v"):
|
||||
display = config.get("platform", platform_name)
|
||||
lines.append(f"This pack was built for {display} {pack_version}. The command "
|
||||
@@ -658,7 +675,10 @@ def setup_version(platform_name: str, record: dict | None = None) -> str:
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
pack_name = release_record.pack_name_for(platform_name, record or {})
|
||||
version = (_pack_version(pack_name) if pack_name else "") or str(config.get("version") or "")
|
||||
version = (
|
||||
_pack_version(pack_name, _pack_platforms(platform_name, record or {}, pack_name))
|
||||
if pack_name else ""
|
||||
) or str(config.get("version") or "")
|
||||
if not version:
|
||||
return ""
|
||||
return f"{config.get('platform', platform_name)} {version.removeprefix('v')}"
|
||||
|
||||
@@ -11,6 +11,7 @@ import zipfile
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
_ALL_ALGORITHMS = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"})
|
||||
@@ -130,6 +131,20 @@ def md5_composite_of(source: str | Path | BinaryIO) -> str:
|
||||
h.update(zf.read(name))
|
||||
return h.hexdigest()
|
||||
|
||||
def cut_md5_owner(cut: str, md5s: Iterable[str]) -> str | None:
|
||||
"""The one md5 a value cut short starts, None when none or several do.
|
||||
|
||||
The single rule for a cut md5, read by the scraper, the resolver and
|
||||
--from-md5. It names a dump, it never verifies one: Batocera compares
|
||||
the whole string (batocera-systems:1088), so every verdict compares
|
||||
exact digests.
|
||||
"""
|
||||
if not cut or len(cut) >= 32:
|
||||
return None
|
||||
found = {md5 for md5 in md5s if md5.startswith(cut)}
|
||||
return found.pop() if len(found) == 1 else None
|
||||
|
||||
|
||||
def parse_hash_list(raw: str | list | int | None) -> list[str]:
|
||||
"""Normalize any hash field into its accepted values, lowercase.
|
||||
|
||||
|
||||
+25
-15
@@ -250,23 +250,33 @@ def _pin_verified(
|
||||
passed off as the one verified. None when the name moved in between.
|
||||
"""
|
||||
revision = os.path.join(dest_dir, VERIFIED_STORE, snapshot[0]["sha1"])
|
||||
os.makedirs(revision, exist_ok=True)
|
||||
final = os.path.join(revision, name)
|
||||
if os.path.isfile(final):
|
||||
return final
|
||||
scratch = os.path.join(revision, f".{os.getpid()}.{os.urandom(4).hex()}.tmp")
|
||||
try:
|
||||
os.link(path, scratch)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
try:
|
||||
if os.stat(scratch).st_ino != snapshot[2]:
|
||||
return None
|
||||
os.replace(scratch, final)
|
||||
return final
|
||||
finally:
|
||||
# A prune removes an empty revision directory: one that went between
|
||||
# makedirs and link is made again, once.
|
||||
for _attempt in range(2):
|
||||
os.makedirs(revision, exist_ok=True)
|
||||
# The link must hold the inode just verified. An older link left by a
|
||||
# copy that lost its cache name has one link left, and the prune of
|
||||
# this very download removed it after it was returned.
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(scratch)
|
||||
if os.stat(final).st_ino == snapshot[2]:
|
||||
return final
|
||||
scratch = os.path.join(revision, f".{os.getpid()}.{os.urandom(4).hex()}.tmp")
|
||||
try:
|
||||
os.link(path, scratch)
|
||||
except FileNotFoundError:
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
continue
|
||||
try:
|
||||
if os.stat(scratch).st_ino != snapshot[2]:
|
||||
return None
|
||||
os.replace(scratch, final)
|
||||
return final
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(scratch)
|
||||
return None
|
||||
|
||||
|
||||
def _prune_verified(dest_dir: str) -> None:
|
||||
|
||||
+47
-13
@@ -1222,8 +1222,14 @@ def _collect_emulator_extras(
|
||||
)
|
||||
continue
|
||||
if claimants is not None:
|
||||
# Keyed as extra_region_key keys the extra that wins the path: a
|
||||
# losing claim kept the bare path, and on RetroDECK (declared
|
||||
# root bios/) verify --region listed Rustation's scph1002b.bin
|
||||
# as packed while the builder had withdrawn bios/scph1002b.bin.
|
||||
claimants[(u.get("emulator", ""), u.get("name", ""), u.get("path") or "")] = (
|
||||
sanitize_pack_path(placements[0][0])
|
||||
prefixed_destination(
|
||||
sanitize_pack_path(placements[0][0]), declared_root(config)
|
||||
)
|
||||
)
|
||||
for dest, full_dest in placements:
|
||||
candidates = [
|
||||
@@ -1633,17 +1639,26 @@ def report_owned_by_systems(report: list[dict], system_ids: list[str]) -> list[d
|
||||
]
|
||||
|
||||
|
||||
def _ships(config: dict, file_entry: dict, db: dict) -> bool:
|
||||
def _ships(
|
||||
config: dict,
|
||||
file_entry: dict,
|
||||
db: dict,
|
||||
zip_contents: dict | None,
|
||||
data_registry: dict | None,
|
||||
override: str | None,
|
||||
) -> bool:
|
||||
"""Whether the builder packs this declaration: resolved, and not refused
|
||||
by a hash frontend. A dump whose md5 Recalbox would reject is left out
|
||||
of the pack, so it cannot win a regional slot over a file that ships.
|
||||
|
||||
Resolved with the builder's own inputs: without the data directories,
|
||||
AmberELEC's bluemsx config.ini read as missing while the pack held it.
|
||||
"""
|
||||
from packresolve import resolve_declaration
|
||||
from validation import baseline_verdict
|
||||
|
||||
# The builder's resolution: no destination hint, release assets included.
|
||||
path, status = resolve_declaration(
|
||||
file_entry, db, "bios", None, None, None, offline=True,
|
||||
file_entry, db, "bios", zip_contents, data_registry, override, offline=True,
|
||||
)
|
||||
if not path:
|
||||
return False
|
||||
@@ -1668,6 +1683,9 @@ def platform_region_groups(
|
||||
extras_out: list[dict] | None = None,
|
||||
held_out: set[str] | None = None,
|
||||
undeclared: list[dict] | None = None,
|
||||
zip_contents: dict | None = None,
|
||||
data_registry: dict | None = None,
|
||||
slot_overrides: dict[str, str] | None = None,
|
||||
) -> tuple[dict[str, list[tuple[str, str]]], dict[tuple[str, str, str], str]]:
|
||||
"""Group a platform's pack candidates the way region filtering reads them.
|
||||
|
||||
@@ -1682,7 +1700,9 @@ def platform_region_groups(
|
||||
two answer differently on the same request. `extras_out` receives the
|
||||
core extras that were grouped, for the install manifest. ``undeclared``
|
||||
is the gap report when the caller holds it (undeclared_report, with the
|
||||
same target_cores and include_all).
|
||||
same target_cores and include_all). ``zip_contents``, ``data_registry``
|
||||
and ``slot_overrides`` (keyed by full pack destination) are the ones the
|
||||
builder resolves with, so a held declaration is the one the pack ships.
|
||||
"""
|
||||
groups: dict[str, list[tuple[str, str]]] = {}
|
||||
for sys_id, system in systems.items():
|
||||
@@ -1695,8 +1715,10 @@ def platform_region_groups(
|
||||
)
|
||||
if dest:
|
||||
members.append((dest, file_entry.get("name", "")))
|
||||
full_dest = f"{base_dest}/{dest}" if base_dest else dest
|
||||
if held_out is not None and db is not None and _ships(
|
||||
config, file_entry, db
|
||||
config, file_entry, db, zip_contents, data_registry,
|
||||
(slot_overrides or {}).get(full_dest),
|
||||
):
|
||||
held_out.add(dest)
|
||||
|
||||
@@ -1799,6 +1821,8 @@ def emulator_region_drops(
|
||||
regions: list[str],
|
||||
required_only: bool = False,
|
||||
db: dict | None = None,
|
||||
zip_contents: dict | None = None,
|
||||
data_registry: dict | None = None,
|
||||
) -> set[str]:
|
||||
"""Destinations an emulator-mode `--region` withdraws.
|
||||
|
||||
@@ -1810,7 +1834,8 @@ def emulator_region_drops(
|
||||
With `db`, a requested-region file the collection lacks never displaces
|
||||
a held one, as in platform mode: a held scph5501.bin was withdrawn for a
|
||||
Japanese image nobody had collected, and the pack shipped no PlayStation
|
||||
BIOS at all.
|
||||
BIOS at all. ``zip_contents`` and ``data_registry`` are the ones the
|
||||
emulator pack resolves with.
|
||||
"""
|
||||
import region as region_mod
|
||||
from packpaths import _resolve_destination
|
||||
@@ -1832,27 +1857,36 @@ def emulator_region_drops(
|
||||
(dest, fe.get("name", ""))
|
||||
)
|
||||
if held is not None and _emulator_ships(
|
||||
fe, emu_name, profile, db, dest, validation_index
|
||||
fe, emu_name, profile, db, validation_index,
|
||||
zip_contents, data_registry,
|
||||
):
|
||||
held.add(dest)
|
||||
return region_mod.resolve_region_drops(region_groups, region_index, regions, held)
|
||||
|
||||
|
||||
def _emulator_ships(
|
||||
file_entry: dict, emu_name: str, profile: dict, db: dict, dest: str,
|
||||
validation_index: dict,
|
||||
file_entry: dict, emu_name: str, profile: dict, db: dict,
|
||||
validation_index: dict, zip_contents: dict | None, data_registry: dict | None,
|
||||
) -> bool:
|
||||
"""Whether the emulator pack ships this entry: held, and not refused by
|
||||
the core's own check, the test _ships applies on the platform side.
|
||||
|
||||
Resolved as generate_emulator_pack resolves: without the data
|
||||
directories, dolphin-mmjr2's nwc24msg.cfg read as missing while the pack
|
||||
held it.
|
||||
"""
|
||||
from packresolve import resolve_file
|
||||
from validation import validated_choice, with_archive_members
|
||||
|
||||
local, _status = resolve_local_file(
|
||||
local, status = resolve_file(
|
||||
{**with_archive_members(file_entry, profile.get("files", [])),
|
||||
"source_profile": emu_name},
|
||||
db, dest_hint=dest,
|
||||
db, "bios", zip_contents,
|
||||
dest_hint=file_entry.get("path", ""),
|
||||
data_dir_registry=data_registry,
|
||||
offline=True,
|
||||
)
|
||||
if not local:
|
||||
if status in ("not_found", "user_provided") or not local:
|
||||
return False
|
||||
_chosen, disagreement = validated_choice(
|
||||
file_entry, local, db, validation_index, "bios", None, emulators={emu_name},
|
||||
|
||||
@@ -14,6 +14,7 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from largefiles import fetch_large_file
|
||||
from hashing import cut_md5_owner
|
||||
from common import (
|
||||
declared_hash_verdict,
|
||||
resolution_is_hash_exact,
|
||||
@@ -203,8 +204,8 @@ def pending_release_asset(
|
||||
return None
|
||||
|
||||
|
||||
# Digest length in hex digits -> kind. 29 is the MD5 Batocera writes with
|
||||
# its last three digits cut, matched by prefix as the resolver does.
|
||||
# Digest length in hex digits -> kind. A 29-digit MD5 is a value cut short,
|
||||
# looked up as the resolver looks one up (cut_md5_owner).
|
||||
_HASH_KINDS = {8: "crc32", 29: "md5", 32: "md5", 40: "sha1", 64: "sha256"}
|
||||
_HEX_DIGITS = re.compile(r"[0-9a-f]+")
|
||||
_WORD = re.compile(r"[0-9A-Za-z]+")
|
||||
@@ -289,9 +290,9 @@ def sha1_for_hash(hash_type: str, hash_val: str, db: dict) -> str | None:
|
||||
elif hash_type == "md5":
|
||||
by_md5 = indexes.get("by_md5", {})
|
||||
sha1 = by_md5.get(hash_val)
|
||||
if sha1 is None and len(hash_val) < 32:
|
||||
prefixed = {v for k, v in by_md5.items() if k.startswith(hash_val)}
|
||||
sha1 = prefixed.pop() if len(prefixed) == 1 else None
|
||||
if sha1 is None:
|
||||
owner = cut_md5_owner(hash_val, by_md5)
|
||||
sha1 = by_md5.get(owner) if owner else None
|
||||
else:
|
||||
sha1 = next(
|
||||
(k for k, e in files_db.items() if e.get("sha256") == hash_val), None
|
||||
|
||||
@@ -30,6 +30,7 @@ from packextras import platform_region_groups
|
||||
from packpaths import _flat
|
||||
from nativemode import reads_file_contents
|
||||
import region as region_mod
|
||||
import slots
|
||||
from packresolve import resolve_declaration, resolve_file
|
||||
from exporter.candidates import BIOS, EntryCategories
|
||||
from gamedata import without_game_data
|
||||
@@ -706,6 +707,7 @@ def verify_pack_against_platform(
|
||||
config, emu_profiles, target_cores
|
||||
)
|
||||
held: set[str] = set()
|
||||
held_zip_contents = build_zip_contents_index(db) if db is not None else {}
|
||||
region_groups, _extra_dests = platform_region_groups(
|
||||
config,
|
||||
expected_systems,
|
||||
@@ -718,6 +720,12 @@ def verify_pack_against_platform(
|
||||
include_all=(source == "truth"),
|
||||
required_only=required_only,
|
||||
held_out=held,
|
||||
zip_contents=held_zip_contents,
|
||||
data_registry=data_registry,
|
||||
slot_overrides=slots.pack_overrides(
|
||||
config, emu_profiles, db, held_zip_contents, data_registry,
|
||||
target_cores=target_cores, source=source,
|
||||
) if db is not None else None,
|
||||
)
|
||||
region_drops = region_mod.resolve_region_drops(
|
||||
region_groups, region_index, regions, held
|
||||
|
||||
@@ -29,14 +29,15 @@ from common import copy_file_atomic, load_database
|
||||
|
||||
|
||||
def gitignored_paths(gitignore: str) -> set[str]:
|
||||
"""Paths the repository keeps out of git, as written in .gitignore."""
|
||||
try:
|
||||
with open(gitignore) as f:
|
||||
return {
|
||||
line.strip() for line in f if line.strip().startswith("bios/")
|
||||
}
|
||||
except FileNotFoundError:
|
||||
return set()
|
||||
"""Paths the repository keeps out of git, as written in .gitignore.
|
||||
|
||||
A missing file raises: read as empty, it left nothing to restore and
|
||||
nothing unsatisfied, and the run exited 0 with every asset absent.
|
||||
"""
|
||||
with open(gitignore) as f:
|
||||
return {
|
||||
line.strip() for line in f if line.strip().startswith("bios/")
|
||||
}
|
||||
|
||||
|
||||
def index_cache(cache_dir: str) -> dict[str, str]:
|
||||
@@ -144,6 +145,8 @@ def main() -> None:
|
||||
help="the .gitignore listing release-asset files (default: %(default)s)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if not os.path.isfile(args.gitignore):
|
||||
parser.error(f"no file at {args.gitignore}: it lists the release assets")
|
||||
_restored, unsatisfied = restore(args.cache, args.db, args.gitignore)
|
||||
if unsatisfied:
|
||||
raise SystemExit(1)
|
||||
|
||||
@@ -18,6 +18,7 @@ from pathlib import Path
|
||||
|
||||
|
||||
from common import yaml_load
|
||||
from hashing import cut_md5_owner
|
||||
from safeparse import yaml_load_upstream
|
||||
|
||||
from .base_scraper import (
|
||||
@@ -312,15 +313,15 @@ def _load_md5_index() -> dict[str, str]:
|
||||
|
||||
|
||||
def _resolve_truncated_md5(md5: str, md5_index: dict[str, str]) -> str:
|
||||
"""Resolve a truncated MD5 to its full 32-char version via prefix match."""
|
||||
"""Resolve a truncated MD5 to its full 32-char version (cut_md5_owner)."""
|
||||
if not md5 or len(md5) == 32:
|
||||
return md5
|
||||
if not _MD5_RE.match(md5):
|
||||
return md5
|
||||
matches = [k for k in md5_index if k.startswith(md5)]
|
||||
if len(matches) == 1:
|
||||
print(f" fixed truncated md5: {md5} -> {matches[0]}", file=sys.stderr)
|
||||
return matches[0]
|
||||
owner = cut_md5_owner(md5, md5_index)
|
||||
if owner:
|
||||
print(f" fixed truncated md5: {md5} -> {owner}", file=sys.stderr)
|
||||
return owner
|
||||
return md5
|
||||
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ feeds the report this module prints.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import zlib
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -892,10 +893,37 @@ def _accepted_by_all(path: str, entries: list[dict], db: dict) -> bool:
|
||||
for field_name in ("md5", "sha1", "sha256", "crc32"):
|
||||
declared = _declared_hashes(entry, field_name)
|
||||
if declared and str(record.get(field_name, "")).lower() not in declared:
|
||||
if field_name == "crc32" and _headerless_crc32(path, record, entry) in declared:
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
COPIER_HEADER = 512
|
||||
|
||||
|
||||
def _headerless_crc32(path: str, record: dict, entry: dict) -> str | None:
|
||||
"""The crc32 of a file past its 512-byte copier header, when the entry
|
||||
declares both the file's size and the size without the header.
|
||||
|
||||
Geargrafx drops the header when the file is 512 bytes over the expected
|
||||
size and computes the crc32 on the rest (src/media.cpp:754-773). The
|
||||
profile lists both sizes. EmuELEC's headered syscard3.pce (crc32
|
||||
64f78e3c) is the 6d9a73ef image the profile names.
|
||||
"""
|
||||
sizes = entry.get("size")
|
||||
declared = sizes if isinstance(sizes, list) else [sizes]
|
||||
size = record.get("size")
|
||||
if not isinstance(size, int) or not {size, size - COPIER_HEADER} <= set(declared):
|
||||
return None
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
f.seek(COPIER_HEADER)
|
||||
return f"{zlib.crc32(f.read()):08x}"
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _same_file_family(paths: list[str]) -> bool:
|
||||
"""Whether the paths are one file and its own pinned variants.
|
||||
|
||||
|
||||
@@ -857,8 +857,8 @@ def frontend_digest_matches(
|
||||
|
||||
resolve_local_file calls a file `hash_mismatch` when any declared hash
|
||||
disagrees, but a frontend compares one digest. verify_entry_md5 reads it
|
||||
too: Batocera compares a truncated md5 by prefix and Recalbox hashes a
|
||||
ZIP by its members. Under SIZE_AND_ANY_HASH (RomM) the declared size must
|
||||
too: Recalbox hashes a ZIP by its members. Batocera compares the whole
|
||||
md5 string (batocera-systems:1088), so a value cut short accepts nothing. Under SIZE_AND_ANY_HASH (RomM) the declared size must
|
||||
match and then any one of md5, sha1 and crc32 does: a mistyped hash next
|
||||
to a right one does not withhold the file, a wrong size does.
|
||||
"""
|
||||
@@ -918,9 +918,7 @@ def digest_verdict(
|
||||
# Recalbox hashes a .zip by its members only (Bios.cpp:64-68), so the
|
||||
# container's own md5 answers nothing there.
|
||||
composite_only = file_entry.get("zip_digest") == "composite"
|
||||
if not composite_only and any(
|
||||
actual == d or (len(d) < 32 and actual.startswith(d)) for d in declared
|
||||
):
|
||||
if not composite_only and actual in declared:
|
||||
return True
|
||||
member_digest = composite()
|
||||
return bool(member_digest) and member_digest.lower() in declared
|
||||
|
||||
+29
-5
@@ -67,6 +67,7 @@ from common import (
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
without_profile_blind,
|
||||
resolve_system_arg,
|
||||
print_platform_listing,
|
||||
select_systems,
|
||||
@@ -1189,12 +1190,13 @@ def verify_platform(
|
||||
else load_emulator_profiles(emulators_dir)
|
||||
)
|
||||
base_dest = config.get("base_destination", "")
|
||||
pack_overrides = slots.pack_overrides(
|
||||
config, profiles, db, zip_contents, data_dir_registry,
|
||||
target_cores=target_cores,
|
||||
)
|
||||
slot_overrides = {
|
||||
(key[len(base_dest) + 1:] if base_dest and key.startswith(f"{base_dest}/") else key): path
|
||||
for key, path in slots.pack_overrides(
|
||||
config, profiles, db, zip_contents, data_dir_registry,
|
||||
target_cores=target_cores,
|
||||
).items()
|
||||
for key, path in pack_overrides.items()
|
||||
}
|
||||
|
||||
# Build HLE + validation indexes from emulator profiles
|
||||
@@ -1258,6 +1260,9 @@ def verify_platform(
|
||||
target_cores=target_cores,
|
||||
required_only=required_only,
|
||||
held_out=held,
|
||||
zip_contents=zip_contents,
|
||||
data_registry=data_dir_registry,
|
||||
slot_overrides=pack_overrides,
|
||||
)
|
||||
region_index = region_mod.platform_region_index(config, profiles, target_cores)
|
||||
region_drops = region_mod.resolve_region_drops(
|
||||
@@ -2138,7 +2143,10 @@ def verify_emulator(
|
||||
if regions:
|
||||
from packextras import emulator_region_drops
|
||||
|
||||
region_drops = emulator_region_drops(selected, standalone, regions, required_only, db=db)
|
||||
region_drops = emulator_region_drops(
|
||||
selected, standalone, regions, required_only, db=db,
|
||||
zip_contents=zip_contents, data_registry=data_registry,
|
||||
)
|
||||
|
||||
# The emulator pack walks profiles in name order and keeps the first
|
||||
# file at a path, folding case: a later profile's different file there
|
||||
@@ -2998,6 +3006,22 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
print(f"ERROR: {reason}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# generate_pack refuses these on a platform whose cores no profile
|
||||
# describes, and the report answers the same request the same way.
|
||||
platforms = without_profile_blind(
|
||||
platforms,
|
||||
[
|
||||
flag for flag, given in (
|
||||
("--region", requested_regions),
|
||||
("--no-game-data", getattr(args, "no_game_data", False)),
|
||||
) if given
|
||||
],
|
||||
args.platforms_dir,
|
||||
emu_profiles,
|
||||
target_cores_cache if args.target else None,
|
||||
args.all,
|
||||
)
|
||||
|
||||
# Group identical platforms (same function as generate_pack)
|
||||
groups = group_identical_platforms(
|
||||
platforms, args.platforms_dir, target_cores_cache if args.target else None
|
||||
|
||||
Reference in new issue
Block a user