chore: merge pack and verify fixes

This commit is contained in:
Abdessamad Derraz committed 2026-11-06 10:28:27 +01:00
commit d0c65dca22
37 files changed
+3267 -142

No files matched your search

+4
View File
@@ -3036,12 +3036,16 @@ SYSTEM_ALIASES: dict[str, str] = {
}
@functools.lru_cache(maxsize=None)
def _norm_system_id(sid: str) -> str:
"""Normalize system ID for cross-platform matching.
Resolves known aliases, then strips manufacturer prefixes and separators
so that platform-specific IDs (e.g., "xbox", "nintendo-wiiu") match
profile IDs (e.g., "microsoft-xbox", "nintendo-wii-u").
A function of the string and two constant tables: a manifest run asks it
1.7 million times for about 1,100 ids.
"""
s = sid.lower().replace("_", "-")
s = SYSTEM_ALIASES.get(s, s)
+22 -4
View File
@@ -26,7 +26,9 @@ sys.path.insert(0, os.path.dirname(__file__))
from common import (
platform_arg,
ProfileSelectionError,
_foreign_tree,
_norm_system_id,
_owner_tokens,
archive_entry,
directory_members,
expand_directory_entries,
@@ -215,8 +217,14 @@ def _resolve_source(
none of the hashes the entry declares: a name never wins over a declared
hash, and zquestclassic's ending.mid read as held because Hydra Castle
Labyrinth has another file of that name.
Another casing of the name in another emulator's tree proves nothing, as
in resolve_local_file's case-insensitive step: dosbox-x's et4000.bin read
as held through pcem's et4000.BIN, which the pack and the installer
refuse it.
"""
declared = _declared_content(file_entry or {})
owners = _owner_tokens(file_entry or {})
def _can_be(sha1s: list | str) -> bool:
if not file_entry or db_files is None:
@@ -233,6 +241,14 @@ def _resolve_source(
"""Whether the files indexed under *name* can be this entry."""
return _can_be(by_name.get(name) or [])
def _folded_hit(name: str) -> bool:
"""_name_hit for a name matched in another casing."""
return _can_be([
sha1 for sha1 in by_name.get(name) or []
if db_files is None
or not _foreign_tree(db_files.get(sha1, {}).get("path", ""), owners)
])
# bios/ via database.json by_name
if fname in by_name and _name_hit(fname):
return "bios"
@@ -241,10 +257,10 @@ def _resolve_source(
if basename and basename in by_name and _name_hit(basename):
return "bios"
key = fname.lower()
if key in by_name_lower and _name_hit(by_name_lower[key]):
if key in by_name_lower and _folded_hit(by_name_lower[key]):
return "bios"
if basename:
if basename.lower() in by_name_lower and _name_hit(
if basename.lower() in by_name_lower and _folded_hit(
by_name_lower[basename.lower()]
):
return "bios"
@@ -257,7 +273,7 @@ def _resolve_source(
if canonical and canonical != fname:
if canonical in by_name and _name_hit(canonical):
return "bios"
if canonical.lower() in by_name_lower and _name_hit(
if canonical.lower() in by_name_lower and _folded_hit(
by_name_lower[canonical.lower()]
):
return "bios"
@@ -901,7 +917,9 @@ def _cross_reference_profile(
source = next(
(
found for version in versions[slot]
if (found := entry_source(version, index)) is not None
if (found := entry_source(
{**version, "source_profile": emu_name}, index,
)) is not None
),
"missing",
)
+146 -49
View File
@@ -26,6 +26,7 @@ import argparse
import contextlib
import functools
import hashlib
import io
import json
import os
import re
@@ -158,6 +159,9 @@ class PackRecord:
self.members: list[str] = []
self.files: list[str] = []
self.trees: list[str] = []
# The collection file each copied member comes from: a split part
# writes a file only where the full pack writes the same bytes.
self.sources: dict[str, str] = {}
def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> None:
@@ -173,6 +177,7 @@ def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> Non
"""
if isinstance(zf, PackRecord):
zf.members.append(arcname)
zf.sources[arcname] = source_path
return
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
info.compress_type = _member_compression(arcname)
@@ -603,6 +608,17 @@ def _pack_data_directories(
return added
def _case_twin_pairs(
twins: list[str], seen_destinations: set[str], base_dest: str, flatten: bool,
) -> list[tuple[str, str]]:
"""Each spelling left out, with the spelling the pack holds."""
held = {
_flat(dest, base_dest, flatten).lower(): _flat(dest, base_dest, flatten)
for dest in seen_destinations
}
return sorted({(twin, held[twin.lower()]) for twin in twins if twin.lower() in held})
def _report_pack_counts(
zip_path: str,
file_status: dict,
@@ -806,6 +822,7 @@ def generate_pack(
pack_name: str | None = None,
no_game_data: bool = False,
record: PackRecord | None = None,
within: dict[str, str] | None = None,
) -> str | None:
"""Generate a ZIP pack for a platform.
@@ -819,6 +836,12 @@ def generate_pack(
``record`` builds the pack in memory (PackRecord): no archive is written,
no external file fetched, and the paths are kept on it.
``within`` is the full pack's PackRecord.sources, given to a split part:
a declared file or core extra is written only where the full pack writes
the same bytes. A part sees neither the other systems' declarations nor
their extras, so a path the full pack settles (a file/directory conflict,
a case twin, a path another system's file takes) was written again.
Returns the path to the generated ZIP, or None on failure.
"""
config = load_platform_config(platform_name, platforms_dir)
@@ -867,6 +890,8 @@ def generate_pack(
user_provided = []
seen_destinations: set[str] = set()
seen_lower: set[str] = set() # only used when case_insensitive=True
# Spellings left out as case twins of a packed file, for the README.
case_twins: list[str] = []
seen_parents: set[str] = (
set()
) # parent dirs of added files (path conflict detection)
@@ -1135,7 +1160,10 @@ def generate_pack(
config.get("hash_rule", ""),
)
if refusal:
file_status[dedup_key] = "excluded"
# An earlier declaration of this path shipped a file this
# one refuses: it is in the pack, untested as verify
# counts it, not excluded.
file_status[dedup_key] = "untested" if already_packed else "excluded"
file_reasons[dedup_key] = refusal
continue
if not hash_mismatch_excludes_file(verification_mode) and (
@@ -1170,7 +1198,17 @@ def generate_pack(
)
if already_packed:
if dedup_key not in seen_destinations:
case_twins.append(_flat(full_dest, base_dest, flatten))
continue
if within is not None and not _same_bytes(
local_path, within.get(_flat(full_dest, base_dest, flatten))
):
# The full pack holds another system's file here.
continue
if file_status.get(dedup_key) == "excluded":
# An earlier declaration refused the file this one ships.
file_status[dedup_key] = "untested"
seen_destinations.add(dedup_key)
_register_path(dedup_key, seen_destinations, seen_parents)
if case_insensitive:
@@ -1227,7 +1265,7 @@ def generate_pack(
if required_only and fe.get("required") is False:
continue
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
if region_drops and dest in region_drops:
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
continue
if not dest:
continue
@@ -1244,10 +1282,13 @@ def generate_pack(
# Taken, a case twin on a case-insensitive filesystem, or a
# file/directory conflict (SGB1.sfc vs SGB1.sfc/): verify replays
# the same rule (packextras.hidden_extras).
if placement_block(
block = placement_block(
full_dest, seen_destinations, seen_lower, seen_parents,
case_insensitive,
):
)
if block:
if block == "case_twin" and extra_resolver.resolve(fe, dest)[0]:
case_twins.append(_flat(full_dest, base_dest, flatten))
continue
# The emulator's own check picks the copy, as for a declared file
@@ -1275,6 +1316,8 @@ def generate_pack(
)
flat_dest = _flat(full_dest, base_dest, flatten)
if within is not None and not _same_bytes(local_path, within.get(flat_dest)):
continue
if local_path.endswith(".zip"):
_add_zip_to_pack(local_path, flat_dest, zf, fe)
else:
@@ -1339,6 +1382,37 @@ def generate_pack(
if _registry_path.exists():
with open(_registry_path) as _rf:
_pack_registry = (yaml_load(_rf) or {}).get("platforms", {})
keeps_case = case_insensitive and any(
d.get("os") not in CASE_FOLDING_OS
for d in (_pack_registry.get(platform_name, {}).get("install", {}) or {})
.get("detect", [])
)
if keeps_case and source != "platform" and precomputed_extras is None:
# The spellings a folded platform counts as declared never reach
# the extras: collected as the filesystem that keeps case sees
# them, as the manifest's case_variants are (Mesen-CE's bios.col
# beside the declared BIOS.col).
unfolded = _collect_emulator_extras(
{**config, "case_insensitive_fs": False}, emulators_dir, db, set(),
base_dest, emu_profiles, target_cores=target_cores,
include_all=(source == "truth"),
)
if system_filter:
unfolded = _extras_for_systems(unfolded, system_filter)
prefix = _detect_extras_prefix(config, base_dest)
for fe in unfolded:
if required_only and fe.get("required") is False:
continue
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
continue
full = extra_full_destination(fe, prefix)
if (
full and full not in seen_destinations and full.lower() in seen_lower
and extra_resolver.resolve(
fe, sanitize_pack_path(fe.get("destination", fe["name"]))
)[0]
):
case_twins.append(_flat(full, base_dest, flatten))
readme_text = _build_readme(
platform_name, platform_display, base_dest, total_files, num_systems,
source=source,
@@ -1364,6 +1438,10 @@ def generate_pack(
) if save_count else ""
),
save_by_system=bool(save_count and save_layout(config).by_system),
case_twins=(
_case_twin_pairs(case_twins, seen_destinations, base_dest, flatten)
if keeps_case else []
),
)
_write_generated_member(zf, "README.txt", readme_text)
@@ -1408,6 +1486,7 @@ def _extract_zip_to_archive(
for name in map(sanitize_pack_path, src.namelist())
if name and not name.endswith("/")
)
target_zf.sources[dest_prefix] = source_zip
return
for info in src.infolist():
if info.is_dir():
@@ -1441,6 +1520,7 @@ def _add_zip_to_pack(
"""
if isinstance(target_zf, PackRecord):
target_zf.members.append(dest_path)
target_zf.sources[dest_path] = source_zip
return
_check_zip_members(source_zip)
_add_pack_member(target_zf, source_zip, dest_path)
@@ -2069,8 +2149,14 @@ def generate_split_packs(
one_per_slot: bool = False,
offline: bool | None = None,
no_game_data: bool = False,
records: dict[str, PackRecord] | None = None,
) -> list[str]:
"""Generate split packs (one ZIP per system or manufacturer)."""
"""Generate split packs (one ZIP per system or manufacturer).
The parts add up to the full pack: each is built against the full pack's
own placement (``within``). ``records`` builds the parts in memory, keyed
by part name, with no archive written.
"""
config = load_platform_config(platform_name, platforms_dir)
# The split directory carries the same tags as the packs inside it, or two
# differently narrowed runs write into one another.
@@ -2093,10 +2179,36 @@ def generate_split_packs(
f"{_platform_pack_stem([platform_name], platform_name, platforms_dir)}{split_tags}"
)
split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split")
os.makedirs(split_dir, exist_ok=True)
if records is None:
os.makedirs(split_dir, exist_ok=True)
if emu_profiles is None:
emu_profiles = load_emulator_profiles(emulators_dir)
narrowing = {
"emulators_dir": emulators_dir, "zip_contents": zip_contents,
"data_registry": data_registry, "emu_profiles": emu_profiles,
"target_cores": target_cores, "required_only": required_only,
"source": source, "regions": regions, "target_name": target_name,
"one_per_slot": one_per_slot, "offline": offline,
"no_game_data": no_game_data,
}
full = PackRecord()
with contextlib.redirect_stdout(io.StringIO()):
generate_pack(
platform_name, platforms_dir, db, bios_dir, split_dir,
record=full, **narrowing,
)
def part(pack_name: str, **scope: object) -> str | None:
record = PackRecord() if records is not None else None
zip_path = generate_pack(
platform_name, platforms_dir, db, bios_dir, split_dir,
pack_name=pack_name, record=record, within=full.sources,
**scope, **narrowing,
)
if record is not None and zip_path:
records[pack_name] = record
return zip_path
# The systems a core of the target runs: a part per declared system sent
# each one the target drops to a refusal naming it as unknown.
systems = platform_scope_systems(config, emu_profiles, target_cores)
@@ -2130,27 +2242,9 @@ def generate_split_packs(
results = []
for group_name, group_system_ids in sorted(groups.items()):
group_extras = _extras_for_systems(all_extras, group_system_ids)
zip_path = generate_pack(
platform_name,
platforms_dir,
db,
bios_dir,
split_dir,
emulators_dir=emulators_dir,
zip_contents=zip_contents,
data_registry=data_registry,
emu_profiles=emu_profiles,
target_cores=target_cores,
required_only=required_only,
system_filter=group_system_ids,
precomputed_extras=group_extras,
source=source,
regions=regions,
target_name=target_name,
one_per_slot=one_per_slot,
offline=offline,
pack_name=f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip",
no_game_data=no_game_data,
zip_path = part(
f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip",
system_filter=group_system_ids, precomputed_extras=group_extras,
)
if zip_path:
results.append(zip_path)
@@ -2166,15 +2260,9 @@ def generate_split_packs(
}
undistributed = [fe for fe in all_extras if id(fe) not in distributed]
if undistributed:
zip_path = generate_pack(
platform_name, platforms_dir, db, bios_dir, split_dir,
emulators_dir=emulators_dir, zip_contents=zip_contents,
data_registry=data_registry, emu_profiles=emu_profiles,
target_cores=target_cores, required_only=required_only,
precomputed_extras=undistributed, extras_only=True, source=source,
regions=regions, target_name=target_name, one_per_slot=one_per_slot,
offline=offline, pack_name=f"{part_prefix}_Other_Cores_BIOS_Pack.zip",
no_game_data=no_game_data,
zip_path = part(
f"{part_prefix}_Other_Cores_BIOS_Pack.zip",
precomputed_extras=undistributed, extras_only=True,
)
if zip_path:
results.append(zip_path)
@@ -4133,7 +4221,8 @@ def _manifest_core_entries(
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
if not dest:
continue
if region_drops and dest in region_drops:
region_key = extra_region_key(fe, declared_root(config))
if region_drops and region_key in region_drops:
continue
full_dest = extra_full_destination(fe, extras_pfx)
if not full_dest:
@@ -4207,7 +4296,7 @@ def _manifest_core_entries(
entry, file_size = _manifest_entry(
local_path, manifest_dest, [source_emu] if source_emu else [],
db, repo_root,
selection.fields(dest, fe.get("name", "")),
selection.fields(region_key, fe.get("name", "")),
)
if entry is not None and save_root:
entry.update(save_entry_fields(fe))
@@ -4321,7 +4410,8 @@ def _manifest_selection(
else set()
)
return drops, SelectionIndex.build(
region_groups, region_index, pack_systems, extras, required_only
region_groups, region_index, pack_systems, extras, required_only,
extras_prefix=declared_root(config),
), held
@@ -4346,7 +4436,9 @@ def _record_absent(
if required_only and fe.get("required") is False:
continue
dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
if not dest or (region_drops and dest in region_drops):
if not dest or (
region_drops and extra_region_key(fe, declared_root(config)) in region_drops
):
continue
full_dest = extra_full_destination(fe, extras_pfx)
if not full_dest:
@@ -4517,16 +4609,18 @@ def generate_manifest(
# (cache file, size). Settled once the trees are walked.
cache_held: dict[str, tuple[str, int]] = {}
rename_notes: set[str] = set()
# The full manifest records every core's answer at a contested slot, so a
# target narrowed from it gets what that target's own pack serves.
whole = target_cores is None and source != "platform" and bool(emu_profiles)
slot_choices = (
slots.slot_choices(config, emu_profiles, db, zip_contents, data_registry)
if whole else {}
)
# Derived from the same choices: computed twice, it was 8% of a run.
slot_overrides = slots.pack_overrides(
config, emu_profiles, db, zip_contents, data_registry,
target_cores=target_cores, source=source,
)
# The full manifest records every core's answer at a contested slot, so a
# target narrowed from it gets what that target's own pack serves.
slot_choices = (
slots.slot_choices(config, emu_profiles, db, zip_contents, data_registry)
if target_cores is None and source != "platform" and emu_profiles
else {}
choices=slot_choices if whole else None,
)
def manifest_destination(full_destination: str) -> str:
@@ -4541,8 +4635,10 @@ def generate_manifest(
reason: str,
cores: list[str] | None,
) -> None:
bare = sanitize_pack_path(
file_entry.get("destination", file_entry.get("name", ""))
# A core extra (cores given) is keyed as the region pass keys it.
bare = (
sanitize_pack_path(file_entry.get("destination", file_entry.get("name", "")))
if cores is None else extra_region_key(file_entry, declared_root(config))
)
save_root = file_entry.get("root") == "save"
omitted_by_destination[full_destination] = {
@@ -5443,6 +5539,7 @@ from packpaths import ( # noqa: E402,F401
from packextras import ( # noqa: E402,F401
ExtraResolver,
extra_full_destination,
extra_region_key,
prefixed_destination,
save_entry_fields,
save_manifest_dest,
+3 -3
View File
@@ -9478,7 +9478,7 @@ def _hash_spans(value: str | list, copy: bool = False) -> str:
def _availability_check(
db: dict, data_names: set[str] | None
db: dict, data_names: set[str] | None, owner: str = "",
) -> Callable[[dict], bool]:
"""A predicate answering whether the collection holds a file.
@@ -9509,7 +9509,7 @@ def _availability_check(
# under the name belongs to another game or release.
if not f.get("name") or f.get("unsourceable"):
return False
return entry_source(f, index) is not None
return entry_source({**f, "source_profile": owner} if owner else f, index) is not None
return _file_available
@@ -10242,7 +10242,7 @@ def emulator_pages(
lines.append("")
lines.extend(markup.aside("info", "Why no files", [_escape_tags(str(exclusion))])[:-1])
else:
_file_available = _availability_check(db, data_names)
_file_available = _availability_check(db, data_names, name)
lines.extend(["## Files", ""])
lines.extend(_emulator_file_summary(files))
+73 -11
View File
@@ -12,6 +12,7 @@ import os
import sys
import re
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
@@ -26,6 +27,13 @@ from hashing import compute_hashes
LARGE_FILES_RELEASE = "large-files"
LARGE_FILES_REPO = "Abdess/retrobios"
LARGE_FILES_CACHE = ".cache/large"
# Verified revisions, hard-linked as <sha1>/<asset> beside the cache. A name
# in the cache is replaced when the release serves new bytes under it, so a
# path handed out by name could hold other bytes by the time the caller read
# them. Nothing ever writes other bytes under a sha1. The asset name stays
# last: callers read the extension (an archive is packed as one).
VERIFIED_STORE = ".by-sha1"
ORPHAN_GRACE = 24 * 3600
GITIGNORE = Path(__file__).resolve().parent.parent / ".gitignore"
_UNSAFE_ASSET_CHARS = re.compile(r"[^A-Za-z0-9._-]+")
@@ -169,14 +177,14 @@ def _fetch_asset(
if os.path.isdir(cached):
return None
if os.path.exists(cached) and not (expected_sha1 or expected_md5):
return cached
if os.path.exists(cached):
snapshot = _read_once(cached)
if snapshot is None:
pass
elif _matches(snapshot[0], expected_sha1, expected_md5):
return cached
pinned = _pin_verified(cached, snapshot, dest_dir, name)
if pinned:
return pinned
elif not _replaced_on_release(name, snapshot[0]["sha256"], snapshot[1], offline):
# A copy of this asset that answers another hash: the caller
# wants a different revision under the same name, and the
@@ -220,17 +228,71 @@ def _fetch_asset(
print(f" large file {candidate}: {exc}", file=sys.stderr)
if not downloaded:
return None
if (expected_sha1 or expected_md5) and not _matches(
compute_hashes(tmp_path), expected_sha1, expected_md5
):
snapshot = _read_once(tmp_path)
if snapshot is None or not _matches(snapshot[0], expected_sha1, expected_md5):
return None
set_plain_mode(tmp_path)
pinned = _pin_verified(tmp_path, snapshot, dest_dir, name)
os.replace(tmp_path, cached)
return cached
_prune_verified(dest_dir)
return pinned
finally:
_drop(tmp_path)
def _pin_verified(
path: str, snapshot: tuple[dict[str, str], int, int], dest_dir: str, name: str,
) -> str | None:
"""The verified revision at *path*, as <sha1>/<name> in VERIFIED_STORE.
*snapshot* is what _read_once read: the link is kept only if it holds
that inode, so a copy another run promoted after the hash is never
passed off as the one verified. None when the name moved in between.
"""
revision = os.path.join(dest_dir, VERIFIED_STORE, snapshot[0]["sha1"])
os.makedirs(revision, exist_ok=True)
final = os.path.join(revision, name)
if os.path.isfile(final):
return final
scratch = os.path.join(revision, f".{os.getpid()}.{os.urandom(4).hex()}.tmp")
try:
os.link(path, scratch)
except FileNotFoundError:
return None
try:
if os.stat(scratch).st_ino != snapshot[2]:
return None
os.replace(scratch, final)
return final
finally:
with contextlib.suppress(FileNotFoundError):
os.unlink(scratch)
def _prune_verified(dest_dir: str) -> None:
"""Drop the revisions no cache name has held for ORPHAN_GRACE seconds.
Losing its cache name changes the inode's ctime, so a revision another
run pinned just before the release replaced it stays readable while that
run packs it.
"""
store = os.path.join(dest_dir, VERIFIED_STORE)
now = time.time()
for revision in os.scandir(store):
if not revision.is_dir():
continue
for entry in os.scandir(revision.path):
if entry.name.endswith(".tmp"):
continue
with contextlib.suppress(FileNotFoundError):
stat = entry.stat()
if stat.st_nlink == 1 and now - stat.st_ctime > ORPHAN_GRACE:
os.unlink(entry.path)
# Another run may be linking a copy into it: left for the next prune.
with contextlib.suppress(FileNotFoundError, OSError):
os.rmdir(revision.path)
def _asset_urls(name: str) -> list[tuple[str, str]]:
"""Release URLs an asset may be published under, with their names.
@@ -252,9 +314,9 @@ def _asset_urls(name: str) -> list[tuple[str, str]]:
]
def _read_once(path: str) -> tuple[dict[str, str], int] | None:
"""Digests and size of the file at *path*, all read from one file, or
None when it is gone.
def _read_once(path: str) -> tuple[dict[str, str], int, int] | None:
"""Digests, size and inode of the file at *path*, all read from one
file, or None when it is gone.
Every judgement on a cache copy reads this one snapshot. Hashed, then
hashed again by path for the release check, a copy another run promoted
@@ -271,7 +333,7 @@ def _read_once(path: str) -> tuple[dict[str, str], int] | None:
except FileNotFoundError:
return None
if (before.st_ino, before.st_mtime_ns) == (after.st_ino, after.st_mtime_ns):
return hashes, after.st_size
return hashes, after.st_size, after.st_ino
return None
+10 -2
View File
@@ -19,7 +19,13 @@ from dataclasses import dataclass
import region as region_mod
from common import _norm_system_id, sanitize_pack_path
from packextras import _kept, _serving_system_ids, extra_region_groups, system_spellings
from packextras import (
_kept,
_serving_system_ids,
extra_region_groups,
extra_region_key,
system_spellings,
)
def _dest(entry: dict) -> str:
@@ -44,6 +50,7 @@ class SelectionIndex:
pack_systems: dict[str, dict],
extras: list[dict],
required_only: bool = False,
extras_prefix: str = "",
) -> SelectionIndex:
by_norm: dict[str, set[str]] = {}
for sys_id in pack_systems:
@@ -60,7 +67,8 @@ class SelectionIndex:
extra_groups: dict[str, set[str]] = {}
spellings = system_spellings(pack_systems)
for extra in extras:
dest = _dest(extra)
# Keyed as platform_region_groups keys it (extra_region_key).
dest = extra_region_key(extra, extras_prefix)
if not dest:
continue
extra_groups.setdefault(dest, set()).update(
+18 -2
View File
@@ -62,6 +62,20 @@ def prefixed_destination(dest: str, prefix: str) -> str:
return f"{prefix}/{dest}"
def extra_region_key(fe: dict, prefix: str) -> str:
"""The key a core extra is grouped, dropped and held under by --region.
Its destination with the BIOS directory the platform writes in front of
its own (declared_root): RetroDECK declares bios/scph5501.bin, and
Beetle PSX's scph5501.bin grouped bare was looked up under another key,
kept by the pack while the declaration of the same file was dropped,
and the installer, which records the path once, dropped it.
"""
return prefixed_destination(
sanitize_pack_path(fe.get("destination", fe.get("name", ""))), prefix
)
def save_entry_fields(fe: dict) -> dict:
"""What a manifest entry adds for a file read below the save directory.
@@ -364,6 +378,7 @@ def hidden_extras(
content[dest.lower()] = sha1_by_path.get(local)
lower = set(content) if case_insensitive else set()
prefix = _detect_extras_prefix(config, base_dest)
region_prefix = declared_root(config)
hidden: dict[tuple[str, str, str], str] = {}
packed_keys: set[tuple[str, str, str]] = set()
extras = _collect_emulator_extras(
@@ -377,7 +392,7 @@ def hidden_extras(
if required_only and fe.get("required") is False:
continue
dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
if not dest or (region_drops and dest in region_drops):
if not dest or (region_drops and extra_region_key(fe, region_prefix) in region_drops):
continue
full = extra_full_destination(fe, prefix)
if not full:
@@ -1705,8 +1720,9 @@ def platform_region_groups(
)
held = {id(extra) for extra in extras}
spellings = system_spellings(systems)
region_prefix = declared_root(config)
for extra in _kept(extras + absent, required_only):
dest = sanitize_pack_path(extra.get("destination", extra.get("name", "")))
dest = extra_region_key(extra, region_prefix)
if not dest:
continue
name = extra.get("name", "")
+19
View File
@@ -207,6 +207,7 @@ def _build_readme(
save_folder: str = "",
save_where: str = "",
save_by_system: bool = False,
case_twins: list[tuple[str, str]] | None = None,
) -> str:
"""Build a personalized step-by-step README for each platform pack.
@@ -216,6 +217,10 @@ def _build_readme(
frontend's save directory, empty when the pack holds none, and
``save_where`` that directory's default place (save_place).
``save_by_system`` is set when the launcher hands each system a folder.
``case_twins`` pairs each spelling the pack leaves out, a case twin of a
packed file, with the one it holds: given for a platform that also runs
where names keep their case (EmuDeck and RetroArch on Linux), where the
core asking for the other spelling finds nothing.
"""
narrowings = narrowings or []
sep = "=" * 50
@@ -245,6 +250,20 @@ def _build_readme(
+ (pack_manual(platform_name) or _generic_manual(bios_paths))
)
if case_twins:
listed = "".join(
f" {missing} (the pack has {held})\n" for missing, held in case_twins
)
guide += (
"NAMES THAT DIFFER ONLY BY CASE\n\n"
" Some cores open the same file under another spelling. This\n"
" pack holds one spelling of each, all Windows and macOS can\n"
" store in one folder. Linux, SteamOS and Android keep both:\n"
" there, run the installer, which adds these spellings, or\n"
" copy each file under the name below.\n\n"
f"{listed}\n"
)
if save_folder:
guide += _save_section(
platform_display, save_folder, save_where, save_by_system,
+106 -10
View File
@@ -55,6 +55,7 @@ import subprocess
import sys
import tempfile
import time
from collections.abc import Callable
from pathlib import Path
# The build scripts need CPython 3.9, which builds the same database,
@@ -562,6 +563,7 @@ def check_emulator_packs(
comparison holds them together. No archive is written and no file read.
"""
import generate_pack # noqa: PLC0415
import packresolve # noqa: PLC0415
import verify # noqa: PLC0415
from common import ( # noqa: PLC0415
ProfileSelectionError,
@@ -580,16 +582,23 @@ def check_emulator_packs(
def compare(names: list[str], standalone: bool, label: str, **narrowing: object) -> bool:
record = generate_pack.PackRecord()
with contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()):
generate_pack.generate_emulator_pack(
names, emulators_dir, db, "bios", "", standalone, zip_contents,
offline=True, platforms_dir=platforms_dir, record=record, **narrowing,
)
report = verify.verify_emulator(
names, emulators_dir, db, standalone, platforms_dir=platforms_dir,
**narrowing,
)
was_offline = packresolve._OFFLINE
# The pack is built offline: verify counts a release asset the cache
# lacks the same way, missing, as the narrowed-pack step does.
packresolve.set_offline(True)
try:
with contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()):
generate_pack.generate_emulator_pack(
names, emulators_dir, db, "bios", "", standalone, zip_contents,
offline=True, platforms_dir=platforms_dir, record=record, **narrowing,
)
report = verify.verify_emulator(
names, emulators_dir, db, standalone, platforms_dir=platforms_dir,
**narrowing,
)
finally:
packresolve.set_offline(was_offline)
lines = _set_difference_lines(
label, set(report["pack_destinations"]), set(record.files)
)
@@ -802,6 +811,92 @@ def check_narrowed_packs(
return all_ok
def split_differences(
full: object, parts: dict[str, object], same_bytes: Callable[[str, str], bool],
) -> list[str]:
"""Where the --split parts of a pack, together, are not the full pack.
A path in a part and not in the full pack, a path of the full pack in no
part, and a path a part fills with other bytes than the full pack.
"""
full_members = {m for m in full.members if not _PACK_NOTES.search(m)}
union = {
m for record in parts.values() for m in record.members
if not _PACK_NOTES.search(m)
}
lines = [f"in a part, not in the full pack: {m}" for m in sorted(union - full_members)]
lines += [f"in the full pack, in no part: {m}" for m in sorted(full_members - union)]
lines += sorted({
f"other bytes in {name}: {member}"
for name, record in parts.items()
for member, source in record.sources.items()
if member in full.sources and not same_bytes(source, full.sources[member])
})
return lines
def check_split_packs(
emulators_dir: str = "emulators", platforms_dir: str = "platforms",
db_path: str = "database.json", include_archived: bool = False,
) -> bool:
"""Build each platform's --split parts in memory and compare them with
the full pack: the parts must add up to it, path for path and byte for
byte. Each part is a pack of its own and used to settle its paths alone:
RetroDECK's parts carried bios/SGB1.sfc as a file and as a directory,
RetroBat's 21 openmsx/ twins of openMSX/ files. Offline, as every
comparison of this step.
"""
import generate_pack # noqa: PLC0415
import packresolve # noqa: PLC0415
from common import ( # noqa: PLC0415
build_zip_contents_index,
group_identical_platforms,
list_registered_platforms,
load_data_dir_registry,
load_database,
load_emulator_profiles,
)
print(f"\n--- {step_title(STEP_LABELS['consistency'])}, split packs ---")
db = load_database(db_path)
zip_contents = build_zip_contents_index(db)
profiles = load_emulator_profiles(emulators_dir)
registry = load_data_dir_registry(platforms_dir)
groups = group_identical_platforms(
list_registered_platforms(platforms_dir, include_archived=include_archived),
platforms_dir,
)
was_offline = packresolve._OFFLINE
packresolve.set_offline(True)
all_ok = True
try:
for _members, platform in groups:
args = dict(
emulators_dir=emulators_dir, zip_contents=zip_contents,
data_registry=registry, emu_profiles=profiles, offline=True,
)
full = generate_pack.PackRecord()
parts: dict[str, generate_pack.PackRecord] = {}
with contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()):
generate_pack.generate_pack(
platform, platforms_dir, db, "bios", "", record=full, **args,
)
generate_pack.generate_split_packs(
platform, platforms_dir, db, "bios", "", records=parts, **args,
)
lines = split_differences(full, parts, generate_pack._same_bytes)
if lines:
all_ok = False
print(f" {platform} --split: {len(lines)} differences")
print("\n".join(f" {line}" for line in lines[:20]))
finally:
packresolve.set_offline(was_offline)
print(f" {len(groups)} split platform packs compared, "
f"{'all equal' if all_ok else 'differences above'}")
return all_ok
class _Skipped:
"""A step that did not run. Truthy, so it never fails the pipeline, and
distinct, so the summary does not report it as work that was done."""
@@ -1365,6 +1460,7 @@ def main() -> None:
ok = check_narrowed_packs(
include_archived=args.include_archived, full=args.full_narrowings,
) and ok
ok = check_split_packs(include_archived=args.include_archived) and ok
print(
f"--- {step_title(STEP_LABELS['consistency'])}, counts and paths: "
f"{'OK' if ok else 'FAILED'} ---"
+36 -3
View File
@@ -17,11 +17,14 @@ Usage:
from __future__ import annotations
import argparse
import contextlib
import hashlib
import os
import sys
import tempfile
sys.path.insert(0, os.path.dirname(__file__))
from artifacts import set_plain_mode
from common import copy_file_atomic, load_database
@@ -56,6 +59,36 @@ def index_cache(cache_dir: str) -> dict[str, str]:
return index
def _copy_verified(source: str, path: str, sha1: str) -> bool:
"""Copy *source* to *path* only if the copy hashes to *sha1*.
The cache was hashed before the copy, and another run can replace a
cached name in between (a re-uploaded asset): the copy is what counts.
Whole or absent: a scan running beside a plain copy hashed the truncated
file and published that in the database.
"""
handle, scratch = tempfile.mkstemp(
dir=os.path.dirname(os.path.abspath(path)),
prefix=f".{os.path.basename(path)}.", suffix=".tmp",
)
os.close(handle)
try:
copy_file_atomic(source, scratch)
digest = hashlib.sha1()
with open(scratch, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
digest.update(chunk)
if digest.hexdigest() != sha1:
print(f"Changed in the cache while restoring: {source}", file=sys.stderr)
return False
set_plain_mode(scratch)
os.replace(scratch, path)
return True
finally:
with contextlib.suppress(FileNotFoundError):
os.unlink(scratch)
def restore(
cache_dir: str, db_path: str, gitignore: str
) -> tuple[int, list[str]]:
@@ -79,9 +112,9 @@ def restore(
unsatisfied.append(path)
continue
os.makedirs(os.path.dirname(path), exist_ok=True)
# Whole or absent: a scan running beside a plain copy hashed the
# truncated file and published that in the database.
copy_file_atomic(source, path)
if not _copy_verified(source, path, sha1):
unsatisfied.append(path)
continue
print(f"Restored: {path}")
restored += 1
print(f"Total: {restored} files restored")
+7 -5
View File
@@ -524,6 +524,7 @@ def pack_overrides(
data_dir_registry: dict | None = None,
target_cores: set[str] | None = None,
source: str = "full",
choices: dict[str, list[tuple[str, str | None]]] | None = None,
) -> dict[str, str]:
"""Full pack destination -> the file a platform pack serves there instead.
@@ -532,7 +533,8 @@ def pack_overrides(
existence mode the frontend never reads the bytes, so serving the
emulator's file satisfies both. The ZIP builder, the install manifest and
verify all read this: two of them deciding alone gave the one-line
installer different bytes than the ZIP.
installer different bytes than the ZIP. ``choices`` is slot_choices
for the same arguments when the caller already holds it.
"""
# A baseline-only pack serves what the platform declares, and a target's
# pack answers to the cores that hardware builds: DuckStation does not
@@ -541,10 +543,10 @@ def pack_overrides(
return {}
cores = set(resolve_platform_cores(config, profiles, target_cores=target_cores))
overrides: dict[str, str] = {}
for destination, choices in slot_choices(
config, profiles, db, zip_contents, data_dir_registry
).items():
path = next((path for emulator, path in choices if emulator in cores), None)
if choices is None:
choices = slot_choices(config, profiles, db, zip_contents, data_dir_registry)
for destination, ordered in choices.items():
path = next((path for emulator, path in ordered if emulator in cores), None)
if path:
overrides[destination] = path
return overrides
+79 -20
View File
@@ -26,11 +26,12 @@ from __future__ import annotations
import argparse
import contextlib
import functools
import json
import os
import sys
from collections import Counter
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from pathlib import Path
import slots
@@ -38,6 +39,8 @@ import slots
sys.path.insert(0, os.path.dirname(__file__))
from hashing import parse_md5_list
from common import (
_foreign_tree,
_owner_tokens,
platform_arg,
target_listing_line,
suggest_names,
@@ -419,14 +422,25 @@ def _build_expected(file_entry: dict, checks: list[str]) -> dict:
return expected
def _in_foreign_tree(db: dict, owners: set[str], sha1: str) -> bool:
"""Whether the collection keeps this sha1 in another emulator's tree."""
return _foreign_tree(db.get("files", {}).get(sha1, {}).get("path", ""), owners)
def _name_in_index(
name: str,
by_name: dict,
by_path_suffix: dict | None = None,
data_names: set[str] | None = None,
by_name_lower: dict[str, str] | None = None,
foreign: Callable[[str], bool] | None = None,
) -> bool:
"""Check if a name is resolvable in the database indexes or data directories."""
"""Check if a name is resolvable in the database indexes or data directories.
*foreign* tells a copy in another emulator's tree from the sha1 of a
name: matched in another casing, such a copy is not the asker's, as in
resolve_local_file's case-insensitive step.
"""
# Strip trailing slash for directory-type entries (e.g. nestopia/samples/foo/)
name = name.rstrip("/")
if name in by_name:
@@ -435,11 +449,11 @@ def _name_in_index(
if basename != name and basename in by_name:
return True
if by_name_lower:
key = name.lower()
if key in by_name_lower:
return True
if basename != name and basename.lower() in by_name_lower:
return True
for key in {name.lower(), basename.lower()}:
if key in by_name_lower and not (
foreign and all(foreign(s) for s in by_name[by_name_lower[key]])
):
return True
if by_path_suffix and name in by_path_suffix:
return True
if data_names:
@@ -667,23 +681,30 @@ def find_undeclared_files(
# Resolution: name, then path basename
# storage: release names where the bytes are served from, not
# that the collection holds them: resolved like any file.
# Resolved as the asker, as the builder's ExtraResolver does:
# another casing of the name in another emulator's tree is
# not this emulator's file (dosbox-x's et4000.bin and pcem's
# et4000.BIN).
owned = {**f, "source_profile": emu_name}
foreign = functools.partial(_in_foreign_tree, db, _owner_tokens(owned))
if shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
# The entry states what its content should be and the builder
# drops a copy that contradicts it, so content decides here
# too. A name match would not do: generic names collide across
# systems, and answering yes on one describes a pack that will
# not contain the file.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
in_repo = _lp is not None and _st != "hash_mismatch"
else:
in_repo = _name_in_index(
fname, by_name, by_path_suffix, data_names, by_name_lower,
foreign,
)
if not in_repo and dest != fname:
path_base = dest.rsplit("/", 1)[-1]
in_repo = _name_in_index(
path_base, by_name, by_path_suffix, data_names,
by_name_lower,
by_name_lower, foreign,
)
if in_repo and not any(
f.get(h) for h in ("sha1", "md5", "crc32", "sha256")
@@ -693,7 +714,7 @@ def find_undeclared_files(
# rejects them all. NanoBoyAdvance's 16 KB bios.bin met
# only PCem's and QEMU's PC BIOS images of that name.
# The fallback below then decides as for any mismatch.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
if _st == "hash_mismatch":
in_repo = False
if not in_repo:
@@ -707,7 +728,7 @@ def find_undeclared_files(
# frontend would reject it, the builder leaves it out, and
# calling it available here would describe a pack that
# does not contain it.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
in_repo = _st != "not_found" and _lp is not None
if in_repo and _st == "hash_mismatch" and shipped_on_mismatch:
in_repo = False
@@ -717,7 +738,7 @@ def find_undeclared_files(
# held for it, the rule the builder's lookup applies
# (packextras.ExtraResolver). The Apple IIGS' 128 KB
# ROM is not Basilisk II's ROM.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
if core_size_refusal(
f, _lp, same_slot_entries(profile.get("files"), fname, dest)
):
@@ -1575,6 +1596,13 @@ def verify_platform(
"exclusion_notes": exclusions,
"region_fallbacks": sorted(region_fallbacks),
**({"left_out": left_out} if no_game_data else {}),
# Every tree of the registry answers a declaration by its tail, so
# one that was never refreshed leaves its files missing here while
# an online pack build refreshes it and packs them.
"data_dir_notices": sorted(
key for key, entry in (data_dir_registry or {}).items()
if not os.path.isdir(entry.get("local_cache", ""))
),
"details": details,
"ground_truth_coverage": {
"with_validation": with_validation,
@@ -1962,6 +1990,13 @@ def print_platform_result(
# them missing, as an offline build leaves them out.
shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "")
print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}")
uncached = result.get("data_dir_notices") or []
if uncached and result.get("status_counts", {}).get(Status.MISSING):
shown = ", ".join(uncached[:10]) + (f" (+{len(uncached) - 10})" if len(uncached) > 10 else "")
print(
f" Data directories not cached ({len(uncached)}), their files count missing "
f"until refresh_data_dirs.py runs (an online pack build refreshes them): {shown}"
)
for line in result.get("left_out") or []:
print(f" {line}")
fallbacks = result.get("region_fallbacks") or []
@@ -2068,7 +2103,7 @@ def verify_emulator(
it and every member it carries are optional, as the builder leaves them.
"""
from packextras import expand_agnostic_entries
from packresolve import resolve_file
from packresolve import pending_release_asset, resolve_file, run_is_offline
from packpaths import (
_archive_destination,
_register_path,
@@ -2094,6 +2129,16 @@ def verify_emulator(
validation_index = _build_validation_index(selected_profiles)
data_registry = load_data_dir_registry(platforms_dir)
def pending(entry: dict, local_path: str | None, status: str) -> tuple[str | None, str]:
# Read from disk and the cache, as _resolve_for_platform does: a
# release asset the cache lacks is fetched by an online build
# (generate_emulator_pack resolves with offline=offline).
if status in ("not_found", "hash_mismatch") and not run_is_offline():
asset = pending_release_asset(entry, db)
if asset:
return asset, RELEASE_ASSET_PENDING
return local_path, status
details = []
file_status: dict[str, str] = {}
file_severity: dict[str, str] = {}
@@ -2233,14 +2278,15 @@ def verify_emulator(
archive = file_entry.get("archive")
if archive and archive not in seen_archives:
seen_archives.add(archive)
local_path, resolve_status = resolve_file(
archive_entry(archive, files),
own_entry = archive_entry(archive, files)
local_path, resolve_status = pending(own_entry, *resolve_file(
own_entry,
db,
DEFAULT_BIOS_DIR,
zip_contents,
data_dir_registry=data_registry,
offline=True,
)
))
required = any(
f.get("archive") == archive and f.get("required", True)
for f in files
@@ -2264,6 +2310,8 @@ def verify_emulator(
"required": required,
"path": local_path,
}
if resolve_status == RELEASE_ASSET_PENDING:
result["release_asset"] = local_path
else:
result = {
"name": archive,
@@ -2324,7 +2372,7 @@ def verify_emulator(
dest_hint = file_entry.get("path", "")
# The emulator pack's resolution, release assets included.
local_path, resolve_status = resolve_file(
local_path, resolve_status = pending(file_entry, *resolve_file(
{**with_archive_members(file_entry, files), "source_profile": emu_name},
db,
DEFAULT_BIOS_DIR,
@@ -2332,7 +2380,7 @@ def verify_emulator(
dest_hint=dest_hint,
data_dir_registry=data_registry,
offline=True,
)
))
if resolve_status in ("not_found", "user_provided", "external"):
local_path = None
name = file_entry.get("name", "")
@@ -2342,6 +2390,15 @@ def verify_emulator(
stored = _storage_verdict(file_entry, emulator_pack=True)
if stored is not None:
result = stored
elif resolve_status == RELEASE_ASSET_PENDING:
result = {
"name": name,
"status": Status.OK,
"required": required,
"path": local_path,
"release_asset": local_path,
"reason": "release asset, fetched when the pack is built",
}
elif not local_path:
result = {"name": name, "status": Status.MISSING, "required": required}
# An entry the profile documents as unsourceable is absent by
@@ -2592,6 +2649,10 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
if unsourceable:
parts.append(f"{unsourceable} unsourceable")
print(f"{label}: {', '.join(parts)} [{mode}]")
fetched = sorted({d["name"] for d in result["details"] if d.get("release_asset")})
if fetched:
shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "")
print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}")
seen = set()
for d in result["details"]:
@@ -2722,8 +2783,6 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
"--destinations describes the full pack and is incompatible with --system"),
(args.destinations and not (args.platform or args.all),
"--destinations requires --platform or --all"),
(args.offline and not (args.platform or args.all),
"--offline requires --platform or --all"),
# Game data is categorized against a platform's list and its cores.
(getattr(args, "no_game_data", False) and not (args.platform or args.all),
"--no-game-data requires --platform or --all"),