mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-06 10:03:49 -06:00
chore: merge pack and verify fixes
This commit is contained in:
commit
d0c65dca22
37 files changed
+3267
-142
No files matched your search
@@ -3036,12 +3036,16 @@ SYSTEM_ALIASES: dict[str, str] = {
|
||||
}
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _norm_system_id(sid: str) -> str:
|
||||
"""Normalize system ID for cross-platform matching.
|
||||
|
||||
Resolves known aliases, then strips manufacturer prefixes and separators
|
||||
so that platform-specific IDs (e.g., "xbox", "nintendo-wiiu") match
|
||||
profile IDs (e.g., "microsoft-xbox", "nintendo-wii-u").
|
||||
|
||||
A function of the string and two constant tables: a manifest run asks it
|
||||
1.7 million times for about 1,100 ids.
|
||||
"""
|
||||
s = sid.lower().replace("_", "-")
|
||||
s = SYSTEM_ALIASES.get(s, s)
|
||||
|
||||
@@ -26,7 +26,9 @@ sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import (
|
||||
platform_arg,
|
||||
ProfileSelectionError,
|
||||
_foreign_tree,
|
||||
_norm_system_id,
|
||||
_owner_tokens,
|
||||
archive_entry,
|
||||
directory_members,
|
||||
expand_directory_entries,
|
||||
@@ -215,8 +217,14 @@ def _resolve_source(
|
||||
none of the hashes the entry declares: a name never wins over a declared
|
||||
hash, and zquestclassic's ending.mid read as held because Hydra Castle
|
||||
Labyrinth has another file of that name.
|
||||
|
||||
Another casing of the name in another emulator's tree proves nothing, as
|
||||
in resolve_local_file's case-insensitive step: dosbox-x's et4000.bin read
|
||||
as held through pcem's et4000.BIN, which the pack and the installer
|
||||
refuse it.
|
||||
"""
|
||||
declared = _declared_content(file_entry or {})
|
||||
owners = _owner_tokens(file_entry or {})
|
||||
|
||||
def _can_be(sha1s: list | str) -> bool:
|
||||
if not file_entry or db_files is None:
|
||||
@@ -233,6 +241,14 @@ def _resolve_source(
|
||||
"""Whether the files indexed under *name* can be this entry."""
|
||||
return _can_be(by_name.get(name) or [])
|
||||
|
||||
def _folded_hit(name: str) -> bool:
|
||||
"""_name_hit for a name matched in another casing."""
|
||||
return _can_be([
|
||||
sha1 for sha1 in by_name.get(name) or []
|
||||
if db_files is None
|
||||
or not _foreign_tree(db_files.get(sha1, {}).get("path", ""), owners)
|
||||
])
|
||||
|
||||
# bios/ via database.json by_name
|
||||
if fname in by_name and _name_hit(fname):
|
||||
return "bios"
|
||||
@@ -241,10 +257,10 @@ def _resolve_source(
|
||||
if basename and basename in by_name and _name_hit(basename):
|
||||
return "bios"
|
||||
key = fname.lower()
|
||||
if key in by_name_lower and _name_hit(by_name_lower[key]):
|
||||
if key in by_name_lower and _folded_hit(by_name_lower[key]):
|
||||
return "bios"
|
||||
if basename:
|
||||
if basename.lower() in by_name_lower and _name_hit(
|
||||
if basename.lower() in by_name_lower and _folded_hit(
|
||||
by_name_lower[basename.lower()]
|
||||
):
|
||||
return "bios"
|
||||
@@ -257,7 +273,7 @@ def _resolve_source(
|
||||
if canonical and canonical != fname:
|
||||
if canonical in by_name and _name_hit(canonical):
|
||||
return "bios"
|
||||
if canonical.lower() in by_name_lower and _name_hit(
|
||||
if canonical.lower() in by_name_lower and _folded_hit(
|
||||
by_name_lower[canonical.lower()]
|
||||
):
|
||||
return "bios"
|
||||
@@ -901,7 +917,9 @@ def _cross_reference_profile(
|
||||
source = next(
|
||||
(
|
||||
found for version in versions[slot]
|
||||
if (found := entry_source(version, index)) is not None
|
||||
if (found := entry_source(
|
||||
{**version, "source_profile": emu_name}, index,
|
||||
)) is not None
|
||||
),
|
||||
"missing",
|
||||
)
|
||||
|
||||
+146
-49
@@ -26,6 +26,7 @@ import argparse
|
||||
import contextlib
|
||||
import functools
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -158,6 +159,9 @@ class PackRecord:
|
||||
self.members: list[str] = []
|
||||
self.files: list[str] = []
|
||||
self.trees: list[str] = []
|
||||
# The collection file each copied member comes from: a split part
|
||||
# writes a file only where the full pack writes the same bytes.
|
||||
self.sources: dict[str, str] = {}
|
||||
|
||||
|
||||
def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> None:
|
||||
@@ -173,6 +177,7 @@ def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> Non
|
||||
"""
|
||||
if isinstance(zf, PackRecord):
|
||||
zf.members.append(arcname)
|
||||
zf.sources[arcname] = source_path
|
||||
return
|
||||
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
|
||||
info.compress_type = _member_compression(arcname)
|
||||
@@ -603,6 +608,17 @@ def _pack_data_directories(
|
||||
return added
|
||||
|
||||
|
||||
def _case_twin_pairs(
|
||||
twins: list[str], seen_destinations: set[str], base_dest: str, flatten: bool,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Each spelling left out, with the spelling the pack holds."""
|
||||
held = {
|
||||
_flat(dest, base_dest, flatten).lower(): _flat(dest, base_dest, flatten)
|
||||
for dest in seen_destinations
|
||||
}
|
||||
return sorted({(twin, held[twin.lower()]) for twin in twins if twin.lower() in held})
|
||||
|
||||
|
||||
def _report_pack_counts(
|
||||
zip_path: str,
|
||||
file_status: dict,
|
||||
@@ -806,6 +822,7 @@ def generate_pack(
|
||||
pack_name: str | None = None,
|
||||
no_game_data: bool = False,
|
||||
record: PackRecord | None = None,
|
||||
within: dict[str, str] | None = None,
|
||||
) -> str | None:
|
||||
"""Generate a ZIP pack for a platform.
|
||||
|
||||
@@ -819,6 +836,12 @@ def generate_pack(
|
||||
``record`` builds the pack in memory (PackRecord): no archive is written,
|
||||
no external file fetched, and the paths are kept on it.
|
||||
|
||||
``within`` is the full pack's PackRecord.sources, given to a split part:
|
||||
a declared file or core extra is written only where the full pack writes
|
||||
the same bytes. A part sees neither the other systems' declarations nor
|
||||
their extras, so a path the full pack settles (a file/directory conflict,
|
||||
a case twin, a path another system's file takes) was written again.
|
||||
|
||||
Returns the path to the generated ZIP, or None on failure.
|
||||
"""
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
@@ -867,6 +890,8 @@ def generate_pack(
|
||||
user_provided = []
|
||||
seen_destinations: set[str] = set()
|
||||
seen_lower: set[str] = set() # only used when case_insensitive=True
|
||||
# Spellings left out as case twins of a packed file, for the README.
|
||||
case_twins: list[str] = []
|
||||
seen_parents: set[str] = (
|
||||
set()
|
||||
) # parent dirs of added files (path conflict detection)
|
||||
@@ -1135,7 +1160,10 @@ def generate_pack(
|
||||
config.get("hash_rule", ""),
|
||||
)
|
||||
if refusal:
|
||||
file_status[dedup_key] = "excluded"
|
||||
# An earlier declaration of this path shipped a file this
|
||||
# one refuses: it is in the pack, untested as verify
|
||||
# counts it, not excluded.
|
||||
file_status[dedup_key] = "untested" if already_packed else "excluded"
|
||||
file_reasons[dedup_key] = refusal
|
||||
continue
|
||||
if not hash_mismatch_excludes_file(verification_mode) and (
|
||||
@@ -1170,7 +1198,17 @@ def generate_pack(
|
||||
)
|
||||
|
||||
if already_packed:
|
||||
if dedup_key not in seen_destinations:
|
||||
case_twins.append(_flat(full_dest, base_dest, flatten))
|
||||
continue
|
||||
if within is not None and not _same_bytes(
|
||||
local_path, within.get(_flat(full_dest, base_dest, flatten))
|
||||
):
|
||||
# The full pack holds another system's file here.
|
||||
continue
|
||||
if file_status.get(dedup_key) == "excluded":
|
||||
# An earlier declaration refused the file this one ships.
|
||||
file_status[dedup_key] = "untested"
|
||||
seen_destinations.add(dedup_key)
|
||||
_register_path(dedup_key, seen_destinations, seen_parents)
|
||||
if case_insensitive:
|
||||
@@ -1227,7 +1265,7 @@ def generate_pack(
|
||||
if required_only and fe.get("required") is False:
|
||||
continue
|
||||
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
|
||||
if region_drops and dest in region_drops:
|
||||
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
|
||||
continue
|
||||
if not dest:
|
||||
continue
|
||||
@@ -1244,10 +1282,13 @@ def generate_pack(
|
||||
# Taken, a case twin on a case-insensitive filesystem, or a
|
||||
# file/directory conflict (SGB1.sfc vs SGB1.sfc/): verify replays
|
||||
# the same rule (packextras.hidden_extras).
|
||||
if placement_block(
|
||||
block = placement_block(
|
||||
full_dest, seen_destinations, seen_lower, seen_parents,
|
||||
case_insensitive,
|
||||
):
|
||||
)
|
||||
if block:
|
||||
if block == "case_twin" and extra_resolver.resolve(fe, dest)[0]:
|
||||
case_twins.append(_flat(full_dest, base_dest, flatten))
|
||||
continue
|
||||
|
||||
# The emulator's own check picks the copy, as for a declared file
|
||||
@@ -1275,6 +1316,8 @@ def generate_pack(
|
||||
)
|
||||
|
||||
flat_dest = _flat(full_dest, base_dest, flatten)
|
||||
if within is not None and not _same_bytes(local_path, within.get(flat_dest)):
|
||||
continue
|
||||
if local_path.endswith(".zip"):
|
||||
_add_zip_to_pack(local_path, flat_dest, zf, fe)
|
||||
else:
|
||||
@@ -1339,6 +1382,37 @@ def generate_pack(
|
||||
if _registry_path.exists():
|
||||
with open(_registry_path) as _rf:
|
||||
_pack_registry = (yaml_load(_rf) or {}).get("platforms", {})
|
||||
keeps_case = case_insensitive and any(
|
||||
d.get("os") not in CASE_FOLDING_OS
|
||||
for d in (_pack_registry.get(platform_name, {}).get("install", {}) or {})
|
||||
.get("detect", [])
|
||||
)
|
||||
if keeps_case and source != "platform" and precomputed_extras is None:
|
||||
# The spellings a folded platform counts as declared never reach
|
||||
# the extras: collected as the filesystem that keeps case sees
|
||||
# them, as the manifest's case_variants are (Mesen-CE's bios.col
|
||||
# beside the declared BIOS.col).
|
||||
unfolded = _collect_emulator_extras(
|
||||
{**config, "case_insensitive_fs": False}, emulators_dir, db, set(),
|
||||
base_dest, emu_profiles, target_cores=target_cores,
|
||||
include_all=(source == "truth"),
|
||||
)
|
||||
if system_filter:
|
||||
unfolded = _extras_for_systems(unfolded, system_filter)
|
||||
prefix = _detect_extras_prefix(config, base_dest)
|
||||
for fe in unfolded:
|
||||
if required_only and fe.get("required") is False:
|
||||
continue
|
||||
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
|
||||
continue
|
||||
full = extra_full_destination(fe, prefix)
|
||||
if (
|
||||
full and full not in seen_destinations and full.lower() in seen_lower
|
||||
and extra_resolver.resolve(
|
||||
fe, sanitize_pack_path(fe.get("destination", fe["name"]))
|
||||
)[0]
|
||||
):
|
||||
case_twins.append(_flat(full, base_dest, flatten))
|
||||
readme_text = _build_readme(
|
||||
platform_name, platform_display, base_dest, total_files, num_systems,
|
||||
source=source,
|
||||
@@ -1364,6 +1438,10 @@ def generate_pack(
|
||||
) if save_count else ""
|
||||
),
|
||||
save_by_system=bool(save_count and save_layout(config).by_system),
|
||||
case_twins=(
|
||||
_case_twin_pairs(case_twins, seen_destinations, base_dest, flatten)
|
||||
if keeps_case else []
|
||||
),
|
||||
)
|
||||
_write_generated_member(zf, "README.txt", readme_text)
|
||||
|
||||
@@ -1408,6 +1486,7 @@ def _extract_zip_to_archive(
|
||||
for name in map(sanitize_pack_path, src.namelist())
|
||||
if name and not name.endswith("/")
|
||||
)
|
||||
target_zf.sources[dest_prefix] = source_zip
|
||||
return
|
||||
for info in src.infolist():
|
||||
if info.is_dir():
|
||||
@@ -1441,6 +1520,7 @@ def _add_zip_to_pack(
|
||||
"""
|
||||
if isinstance(target_zf, PackRecord):
|
||||
target_zf.members.append(dest_path)
|
||||
target_zf.sources[dest_path] = source_zip
|
||||
return
|
||||
_check_zip_members(source_zip)
|
||||
_add_pack_member(target_zf, source_zip, dest_path)
|
||||
@@ -2069,8 +2149,14 @@ def generate_split_packs(
|
||||
one_per_slot: bool = False,
|
||||
offline: bool | None = None,
|
||||
no_game_data: bool = False,
|
||||
records: dict[str, PackRecord] | None = None,
|
||||
) -> list[str]:
|
||||
"""Generate split packs (one ZIP per system or manufacturer)."""
|
||||
"""Generate split packs (one ZIP per system or manufacturer).
|
||||
|
||||
The parts add up to the full pack: each is built against the full pack's
|
||||
own placement (``within``). ``records`` builds the parts in memory, keyed
|
||||
by part name, with no archive written.
|
||||
"""
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
# The split directory carries the same tags as the packs inside it, or two
|
||||
# differently narrowed runs write into one another.
|
||||
@@ -2093,10 +2179,36 @@ def generate_split_packs(
|
||||
f"{_platform_pack_stem([platform_name], platform_name, platforms_dir)}{split_tags}"
|
||||
)
|
||||
split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split")
|
||||
os.makedirs(split_dir, exist_ok=True)
|
||||
if records is None:
|
||||
os.makedirs(split_dir, exist_ok=True)
|
||||
|
||||
if emu_profiles is None:
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
narrowing = {
|
||||
"emulators_dir": emulators_dir, "zip_contents": zip_contents,
|
||||
"data_registry": data_registry, "emu_profiles": emu_profiles,
|
||||
"target_cores": target_cores, "required_only": required_only,
|
||||
"source": source, "regions": regions, "target_name": target_name,
|
||||
"one_per_slot": one_per_slot, "offline": offline,
|
||||
"no_game_data": no_game_data,
|
||||
}
|
||||
full = PackRecord()
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
generate_pack(
|
||||
platform_name, platforms_dir, db, bios_dir, split_dir,
|
||||
record=full, **narrowing,
|
||||
)
|
||||
|
||||
def part(pack_name: str, **scope: object) -> str | None:
|
||||
record = PackRecord() if records is not None else None
|
||||
zip_path = generate_pack(
|
||||
platform_name, platforms_dir, db, bios_dir, split_dir,
|
||||
pack_name=pack_name, record=record, within=full.sources,
|
||||
**scope, **narrowing,
|
||||
)
|
||||
if record is not None and zip_path:
|
||||
records[pack_name] = record
|
||||
return zip_path
|
||||
# The systems a core of the target runs: a part per declared system sent
|
||||
# each one the target drops to a refusal naming it as unknown.
|
||||
systems = platform_scope_systems(config, emu_profiles, target_cores)
|
||||
@@ -2130,27 +2242,9 @@ def generate_split_packs(
|
||||
results = []
|
||||
for group_name, group_system_ids in sorted(groups.items()):
|
||||
group_extras = _extras_for_systems(all_extras, group_system_ids)
|
||||
zip_path = generate_pack(
|
||||
platform_name,
|
||||
platforms_dir,
|
||||
db,
|
||||
bios_dir,
|
||||
split_dir,
|
||||
emulators_dir=emulators_dir,
|
||||
zip_contents=zip_contents,
|
||||
data_registry=data_registry,
|
||||
emu_profiles=emu_profiles,
|
||||
target_cores=target_cores,
|
||||
required_only=required_only,
|
||||
system_filter=group_system_ids,
|
||||
precomputed_extras=group_extras,
|
||||
source=source,
|
||||
regions=regions,
|
||||
target_name=target_name,
|
||||
one_per_slot=one_per_slot,
|
||||
offline=offline,
|
||||
pack_name=f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip",
|
||||
no_game_data=no_game_data,
|
||||
zip_path = part(
|
||||
f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip",
|
||||
system_filter=group_system_ids, precomputed_extras=group_extras,
|
||||
)
|
||||
if zip_path:
|
||||
results.append(zip_path)
|
||||
@@ -2166,15 +2260,9 @@ def generate_split_packs(
|
||||
}
|
||||
undistributed = [fe for fe in all_extras if id(fe) not in distributed]
|
||||
if undistributed:
|
||||
zip_path = generate_pack(
|
||||
platform_name, platforms_dir, db, bios_dir, split_dir,
|
||||
emulators_dir=emulators_dir, zip_contents=zip_contents,
|
||||
data_registry=data_registry, emu_profiles=emu_profiles,
|
||||
target_cores=target_cores, required_only=required_only,
|
||||
precomputed_extras=undistributed, extras_only=True, source=source,
|
||||
regions=regions, target_name=target_name, one_per_slot=one_per_slot,
|
||||
offline=offline, pack_name=f"{part_prefix}_Other_Cores_BIOS_Pack.zip",
|
||||
no_game_data=no_game_data,
|
||||
zip_path = part(
|
||||
f"{part_prefix}_Other_Cores_BIOS_Pack.zip",
|
||||
precomputed_extras=undistributed, extras_only=True,
|
||||
)
|
||||
if zip_path:
|
||||
results.append(zip_path)
|
||||
@@ -4133,7 +4221,8 @@ def _manifest_core_entries(
|
||||
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
|
||||
if not dest:
|
||||
continue
|
||||
if region_drops and dest in region_drops:
|
||||
region_key = extra_region_key(fe, declared_root(config))
|
||||
if region_drops and region_key in region_drops:
|
||||
continue
|
||||
full_dest = extra_full_destination(fe, extras_pfx)
|
||||
if not full_dest:
|
||||
@@ -4207,7 +4296,7 @@ def _manifest_core_entries(
|
||||
entry, file_size = _manifest_entry(
|
||||
local_path, manifest_dest, [source_emu] if source_emu else [],
|
||||
db, repo_root,
|
||||
selection.fields(dest, fe.get("name", "")),
|
||||
selection.fields(region_key, fe.get("name", "")),
|
||||
)
|
||||
if entry is not None and save_root:
|
||||
entry.update(save_entry_fields(fe))
|
||||
@@ -4321,7 +4410,8 @@ def _manifest_selection(
|
||||
else set()
|
||||
)
|
||||
return drops, SelectionIndex.build(
|
||||
region_groups, region_index, pack_systems, extras, required_only
|
||||
region_groups, region_index, pack_systems, extras, required_only,
|
||||
extras_prefix=declared_root(config),
|
||||
), held
|
||||
|
||||
|
||||
@@ -4346,7 +4436,9 @@ def _record_absent(
|
||||
if required_only and fe.get("required") is False:
|
||||
continue
|
||||
dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
|
||||
if not dest or (region_drops and dest in region_drops):
|
||||
if not dest or (
|
||||
region_drops and extra_region_key(fe, declared_root(config)) in region_drops
|
||||
):
|
||||
continue
|
||||
full_dest = extra_full_destination(fe, extras_pfx)
|
||||
if not full_dest:
|
||||
@@ -4517,16 +4609,18 @@ def generate_manifest(
|
||||
# (cache file, size). Settled once the trees are walked.
|
||||
cache_held: dict[str, tuple[str, int]] = {}
|
||||
rename_notes: set[str] = set()
|
||||
# The full manifest records every core's answer at a contested slot, so a
|
||||
# target narrowed from it gets what that target's own pack serves.
|
||||
whole = target_cores is None and source != "platform" and bool(emu_profiles)
|
||||
slot_choices = (
|
||||
slots.slot_choices(config, emu_profiles, db, zip_contents, data_registry)
|
||||
if whole else {}
|
||||
)
|
||||
# Derived from the same choices: computed twice, it was 8% of a run.
|
||||
slot_overrides = slots.pack_overrides(
|
||||
config, emu_profiles, db, zip_contents, data_registry,
|
||||
target_cores=target_cores, source=source,
|
||||
)
|
||||
# The full manifest records every core's answer at a contested slot, so a
|
||||
# target narrowed from it gets what that target's own pack serves.
|
||||
slot_choices = (
|
||||
slots.slot_choices(config, emu_profiles, db, zip_contents, data_registry)
|
||||
if target_cores is None and source != "platform" and emu_profiles
|
||||
else {}
|
||||
choices=slot_choices if whole else None,
|
||||
)
|
||||
|
||||
def manifest_destination(full_destination: str) -> str:
|
||||
@@ -4541,8 +4635,10 @@ def generate_manifest(
|
||||
reason: str,
|
||||
cores: list[str] | None,
|
||||
) -> None:
|
||||
bare = sanitize_pack_path(
|
||||
file_entry.get("destination", file_entry.get("name", ""))
|
||||
# A core extra (cores given) is keyed as the region pass keys it.
|
||||
bare = (
|
||||
sanitize_pack_path(file_entry.get("destination", file_entry.get("name", "")))
|
||||
if cores is None else extra_region_key(file_entry, declared_root(config))
|
||||
)
|
||||
save_root = file_entry.get("root") == "save"
|
||||
omitted_by_destination[full_destination] = {
|
||||
@@ -5443,6 +5539,7 @@ from packpaths import ( # noqa: E402,F401
|
||||
from packextras import ( # noqa: E402,F401
|
||||
ExtraResolver,
|
||||
extra_full_destination,
|
||||
extra_region_key,
|
||||
prefixed_destination,
|
||||
save_entry_fields,
|
||||
save_manifest_dest,
|
||||
|
||||
@@ -9478,7 +9478,7 @@ def _hash_spans(value: str | list, copy: bool = False) -> str:
|
||||
|
||||
|
||||
def _availability_check(
|
||||
db: dict, data_names: set[str] | None
|
||||
db: dict, data_names: set[str] | None, owner: str = "",
|
||||
) -> Callable[[dict], bool]:
|
||||
"""A predicate answering whether the collection holds a file.
|
||||
|
||||
@@ -9509,7 +9509,7 @@ def _availability_check(
|
||||
# under the name belongs to another game or release.
|
||||
if not f.get("name") or f.get("unsourceable"):
|
||||
return False
|
||||
return entry_source(f, index) is not None
|
||||
return entry_source({**f, "source_profile": owner} if owner else f, index) is not None
|
||||
|
||||
return _file_available
|
||||
|
||||
@@ -10242,7 +10242,7 @@ def emulator_pages(
|
||||
lines.append("")
|
||||
lines.extend(markup.aside("info", "Why no files", [_escape_tags(str(exclusion))])[:-1])
|
||||
else:
|
||||
_file_available = _availability_check(db, data_names)
|
||||
_file_available = _availability_check(db, data_names, name)
|
||||
|
||||
lines.extend(["## Files", ""])
|
||||
lines.extend(_emulator_file_summary(files))
|
||||
|
||||
+73
-11
@@ -12,6 +12,7 @@ import os
|
||||
import sys
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -26,6 +27,13 @@ from hashing import compute_hashes
|
||||
LARGE_FILES_RELEASE = "large-files"
|
||||
LARGE_FILES_REPO = "Abdess/retrobios"
|
||||
LARGE_FILES_CACHE = ".cache/large"
|
||||
# Verified revisions, hard-linked as <sha1>/<asset> beside the cache. A name
|
||||
# in the cache is replaced when the release serves new bytes under it, so a
|
||||
# path handed out by name could hold other bytes by the time the caller read
|
||||
# them. Nothing ever writes other bytes under a sha1. The asset name stays
|
||||
# last: callers read the extension (an archive is packed as one).
|
||||
VERIFIED_STORE = ".by-sha1"
|
||||
ORPHAN_GRACE = 24 * 3600
|
||||
GITIGNORE = Path(__file__).resolve().parent.parent / ".gitignore"
|
||||
|
||||
_UNSAFE_ASSET_CHARS = re.compile(r"[^A-Za-z0-9._-]+")
|
||||
@@ -169,14 +177,14 @@ def _fetch_asset(
|
||||
|
||||
if os.path.isdir(cached):
|
||||
return None
|
||||
if os.path.exists(cached) and not (expected_sha1 or expected_md5):
|
||||
return cached
|
||||
if os.path.exists(cached):
|
||||
snapshot = _read_once(cached)
|
||||
if snapshot is None:
|
||||
pass
|
||||
elif _matches(snapshot[0], expected_sha1, expected_md5):
|
||||
return cached
|
||||
pinned = _pin_verified(cached, snapshot, dest_dir, name)
|
||||
if pinned:
|
||||
return pinned
|
||||
elif not _replaced_on_release(name, snapshot[0]["sha256"], snapshot[1], offline):
|
||||
# A copy of this asset that answers another hash: the caller
|
||||
# wants a different revision under the same name, and the
|
||||
@@ -220,17 +228,71 @@ def _fetch_asset(
|
||||
print(f" large file {candidate}: {exc}", file=sys.stderr)
|
||||
if not downloaded:
|
||||
return None
|
||||
if (expected_sha1 or expected_md5) and not _matches(
|
||||
compute_hashes(tmp_path), expected_sha1, expected_md5
|
||||
):
|
||||
snapshot = _read_once(tmp_path)
|
||||
if snapshot is None or not _matches(snapshot[0], expected_sha1, expected_md5):
|
||||
return None
|
||||
set_plain_mode(tmp_path)
|
||||
pinned = _pin_verified(tmp_path, snapshot, dest_dir, name)
|
||||
os.replace(tmp_path, cached)
|
||||
return cached
|
||||
_prune_verified(dest_dir)
|
||||
return pinned
|
||||
finally:
|
||||
_drop(tmp_path)
|
||||
|
||||
|
||||
def _pin_verified(
|
||||
path: str, snapshot: tuple[dict[str, str], int, int], dest_dir: str, name: str,
|
||||
) -> str | None:
|
||||
"""The verified revision at *path*, as <sha1>/<name> in VERIFIED_STORE.
|
||||
|
||||
*snapshot* is what _read_once read: the link is kept only if it holds
|
||||
that inode, so a copy another run promoted after the hash is never
|
||||
passed off as the one verified. None when the name moved in between.
|
||||
"""
|
||||
revision = os.path.join(dest_dir, VERIFIED_STORE, snapshot[0]["sha1"])
|
||||
os.makedirs(revision, exist_ok=True)
|
||||
final = os.path.join(revision, name)
|
||||
if os.path.isfile(final):
|
||||
return final
|
||||
scratch = os.path.join(revision, f".{os.getpid()}.{os.urandom(4).hex()}.tmp")
|
||||
try:
|
||||
os.link(path, scratch)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
try:
|
||||
if os.stat(scratch).st_ino != snapshot[2]:
|
||||
return None
|
||||
os.replace(scratch, final)
|
||||
return final
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(scratch)
|
||||
|
||||
|
||||
def _prune_verified(dest_dir: str) -> None:
|
||||
"""Drop the revisions no cache name has held for ORPHAN_GRACE seconds.
|
||||
|
||||
Losing its cache name changes the inode's ctime, so a revision another
|
||||
run pinned just before the release replaced it stays readable while that
|
||||
run packs it.
|
||||
"""
|
||||
store = os.path.join(dest_dir, VERIFIED_STORE)
|
||||
now = time.time()
|
||||
for revision in os.scandir(store):
|
||||
if not revision.is_dir():
|
||||
continue
|
||||
for entry in os.scandir(revision.path):
|
||||
if entry.name.endswith(".tmp"):
|
||||
continue
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
stat = entry.stat()
|
||||
if stat.st_nlink == 1 and now - stat.st_ctime > ORPHAN_GRACE:
|
||||
os.unlink(entry.path)
|
||||
# Another run may be linking a copy into it: left for the next prune.
|
||||
with contextlib.suppress(FileNotFoundError, OSError):
|
||||
os.rmdir(revision.path)
|
||||
|
||||
|
||||
def _asset_urls(name: str) -> list[tuple[str, str]]:
|
||||
"""Release URLs an asset may be published under, with their names.
|
||||
|
||||
@@ -252,9 +314,9 @@ def _asset_urls(name: str) -> list[tuple[str, str]]:
|
||||
]
|
||||
|
||||
|
||||
def _read_once(path: str) -> tuple[dict[str, str], int] | None:
|
||||
"""Digests and size of the file at *path*, all read from one file, or
|
||||
None when it is gone.
|
||||
def _read_once(path: str) -> tuple[dict[str, str], int, int] | None:
|
||||
"""Digests, size and inode of the file at *path*, all read from one
|
||||
file, or None when it is gone.
|
||||
|
||||
Every judgement on a cache copy reads this one snapshot. Hashed, then
|
||||
hashed again by path for the release check, a copy another run promoted
|
||||
@@ -271,7 +333,7 @@ def _read_once(path: str) -> tuple[dict[str, str], int] | None:
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if (before.st_ino, before.st_mtime_ns) == (after.st_ino, after.st_mtime_ns):
|
||||
return hashes, after.st_size
|
||||
return hashes, after.st_size, after.st_ino
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -19,7 +19,13 @@ from dataclasses import dataclass
|
||||
|
||||
import region as region_mod
|
||||
from common import _norm_system_id, sanitize_pack_path
|
||||
from packextras import _kept, _serving_system_ids, extra_region_groups, system_spellings
|
||||
from packextras import (
|
||||
_kept,
|
||||
_serving_system_ids,
|
||||
extra_region_groups,
|
||||
extra_region_key,
|
||||
system_spellings,
|
||||
)
|
||||
|
||||
|
||||
def _dest(entry: dict) -> str:
|
||||
@@ -44,6 +50,7 @@ class SelectionIndex:
|
||||
pack_systems: dict[str, dict],
|
||||
extras: list[dict],
|
||||
required_only: bool = False,
|
||||
extras_prefix: str = "",
|
||||
) -> SelectionIndex:
|
||||
by_norm: dict[str, set[str]] = {}
|
||||
for sys_id in pack_systems:
|
||||
@@ -60,7 +67,8 @@ class SelectionIndex:
|
||||
extra_groups: dict[str, set[str]] = {}
|
||||
spellings = system_spellings(pack_systems)
|
||||
for extra in extras:
|
||||
dest = _dest(extra)
|
||||
# Keyed as platform_region_groups keys it (extra_region_key).
|
||||
dest = extra_region_key(extra, extras_prefix)
|
||||
if not dest:
|
||||
continue
|
||||
extra_groups.setdefault(dest, set()).update(
|
||||
|
||||
+18
-2
@@ -62,6 +62,20 @@ def prefixed_destination(dest: str, prefix: str) -> str:
|
||||
return f"{prefix}/{dest}"
|
||||
|
||||
|
||||
def extra_region_key(fe: dict, prefix: str) -> str:
|
||||
"""The key a core extra is grouped, dropped and held under by --region.
|
||||
|
||||
Its destination with the BIOS directory the platform writes in front of
|
||||
its own (declared_root): RetroDECK declares bios/scph5501.bin, and
|
||||
Beetle PSX's scph5501.bin grouped bare was looked up under another key,
|
||||
kept by the pack while the declaration of the same file was dropped,
|
||||
and the installer, which records the path once, dropped it.
|
||||
"""
|
||||
return prefixed_destination(
|
||||
sanitize_pack_path(fe.get("destination", fe.get("name", ""))), prefix
|
||||
)
|
||||
|
||||
|
||||
def save_entry_fields(fe: dict) -> dict:
|
||||
"""What a manifest entry adds for a file read below the save directory.
|
||||
|
||||
@@ -364,6 +378,7 @@ def hidden_extras(
|
||||
content[dest.lower()] = sha1_by_path.get(local)
|
||||
lower = set(content) if case_insensitive else set()
|
||||
prefix = _detect_extras_prefix(config, base_dest)
|
||||
region_prefix = declared_root(config)
|
||||
hidden: dict[tuple[str, str, str], str] = {}
|
||||
packed_keys: set[tuple[str, str, str]] = set()
|
||||
extras = _collect_emulator_extras(
|
||||
@@ -377,7 +392,7 @@ def hidden_extras(
|
||||
if required_only and fe.get("required") is False:
|
||||
continue
|
||||
dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
|
||||
if not dest or (region_drops and dest in region_drops):
|
||||
if not dest or (region_drops and extra_region_key(fe, region_prefix) in region_drops):
|
||||
continue
|
||||
full = extra_full_destination(fe, prefix)
|
||||
if not full:
|
||||
@@ -1705,8 +1720,9 @@ def platform_region_groups(
|
||||
)
|
||||
held = {id(extra) for extra in extras}
|
||||
spellings = system_spellings(systems)
|
||||
region_prefix = declared_root(config)
|
||||
for extra in _kept(extras + absent, required_only):
|
||||
dest = sanitize_pack_path(extra.get("destination", extra.get("name", "")))
|
||||
dest = extra_region_key(extra, region_prefix)
|
||||
if not dest:
|
||||
continue
|
||||
name = extra.get("name", "")
|
||||
|
||||
@@ -207,6 +207,7 @@ def _build_readme(
|
||||
save_folder: str = "",
|
||||
save_where: str = "",
|
||||
save_by_system: bool = False,
|
||||
case_twins: list[tuple[str, str]] | None = None,
|
||||
) -> str:
|
||||
"""Build a personalized step-by-step README for each platform pack.
|
||||
|
||||
@@ -216,6 +217,10 @@ def _build_readme(
|
||||
frontend's save directory, empty when the pack holds none, and
|
||||
``save_where`` that directory's default place (save_place).
|
||||
``save_by_system`` is set when the launcher hands each system a folder.
|
||||
``case_twins`` pairs each spelling the pack leaves out, a case twin of a
|
||||
packed file, with the one it holds: given for a platform that also runs
|
||||
where names keep their case (EmuDeck and RetroArch on Linux), where the
|
||||
core asking for the other spelling finds nothing.
|
||||
"""
|
||||
narrowings = narrowings or []
|
||||
sep = "=" * 50
|
||||
@@ -245,6 +250,20 @@ def _build_readme(
|
||||
+ (pack_manual(platform_name) or _generic_manual(bios_paths))
|
||||
)
|
||||
|
||||
if case_twins:
|
||||
listed = "".join(
|
||||
f" {missing} (the pack has {held})\n" for missing, held in case_twins
|
||||
)
|
||||
guide += (
|
||||
"NAMES THAT DIFFER ONLY BY CASE\n\n"
|
||||
" Some cores open the same file under another spelling. This\n"
|
||||
" pack holds one spelling of each, all Windows and macOS can\n"
|
||||
" store in one folder. Linux, SteamOS and Android keep both:\n"
|
||||
" there, run the installer, which adds these spellings, or\n"
|
||||
" copy each file under the name below.\n\n"
|
||||
f"{listed}\n"
|
||||
)
|
||||
|
||||
if save_folder:
|
||||
guide += _save_section(
|
||||
platform_display, save_folder, save_where, save_by_system,
|
||||
|
||||
+106
-10
@@ -55,6 +55,7 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
# The build scripts need CPython 3.9, which builds the same database,
|
||||
@@ -562,6 +563,7 @@ def check_emulator_packs(
|
||||
comparison holds them together. No archive is written and no file read.
|
||||
"""
|
||||
import generate_pack # noqa: PLC0415
|
||||
import packresolve # noqa: PLC0415
|
||||
import verify # noqa: PLC0415
|
||||
from common import ( # noqa: PLC0415
|
||||
ProfileSelectionError,
|
||||
@@ -580,16 +582,23 @@ def check_emulator_packs(
|
||||
|
||||
def compare(names: list[str], standalone: bool, label: str, **narrowing: object) -> bool:
|
||||
record = generate_pack.PackRecord()
|
||||
with contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(io.StringIO()):
|
||||
generate_pack.generate_emulator_pack(
|
||||
names, emulators_dir, db, "bios", "", standalone, zip_contents,
|
||||
offline=True, platforms_dir=platforms_dir, record=record, **narrowing,
|
||||
)
|
||||
report = verify.verify_emulator(
|
||||
names, emulators_dir, db, standalone, platforms_dir=platforms_dir,
|
||||
**narrowing,
|
||||
)
|
||||
was_offline = packresolve._OFFLINE
|
||||
# The pack is built offline: verify counts a release asset the cache
|
||||
# lacks the same way, missing, as the narrowed-pack step does.
|
||||
packresolve.set_offline(True)
|
||||
try:
|
||||
with contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(io.StringIO()):
|
||||
generate_pack.generate_emulator_pack(
|
||||
names, emulators_dir, db, "bios", "", standalone, zip_contents,
|
||||
offline=True, platforms_dir=platforms_dir, record=record, **narrowing,
|
||||
)
|
||||
report = verify.verify_emulator(
|
||||
names, emulators_dir, db, standalone, platforms_dir=platforms_dir,
|
||||
**narrowing,
|
||||
)
|
||||
finally:
|
||||
packresolve.set_offline(was_offline)
|
||||
lines = _set_difference_lines(
|
||||
label, set(report["pack_destinations"]), set(record.files)
|
||||
)
|
||||
@@ -802,6 +811,92 @@ def check_narrowed_packs(
|
||||
return all_ok
|
||||
|
||||
|
||||
def split_differences(
|
||||
full: object, parts: dict[str, object], same_bytes: Callable[[str, str], bool],
|
||||
) -> list[str]:
|
||||
"""Where the --split parts of a pack, together, are not the full pack.
|
||||
|
||||
A path in a part and not in the full pack, a path of the full pack in no
|
||||
part, and a path a part fills with other bytes than the full pack.
|
||||
"""
|
||||
full_members = {m for m in full.members if not _PACK_NOTES.search(m)}
|
||||
union = {
|
||||
m for record in parts.values() for m in record.members
|
||||
if not _PACK_NOTES.search(m)
|
||||
}
|
||||
lines = [f"in a part, not in the full pack: {m}" for m in sorted(union - full_members)]
|
||||
lines += [f"in the full pack, in no part: {m}" for m in sorted(full_members - union)]
|
||||
lines += sorted({
|
||||
f"other bytes in {name}: {member}"
|
||||
for name, record in parts.items()
|
||||
for member, source in record.sources.items()
|
||||
if member in full.sources and not same_bytes(source, full.sources[member])
|
||||
})
|
||||
return lines
|
||||
|
||||
|
||||
def check_split_packs(
|
||||
emulators_dir: str = "emulators", platforms_dir: str = "platforms",
|
||||
db_path: str = "database.json", include_archived: bool = False,
|
||||
) -> bool:
|
||||
"""Build each platform's --split parts in memory and compare them with
|
||||
the full pack: the parts must add up to it, path for path and byte for
|
||||
byte. Each part is a pack of its own and used to settle its paths alone:
|
||||
RetroDECK's parts carried bios/SGB1.sfc as a file and as a directory,
|
||||
RetroBat's 21 openmsx/ twins of openMSX/ files. Offline, as every
|
||||
comparison of this step.
|
||||
"""
|
||||
import generate_pack # noqa: PLC0415
|
||||
import packresolve # noqa: PLC0415
|
||||
from common import ( # noqa: PLC0415
|
||||
build_zip_contents_index,
|
||||
group_identical_platforms,
|
||||
list_registered_platforms,
|
||||
load_data_dir_registry,
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
)
|
||||
|
||||
print(f"\n--- {step_title(STEP_LABELS['consistency'])}, split packs ---")
|
||||
db = load_database(db_path)
|
||||
zip_contents = build_zip_contents_index(db)
|
||||
profiles = load_emulator_profiles(emulators_dir)
|
||||
registry = load_data_dir_registry(platforms_dir)
|
||||
groups = group_identical_platforms(
|
||||
list_registered_platforms(platforms_dir, include_archived=include_archived),
|
||||
platforms_dir,
|
||||
)
|
||||
was_offline = packresolve._OFFLINE
|
||||
packresolve.set_offline(True)
|
||||
all_ok = True
|
||||
try:
|
||||
for _members, platform in groups:
|
||||
args = dict(
|
||||
emulators_dir=emulators_dir, zip_contents=zip_contents,
|
||||
data_registry=registry, emu_profiles=profiles, offline=True,
|
||||
)
|
||||
full = generate_pack.PackRecord()
|
||||
parts: dict[str, generate_pack.PackRecord] = {}
|
||||
with contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(io.StringIO()):
|
||||
generate_pack.generate_pack(
|
||||
platform, platforms_dir, db, "bios", "", record=full, **args,
|
||||
)
|
||||
generate_pack.generate_split_packs(
|
||||
platform, platforms_dir, db, "bios", "", records=parts, **args,
|
||||
)
|
||||
lines = split_differences(full, parts, generate_pack._same_bytes)
|
||||
if lines:
|
||||
all_ok = False
|
||||
print(f" {platform} --split: {len(lines)} differences")
|
||||
print("\n".join(f" {line}" for line in lines[:20]))
|
||||
finally:
|
||||
packresolve.set_offline(was_offline)
|
||||
print(f" {len(groups)} split platform packs compared, "
|
||||
f"{'all equal' if all_ok else 'differences above'}")
|
||||
return all_ok
|
||||
|
||||
|
||||
class _Skipped:
|
||||
"""A step that did not run. Truthy, so it never fails the pipeline, and
|
||||
distinct, so the summary does not report it as work that was done."""
|
||||
@@ -1365,6 +1460,7 @@ def main() -> None:
|
||||
ok = check_narrowed_packs(
|
||||
include_archived=args.include_archived, full=args.full_narrowings,
|
||||
) and ok
|
||||
ok = check_split_packs(include_archived=args.include_archived) and ok
|
||||
print(
|
||||
f"--- {step_title(STEP_LABELS['consistency'])}, counts and paths: "
|
||||
f"{'OK' if ok else 'FAILED'} ---"
|
||||
|
||||
@@ -17,11 +17,14 @@ Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from artifacts import set_plain_mode
|
||||
from common import copy_file_atomic, load_database
|
||||
|
||||
|
||||
@@ -56,6 +59,36 @@ def index_cache(cache_dir: str) -> dict[str, str]:
|
||||
return index
|
||||
|
||||
|
||||
def _copy_verified(source: str, path: str, sha1: str) -> bool:
|
||||
"""Copy *source* to *path* only if the copy hashes to *sha1*.
|
||||
|
||||
The cache was hashed before the copy, and another run can replace a
|
||||
cached name in between (a re-uploaded asset): the copy is what counts.
|
||||
Whole or absent: a scan running beside a plain copy hashed the truncated
|
||||
file and published that in the database.
|
||||
"""
|
||||
handle, scratch = tempfile.mkstemp(
|
||||
dir=os.path.dirname(os.path.abspath(path)),
|
||||
prefix=f".{os.path.basename(path)}.", suffix=".tmp",
|
||||
)
|
||||
os.close(handle)
|
||||
try:
|
||||
copy_file_atomic(source, scratch)
|
||||
digest = hashlib.sha1()
|
||||
with open(scratch, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
if digest.hexdigest() != sha1:
|
||||
print(f"Changed in the cache while restoring: {source}", file=sys.stderr)
|
||||
return False
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, path)
|
||||
return True
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(scratch)
|
||||
|
||||
|
||||
def restore(
|
||||
cache_dir: str, db_path: str, gitignore: str
|
||||
) -> tuple[int, list[str]]:
|
||||
@@ -79,9 +112,9 @@ def restore(
|
||||
unsatisfied.append(path)
|
||||
continue
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
# Whole or absent: a scan running beside a plain copy hashed the
|
||||
# truncated file and published that in the database.
|
||||
copy_file_atomic(source, path)
|
||||
if not _copy_verified(source, path, sha1):
|
||||
unsatisfied.append(path)
|
||||
continue
|
||||
print(f"Restored: {path}")
|
||||
restored += 1
|
||||
print(f"Total: {restored} files restored")
|
||||
|
||||
+7
-5
@@ -524,6 +524,7 @@ def pack_overrides(
|
||||
data_dir_registry: dict | None = None,
|
||||
target_cores: set[str] | None = None,
|
||||
source: str = "full",
|
||||
choices: dict[str, list[tuple[str, str | None]]] | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Full pack destination -> the file a platform pack serves there instead.
|
||||
|
||||
@@ -532,7 +533,8 @@ def pack_overrides(
|
||||
existence mode the frontend never reads the bytes, so serving the
|
||||
emulator's file satisfies both. The ZIP builder, the install manifest and
|
||||
verify all read this: two of them deciding alone gave the one-line
|
||||
installer different bytes than the ZIP.
|
||||
installer different bytes than the ZIP. ``choices`` is slot_choices
|
||||
for the same arguments when the caller already holds it.
|
||||
"""
|
||||
# A baseline-only pack serves what the platform declares, and a target's
|
||||
# pack answers to the cores that hardware builds: DuckStation does not
|
||||
@@ -541,10 +543,10 @@ def pack_overrides(
|
||||
return {}
|
||||
cores = set(resolve_platform_cores(config, profiles, target_cores=target_cores))
|
||||
overrides: dict[str, str] = {}
|
||||
for destination, choices in slot_choices(
|
||||
config, profiles, db, zip_contents, data_dir_registry
|
||||
).items():
|
||||
path = next((path for emulator, path in choices if emulator in cores), None)
|
||||
if choices is None:
|
||||
choices = slot_choices(config, profiles, db, zip_contents, data_dir_registry)
|
||||
for destination, ordered in choices.items():
|
||||
path = next((path for emulator, path in ordered if emulator in cores), None)
|
||||
if path:
|
||||
overrides[destination] = path
|
||||
return overrides
|
||||
|
||||
+79
-20
@@ -26,11 +26,12 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from pathlib import Path
|
||||
|
||||
import slots
|
||||
@@ -38,6 +39,8 @@ import slots
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from hashing import parse_md5_list
|
||||
from common import (
|
||||
_foreign_tree,
|
||||
_owner_tokens,
|
||||
platform_arg,
|
||||
target_listing_line,
|
||||
suggest_names,
|
||||
@@ -419,14 +422,25 @@ def _build_expected(file_entry: dict, checks: list[str]) -> dict:
|
||||
return expected
|
||||
|
||||
|
||||
def _in_foreign_tree(db: dict, owners: set[str], sha1: str) -> bool:
|
||||
"""Whether the collection keeps this sha1 in another emulator's tree."""
|
||||
return _foreign_tree(db.get("files", {}).get(sha1, {}).get("path", ""), owners)
|
||||
|
||||
|
||||
def _name_in_index(
|
||||
name: str,
|
||||
by_name: dict,
|
||||
by_path_suffix: dict | None = None,
|
||||
data_names: set[str] | None = None,
|
||||
by_name_lower: dict[str, str] | None = None,
|
||||
foreign: Callable[[str], bool] | None = None,
|
||||
) -> bool:
|
||||
"""Check if a name is resolvable in the database indexes or data directories."""
|
||||
"""Check if a name is resolvable in the database indexes or data directories.
|
||||
|
||||
*foreign* tells a copy in another emulator's tree from the sha1 of a
|
||||
name: matched in another casing, such a copy is not the asker's, as in
|
||||
resolve_local_file's case-insensitive step.
|
||||
"""
|
||||
# Strip trailing slash for directory-type entries (e.g. nestopia/samples/foo/)
|
||||
name = name.rstrip("/")
|
||||
if name in by_name:
|
||||
@@ -435,11 +449,11 @@ def _name_in_index(
|
||||
if basename != name and basename in by_name:
|
||||
return True
|
||||
if by_name_lower:
|
||||
key = name.lower()
|
||||
if key in by_name_lower:
|
||||
return True
|
||||
if basename != name and basename.lower() in by_name_lower:
|
||||
return True
|
||||
for key in {name.lower(), basename.lower()}:
|
||||
if key in by_name_lower and not (
|
||||
foreign and all(foreign(s) for s in by_name[by_name_lower[key]])
|
||||
):
|
||||
return True
|
||||
if by_path_suffix and name in by_path_suffix:
|
||||
return True
|
||||
if data_names:
|
||||
@@ -667,23 +681,30 @@ def find_undeclared_files(
|
||||
# Resolution: name, then path basename
|
||||
# storage: release names where the bytes are served from, not
|
||||
# that the collection holds them: resolved like any file.
|
||||
# Resolved as the asker, as the builder's ExtraResolver does:
|
||||
# another casing of the name in another emulator's tree is
|
||||
# not this emulator's file (dosbox-x's et4000.bin and pcem's
|
||||
# et4000.BIN).
|
||||
owned = {**f, "source_profile": emu_name}
|
||||
foreign = functools.partial(_in_foreign_tree, db, _owner_tokens(owned))
|
||||
if shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
|
||||
# The entry states what its content should be and the builder
|
||||
# drops a copy that contradicts it, so content decides here
|
||||
# too. A name match would not do: generic names collide across
|
||||
# systems, and answering yes on one describes a pack that will
|
||||
# not contain the file.
|
||||
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
|
||||
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
|
||||
in_repo = _lp is not None and _st != "hash_mismatch"
|
||||
else:
|
||||
in_repo = _name_in_index(
|
||||
fname, by_name, by_path_suffix, data_names, by_name_lower,
|
||||
foreign,
|
||||
)
|
||||
if not in_repo and dest != fname:
|
||||
path_base = dest.rsplit("/", 1)[-1]
|
||||
in_repo = _name_in_index(
|
||||
path_base, by_name, by_path_suffix, data_names,
|
||||
by_name_lower,
|
||||
by_name_lower, foreign,
|
||||
)
|
||||
if in_repo and not any(
|
||||
f.get(h) for h in ("sha1", "md5", "crc32", "sha256")
|
||||
@@ -693,7 +714,7 @@ def find_undeclared_files(
|
||||
# rejects them all. NanoBoyAdvance's 16 KB bios.bin met
|
||||
# only PCem's and QEMU's PC BIOS images of that name.
|
||||
# The fallback below then decides as for any mismatch.
|
||||
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
|
||||
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
|
||||
if _st == "hash_mismatch":
|
||||
in_repo = False
|
||||
if not in_repo:
|
||||
@@ -707,7 +728,7 @@ def find_undeclared_files(
|
||||
# frontend would reject it, the builder leaves it out, and
|
||||
# calling it available here would describe a pack that
|
||||
# does not contain it.
|
||||
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
|
||||
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
|
||||
in_repo = _st != "not_found" and _lp is not None
|
||||
if in_repo and _st == "hash_mismatch" and shipped_on_mismatch:
|
||||
in_repo = False
|
||||
@@ -717,7 +738,7 @@ def find_undeclared_files(
|
||||
# held for it, the rule the builder's lookup applies
|
||||
# (packextras.ExtraResolver). The Apple IIGS' 128 KB
|
||||
# ROM is not Basilisk II's ROM.
|
||||
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
|
||||
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
|
||||
if core_size_refusal(
|
||||
f, _lp, same_slot_entries(profile.get("files"), fname, dest)
|
||||
):
|
||||
@@ -1575,6 +1596,13 @@ def verify_platform(
|
||||
"exclusion_notes": exclusions,
|
||||
"region_fallbacks": sorted(region_fallbacks),
|
||||
**({"left_out": left_out} if no_game_data else {}),
|
||||
# Every tree of the registry answers a declaration by its tail, so
|
||||
# one that was never refreshed leaves its files missing here while
|
||||
# an online pack build refreshes it and packs them.
|
||||
"data_dir_notices": sorted(
|
||||
key for key, entry in (data_dir_registry or {}).items()
|
||||
if not os.path.isdir(entry.get("local_cache", ""))
|
||||
),
|
||||
"details": details,
|
||||
"ground_truth_coverage": {
|
||||
"with_validation": with_validation,
|
||||
@@ -1962,6 +1990,13 @@ def print_platform_result(
|
||||
# them missing, as an offline build leaves them out.
|
||||
shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "")
|
||||
print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}")
|
||||
uncached = result.get("data_dir_notices") or []
|
||||
if uncached and result.get("status_counts", {}).get(Status.MISSING):
|
||||
shown = ", ".join(uncached[:10]) + (f" (+{len(uncached) - 10})" if len(uncached) > 10 else "")
|
||||
print(
|
||||
f" Data directories not cached ({len(uncached)}), their files count missing "
|
||||
f"until refresh_data_dirs.py runs (an online pack build refreshes them): {shown}"
|
||||
)
|
||||
for line in result.get("left_out") or []:
|
||||
print(f" {line}")
|
||||
fallbacks = result.get("region_fallbacks") or []
|
||||
@@ -2068,7 +2103,7 @@ def verify_emulator(
|
||||
it and every member it carries are optional, as the builder leaves them.
|
||||
"""
|
||||
from packextras import expand_agnostic_entries
|
||||
from packresolve import resolve_file
|
||||
from packresolve import pending_release_asset, resolve_file, run_is_offline
|
||||
from packpaths import (
|
||||
_archive_destination,
|
||||
_register_path,
|
||||
@@ -2094,6 +2129,16 @@ def verify_emulator(
|
||||
validation_index = _build_validation_index(selected_profiles)
|
||||
data_registry = load_data_dir_registry(platforms_dir)
|
||||
|
||||
def pending(entry: dict, local_path: str | None, status: str) -> tuple[str | None, str]:
|
||||
# Read from disk and the cache, as _resolve_for_platform does: a
|
||||
# release asset the cache lacks is fetched by an online build
|
||||
# (generate_emulator_pack resolves with offline=offline).
|
||||
if status in ("not_found", "hash_mismatch") and not run_is_offline():
|
||||
asset = pending_release_asset(entry, db)
|
||||
if asset:
|
||||
return asset, RELEASE_ASSET_PENDING
|
||||
return local_path, status
|
||||
|
||||
details = []
|
||||
file_status: dict[str, str] = {}
|
||||
file_severity: dict[str, str] = {}
|
||||
@@ -2233,14 +2278,15 @@ def verify_emulator(
|
||||
archive = file_entry.get("archive")
|
||||
if archive and archive not in seen_archives:
|
||||
seen_archives.add(archive)
|
||||
local_path, resolve_status = resolve_file(
|
||||
archive_entry(archive, files),
|
||||
own_entry = archive_entry(archive, files)
|
||||
local_path, resolve_status = pending(own_entry, *resolve_file(
|
||||
own_entry,
|
||||
db,
|
||||
DEFAULT_BIOS_DIR,
|
||||
zip_contents,
|
||||
data_dir_registry=data_registry,
|
||||
offline=True,
|
||||
)
|
||||
))
|
||||
required = any(
|
||||
f.get("archive") == archive and f.get("required", True)
|
||||
for f in files
|
||||
@@ -2264,6 +2310,8 @@ def verify_emulator(
|
||||
"required": required,
|
||||
"path": local_path,
|
||||
}
|
||||
if resolve_status == RELEASE_ASSET_PENDING:
|
||||
result["release_asset"] = local_path
|
||||
else:
|
||||
result = {
|
||||
"name": archive,
|
||||
@@ -2324,7 +2372,7 @@ def verify_emulator(
|
||||
|
||||
dest_hint = file_entry.get("path", "")
|
||||
# The emulator pack's resolution, release assets included.
|
||||
local_path, resolve_status = resolve_file(
|
||||
local_path, resolve_status = pending(file_entry, *resolve_file(
|
||||
{**with_archive_members(file_entry, files), "source_profile": emu_name},
|
||||
db,
|
||||
DEFAULT_BIOS_DIR,
|
||||
@@ -2332,7 +2380,7 @@ def verify_emulator(
|
||||
dest_hint=dest_hint,
|
||||
data_dir_registry=data_registry,
|
||||
offline=True,
|
||||
)
|
||||
))
|
||||
if resolve_status in ("not_found", "user_provided", "external"):
|
||||
local_path = None
|
||||
name = file_entry.get("name", "")
|
||||
@@ -2342,6 +2390,15 @@ def verify_emulator(
|
||||
stored = _storage_verdict(file_entry, emulator_pack=True)
|
||||
if stored is not None:
|
||||
result = stored
|
||||
elif resolve_status == RELEASE_ASSET_PENDING:
|
||||
result = {
|
||||
"name": name,
|
||||
"status": Status.OK,
|
||||
"required": required,
|
||||
"path": local_path,
|
||||
"release_asset": local_path,
|
||||
"reason": "release asset, fetched when the pack is built",
|
||||
}
|
||||
elif not local_path:
|
||||
result = {"name": name, "status": Status.MISSING, "required": required}
|
||||
# An entry the profile documents as unsourceable is absent by
|
||||
@@ -2592,6 +2649,10 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
|
||||
if unsourceable:
|
||||
parts.append(f"{unsourceable} unsourceable")
|
||||
print(f"{label}: {', '.join(parts)} [{mode}]")
|
||||
fetched = sorted({d["name"] for d in result["details"] if d.get("release_asset")})
|
||||
if fetched:
|
||||
shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "")
|
||||
print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}")
|
||||
|
||||
seen = set()
|
||||
for d in result["details"]:
|
||||
@@ -2722,8 +2783,6 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
|
||||
"--destinations describes the full pack and is incompatible with --system"),
|
||||
(args.destinations and not (args.platform or args.all),
|
||||
"--destinations requires --platform or --all"),
|
||||
(args.offline and not (args.platform or args.all),
|
||||
"--offline requires --platform or --all"),
|
||||
# Game data is categorized against a platform's list and its cores.
|
||||
(getattr(args, "no_game_data", False) and not (args.platform or args.all),
|
||||
"--no-game-data requires --platform or --all"),
|
||||
|
||||
Reference in new issue
Block a user