kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-06 10:03:49 -06:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge site insights

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-11-06 12:56:13 +01:00
2 parents b23840bc21 f55e473183
commit c1b0149a81
146 files changed
+10427 -797
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/artifacts.py
+13
View File
@@ -19,6 +19,7 @@ import re
_TIMESTAMP_PATTERNS = [
re.compile(r'"generated_at":\s*"[^"]*"'), # database.json
re.compile(r'"imported_at":\s*"[^"]*"'), # provenance snapshots
re.compile(r'"fetched":\s*"[^"]*"'), # games/catalogs/
re.compile(r"\*Auto-generated on [^*]*\*"), # README.md
re.compile(r"\*Generated on [^*]*\*"), # docs site pages
# The decorated pages carry the same stamp again as a rendered element,
@@ -208,6 +209,10 @@ def _strip_timestamps(text: str) -> str:
class ArtifactLockBusy(RuntimeError):
"""Raised when another process already holds the artifact directory."""
class ArtifactDirectoryMissing(FileNotFoundError):
"""A reader named a directory that does not exist."""
# A run that holds artifact directories for its whole duration names them
# here, os.pathsep apart, for the steps it spawns: they would otherwise be
# refused by it. A pipeline holds the repository root and its --output-dir.
@@ -227,6 +232,14 @@ def artifact_lock(directory: str, exclusive: bool = True) -> Iterator[None]:
Writers take the lock exclusively, readers share it. On platforms
without flock the lock is a no-op.
"""
# A reader creating the directory it was pointed at left a typo behind
# as a directory holding a .lock, and the next run with the same typo
# no longer said the path did not exist.
if not exclusive and not os.path.isdir(directory):
raise ArtifactDirectoryMissing(
f"{directory}: no such directory, pass the --output-dir the packs "
"were written to (default dist/)"
)
try:
import fcntl
except ImportError:
scripts/bios_integrity.py
+17 -2
View File
@@ -91,8 +91,19 @@ def sha1_of(path: Path) -> str:
return digest.hexdigest()
class GitFailed(RuntimeError):
"""A git command failed: its empty output says nothing about the file."""
def git(root: Path, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
"""Run git in root. A failure raises: an unreadable index read as an empty
status made a session's working change look unmodified, and --apply wrote
HEAD's bytes over it."""
done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
if done.returncode:
message = done.stderr.decode(errors="replace").strip()
raise GitFailed(f"git {' '.join(args)} exited {done.returncode}: {message}")
return done
def head_blob(root: Path, path: str) -> str:
@@ -197,7 +208,11 @@ def main() -> int:
from common import load_database
root = Path.cwd()
report = check(root, load_database(args.db), args.apply)
try:
report = check(root, load_database(args.db), args.apply)
except GitFailed as exc:
print(f"ERROR: {exc}. Nothing was restored past this point.", file=sys.stderr)
return 2
if args.json:
print(json.dumps(report.as_dict(), indent=2))
else:
scripts/check_freshness.py
+25 -17
View File
@@ -32,7 +32,6 @@ from __future__ import annotations
import argparse
import contextlib
import hashlib
import io
import json
import re
@@ -51,6 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import export_native # noqa: E402
import pin_installer # noqa: E402
import refresh_data_dirs # noqa: E402
import upstream # noqa: E402
from common import ( # noqa: E402
@@ -83,7 +83,6 @@ PYPI_URL = "https://pypi.org/pypi/{name}/json"
NPM_URL = "https://registry.npmjs.org/{name}/latest"
# The site's npm dependencies, pinned to exact versions and built by Deploy Site.
SITE_PACKAGE = REPO_ROOT / "website" / "package.json"
INSTALLERS = ("install.sh", "install.ps1")
SHOWN_ITEMS = 4 # items named in a diff summary before ", ..."
SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..."
@@ -509,7 +508,10 @@ def check_native(
) -> list[Finding]:
exporters = discover_exporters()
index = cache_dir / export_native.SOURCES_INDEX
recorded = export_native.load_sources(cache_dir)
try:
recorded = export_native.load_sources(cache_dir)
except export_native.SourcesIndexUnreadable as exc:
return [Finding("native", index.name, ERROR, detail=str(exc))]
findings: list[Finding] = []
for name in list_registered_platforms(str(platforms_dir), include_archived=True):
exporter_class = exporters.get(name)
@@ -1462,18 +1464,16 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]:
def _check_installer_pins() -> Finding:
"""The bootstraps pin install.py by SHA-256, so a stale pin refuses every install."""
digest = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
stale = []
for name in INSTALLERS:
text = (REPO_ROOT / name).read_text(encoding="utf-8", errors="replace")
if digest not in text:
stale.append(name)
stale = pin_installer.stale_pins(REPO_ROOT)
return Finding(
"ci",
"install.py pin",
STALE if stale else OK,
local=digest[:12],
detail=("pin outdated in " + ", ".join(stale)) if stale else "install.sh and install.ps1 match",
local=pin_installer.installer_sha256(REPO_ROOT)[:12],
detail=(
f"pin outdated in {', '.join(stale)}, {pin_installer.COMMAND} writes it"
if stale else "install.sh and install.ps1 match"
),
)
@@ -1492,6 +1492,11 @@ def check_source_paths(profiles: dict) -> Finding:
sitesources.tree_key(repo, pin)
for profile in profiles.values() if sitesources.cited_paths(profile)
for repo, pin in sitesources.profile_trees(profile)
} | {
# The platform build files pin_build names, linked from a page too.
sitesources.tree_key(repo, pin)
for profile in profiles.values()
for repo, pin, _path in sitesources.build_files(profile)
}
try:
recorded = set(json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")))
@@ -1757,15 +1762,18 @@ def render(findings: list[Finding]) -> str:
# Columns are padded, never cut: the profile_sync row says in its
# local and remote values what to re-read, and lost the end of it.
local_width = min(max((len(f.local) for f in rows), default=0), 44)
remote_width = min(max((len(f.remote) for f in rows), default=0), 44)
# The arrow and the remote value, or as many blanks where a row has none.
remote_span = 4 + remote_width if remote_width else 0
for f in rows:
cols = f" {f.status:7} {f.subject:{width}}"
if f.remote:
cols += f" {f.local:{local_width}} -> {f.remote}"
elif f.local or (f.detail and local_width):
# An arrow pointing at nothing read as a lost value. An empty
# local value still takes its column, or the detail starts
# where the other rows show theirs.
cols += f" {f.local:{local_width}}"
cols += f" {f.local:{local_width}} -> {f.remote:{remote_width}}"
elif f.local or (f.detail and (local_width or remote_span)):
# An arrow pointing at nothing read as a lost value. Empty
# local and remote values still take their columns, or the
# detail starts where the other rows show theirs.
cols += f" {f.local:{local_width}}" + " " * remote_span
if f.detail:
cols += f" {f.detail}"
lines.append(cols.rstrip())
scripts/common.py
+6 -1
View File
@@ -3036,12 +3036,16 @@ SYSTEM_ALIASES: dict[str, str] = {
}
@functools.lru_cache(maxsize=None)
def _norm_system_id(sid: str) -> str:
"""Normalize system ID for cross-platform matching.
Resolves known aliases, then strips manufacturer prefixes and separators
so that platform-specific IDs (e.g., "xbox", "nintendo-wiiu") match
profile IDs (e.g., "microsoft-xbox", "nintendo-wii-u").
A function of the string and two constant tables: a manifest run asks it
1.7 million times for about 1,100 ids.
"""
s = sid.lower().replace("_", "-")
s = SYSTEM_ALIASES.get(s, s)
@@ -3587,6 +3591,7 @@ from artifacts import ( # noqa: E402,F401
copy_file_atomic,
plain_file_mode,
set_plain_mode,
ArtifactDirectoryMissing,
ArtifactLockBusy,
artifact_lock,
hold_artifact_lock,
@@ -3634,7 +3639,7 @@ def run_cli(main: Callable[[], int | None]) -> None:
try:
code = main()
sys.stdout.flush()
except (InputMissing, ArtifactLockBusy) as exc:
except (InputMissing, ArtifactLockBusy, ArtifactDirectoryMissing) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
sys.exit(1)
except BrokenPipeError:
scripts/cross_reference.py
+22 -4
View File
@@ -26,7 +26,9 @@ sys.path.insert(0, os.path.dirname(__file__))
from common import (
platform_arg,
ProfileSelectionError,
_foreign_tree,
_norm_system_id,
_owner_tokens,
archive_entry,
directory_members,
expand_directory_entries,
@@ -215,8 +217,14 @@ def _resolve_source(
none of the hashes the entry declares: a name never wins over a declared
hash, and zquestclassic's ending.mid read as held because Hydra Castle
Labyrinth has another file of that name.
Another casing of the name in another emulator's tree proves nothing, as
in resolve_local_file's case-insensitive step: dosbox-x's et4000.bin read
as held through pcem's et4000.BIN, which the pack and the installer
refuse it.
"""
declared = _declared_content(file_entry or {})
owners = _owner_tokens(file_entry or {})
def _can_be(sha1s: list | str) -> bool:
if not file_entry or db_files is None:
@@ -233,6 +241,14 @@ def _resolve_source(
"""Whether the files indexed under *name* can be this entry."""
return _can_be(by_name.get(name) or [])
def _folded_hit(name: str) -> bool:
"""_name_hit for a name matched in another casing."""
return _can_be([
sha1 for sha1 in by_name.get(name) or []
if db_files is None
or not _foreign_tree(db_files.get(sha1, {}).get("path", ""), owners)
])
# bios/ via database.json by_name
if fname in by_name and _name_hit(fname):
return "bios"
@@ -241,10 +257,10 @@ def _resolve_source(
if basename and basename in by_name and _name_hit(basename):
return "bios"
key = fname.lower()
if key in by_name_lower and _name_hit(by_name_lower[key]):
if key in by_name_lower and _folded_hit(by_name_lower[key]):
return "bios"
if basename:
if basename.lower() in by_name_lower and _name_hit(
if basename.lower() in by_name_lower and _folded_hit(
by_name_lower[basename.lower()]
):
return "bios"
@@ -257,7 +273,7 @@ def _resolve_source(
if canonical and canonical != fname:
if canonical in by_name and _name_hit(canonical):
return "bios"
if canonical.lower() in by_name_lower and _name_hit(
if canonical.lower() in by_name_lower and _folded_hit(
by_name_lower[canonical.lower()]
):
return "bios"
@@ -901,7 +917,9 @@ def _cross_reference_profile(
source = next(
(
found for version in versions[slot]
if (found := entry_source(version, index)) is not None
if (found := entry_source(
{**version, "source_profile": emu_name}, index,
)) is not None
),
"missing",
)
scripts/export_native.py
+17 -2
View File
@@ -36,6 +36,7 @@ from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_tex
import yaml
from common import (
InputMissing,
build_zip_contents_index,
platform_arg,
list_registered_platforms,
@@ -62,13 +63,27 @@ _USER_AGENT = "retrobios-exporter/1.0"
_MAX_BYTES = 64 * 1024 * 1024
class SourcesIndexUnreadable(InputMissing):
"""The cache's URL index exists and does not read as JSON."""
def _load_sources(index: Path | None) -> dict[str, str]:
"""The URL each cached file was fetched from.
An absent index records nothing. An index that does not read raises:
read as empty it let every cached original pass for current, and the
next download rewrote it with that one entry.
"""
if index is None or not index.is_file():
return {}
try:
return json.loads(index.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
except (OSError, json.JSONDecodeError) as exc:
raise SourcesIndexUnreadable(
f"{index} does not read ({exc}): remove it, then"
" python scripts/export_native.py --all --refresh-cache"
" downloads every original again and records its URL"
) from exc
def source_key(destination: Path, index: Path | None) -> str:
scripts/exporter/batocera_exporter.py
+11 -7
View File
@@ -19,7 +19,9 @@ SOURCE_URL = (
"/package/batocera/core/batocera-scripts/scripts/batocera-systems"
)
_ENTRY_START = re.compile(r'^(\s{4})"([^"]+)":\s*\{')
# Any indent: Batocera writes four spaces, EmuELEC one tab for 23 systems,
# and an entry the pattern missed was appended again whole.
_ENTRY_START = re.compile(r'^(\s+)"([^"]+)":\s*\{')
_INDENT = " " * 4
# One BIOS item: a dict literal with no nested braces.
_ITEM = re.compile(r"\{[^{}\n]*\}")
@@ -382,12 +384,13 @@ class Exporter(BaseExporter):
return inner if isinstance(inner, dict) else {}
def _entry_line(
self, system: NativeSystem, files: list[NativeFile], before: dict | None = None
self, system: NativeSystem, files: list[NativeFile], before: dict | None = None,
indent: str = _INDENT,
) -> str:
own = self._own_entry(system, before)
runner = "".join(f"{key}, " for key in self._runner_keys(own))
return (
f'{_INDENT}"{system.native_id}": '
f'{indent}"{system.native_id}": '
f'{{ "name": "{self.display_name(system)}", {runner}'
f'"biosFiles": [ {self._bios_files(files, own)} ] }},'
)
@@ -440,16 +443,17 @@ class Exporter(BaseExporter):
before = self._parse_entry(original)
own = self._own_entry(system, before)
items = self._bios_items(files, own)
indent = _ENTRY_START.match(original[0]).group(1)
one_line = len(original) == 1 and not trailing and not leading
if one_line:
return [self._entry_line(system, files, before)]
return [self._entry_line(system, files, before, indent)]
runner = "".join(f"{key}, " for key in self._runner_keys(own))
head = (
f'{_INDENT}"{system.native_id}": '
f'{indent}"{system.native_id}": '
f'{{ "name": "{self.display_name(system)}", {runner}"biosFiles": ['
)
pad = " " * (len(_INDENT) + 4)
pad = indent + " " * 4
lines = [head]
for index, item in enumerate(items):
key = _note_key(item)
@@ -458,7 +462,7 @@ class Exporter(BaseExporter):
note = trailing.get(key, "")
suffix = f" {note}" if note else ""
lines.append(f"{pad}{item}{separator}{suffix}")
lines.append(f"{_INDENT}] }},")
lines.append(f"{indent}] }},")
return lines
def _patched_entry(
scripts/generate_db.py
+23 -4
View File
@@ -576,12 +576,26 @@ def _collect_all_aliases(
# naming its content once 64DD_IPL.bin sorted first, and its aliases
# 64dd_ipl_J.bin and 64DD_IPL_JP.n64 went with it.
holders: dict[str, set[str]] = {}
# The files a name designates once case is folded, as the resolver's
# name step folds it, a primary and its own .variants/ counted as one
# file. dosbox's SC-55 ROM2.BIN found Galaksija's ROM2.BIN the only
# exact holder, and its alias ROM2_ST.BIN went to a 4 KiB BASIC ROM,
# while SC-55's rom2.bin sat under the same folded name.
families: dict[str, set[str]] = {}
def _hold(name: str, path: str, sha1: str) -> None:
holders.setdefault(name, set()).add(sha1)
directory = path.rpartition("/")[0].removesuffix("/.variants")
families.setdefault(name.casefold(), set()).add(
f"{directory}/{variant_base_name(name)}".casefold()
)
for sha1, entry in files.items():
holders.setdefault(entry["name"], set()).add(sha1)
_hold(entry["name"], entry["path"], sha1)
for sha1, copy_list in (copies or {}).items():
if sha1 in files:
for copy in copy_list:
holders.setdefault(copy["name"], set()).add(sha1)
_hold(copy["name"], copy["path"], sha1)
aliases = {}
@@ -691,11 +705,16 @@ def _collect_all_aliases(
]
if taken:
shadowed.append((entry_name, set(matched), taken))
if not matched and len(holders.get(entry_name, ())) == 1:
held = holders.get(entry_name, set())
if (
not matched
and len(held) == 1
and len(families[entry_name.casefold()]) == 1
):
# A name carried by several contents names none of them:
# quasi88's disk.rom aliases went to whichever
# disk.rom the scan met last, a Tandy CoCo ROM.
matched = set(holders[entry_name])
matched = set(held)
for sha in sorted(matched):
for alias_name in entry_aliases:
_add_alias(alias_name, sha)
scripts/generate_pack.py
+161 -50
View File
@@ -26,6 +26,7 @@ import argparse
import contextlib
import functools
import hashlib
import io
import json
import os
import re
@@ -158,6 +159,9 @@ class PackRecord:
self.members: list[str] = []
self.files: list[str] = []
self.trees: list[str] = []
# The collection file each copied member comes from: a split part
# writes a file only where the full pack writes the same bytes.
self.sources: dict[str, str] = {}
def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> None:
@@ -173,6 +177,7 @@ def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> Non
"""
if isinstance(zf, PackRecord):
zf.members.append(arcname)
zf.sources[arcname] = source_path
return
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
info.compress_type = _member_compression(arcname)
@@ -603,6 +608,17 @@ def _pack_data_directories(
return added
def _case_twin_pairs(
twins: list[str], seen_destinations: set[str], base_dest: str, flatten: bool,
) -> list[tuple[str, str]]:
"""Each spelling left out, with the spelling the pack holds."""
held = {
_flat(dest, base_dest, flatten).lower(): _flat(dest, base_dest, flatten)
for dest in seen_destinations
}
return sorted({(twin, held[twin.lower()]) for twin in twins if twin.lower() in held})
def _report_pack_counts(
zip_path: str,
file_status: dict,
@@ -806,6 +822,7 @@ def generate_pack(
pack_name: str | None = None,
no_game_data: bool = False,
record: PackRecord | None = None,
within: dict[str, str] | None = None,
) -> str | None:
"""Generate a ZIP pack for a platform.
@@ -819,6 +836,12 @@ def generate_pack(
``record`` builds the pack in memory (PackRecord): no archive is written,
no external file fetched, and the paths are kept on it.
``within`` is the full pack's PackRecord.sources, given to a split part:
a declared file or core extra is written only where the full pack writes
the same bytes. A part sees neither the other systems' declarations nor
their extras, so a path the full pack settles (a file/directory conflict,
a case twin, a path another system's file takes) was written again.
Returns the path to the generated ZIP, or None on failure.
"""
config = load_platform_config(platform_name, platforms_dir)
@@ -867,6 +890,8 @@ def generate_pack(
user_provided = []
seen_destinations: set[str] = set()
seen_lower: set[str] = set() # only used when case_insensitive=True
# Spellings left out as case twins of a packed file, for the README.
case_twins: list[str] = []
seen_parents: set[str] = (
set()
) # parent dirs of added files (path conflict detection)
@@ -1135,7 +1160,10 @@ def generate_pack(
config.get("hash_rule", ""),
)
if refusal:
file_status[dedup_key] = "excluded"
# An earlier declaration of this path shipped a file this
# one refuses: it is in the pack, untested as verify
# counts it, not excluded.
file_status[dedup_key] = "untested" if already_packed else "excluded"
file_reasons[dedup_key] = refusal
continue
if not hash_mismatch_excludes_file(verification_mode) and (
@@ -1170,7 +1198,17 @@ def generate_pack(
)
if already_packed:
if dedup_key not in seen_destinations:
case_twins.append(_flat(full_dest, base_dest, flatten))
continue
if within is not None and not _same_bytes(
local_path, within.get(_flat(full_dest, base_dest, flatten))
):
# The full pack holds another system's file here.
continue
if file_status.get(dedup_key) == "excluded":
# An earlier declaration refused the file this one ships.
file_status[dedup_key] = "untested"
seen_destinations.add(dedup_key)
_register_path(dedup_key, seen_destinations, seen_parents)
if case_insensitive:
@@ -1227,7 +1265,7 @@ def generate_pack(
if required_only and fe.get("required") is False:
continue
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
if region_drops and dest in region_drops:
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
continue
if not dest:
continue
@@ -1244,10 +1282,13 @@ def generate_pack(
# Taken, a case twin on a case-insensitive filesystem, or a
# file/directory conflict (SGB1.sfc vs SGB1.sfc/): verify replays
# the same rule (packextras.hidden_extras).
if placement_block(
block = placement_block(
full_dest, seen_destinations, seen_lower, seen_parents,
case_insensitive,
):
)
if block:
if block == "case_twin" and extra_resolver.resolve(fe, dest)[0]:
case_twins.append(_flat(full_dest, base_dest, flatten))
continue
# The emulator's own check picks the copy, as for a declared file
@@ -1275,6 +1316,8 @@ def generate_pack(
)
flat_dest = _flat(full_dest, base_dest, flatten)
if within is not None and not _same_bytes(local_path, within.get(flat_dest)):
continue
if local_path.endswith(".zip"):
_add_zip_to_pack(local_path, flat_dest, zf, fe)
else:
@@ -1339,6 +1382,37 @@ def generate_pack(
if _registry_path.exists():
with open(_registry_path) as _rf:
_pack_registry = (yaml_load(_rf) or {}).get("platforms", {})
keeps_case = case_insensitive and any(
d.get("os") not in CASE_FOLDING_OS
for d in (_pack_registry.get(platform_name, {}).get("install", {}) or {})
.get("detect", [])
)
if keeps_case and source != "platform" and precomputed_extras is None:
# The spellings a folded platform counts as declared never reach
# the extras: collected as the filesystem that keeps case sees
# them, as the manifest's case_variants are (Mesen-CE's bios.col
# beside the declared BIOS.col).
unfolded = _collect_emulator_extras(
{**config, "case_insensitive_fs": False}, emulators_dir, db, set(),
base_dest, emu_profiles, target_cores=target_cores,
include_all=(source == "truth"),
)
if system_filter:
unfolded = _extras_for_systems(unfolded, system_filter)
prefix = _detect_extras_prefix(config, base_dest)
for fe in unfolded:
if required_only and fe.get("required") is False:
continue
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
continue
full = extra_full_destination(fe, prefix)
if (
full and full not in seen_destinations and full.lower() in seen_lower
and extra_resolver.resolve(
fe, sanitize_pack_path(fe.get("destination", fe["name"]))
)[0]
):
case_twins.append(_flat(full, base_dest, flatten))
readme_text = _build_readme(
platform_name, platform_display, base_dest, total_files, num_systems,
source=source,
@@ -1364,6 +1438,10 @@ def generate_pack(
) if save_count else ""
),
save_by_system=bool(save_count and save_layout(config).by_system),
case_twins=(
_case_twin_pairs(case_twins, seen_destinations, base_dest, flatten)
if keeps_case else []
),
)
_write_generated_member(zf, "README.txt", readme_text)
@@ -1408,6 +1486,7 @@ def _extract_zip_to_archive(
for name in map(sanitize_pack_path, src.namelist())
if name and not name.endswith("/")
)
target_zf.sources[dest_prefix] = source_zip
return
for info in src.infolist():
if info.is_dir():
@@ -1441,6 +1520,7 @@ def _add_zip_to_pack(
"""
if isinstance(target_zf, PackRecord):
target_zf.members.append(dest_path)
target_zf.sources[dest_path] = source_zip
return
_check_zip_members(source_zip)
_add_pack_member(target_zf, source_zip, dest_path)
@@ -2069,8 +2149,14 @@ def generate_split_packs(
one_per_slot: bool = False,
offline: bool | None = None,
no_game_data: bool = False,
records: dict[str, PackRecord] | None = None,
) -> list[str]:
"""Generate split packs (one ZIP per system or manufacturer)."""
"""Generate split packs (one ZIP per system or manufacturer).
The parts add up to the full pack: each is built against the full pack's
own placement (``within``). ``records`` builds the parts in memory, keyed
by part name, with no archive written.
"""
config = load_platform_config(platform_name, platforms_dir)
# The split directory carries the same tags as the packs inside it, or two
# differently narrowed runs write into one another.
@@ -2093,10 +2179,36 @@ def generate_split_packs(
f"{_platform_pack_stem([platform_name], platform_name, platforms_dir)}{split_tags}"
)
split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split")
os.makedirs(split_dir, exist_ok=True)
if records is None:
os.makedirs(split_dir, exist_ok=True)
if emu_profiles is None:
emu_profiles = load_emulator_profiles(emulators_dir)
narrowing = {
"emulators_dir": emulators_dir, "zip_contents": zip_contents,
"data_registry": data_registry, "emu_profiles": emu_profiles,
"target_cores": target_cores, "required_only": required_only,
"source": source, "regions": regions, "target_name": target_name,
"one_per_slot": one_per_slot, "offline": offline,
"no_game_data": no_game_data,
}
full = PackRecord()
with contextlib.redirect_stdout(io.StringIO()):
generate_pack(
platform_name, platforms_dir, db, bios_dir, split_dir,
record=full, **narrowing,
)
def part(pack_name: str, **scope: object) -> str | None:
record = PackRecord() if records is not None else None
zip_path = generate_pack(
platform_name, platforms_dir, db, bios_dir, split_dir,
pack_name=pack_name, record=record, within=full.sources,
**scope, **narrowing,
)
if record is not None and zip_path:
records[pack_name] = record
return zip_path
# The systems a core of the target runs: a part per declared system sent
# each one the target drops to a refusal naming it as unknown.
systems = platform_scope_systems(config, emu_profiles, target_cores)
@@ -2130,27 +2242,9 @@ def generate_split_packs(
results = []
for group_name, group_system_ids in sorted(groups.items()):
group_extras = _extras_for_systems(all_extras, group_system_ids)
zip_path = generate_pack(
platform_name,
platforms_dir,
db,
bios_dir,
split_dir,
emulators_dir=emulators_dir,
zip_contents=zip_contents,
data_registry=data_registry,
emu_profiles=emu_profiles,
target_cores=target_cores,
required_only=required_only,
system_filter=group_system_ids,
precomputed_extras=group_extras,
source=source,
regions=regions,
target_name=target_name,
one_per_slot=one_per_slot,
offline=offline,
pack_name=f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip",
no_game_data=no_game_data,
zip_path = part(
f"{part_prefix}_{_name_part(group_name, '_')}_BIOS_Pack.zip",
system_filter=group_system_ids, precomputed_extras=group_extras,
)
if zip_path:
results.append(zip_path)
@@ -2166,15 +2260,9 @@ def generate_split_packs(
}
undistributed = [fe for fe in all_extras if id(fe) not in distributed]
if undistributed:
zip_path = generate_pack(
platform_name, platforms_dir, db, bios_dir, split_dir,
emulators_dir=emulators_dir, zip_contents=zip_contents,
data_registry=data_registry, emu_profiles=emu_profiles,
target_cores=target_cores, required_only=required_only,
precomputed_extras=undistributed, extras_only=True, source=source,
regions=regions, target_name=target_name, one_per_slot=one_per_slot,
offline=offline, pack_name=f"{part_prefix}_Other_Cores_BIOS_Pack.zip",
no_game_data=no_game_data,
zip_path = part(
f"{part_prefix}_Other_Cores_BIOS_Pack.zip",
precomputed_extras=undistributed, extras_only=True,
)
if zip_path:
results.append(zip_path)
@@ -4133,7 +4221,8 @@ def _manifest_core_entries(
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
if not dest:
continue
if region_drops and dest in region_drops:
region_key = extra_region_key(fe, declared_root(config))
if region_drops and region_key in region_drops:
continue
full_dest = extra_full_destination(fe, extras_pfx)
if not full_dest:
@@ -4207,7 +4296,7 @@ def _manifest_core_entries(
entry, file_size = _manifest_entry(
local_path, manifest_dest, [source_emu] if source_emu else [],
db, repo_root,
selection.fields(dest, fe.get("name", "")),
selection.fields(region_key, fe.get("name", "")),
)
if entry is not None and save_root:
entry.update(save_entry_fields(fe))
@@ -4321,7 +4410,8 @@ def _manifest_selection(
else set()
)
return drops, SelectionIndex.build(
region_groups, region_index, pack_systems, extras, required_only
region_groups, region_index, pack_systems, extras, required_only,
extras_prefix=declared_root(config),
), held
@@ -4346,7 +4436,9 @@ def _record_absent(
if required_only and fe.get("required") is False:
continue
dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
if not dest or (region_drops and dest in region_drops):
if not dest or (
region_drops and extra_region_key(fe, declared_root(config)) in region_drops
):
continue
full_dest = extra_full_destination(fe, extras_pfx)
if not full_dest:
@@ -4517,16 +4609,18 @@ def generate_manifest(
# (cache file, size). Settled once the trees are walked.
cache_held: dict[str, tuple[str, int]] = {}
rename_notes: set[str] = set()
# The full manifest records every core's answer at a contested slot, so a
# target narrowed from it gets what that target's own pack serves.
whole = target_cores is None and source != "platform" and bool(emu_profiles)
slot_choices = (
slots.slot_choices(config, emu_profiles, db, zip_contents, data_registry)
if whole else {}
)
# Derived from the same choices: computed twice, it was 8% of a run.
slot_overrides = slots.pack_overrides(
config, emu_profiles, db, zip_contents, data_registry,
target_cores=target_cores, source=source,
)
# The full manifest records every core's answer at a contested slot, so a
# target narrowed from it gets what that target's own pack serves.
slot_choices = (
slots.slot_choices(config, emu_profiles, db, zip_contents, data_registry)
if target_cores is None and source != "platform" and emu_profiles
else {}
choices=slot_choices if whole else None,
)
def manifest_destination(full_destination: str) -> str:
@@ -4541,8 +4635,10 @@ def generate_manifest(
reason: str,
cores: list[str] | None,
) -> None:
bare = sanitize_pack_path(
file_entry.get("destination", file_entry.get("name", ""))
# A core extra (cores given) is keyed as the region pass keys it.
bare = (
sanitize_pack_path(file_entry.get("destination", file_entry.get("name", "")))
if cores is None else extra_region_key(file_entry, declared_root(config))
)
save_root = file_entry.get("root") == "save"
omitted_by_destination[full_destination] = {
@@ -4949,9 +5045,12 @@ def _manifest_data_directories(
(strip_components for a ZIP, source_path for a tarball, exclude) come
from _data_dirs.yml, the same refresh_data_dirs applies. Every file
carries the sha1 and size the cache holds, so a member the source
changed since is refused rather than installed.
changed since is refused rather than installed. archive_size is the
size of the archive refresh_data_dirs last downloaded, what fetching
the tree costs: a tarball whose tree is 10 KB is a download of 21 MB.
"""
grouped: dict[tuple[str, str], dict] = {}
versions: dict[str, dict[str, dict]] = {}
for src, full in members:
ref, systems, prefix = refs[full]
entry = registry.get(ref, {})
@@ -4976,6 +5075,17 @@ def _manifest_data_directories(
group["source_path"] = str(entry.get("source_path", "")).format(version=version)
if entry.get("exclude"):
group["exclude"] = [str(e) for e in entry["exclude"]]
cache = entry.get("local_cache")
if cache:
recorded = Path(cache).parent / ".versions.json"
if str(recorded) not in versions:
try:
versions[str(recorded)] = json.loads(recorded.read_text())
except FileNotFoundError:
versions[str(recorded)] = {}
size = versions[str(recorded)].get(ref, {}).get("archive_size")
if isinstance(size, int) and size > 0:
group["archive_size"] = size
group["systems"].extend(s for s in systems if s not in group["systems"])
rel = full[len(prefix):].lstrip("/") if prefix else full
hashes = compute_hashes(src, frozenset({"sha1"}))
@@ -5429,6 +5539,7 @@ from packpaths import ( # noqa: E402,F401
from packextras import ( # noqa: E402,F401
ExtraResolver,
extra_full_destination,
extra_region_key,
prefixed_destination,
save_entry_fields,
save_manifest_dest,
scripts/generate_readme.py
+5 -1
View File
@@ -608,7 +608,7 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json"
"",
"## Download BIOS packs",
"",
"A pack holds everything its platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **" + _installer_words().TUI_SYSTEMS_ROW + "**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.",
f"A pack holds its platform's own BIOS list plus the files its emulator cores load. What no pack carries yet is in the [gap analysis]({SITE_URL}gaps/). Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **" + _installer_words().TUI_SYSTEMS_ROW + "**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.",
"",
"A pack over 2 GB comes in several parts (`Pack.part1of2.zip`,"
" `Pack.part2of2.zip`), and every part is needed. Each part is an"
@@ -941,6 +941,10 @@ the most useful contribution.
hash differs from the one already held under its name goes to that folder's
`.variants/`.
A ZIP that no list declares as a whole is read file by file, and the report
says which of them the collection already holds: upload the files
themselves, unzipped.
From a clone, the file goes to `bios/Manufacturer/Console/filename`, and
`python scripts/validate_pr.py path/to/file` names that folder for a file still
outside `bios/`, and `--place` copies it there under its declared name. Given a
scripts/generate_site.py
+179 -67
View File
@@ -86,9 +86,10 @@ from generate_readme import (
extract_targets,
release_totals,
)
from provenance_report import build_report, distinct_missing
from provenance_report import build_report, distinct_missing, profile_needs
import sitemarkup as markup
import siteentries
import siteinputs
import retired_files
import sitestarlight
from siteexports import generate_exports_page, load_export_reports
@@ -662,7 +663,7 @@ def release_line(platform_name: str, record: dict) -> list[str]:
f"[SHA256SUMS.txt]({base}/SHA256SUMS.txt) and its "
f"[signature]({base}/SHA256SUMS.txt.sig).",
]
config = load_platform_config(platform_name, "platforms")
config = load_platform_config(platform_name, str(siteinputs.current().platforms))
page_version = str(config.get("version") or "")
pack_version = _pack_version(pack_name)
if page_version and pack_version and page_version.lstrip("v") != pack_version.lstrip("v"):
@@ -687,7 +688,7 @@ def setup_version(platform_name: str, record: dict | None = None) -> str:
platform YAML was read at. Empty when neither says, or when the
platform has no YAML to read a version from."""
try:
config = load_platform_config(platform_name, "platforms")
config = load_platform_config(platform_name, str(siteinputs.current().platforms))
except FileNotFoundError:
return ""
pack_name = release_record.pack_name_for(platform_name, record or {})
@@ -714,8 +715,7 @@ NO_ZIP_FAQ = "wiki/faq.md#why-is-there-no-zip-for-my-setup"
def has_release_pack(platform_name: str) -> bool:
"""Whether a release publishes a ZIP for the platform (`release_pack`)."""
entry = load_platform_registry("platforms").get(platform_name) or {}
return entry.get("release_pack") is True
return platform_name in _release_pack_platforms()
def no_zip_line(display: str) -> str:
@@ -734,7 +734,7 @@ def get_page_title(platform_name: str, display: str) -> str:
the system's own download. A setup that is also an emulator (BizHawk)
names it last: "BizHawk BIOS files" is the title of its emulator page."""
kind = "BIOS pack" if has_release_pack(platform_name) else "BIOS files"
if (_REPO_ROOT / "emulators" / f"{platform_name}.yml").is_file():
if (siteinputs.current().emulators / f"{platform_name}.yml").is_file():
return f"{kind} for {display}"
return f"{display} {kind}"
@@ -779,7 +779,7 @@ def _pack_buttons(platform_name: str, label: str = "Pack", prefix: str = "../",
frontend user asks. A setup with no release pack gets the line that
says so, and why."""
if not has_release_pack(platform_name):
display = display or load_platform_config(platform_name, "platforms").get(
display = display or load_platform_config(platform_name, str(siteinputs.current().platforms)).get(
"platform", platform_name)
return f"{no_zip_line(display)} [Why no ZIP?]({prefix}{NO_ZIP_FAQ})"
record = release_record.load_advertised()
@@ -804,7 +804,7 @@ def generate_get_page(platform_name: str) -> str:
the files of the pack the current release holds, read from release.json,
or the command when no release publishes one. The release process
never touches it: committing release.json rebuilds it."""
config = load_platform_config(platform_name, "platforms")
config = load_platform_config(platform_name, str(siteinputs.current().platforms))
display = config.get("platform", platform_name)
record = release_record.load_advertised()
pack = release_record.pack_for(platform_name, record)
@@ -823,7 +823,7 @@ def generate_get_page(platform_name: str) -> str:
if released and pack and pack.get("assets"):
parts = len(pack["assets"])
release = release_line(platform_name, record)
extract_to = (load_platform_registry("platforms").get(platform_name) or {}).get("extract_to")
extract_to = (load_platform_registry(str(siteinputs.current().platforms)).get(platform_name) or {}).get("extract_to")
lines += [
"This address stays the same. It always gives the latest ZIP.",
"",
@@ -872,7 +872,7 @@ def generate_get_page(platform_name: str) -> str:
def generate_get_index() -> str:
"""get/: the permanent page of every setup, grouped as the home tiles
are. A link trimmed to /get/ answered 404."""
registry = load_platform_registry("platforms")
registry = load_platform_registry(str(siteinputs.current().platforms))
lines = [
"---", "search: exclude",
'description: "The BIOS files of every setup, one page each. Each address stays the same."',
@@ -886,7 +886,7 @@ def generate_get_index() -> str:
continue
lines += [f"## {label}", ""]
for name in names:
display = load_platform_config(name, "platforms").get("platform", name)
display = load_platform_config(name, str(siteinputs.current().platforms)).get("platform", name)
lines.append(f"- [{get_page_title(name, display)}]({name}.md)")
lines.append("")
return "\n".join(lines)
@@ -1067,10 +1067,13 @@ def _split_contract(config: dict) -> tuple[dict, dict]:
return contract, registry
@functools.lru_cache(maxsize=1)
def _release_pack_platforms() -> frozenset[str]:
return frozenset(list_release_pack_platforms(
str(_REPO_ROOT / "platforms"), include_archived=True))
return _release_packs_in(str(siteinputs.current().platforms))
@functools.lru_cache(maxsize=None)
def _release_packs_in(platforms_dir: str) -> frozenset[str]:
return frozenset(list_release_pack_platforms(platforms_dir, include_archived=True))
# Per-entry fields a folder check needs beyond the hashes: an md5 is a ZIP
@@ -1617,6 +1620,7 @@ def generate_data_exports(
members = data_directory_members()
locations = file_location_items(db, profiles, members)
wanted = _provenance_wanted_rows(db)
envelopes = {
"platforms.json": ("platforms", platform_items),
"emulators.json": ("emulators", emulator_items),
@@ -1625,7 +1629,7 @@ def generate_data_exports(
}
write_lookup_shards(
api, file_lookup_records(db, coverages, profiles, locations, members, consumers,
retired_files.load(_REPO_ROOT / "retired.json")),
retired_files.load(_REPO_ROOT / "retired.json"), wanted),
generated_at, commit,
)
for filename, (kind, items) in envelopes.items():
@@ -1744,7 +1748,7 @@ def generate_data_exports(
_record(downloads / "retrobios.sqlite")
write_if_changed(
str(downloads / "provenance-wanted.csv"),
_csv_document(list(WANTED_FIELDS), _provenance_wanted_rows(db)),
_csv_document(list(WANTED_FIELDS), wanted),
)
assets = [
@@ -2104,7 +2108,7 @@ def _short_sha256(item: dict, catalog: str) -> str:
def _platforms_without_a_list(platforms_dir: Path | None = None) -> list[str]:
"""The names of the platforms whose registry says they keep no list."""
platforms_dir = platforms_dir or _REPO_ROOT / "platforms"
platforms_dir = platforms_dir or siteinputs.current().platforms
names = []
for key, entry in sorted(load_platform_registry(str(platforms_dir)).items()):
if entry.get("native_list") != "none":
@@ -3236,7 +3240,9 @@ def _front_matter(markdown: str) -> dict:
return data if isinstance(data, dict) else {}
_SHA1_BY_PREFIX: dict[str, str] = {}
@functools.lru_cache(maxsize=None)
def _sha1_by_prefix(db_path: str) -> dict[str, str]:
return {sha1[:12]: sha1 for sha1 in load_database(db_path)["files"]}
def _cards_elsewhere(markdown: str) -> str:
@@ -3247,11 +3253,9 @@ def _cards_elsewhere(markdown: str) -> str:
away = {ident for ident in links if f'id="{ident}"' not in markdown}
if not away:
return markdown
if not _SHA1_BY_PREFIX:
db = load_database(str(_REPO_ROOT / "database.json"))
_SHA1_BY_PREFIX.update({sha1[:12]: sha1 for sha1 in db["files"]})
by_prefix = _sha1_by_prefix(str(siteinputs.current().db))
for ident in away:
full = _SHA1_BY_PREFIX[ident.removeprefix("sha1-")]
full = by_prefix[ident.removeprefix("sha1-")]
markdown = markdown.replace(f"](#{ident})", f"]({markup.site_href(f'files/?sha1={full}')})")
return markdown
@@ -3562,10 +3566,14 @@ def _list_pages(
f"[part {i}]({other}.md)" for i, other in enumerate(stems, 1))
@functools.lru_cache(maxsize=1)
def _collection_sha1_by_path() -> dict[str, str]:
"""The sha1 of each file the collection holds, by its path."""
with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle:
return _sha1_by_path_in(str(siteinputs.current().db))
@functools.lru_cache(maxsize=None)
def _sha1_by_path_in(db_path: str) -> dict[str, str]:
with open(db_path, encoding="utf-8") as handle:
files = json.load(handle).get("files", {})
return {str(entry.get("path", "")): sha1 for sha1, entry in files.items()}
@@ -3595,7 +3603,7 @@ def _system_named_link(sys_id: str, prefix: str = "") -> str:
def _emulator_named_link(key: str, prefix: str = "") -> str:
"""An emulator by the name its page carries, linked to that page. A
profile with no page (an alias, a test) is named only."""
profile = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False).get(key, {})
profile = _profiles_on_disk().get(key, {})
label = _escape_tags(str(profile.get("emulator") or key))
if profile.get("type") in ("alias", "test"):
return label
@@ -3985,10 +3993,9 @@ def _platform_data_links(name: str) -> str:
return ", ".join(links)
@functools.lru_cache(maxsize=1)
def _site_profiles() -> dict[str, dict]:
"""Every emulator profile, read once for the citations of a page."""
return load_emulator_profiles(str(_REPO_ROOT / "emulators"))
return _profiles_in(str(siteinputs.current().emulators), skip_aliases=True)
# Whose code a source reference cites, in one word: a platform page names
@@ -4072,13 +4079,17 @@ def _category_evidence_md(evidence: dict, launched: list[str] | None = None) ->
return f"No reader found ({cores})"
@functools.lru_cache(maxsize=1)
def _profiles_by_core_name() -> dict[str, list[str]]:
"""Core name as a platform writes it -> the profiles with a page for it.
MinUI writes `mednafen_vb`, the profile and its page are `beetle_vb`.
"""
profiles = load_emulator_profiles(str(_REPO_ROOT / "emulators"))
return _core_names_in(str(siteinputs.current().emulators))
@functools.lru_cache(maxsize=None)
def _core_names_in(emulators_dir: str) -> dict[str, list[str]]:
profiles = _profiles_in(emulators_dir, skip_aliases=True)
with_page = {k for k, v in profiles.items() if v.get("type") not in ("alias", "test")}
return {
core: sorted(k for k in keys if k in with_page)
@@ -4248,7 +4259,7 @@ def _card_route(name: str, display: str, guide: Guide) -> list[str]:
sees it is the reader's to name: no guide cites it."""
if guide.on_card or _offline_line(name):
return []
extract_to = (load_platform_registry("platforms").get(name) or {}).get("extract_to")
extract_to = (load_platform_registry(str(siteinputs.current().platforms)).get(name) or {}).get("extract_to")
leaf = (extract_to or "bios").rstrip("/").rsplit("/", 1)[-1].lstrip("~") or "bios"
tabs = guide.card_run or ("windows", "macos", "linux")
args = {key: f"--platform {name} --dest "
@@ -4449,7 +4460,7 @@ def generate_handheld_guide(guide: HandheldGuide) -> str:
A handheld owner starts from the device name. The systems were named
only in a closed note, with a path seen from inside the device."""
uses = GUIDES[guide.uses]
display = load_platform_config(guide.uses, "platforms").get("platform", guide.uses)
display = load_platform_config(guide.uses, str(siteinputs.current().platforms)).get("platform", guide.uses)
args = {"windows": f"--platform {guide.uses} --dest E:\\BIOS",
"macos": f"--platform {guide.uses} --dest /path/to/card/BIOS",
"linux": f"--platform {guide.uses} --dest /path/to/card/BIOS"}
@@ -4504,7 +4515,7 @@ def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list:
lines.append("")
for note in guide.notes:
lines += [note, ""]
lines += [line for note in extract_notes("platforms", released_only=True)
lines += [line for note in extract_notes(str(siteinputs.current().platforms), released_only=True)
if f"The {display} pack " in note
for line in (note, "")]
if guide.more_folders:
@@ -4979,7 +4990,7 @@ def _data_member_digests(
from hashing import compute_hashes # noqa: PLC0415
if data_registry is None:
data_registry = load_data_dir_registry(str(root / "platforms"))
data_registry = load_data_dir_registry(str(siteinputs.current().platforms))
found: dict[str, dict[str, str]] = {}
for ref, tree in sorted(members.items()):
cache = (data_registry.get(ref) or {}).get("local_cache")
@@ -5062,12 +5073,14 @@ def file_lookup_records(
members: dict | None = None,
consumers: FileConsumers | None = None,
retired: dict | None = None,
wanted: list[dict] | None = None,
) -> list[dict]:
"""Every hash the site knows, one record per value, for files/?hash=.
The collection's own files lead to their card, a platform's declared
value to the system section of its page, a profile's to the entry's
card. The site search indexed these values as text, 4 MB of the 21 MB
card, a wanted dump (``wanted``, the provenance-wanted.csv rows) to
its catalog on the provenance page. The site search indexed these values as text, 4 MB of the 21 MB
it made every visitor download: the lookup reads one shard instead.
"""
files = db.get("files", {})
@@ -5140,6 +5153,7 @@ def file_lookup_records(
**({"system_name": entry_label} if entry_label else {}),
})
records += retired_records(retired or {}, files, locations)
records += wanted_records(wanted or [])
records += _held_under_every_hash(records, files, db.get("indexes") or {})
for record in records:
if record["layer"] in ("platform", "emulator"):
@@ -5151,6 +5165,23 @@ def file_lookup_records(
r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"]))
def wanted_records(rows: list[dict]) -> list[dict]:
"""A record under each hash of a dump a catalog lists and the collection
lacks, from the rows of provenance-wanted.csv. A pasted hash of a
wanted dump (Redump's ps2-dvd-212k.bin) answered that nothing the site
knows has it, and never said the dump is wanted."""
records = []
for row in rows:
label = _PROVENANCE_LABELS.get(row["catalog"], row["catalog"])
url = _site_path(f"provenance.md#{_slugify_anchor(label)}")
for algorithm in ("sha1", "md5", "crc32"):
value = str(row.get(algorithm) or "").lower()
if value:
records.append({"value": value, "algorithm": algorithm, "layer": "wanted",
"owner": label, "name": row["name"], "url": url})
return records
def _read_by_name(records: list[dict], files: dict, consumers: FileConsumers) -> list[dict]:
"""The emulators the file's card names that no declared hash reached.
@@ -5836,6 +5867,13 @@ def _wanted_list(source: str, label: str, data: dict, by_dat: dict[str, list],
_RELEASE_ASSET_NAMES: dict[str, str] | None = None
def _download_name(path: str) -> str:
"""The name a browser saves a collection file under: the last segment
of its download URL. Neither host sends a Content-Disposition, and the
download attribute is ignored on another origin."""
return urllib.parse.unquote(_fetch_url(path).rsplit("/", 1)[-1])
def _fetch_url(path: str) -> str:
"""Where a collection file downloads from: its raw URL, or the
large-files release asset when .gitignore keeps it out of git."""
@@ -5867,7 +5905,7 @@ def _platform_display(name: str) -> str:
display = _platform_names.get(name)
if display is None:
try:
config = load_platform_config(name, str(_REPO_ROOT / "platforms"))
config = load_platform_config(name, str(siteinputs.current().platforms))
except FileNotFoundError:
config = {}
display = _platform_names[name] = str(config.get("platform") or name)
@@ -6210,6 +6248,12 @@ def _system_file_card(
# Each fact: its label, its values, whether the search reads it. What a
# setup does with the file comes first, the fingerprints last.
facts: list[tuple[str, list[str], bool]] = []
saved = _download_name(f["path"]) if f.get("path") else name
if saved != name:
# A .variants/ copy is stored as advision.zip.1c086064, and the
# browser names the download after the URL: copied as it came, the
# file a platform pins was not found under its name.
facts.append(("Save as", [f"{_code(name)}, the download is named {_code(saved)}"], False))
if plats:
# Each link lands on the platform's system section, and says where
# that platform puts the file. Out of the search: the card's title
@@ -6268,7 +6312,7 @@ def _system_file_card(
# What a system card shows at a glance: who lists the file and who reads it.
# Its fingerprints and the rest open on demand, so a page of 250 cards reads
# as a list of files, not a wall of hashes.
_CARD_KEY_FACTS = ("On the setup's list", "Packed for its emulators by", "Emulators")
_CARD_KEY_FACTS = ("Save as", "On the setup's list", "Packed for its emulators by", "Emulators")
_CARD_HASHES = ("SHA1", "MD5", "CRC32", "SHA256")
# The words a card's fold names its other facts by.
_CARD_MORE_WORDS = {"Romset": "romset", "Collection path": "collection path",
@@ -6664,11 +6708,13 @@ def _reading_pairs(sha1: str, consumers: FileConsumers) -> list[tuple[str, dict]
return pairs
@functools.lru_cache(maxsize=1)
def _profiles_on_disk() -> dict[str, dict]:
from common import load_emulator_profiles # noqa: PLC0415
return _profiles_in(str(siteinputs.current().emulators), skip_aliases=False)
return load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False)
@functools.lru_cache(maxsize=None)
def _profiles_in(emulators_dir: str, skip_aliases: bool) -> dict[str, dict]:
return load_emulator_profiles(emulators_dir, skip_aliases=skip_aliases)
def _entry_systems(emulator: str, entry: dict) -> list[str]:
@@ -8212,6 +8258,26 @@ def _exact_size(size: int) -> str:
return _fmt_size(size) if size < 1024 else f"{_fmt_size(size)} ({size:,} bytes)"
def _build_file_links(pin_build: list[dict]) -> list[str]:
"""Each platform build file that holds the revision, linked at the
commit it was read at when the source record holds it. The prose of
pin_frozen named the file as plain text."""
from sitesources import _browse_repo, _source_permalink, tree_holds # noqa: PLC0415
links = []
for build in pin_build:
repo = _browse_repo(str(build["repo"]))
path = str(build["path"])
revision = str(build["revision"])
text = f"{_escape_tags(_platform_display(str(build['platform'])))} {_code(path)}"
# Linked where the record saw the file at that commit, as every
# source link is: a tag can move, and is never recorded.
if repo is not None and tree_holds(repo, revision, path):
text = f"[{text}]({_source_permalink(repo, revision, path, None, None)})"
links.append(f"{text} at {_code(str(build['revision']))} sets {_code(str(build['variable']))}")
return links
def _build_rows(profile: dict) -> list[str]:
"""Meta card rows for who ships the build, how its files are found, why
its revision is held and where its releases are announced."""
@@ -8232,6 +8298,9 @@ def _build_rows(profile: dict) -> list[str]:
if frozen:
held = _escape_tags(' '.join(str(frozen).split())).replace('|', r'\|')
rows.append(f"| {label} | {held} |")
builds = _build_file_links(profile.get("pin_build") or [])
if builds:
rows.append(f"| Build file | {'<br>'.join(builds)} |")
announced = profile.get("version_source")
if isinstance(announced, dict) and announced.get("url"):
url = str(announced["url"])
@@ -8483,12 +8552,7 @@ def _render_emulator_file(
if fnote:
lines.append(f"- {data_root_words(_escape_tags(link_prose_citations(profile, str(fnote))), data_href)}")
if not in_repo and f.get("unsourceable"):
# The badge alone sent readers after a dump that cannot exist, and the
# reason was only on the gaps page, which the card did not link.
lines.append(
f"- Not collectable: {_escape_tags(str(f['unsourceable']))} "
"([acknowledged gaps](../gaps.md#acknowledged-gaps))"
)
lines.append(f"- {_unsourceable_line(f)}")
lines.extend(_contents_lines(contents))
lines[body_start:] = _search_skipped_apart(lines[body_start:])
lines.append("")
@@ -8600,12 +8664,7 @@ def _emulator_card_folds(
open_lines.append(data_root_words(
_escape_tags(link_prose_citations(profile, str(fnote))), data_href))
if not in_repo and f.get("unsourceable"):
# The badge alone sent readers after a dump that cannot exist, and the
# reason was only on the gaps page, which the card did not link.
open_lines.append(
f"Not collectable: {_escape_tags(str(f['unsourceable']))} "
"([acknowledged gaps](../gaps.md#acknowledged-gaps))"
)
open_lines.append(_unsourceable_line(f))
lines = [f"- {d}" for d in open_lines]
shut_fold = [f"- {d}" for d in hashes + [m for m in more if skipped(m)]]
read_fold = [f"- {m}" for m in more if not skipped(m)]
@@ -8988,10 +9047,14 @@ def emulator_install_route(name: str, profile: dict) -> tuple[str, list[str], tu
return command, setups, oses
@functools.cache
def _standalone_cores(platform: str) -> frozenset[str]:
"""The emulators a setup launches as apps, by its platform file."""
config = load_platform_config(platform, str(_REPO_ROOT / "platforms"))
return _standalone_cores_in(str(siteinputs.current().platforms), platform)
@functools.cache
def _standalone_cores_in(platforms_dir: str, platform: str) -> frozenset[str]:
config = load_platform_config(platform, platforms_dir)
return frozenset(str(c) for c in config.get("standalone_cores") or [])
@@ -9478,7 +9541,7 @@ def _hash_spans(value: str | list, copy: bool = False) -> str:
def _availability_check(
db: dict, data_names: set[str] | None
db: dict, data_names: set[str] | None, owner: str = "",
) -> Callable[[dict], bool]:
"""A predicate answering whether the collection holds a file.
@@ -9509,7 +9572,7 @@ def _availability_check(
# under the name belongs to another game or release.
if not f.get("name") or f.get("unsourceable"):
return False
return entry_source(f, index) is not None
return entry_source({**f, "source_profile": owner} if owner else f, index) is not None
return _file_available
@@ -10242,7 +10305,7 @@ def emulator_pages(
lines.append("")
lines.extend(markup.aside("info", "Why no files", [_escape_tags(str(exclusion))])[:-1])
else:
_file_available = _availability_check(db, data_names)
_file_available = _availability_check(db, data_names, name)
lines.extend(["## Files", ""])
lines.extend(_emulator_file_summary(files))
@@ -10505,6 +10568,33 @@ def _hash_cell(hashes: dict[str, list[str]]) -> str:
)
def _unsourceable_line(f: dict) -> str:
"""Why the collection holds no copy of a file a profile marks
`unsourceable:`, linked to the list of them. The flag only keeps a
same-named file from being served in its place: the file is still
wanted, and "Not collectable" sent readers away from KANJI2.ROM, which
MAME's fm7740sx.zip carries."""
return (f"Not collected yet: {_escape_tags(str(f['unsourceable']))} "
"([files not collected yet](../gaps.md#not-collected-yet))")
def _catalogued_needs(report: dict | None) -> dict[tuple[str, str], list[str]]:
"""(profile, file name casefolded) -> where a dump catalog or an
imported romset lists that file, from a provenance report built with
the profiles' needs."""
out: dict[tuple[str, str], list[str]] = {}
for source, data in sorted((report or {}).items()):
label = _PROVENANCE_LABELS.get(source, source)
for entry in data.get("missing", []):
where = [f"{label} `{_escape_tags(str(entry.get('name', '')))}`"]
where += [f"{_escape_tags(str(hit['source']))} `{_escape_tags(str(hit['archive']))}`"
for hit in entry.get("romsets", [])]
for need in entry.get("needed_by", []):
held = out.setdefault((need["emulator"], str(need["name"]).casefold()), [])
held += [w for w in where if w not in held]
return out
def _reason_cell(text: str) -> str:
"""A reason in a gaps table, in the class that keeps its column readable."""
return f'<span class="rb-reason">{text}</span>' if text else "-"
@@ -10799,9 +10889,14 @@ def generate_gap_analysis(
registry: dict | None = None,
gap_report: dict | None = None,
moved: dict[str, str] | None = None,
catalogued: dict | None = None,
) -> str:
"""Generate a unified gap analysis page.
``catalogued`` is a provenance report built with the profiles' needs:
a file the collection does not hold yet names the catalog or romset
that lists it.
With ``moved``, the list of files to source goes to pages of its own,
keyed by stem: on the page it made 662 KB of HTML.
@@ -11232,7 +11327,7 @@ def generate_gap_analysis(
"",
])
# ---- Section 4: Acknowledged gaps (unsourceable files) ----
# ---- Section 4: files the collection does not hold yet (unsourceable) ----
all_unsourceable: list[dict] = []
for emu_name, data in sorted(report_all.items()):
@@ -11246,19 +11341,26 @@ def generate_gap_analysis(
})
if all_unsourceable:
listed = _catalogued_needs(catalogued)
# The old heading's address still leads here.
lines.extend([
"## Acknowledged gaps",
'<span id="acknowledged-gaps"></span>',
"",
f"{len(all_unsourceable)} files documented as unsourceable "
"(verified from source code).",
"## Not collected yet",
"",
"| File | Emulator | Reason | Source ref |",
"|------|----------|--------|-----------|",
f"{len(all_unsourceable)} files the collection does not hold. A file "
"with the same name is never served in their place. Each is still "
"wanted, and the last column names a catalog or romset that lists it.",
"",
"| File | Emulator | Reason | Source ref | Listed in |",
"|------|----------|--------|-----------|-----------|",
])
for u in sorted(all_unsourceable, key=lambda x: x["name"]):
where = listed.get((u["emu_key"], str(u["name"]).casefold()), [])
lines.append(
f"| `{u['name']}` | {u['emulator']} | {_reason_cell(u['reason'])} "
f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} |"
f"| {_source_ref_markdown(profiles[u['emu_key']], u['source_ref'])} "
f"| {', '.join(where) or '-'} |"
)
lines.append("")
@@ -11844,7 +11946,7 @@ def _own_root_folders(registry: dict) -> dict[str, str]:
destination carries its root in every path."""
folders = {}
for key, entry in registry.items():
config = load_platform_config(key, "platforms")
config = load_platform_config(key, str(siteinputs.current().platforms))
if config.get("base_destination") == "" and entry.get("extract_to"):
folders[key] = entry["extract_to"]
return folders
@@ -11944,9 +12046,10 @@ def generate_which_pack() -> str:
The tiles are the choice. Open tables under them listed every setup a
second time and made the page 11,800 px tall on a phone."""
registry = load_platform_registry("platforms")
registry = load_platform_registry(str(siteinputs.current().platforms))
record = release_record.load_advertised()
display = {key: load_platform_config(key, "platforms").get("platform", key) for key in registry}
display = {key: load_platform_config(key, str(siteinputs.current().platforms)).get("platform", key)
for key in registry}
all_tabs = ("windows", "macos", "linux", "steamdeck", "android")
lines = [
"---",
@@ -12322,6 +12425,12 @@ def main() -> None:
def _generate(args: argparse.Namespace) -> None:
# Every page reads these, never the repository's own files.
siteinputs.use(siteinputs.SiteInputs(
db=Path(args.db).resolve(),
emulators=Path(args.emulators_dir).resolve(),
platforms=Path(args.platforms_dir).resolve(),
))
db = load_database(args.db)
docs = Path(args.docs_dir)
# Files the site serves as they are (data, downloads, images), in the
@@ -12602,12 +12711,16 @@ def _generate(args: argparse.Namespace) -> None:
sitestarlight.MOVED_PAGES.setdefault(
f"cross-reference-{pname}.md", f"platforms/{pname}.md")
provenance_report = build_report(
db, load_provenance_snapshots(), needs=profile_needs(profiles, db))
print("Generating gap analysis page...")
gap_pages: dict[str, str] = {}
write_if_changed(
str(docs / "gaps.md"),
tables_out_of_search(generate_gap_analysis(
profiles, coverages, db, suppl_names, registry, gap_report, gap_pages,
catalogued=provenance_report,
)) + reference_next(),
)
for stem, page in gap_pages.items():
@@ -12623,7 +12736,6 @@ def _generate(args: argparse.Namespace) -> None:
)
print("Generating dump provenance page...")
provenance_report = build_report(db, load_provenance_snapshots())
provenance_pages: dict[str, str] = {}
write_if_changed(
str(docs / "provenance.md"),
scripts/generate_truth.py
+15 -12
View File
@@ -14,13 +14,13 @@ import sys
sys.path.insert(0, os.path.dirname(__file__))
from common import (
build_target_cores_cache,
platform_arg,
list_registered_platforms,
load_data_dir_registry,
load_database,
load_emulator_profiles,
load_platform_config,
load_target_config,
require_directory,
require_platform,
require_yaml,
@@ -97,6 +97,19 @@ def main(argv: list[str] | None = None) -> None:
require_platform(args.platform, args.platforms_dir)
platforms = [args.platform]
# A named platform without that target is refused, as verify and
# generate_pack refuse it: skipped, it printed Done. and exited 0 with
# no model written. --all leaves such platforms out and says so.
target_cores_cache: dict[str, set[str] | None] = {}
if args.target:
try:
target_cores_cache, platforms = build_target_cores_cache(
platforms, args.target, args.platforms_dir, is_all=args.all
)
except (FileNotFoundError, ValueError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
sys.exit(1)
# A target-narrowed model is a different artifact: written under the
# platform's own name it overwrote the full one, and diff_truth then
# compared a narrowed model against the whole scrape. Its directory is
@@ -109,17 +122,7 @@ def main(argv: list[str] | None = None) -> None:
output_dir, [name], args.target, args.platforms_dir
)
os.makedirs(output_dir, exist_ok=True)
target_cores: set[str] | None = None
if args.target:
try:
target_cores = load_target_config(
name,
args.target,
args.platforms_dir,
)
except FileNotFoundError:
print(f" {name}: no target config, skipped")
continue
target_cores = target_cores_cache.get(name)
# Load platform config (with inheritance) and registry entry
try:
scripts/largefiles.py
+73 -11
View File
@@ -12,6 +12,7 @@ import os
import sys
import re
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
@@ -26,6 +27,13 @@ from hashing import compute_hashes
LARGE_FILES_RELEASE = "large-files"
LARGE_FILES_REPO = "Abdess/retrobios"
LARGE_FILES_CACHE = ".cache/large"
# Verified revisions, hard-linked as <sha1>/<asset> beside the cache. A name
# in the cache is replaced when the release serves new bytes under it, so a
# path handed out by name could hold other bytes by the time the caller read
# them. Nothing ever writes other bytes under a sha1. The asset name stays
# last: callers read the extension (an archive is packed as one).
VERIFIED_STORE = ".by-sha1"
ORPHAN_GRACE = 24 * 3600
GITIGNORE = Path(__file__).resolve().parent.parent / ".gitignore"
_UNSAFE_ASSET_CHARS = re.compile(r"[^A-Za-z0-9._-]+")
@@ -169,14 +177,14 @@ def _fetch_asset(
if os.path.isdir(cached):
return None
if os.path.exists(cached) and not (expected_sha1 or expected_md5):
return cached
if os.path.exists(cached):
snapshot = _read_once(cached)
if snapshot is None:
pass
elif _matches(snapshot[0], expected_sha1, expected_md5):
return cached
pinned = _pin_verified(cached, snapshot, dest_dir, name)
if pinned:
return pinned
elif not _replaced_on_release(name, snapshot[0]["sha256"], snapshot[1], offline):
# A copy of this asset that answers another hash: the caller
# wants a different revision under the same name, and the
@@ -220,17 +228,71 @@ def _fetch_asset(
print(f" large file {candidate}: {exc}", file=sys.stderr)
if not downloaded:
return None
if (expected_sha1 or expected_md5) and not _matches(
compute_hashes(tmp_path), expected_sha1, expected_md5
):
snapshot = _read_once(tmp_path)
if snapshot is None or not _matches(snapshot[0], expected_sha1, expected_md5):
return None
set_plain_mode(tmp_path)
pinned = _pin_verified(tmp_path, snapshot, dest_dir, name)
os.replace(tmp_path, cached)
return cached
_prune_verified(dest_dir)
return pinned
finally:
_drop(tmp_path)
def _pin_verified(
path: str, snapshot: tuple[dict[str, str], int, int], dest_dir: str, name: str,
) -> str | None:
"""The verified revision at *path*, as <sha1>/<name> in VERIFIED_STORE.
*snapshot* is what _read_once read: the link is kept only if it holds
that inode, so a copy another run promoted after the hash is never
passed off as the one verified. None when the name moved in between.
"""
revision = os.path.join(dest_dir, VERIFIED_STORE, snapshot[0]["sha1"])
os.makedirs(revision, exist_ok=True)
final = os.path.join(revision, name)
if os.path.isfile(final):
return final
scratch = os.path.join(revision, f".{os.getpid()}.{os.urandom(4).hex()}.tmp")
try:
os.link(path, scratch)
except FileNotFoundError:
return None
try:
if os.stat(scratch).st_ino != snapshot[2]:
return None
os.replace(scratch, final)
return final
finally:
with contextlib.suppress(FileNotFoundError):
os.unlink(scratch)
def _prune_verified(dest_dir: str) -> None:
"""Drop the revisions no cache name has held for ORPHAN_GRACE seconds.
Losing its cache name changes the inode's ctime, so a revision another
run pinned just before the release replaced it stays readable while that
run packs it.
"""
store = os.path.join(dest_dir, VERIFIED_STORE)
now = time.time()
for revision in os.scandir(store):
if not revision.is_dir():
continue
for entry in os.scandir(revision.path):
if entry.name.endswith(".tmp"):
continue
with contextlib.suppress(FileNotFoundError):
stat = entry.stat()
if stat.st_nlink == 1 and now - stat.st_ctime > ORPHAN_GRACE:
os.unlink(entry.path)
# Another run may be linking a copy into it: left for the next prune.
with contextlib.suppress(FileNotFoundError, OSError):
os.rmdir(revision.path)
def _asset_urls(name: str) -> list[tuple[str, str]]:
"""Release URLs an asset may be published under, with their names.
@@ -252,9 +314,9 @@ def _asset_urls(name: str) -> list[tuple[str, str]]:
]
def _read_once(path: str) -> tuple[dict[str, str], int] | None:
"""Digests and size of the file at *path*, all read from one file, or
None when it is gone.
def _read_once(path: str) -> tuple[dict[str, str], int, int] | None:
"""Digests, size and inode of the file at *path*, all read from one
file, or None when it is gone.
Every judgement on a cache copy reads this one snapshot. Hashed, then
hashed again by path for the release check, a copy another run promoted
@@ -271,7 +333,7 @@ def _read_once(path: str) -> tuple[dict[str, str], int] | None:
except FileNotFoundError:
return None
if (before.st_ino, before.st_mtime_ns) == (after.st_ino, after.st_mtime_ns):
return hashes, after.st_size
return hashes, after.st_size, after.st_ino
return None
scripts/manifestselect.py
+10 -2
View File
@@ -19,7 +19,13 @@ from dataclasses import dataclass
import region as region_mod
from common import _norm_system_id, sanitize_pack_path
from packextras import _kept, _serving_system_ids, extra_region_groups, system_spellings
from packextras import (
_kept,
_serving_system_ids,
extra_region_groups,
extra_region_key,
system_spellings,
)
def _dest(entry: dict) -> str:
@@ -44,6 +50,7 @@ class SelectionIndex:
pack_systems: dict[str, dict],
extras: list[dict],
required_only: bool = False,
extras_prefix: str = "",
) -> SelectionIndex:
by_norm: dict[str, set[str]] = {}
for sys_id in pack_systems:
@@ -60,7 +67,8 @@ class SelectionIndex:
extra_groups: dict[str, set[str]] = {}
spellings = system_spellings(pack_systems)
for extra in extras:
dest = _dest(extra)
# Keyed as platform_region_groups keys it (extra_region_key).
dest = extra_region_key(extra, extras_prefix)
if not dest:
continue
extra_groups.setdefault(dest, set()).update(
scripts/packextras.py
+18 -2
View File
@@ -62,6 +62,20 @@ def prefixed_destination(dest: str, prefix: str) -> str:
return f"{prefix}/{dest}"
def extra_region_key(fe: dict, prefix: str) -> str:
"""The key a core extra is grouped, dropped and held under by --region.
Its destination with the BIOS directory the platform writes in front of
its own (declared_root): RetroDECK declares bios/scph5501.bin, and
Beetle PSX's scph5501.bin grouped bare was looked up under another key,
kept by the pack while the declaration of the same file was dropped,
and the installer, which records the path once, dropped it.
"""
return prefixed_destination(
sanitize_pack_path(fe.get("destination", fe.get("name", ""))), prefix
)
def save_entry_fields(fe: dict) -> dict:
"""What a manifest entry adds for a file read below the save directory.
@@ -364,6 +378,7 @@ def hidden_extras(
content[dest.lower()] = sha1_by_path.get(local)
lower = set(content) if case_insensitive else set()
prefix = _detect_extras_prefix(config, base_dest)
region_prefix = declared_root(config)
hidden: dict[tuple[str, str, str], str] = {}
packed_keys: set[tuple[str, str, str]] = set()
extras = _collect_emulator_extras(
@@ -377,7 +392,7 @@ def hidden_extras(
if required_only and fe.get("required") is False:
continue
dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
if not dest or (region_drops and dest in region_drops):
if not dest or (region_drops and extra_region_key(fe, region_prefix) in region_drops):
continue
full = extra_full_destination(fe, prefix)
if not full:
@@ -1705,8 +1720,9 @@ def platform_region_groups(
)
held = {id(extra) for extra in extras}
spellings = system_spellings(systems)
region_prefix = declared_root(config)
for extra in _kept(extras + absent, required_only):
dest = sanitize_pack_path(extra.get("destination", extra.get("name", "")))
dest = extra_region_key(extra, region_prefix)
if not dest:
continue
name = extra.get("name", "")
scripts/packreadme.py
+22 -3
View File
@@ -207,6 +207,7 @@ def _build_readme(
save_folder: str = "",
save_where: str = "",
save_by_system: bool = False,
case_twins: list[tuple[str, str]] | None = None,
) -> str:
"""Build a personalized step-by-step README for each platform pack.
@@ -216,6 +217,10 @@ def _build_readme(
frontend's save directory, empty when the pack holds none, and
``save_where`` that directory's default place (save_place).
``save_by_system`` is set when the launcher hands each system a folder.
``case_twins`` pairs each spelling the pack leaves out, a case twin of a
packed file, with the one it holds: given for a platform that also runs
where names keep their case (EmuDeck and RetroArch on Linux), where the
core asking for the other spelling finds nothing.
"""
narrowings = narrowings or []
sep = "=" * 50
@@ -245,6 +250,20 @@ def _build_readme(
+ (pack_manual(platform_name) or _generic_manual(bios_paths))
)
if case_twins:
listed = "".join(
f" {missing} (the pack has {held})\n" for missing, held in case_twins
)
guide += (
"NAMES THAT DIFFER ONLY BY CASE\n\n"
" Some cores open the same file under another spelling. This\n"
" pack holds one spelling of each, all Windows and macOS can\n"
" store in one folder. Linux, SteamOS and Android keep both:\n"
" there, run the installer, which adds these spellings, or\n"
" copy each file under the name below.\n\n"
f"{listed}\n"
)
if save_folder:
guide += _save_section(
platform_display, save_folder, save_where, save_by_system,
@@ -300,8 +319,8 @@ def _build_readme(
if fallback_systems:
listed = "\n".join(f" {_group_label(s)}" for s in fallback_systems)
region_info += (
"\n These systems have no BIOS in those regions, so all\n"
" of theirs were kept:\n"
"\n These systems have no BIOS in those regions, so every\n"
" BIOS file for them was kept:\n"
f"{listed}\n"
)
region_info += (
@@ -316,7 +335,7 @@ def _build_readme(
slot_info = (
" Where a core declares which BIOS it prefers, only that one was\n"
" kept for each system and region. Systems whose cores declare no\n"
" order keep all of theirs.\n"
" order keep every BIOS file.\n"
)
if undecidable_slots:
slot_info += (
scripts/pin_installer.py
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env python3
"""Write the SHA-256 of install.py into the bootstraps that pin it.
install.sh and install.ps1 refuse an installer whose hash is not the one
they carry, so every change to install.py moves both pins in the same
commit. --check writes nothing and exits 1 naming each stale pin.
Usage:
python scripts/pin_installer.py
python scripts/pin_installer.py --check
"""
from __future__ import annotations
import argparse
import hashlib
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
INSTALLER = "install.py"
# Each bootstrap and its one pin line, the value between the two groups.
PINS: dict[str, re.Pattern[str]] = {
"install.sh": re.compile(r'^(DEFAULT_INSTALL_SHA256=")[0-9a-f]{64}(")', re.M),
"install.ps1": re.compile(r'^(\s*\$defaultInstallSha256 = ")[0-9a-f]{64}(")', re.M),
}
COMMAND = "python scripts/pin_installer.py"
def installer_sha256(root: Path = ROOT) -> str:
return hashlib.sha256((root / INSTALLER).read_bytes()).hexdigest()
def stale_pins(root: Path = ROOT) -> list[str]:
"""The bootstraps whose pin is not the SHA-256 of install.py."""
digest = installer_sha256(root)
stale = []
for name, pin in PINS.items():
found = pin.search((root / name).read_bytes().decode("utf-8"))
if found is None:
raise ValueError(f"{name}: no installer pin line")
if found.group(0) != f"{found.group(1)}{digest}{found.group(2)}":
stale.append(name)
return stale
def write_pins(root: Path = ROOT) -> list[str]:
"""Rewrite each stale pin. Returns the bootstraps written."""
digest = installer_sha256(root)
written = []
for name in stale_pins(root):
path = root / name
text = path.read_bytes().decode("utf-8")
new, count = PINS[name].subn(rf"\g<1>{digest}\g<2>", text)
if count != 1:
raise ValueError(f"{name}: {count} installer pin lines, expected 1")
path.write_bytes(new.encode("utf-8"))
written.append(name)
return written
def main() -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
parser.add_argument("--check", action="store_true",
help="write nothing, exit 1 if a pin is stale")
args = parser.parse_args()
if args.check:
stale = stale_pins()
for name in stale:
print(f"{name}: pin is not the SHA-256 of {INSTALLER}, {COMMAND} writes it",
file=sys.stderr)
return 1 if stale else 0
for name in write_pins():
print(f"{name}: pinned {installer_sha256()}")
return 0
if __name__ == "__main__":
from common import run_cli
run_cli(main)
scripts/pipeline.py
+133 -10
View File
@@ -55,6 +55,7 @@ import subprocess
import sys
import tempfile
import time
from collections.abc import Callable
from pathlib import Path
# The build scripts need CPython 3.9, which builds the same database,
@@ -562,6 +563,7 @@ def check_emulator_packs(
comparison holds them together. No archive is written and no file read.
"""
import generate_pack # noqa: PLC0415
import packresolve # noqa: PLC0415
import verify # noqa: PLC0415
from common import ( # noqa: PLC0415
ProfileSelectionError,
@@ -580,16 +582,23 @@ def check_emulator_packs(
def compare(names: list[str], standalone: bool, label: str, **narrowing: object) -> bool:
record = generate_pack.PackRecord()
with contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()):
generate_pack.generate_emulator_pack(
names, emulators_dir, db, "bios", "", standalone, zip_contents,
offline=True, platforms_dir=platforms_dir, record=record, **narrowing,
)
report = verify.verify_emulator(
names, emulators_dir, db, standalone, platforms_dir=platforms_dir,
**narrowing,
)
was_offline = packresolve._OFFLINE
# The pack is built offline: verify counts a release asset the cache
# lacks the same way, missing, as the narrowed-pack step does.
packresolve.set_offline(True)
try:
with contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()):
generate_pack.generate_emulator_pack(
names, emulators_dir, db, "bios", "", standalone, zip_contents,
offline=True, platforms_dir=platforms_dir, record=record, **narrowing,
)
report = verify.verify_emulator(
names, emulators_dir, db, standalone, platforms_dir=platforms_dir,
**narrowing,
)
finally:
packresolve.set_offline(was_offline)
lines = _set_difference_lines(
label, set(report["pack_destinations"]), set(record.files)
)
@@ -802,6 +811,92 @@ def check_narrowed_packs(
return all_ok
def split_differences(
full: object, parts: dict[str, object], same_bytes: Callable[[str, str], bool],
) -> list[str]:
"""Where the --split parts of a pack, together, are not the full pack.
A path in a part and not in the full pack, a path of the full pack in no
part, and a path a part fills with other bytes than the full pack.
"""
full_members = {m for m in full.members if not _PACK_NOTES.search(m)}
union = {
m for record in parts.values() for m in record.members
if not _PACK_NOTES.search(m)
}
lines = [f"in a part, not in the full pack: {m}" for m in sorted(union - full_members)]
lines += [f"in the full pack, in no part: {m}" for m in sorted(full_members - union)]
lines += sorted({
f"other bytes in {name}: {member}"
for name, record in parts.items()
for member, source in record.sources.items()
if member in full.sources and not same_bytes(source, full.sources[member])
})
return lines
def check_split_packs(
emulators_dir: str = "emulators", platforms_dir: str = "platforms",
db_path: str = "database.json", include_archived: bool = False,
) -> bool:
"""Build each platform's --split parts in memory and compare them with
the full pack: the parts must add up to it, path for path and byte for
byte. Each part is a pack of its own and used to settle its paths alone:
RetroDECK's parts carried bios/SGB1.sfc as a file and as a directory,
RetroBat's 21 openmsx/ twins of openMSX/ files. Offline, as every
comparison of this step.
"""
import generate_pack # noqa: PLC0415
import packresolve # noqa: PLC0415
from common import ( # noqa: PLC0415
build_zip_contents_index,
group_identical_platforms,
list_registered_platforms,
load_data_dir_registry,
load_database,
load_emulator_profiles,
)
print(f"\n--- {step_title(STEP_LABELS['consistency'])}, split packs ---")
db = load_database(db_path)
zip_contents = build_zip_contents_index(db)
profiles = load_emulator_profiles(emulators_dir)
registry = load_data_dir_registry(platforms_dir)
groups = group_identical_platforms(
list_registered_platforms(platforms_dir, include_archived=include_archived),
platforms_dir,
)
was_offline = packresolve._OFFLINE
packresolve.set_offline(True)
all_ok = True
try:
for _members, platform in groups:
args = dict(
emulators_dir=emulators_dir, zip_contents=zip_contents,
data_registry=registry, emu_profiles=profiles, offline=True,
)
full = generate_pack.PackRecord()
parts: dict[str, generate_pack.PackRecord] = {}
with contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()):
generate_pack.generate_pack(
platform, platforms_dir, db, "bios", "", record=full, **args,
)
generate_pack.generate_split_packs(
platform, platforms_dir, db, "bios", "", records=parts, **args,
)
lines = split_differences(full, parts, generate_pack._same_bytes)
if lines:
all_ok = False
print(f" {platform} --split: {len(lines)} differences")
print("\n".join(f" {line}" for line in lines[:20]))
finally:
packresolve.set_offline(was_offline)
print(f" {len(groups)} split platform packs compared, "
f"{'all equal' if all_ok else 'differences above'}")
return all_ok
class _Skipped:
"""A step that did not run. Truthy, so it never fails the pipeline, and
distinct, so the summary does not report it as work that was done."""
@@ -883,6 +978,32 @@ def _refuse_offline_without_caches(
)
def _refuse_offline_export_without_originals(
parser: argparse.ArgumentParser, args: argparse.Namespace,
cache: str = ".cache/upstream-native",
) -> None:
"""An offline run with --with-export needs each patched file cached.
Seven formats carry code, so step 2e patches the platform's own file,
which --offline does not fetch. On a fresh clone the database, verify
and the truth model were built before step 2e failed on every one.
"""
if not (args.offline and args.with_export):
return
from exporter import discover_exporters # noqa: PLC0415
lacking = sorted(
name for name, exporter in discover_exporters().items()
if exporter.needs_original() and not any(Path(cache, name).glob("*"))
)
if lacking:
parser.error(
f"--with-export patches each platform's own file, and {cache}/ holds none "
f"for {', '.join(lacking)}: run python scripts/export_native.py --all "
"--refresh-cache once, or drop --offline"
)
def _truth_steps(args: argparse.Namespace, results: dict) -> bool:
"""Steps 2c to 2e: truth model, its diff, the native export."""
all_ok = True
@@ -1010,6 +1131,7 @@ def main() -> None:
args = parser.parse_args()
_refuse_unapplied_flags(parser, args)
_refuse_offline_without_caches(parser, args)
_refuse_offline_export_without_originals(parser, args)
if not args.offline:
# Read once and put in the environment every step inherits: the
# scrapers and refresh_data_dirs went anonymous unless the user had
@@ -1365,6 +1487,7 @@ def main() -> None:
ok = check_narrowed_packs(
include_archived=args.include_archived, full=args.full_narrowings,
) and ok
ok = check_split_packs(include_archived=args.include_archived) and ok
print(
f"--- {step_title(STEP_LABELS['consistency'])}, counts and paths: "
f"{'OK' if ok else 'FAILED'} ---"
scripts/profile_sync.py
+58 -63
View File
@@ -2665,6 +2665,9 @@ def drift_score(
# set again before any ref can be read, so each outranks a GONE ref:
# pcem ranked last at 0 with an orphaned upstream_commit.
score += PIN_WEIGHT * (len(report.orphaned) + len(report.moved_repos))
# A profile no repository answers for cannot be read at all, and
# --strict fails on it: it ranked at 0 under the clean profiles.
score += PIN_WEIGHT * report.uncheckable()
score += min(commits, 20) // 4
return score
@@ -3742,15 +3745,6 @@ def _git_file_at(path: Path, sha: str) -> dict | None:
return document if isinstance(document, dict) else None
def _scalar_values(document: dict) -> dict[str, str]:
"""String scalars of the document, keyed by field path."""
return {
field: value
for kind, field, _, _, value in _walk_document(document)
if kind == "str"
}
def _pin_of(
document: dict, field: str, mode: str = "",
repo: upstream.Repo | None = None,
@@ -3788,26 +3782,6 @@ def _pin_of(
return value
_CITATION_RUN_RE = re.compile(
PROSE_CITE_RE.pattern + r"(?:, *\d+(?:-\d+)?)*"
)
def _scalar_citations(text: str | None) -> tuple[str, ...] | None:
"""The citations a scalar carries, in order, without the prose around them.
A note reworded around unchanged citations was still written under the
pin those citations describe: comparing whole texts moved the writing
revision to the rewording commit, whose pin is current, and the stale
citations were never offered for realignment again.
"""
if text is None:
return None
return tuple(
_citation_key(m.group(0)) for m in _CITATION_RUN_RE.finditer(text)
)
def _writing_pins(
revisions: list[tuple[str, dict]], intro_sha: str, field: str, mode: str = "",
repo: upstream.Repo | None = None,
@@ -3943,7 +3917,8 @@ def _realign_part(
def _writing_pairs(
document: dict, repos: list, revisions: list[tuple[str, dict]], intro_sha: str
document: dict, repos: list, revisions: list[tuple[str, dict]], intro_sha: str,
served: dict | None = None,
) -> tuple[list[list], list[str]]:
"""Every reading of where the text was written, and the ambiguities.
@@ -3973,8 +3948,9 @@ def _writing_pairs(
ambiguous.append(f"{pin_field}_commit {' -> '.join(written)}")
# The current pin among the candidates reads as "no move" (None) for
# that repository, which the other candidates must then agree with.
reader = (served or {}).get(repo, repo)
options.append([
(repo, pin, current) if pin != current else None for pin in written
(reader, pin, current) if pin != current else None for pin in written
])
readings = [
[pair for pair in combo if pair is not None]
@@ -4055,8 +4031,20 @@ def realign_prose(
]
if not repos:
return []
# A source declared gone is read through its mirror, which carries the
# same commits: teknoparrot's notes stayed blocked once
# teknogods/TeknoParrotUI answered 404, every reading of the old pin
# calling the files absent.
mirror = (
upstream.parse_repo(str(document.get("source_mirror") or ""))
if document.get("upstream_gone") else None
)
served = {
repo: mirror for field, _mode, repo in repos
if mirror is not None and field == "source"
}
pinned = [
(repo, pin)
(served.get(repo, repo), pin)
for field, mode, repo in repos
if (pin := _pin_of(document, f"{field}_commit", mode))
]
@@ -4066,72 +4054,79 @@ def realign_prose(
if not history:
return []
current_values = _scalar_values(document)
fields = {citation.field for citation in citations}
alive = dict.fromkeys(fields, True)
intro: dict[str, str | None] = dict.fromkeys(fields)
# Each citation is followed on its own: a field's text changes whenever
# one of its citations is corrected by hand, and dating the whole field
# from that commit put idtech4a++'s untouched Game.java:69-72 under the
# current pin, so the stale range was never offered again.
keys = [(c.field, _citation_key(c.ref)) for c in citations]
alive = [True] * len(citations)
intro: list[str | None] = [None] * len(citations)
revisions: list[tuple[str, dict]] = []
# Fields whose writing pin lies past a revision that cannot be read:
# Citations whose writing pin lies past a revision that cannot be read:
# where their text came in is unknown, so they are not moved.
unknown: dict[str, str] = {}
unknown: dict[int, str] = {}
# A revision that does not parse is passed over unless the pins differ
# on its two sides while the field's citations changed across it: then
# the text may have come in there under either pin. Otherwise nothing
# on its two sides while the citation is absent across it: then the
# text may have come in there under either pin. Otherwise nothing
# it could hold changes the reading. lindbergh-loader
# committed invalid YAML once (df7346b3), fixed the same day, and every
# note older than that commit stayed blocked for good.
gap: str | None = None
def carried_by(past: dict) -> set[tuple[str, str]]:
return {
(c.field, _citation_key(c.ref))
for c in collect_citations(past) if c.kind == "prose"
}
for sha in history:
if not any(alive.values()):
if not any(alive):
break
past = _git_file_at(path, sha)
if past is None:
gap = gap or sha
continue
held = carried_by(past)
if gap:
newer = revisions[-1][1] if revisions else document
moved = any(
newer.get(pin) != past.get(pin)
for pin in ("source_commit", "upstream_commit")
)
values = _scalar_values(past)
for field, live in alive.items():
if live and moved and _scalar_citations(values.get(field)) != (
_scalar_citations(current_values.get(field))
):
unknown[field] = gap
alive[field] = False
for index, live in enumerate(alive):
if live and moved and keys[index] not in held:
unknown[index] = gap
alive[index] = False
gap = None
revisions.append((sha, past))
values = _scalar_values(past)
for field in fields:
if not alive[field]:
for index, live in enumerate(alive):
if not live:
continue
if _scalar_citations(values.get(field)) == _scalar_citations(
current_values.get(field)
):
intro[field] = sha
if keys[index] in held:
intro[index] = sha
else:
alive[field] = False
alive[index] = False
if gap:
# The oldest revisions do not parse: where the text came in is past
# what can be read.
unknown.update({field: gap for field, live in alive.items() if live})
unknown.update({index: gap for index, live in enumerate(alive) if live})
messages: list[str] = []
jobs: list[tuple[Citation, dict]] = []
for citation in citations:
if citation.field in unknown:
for index, citation in enumerate(citations):
if index in unknown:
messages.append(
f"read again: {citation.field}: {citation.ref} "
f"(revision {unknown[citation.field][:8]} unreadable)"
f"(revision {unknown[index][:8]} unreadable)"
)
continue
intro_sha = intro.get(citation.field)
intro_sha = intro[index]
if intro_sha is None:
# The scalar is not committed yet: written now, under this pin.
continue
readings, ambiguous = _writing_pairs(document, repos, revisions, intro_sha)
readings, ambiguous = _writing_pairs(
document, repos, revisions, intro_sha, served
)
if not any(readings):
continue
moves, blocked = _realign_citation(
@@ -5042,7 +5037,7 @@ def _print_triage(
state = triage_state(report)
if refused:
state += f", {refused}"
pending += bool(report.needs_review())
pending += bool(report.needs_review() or report.uncheckable())
print(f"{score:5d} {report.name:30s} {state}")
print(f"{len(ranked)} profiles, {pending} with something to review")
scripts/refresh_data_dirs.py
+81 -6
View File
@@ -24,7 +24,7 @@ import time
import urllib.error
import urllib.request
import zipfile
from collections.abc import Iterator
from collections.abc import Callable, Iterator
from pathlib import Path
from artifacts import file_lock as _file_lock, set_plain_mode
from common import InputMissing, github_token, platform_arg, require_platform, suggest_names, yaml_load
@@ -71,6 +71,24 @@ def cache_lock(
return _announced_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared, cache_dir)
def _wait_message(lock_path: Path, shared: bool, cache_dir: Path) -> str:
"""Who to wait for, and, for a refresh, how not to.
The bare "another run holds it" named neither the lock nor its holder,
and the build waited with no limit.
"""
message = (
f"waiting for {cache_dir}: another run holds {lock_path} "
f"(lsof {lock_path} names it)"
)
if not shared:
message += (
". generate_pack.py --no-refresh-data, or --offline there and in"
" pipeline.py, builds from the cache as it is without waiting"
)
return message
@contextlib.contextmanager
def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[None]:
"""file_lock, saying so when it has to wait.
@@ -90,7 +108,7 @@ def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[
try:
fcntl.flock(handle, mode | fcntl.LOCK_NB)
except BlockingIOError:
log.warning("waiting for %s: another run holds it", cache_dir)
log.warning(_wait_message(lock_path, shared, cache_dir))
fcntl.flock(handle, mode)
try:
yield
@@ -270,10 +288,12 @@ def _download_and_extract(
source_path: str,
local_cache: str,
exclude: list[str] | None = None,
measured: Callable[[int], None] | None = None,
) -> int:
"""Download tarball, extract source_path subtree to local_cache.
Returns the number of files extracted.
Returns the number of files extracted. measured receives the size of
the archive downloaded.
"""
exclude = exclude or []
cache_dir = Path(local_cache)
@@ -290,6 +310,8 @@ def _download_and_extract(
if not chunk:
break
f.write(chunk)
if measured is not None:
measured(tarball_path.stat().st_size)
log.info("extracting %s -> %s", source_path, local_cache)
@@ -347,6 +369,7 @@ def _download_and_extract_zip(
local_cache: str,
exclude: list[str] | None = None,
strip_components: int = 0,
measured: Callable[[int], None] | None = None,
) -> int:
"""Download ZIP, extract to local_cache. Returns file count.
@@ -369,6 +392,8 @@ def _download_and_extract_zip(
if not chunk:
break
f.write(chunk)
if measured is not None:
measured(zip_path.stat().st_size)
extract_dir = Path(tmpdir) / "extract"
extract_dir.mkdir()
@@ -408,6 +433,46 @@ def _download_and_extract_zip(
return file_count
def _measure_archive(source_url: str) -> int | None:
"""The size of the archive at source_url, read by streaming it: a
tarball GitHub builds on request is sent without a Content-Length."""
req = urllib.request.Request(source_url, headers={"User-Agent": USER_AGENT})
size = 0
try:
with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp:
while chunk := resp.read(65536):
size += len(chunk)
except _TRANSIENT as exc:
log.warning("could not measure %s: %s", source_url, exc)
return None
return size or None
def _record_archive_size(key: str, entry: dict, versions_path: str) -> None:
"""Record the archive size of a tree current since before it was kept.
The installer announces a tree's download as that size: a tarball
whose tree is 10 KB is a download of 21 MB. Measured, not refetched,
so the tree the packs carry stays the one recorded.
"""
path = Path(versions_path)
with _file_lock(path.with_name(f".{path.name}.lock")):
versions = _load_versions(versions_path)
record = versions.get(key)
if record is None or "archive_size" in record:
return
version = entry.get("version", "master")
size = _measure_archive(entry["source_url"].format(version=version))
if size is None:
return
with _file_lock(path.with_name(f".{path.name}.lock")):
versions = _load_versions(versions_path)
if key in versions and "archive_size" not in versions[key]:
versions[key]["archive_size"] = size
_save_versions(versions, versions_path)
log.info("[%s] archive size recorded: %d bytes", key, size)
def _get_remote_etag(source_url: str) -> str | None:
"""HEAD request to get ETag or Last-Modified for freshness check."""
try:
@@ -456,6 +521,7 @@ def refresh_entry(
return None
if not needs:
log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag))
_record_archive_size(key, entry, versions_path)
return False
with cache_lock(entry["local_cache"]):
return _refresh_entry(key, entry, force, dry_run, versions_path)
@@ -516,13 +582,19 @@ def _refresh_entry(
else:
remote_tag = get_remote_sha(entry["source_url"], version)
archive_sizes: list[int] = []
def fetch() -> int:
archive_sizes.clear()
if source_type == "zip":
return _download_and_extract_zip(
source_url, local_cache, exclude, entry.get("strip_components", 0)
source_url, local_cache, exclude, entry.get("strip_components", 0),
measured=archive_sizes.append,
)
source_path = entry["source_path"].format(version=version)
return _download_and_extract(source_url, source_path, local_cache, exclude)
return _download_and_extract(
source_url, source_path, local_cache, exclude, measured=archive_sizes.append
)
# An archive that reads but holds nothing under the path is the same on
# every try: only a failed or unreadable download is tried again.
@@ -540,7 +612,10 @@ def _refresh_entry(
log.warning("[%s] download failed (%s), trying again in %d s", key, exc, wait)
time.sleep(wait)
_record_version(key, {"sha": remote_tag or "", "version": version}, versions_path)
record = {"sha": remote_tag or "", "version": version}
if archive_sizes:
record["archive_size"] = archive_sizes[-1]
_record_version(key, record, versions_path)
log.info("[%s] refreshed: %d files extracted to %s", key, file_count, local_cache)
return True
scripts/refresh_stale.py
+30 -4
View File
@@ -34,7 +34,10 @@ has ended, and its export is rewritten even when no row named it.
Surfaced only (no refresh): data rows that are not a _data_dirs.yml key
(the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs,
CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs
CI pins (PyPI, pinned actions, while scripts/pin_installer.py writes the
install.py SHA-256 pin), profile_sync, rom-folder
pins (moved by hand, then python -m scripts.scraper.romfolders --setup NAME
reads them again). Each needs
a human read (new .info file, manual DAT download, PyPI version bump).
Usage:
@@ -65,7 +68,9 @@ PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
MANUAL_AREAS = ("coreinfo", "ci", "profiles")
# A rom-folder pin is moved by hand in platforms/romfolders/_sources/,
# then `python -m scripts.scraper.romfolders --setup NAME` reads it again.
MANUAL_AREAS = ("romfolders", "coreinfo", "ci", "profiles")
JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10
@@ -397,6 +402,10 @@ def _command_for(
"--force",
]
if area == "ci" and subject == "install.py pin":
# The pin is the hash of the committed install.py, nothing to judge.
return [sys.executable, "scripts/pin_installer.py"]
if area == "catalogs":
if subject == "mame recipes":
return [
@@ -563,7 +572,8 @@ def main() -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
parser.add_argument(
"--only",
help="comma-separated areas to check: " + ",".join(AUTO_AREAS),
help="comma-separated areas to check: " + ",".join(AUTO_AREAS)
+ " (refreshed), " + ",".join(MANUAL_AREAS) + " (listed for a human)",
)
parser.add_argument(
"--dry-run",
@@ -579,7 +589,7 @@ def main() -> int:
parser.add_argument(
"--json",
action="store_true",
help="machine-readable summary (jobs, surfaced, ignored)",
help="machine-readable summary (jobs, surfaced, ignored), the plan with --dry-run",
)
parser.add_argument(
"--findings",
@@ -618,6 +628,22 @@ def main() -> int:
registry = _load_platform_registry()
jobs, surfaced, ignored = plan_jobs(findings, registry)
if args.dry_run and args.json:
print(json.dumps({
"planned": [
{
"area": job.area,
"subject": job.subject,
"command": job.command,
"after": sorted(job.after),
"then": [list(command) for command in job.then],
}
for job in sorted(jobs, key=lambda j: (j.area, j.subject))
],
"surfaced": surfaced,
"ignored_count": len(ignored),
}, indent=2))
return 0
if args.dry_run:
print(f"\n[plan] {len(jobs)} refreshable, {len(surfaced)} manual, "
f"{len(ignored)} already clean/skipped")
scripts/release_notes.py
+24 -2
View File
@@ -7,7 +7,8 @@ issue 81 happened. Install commands, the parts sentence and the table come
from the record and the registry. Each row links the platform's page under
get/ on the site, which outlives the release, and the setups the registry
does not mark release_pack are named with the command. The maintainer writes
what changed, and that text is appended as given.
what changed, and that text is appended as given once it holds none of the
REFUSED phrases.
Usage:
python scripts/release_notes.py --whats-new tmp/whats-new.md --output notes.md
@@ -102,6 +103,20 @@ def _link(name: str, pack: dict) -> str:
return f"[{label}]({release_record.get_page(_owner(name, pack), SITE)})"
# Words the published notes of v2026.10.10 carried by hand: "never from a
# forum list" judged lists other people keep, and "a game never stops on a
# missing file" was a claim the gap analysis contradicts. Text about other
# projects says what changed, not who erred (wiki/release-process.md).
REFUSED = ("their file", "theirs", "of ours", "corrected", "wrong", "drift",
"left out", "never seen", "forum", "never stops", "never fails")
def refused_words(text: str) -> list[str]:
"""The refused phrases the text holds, case ignored."""
lowered = text.lower()
return [phrase for phrase in REFUSED if phrase in lowered]
def render(record: dict, platforms_dir: str, whats_new: str) -> str:
packs = record.get("packs", {})
folders = _extract_folders(record, platforms_dir)
@@ -124,7 +139,8 @@ def render(record: dict, platforms_dir: str, whats_new: str) -> str:
"### Download a pack",
"",
("One complete pack for each platform below: the BIOS list the platform"
" publishes plus every file its emulators open. Each link is a page"
" publishes plus the files its emulators open. Files no pack carries yet"
f" are listed in the [gap analysis]({SITE}gaps/). Each link is a page"
" that keeps its address from one release to the next. Files is what the pack"
" holds once extracted, the count a file manager shows. Extracted size"
" is the space those files take, and the download is smaller."),
@@ -173,6 +189,12 @@ def main() -> int:
if not whats_new.strip():
print(f"Error: {args.whats_new} is empty", file=sys.stderr)
return 1
refused = refused_words(whats_new)
if refused:
print(f"Error: {args.whats_new} says {', '.join(repr(w) for w in refused)}. State what"
" changed and how it was checked, not a verdict on other projects or a"
" promise the gap analysis contradicts", file=sys.stderr)
return 1
write_text_atomic(str(args.output), render(record, args.platforms_dir, whats_new))
print(f"Wrote {args.output} for {record['tag']}, {len(record['packs'])} packs")
return 0
scripts/restore_large_files.py
+36 -3
View File
@@ -17,11 +17,14 @@ Usage:
from __future__ import annotations
import argparse
import contextlib
import hashlib
import os
import sys
import tempfile
sys.path.insert(0, os.path.dirname(__file__))
from artifacts import set_plain_mode
from common import copy_file_atomic, load_database
@@ -56,6 +59,36 @@ def index_cache(cache_dir: str) -> dict[str, str]:
return index
def _copy_verified(source: str, path: str, sha1: str) -> bool:
"""Copy *source* to *path* only if the copy hashes to *sha1*.
The cache was hashed before the copy, and another run can replace a
cached name in between (a re-uploaded asset): the copy is what counts.
Whole or absent: a scan running beside a plain copy hashed the truncated
file and published that in the database.
"""
handle, scratch = tempfile.mkstemp(
dir=os.path.dirname(os.path.abspath(path)),
prefix=f".{os.path.basename(path)}.", suffix=".tmp",
)
os.close(handle)
try:
copy_file_atomic(source, scratch)
digest = hashlib.sha1()
with open(scratch, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
digest.update(chunk)
if digest.hexdigest() != sha1:
print(f"Changed in the cache while restoring: {source}", file=sys.stderr)
return False
set_plain_mode(scratch)
os.replace(scratch, path)
return True
finally:
with contextlib.suppress(FileNotFoundError):
os.unlink(scratch)
def restore(
cache_dir: str, db_path: str, gitignore: str
) -> tuple[int, list[str]]:
@@ -79,9 +112,9 @@ def restore(
unsatisfied.append(path)
continue
os.makedirs(os.path.dirname(path), exist_ok=True)
# Whole or absent: a scan running beside a plain copy hashed the
# truncated file and published that in the database.
copy_file_atomic(source, path)
if not _copy_verified(source, path, sha1):
unsatisfied.append(path)
continue
print(f"Restored: {path}")
restored += 1
print(f"Total: {restored} files restored")
scripts/scraper/arcade_parent_importer.py
+3 -10
View File
@@ -49,7 +49,7 @@ from collections.abc import Callable, Iterator
from dataclasses import dataclass, field
from pathlib import Path
from ..artifacts import write_text_atomic
from ..artifacts import write_if_changed
from ..common import load_emulator_profiles, run_cli
from .romset_dat_importer import (
LISTXML_CHUNK,
@@ -1114,14 +1114,6 @@ def dump(doc: dict) -> str:
return json.dumps(doc, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
def write_if_changed(path: Path, text: str) -> bool:
if path.is_file() and path.read_text(encoding="utf-8") == text:
return False
path.parent.mkdir(parents=True, exist_ok=True)
write_text_atomic(str(path), text)
return True
def main() -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
which = parser.add_mutually_exclusive_group(required=True)
@@ -1197,7 +1189,8 @@ def main() -> int:
continue
if not args.dry_run:
target = args.output_dir / f"{profile}.json"
if write_if_changed(target, dump(doc)):
target.parent.mkdir(parents=True, exist_ok=True)
if write_if_changed(str(target), dump(doc)):
print(f" wrote {target}")
return 1 if failed else 0
scripts/scraper/emudeck_launchers.py
+9 -4
View File
@@ -46,14 +46,18 @@ STORAGE = "${storagePath}"
FLATPAK_DATA = "${HOME}/.var/app/"
# EmuScripts file names that differ from the emulator profile they set up.
# The platform and target scrapers both read them here.
SCRIPT_EMULATORS = {
"pcsx2qt": "pcsx2",
"rpcs3legacy": "rpcs3",
"cemuproton": "cemu",
"rmg": "mupen64plus_next",
}
# The main RetroArch config script installs no emulator.
NOT_EMULATORS = {"retroarch_maincfg"}
# The main RetroArch config script installs no emulator, and emudeck-we's
# emuDeckTemplate.ps1 is the skeleton new installers are copied from.
NOT_EMULATORS = {"retroarch_maincfg", "template"}
# EmuDeck's SteamOS installers are shell scripts, emudeck-we's PowerShell.
SCRIPT_SUFFIXES = (".sh", ".ps1")
# What the emulator does with a directory written into each key.
DIRECTORY_KINDS = {
@@ -66,9 +70,10 @@ DIRECTORY_KINDS = {
def emulator_of_script(file_name: str) -> str | None:
"""The emulator an EmuScripts file sets up, None for a helper script."""
if not file_name.endswith(".sh"):
stem, dot, suffix = file_name.rpartition(".")
if not dot or f".{suffix}" not in SCRIPT_SUFFIXES:
return None
key = re.sub(r"^emuDeck", "", file_name[: -len(".sh")], flags=re.IGNORECASE).lower()
key = re.sub(r"^emuDeck", "", stem, flags=re.IGNORECASE).lower()
if not key or key in NOT_EMULATORS:
return None
return SCRIPT_EMULATORS.get(key, key)
scripts/scraper/emudeck_scraper.py
+8 -7
View File
@@ -469,9 +469,10 @@ class Scraper(BaseScraper):
version = fetch_github_latest_version("dragoonDorise/EmuDeck")
cores = self._fetch_installed_emulators()
revision = launcher_revision(self.checkbios_url)
cores = self._fetch_installed_emulators(revision)
standalone = [core for core in cores if core != "retroarch"]
tree = RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url))
tree = RepoTree.github(EMUDECK_REPO, revision)
libretro = set(installed_libretro_cores(tree.read(RETROARCH_SCRIPT)))
return {
@@ -501,16 +502,16 @@ class Scraper(BaseScraper):
"launchers": extract_launchers(tree),
}
def _fetch_installed_emulators(self) -> list[str]:
"""Fetch the list of emulators installed by EmuDeck from EmuScripts.
Returns core names normalized to match emulator profile keys.
def _fetch_installed_emulators(self, revision: str) -> list[str]:
"""The emulators EmuDeck's EmuScripts install at the pinned revision,
named as their profiles are: the launcher layer and the SteamOS
target are read at that revision too.
"""
import json
api_url = (
"https://api.github.com/repos/dragoonDorise/EmuDeck/"
"contents/functions/EmuScripts"
f"contents/functions/EmuScripts?ref={revision}"
)
try:
req = urllib.request.Request(api_url, headers=github_headers())
scripts/scraper/game_dat_importer.py
+4 -10
View File
@@ -38,7 +38,7 @@ _SCRIPTS_DIR = str(Path(__file__).resolve().parents[1])
if _SCRIPTS_DIR not in sys.path:
sys.path.insert(0, _SCRIPTS_DIR)
from artifacts import copy_stream_atomic, write_text_atomic # noqa: E402
from artifacts import copy_stream_atomic, write_if_changed, write_text_atomic # noqa: E402
from common import parse_untrusted_xml # noqa: E402
from region import REGIONS, WORLD # noqa: E402
from region_audit import NOINTRO # noqa: E402
@@ -275,14 +275,6 @@ def dump_catalog(doc: dict) -> str:
return f'{text},"games":[\n{games}\n]}}\n'
def write_if_changed(path: Path, text: str) -> bool:
if path.is_file() and path.read_text(encoding="utf-8") == text:
return False
path.parent.mkdir(parents=True, exist_ok=True)
write_text_atomic(str(path), text)
return True
# Fetching ---------------------------------------------------------------
@@ -503,7 +495,9 @@ def main() -> int:
failures.append(f"{doc['source']}/{doc['system']}: {cov['games']} games, "
f"floor {floor_of(doc)}")
if not args.dry_run:
write_if_changed(args.output_dir / doc["source"] / f"{doc['system']}.json", text)
target = args.output_dir / doc["source"] / f"{doc['system']}.json"
target.parent.mkdir(parents=True, exist_ok=True)
write_if_changed(str(target), text)
print(f"{len(docs)} catalogs, {raw_total} bytes raw, {gz_total} bytes gzip")
for failure in failures:
print(f"FAIL {failure}", file=sys.stderr)
scripts/scraper/targets/amberelec_targets_scraper.py
+157
View File
@@ -0,0 +1,157 @@
"""Scraper for AmberELEC per-device emulator availability.
Sources (AmberELEC/AmberELEC, at the commit platforms/amberelec.yml names):
- Makefile -- one `DEVICE=<name> ARCH=<arch> ./scripts/build_distro` target
per image (RG351P, RG351V, RG351MP, RG552, RG353, RG503 at a21975bf)
- packages/amberelec/package.mk -- LIBRETRO_CORES_EXTRA joins PKG_EMUS
only on the devices its `if` names (lines 23-31 at a21975bf: RG552,
RG353, RG503), and on every other device makeinstall_target deletes
each of those cores from es_systems.cfg (lines 45-58)
- packages/ui/emulationstation/config/es_systems.cfg -- read the way the
platform scraper reads it, after that deletion
An RG351P image carries neither melonDS nor bsnes: without a target list,
its pack shipped the Nintendo DS firmware anyway.
"""
from __future__ import annotations
import argparse
import re
import sys
from datetime import datetime, timezone
import yaml
from . import BaseTargetScraper
PLATFORM_NAME = "amberelec"
RAW_BASE = "https://raw.githubusercontent.com/AmberELEC/AmberELEC/{ref}"
META_MK = "packages/amberelec/package.mk"
try:
from ..amberelec_scraper import BUILDS_OF, ES_SYSTEMS_TEMPLATE, RUNNERS
from ..base_scraper import fetch_bytes
from ..batocera_scraper import _not_cores
from ..es_emulators import es_cores, xml_systems
except ImportError:
from scraper.amberelec_scraper import BUILDS_OF, ES_SYSTEMS_TEMPLATE, RUNNERS
from scraper.base_scraper import fetch_bytes
from scraper.batocera_scraper import _not_cores
from scraper.es_emulators import es_cores, xml_systems
_DEVICE_RE = re.compile(
r"^(\w+):\n\tDEVICE=(\w+) ARCH=(\w+) \./scripts/build_distro$", re.MULTILINE
)
_EXTRA_RE = re.compile(r'LIBRETRO_CORES_EXTRA="([^"]*)"')
# The `if` whose body adds the extra cores to PKG_EMUS.
_EXTRA_IF_RE = re.compile(
r"if \[\[ (?P<test>[^\n]*?) \]\]; then\s*\n\s*PKG_EMUS\+=\" \$\{LIBRETRO_CORES_EXTRA\}\""
)
# What makeinstall_target does to es_systems.cfg on the other devices.
_REMOVAL_LINES = (
"for CORE in ${LIBRETRO_CORES_EXTRA}; do",
'sed -i "s|<core>$CORE</core>||g"',
r"sed -i -e '/<emulator name/{:a;N;/<\/emulator>/!ba;/melonds/d}'",
)
_EMULATOR_BLOCK_RE = re.compile(r"[^\n]*<emulator name.*?</emulator>[^\n]*\n?", re.DOTALL)
def devices(makefile: str) -> dict[str, str]:
"""{device: architecture} of each image the Makefile builds."""
found = {device: arch for target, device, arch in _DEVICE_RE.findall(makefile)
if target == device}
if not found:
raise RuntimeError("the AmberELEC Makefile builds no device image")
return found
def extra_cores(package_mk: str) -> tuple[list[str], set[str]]:
"""(the extra cores, the devices that keep them) from the metapackage."""
listed = _EXTRA_RE.search(package_mk)
test = _EXTRA_IF_RE.search(package_mk)
if listed is None or test is None:
raise RuntimeError(f"{META_MK}: no LIBRETRO_CORES_EXTRA or no test adding it")
missing = [line for line in _REMOVAL_LINES if line not in package_mk]
if missing:
raise RuntimeError(f"{META_MK}: makeinstall_target no longer reads {missing[0]!r}")
keep = set(re.findall(r'"\$\{DEVICE\}" == "(\w+)"', test.group("test")))
for alternatives in re.findall(r'"\$\{DEVICE\}" =~ \^\(([\w|]+)\)\$', test.group("test")):
keep.update(alternatives.split("|"))
if not keep:
raise RuntimeError(f"{META_MK}: the test adding the extra cores names no device")
return listed.group(1).split(), keep
def without_cores(es_systems: str, cores: list[str]) -> str:
"""es_systems.cfg as makeinstall_target leaves it on a device without them."""
for core in cores:
if core == "melonds":
es_systems = _EMULATOR_BLOCK_RE.sub(
lambda block: "" if "melonds" in block.group(0) else block.group(0),
es_systems,
)
es_systems = es_systems.replace(f"<core>{core}</core>", "")
return es_systems
def device_cores(es_systems: str, extra: list[str], keeps_extra: bool) -> list[str]:
"""Every core and standalone emulator one device's es_systems.cfg offers."""
text = es_systems if keeps_extra else without_cores(es_systems, extra)
cores, _standalone, _dual = es_cores(
xml_systems(text), RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME)
)
return sorted(set(cores))
class Scraper(BaseTargetScraper):
"""Per-device core lists from AmberELEC's metapackage."""
def __init__(self, url: str = "https://github.com/AmberELEC/AmberELEC") -> None:
super().__init__(url=url)
self.ref = ""
def _raw(self, path: str) -> str:
return fetch_bytes(f"{RAW_BASE.format(ref=self.ref)}/{path}").decode("utf-8")
def fetch_targets(self) -> dict:
self.ref = self.pinned_revision(PLATFORM_NAME)
extra, keep = extra_cores(self._raw(META_MK))
es_systems = fetch_bytes(ES_SYSTEMS_TEMPLATE.format(ref=self.ref)).decode("utf-8")
targets: dict[str, dict] = {}
for device, arch in sorted(devices(self._raw("Makefile")).items()):
cores = device_cores(es_systems, extra, device in keep)
targets[device.lower()] = {"architecture": arch, "cores": cores}
print(f" {device.lower()} ({arch}): {len(cores)} cores", file=sys.stderr)
return {
"platform": PLATFORM_NAME,
"source": f"{self.url}/blob/{self.ref}/{META_MK}",
"scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"targets": targets,
}
def main() -> None:
parser = argparse.ArgumentParser(
allow_abbrev=False, description="Scrape AmberELEC per-device cores"
)
parser.add_argument("--dry-run", action="store_true", help="Show target summary")
parser.add_argument("--output", "-o", help="Output YAML file")
args = parser.parse_args()
scraper = Scraper()
data = scraper.fetch_targets()
if args.dry_run:
for name, info in data["targets"].items():
print(f" {name} ({info['architecture']}): {len(info['cores'])} cores")
return
if args.output:
scraper.write_output(data, args.output)
print(f"Written to {args.output}")
return
print(yaml.dump(data, default_flow_style=False, sort_keys=False))
if __name__ == "__main__":
main()
scripts/scraper/targets/emudeck_targets_scraper.py
+7 -39
View File
@@ -12,7 +12,6 @@ from __future__ import annotations
import argparse
import json
import re
import sys
import urllib.error
import urllib.request
@@ -22,6 +21,7 @@ import yaml
from common import github_token, yaml_load
from ..emudeck_launchers import SCRIPT_SUFFIXES, emulator_of_script
from . import BaseTargetScraper
PLATFORM_NAME = "emudeck"
@@ -33,24 +33,6 @@ WINDOWS_API = (
"https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts"
)
# Map EmuDeck script names to emulator profile keys
# Script naming: emuDeckDolphin.sh -> dolphin
# Some need explicit mapping when names differ
_NAME_OVERRIDES: dict[str, str] = {
"pcsx2qt": "pcsx2",
"rpcs3legacy": "rpcs3",
"cemuproton": "cemu",
"rmg": "mupen64plus_next",
"bigpemu": "bigpemu",
"eden": "eden",
"suyu": "suyu",
"ares": "ares",
}
# Scripts that are not emulators (config helpers, etc.)
_SKIP = {"retroarch_maincfg", "retroarch"}
def _fetch(url: str) -> str:
headers = {"User-Agent": "retrobios-scraper/1.0"}
token = github_token()
@@ -69,26 +51,13 @@ def _list_emuscripts(api_url: str) -> list[str]:
"""List emulator script filenames from GitHub API."""
entries = json.loads(_fetch(api_url))
names = [
e["name"] for e in entries if e.get("name", "").endswith((".sh", ".ps1"))
e["name"] for e in entries if e.get("name", "").endswith(SCRIPT_SUFFIXES)
]
if not names:
raise RuntimeError(f"no emulator scripts listed at {api_url}")
return names
def _script_to_core(filename: str) -> str | None:
"""Convert EmuScripts filename to core profile key."""
# Strip extension and emuDeck prefix
name = re.sub(r"\.(sh|ps1)$", "", filename, flags=re.IGNORECASE)
name = re.sub(r"^emuDeck", "", name, flags=re.IGNORECASE)
if not name:
return None
key = name.lower()
if key in _SKIP:
return None
return _NAME_OVERRIDES.get(key, key)
class Scraper(BaseTargetScraper):
"""Fetches emulator lists for EmuDeck SteamOS and Windows targets."""
@@ -114,14 +83,13 @@ class Scraper(BaseTargetScraper):
seen: set[str] = set()
has_retroarch = False
for script in scripts:
core = _script_to_core(script)
if core and core not in seen:
core = emulator_of_script(script)
# RetroArch stands for its libretro cores, added below.
if core == "retroarch":
has_retroarch = True
elif core and core not in seen:
seen.add(core)
cores.append(core)
# Detect RetroArch presence (provides all libretro cores)
name = re.sub(r"\.(sh|ps1)$", "", script, flags=re.IGNORECASE)
if name.lower() in ("emudeckretroarch", "retroarch_maincfg"):
has_retroarch = True
standalone_count = len(cores)
# EmuDeck ships RetroArch = all its libretro cores are available
scripts/scraper/targets/emuelec_targets_scraper.py
+662
View File
@@ -0,0 +1,662 @@
"""Scraper for EmuELEC per-device emulator availability.
Sources (EmuELEC/EmuELEC, at the tag platforms/emuelec.yml names, v4.8):
- the release's asset names: one `EmuELEC-<DEVICE>.<ARCH>-<version>`
image per device (Amlogic-ng, GameForce, OdroidGoAdvance, OdroidM1 at
v4.8). The tree names no device list: projects/ holds the directories
of many more devices than the release builds.
- packages/sx05re/emuelec/package.mk -- the metapackage, whose
PKG_DEPENDS_TARGET is each device's package list (lines 8-77)
- distributions/EmuELEC/options -- LIBRETRO_CORES and
LIBRETRO_S922X_CORES (lines 274-373)
- each core package's package.mk -- PKG_ARCH, which scripts/build:28-30
reads to skip a package, and the install step naming the .so or binary
- packages/sx05re/emuelec-emulationstation/package.mk -- the cores and
emulators makeinstall_target deletes from es_systems.cfg per device
(lines 114-143)
- es_systems.json and ports.yaml, read the way the platform scraper reads
them
A device keeps a core when a package it builds installs that core and its
es_systems.cfg still offers it. The OdroidGoAdvance image carries neither
DuckStation nor Dolphin nor the MAME core: without a target list, its pack
shipped their firmware anyway.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from datetime import datetime, timezone
from typing import NamedTuple
import yaml
from . import BaseTargetScraper
PLATFORM_NAME = "emuelec"
REPO = "EmuELEC/EmuELEC"
RAW = "https://raw.githubusercontent.com/EmuELEC/EmuELEC/{ref}/{path}"
RELEASE_API = "https://api.github.com/repos/EmuELEC/EmuELEC/releases/tags/{ref}"
META_MK = "packages/sx05re/emuelec/package.mk"
OPTIONS = "distributions/EmuELEC/options"
ES_MK = "packages/sx05re/emuelec-emulationstation/package.mk"
try:
from ..base_scraper import fetch_bytes, github_headers
from ..batocera_scraper import _not_cores
from ..emuelec_scraper import (
BUILDS_OF, ES_SYSTEMS_TEMPLATE, PORTS_TEMPLATE, RUNNERS, port_engines, with_ports,
)
from ..es_emulators import SystemEmulators, es_cores
except ImportError:
from scraper.base_scraper import fetch_bytes, github_headers
from scraper.batocera_scraper import _not_cores
from scraper.emuelec_scraper import (
BUILDS_OF, ES_SYSTEMS_TEMPLATE, PORTS_TEMPLATE, RUNNERS, port_engines, with_ports,
)
from scraper.es_emulators import SystemEmulators, es_cores
_ASSET_RE = re.compile(r"^EmuELEC-(?P<device>[A-Za-z0-9-]+)\.(?P<arch>[a-z0-9_]+)-")
# The metapackage, line for line. Each rule below transcribes these lines;
# a tag that rewrites one stops the scrape instead of reading it wrong.
_META_LINES = (
'PKG_DEPENDS_TARGET="toolchain ${OPENGLES} emuelec-emulationstation retroarch"',
'PKG_DEPENDS_TARGET+=" emuelec-tools ${PKG_EMUS} ${PKG_EXPERIMENTAL}"',
'if [ "${DEVICE}" == "Amlogic-ng" ] || [ "${DEVICE}" == "Amlogic-no" ] || '
'[ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then',
'PKG_DEPENDS_TARGET+=" ${LIBRETRO_S922X_CORES}"',
'if [ "${DEVICE}" == "OdroidGoAdvance" ] || [ "${DEVICE}" == "GameForce" ]; then',
'PKG_DEPENDS_TARGET+=" kmscon odroidgoa-utils"',
"for discore in duckstation mesen-s virtualjaguar quicknes MC; do",
'PKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore} | |")',
'PKG_DEPENDS_TARGET+=" yabasanshiro"',
'PKG_DEPENDS_TARGET+=" fbterm"',
'if [ "${ARCH}" == "aarch64" ]; then',
"for discore in quicknes parallel-n64 pcsx_rearmed; do",
'PKG_DEPENDS_TARGET=$(echo ${PKG_DEPENDS_TARGET} | sed "s|${discore}| |")',
'PKG_DEPENDS_TARGET+=" dolphinSA"',
'if [ "${DEVICE}" == "Amlogic-old" ]; then',
"for discore in yabasanshiroSA_1_11 yabasanshiroSA_1_5 yabasanshiro same_cdi duckstation; do",
'PKG_DEPENDS_TARGET+=" mame"',
'if [ "${DEVICE}" == "RK356x" ] || [ "${DEVICE}" == "OdroidM1" ]; then',
"for discore in flycast-dojo; do",
)
_LIB32_RE = re.compile(r'PKG_DEPENDS_TARGET\+=" (swanstation [^"]*)"')
_BIG = ("Amlogic-ng", "Amlogic-no", "RK356x", "OdroidM1")
_HANDHELD = ("OdroidGoAdvance", "GameForce")
# What emuelec-emulationstation's makeinstall_target does to es_systems.cfg.
_ES_LINES = (
'if [[ "${DEVICE}" != "Amlogic-ng" || "${DEVICE}" != "Amlogic-ne" || '
'"${DEVICE}" != "Amlogic-no" ]]; then',
'if [[ ${DEVICE} == "OdroidGoAdvance" || "${DEVICE}" == "GameForce" ]]; then',
'remove_cores="mesen-s quicknes mame2016 mesen"',
'remove_cores="mesen-s quicknes mame2016 mesen yabasanshiroSA yabasanshiro"',
"xmlstarlet ed -L -P -d \"/systemList/system[name='saturn']\" ${CORESFILE}",
"xmlstarlet ed -L -P -d \"/systemList/system[name='phillips-cdi']\" ${CORESFILE}",
'sed -i "s|<core>${discore}</core>||g" ${CORESFILE}',
'if [[ ${DEVICE} != "OdroidGoAdvance" ]] && [[ "${DEVICE}" != "GameForce" ]]; then',
"xmlstarlet ed -L -P -d \"/systemList/system/emulators/emulator[@name='retrorun']\" ${CORESFILE}",
"xmlstarlet ed -L -P -d \"/systemList/system/emulators/emulator[@name='Duckstation']\" ${CORESFILE}",
)
class Install(NamedTuple):
"""One line of a package.mk that puts a core on the image."""
path: str
line: str
cores: tuple[str, ...]
# The ARCH test the line sits under in makeinstall_target, if any.
arch: str | None = None
def _lr(package: str, line: str, *cores: str, arch: str | None = None) -> Install:
return Install(f"packages/sx05re/libretro/{package}/package.mk", line, cores, arch)
def _sa(package: str, line: str, *cores: str) -> Install:
return Install(f"packages/sx05re/emulators/{package}/package.mk", line, cores)
def _lib32(package: str, line: str, *cores: str) -> Install:
return Install(f"packages/lib32/emuelec/libretro/{package}/package.mk", line, cores)
_TO = "${INSTALL}/usr/lib/libretro/"
# Package -> the ES core names its install step puts on the image, with the
# line that does it (v4.8). A libretro core is the .so it copies to
# /usr/lib/libretro, the name emuelecRunEmu.sh:101-116 loads as
# ${CORE}_libretro. A standalone name is the program the runner's case for
# it starts (emuelecRunEmu.sh:164-353), installed by the line cited, or by
# the package's default make/cmake/meson install where the line cited only
# adds the launch script beside it.
PROVIDERS: dict[str, tuple[Install, ...]] = {
"2048": (_lr("2048", f"cp 2048_libretro.so {_TO}", "2048"),),
"81": (_lr("81", f"cp 81_libretro.so {_TO}", "81"),),
"a5200": (_lr("a5200", f"cp a5200_libretro.so {_TO}", "a5200"),),
"atari800": (_lr("atari800", f"cp atari800_libretro.so {_TO}", "atari800"),),
"beetle-lynx": (_lr("beetle-lynx", f"cp mednafen_lynx_libretro.so {_TO}", "mednafen_lynx"),),
"beetle-ngp": (_lr("beetle-ngp", f"cp mednafen_ngp_libretro.so {_TO}", "mednafen_ngp"),),
"beetle-pce": (_lr("beetle-pce", f"cp mednafen_pce_fast_libretro.so {_TO}",
"mednafen_pce_fast"),),
"beetle-pcfx": (_lr("beetle-pcfx", f"cp mednafen_pcfx_libretro.so {_TO}", "mednafen_pcfx"),),
# line 20: the build makes mednafen_saturn_libretro.so (Makefile TARGET_NAME).
"beetle-saturn": (_lr("beetle-saturn", "cp mednafen_saturn_*.so $INSTALL/usr/lib/libretro/",
"mednafen_saturn"),),
"beetle-supergrafx": (_lr("beetle-supergrafx", f"cp mednafen_supergrafx_libretro.so {_TO}",
"mednafen_supergrafx"),),
"beetle-vb": (_lr("beetle-vb", f"cp mednafen_vb_libretro.so {_TO}", "mednafen_vb"),),
"beetle-wswan": (_lr("beetle-wswan", f"cp mednafen_wswan_libretro.so {_TO}",
"mednafen_wswan"),),
"bk": (_lr("bk", f"cp ${{PKG_BUILD}}/bk_libretro.so {_TO}", "bk"),),
"bluemsx": (_lr("bluemsx", f"cp bluemsx_libretro.so {_TO}", "bluemsx"),),
"cannonball": (_lr("cannonball", f"cp cannonball_libretro.so {_TO}", "cannonball"),),
"cap32": (_lr("cap32", f"cp cap32_libretro.so {_TO}", "cap32"),),
"daphne": (_lr("daphne", f"cp daphne_libretro.so {_TO}", "daphne"),),
"dinothawr": (_lr("dinothawr", f"cp dinothawr_libretro.so {_TO}", "dinothawr"),),
"dosbox-pure": (_lr("dosbox-pure", f"cp dosbox_pure_libretro.so {_TO}dosbox_pure_libretro.so",
"dosbox_pure"),),
"dosbox-svn": (_lr("dosbox-svn", "cp ${PKG_BUILD}/libretro/dosbox_svn_libretro.so "
"${INSTALL}/usr/lib/libretro", "dosbox_svn"),),
"easyrpg": (_lr("easyrpg", f"cp ${{PKG_BUILD}}/.${{TARGET_NAME}}/easyrpg_libretro.so {_TO}",
"easyrpg"),),
"fake_08": (_lr("fake_08", f"cp -v ${{PKG_BUILD}}/platform/libretro/fake08_libretro.so {_TO}",
"fake08"),),
"fbalpha2012": (_lr("fbalpha2012", f"cp fbalpha2012_libretro.so {_TO}", "fbalpha2012"),),
"fbneo": (_lr("fbneo", f"cp ${{PKG_BUILD}}/src/burner/libretro/fbneo_libretro.so {_TO}",
"fbneo"),),
"fceumm": (_lr("fceumm", f"cp fceumm_libretro.so {_TO}", "fceumm"),),
"fceumm-mod": (_lr("fceumm-mod", f"cp ${{PKG_BUILD}}/fceumm_mod_libretro.* {_TO}",
"fceumm_mod"),),
# lines 47-51: flycast_32b on arm, flycast elsewhere.
"flycast": (
_lr("flycast", f"cp flycast_libretro.so {_TO}flycast_32b_libretro.so", "flycast_32b",
arch="arm"),
_lr("flycast", f"cp flycast_libretro.so {_TO}", "flycast", arch="!arm"),
),
"fmsx-libretro": (_lr("fmsx-libretro", f"cp fmsx_libretro.so {_TO}", "fmsx"),),
"freeintv": (_lr("freeintv", f"cp freeintv_libretro.so {_TO}", "freeintv"),),
"freej2me": (_lr("freej2me", 'cp `find . -name "freej2me_libretro.so" | xargs echo` '
"${INSTALL}/usr/lib/libretro", "freej2me"),),
"fuse-libretro": (_lr("fuse-libretro", f"cp fuse_libretro.so {_TO}", "fuse"),),
"gambatte": (_lr("gambatte", f"cp gambatte_libretro.so {_TO}", "gambatte"),),
"gearboy": (_lr("gearboy", f"cp platforms/libretro/gearboy_libretro.so {_TO}", "gearboy"),),
"gearcoleco": (_lr("gearcoleco", f"cp platforms/libretro/gearcoleco_libretro.so {_TO}",
"gearcoleco"),),
"geargrafx": (_lr("geargrafx", f"cp platforms/libretro/geargrafx_libretro.so {_TO}",
"geargrafx"),),
"gearlynx": (_lr("gearlynx", f"cp platforms/libretro/gearlynx_libretro.so {_TO}",
"gearlynx"),),
"gearsystem": (_lr("gearsystem", f"cp platforms/libretro/gearsystem_libretro.so {_TO}",
"gearsystem"),),
"genesis-plus-gx": (_lr("genesis-plus-gx",
f"cp genesis_plus_gx_libretro.so {_TO}genesis_plus_gx_libretro.so",
"genesis_plus_gx"),),
"genesis-plus-gx-wide": (_lr("genesis-plus-gx-wide", "cp genesis_plus_gx_wide_libretro.so "
f"{_TO}genesis_plus_gx_wide_libretro.so",
"genesis_plus_gx_wide"),),
# PKG_NAME is genesis_plus_gx_cart_special (line 4).
"genesis_plus_gx_cart_special": (_lr("genesis_plus_gx_cart_special",
f"cp genesis_plus_gx_libretro.so {_TO}"
"${PKG_NAME}_libretro.so",
"genesis_plus_gx_cart_special"),),
"geolith": (_lr("geolith", f"cp geolith_libretro.so {_TO}", "geolith"),),
"gpsp": (_lr("gpsp", f"cp -v gpsp_libretro.so {_TO}", "gpsp"),),
"gw-libretro": (_lr("gw-libretro", f"cp gw_libretro.so {_TO}", "gw"),),
"handy": (_lr("handy", f"cp handy_libretro.so {_TO}", "handy"),),
"hatari": (_lr("hatari", f"cp ../hatari_libretro.so {_TO}", "hatari"),),
# line 65: the build's only .so is mame_libretro.so.
"mame": (_lr("mame", f"cp *.so {_TO}", "mame"),),
"mame2003-plus": (_lr("mame2003-plus", f"cp mame2003_plus_libretro.so {_TO}",
"mame2003_plus"),),
"mame2010": (_lr("mame2010", f"cp mame2010_libretro.so {_TO}", "mame2010"),),
"meowpc98": (_lr("meowpc98", f"cp libretro/nekop2_libretro.so {_TO}", "nekop2"),),
"mesen": (_lr("mesen", f"cp Libretro/mesen_libretro.so {_TO}", "mesen"),),
"mesen-s": (_lr("mesen-s", f"cp Libretro/mesen-s_libretro.so {_TO}", "mesen-s"),),
"mgba": (_lr("mgba", f"cp mgba_libretro.so {_TO}", "mgba"),),
"mrboom": (_lr("mrboom", f"cp mrboom_libretro.so {_TO}", "mrboom"),),
# PKG_ARCH="arm" (line 25): scripts/build skips it on aarch64.
"mupen64plus": (_lr("mupen64plus", f"cp mupen64plus_libretro.so {_TO}"
"mupen64plus_32b_libretro.so", "mupen64plus_32b"),),
"mupen64plus-nx": (_lr("mupen64plus-nx", f"cp mupen64plus_next_libretro.so {_TO}",
"mupen64plus_next"),),
"mupen64plus-nx-alt": (_lr("mupen64plus-nx-alt", f"cp mupen64plus_next_libretro.so {_TO}"
"mupen64plus_next_alt_libretro.so", "mupen64plus_next_alt"),),
"neocd_libretro": (_lr("neocd_libretro", f"cp ${{PKG_BUILD}}/neocd_libretro.so {_TO}",
"neocd"),),
"nestopia": (_lr("nestopia", f"cp libretro/nestopia_libretro.so {_TO}", "nestopia"),),
"np2kai": (_lr("np2kai", f"cp ${{PKG_BUILD}}/sdl/np2kai_libretro.so {_TO}", "np2kai"),),
"nxengine": (_lr("nxengine", f"cp nxengine_libretro.so {_TO}", "nxengine"),),
"o2em": (_lr("o2em", f"cp o2em_libretro.so {_TO}", "o2em"),),
"opera": (_lr("opera", f"cp opera_libretro.so {_TO}", "opera"),),
# lines 42-46: pcsx_rearmed_32b on arm, pcsx_rearmed elsewhere.
"pcsx_rearmed": (
_lr("pcsx_rearmed", "cp pcsx_rearmed_libretro.so "
"${INSTALL}${INSTALLTO}/pcsx_rearmed_32b_libretro.so", "pcsx_rearmed_32b", arch="arm"),
_lr("pcsx_rearmed", "cp pcsx_rearmed_libretro.so ${INSTALL}${INSTALLTO}",
"pcsx_rearmed", arch="!arm"),
),
"picodrive": (_lr("picodrive", f"cp ${{PKG_BUILD}}/picodrive_libretro.so {_TO}",
"picodrive"),),
"pocketcdg": (_lr("pocketcdg", f"cp pocketcdg_libretro.so {_TO}", "pocketcdg"),),
"pokemini": (_lr("pokemini", f"cp pokemini_libretro.so {_TO}", "pokemini"),),
"potator": (_lr("potator", "cp ${PKG_BUILD}/platform/libretro/potator_libretro.so "
f"{_TO}potator_libretro.so", "potator"),),
"prboom": (_lr("prboom", f"cp prboom_libretro.so {_TO}prboom_libretro.so", "prboom"),),
"prosystem": (_lr("prosystem", f"cp prosystem_libretro.so {_TO}", "prosystem"),),
"puae": (_lr("puae", f"cp puae_libretro.so {_TO}", "puae"),),
"puae2021": (_lr("puae2021", f"cp -v puae2021_libretro.so {_TO}", "puae2021"),),
"px68k": (_lr("px68k", f"cp px68k_libretro.so {_TO}", "px68k"),),
"reminiscence": (_lr("reminiscence", f"cp reminiscence_libretro.so {_TO}", "reminiscence"),),
"same_cdi": (_lr("same_cdi", f"cp -v same_cdi_libretro.so {_TO}", "same_cdi"),),
"sameboy": (_lr("sameboy", f"cp libretro/sameboy_libretro.so {_TO}", "sameboy"),),
"sameduck": (_lr("sameduck", f"cp build/bin/sameduck_libretro.so {_TO}", "sameduck"),),
"scummvm": (_lr("scummvm", 'cp "${PKG_BUILD}/backends/platform/libretro/scummvm_libretro."'
"{so,info} ${INSTALL}/usr/lib/libretro/", "scummvm"),),
"snes9x": (_lr("snes9x", f"cp libretro/snes9x_libretro.so {_TO}", "snes9x"),),
"snes9x2002": (_lr("snes9x2002", f"cp snes9x2002_libretro.so {_TO}", "snes9x2002"),),
"snes9x2005_plus": (_lr("snes9x2005_plus", f"cp snes9x2005_plus_libretro.so {_TO}",
"snes9x2005_plus"),),
"snes9x2010": (_lr("snes9x2010", f"cp snes9x2010_libretro.so {_TO}", "snes9x2010"),),
"stella": (_lr("stella", f"cp ${{PKG_BUILD}}/src/os/libretro/stella_libretro.so {_TO}",
"stella"),),
"supafaust": (_lr("supafaust", f"cp -v mednafen_supafaust_libretro.so {_TO}",
"mednafen_supafaust"),),
"swanstation": (_lr("swanstation", "cp ${PKG_BUILD}/.${TARGET_NAME}/swanstation_libretro.so "
f"{_TO}", "swanstation"),),
"tgbdual": (_lr("tgbdual", f"cp tgbdual_libretro.so {_TO}", "tgbdual"),),
"tic-80": (_lr("tic-80", "cp ${PKG_BUILD}/.${TARGET_NAME}/bin/tic80_libretro.so "
f"{_TO}tic80_libretro.so", "tic80"),),
"tyrquake": (_lr("tyrquake", f"cp tyrquake_libretro.so {_TO}", "tyrquake"),),
# PKG_ARCH="arm" (line 25): scripts/build skips it on aarch64.
"uae4arm": (_lr("uae4arm", f"cp uae4arm_libretro.so {_TO}uae4arm_32b_libretro.so",
"uae4arm_32b"),),
"uzem": (_lr("uzem", f"cp uzem_libretro.so {_TO}", "uzem"),),
"vba-next": (_lr("vba-next", f"cp vba_next_libretro.so {_TO}", "vba_next"),),
"vbam": (_lr("vbam", f"cp ../src/libretro/vbam_libretro.so {_TO}", "vbam"),),
"vecx": (_lr("vecx", f"cp vecx_libretro.so {_TO}vecx_libretro.so", "vecx"),),
# line 55: make_target builds one vice_<emu>_libretro.so per emulator it
# loops over (lines 37-50), x64, x64sc, x128, xplus4 and xvic among them.
"vice": (_lr("vice", f"cp built/vice_*_libretro.so {_TO}",
"vice_x128", "vice_x64", "vice_x64sc", "vice_xplus4", "vice_xvic"),),
"virtualjaguar": (_lr("virtualjaguar", f"cp virtualjaguar_libretro.so {_TO}",
"virtualjaguar"),),
"xmil": (_lr("xmil", f"cp ${{PKG_BUILD}}/libretro/x1_libretro.so {_TO}", "x1"),),
"xrick": (_lr("xrick", f"cp xrick_libretro.so {_TO}", "xrick"),),
"yabasanshiro": (_lr("yabasanshiro", f"cp yabause/src/libretro/yabasanshiro_libretro.so {_TO}",
"yabasanshiro"),),
"lib32-flycast": (_lib32("lib32-flycast", "cp -va flycast_libretro.so "
f"{_TO}flycast_32b_libretro.so", "flycast_32b"),),
"lib32-mupen64plus": (_lib32("lib32-mupen64plus", "cp mupen64plus_libretro.so "
f"{_TO}mupen64plus_32b_libretro.so", "mupen64plus_32b"),),
"lib32-parallel-n64": (_lib32("lib32-parallel-n64", "cp parallel_n64_libretro.so "
f"{_TO}parallel_n64_32b_libretro.so", "parallel_n64_32b"),),
"lib32-pcsx_rearmed": (_lib32("lib32-pcsx_rearmed", "cp pcsx_rearmed_libretro.so "
f"{_TO}pcsx_rearmed_32b_libretro.so", "pcsx_rearmed_32b"),),
"lib32-uae4arm": (_lib32("lib32-uae4arm", "cp uae4arm_libretro.so "
f"{_TO}uae4arm_32b_libretro.so", "uae4arm_32b"),),
# Standalone emulators, by the program the runner starts.
"PPSSPPSDL": (_sa("PPSSPPSDL", 'cp `find . -name "PPSSPPSDL" | xargs echo` '
"${INSTALL}/usr/bin/PPSSPPSDL", "PPSSPPSDL"),),
"advancemame": (_sa("advancemame", "cp -r ${PKG_BUILD}/obj/mame/linux/blend/advmame "
"${INSTALL}/usr/bin", "AdvanceMame"),),
"amiberry": (_sa("amiberry", "cp -a amiberry* ${INSTALL}/usr/bin/amiberry", "AMIBERRY"),),
"dolphinSA": (_sa("dolphinSA", "cp -rf ${PKG_BUILD}/.${TARGET_NAME}/Binaries/"
"dolphin-emu-nogui ${INSTALL}/usr/bin", "dolphin"),),
# dosbox comes from the meson install, dosbox.start from line 28.
"dosbox-staging": (_sa("dosbox-staging", "cp -a ${PKG_DIR}/scripts/* ${INSTALL}/usr/bin/",
"DOSBOXSDL2"),),
"duckstation": (_sa("duckstation", "cp -rf ${PKG_BUILD}/.${TARGET_NAME}/bin/"
"duckstation-nogui ${INSTALL}/usr/bin", "Duckstation"),),
"ecwolf": (_sa("ecwolf", "cp ${PKG_BUILD}/ecwolf ${INSTALL}/usr/bin/ecwolf", "ECWolf"),),
"fbneoSA": (_sa("fbneoSA", "cp -rf ${PKG_BUILD}/fbneo ${INSTALL}/usr/bin", "FBNEOSA"),),
"flycastsa": (_sa("flycastsa", "cp ${PKG_BUILD}/.${TARGET_NAME}/flycast "
"${INSTALL}/usr/bin/flycast", "flycastsa"),),
"hatarisa": (_sa("hatarisa", "cp src/hatari ${INSTALL}/usr/bin", "HATARISA"),),
# hypseus comes from the cmake install (PKG_TOOLCHAIN, line 13).
"hypseus-singe": (_sa("hypseus-singe", 'PKG_TOOLCHAIN="cmake"', "HYPSEUS"),),
"ikemen-go": (_sa("ikemen-go", 'cp "$PKG_BUILD/bin/Ikemen_Go_"* "$INSTALL/usr/bin/Ikemen_Go"',
"ikemen"),),
"jzintv": (_sa("jzintv", "cp bin/jzintv ${INSTALL}/usr/bin", "jzintv"),),
# The metapackage's dependencies (line 10). mupen64plus itself comes from
# mupen64plussa-ui-console's makeinstall_target.
"mupen64plussa": (
_sa("mupen64plussa", 'PKG_DEPENDS_TARGET="toolchain mupen64plussa-core '
"mupen64plussa-ui-console mupen64plussa-audio-sdl mupen64plussa-input-sdl "
'mupen64plussa-rsp-hle mupen64plussa-video-rice mupen64plussa-video-glide64mk2"',
"mupen64plussa"),
Install("packages/sx05re/emulators/mupen64plussa/mupen64plussa-ui-console/package.mk",
"cp ${PKG_BUILD}/projects/unix/mupen64plus ${UBINDIR}", ()),
),
"openbor": (_sa("openbor", 'cp `find . -name "OpenBOR.elf" | xargs echo` '
"${INSTALL}/usr/bin/OpenBOR", "OPENBOR"),),
# scummvm comes from the make install, moved to /usr/bin at line 51.
"scummvmsa": (_sa("scummvmsa", "mv ${INSTALL}/usr/local/bin ${INSTALL}/usr/", "SCUMMVMSA"),),
# solarus-run comes from the cmake install, solarus.sh from line 22.
"solarus": (_sa("solarus", "cp ${PKG_DIR}/scripts/* ${INSTALL}/usr/bin", "Solarus"),),
"stellasa": (_sa("stellasa", "cp -rf ${PKG_BUILD}/stella ${INSTALL}/usr/bin", "STELLASA"),),
"yabasanshiroSA_1_11": (_sa("yabasanshiroSA_1_11", "cp -a ${PKG_BUILD}/src/retro_arena/"
"yabasanshiro ${INSTALL}/usr/bin", "yabasanshiro"),),
"lib32-bennugd-monolithic": (Install(
"packages/lib32/emuelec/ports/lib32-bennugd-monolithic/package.mk",
"cp ${PKG_BUILD}/projects/cmake/bgdi/bgdi ${INSTALL}/usr/bin", ("bgdi",)),),
# The metapackage's own bin/ (line 82) carries pico8.sh, which starts the
# PICO-8 binary the user copies to bios/pico-8.
"emuelec": (Install(META_MK, "cp -rf ${PKG_DIR}/bin ${INSTALL}/usr", ("Pico-8",)),),
}
# Packages of a device list whose install step puts no ES core on the image.
NO_ES_CORE: dict[str, str] = {
"toolchain": "build toolchain",
"${OPENGLES}": "graphics driver",
"emuelec-emulationstation": "the frontend",
"retroarch": "the libretro frontend",
"lib32-retroarch": "the 32-bit libretro frontend, retroarch32",
"emuelec-tools": "system tools",
"emuelec-32bit-info": "the .info files of the 32-bit cores",
"lib32-essential": "32-bit runtime",
"lib32-libusb": "32-bit library",
"lib32-box86": "x86 translator",
"fbterm": "terminal",
"kmscon": "terminal",
"odroidgoa-utils": "device tools and retrorun, a runner",
"amiberry-lite": "amiberry-lite, which es_systems.json names nowhere",
"yabasanshiroSA_1_5": "yabasanshiro1_5, which es_systems.json names nowhere",
"lib32-droidports": "gmloader, which es_cores drops (_overrides.yml)",
"gme": "gme_libretro.so, which es_systems.json names nowhere",
"nestopiaCV": "nestopiaCV_libretro.so, which es_systems.json names nowhere",
"quasi88": "quasi88_libretro.so, which es_systems.json names nowhere",
"spring": "spring_*_libretro.so, which es_systems.json names nowhere",
}
# ES core names no package of any device builds, by the package that would.
# The port engines are dependencies of emuelec-ports (its package.mk,
# lines 17-59), and no package of the image depends on emuelec-ports;
# flycast-dojo is in no device list.
UNBUILT: dict[str, str] = {
**dict.fromkeys(
("CGeniusExe", "Chocolate-Doom", "GZDoom", "LZDoom", "OpenJazz", "RigelEngine",
"VVVVVV", "abuse", "bs", "bstone", "ccleste", "cdogs-sdl", "devilutionx",
"eduke32", "fallout-ce", "fallout2-ce", "fheroes2", "hcl", "hode", "hurrican",
"opentyrian", "prince", "smw", "sonic2013", "soniccd", "sonicmania", "supertux2",
"supertuxkart", "xash3d"),
"emuelec-ports",
),
"flycast_dojo": "flycast-dojo",
}
def release_devices(asset_names: list[str]) -> dict[str, str]:
"""{device: architecture} of each image a release publishes."""
found: dict[str, str] = {}
for name in asset_names:
match = _ASSET_RE.match(name)
if match:
found.setdefault(match.group("device"), match.group("arch"))
if not found:
raise RuntimeError("the EmuELEC release publishes no device image")
return found
def _require(text: str, lines: tuple[str, ...], where: str) -> None:
present = {line.strip() for line in text.splitlines()}
missing = [line for line in lines if line not in present]
if missing:
raise RuntimeError(f"{where} no longer reads {missing[0]!r}")
def _quoted(text: str, name: str, where: str) -> list[str]:
"""The words of `NAME="..."`; a backslash-newline inside the quotes is
removed by bash, so the value reads as one line."""
match = re.search(rf'^\s*{name}="([^"]*)"', text, re.MULTILINE)
if match is None:
raise RuntimeError(f"{where} sets no {name}")
return match.group(1).replace("\\\n", " ").split()
class Metapackage(NamedTuple):
emus: list[str]
experimental: list[str]
s922x: list[str]
lib32: list[str]
def read_metapackage(package_mk: str, options: str) -> Metapackage:
"""The lists the metapackage rules work on, with PKG_EMUS expanded."""
_require(package_mk, _META_LINES, META_MK)
emus = _quoted(package_mk, "PKG_EMUS", META_MK)
if emus[:1] != ["${LIBRETRO_CORES}"]:
raise RuntimeError(f"{META_MK}: PKG_EMUS no longer starts with ${{LIBRETRO_CORES}}")
lib32 = _LIB32_RE.search(package_mk)
if lib32 is None:
raise RuntimeError(f"{META_MK}: no aarch64 lib32 list")
return Metapackage(
emus=_quoted(options, "LIBRETRO_CORES", OPTIONS) + emus[1:],
experimental=_quoted(package_mk, "PKG_EXPERIMENTAL", META_MK),
s922x=_quoted(options, "LIBRETRO_S922X_CORES", OPTIONS),
lib32=lib32.group(1).replace("\\\n", " ").split(),
)
def _sed_first(words: str, pattern: str) -> str:
"""`echo ${PKG_DEPENDS_TARGET} | sed "s|<pattern>| |"`: the unquoted echo
joins the words with single spaces, sed replaces the first match only,
anywhere in the line, a word's tail included."""
line = " ".join(words.split())
at = line.find(pattern)
return line if at < 0 else f"{line[:at]} {line[at + len(pattern):]}"
def device_packages(meta: Metapackage, device: str, arch: str) -> list[str]:
"""PKG_DEPENDS_TARGET of the metapackage for one device (lines 8-77)."""
depends = "toolchain ${OPENGLES} emuelec-emulationstation retroarch"
depends += " emuelec-tools " + " ".join(meta.emus) + " " + " ".join(meta.experimental)
if device in _BIG:
depends += " " + " ".join(meta.s922x)
if device in _HANDHELD:
depends += " kmscon odroidgoa-utils"
for discore in ("duckstation", "mesen-s", "virtualjaguar", "quicknes", "MC"):
depends = _sed_first(depends, f"{discore} ")
depends += " yabasanshiro"
else:
depends += " fbterm"
if arch == "aarch64":
for discore in ("quicknes", "parallel-n64", "pcsx_rearmed"):
depends = _sed_first(depends, discore)
depends += " " + " ".join(meta.lib32)
if device in _BIG:
depends += " dolphinSA"
if device == "Amlogic-old":
for discore in ("yabasanshiroSA_1_11", "yabasanshiroSA_1_5", "yabasanshiro",
"same_cdi", "duckstation"):
depends = _sed_first(depends, f"{discore} ")
if device in _BIG:
depends += " mame"
if device in ("RK356x", "OdroidM1"):
depends = _sed_first(depends, "flycast-dojo")
return depends.split()
def arch_builds(pkg_arch: str, arch: str) -> bool:
"""scripts/build:28-30: an empty PKG_ARCH builds everywhere, `!<arch>`
excludes, otherwise the list names the arch or `any`."""
words = pkg_arch.split()
if not words:
return True
if f"!{arch}" in words:
return False
return arch in words or "any" in words
def pkg_arch(package_mk: str) -> str:
match = re.search(r'^PKG_ARCH="([^"]*)"', package_mk, re.MULTILINE)
return match.group(1) if match else ""
def es_systems_for(es_json: str, es_mk: str, device: str) -> SystemEmulators:
"""es_systems.json as es_systems.cfg reads after makeinstall_target's
deletions on one device (emuelec-emulationstation/package.mk:114-143).
`sed "s|<core>X</core>||g"` matches a core written without attributes:
generate_es_systems.py gives `default="true"` to the first default core
of a system and `incompatible_extensions` to a core listing them.
"""
_require(es_mk, _ES_LINES, ES_MK)
if device in _HANDHELD:
remove_cores = {"mesen-s", "quicknes", "mame2016", "mesen"}
drop_systems: set[str] = set()
drop_emulators = {"Duckstation"}
elif device == "Amlogic-old":
remove_cores = {"mesen-s", "quicknes", "mame2016", "mesen", "yabasanshiroSA",
"yabasanshiro"}
drop_systems = {"saturn", "phillips-cdi"}
drop_emulators = {"Duckstation", "retrorun"}
else:
# Line 117 holds for every device: no name is all three at once.
remove_cores = set()
drop_systems = set()
drop_emulators = {"retrorun"}
systems: SystemEmulators = {}
for system in json.loads(es_json).get("systems", []):
if system["name"] in drop_systems:
continue
default_set = False
emulators = []
for emulator in system.get("emulators") or []:
kept = []
for core in emulator.get("cores") or []:
bare = "incompatible" not in core
if core.get("default", False) and not default_set:
default_set = True
bare = False
if not (bare and core["name"] in remove_cores):
kept.append(core["name"])
if emulator["name"] not in drop_emulators:
emulators.append((emulator["name"], kept))
systems[system["name"]] = emulators
return systems
def device_cores(
packages: list[str],
arch: str,
archs: dict[str, str],
systems: SystemEmulators,
) -> list[str]:
"""ES core names a device both installs and offers.
`archs` maps each package of PROVIDERS to its PKG_ARCH.
"""
unknown = sorted(
p for p in set(packages) if p not in PROVIDERS and p not in NO_ES_CORE
and p not in UNBUILT.values()
)
if unknown:
raise RuntimeError(f"{META_MK}: no install step read for {', '.join(unknown)}")
unbuilt = sorted(set(packages) & set(UNBUILT.values()))
if unbuilt:
raise RuntimeError(f"{META_MK}: {unbuilt[0]} is built and its cores are read as absent")
installed: set[str] = set()
for package in [*set(packages), "emuelec"]:
if package not in PROVIDERS or not arch_builds(archs[package], arch):
continue
for install in PROVIDERS[package]:
if install.arch is None or (install.arch == arch) or (
install.arch.startswith("!") and install.arch[1:] != arch
):
installed.update(install.cores)
offered, _standalone, _dual = es_cores(
systems, RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME)
)
return sorted(installed & set(offered))
def check_names(systems: SystemEmulators) -> None:
"""Every ES core name is installed by a package read or known unbuilt."""
offered, _standalone, _dual = es_cores(
systems, RUNNERS, BUILDS_OF, _not_cores(PLATFORM_NAME)
)
provided = {core for installs in PROVIDERS.values() for i in installs for core in i.cores}
unread = sorted(set(offered) - provided - set(UNBUILT))
if unread:
raise RuntimeError(f"es_systems.json names {', '.join(unread)}, no package read installs it")
def check_installs(texts: dict[str, str]) -> dict[str, str]:
"""{package: PKG_ARCH}, after checking each cited install line is there."""
archs: dict[str, str] = {}
for package, installs in PROVIDERS.items():
for install in installs:
_require(texts[install.path], (install.line,), install.path)
archs[package] = pkg_arch(texts[installs[0].path])
return archs
class Scraper(BaseTargetScraper):
"""Per-device core lists from EmuELEC's metapackage."""
def __init__(self, url: str = "https://github.com/" + REPO) -> None:
super().__init__(url=url)
self.ref = ""
def _raw(self, path: str) -> str:
return fetch_bytes(RAW.format(ref=self.ref, path=path)).decode("utf-8")
def fetch_targets(self) -> dict:
self.ref = self.pinned_revision(PLATFORM_NAME)
release = json.loads(
fetch_bytes(RELEASE_API.format(ref=self.ref), headers=github_headers())
)
devices = release_devices([asset["name"] for asset in release.get("assets", [])])
meta = read_metapackage(self._raw(META_MK), self._raw(OPTIONS))
paths = sorted({i.path for installs in PROVIDERS.values() for i in installs})
archs = check_installs({path: self._raw(path) for path in paths})
es_json = fetch_bytes(ES_SYSTEMS_TEMPLATE.format(ref=self.ref)).decode("utf-8")
ports = port_engines(fetch_bytes(PORTS_TEMPLATE.format(ref=self.ref)).decode("utf-8"))
es_mk = self._raw(ES_MK)
check_names(with_ports(es_systems_for(es_json, es_mk, "Amlogic-ng"), ports))
targets: dict[str, dict] = {}
for device, arch in sorted(devices.items()):
systems = with_ports(es_systems_for(es_json, es_mk, device), ports)
cores = device_cores(device_packages(meta, device, arch), arch, archs, systems)
targets[device.lower()] = {"architecture": arch, "cores": cores}
print(f" {device.lower()} ({arch}): {len(cores)} cores", file=sys.stderr)
return {
"platform": PLATFORM_NAME,
"source": f"{self.url}/blob/{self.ref}/{META_MK}",
"devices_source": release["html_url"],
"scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"targets": targets,
}
def main() -> None:
parser = argparse.ArgumentParser(
allow_abbrev=False, description="Scrape EmuELEC per-device cores"
)
parser.add_argument("--dry-run", action="store_true", help="Show target summary")
parser.add_argument("--output", "-o", help="Output YAML file")
args = parser.parse_args()
scraper = Scraper()
data = scraper.fetch_targets()
if args.dry_run:
for name, info in data["targets"].items():
print(f" {name} ({info['architecture']}): {len(info['cores'])} cores")
return
if args.output:
scraper.write_output(data, args.output)
print(f"Written to {args.output}")
return
print(yaml.dump(data, default_flow_style=False, sort_keys=False))
if __name__ == "__main__":
main()
scripts/sitediagrams.py
+27 -10
View File
@@ -36,6 +36,7 @@ from xml.sax.saxutils import escape
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
import siteinputs # noqa: E402
import sitemarkup as markup # noqa: E402
PLACEHOLDER = re.compile(r"(?m)^([ \t]*)<!--\s*image:\s*([A-Za-z0-9_-]+)\s*-->[ \t]*$")
@@ -314,21 +315,29 @@ def svg(figure: Figure, layout: str, suffix: str) -> str:
# Data
@lru_cache(maxsize=None)
def registry() -> dict:
return _registry_in(str(siteinputs.current().platforms))
@lru_cache(maxsize=None)
def _registry_in(platforms_dir: str) -> dict:
import yaml
with open(ROOT / "platforms" / "_registry.yml", encoding="utf-8") as handle:
with open(Path(platforms_dir) / "_registry.yml", encoding="utf-8") as handle:
return yaml.safe_load(handle).get("platforms", {})
@lru_cache(maxsize=None)
def setups() -> tuple[str, ...]:
"""The registered setups, archived ones included: each has its own page,
pack and file list, so every count of setups on the site is this one."""
return _setups_in(str(siteinputs.current().platforms))
@lru_cache(maxsize=None)
def _setups_in(platforms_dir: str) -> tuple[str, ...]:
from common import list_registered_platforms
return tuple(sorted(list_registered_platforms(str(ROOT / "platforms"), include_archived=True)))
return tuple(sorted(list_registered_platforms(platforms_dir, include_archived=True)))
def setups_named(named: list[str]) -> str:
@@ -339,15 +348,19 @@ def setups_named(named: list[str]) -> str:
return ", ".join(named) + (f" and {others} more" if others else "")
@lru_cache(maxsize=None)
def platform_modes() -> dict[str, tuple[str, str, str | None]]:
"""name -> (display name, verification mode, hash rule) for every platform."""
return _platform_modes_in(str(siteinputs.current().platforms))
@lru_cache(maxsize=None)
def _platform_modes_in(platforms_dir: str) -> dict[str, tuple[str, str, str | None]]:
from common import load_platform_config
out: dict[str, tuple[str, str, str | None]] = {}
for name in setups():
config = load_platform_config(name, str(ROOT / "platforms"))
rule = registry().get(name, {}).get("hash_rule") or config.get("hash_rule")
for name in _setups_in(platforms_dir):
config = load_platform_config(name, platforms_dir)
rule = _registry_in(platforms_dir).get(name, {}).get("hash_rule") or config.get("hash_rule")
out[name] = (config.get("platform", name), config.get("verification_mode", ""), rule)
return out
@@ -365,13 +378,17 @@ def _folder(name: str) -> str:
CHAIN_EXAMPLES = (("batocera", "duckstation"), ("emudeck", "pcsx2"), ("retroarch", "beetle_psx"))
@lru_cache(maxsize=None)
def emulator_names() -> dict[str, str]:
return _emulator_names_in(str(siteinputs.current().emulators))
@lru_cache(maxsize=None)
def _emulator_names_in(emulators_dir: str) -> dict[str, str]:
import yaml
names: dict[str, str] = {}
for _, emulator in CHAIN_EXAMPLES:
with open(ROOT / "emulators" / f"{emulator}.yml", encoding="utf-8") as handle:
with open(Path(emulators_dir) / f"{emulator}.yml", encoding="utf-8") as handle:
label = str(yaml.safe_load(handle).get("emulator", emulator))
names[emulator] = re.sub(r"\s*\(.*\)$", "", label)
return names
scripts/siteinputs.py
+32
View File
@@ -0,0 +1,32 @@
"""The collection, profiles and platform files the site describes.
generate_site.py takes them as --db, --emulators-dir and --platforms-dir and
installs them here before writing a page. Every page module reads its inputs
from current(), so a run on another collection describes that collection on
every page. A module used on its own reads the repository's files.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
@dataclass(frozen=True)
class SiteInputs:
db: Path = ROOT / "database.json"
emulators: Path = ROOT / "emulators"
platforms: Path = ROOT / "platforms"
_current = [SiteInputs()]
def current() -> SiteInputs:
return _current[0]
def use(inputs: SiteInputs) -> None:
_current[0] = inputs
scripts/siterender.py
+8 -5
View File
@@ -7,6 +7,7 @@ import urllib.parse
from collections.abc import Callable
from pathlib import Path
import siteinputs
import sitemarkup as markup
@@ -401,7 +402,8 @@ def _build_boards() -> None:
from common import load_emulator_profiles
database = _REPO_ROOT / "database.json"
inputs = siteinputs.current()
database = inputs.db
if not database.exists():
_archive_boards.clear()
_board_names.clear()
@@ -409,7 +411,7 @@ def _build_boards() -> None:
return
with open(database, encoding="utf-8") as handle:
db = json.load(handle)
profiles = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False)
profiles = load_emulator_profiles(str(inputs.emulators), skip_aliases=False)
_arcade_source[0] = (profiles, db)
profiles, db = _arcade_source[0]
files = db.get("files", {})
@@ -780,9 +782,10 @@ def _ensure_console_sections() -> None:
from common import load_emulator_profiles
from generate_site import _group_by_manufacturer
with open(_REPO_ROOT / "database.json", encoding="utf-8") as handle:
inputs = siteinputs.current()
with open(inputs.db, encoding="utf-8") as handle:
db = json.load(handle)
profiles = load_emulator_profiles(str(_REPO_ROOT / "emulators"), skip_aliases=False)
profiles = load_emulator_profiles(str(inputs.emulators), skip_aliases=False)
set_console_sections(_group_by_manufacturer(db), program_names(profiles))
@@ -1431,7 +1434,7 @@ def system_search_aliases(sys_id: str) -> list[str]:
platform_system_names,
)
platforms_dir = str(_REPO_ROOT / "platforms")
platforms_dir = str(siteinputs.current().platforms)
index: dict[str, set[str]] = {}
for platform in list_registered_platforms(platforms_dir, include_archived=True):
config = load_platform_config(platform, platforms_dir)
scripts/sitesources.py
+19
View File
@@ -321,9 +321,28 @@ def build_source_paths(
# Recorded even when it holds no cited path, so a pinned tree
# missing from the record is one no run has read.
held.setdefault(tree_key(repo, pin), set()).update(paths & tree)
for repo, pin, path in build_files(profile):
tree = (_cached_tree(repo, pin, cache_dirs, fetch)
or _listed_directories(repo, pin, {path}, cache_dirs, fetch))
if tree is None:
unread.add(tree_key(repo, pin))
continue
held.setdefault(tree_key(repo, pin), set()).update({path} & tree)
return {key: sorted(paths) for key, paths in sorted(held.items())}, sorted(unread)
def build_files(profile: dict) -> list[tuple[upstream.Repo, str, str]]:
"""The platform build files `pin_build` names at a commit, as (repo,
revision, path). A tag can move, so a build read at a tag is not linked."""
found = []
for build in profile.get("pin_build") or []:
repo = _browse_repo(str(build.get("repo", "")))
revision = str(build.get("revision", ""))
if repo is not None and re.fullmatch(r"[0-9a-f]{40}", revision):
found.append((repo, revision, str(build.get("path", ""))))
return found
def keep_unread(
held: dict[str, list[str]], unread: list[str], previous: dict[str, list[str]]
) -> dict[str, list[str]]:
scripts/slots.py
+76 -17
View File
@@ -43,7 +43,12 @@ from common import (
variant_base_name,
)
from packresolve import preferred_entries
from validation import outside_gap_scope, read_from_save_dir, with_archive_members
from validation import (
content_alternatives,
outside_gap_scope,
read_from_save_dir,
with_archive_members,
)
if TYPE_CHECKING:
from packextras import ExtraPlacement
@@ -85,6 +90,9 @@ class Claim:
entry: dict = field(default_factory=dict)
local_path: str | None = None
status: str = ""
# A core that identifies its firmware by content reads no file by this
# name: any dump it takes for the same system and region answers for it.
accepts: tuple[dict, ...] = ()
@property
def is_proven(self) -> bool:
@@ -260,6 +268,7 @@ def profile_claims(
)
if status == "name_exact" and _size_the_code_reads(entry, local):
status = SIZE_EXACT
accepts = tuple(content_alternatives(entry, profile))
claims.append(
Claim(
origin="profile",
@@ -269,6 +278,7 @@ def profile_claims(
entry=entry,
local_path=local,
status=status,
accepts=accepts,
)
)
return claims
@@ -514,6 +524,7 @@ def pack_overrides(
data_dir_registry: dict | None = None,
target_cores: set[str] | None = None,
source: str = "full",
choices: dict[str, list[tuple[str, str | None]]] | None = None,
) -> dict[str, str]:
"""Full pack destination -> the file a platform pack serves there instead.
@@ -522,7 +533,8 @@ def pack_overrides(
existence mode the frontend never reads the bytes, so serving the
emulator's file satisfies both. The ZIP builder, the install manifest and
verify all read this: two of them deciding alone gave the one-line
installer different bytes than the ZIP.
installer different bytes than the ZIP. ``choices`` is slot_choices
for the same arguments when the caller already holds it.
"""
# A baseline-only pack serves what the platform declares, and a target's
# pack answers to the cores that hardware builds: DuckStation does not
@@ -531,10 +543,10 @@ def pack_overrides(
return {}
cores = set(resolve_platform_cores(config, profiles, target_cores=target_cores))
overrides: dict[str, str] = {}
for destination, choices in slot_choices(
config, profiles, db, zip_contents, data_dir_registry
).items():
path = next((path for emulator, path in choices if emulator in cores), None)
if choices is None:
choices = slot_choices(config, profiles, db, zip_contents, data_dir_registry)
for destination, ordered in choices.items():
path = next((path for emulator, path in ordered if emulator in cores), None)
if path:
overrides[destination] = path
return overrides
@@ -677,12 +689,21 @@ class Collision:
resolved: list[str]
# Each declaration with the file it resolved to, in the platform's order.
declarations: list[tuple[dict, str | None]] = field(default_factory=list)
# The system of each declaration, in the same order.
systems: list[str] = field(default_factory=list)
# The file the pack ships at the destination, as the builder decides it.
shipped: str | None = None
def _collision_json(collision: Collision) -> dict:
return {
"destination": collision.destination,
"resolved": list(collision.resolved),
"shipped": collision.shipped,
"declarations": [
{"system": system, "name": entry.get("name", ""), "file": local}
for system, (entry, local) in zip(collision.systems, collision.declarations)
],
}
@@ -702,19 +723,27 @@ def find_collisions(
CoCo disk ROM at one bios/disk.rom.
"""
fold = bool(config.get("case_insensitive_fs", False))
by_dest: dict[str, list[dict]] = {}
for system in (config.get("systems") or {}).values():
systems = config.get("systems") or {}
# The builder lets a hash-constrained declaration that resolves by
# content take the path, else the first it reaches walking the systems
# sorted: the same choice says which file ships.
preferred = set(
preferred_entries(systems, db, "bios", "", False, zip_contents, True).values()
)
by_dest: dict[str, list[tuple[str, dict]]] = {}
for sys_id, system in systems.items():
for entry in system.get("files") or []:
if not isinstance(entry, dict):
continue
dest = entry.get("destination") or entry.get("name") or ""
if dest:
by_dest.setdefault(_normalize(dest, fold), []).append(entry)
by_dest.setdefault(_normalize(dest, fold), []).append((sys_id, entry))
collisions = []
for key, entries in sorted(by_dest.items()):
if len(entries) < 2:
for key, claims in sorted(by_dest.items()):
if len(claims) < 2:
continue
entries = [entry for _sys_id, entry in claims]
resolved = []
declarations = []
for entry in entries:
@@ -734,8 +763,22 @@ def find_collisions(
# also names the first one's file.
if any(_accepted_by_all(path, entries, db) for path in resolved):
continue
reached = sorted(
(sys_id, local)
for (sys_id, _entry), (_e, local) in zip(claims, declarations) if local
)
shipped = next(
(local for entry, local in declarations if local and id(entry) in preferred),
reached[0][1],
)
collisions.append(
Collision(destination=key, resolved=resolved, declarations=declarations)
Collision(
destination=key,
resolved=resolved,
declarations=declarations,
systems=[sys_id for sys_id, _entry in claims],
shipped=shipped,
)
)
return collisions
@@ -746,6 +789,10 @@ def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool:
They agree when they resolve to the same bytes, when the platform's own
hash list accepts the profile's file, or when the profile's accepts the
platform's. A contradiction needs both lists to reject the other side.
A core found by content (found_by: content) reads no file by the
destination's name, so any dump it takes for the same system and region
settles the slot: DuckStation takes Batocera's scph1001.bin v2.0, which its profile
labels scph1001_v20.bin.
"""
if profile.local_path == platform.local_path:
return True
@@ -764,6 +811,11 @@ def _satisfies(profile: Claim, platform: Claim, db: dict) -> bool:
and _accepted_by_all(profile.local_path, [platform.entry], db)
):
return True
if platform.local_path and any(
_lists_hashes(label) and _accepted_by_all(platform.local_path, [label], db)
for label in profile.accepts
):
return True
return bool(
platform.local_path
and _lists_hashes(profile.entry)
@@ -863,11 +915,18 @@ def _same_file_family(paths: list[str]) -> bool:
def format_collision(collision: Collision) -> str:
"""One line naming the destination and the files fighting for it."""
return (
f"{collision.destination}: declared for "
+ " and for ".join(collision.resolved)
)
"""One line naming the destination, which system declares which file,
and the one the pack ships there."""
owners: dict[str, list[str]] = {}
for system, (_entry, local) in zip(collision.systems, collision.declarations):
if local and system not in owners.setdefault(local, []):
owners[local].append(system)
parts = [
f"{path} ({', '.join(owners[path])}) "
+ ("packed" if path == collision.shipped else "left out")
for path in collision.resolved
]
return f"{collision.destination}: " + ", ".join(parts)
def format_conflict(conflict: Conflict) -> str:
scripts/sourceref.py
+88 -38
View File
@@ -220,10 +220,21 @@ PROSE_EXTENSIONLESS = (
"COPYING",
"CHANGELOG",
)
PROSE_CITE_RE = re.compile(
# Any other dotless file is cited with its directory, which a constant or a
# hex address never carries: lightspark's tools/errorgen:20.
_PROSE_PATH = (
r"(?P<path>[A-Za-z0-9_][\w./+-]*\.[A-Za-z]\w*"
r"|(?:[\w./+-]*/)?(?:" + "|".join(PROSE_EXTENSIONLESS) + r"))"
r":(?P<range>\d+(?:-\d+)?)"
r"|(?:[\w./+-]*/)?(?:" + "|".join(PROSE_EXTENSIONLESS) + r")"
r"|(?:[\w.+-]+/)+[A-Za-z_][\w+-]*)"
)
PROSE_CITE_RE = re.compile(_PROSE_PATH + r":(?P<range>\d+(?:-\d+)?)")
# A file named before its locations, `Source/engine/assets.cpp:` then a
# list, and a location that leaves the file out, `(:507)`, which reads in
# the file named last: devilutionx, corsixth and starship cite their load
# order this way. Neither was collected, so no bump or guard followed them.
_PROSE_FILE_NAMED_RE = re.compile(_PROSE_PATH + r":(?=\s|$)")
_PROSE_ABBREVIATED_RE = re.compile(
r"(?:(?<=[(\s])|^):(?P<range>\d+(?:-\d+)?)(?![\w:-])(?!\.\d)"
)
PROSE_LIST_ITEM_RE = re.compile(r"\d+[.)](?:\s|$)")
PROSE_WRAPPED_RANGE_RE = re.compile(r":\d+(?:-\d+)?(?:[,)\s]|$)")
@@ -344,6 +355,12 @@ def _prose_runs(
joined line, so a writer can put each span back on its own line.
"""
lines = text.splitlines()
# The file an abbreviated location reads in, and the role it was named
# under, carried from line to line within the scalar.
named: tuple[str, str] | None = None
# Where the run wrapped from the line before ends in this one: its
# ranges are that run's, not abbreviated locations of their own.
carried = 0
for number, own in enumerate(lines):
# A literal block keeps its line breaks, so a citation wrapped after
# `file:` or after a range's comma lost the ranges on the next line
@@ -369,46 +386,35 @@ def _prose_runs(
elif following and line.endswith(","):
joined_at = len(line) + 1
line += " " + following
consumed = 0
for match in PROSE_CITE_RE.finditer(line):
consumed, carried = carried, 0
events = sorted(
[(m.start(), 0, m) for m in PROSE_CITE_RE.finditer(line)]
+ [(m.start(), 1, m) for m in _PROSE_FILE_NAMED_RE.finditer(line)]
+ [(m.start(), 2, m) for m in _PROSE_ABBREVIATED_RE.finditer(line)],
key=lambda event: (event[0], event[1]),
)
for _start, kind, match in events:
if match.start() >= boundary:
break
if match.start() < consumed:
continue
token_start = max(line.rfind(" ", 0, match.start()),
line.rfind("\t", 0, match.start())) + 1
token = line[token_start:].split()[0] if line[token_start:] else ""
if "://" in token:
if kind == 1:
if "://" not in line[max(0, match.start() - 3):match.end()]:
named = (match.group("path"), "")
continue
path = match.group("path")
path_start = match.start("path")
# A dotfile citation opens its token with the dot: `.gitlab-ci.yml`
# after a delimiter is the filename, while the dot of `boot.cfg`
# inside a word belongs to the word before it.
if (
path_start > 0
and line[path_start - 1] == "."
and (path_start == 1 or line[path_start - 2] in " \t(")
):
path_start -= 1
path = "." + path
# A path opening its line reads its role or project word at the
# end of the line before: genesis_plus_gx_expanded wrapped
# `(Batocera` and `package/batocera/...mk:7` one line apart.
lead = line[:path_start]
if number and not lead.strip():
lead = lines[number - 1].rstrip() + " "
role = _prose_role(lead, len(lead))
if not role and _prose_external(lead, len(lead), known):
# Kept with its project word, the source_ref convention: the
# report looks for the project inside the declared trees
# (mupen64plus-ae vendors `rice Config.cpp`) and calls the
# citation EXTERNAL only when none carries it.
before = lead.rstrip()
word = before[max(before.rfind(" "), before.rfind("\t"),
before.rfind("(")) + 1:]
path = f"{word} {path}"
path_span = (path_start, match.end("path"))
if kind == 2:
if named is None:
continue
path, role = named
path_start = match.start()
path_span = (path_start, path_start)
else:
path, role, path_start, path_span = _prose_path(
line, lines, number, match, known
)
if path is None:
continue
named = (path, role)
spans = [(match.start("range"), match.end("range"))]
end = match.end()
while True:
@@ -429,6 +435,9 @@ def _prose_runs(
spans.append((cont.start("range"), cont.end("range")))
end = cont.end()
consumed = end
if end > boundary:
indent = len(lines[number + 1]) - len(lines[number + 1].lstrip())
carried = indent + end - joined_at
parts = []
for lo, hi in spans:
rng = line[lo:hi]
@@ -438,3 +447,44 @@ def _prose_runs(
number, path_span, tuple(spans), tuple(parts),
line[path_start:end], joined_at if end > boundary else 0,
)
def _prose_path(
line: str, lines: list[str], number: int, match: re.Match,
known: frozenset[str] | set[str],
) -> tuple[str | None, str, int, tuple[int, int]]:
"""(path, role, start, span) of one written citation, None for a URL."""
token_start = max(line.rfind(" ", 0, match.start()),
line.rfind("\t", 0, match.start())) + 1
token = line[token_start:].split()[0] if line[token_start:] else ""
if "://" in token:
return None, "", 0, (0, 0)
path = match.group("path")
path_start = match.start("path")
# A dotfile citation opens its token with the dot: `.gitlab-ci.yml`
# after a delimiter is the filename, while the dot of `boot.cfg`
# inside a word belongs to the word before it.
if (
path_start > 0
and line[path_start - 1] == "."
and (path_start == 1 or line[path_start - 2] in " \t(")
):
path_start -= 1
path = "." + path
# A path opening its line reads its role or project word at the
# end of the line before: genesis_plus_gx_expanded wrapped
# `(Batocera` and `package/batocera/...mk:7` one line apart.
lead = line[:path_start]
if number and not lead.strip():
lead = lines[number - 1].rstrip() + " "
role = _prose_role(lead, len(lead))
if not role and _prose_external(lead, len(lead), known):
# Kept with its project word, the source_ref convention: the
# report looks for the project inside the declared trees
# (mupen64plus-ae vendors `rice Config.cpp`) and calls the
# citation EXTERNAL only when none carries it.
before = lead.rstrip()
word = before[max(before.rfind(" "), before.rfind("\t"),
before.rfind("(")) + 1:]
path = f"{word} {path}"
return path, role, path_start, (path_start, match.end("path"))
scripts/split_pack.py
+4
View File
@@ -338,6 +338,10 @@ def main() -> int:
# The packs are rewritten in place by generate_pack under this lock: a
# build running beside the split mixes its bytes into the pack being
# read, or writes a pack that the split then deletes.
if not args.target.exists():
print(f"Error: {args.target}: no such pack or directory, pass a pack or the "
"--output-dir generate_pack wrote (default dist/)", file=sys.stderr)
return 1
directory = args.target if args.target.is_dir() else args.target.parent
try:
with artifact_lock(str(directory)):
scripts/validate_pr.py
+74 -6
View File
@@ -18,6 +18,7 @@ Outputs a structured report suitable for PR comments.
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
@@ -25,6 +26,7 @@ import shutil
import subprocess
import sys
import urllib.parse
import zipfile
from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__))
@@ -335,6 +337,53 @@ def load_profile_hashes(
return known
def _archive_members(
filepath: str, db: dict | None, platform_hashes: dict, profile_hashes: dict
) -> list[str] | None:
"""What each file of a ZIP is, None when filepath is not a ZIP.
Read only for an archive no list declares as a whole: MAME and FBNeo
sets are ZIP BIOS files in their own right.
"""
if not zipfile.is_zipfile(filepath):
return None
held = (db or {}).get("files", {})
described = []
try:
with zipfile.ZipFile(filepath) as archive:
for info in archive.infolist():
if info.is_dir():
continue
if info.file_size > MAX_FILE_SIZE:
described.append(f"`{info.filename}` too large to read here")
continue
sha1, md5 = hashlib.sha1(), hashlib.md5()
with archive.open(info) as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
sha1.update(chunk)
md5.update(chunk)
sha1_hex, md5_hex = sha1.hexdigest(), md5.hexdigest()
crc = f"{info.CRC:08x}"
if sha1_hex in held:
state = f"already held at `{held[sha1_hex]['path']}`"
elif (
sha1_hex in platform_hashes.get("sha1", set())
or md5_hex in platform_hashes.get("md5", set())
or (crc, info.file_size) in platform_hashes.get("crc32", set())
or profile_hashes.get("sha1", {}).get(sha1_hex)
or profile_hashes.get("md5", {}).get(md5_hex)
or profile_hashes.get("crc32", {}).get((crc, info.file_size))
):
state = "declared and not held"
else:
state = "not recognised"
described.append(f"`{info.filename}` {state}")
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
# An encrypted member raises RuntimeError.
described.append(f"unreadable ({exc})")
return described
def validate_file(
filepath: str,
db: dict | None,
@@ -446,6 +495,8 @@ def validate_file(
or profile_hashes.get("crc32", {}).get((result.crc32, None))
)
# The files of a ZIP no list declares whole, when the upload is one.
members: list[str] | None = None
if sha1_known:
result.add_check(True, "SHA1 matches known platform requirement")
elif md5_known:
@@ -466,11 +517,26 @@ def validate_file(
)
else:
result.outcome = "unknown"
result.add_warning(
"Not recognised: no platform list or emulator profile declares this "
f"file. Say in the {'issue' if too_large else 'pull request'} which "
"emulator reads it, for which system, and where the dump comes from"
members = None if too_large else _archive_members(
filepath, db, platform_hashes, profile_hashes
)
if members is not None:
# A folder zipped to upload many files at once was hashed as one
# file and its contributor asked where that dump came from.
shown = "; ".join(members[:10]) + (
f"; and {len(members) - 10} more" if len(members) > 10 else ""
)
result.add_warning(
"A ZIP no platform list or emulator profile declares as a whole: "
"add the files it holds, unzipped, not the archive. Inside: "
+ (shown or "nothing")
)
else:
result.add_warning(
"Not recognised: no platform list or emulator profile declares this "
f"file. Say in the {'issue' if too_large else 'pull request'} which "
"emulator reads it, for which system, and where the dump comes from"
)
normalized = os.path.normpath(relative)
# A platform's declaration first, then a profile's: most files the gaps
@@ -503,7 +569,8 @@ def validate_file(
result.add_check(True, f"Correct placement: bios/{parts[1]}/{parts[2]}/")
else:
result.add_warning("File should be in bios/Manufacturer/Console/ structure")
elif result.outcome == "held":
elif result.outcome == "held" or members is not None:
# Nothing to place: a copy, or an archive whose files go in unzipped.
pass
elif expected_folders:
# The contributor asks before forking: say where it goes.
@@ -847,7 +914,8 @@ def main() -> None:
{
"file": r.filepath,
"passed": r.passed,
"outcome": r.outcome or "new",
# A file that failed, unread or refused, is not a new one.
"outcome": r.outcome or ("new" if r.passed else "failed"),
"destination": r.destination,
"sha1": r.sha1,
"md5": r.md5,
scripts/validate_schemas.py
+9 -1
View File
@@ -14,7 +14,15 @@ from pathlib import Path, PurePosixPath
import yaml
from common import source_yaml_paths, yaml_load
from jsonschema import Draft202012Validator, FormatChecker
try:
from jsonschema import Draft202012Validator, FormatChecker
except ImportError:
# Tests import this module and skip on the ImportError, the command
# names the install pipeline.py names for the same step.
if __name__ != "__main__":
raise
sys.exit('Error: jsonschema required (pip install "jsonschema[format-nongpl]")')
ROOT = Path(__file__).resolve().parent.parent
SCHEMAS = ROOT / "schemas"
scripts/validation.py
+68 -1
View File
@@ -16,6 +16,7 @@ from common import (
adler32_of,
archive_members,
compute_hashes,
entry_system_ids,
name_match_size_ok,
sanitize_pack_path,
size_fits,
@@ -145,6 +146,72 @@ def _rule_failure(
return None
def _found_by_content(entry: dict, profile: dict) -> bool:
"""Whether every build of the core identifies this entry by content."""
return not opens_by_name(entry, profile) and not opens_by_name(
entry, profile, standalone=True
)
def _regions(entry: dict) -> set[str]:
value = entry.get("region")
return set(value) if isinstance(value, list) else {value} if value else set()
def content_alternatives(entry: dict, profile: dict) -> list[dict]:
"""The labels a content-found core takes in place of this one.
rustation scans its folder by sha256 (found_by: content): its
scph1001.bin is a label for v2.2, and the v2.0 dump ROCKNIX ships under
that name boots the same North American console, labelled
scph1001_v20.bin. Alternatives share the entry's system and declared
region, the slot slot.py groups by: openMSX also finds its ROMs by
sha1, but each one serves its own machine and declares no region.
"""
if not _found_by_content(entry, profile):
return []
regions = _regions(entry)
if not regions:
return []
systems = set(entry_system_ids(entry, profile))
return [
other for other in profile.get("files") or []
if isinstance(other, dict)
and _found_by_content(other, profile)
and regions & _regions(other)
and systems == set(entry_system_ids(other, profile))
]
def _labels_widened(profile: dict) -> list[dict]:
"""The profile's entries, each content-found label accepting the dumps
of its alternatives (content_alternatives)."""
widened = []
for f in profile.get("files") or []:
if not isinstance(f, dict):
continue
siblings = content_alternatives(f, profile)
if len(siblings) < 2:
widened.append(f)
continue
merged = dict(f)
for field_name in ("crc32", "md5", "sha1", "sha256"):
if f.get(field_name):
merged[field_name] = list(dict.fromkeys(
value for other in siblings
for value in parse_hash_list(other.get(field_name))
))
if f.get("size") is not None:
merged["size"] = sorted({
size for other in siblings
for size in (other["size"] if isinstance(other.get("size"), list)
else [other.get("size")])
if size is not None
})
widened.append(merged)
return widened
def _build_validation_index(profiles: dict) -> dict[str, dict]:
"""Build per-filename validation rules from emulator profiles.
@@ -170,7 +237,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
for emu_name, profile in profiles.items():
if profile.get("type") in ("launcher", "alias"):
continue
for f in profile.get("files", []):
for f in _labels_widened(profile):
fname = f.get("name", "")
if not fname:
continue
scripts/verify.py
+100 -31
View File
@@ -26,11 +26,12 @@ from __future__ import annotations
import argparse
import contextlib
import functools
import json
import os
import sys
from collections import Counter
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from pathlib import Path
import slots
@@ -38,6 +39,8 @@ import slots
sys.path.insert(0, os.path.dirname(__file__))
from hashing import parse_md5_list
from common import (
_foreign_tree,
_owner_tokens,
platform_arg,
target_listing_line,
suggest_names,
@@ -419,14 +422,25 @@ def _build_expected(file_entry: dict, checks: list[str]) -> dict:
return expected
def _in_foreign_tree(db: dict, owners: set[str], sha1: str) -> bool:
"""Whether the collection keeps this sha1 in another emulator's tree."""
return _foreign_tree(db.get("files", {}).get(sha1, {}).get("path", ""), owners)
def _name_in_index(
name: str,
by_name: dict,
by_path_suffix: dict | None = None,
data_names: set[str] | None = None,
by_name_lower: dict[str, str] | None = None,
foreign: Callable[[str], bool] | None = None,
) -> bool:
"""Check if a name is resolvable in the database indexes or data directories."""
"""Check if a name is resolvable in the database indexes or data directories.
*foreign* tells a copy in another emulator's tree from the sha1 of a
name: matched in another casing, such a copy is not the asker's, as in
resolve_local_file's case-insensitive step.
"""
# Strip trailing slash for directory-type entries (e.g. nestopia/samples/foo/)
name = name.rstrip("/")
if name in by_name:
@@ -435,11 +449,11 @@ def _name_in_index(
if basename != name and basename in by_name:
return True
if by_name_lower:
key = name.lower()
if key in by_name_lower:
return True
if basename != name and basename.lower() in by_name_lower:
return True
for key in {name.lower(), basename.lower()}:
if key in by_name_lower and not (
foreign and all(foreign(s) for s in by_name[by_name_lower[key]])
):
return True
if by_path_suffix and name in by_path_suffix:
return True
if data_names:
@@ -667,23 +681,30 @@ def find_undeclared_files(
# Resolution: name, then path basename
# storage: release names where the bytes are served from, not
# that the collection holds them: resolved like any file.
# Resolved as the asker, as the builder's ExtraResolver does:
# another casing of the name in another emulator's tree is
# not this emulator's file (dosbox-x's et4000.bin and pcem's
# et4000.BIN).
owned = {**f, "source_profile": emu_name}
foreign = functools.partial(_in_foreign_tree, db, _owner_tokens(owned))
if shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
# The entry states what its content should be and the builder
# drops a copy that contradicts it, so content decides here
# too. A name match would not do: generic names collide across
# systems, and answering yes on one describes a pack that will
# not contain the file.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
in_repo = _lp is not None and _st != "hash_mismatch"
else:
in_repo = _name_in_index(
fname, by_name, by_path_suffix, data_names, by_name_lower,
foreign,
)
if not in_repo and dest != fname:
path_base = dest.rsplit("/", 1)[-1]
in_repo = _name_in_index(
path_base, by_name, by_path_suffix, data_names,
by_name_lower,
by_name_lower, foreign,
)
if in_repo and not any(
f.get(h) for h in ("sha1", "md5", "crc32", "sha256")
@@ -693,7 +714,7 @@ def find_undeclared_files(
# rejects them all. NanoBoyAdvance's 16 KB bios.bin met
# only PCem's and QEMU's PC BIOS images of that name.
# The fallback below then decides as for any mismatch.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
if _st == "hash_mismatch":
in_repo = False
if not in_repo:
@@ -707,7 +728,7 @@ def find_undeclared_files(
# frontend would reject it, the builder leaves it out, and
# calling it available here would describe a pack that
# does not contain it.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
in_repo = _st != "not_found" and _lp is not None
if in_repo and _st == "hash_mismatch" and shipped_on_mismatch:
in_repo = False
@@ -717,7 +738,7 @@ def find_undeclared_files(
# held for it, the rule the builder's lookup applies
# (packextras.ExtraResolver). The Apple IIGS' 128 KB
# ROM is not Basilisk II's ROM.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
_lp, _st = resolve_local_file(owned, db, dest_hint=dest)
if core_size_refusal(
f, _lp, same_slot_entries(profile.get("files"), fname, dest)
):
@@ -1575,6 +1596,13 @@ def verify_platform(
"exclusion_notes": exclusions,
"region_fallbacks": sorted(region_fallbacks),
**({"left_out": left_out} if no_game_data else {}),
# Every tree of the registry answers a declaration by its tail, so
# one that was never refreshed leaves its files missing here while
# an online pack build refreshes it and packs them.
"data_dir_notices": sorted(
key for key, entry in (data_dir_registry or {}).items()
if not os.path.isdir(entry.get("local_cache", ""))
),
"details": details,
"ground_truth_coverage": {
"with_validation": with_validation,
@@ -1845,6 +1873,13 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
game_data, bios_files, unplaced = _split_undeclared(undeclared)
# A file installed with its emulator is never collected, firmware or
# game data, required or not: cross_reference counts it apart. The
# firmware already left the pack lists (_mark_unplaced), the game data
# did not, and Abuse's three bundled data files read as missing.
bundled = [u for u in undeclared if not u["in_repo"] and u.get("bundled")]
game_data = [u for u in game_data if u["in_repo"] or not u.get("bundled")]
req_not_in_repo = _core_required_missing(bios_files)
req_hle_not_in_repo = [
u
@@ -1854,12 +1889,6 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
req_in_repo = [u for u in bios_files if u["required"] and u["in_repo"]]
opt_in_repo = [u for u in bios_files if not u["required"] and u["in_repo"]]
opt_not_in_repo = [u for u in bios_files if not u["required"] and not u["in_repo"]]
bundled = [
u for u in bios_files
if u["required"] and not u["in_repo"] and u.get("bundled")
and not u.get("hle_fallback")
]
# The builder's count, one file per pack path, when the platform pass
# replayed it. Counting entries said 1130 for RetroArch, 48 more than the
# pack holds: rom.key alone was claimed by three Amiga cores.
@@ -1886,8 +1915,17 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
for u in opt_not_in_repo:
_print_undeclared_entry(u, "MISSING (optional)", verbose, repeated)
if bundled:
names = ", ".join(sorted({u["name"] for u in bundled}))
print(f" Bundled with their emulator, not collected: {names}")
# The required firmware is named, as a missing one would be. The
# rest (shaders, runtime libraries, engine data) on request.
shown = bundled if verbose else [
u for u in bundled
if u["required"] and u.get("category", "bios") != "game_data"
]
names = ", ".join(sorted({u["name"] for u in shown}))
print(
f" Bundled with their emulator, not collected: {len(bundled)}"
+ (f" ({names})" if names else "")
)
if unplaced:
print(f" Core files held but not packed: {len(unplaced)} "
f"({_omission_reasons(unplaced)})")
@@ -1952,6 +1990,13 @@ def print_platform_result(
# them missing, as an offline build leaves them out.
shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "")
print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}")
uncached = result.get("data_dir_notices") or []
if uncached and result.get("status_counts", {}).get(Status.MISSING):
shown = ", ".join(uncached[:10]) + (f" (+{len(uncached) - 10})" if len(uncached) > 10 else "")
print(
f" Data directories not cached ({len(uncached)}), their files count missing "
f"until refresh_data_dirs.py runs (an online pack build refreshes them): {shown}"
)
for line in result.get("left_out") or []:
print(f" {line}")
fallbacks = result.get("region_fallbacks") or []
@@ -2058,7 +2103,7 @@ def verify_emulator(
it and every member it carries are optional, as the builder leaves them.
"""
from packextras import expand_agnostic_entries
from packresolve import resolve_file
from packresolve import pending_release_asset, resolve_file, run_is_offline
from packpaths import (
_archive_destination,
_register_path,
@@ -2084,6 +2129,16 @@ def verify_emulator(
validation_index = _build_validation_index(selected_profiles)
data_registry = load_data_dir_registry(platforms_dir)
def pending(entry: dict, local_path: str | None, status: str) -> tuple[str | None, str]:
# Read from disk and the cache, as _resolve_for_platform does: a
# release asset the cache lacks is fetched by an online build
# (generate_emulator_pack resolves with offline=offline).
if status in ("not_found", "hash_mismatch") and not run_is_offline():
asset = pending_release_asset(entry, db)
if asset:
return asset, RELEASE_ASSET_PENDING
return local_path, status
details = []
file_status: dict[str, str] = {}
file_severity: dict[str, str] = {}
@@ -2223,14 +2278,15 @@ def verify_emulator(
archive = file_entry.get("archive")
if archive and archive not in seen_archives:
seen_archives.add(archive)
local_path, resolve_status = resolve_file(
archive_entry(archive, files),
own_entry = archive_entry(archive, files)
local_path, resolve_status = pending(own_entry, *resolve_file(
own_entry,
db,
DEFAULT_BIOS_DIR,
zip_contents,
data_dir_registry=data_registry,
offline=True,
)
))
required = any(
f.get("archive") == archive and f.get("required", True)
for f in files
@@ -2254,6 +2310,8 @@ def verify_emulator(
"required": required,
"path": local_path,
}
if resolve_status == RELEASE_ASSET_PENDING:
result["release_asset"] = local_path
else:
result = {
"name": archive,
@@ -2314,7 +2372,7 @@ def verify_emulator(
dest_hint = file_entry.get("path", "")
# The emulator pack's resolution, release assets included.
local_path, resolve_status = resolve_file(
local_path, resolve_status = pending(file_entry, *resolve_file(
{**with_archive_members(file_entry, files), "source_profile": emu_name},
db,
DEFAULT_BIOS_DIR,
@@ -2322,7 +2380,7 @@ def verify_emulator(
dest_hint=dest_hint,
data_dir_registry=data_registry,
offline=True,
)
))
if resolve_status in ("not_found", "user_provided", "external"):
local_path = None
name = file_entry.get("name", "")
@@ -2332,6 +2390,15 @@ def verify_emulator(
stored = _storage_verdict(file_entry, emulator_pack=True)
if stored is not None:
result = stored
elif resolve_status == RELEASE_ASSET_PENDING:
result = {
"name": name,
"status": Status.OK,
"required": required,
"path": local_path,
"release_asset": local_path,
"reason": "release asset, fetched when the pack is built",
}
elif not local_path:
result = {"name": name, "status": Status.MISSING, "required": required}
# An entry the profile documents as unsourceable is absent by
@@ -2582,6 +2649,10 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
if unsourceable:
parts.append(f"{unsourceable} unsourceable")
print(f"{label}: {', '.join(parts)} [{mode}]")
fetched = sorted({d["name"] for d in result["details"] if d.get("release_asset")})
if fetched:
shown = ", ".join(fetched[:10]) + (f" (+{len(fetched) - 10})" if len(fetched) > 10 else "")
print(f" Release assets fetched when the pack is built ({len(fetched)}): {shown}")
seen = set()
for d in result["details"]:
@@ -2712,8 +2783,6 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
"--destinations describes the full pack and is incompatible with --system"),
(args.destinations and not (args.platform or args.all),
"--destinations requires --platform or --all"),
(args.offline and not (args.platform or args.all),
"--offline requires --platform or --all"),
# Game data is categorized against a platform's list and its cores.
(getattr(args, "no_game_data", False) and not (args.platform or args.all),
"--no-game-data requires --platform or --all"),
@@ -2988,13 +3057,13 @@ def _main(held: contextlib.ExitStack) -> None:
print(_overall_summary(group_results))
if args.destinations:
held = {}
destinations = {}
for (result, names), (group, _rep) in zip(group_results, groups):
for name, key in zip(names, group):
held[name] = {
destinations[name] = {
"platform": key, "pack_destinations": result["pack_destinations"],
}
Path(args.destinations).write_text(json.dumps(held, indent=1, sort_keys=True))
Path(args.destinations).write_text(json.dumps(destinations, indent=1, sort_keys=True))
if args.json:
for r in all_results.values():