mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-22 03:39:14 -05:00
docs: write prose without dashes or semicolons
Profiles, the wiki, comments, docstrings, command output and schema descriptions carried em-dashes, clause-joining semicolons, unicode arrows, box-drawing banners and check-mark glyphs. Each sentence was rewritten rather than having its character swapped. Hashes, paths, source_ref values and every cited file:line are unchanged, which a comparison of each parsed profile against the previous revision confirms. validate_pr comments now say PASS and FAIL in words, and source_ref values in a profile's structured blocks render as code. test_prose_style fails if any of these comes back, in tracked files or in the generated site.
This commit is contained in:
1 parent
583031eda5
commit
4231902a6c
495 files changed
+2486
-2301
No files matched your search
+1
-1
@@ -3,7 +3,7 @@
|
||||
These modules are run as scripts (``python scripts/verify.py``), as modules
|
||||
(``python -m scripts.scraper.libretro_scraper``), and imported as a package by
|
||||
the tests and the type checker. The first form puts this directory on the path
|
||||
itself; the other two do not, so importing the package adds it. Without that,
|
||||
itself. The other two do not, so importing the package adds it. Without that,
|
||||
``import scripts.common`` fails on the first sibling import it reaches, and
|
||||
every absolute import between siblings would have to be rewritten to be
|
||||
relative -a change that would break running a script directly.
|
||||
|
||||
@@ -123,7 +123,7 @@ def _accepted(entry: dict, field: str) -> set[str]:
|
||||
"""Every value an entry accepts for a hash field.
|
||||
|
||||
A profile lists several revisions as a YAML list (daedalusx64's
|
||||
n64.psh), a platform as a comma-separated string; read as one value,
|
||||
n64.psh), a platform as a comma-separated string. Read as one value,
|
||||
the list stopped the run with an unhashable-type error.
|
||||
"""
|
||||
value = entry.get(field)
|
||||
@@ -264,7 +264,7 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
|
||||
sha1s = sorted(_accepted(entry, "sha1"))
|
||||
if not sha1s:
|
||||
return None
|
||||
# Each accepted revision is a query of its own; archive.org matches one.
|
||||
# Each accepted revision is a query of its own: archive.org matches one.
|
||||
sha1 = " OR ".join(f"sha1:{value}" for value in sha1s)
|
||||
|
||||
search_url = (
|
||||
|
||||
@@ -153,7 +153,7 @@ def print_report(report: dict) -> None:
|
||||
|
||||
|
||||
def update_changed(report: dict) -> list[str]:
|
||||
"""Refresh entries that have changed; the keys whose refresh failed."""
|
||||
"""Refresh entries that have changed and return the keys whose refresh failed."""
|
||||
failed: list[str] = []
|
||||
for e in report.get("entries", []):
|
||||
if e["status"] == "UPDATED" and e.get("key"):
|
||||
|
||||
@@ -565,7 +565,7 @@ def _unresolved_findings(platform: str, missing: dict[str, list[str]]) -> list[F
|
||||
"""One row per platform: the names its targets list that no profile claims.
|
||||
|
||||
A buildbot name without a profile is a core the target filter cannot
|
||||
see; a platform's own emulator id without one is an emulator nobody has
|
||||
see, and a platform's own emulator id without one is an emulator nobody has
|
||||
profiled yet. Both are actionable, so both are listed, but as one row so
|
||||
the backlog does not bury the day's changes.
|
||||
"""
|
||||
@@ -702,7 +702,7 @@ def check_data(offline: bool) -> list[Finding]:
|
||||
def _check_buildbot_system() -> Finding:
|
||||
"""The buildbot's system directory against what _data_dirs.yml tracks.
|
||||
|
||||
The registry entries are refreshed above; an asset published since, or
|
||||
The registry entries are refreshed above. An asset published since, or
|
||||
one the buildbot no longer serves, is only seen by reading its .index.
|
||||
"""
|
||||
import check_buildbot_system # noqa: PLC0415
|
||||
@@ -1030,7 +1030,7 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]:
|
||||
|
||||
|
||||
def _check_installer_pins() -> Finding:
|
||||
"""The bootstraps pin install.py by SHA-256; a stale pin refuses every install."""
|
||||
"""The bootstraps pin install.py by SHA-256, so a stale pin refuses every install."""
|
||||
digest = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
|
||||
stale = []
|
||||
for name in INSTALLERS:
|
||||
@@ -1084,7 +1084,7 @@ def check_profiles(run: bool, offline: bool) -> list[Finding]:
|
||||
def summarize_profile_sync(report: object) -> list[Finding]:
|
||||
"""One row per profile with something to look at, from profile_sync's JSON.
|
||||
|
||||
The JSON carries the ref verdicts and the pin against the tip; a profile
|
||||
The JSON carries the ref verdicts and the pin against the tip. A profile
|
||||
whose refs need review is stale, one the forge could not serve is unknown,
|
||||
and one whose upstream moved while every ref still anchors is reported as
|
||||
such without being counted as stale.
|
||||
|
||||
@@ -160,7 +160,7 @@ def main() -> int:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sums", metavar="FILE",
|
||||
help=f"local {SUMS} the packs were listed in; its copy and signature on "
|
||||
help=f"local {SUMS} the packs were listed in. Its copy and signature on "
|
||||
f"the release must be these files (default: the list attached to the release)",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="JSON output")
|
||||
@@ -169,7 +169,7 @@ def main() -> int:
|
||||
record = load_record(args.record)
|
||||
tag = args.tag or record.get("tag")
|
||||
if not tag:
|
||||
print(f"ERROR: {args.record} names no tag; pass --tag", file=sys.stderr)
|
||||
print(f"ERROR: {args.record} names no tag, pass --tag", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
assets, draft = fetch_assets(tag)
|
||||
|
||||
@@ -68,7 +68,7 @@ def compare(
|
||||
against the collection's: a dump rebuilt at its old size and never
|
||||
re-uploaded was invisible to a size comparison.
|
||||
|
||||
*registered* is every path .gitignore lists; an asset name depends on
|
||||
*registered* is every path .gitignore lists. An asset name depends on
|
||||
which other paths share its basename, so it defaults to *expected* only
|
||||
when the caller has nothing wider. A release description that no longer
|
||||
matches what render_notes() would write is a finding of its own: the
|
||||
@@ -182,8 +182,8 @@ def render_notes(
|
||||
"""The release page, from what the release actually serves.
|
||||
|
||||
Every asset gets one row. A file the database indexes carries its
|
||||
database SHA1; a data-directory bundle carries the hash of its cached
|
||||
copy when one exists; anything else is listed apart with its size only,
|
||||
database SHA1, a data-directory bundle carries the hash of its cached
|
||||
copy when one exists, and anything else is listed apart with its size only,
|
||||
so the page never vouches for bytes nobody indexed. Descriptions and
|
||||
sections already written by hand are kept by name.
|
||||
"""
|
||||
|
||||
+17
-17
@@ -147,7 +147,7 @@ def held_adler32_pair(file_entry: dict, db: dict) -> str | None:
|
||||
Dolphin hashes dsp_rom.bin and dsp_coef.bin together and warns on any
|
||||
combination outside its table: two files each accepted alone can still
|
||||
be a pair it refuses. ``adler32_pairs`` lists the accepted pairs, the
|
||||
entry's own value first, in the code's table order; both members walk
|
||||
entry's own value first, in the code's table order. Both members walk
|
||||
the same pairs over the same collection, so they settle on one pair.
|
||||
None when the entry declares no pair or none is held whole.
|
||||
"""
|
||||
@@ -195,7 +195,7 @@ def adler32_fits(file_entry: dict, path: str, db: dict | None = None) -> bool:
|
||||
def size_fits(file_entry: dict, candidate_size: int) -> bool:
|
||||
"""Whether a size is one the entry declares.
|
||||
|
||||
``size`` is an exact size or the list of sizes the code accepts; it
|
||||
``size`` is an exact size or the list of sizes the code accepts, and it
|
||||
outranks a ``min_size``/``max_size`` range. Nothing declared fits all.
|
||||
"""
|
||||
declared = file_entry.get("size")
|
||||
@@ -265,8 +265,8 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
|
||||
|
||||
# Resolve shared group includes
|
||||
shared_path = os.path.join(platforms_dir, "_shared.yml")
|
||||
# Merge metadata from _registry.yml. The registry is our curated source;
|
||||
# the scraped YAML may be incomplete (missing cores, metadata fields).
|
||||
# Merge metadata from _registry.yml. The registry is our curated source,
|
||||
# while the scraped YAML may be incomplete (missing cores, metadata fields).
|
||||
# Registry fields supplement (not replace) the scraped config.
|
||||
registry_path = os.path.join(platforms_dir, "_registry.yml")
|
||||
if os.path.exists(registry_path):
|
||||
@@ -281,7 +281,7 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
|
||||
for depth, name in enumerate(chain):
|
||||
reg_entry = reg.get("platforms", {}).get(name, {})
|
||||
|
||||
# Merge cores (union for lists, override for all_libretro); a
|
||||
# Merge cores (union for lists, override for all_libretro), a
|
||||
# parent's only where the platform names none.
|
||||
reg_cores = reg_entry.get("cores")
|
||||
if reg_cores is not None and (depth == 0 or "cores" not in config):
|
||||
@@ -452,9 +452,9 @@ def suggest_names(typed: list[str], available: list[str], listing_flag: str) ->
|
||||
def platform_arg(value: str) -> str:
|
||||
"""--platform as typed, mapped to the registry's slug when it names one.
|
||||
|
||||
The site and the pack names say RetroArch and MiSTer FPGA; verify and
|
||||
The site and the pack names say RetroArch and MiSTer FPGA, but verify and
|
||||
generate_pack refused both while install.py took them. Letters and
|
||||
digits decide, as for pack names; anything else reaches require_platform
|
||||
digits decide, as for pack names. Anything else reaches require_platform
|
||||
unchanged and gets its suggestion.
|
||||
"""
|
||||
key = re.sub(r"[^a-z0-9]", "", value.lower())
|
||||
@@ -993,7 +993,7 @@ def _member_held(member: dict, crcs: frozenset[str], names: frozenset[str]) -> b
|
||||
def _holding_members(paths: list[str], members: list[dict]) -> list[str]:
|
||||
"""The same-named archives holding the most required members, then the most members.
|
||||
|
||||
FBNeo reads neocd.bin and 000-lo.lo inside neocdz.zip; the primary copy,
|
||||
FBNeo reads neocd.bin and 000-lo.lo inside neocdz.zip. The primary copy,
|
||||
a set without 000-lo.lo, won on the name while a variant held both.
|
||||
"""
|
||||
def held(path: str) -> tuple[int, int]:
|
||||
@@ -1034,7 +1034,7 @@ def _by_affinity(paths: list[str], file_entry: dict, dest_hint: str) -> list[str
|
||||
TheXTech coin.ogg for Syobon Action, a Hurrican sound for ARMSX2, the
|
||||
1 MB CM-32L PCM ROM for ScummVM's 512 KB MT32_PCM.ROM. A size the entry
|
||||
declares orders the candidates first (ordering rejects nothing, so an
|
||||
unvalidated size keeps its informative status); then the destination's
|
||||
unvalidated size keeps its informative status). Then the destination's
|
||||
directories and the profile that asks name where its own copy lives. The
|
||||
order is otherwise kept.
|
||||
"""
|
||||
@@ -1210,7 +1210,7 @@ def resolve_local_file(
|
||||
return path, status
|
||||
|
||||
# 1c. CRC32 lookup, only when no stronger hash is declared. A declared
|
||||
# size confirms it when available; a handful of emulators validate CRC32
|
||||
# size confirms it when available. A handful of emulators validate CRC32
|
||||
# alone, so those entries retain the same (weaker) evidence as the core.
|
||||
if (
|
||||
crc_candidates
|
||||
@@ -1287,11 +1287,11 @@ def resolve_local_file(
|
||||
# shortened tail can be claimed by several emulators shipping the same
|
||||
# asset tree, and it then proves nothing: skip it. The full tail can be
|
||||
# claimed twice as well, one storing it at the root of its own
|
||||
# directory and one nested deeper; there the file whose path carries
|
||||
# directory and one nested deeper. There the file whose path carries
|
||||
# the least on top of the destination is the one that IS the
|
||||
# destination.
|
||||
# The descent drops at most one leading segment. A destination and the
|
||||
# tree differ by the wrapper directory at most; beyond that the tail
|
||||
# tree differ by the wrapper directory at most. Beyond that the tail
|
||||
# stops describing this file and starts describing whichever emulator
|
||||
# happens to store the same asset tree.
|
||||
hint_parts = dest_hint.split("/")
|
||||
@@ -1418,7 +1418,7 @@ def resolve_local_file(
|
||||
# set aside: the name says nothing about this profile's file,
|
||||
# and a data directory it reads may hold it. Dolphin got
|
||||
# Dolphin-MMJR's codehandler.bin while data/dolphin-sys held
|
||||
# its own; blueMSX got the 16 KB RS232.ROM once msx-emu's
|
||||
# its own, and blueMSX got the 16 KB RS232.ROM once msx-emu's
|
||||
# 8 KB copy was set aside, while data/bluemsx held its own,
|
||||
# and MSX2P.ROM of another dump once only msx-emu answered
|
||||
# the exact casing.
|
||||
@@ -1649,7 +1649,7 @@ def profiles_for_systems(
|
||||
def entry_serves_systems(entry: dict, profile: dict, system_ids: list[str]) -> bool:
|
||||
"""Whether a profile entry belongs to one of the systems, spelled either way.
|
||||
|
||||
The entry's own `system:` decides; one without it serves every system
|
||||
The entry's own `system:` decides. One without it serves every system
|
||||
of its profile, the rule a platform pack applies to core extras.
|
||||
"""
|
||||
own = entry.get("system")
|
||||
@@ -1990,7 +1990,7 @@ def expand_directory_entries(
|
||||
) -> list[dict]:
|
||||
"""Replace each ``type: directory`` entry by the files held under it.
|
||||
|
||||
A profile declares the directory its code walks; what a pack can carry
|
||||
A profile declares the directory its code walks, but what a pack can carry
|
||||
is the files. Each member keeps the entry's fields and is pinned to the
|
||||
content the collection holds at that destination. A directory nothing
|
||||
answers is kept as declared, one gap, and a member the profile also
|
||||
@@ -2277,7 +2277,7 @@ def designated_profiles(profiles: dict[str, dict], names: set[str]) -> set[str]:
|
||||
"""The profiles a list of upstream core names designates.
|
||||
|
||||
The buildbot and es_systems name cores their own way (`mednafen_psx` for
|
||||
beetle_psx); a profile's `cores:` lists those names. A name no profile
|
||||
beetle_psx), and a profile's `cores:` lists those names. A name no profile
|
||||
claims stands for itself.
|
||||
"""
|
||||
index = upstream_profile_index(profiles, include_aliases=True)
|
||||
@@ -2385,7 +2385,7 @@ def platform_declarations(
|
||||
A name alone does not say which file a declaration is: RetroArch declares
|
||||
galaksija/ROM1.BIN, a 4 KiB ROM, and DOSBox reads a 32 KiB SC-55 ROM1.BIN.
|
||||
The size, from the platform entry or else from the file its hash names,
|
||||
tells a declaration of the same file from a same-named one; None stands
|
||||
tells a declaration of the same file from a same-named one. None stands
|
||||
for a size nobody knows. The slot is the declared destination, sanitized
|
||||
and case-folded, relative to the directory core files go to: RetroDECK
|
||||
writes bios/ in front of every destination, the others put it in
|
||||
|
||||
@@ -136,7 +136,7 @@ def _rsa_verify_pkcs1v15_sha256(
|
||||
|
||||
|
||||
# AES-128-CBC decryption, standard library only. The build promises stdlib +
|
||||
# pyyaml; a contributor without cryptography, pycryptodome or openssl on PATH
|
||||
# pyyaml, and a contributor without cryptography, pycryptodome or openssl on PATH
|
||||
# got a RuntimeError from verify.py on otp.bin. 256 bytes need no speed.
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -196,7 +196,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
|
||||
previous = {}
|
||||
except json.JSONDecodeError as e:
|
||||
raise SystemExit(
|
||||
f"{clone_path} is not valid JSON ({e}); restore it from git "
|
||||
f"{clone_path} is not valid JSON ({e}). Restore it from git "
|
||||
"before deduplicating again"
|
||||
) from e
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
"""Catalog entries whose bytes a held dump already contains.
|
||||
|
||||
A catalog lists chips as they come off the board; a collection often holds
|
||||
A catalog lists chips as they come off the board, while a collection often holds
|
||||
the image an emulator reads. The Amiga 3.1 Kickstart pair 391773/391774 is
|
||||
the high and low 16-bit words of amiga-os-310-a1200.rom, byte-swapped; the
|
||||
the high and low 16-bit words of amiga-os-310-a1200.rom, byte-swapped, and the
|
||||
TOSEC "Basic V7" is basiclo and basichi end to end. Listed as missing, they
|
||||
send a maintainer after files that are already here. Each transform below is
|
||||
one the source boards use; a match is by sha1, never by size or name.
|
||||
one the source boards use. A match is by sha1, never by size or name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -172,7 +172,7 @@ def _held_sources(db: dict) -> list[dict]:
|
||||
|
||||
def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]:
|
||||
"""Missing sha1 to the held file or member and the transform that
|
||||
produce its bytes; archives stay open for the whole pass."""
|
||||
produce its bytes. Archives stay open for the whole pass."""
|
||||
archives: dict[str, zipfile.ZipFile] = {}
|
||||
try:
|
||||
return _derivable(entries, db, archives)
|
||||
|
||||
+2
-2
@@ -226,7 +226,7 @@ def download_file(
|
||||
else:
|
||||
print(f" {label}{downloaded:,} bytes")
|
||||
if expected_size and downloaded != expected_size:
|
||||
raise ValueError(f"downloaded {downloaded} bytes; expected {expected_size}")
|
||||
raise ValueError(f"downloaded {downloaded} bytes, expected {expected_size}")
|
||||
|
||||
|
||||
def join_volumes(parts: list[Path], dest: Path) -> None:
|
||||
@@ -263,7 +263,7 @@ def _check(archive: Path, checksums: dict[str, str]) -> None:
|
||||
print(
|
||||
f"Error: checksum mismatch for {archive.name}\n"
|
||||
f" expected {expected}\n got {actual}\n"
|
||||
"Download the parts again; a truncated part gives this.",
|
||||
"Download the parts again: a truncated part gives this.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
+1
-1
@@ -209,7 +209,7 @@ verify_checksum() {
|
||||
echo "Error: checksum mismatch for ${archive}" >&2
|
||||
echo " expected ${expected}" >&2
|
||||
echo " got ${actual}" >&2
|
||||
echo "Download the parts again; a truncated part gives this." >&2
|
||||
echo "Download the parts again: a truncated part gives this." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ def load_provenance_snapshots(provenance_dir: str = DEFAULT_PROVENANCE_DIR) -> d
|
||||
def build_provenance_index(snapshots: dict) -> dict:
|
||||
"""Index snapshot entries by sha1 and by (md5, size) per source.
|
||||
|
||||
First entry wins on hash collisions within a source; entries are
|
||||
First entry wins on hash collisions within a source. Entries are
|
||||
pre-sorted at snapshot write time so the outcome is deterministic.
|
||||
"""
|
||||
index = {}
|
||||
@@ -95,7 +95,7 @@ def build_snapshot(
|
||||
|
||||
|
||||
def write_snapshot(path: str, snapshot: dict) -> bool:
|
||||
"""Write a snapshot; timestamps alone never count as a change."""
|
||||
"""Write a snapshot. Timestamps alone never count as a change."""
|
||||
return write_if_changed(path, json.dumps(snapshot, indent=2) + "\n")
|
||||
|
||||
|
||||
|
||||
@@ -170,7 +170,7 @@ def pinned_base(wanted: dict[str, str], scraped: dict | None) -> str:
|
||||
|
||||
|
||||
def _raw_url(url: str) -> str:
|
||||
"""A GitHub blob page names a revision but serves HTML; raw serves bytes."""
|
||||
"""A GitHub blob page names a revision but serves HTML, raw serves bytes."""
|
||||
marker = "/blob/"
|
||||
if url.startswith("https://github.com/") and marker in url:
|
||||
owner_repo, _, path = url[len("https://github.com/") :].partition(marker)
|
||||
@@ -203,7 +203,7 @@ _RAW_GITHUB = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/
|
||||
|
||||
|
||||
def _at_revision(url: str, base: str) -> str:
|
||||
"""A file of the pinned repository, read at the pinned revision; empty
|
||||
"""A file of the pinned repository, read at the pinned revision, empty
|
||||
when the URL is not a raw GitHub URL of that repository.
|
||||
|
||||
ROCKNIX's checker sits in packages/rocknix/, the emulators it names in
|
||||
|
||||
@@ -66,7 +66,7 @@ class NativeFile:
|
||||
# An addition filed under our own slug: the platform names no such system.
|
||||
foreign_system: bool = False
|
||||
# For an addition, each core that wants it as (runs standalone, name the
|
||||
# platform gives it); empty when no core the platform runs is named.
|
||||
# platform gives it), empty when no core the platform runs is named.
|
||||
runners: list[tuple[bool, str]] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
@@ -103,7 +103,7 @@ class NativeFile:
|
||||
def hashes(self, field_name: str) -> list[str]:
|
||||
"""Accepted values for a hash, truth first when it has an opinion.
|
||||
|
||||
The truth is read from the emulator's source; the platform list is a
|
||||
The truth is read from the emulator's source. The platform list is a
|
||||
secondary source. When both speak and disagree, the truth decides and
|
||||
the divergence is recorded, never silently merged: an emulator that
|
||||
rejects a file will reject it whatever the platform declares.
|
||||
@@ -134,7 +134,7 @@ class NativeFile:
|
||||
with the v2.0 crc (the truth's first) beside the v2.1 md5 and sha1
|
||||
(System.dat's), a record no file satisfies. When the two sides
|
||||
agree on the dump, the platform's record is kept whole, its value
|
||||
chosen among those the truth accepts; a field only the truth names
|
||||
chosen among those the truth accepts. A field only the truth names
|
||||
is written when it names one value, which is then that dump's.
|
||||
"""
|
||||
source = self._identity_source()
|
||||
@@ -526,7 +526,7 @@ def _attach_runners(systems: dict[str, NativeSystem], scraped: dict | None) -> N
|
||||
|
||||
The platform lists its cores by its own names (Batocera's
|
||||
`mednafen_psx`, Recalbox's `mednafen_psx`), our truth by profile key
|
||||
(`beetle_psx`); a profile's `cores:` holds the names it answers to.
|
||||
(`beetle_psx`), and a profile's `cores:` holds the names it answers to.
|
||||
The build a platform runs an emulator as decides the runner.
|
||||
"""
|
||||
if not scraped:
|
||||
|
||||
@@ -26,7 +26,7 @@ _INDENT = " " * 4
|
||||
class Exporter(BaseExporter):
|
||||
"""Write Batocera's batocera-systems, corrected."""
|
||||
|
||||
# checkBios compares one md5 for a plain file; altmd5 is read only for
|
||||
# checkBios compares one md5 for a plain file. altmd5 is read only for
|
||||
# a zippedFile: retrodream's boot.bin written with the first of four
|
||||
# md5 refused the three other dumps the core accepts.
|
||||
SINGLE_HASH_FIELD = "md5"
|
||||
@@ -130,7 +130,7 @@ class Exporter(BaseExporter):
|
||||
|
||||
@staticmethod
|
||||
def needs_original() -> bool:
|
||||
# The data block is a fraction of the file; the rest is the checker.
|
||||
# The data block is a fraction of the file, the rest is the checker.
|
||||
return True
|
||||
|
||||
def _bios_files(self, files: list[NativeFile], before: dict | None = None) -> str:
|
||||
@@ -163,7 +163,7 @@ class Exporter(BaseExporter):
|
||||
return parts
|
||||
|
||||
def _item_runners(self, fe: NativeFile) -> list[list[str] | None]:
|
||||
"""One item per runner an addition names; None keeps the original's."""
|
||||
"""One item per runner an addition names. None keeps the original's."""
|
||||
if fe.platform is not None:
|
||||
return [None]
|
||||
pairs = self._addition_keys(fe) or []
|
||||
@@ -213,7 +213,7 @@ class Exporter(BaseExporter):
|
||||
|
||||
Rewriting an entry as one line would drop them, and a note like
|
||||
"ideally - 94bc50..." is the only record of why a hash is blank.
|
||||
A comment on its own line belongs to the file below it; a comment at
|
||||
A comment on its own line belongs to the file below it, a comment at
|
||||
the end of a line belongs to the file on it.
|
||||
"""
|
||||
trailing: dict[str, str] = {}
|
||||
|
||||
@@ -53,7 +53,7 @@ class Exporter(BaseExporter):
|
||||
"""No hash or size is written, so no correction is delivered.
|
||||
|
||||
Every call holding a hash holds one BizHawk declares, the authority
|
||||
for its cores; the calls without one are SHA1Checksum.Dummy, a
|
||||
for its cores. The calls without one are SHA1Checksum.Dummy, a
|
||||
per-console dump (DSi firmware, NAND) BizHawk leaves unhashed on
|
||||
purpose. The rewrite this exporter had could only ever fire on a
|
||||
same-named call of another system, giving it the wrong system's hash.
|
||||
|
||||
@@ -98,7 +98,7 @@ class Exporter(BaseExporter):
|
||||
def _expected_md5s(
|
||||
self, systems: dict[str, NativeSystem], name: str, system_id: str
|
||||
) -> list[str]:
|
||||
"""What a check must now hold; a refused rewrite keeps EmuDeck's own array."""
|
||||
"""What a check must now hold. A refused rewrite keeps EmuDeck's own array."""
|
||||
if name in getattr(self, "_withdrawn", {}):
|
||||
return []
|
||||
return self._md5s(systems, system_id)
|
||||
@@ -187,7 +187,7 @@ class Exporter(BaseExporter):
|
||||
# it would stop passing the check. A rewrite that withdraws one
|
||||
# is refused and reported instead.
|
||||
# A correction replaces the value it corrects, which is not a
|
||||
# withdrawal; any other value missing from ours would be.
|
||||
# withdrawal, but any other value missing from ours would be.
|
||||
withdrawn = theirs - set(md5s) - self._corrected_away(systems, system_id)
|
||||
if withdrawn:
|
||||
self._withdrawn.setdefault(name, set()).update(withdrawn)
|
||||
|
||||
@@ -111,7 +111,7 @@ class Exporter(BaseExporter):
|
||||
|
||||
@classmethod
|
||||
def writable(cls, fe: NativeFile, require: str = "") -> bool:
|
||||
"""es_bios.xsd makes md5 and core required; without them, no element.
|
||||
"""es_bios.xsd makes md5 and core required: without them, no element.
|
||||
|
||||
An entry Recalbox already ships has both, so this only ever gates
|
||||
what we would be adding.
|
||||
|
||||
@@ -79,7 +79,7 @@ class Exporter(BaseExporter):
|
||||
|
||||
@staticmethod
|
||||
def needs_original() -> bool:
|
||||
# A manifest is mostly presets and launch configuration; rebuilding
|
||||
# A manifest is mostly presets and launch configuration, so rebuilding
|
||||
# one from BIOS data alone would throw the component away.
|
||||
return True
|
||||
|
||||
@@ -104,7 +104,7 @@ class Exporter(BaseExporter):
|
||||
|
||||
A component is one emulator's manifest, and only the platform's
|
||||
entries name it. A system whose entries all sit in one component
|
||||
takes the addition there; one spread over several, or with none of
|
||||
takes the addition there. One spread over several, or with none of
|
||||
its own, cannot say which manifest is meant, and the addition is
|
||||
left out and counted as such.
|
||||
"""
|
||||
@@ -197,7 +197,7 @@ class Exporter(BaseExporter):
|
||||
additions: list[OrderedDict] | None = None,
|
||||
appended: set[tuple[str, str]] | None = None,
|
||||
) -> list[OrderedDict]:
|
||||
"""Correct the component's own list; never replace it.
|
||||
"""Correct the component's own list, never replace it.
|
||||
|
||||
Assigning our entries wholesale dropped every file RetroDECK declares
|
||||
that our model does not carry -- 177 of them across the components.
|
||||
|
||||
@@ -65,7 +65,7 @@ class Exporter(BaseExporter):
|
||||
|
||||
@staticmethod
|
||||
def carries() -> frozenset[str]:
|
||||
# The help names files; it states no hash and no requirement flag.
|
||||
# The help names files. It states no hash and no requirement flag.
|
||||
return frozenset({"name"})
|
||||
|
||||
@staticmethod
|
||||
@@ -181,7 +181,7 @@ class Exporter(BaseExporter):
|
||||
|
||||
Someone reading the sentence copies the files in the order it
|
||||
gives, so the one the emulator prefers is named first. The model
|
||||
already holds them in that order; this only removes the duplicates
|
||||
already holds them in that order, so this only removes the duplicates
|
||||
a name can pick up from several cores.
|
||||
"""
|
||||
names: list[str] = []
|
||||
|
||||
@@ -37,7 +37,7 @@ class Exporter(BatoceraExporter):
|
||||
def _runner_pair(self, runner: tuple[bool, str]) -> tuple[str, str | None] | None:
|
||||
"""ROCKNIX registers libretro cores under `retroarch` and each
|
||||
standalone under its own emulator (`azahar azahar-sa`), in the
|
||||
emulators package; the `libretro` Batocera writes names nothing
|
||||
emulators package. The `libretro` Batocera writes names nothing
|
||||
there."""
|
||||
_standalone, name = runner
|
||||
emulator = self._emulator_of.get(name)
|
||||
|
||||
@@ -69,7 +69,7 @@ class Exporter(BaseExporter):
|
||||
return native_id in SLUG_MAP
|
||||
|
||||
def writable(self, fe: NativeFile, require: str = "") -> bool: # noqa: ARG002
|
||||
"""What RomM already ships stays; the conditions gate additions.
|
||||
"""What RomM already ships stays. The conditions gate additions.
|
||||
|
||||
An entry of theirs that could never verify is still theirs, and the
|
||||
round trip is not the place to decide otherwise. An addition whose
|
||||
@@ -129,10 +129,10 @@ class Exporter(BaseExporter):
|
||||
if not self.writable(fe):
|
||||
continue
|
||||
key = f"{system.native_id}:{fe.name}"
|
||||
# Their entry keeps its fields and their order; ours fill in.
|
||||
# Their entry keeps its fields and their order, ours fill in.
|
||||
entry: OrderedDict[str, str] = OrderedDict(output.get(key, {}))
|
||||
# The fixture states every value as a string, size included.
|
||||
# Where the model settles no size, RomM's own value stands;
|
||||
# Where the model settles no size, RomM's own value stands.
|
||||
# str(None) wrote "None", and RomM's int() on it raised.
|
||||
size = fe.size()
|
||||
if size is None and fe.platform is not None:
|
||||
|
||||
@@ -9,7 +9,7 @@ required or loaded by this core" while its source had grown eleven filenames
|
||||
read from the system directory.
|
||||
|
||||
The signal is the request for a directory to read from. A libretro core asks
|
||||
the frontend with RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY; other shapes ask the
|
||||
the frontend with RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY. Other shapes ask the
|
||||
environment or build a path from a home directory. Finding one in a file the
|
||||
profile itself cites does not prove a file is loaded, and this reports rather
|
||||
than concludes: it names the profiles worth a reading, so the other hundred and
|
||||
@@ -64,12 +64,12 @@ def audit(
|
||||
return None
|
||||
if str(document.get("exclusion_note") or "").strip():
|
||||
# Someone read this one and wrote down why nothing is declared. The
|
||||
# check exists to find the profiles nobody has answered yet; an
|
||||
# check exists to find the profiles nobody has answered yet, and an
|
||||
# answer that goes stale shows up as a drifting ref instead.
|
||||
return None
|
||||
if document.get("data_directories"):
|
||||
# The load is declared, as a directory rather than a file. dinothawr
|
||||
# reads system_dir/dinothawr/ and says so there; that is coverage, not
|
||||
# reads system_dir/dinothawr/ and says so there. That is coverage, not
|
||||
# an assertion waiting to age.
|
||||
return None
|
||||
views = select_views(document, cache_dir, offline)
|
||||
@@ -117,7 +117,7 @@ def main() -> None:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--offline", action="store_true",
|
||||
help="read only what the cache holds; no network access",
|
||||
help="read only what the cache holds, no network access",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.all == bool(args.emulators):
|
||||
|
||||
@@ -273,7 +273,7 @@ def _restore(cached: str, path: str) -> None:
|
||||
"""Copy a fetched asset to its registered path, whole or not at all.
|
||||
|
||||
An interrupted copy at the final name is a truncated file the next scan
|
||||
hashes and publishes; a scratch beside it is swapped in only once full.
|
||||
hashes and publishes. A scratch beside it is swapped in only once full.
|
||||
"""
|
||||
directory = os.path.dirname(path)
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
@@ -314,7 +314,7 @@ def _preserve_large_file_entries(files: dict, db_path: str) -> int:
|
||||
# checkout without the release assets would drop every entry they
|
||||
# hold, and the manifests regenerated from it would omit them.
|
||||
raise SystemExit(
|
||||
f"{db_path} is not valid JSON ({e}); restore it from git "
|
||||
f"{db_path} is not valid JSON ({e}). Restore it from git "
|
||||
"or regenerate it on a checkout holding the release assets before "
|
||||
"scanning again"
|
||||
) from e
|
||||
|
||||
+37
-37
@@ -261,7 +261,7 @@ def _bounded_join(parts: list[str], keys: list[str]) -> str:
|
||||
"""Parts joined while they fit, else the first, a count and a digest.
|
||||
|
||||
Thirty systems joined made a 261-byte file name and the ZIP could not
|
||||
be opened; the digest keeps two different selections apart. The joiner
|
||||
be opened. The digest keeps two different selections apart. The joiner
|
||||
is one no part holds: with "_", sega-dreamcast + arcade and
|
||||
sega-dreamcast-arcade both named Dreamcast_Arcade, and the second build
|
||||
replaced the first.
|
||||
@@ -279,7 +279,7 @@ _UNSAFE_IN_NAME = re.compile(r'[/\\:*?"<>|]')
|
||||
def _name_part(text: str, space: str = "") -> str:
|
||||
"""A display name made safe to stand in a file name.
|
||||
|
||||
'FinalBurn Neo (CPS-1/CPS-2)' named a directory that did not exist; a
|
||||
'FinalBurn Neo (CPS-1/CPS-2)' named a directory that did not exist, and a
|
||||
Windows user cannot write ':' or '?' either.
|
||||
"""
|
||||
return _UNSAFE_IN_NAME.sub("-", text.replace(" ", space))
|
||||
@@ -290,8 +290,8 @@ def _platform_pack_stem(
|
||||
) -> str:
|
||||
"""Platform part of a pack name, version included.
|
||||
|
||||
A group of identical platforms shares one pack named after all of them;
|
||||
the builder and --verify-packs both read the name from here.
|
||||
A group of identical platforms shares one pack named after all of them,
|
||||
and the builder and --verify-packs both read the name from here.
|
||||
"""
|
||||
rep_cfg = load_platform_config(representative, platforms_dir)
|
||||
version = rep_cfg.get("version", rep_cfg.get("dat_version", ""))
|
||||
@@ -602,7 +602,7 @@ def _matches_an_alternative(fe: dict, local_path: str, profiles: dict) -> bool:
|
||||
"""Whether the copy is another version the same profile accepts there.
|
||||
|
||||
A profile lists every version its code takes under one name (each GRP
|
||||
of Raze's table, eduke32's demo and retail NWINTER.GRP); the extra
|
||||
of Raze's table, eduke32's demo and retail NWINTER.GRP). The extra
|
||||
carries the first, and the held copy matching a sibling is no mismatch.
|
||||
"""
|
||||
profile = profiles.get(fe.get("source_profile") or "", {})
|
||||
@@ -660,7 +660,7 @@ def generate_pack(
|
||||
if zip_contents is None:
|
||||
zip_contents = {}
|
||||
if emu_profiles is None:
|
||||
# The pipeline always passes them; a caller that does not must get the
|
||||
# The pipeline always passes them, but a caller that does not must get the
|
||||
# same pack, not one built without validation, slots or agnostic cores.
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
|
||||
@@ -913,7 +913,7 @@ def generate_pack(
|
||||
|
||||
if status == "not_found":
|
||||
# A filename-agnostic core takes any image of its system,
|
||||
# renamed; only an existence frontend accepts that.
|
||||
# renamed. Only an existence frontend accepts that.
|
||||
files_db = db.get("files", {})
|
||||
agnostic_path = None
|
||||
agnostic_resolved = False
|
||||
@@ -982,7 +982,7 @@ def generate_pack(
|
||||
existence_discrepancy(file_entry, local_path, platform_display),
|
||||
)
|
||||
if file_status.get(dedup_key) == "missing":
|
||||
# An earlier declaration of this path found nothing; this
|
||||
# An earlier declaration of this path found nothing, and this
|
||||
# one fills it, as verify counts it.
|
||||
del file_status[dedup_key]
|
||||
file_reasons.pop(dedup_key, None)
|
||||
@@ -1105,7 +1105,7 @@ def generate_pack(
|
||||
):
|
||||
# The core's declared hash comes from its source, the local dump
|
||||
# is what the collection holds. Shipping it keeps the emulator
|
||||
# working when the code never reads the hash; the divergence is
|
||||
# working when the code never reads the hash, and the divergence is
|
||||
# reported so it can be resolved at the profile or the dump.
|
||||
# file_status tracks platform declarations only, so the report
|
||||
# goes to its own list and never moves the pack's own count.
|
||||
@@ -1315,7 +1315,7 @@ def _pack_data_tree(
|
||||
) -> int:
|
||||
"""Write one data directory into an emulator pack, under its shared lock.
|
||||
|
||||
A refresh swaps the tree by rename; without the lock a walk half done
|
||||
A refresh swaps the tree by rename. Without the lock a walk half done
|
||||
read the old tree and wrote files of the new one. ``packed_at`` records
|
||||
each member, so a later profile asking for the same bytes there is
|
||||
served rather than reported missing.
|
||||
@@ -1349,8 +1349,8 @@ def _same_bytes(path: str, other: str | None) -> bool:
|
||||
return compute_hashes(path)["sha1"] == compute_hashes(other)["sha1"]
|
||||
|
||||
|
||||
# A file name holds 255 bytes on every filesystem a pack is written to;
|
||||
# the margin leaves room for the split part suffix and a scratch prefix.
|
||||
# A file name holds 255 bytes on every filesystem a pack is written to,
|
||||
# and the margin leaves room for the split part suffix and a scratch prefix.
|
||||
MAX_PACK_NAME_BYTES = 200
|
||||
|
||||
|
||||
@@ -1499,7 +1499,7 @@ def generate_emulator_pack(
|
||||
taken = archive_dest.lower() in seen_lower
|
||||
if taken or _has_path_conflict(archive_dest, seen_destinations, seen_parents):
|
||||
# Another profile's file holds this path. The same archive
|
||||
# asked for twice is served; anything else leaves this
|
||||
# asked for twice is served. Anything else leaves this
|
||||
# profile without its archive, which used to pass in
|
||||
# silence: a neocdz.zip lacking 000-lo.lo shipped as the
|
||||
# one the second core needs.
|
||||
@@ -1606,8 +1606,8 @@ def generate_emulator_pack(
|
||||
chosen = (fe, better, None)
|
||||
break
|
||||
if status == "hash_mismatch":
|
||||
# Not a check the core runs, so the file still serves;
|
||||
# it is not the dump the profile documents.
|
||||
# Not a check the core runs, so the file still serves,
|
||||
# but it is not the dump the profile documents.
|
||||
fallback = fallback or (fe, local_path, "mismatch")
|
||||
continue
|
||||
chosen = (fe, local_path, None)
|
||||
@@ -1870,7 +1870,7 @@ def _group_systems_by_manufacturer(
|
||||
def _extras_for_systems(extras: list[dict], system_ids: list[str]) -> list[dict]:
|
||||
"""Core extras owned by one of the given systems.
|
||||
|
||||
Extras carry their profile/system identity directly; display labels are
|
||||
Extras carry their profile/system identity directly. Display labels are
|
||||
presentation only and must never drive routing.
|
||||
"""
|
||||
from common import _norm_system_id
|
||||
@@ -1915,7 +1915,7 @@ def generate_split_packs(
|
||||
source, regions, target_name, one_per_slot, required_only
|
||||
)
|
||||
)
|
||||
# Two groupings write different files; without the tag they accumulate in
|
||||
# Two groupings write different files. Without the tag they accumulate in
|
||||
# one directory under one SHA256SUMS.txt.
|
||||
group_tag = "" if group_by == "system" else f"_By{group_by.title()}"
|
||||
# The parts carry the version, so the directory does too: a rescrape that
|
||||
@@ -2004,7 +2004,7 @@ def generate_split_packs(
|
||||
def _entry_for_file(candidates: list[dict], held: dict) -> dict:
|
||||
"""The declaration a held file answers when several share its name.
|
||||
|
||||
BBKEmu declares 8.BIN once per model; the last one won, and the A4980
|
||||
BBKEmu declares 8.BIN once per model. The last one won, and the A4980
|
||||
dump went to the A4988 folder. The declared hash decides, then the
|
||||
destination the collection's own path ends with, then the last one.
|
||||
"""
|
||||
@@ -2547,7 +2547,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
|
||||
continue
|
||||
tc = _target_cores_for(platform_name, args.target, args.platforms_dir)
|
||||
# A group pack and a pack built for the platform alone can both be
|
||||
# present; each one is a pack of this platform.
|
||||
# present, and each one is a pack of this platform.
|
||||
for pack_name in found:
|
||||
zip_path = os.path.join(args.output_dir, pack_name)
|
||||
result = verify_pack_against_platform(
|
||||
@@ -2612,7 +2612,7 @@ def _skip_empty_truth(
|
||||
tc: set[str] | None,
|
||||
failed: list[str],
|
||||
) -> bool:
|
||||
"""Leave out a truth pack that would be empty; refuse it when asked for."""
|
||||
"""Leave out a truth pack that would be empty, refuse it when asked for."""
|
||||
if source != "truth" or not _truth_layer_is_empty(
|
||||
representative, args.platforms_dir, emu_profiles, tc
|
||||
):
|
||||
@@ -2778,7 +2778,7 @@ def _run_platform_packs(
|
||||
)
|
||||
all_ok = all_ok and ok
|
||||
if failed:
|
||||
# Verification only sees the ZIPs that exist; a pack that was never
|
||||
# Verification only sees the ZIPs that exist, so a pack that was never
|
||||
# written does not fail it.
|
||||
print(f"ERROR: no pack for {', '.join(sorted(set(failed)))}")
|
||||
sys.exit(1)
|
||||
@@ -2830,7 +2830,7 @@ def _refuse_refresh_data(args: argparse.Namespace, parser: argparse.ArgumentPars
|
||||
def _refuse_include_archived(
|
||||
args: argparse.Namespace, parser: argparse.ArgumentParser
|
||||
) -> None:
|
||||
"""Only --all chooses among registered platforms; every other mode names
|
||||
"""Only --all chooses among registered platforms. Every other mode names
|
||||
its own platform, emulator, system or hash."""
|
||||
if args.include_archived and not args.all:
|
||||
parser.error("--include-archived requires --all")
|
||||
@@ -3079,8 +3079,8 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
# Quick-exit modes: --verify-packs alone = verify existing packs only
|
||||
# Combined with --all-variants, generation runs first then verify
|
||||
if args.verify_packs and not args.all_variants:
|
||||
# The check resolves core files against data/ like the build did;
|
||||
# a refresh swapping a tree mid-check would judge another tree.
|
||||
# The check resolves core files against data/ like the build did,
|
||||
# and a refresh swapping a tree mid-check would judge another tree.
|
||||
hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
|
||||
with _pack_output_lock(args.output_dir, exclusive=False):
|
||||
_run_verify_packs(args)
|
||||
@@ -3295,7 +3295,7 @@ def _registered(repo_root: str) -> frozenset[str]:
|
||||
def _is_release_asset(local_path: str, repo_root: str) -> bool:
|
||||
"""Whether the installer fetches this file from the large-files release.
|
||||
|
||||
The repository serves every committed file from its raw URL; only the
|
||||
The repository serves every committed file from its raw URL. Only the
|
||||
files kept out of git live as release assets, and .gitignore is the
|
||||
ledger of those. Size is not the criterion: a committed file over 50 MB
|
||||
is still served by the repository, and announcing it as a release asset
|
||||
@@ -3358,7 +3358,7 @@ def _placed_copies(copies: list[dict], dests: list[str]) -> list[dict]:
|
||||
aes_keys.txt at the root and Batocera prod.keys under switch/, so the
|
||||
installer looked for Citra/sysdata/aes_keys.txt, found nothing and
|
||||
copied nothing. `file` stays as written, since it also lays out a custom
|
||||
emulator root; `source` says where the installed file is.
|
||||
emulator root. `source` says where the installed file is.
|
||||
"""
|
||||
by_name: dict[str, list[str]] = {}
|
||||
for dest in dests:
|
||||
@@ -3408,7 +3408,7 @@ def _stable_hashes(local_path: str, db: dict) -> dict:
|
||||
hashes = compute_hashes(local_path, _MANIFEST_HASHES)
|
||||
raise RuntimeError(
|
||||
f"{local_path}: sha1 {hashes['sha1']} contradicts the database scan "
|
||||
f"({expected}); rehash the file or regenerate database.json"
|
||||
f"({expected}). Rehash the file or regenerate database.json"
|
||||
)
|
||||
|
||||
|
||||
@@ -3455,7 +3455,7 @@ def _manifest_entry(
|
||||
if is_release_asset:
|
||||
entry["storage"] = "release"
|
||||
entry["release_asset"] = _release_asset_name(local_path, repo_root)
|
||||
# The installer writes what it downloads with the umask's mode; a program
|
||||
# The installer writes what it downloads with the umask's mode, but a program
|
||||
# (RetroDECK's pico8, vox) has to stay runnable. The git mode cannot say:
|
||||
# hundreds of MAME archives carry an execute bit nothing reads.
|
||||
if local_path and _is_program(local_path):
|
||||
@@ -3500,7 +3500,7 @@ def _attach_claims(
|
||||
one's copy.
|
||||
|
||||
A claim joins the copy just before it when the bytes agree and starts an
|
||||
alternative otherwise; it never joins an earlier copy, which would let it
|
||||
alternative otherwise. It never joins an earlier copy, which would let it
|
||||
win over a core the builder tries first. Only a core's first claim can
|
||||
win: its later ones are reached only when that one is. A platform entry
|
||||
names no core, so its first claim always starts an alternative.
|
||||
@@ -3703,7 +3703,7 @@ def _manifest_core_entries(
|
||||
|
||||
source_emu = fe.get("source_profile") or fe.get("source_emulator", "")
|
||||
|
||||
# Manifest dests are relative to base_destination; keep the inferred
|
||||
# Manifest dests are relative to base_destination. Keep the inferred
|
||||
# extras prefix when it is an internal layout dir (RetroDECK bios/).
|
||||
manifest_dest = full_dest
|
||||
if base_dest and manifest_dest.startswith(f"{base_dest}/"):
|
||||
@@ -3755,7 +3755,7 @@ def _claim_platform_omission(held: dict | None, emulator: str) -> bool:
|
||||
"""Record a core's claim on a destination the platform already omits.
|
||||
|
||||
The platform's own omission stands while one of its declaring systems
|
||||
does; a pack built without them falls to the first core claiming the
|
||||
does. A pack built without them falls to the first core claiming the
|
||||
path, as for a file present. Overwriting the record lost the platform's
|
||||
claim: a target without that core dropped an omission its pack reports.
|
||||
"""
|
||||
@@ -3936,7 +3936,7 @@ def generate_manifest(
|
||||
# platform file, for the core claims a target build would fall back on.
|
||||
platform_entries: dict[str, tuple[dict, str, str]] = {}
|
||||
# Full destination -> the platform systems declaring it. A file's
|
||||
# `systems` also names the systems whose cores read it there; a pack built
|
||||
# `systems` also names the systems whose cores read it there. A pack built
|
||||
# for a target keeps the file only while a declaring system stays.
|
||||
declared_by: dict[str, set[str]] = {}
|
||||
seen_lower: set[str] = set()
|
||||
@@ -4000,11 +4000,11 @@ def generate_manifest(
|
||||
if source != "truth":
|
||||
# The same destination can be declared by more than one system, bare
|
||||
# in one and hash-constrained in another. The builder lets the
|
||||
# constrained sibling claim it; without the same rule here the
|
||||
# constrained sibling claim it. Without the same rule here the
|
||||
# manifest named whatever answered to the name, so the pack and the
|
||||
# installer disagreed about which file a destination means.
|
||||
# Decided among the declarations this manifest keeps, as the pack
|
||||
# does; a manifest carries no data directories.
|
||||
# does. A manifest carries no data directories.
|
||||
preferred_entries = _preferred_entries(
|
||||
pack_systems, db, bios_dir, base_dest, required_only,
|
||||
zip_contents, offline,
|
||||
@@ -4284,7 +4284,7 @@ def generate_manifest(
|
||||
# RetroArch also runs where case is kept: there Mesen-CE opens
|
||||
# bios.col and np2kai np2kai/2608_bd.wav, which the folded list
|
||||
# serves only as BIOS.col and 2608_BD.WAV. The installer adds these
|
||||
# where the directory keeps case; the pack, extracted on Windows
|
||||
# where the directory keeps case, while the pack, extracted on Windows
|
||||
# or macOS too, stays folded.
|
||||
kept = generate_manifest(
|
||||
platform_name, platforms_dir, db, bios_dir, registry_path,
|
||||
@@ -4313,7 +4313,7 @@ def _manifest_data_directories(
|
||||
|
||||
The source archive and the rule that maps its members onto the tree
|
||||
(strip_components for a ZIP, source_path for a tarball, exclude) come
|
||||
from _data_dirs.yml, the same refresh_data_dirs applies; every file
|
||||
from _data_dirs.yml, the same refresh_data_dirs applies. Every file
|
||||
carries the sha1 and size the cache holds, so a member the source
|
||||
changed since is refused rather than installed.
|
||||
"""
|
||||
@@ -4362,7 +4362,7 @@ def _manifest_core_index(
|
||||
buildbot does, and a file names the profile that reads it: `core_names`
|
||||
gives each profile the other names it goes by. `system_cores` gives each
|
||||
platform system the cores whose presence keeps it, as
|
||||
`filter_systems_by_target` decides; a system absent from it is kept
|
||||
`filter_systems_by_target` decides. A system absent from it is kept
|
||||
whatever is available.
|
||||
"""
|
||||
reach = system_core_reach(
|
||||
|
||||
@@ -280,7 +280,7 @@ def _catalog_matched_line(db: dict) -> list[str]:
|
||||
labels = ", ".join(_CATALOG_LABELS.get(s, s) for s in sorted(sources))
|
||||
return [
|
||||
f"- **{matched:,} of {system_files:,} system files** matched to"
|
||||
f" dump-preservation catalogs ({labels}); arcade sets and engine data"
|
||||
f" dump-preservation catalogs ({labels}). Arcade sets and engine data"
|
||||
f" fall outside what those catalogs index"
|
||||
]
|
||||
|
||||
@@ -368,12 +368,12 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"",
|
||||
"## Download BIOS packs",
|
||||
"",
|
||||
"One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file. At a terminal it asks whether to install everything or to choose systems, cores and regions, and `--system`, `--core`, `--region` and `--target` make the same choice without questions; for a bare minimum, build your own pack below.",
|
||||
"One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file. At a terminal it asks whether to install everything or to choose systems, cores and regions, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.",
|
||||
"Files is how many the pack holds once extracted, the figure a file"
|
||||
" manager shows for the folder. It differs from one platform to the"
|
||||
" next because a pack carries only what that platform's emulators"
|
||||
" load."
|
||||
" The size is what the files occupy once extracted; the ZIP itself"
|
||||
" The size is what the files occupy once extracted, and the ZIP itself"
|
||||
" downloads smaller.",
|
||||
"",
|
||||
"A pack over 2 GB comes in several parts, and every part is needed."
|
||||
@@ -429,7 +429,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"BIOS, firmware, and system files for consoles from Atari to PlayStation 3.",
|
||||
"These are the files an emulator loads from disk instead of"
|
||||
" carrying inside itself. Some are required to boot a system,"
|
||||
" others improve accuracy or unlock a feature; the packs carry"
|
||||
" others improve accuracy or unlock a feature, and the packs carry"
|
||||
" both.",
|
||||
"",
|
||||
"Each file is checked the way your platform checks it. Most compare"
|
||||
@@ -445,7 +445,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
f"- **{acquirable_total} files** the platforms' emulators load"
|
||||
f" are still to be found, and {unsourceable_total} more cannot"
|
||||
f" be sourced at all (per-user keys, user-filled slots, dumps"
|
||||
f" nobody has made); both are named in the"
|
||||
f" nobody has made). Both are named in the"
|
||||
f" [gap analysis]({SITE_URL}gaps/)"
|
||||
if missing_total
|
||||
else "- **Nothing missing**: every file the platforms'"
|
||||
@@ -594,11 +594,11 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"",
|
||||
"Documentation and metadata can drift from what emulators actually load.",
|
||||
"To keep packs accurate, platform lists are checked against emulator"
|
||||
" source code, file by file where a profile exists; when the two"
|
||||
" source code, file by file where a profile exists. When the two"
|
||||
" disagree, the code wins.",
|
||||
"",
|
||||
"Hashes document what emulator code loads and accepts, not dump"
|
||||
" provenance; that boundary, and how it relates to preservation"
|
||||
" provenance. That boundary, and how it relates to preservation"
|
||||
" catalogs such as No-Intro, is drawn in the"
|
||||
f" [FAQ]({SITE_URL}wiki/faq/"
|
||||
"#are-these-files-verified-against-original-hardware-dumps).",
|
||||
@@ -715,8 +715,8 @@ Full walkthrough: [profiling guide]({link("wiki/profiling")}).
|
||||
- `bios/Manufacturer/Console/.variants/filename.sha1prefix` for alternate versions
|
||||
- Files >50 MB go in GitHub release assets (`large-files` release)
|
||||
- RPG Maker and ScummVM directories are excluded from deduplication
|
||||
- Two paths differing only by case break clones on Windows and macOS;
|
||||
`tests/test_no_case_collisions.py` enforces this
|
||||
- Two paths differing only by case break clones on Windows and macOS,
|
||||
and `tests/test_no_case_collisions.py` enforces this
|
||||
|
||||
## Before opening a PR
|
||||
|
||||
|
||||
+34
-34
@@ -152,7 +152,7 @@ def _build_system_page_map_from_data(
|
||||
Uses platform file paths to trace system_id -> bios directory -> manufacturer page.
|
||||
"""
|
||||
# Every console folder a filename sits in. A name in one folder only can
|
||||
# place a system; a name in several says nothing (MiSTer's boot.rom lives
|
||||
# place a system, while a name in several says nothing (MiSTer's boot.rom lives
|
||||
# under a dozen consoles, and the last one written used to win).
|
||||
name_consoles: dict[str, set[tuple[str, str]]] = {}
|
||||
for mfr, consoles in manufacturers.items():
|
||||
@@ -191,7 +191,7 @@ def _build_system_page_map_from_data(
|
||||
if sys_id in _system_page_map:
|
||||
continue
|
||||
# Strategy 1: the folder holding the files the system declares,
|
||||
# found by content; most votes win. A bare name stands in only
|
||||
# found by content, and most votes win. A bare name stands in only
|
||||
# where the collection files it in a single console.
|
||||
place = _system_folder(system.get("files", []), db, name_consoles)
|
||||
if place is not None:
|
||||
@@ -321,7 +321,7 @@ def generate_home(
|
||||
[
|
||||
"",
|
||||
"Checked by is the test each platform runs on its own, replicated "
|
||||
"from its source code. RetroArch only looks for the filename; "
|
||||
"from its source code. RetroArch only looks for the filename, while "
|
||||
"Batocera compares MD5 checksums, the fingerprint of a file's "
|
||||
"contents. "
|
||||
"[How each mode works](wiki/verification-modes.md).",
|
||||
@@ -594,7 +594,7 @@ def build_emulator_gap_report(
|
||||
"""Files a core loads that no platform declares, per emulator profile.
|
||||
|
||||
The gap analysis page and the published gaps export must not compute this
|
||||
twice and drift; both read this one report.
|
||||
twice and drift. Both read this one report.
|
||||
"""
|
||||
from common import _norm_system_id, merged_platform_declarations
|
||||
from cross_reference import cross_reference as run_cross_reference
|
||||
@@ -810,7 +810,7 @@ def _write_sqlite_export(
|
||||
"schema_version": "1",
|
||||
"generated_at": str(db.get("generated_at") or _timestamp()),
|
||||
"source": REPO_URL,
|
||||
"scope": "metadata only; no BIOS or firmware payload bytes",
|
||||
"scope": "metadata only, no BIOS or firmware payload bytes",
|
||||
}
|
||||
connection.executemany(
|
||||
"INSERT INTO metadata(key, value) VALUES (?, ?)",
|
||||
@@ -1091,7 +1091,7 @@ def generate_data_page(exports: list[dict]) -> str:
|
||||
"The DAT is Logiqx XML, the format clrmamepro and RomVault read: one game "
|
||||
"per `bios/<Manufacturer>/<Console>/` folder of the collection, each file a "
|
||||
"rom named by its path below that folder. It audits a copy of the "
|
||||
"collection's `bios/` tree, or rebuilds that layout from your own dumps; a "
|
||||
"collection's `bios/` tree, or rebuilds that layout from your own dumps. A "
|
||||
"platform pack has its own layout and is checked with `install.py --check`.",
|
||||
"",
|
||||
"## Stable JSON endpoints",
|
||||
@@ -1113,7 +1113,7 @@ def generate_data_page(exports: list[dict]) -> str:
|
||||
lines.extend([
|
||||
"",
|
||||
"Every JSON document carries `schema_version`. Breaking changes use a new "
|
||||
"URL prefix (`/api/v2/`); fields may only be added compatibly within v1.",
|
||||
"URL prefix (`/api/v2/`), and fields may only be added compatibly within v1.",
|
||||
"",
|
||||
"One emulator profile without the whole list: "
|
||||
"`api/v1/emulators/<id>.json`, the envelope of `emulators.json` holding "
|
||||
@@ -1168,8 +1168,8 @@ def generate_data_page(exports: list[dict]) -> str:
|
||||
"catalog, or emulator compatibility is not a statement about copyright, "
|
||||
"ownership, or redistribution rights.",
|
||||
"",
|
||||
f"The data can be newer than the [latest published pack]({RELEASE_URL}); "
|
||||
"pack publication is manual and only occurs after all release gates pass.",
|
||||
f"The data can be newer than the [latest published pack]({RELEASE_URL}). "
|
||||
"A pack is published by hand, once every release check has passed.",
|
||||
"",
|
||||
"See the [data model](wiki/data-model.md), [verification modes]"
|
||||
"(wiki/verification-modes.md), and [methodology](wiki/architecture.md) "
|
||||
@@ -1355,7 +1355,7 @@ def platform_structured_data(name: str, coverage: dict) -> dict:
|
||||
|
||||
The page printed thousands of hashes while its JSON-LD named only the
|
||||
page: a crawler could not learn which md5 Batocera requires for a file.
|
||||
The software and its version are the subject; each declared file, with
|
||||
The software and its version are the subject, and each declared file, with
|
||||
the hashes the platform checks, is a part. Past SYSTEM_PARTS_IN_FULL the
|
||||
parts give way to the platform-files table, as system pages do.
|
||||
"""
|
||||
@@ -1530,8 +1530,8 @@ def emulator_structured_data(profile: dict) -> dict:
|
||||
return data
|
||||
|
||||
|
||||
# Working notes kept under docs/ during development, never part of the site;
|
||||
# no generated page lives in a directory of these names.
|
||||
# Working notes kept under docs/ during development, never part of the site,
|
||||
# and no generated page lives in a directory of these names.
|
||||
LOCAL_NOTE_DIRS = ("plans", "specs")
|
||||
|
||||
|
||||
@@ -1937,7 +1937,7 @@ def generate_platform_page(
|
||||
def generate_files_page() -> str:
|
||||
"""The content address of every held file: files/?sha1=<sha1>.
|
||||
|
||||
site.js reads api/v1/file-locations.json and goes to the card; without
|
||||
site.js reads api/v1/file-locations.json and goes to the card. Without
|
||||
a sha1, or with one the collection does not hold, the page says so.
|
||||
"""
|
||||
return "\n".join([
|
||||
@@ -2061,7 +2061,7 @@ def _provenance_lines(provenance: dict, prefix: str) -> list[str]:
|
||||
"""One visible line per catalog: its badge and the entry's own name.
|
||||
|
||||
The name was only a title attribute, unreadable on a touch screen and
|
||||
absent from the search for 793 of 803 provenances; and a TOSEC entry
|
||||
absent from the search for 793 of 803 provenances, and a TOSEC entry
|
||||
flagged [b] or [o] wore the same green badge as a clean dump.
|
||||
"""
|
||||
lines: list[str] = []
|
||||
@@ -2311,7 +2311,7 @@ def _system_file_card(f: dict, consumers: FileConsumers) -> list[str]:
|
||||
anchor = f"sha1-{sha1_full[:12]}" if sha1_full != "unknown" else ""
|
||||
id_attr = f' id="{anchor}"' if anchor else ""
|
||||
lines = [f'<div class="rb-sys-file"{id_attr} markdown>', ""]
|
||||
# The anchor is this page's; the second link is the file's own address,
|
||||
# The anchor is this page's, the second link is the file's own address,
|
||||
# resolved by content wherever its card moves.
|
||||
permalink = (
|
||||
f' <a class="rb-permalink" href="#{anchor}" title="Link to this card">#</a>'
|
||||
@@ -2750,7 +2750,7 @@ def _render_emulator_file(
|
||||
if fnote:
|
||||
lines.append(f"- {_escape_tags(str(fnote))}")
|
||||
if not in_repo and f.get("unsourceable"):
|
||||
# The badge alone sent readers after a dump that cannot exist; the
|
||||
# The badge alone sent readers after a dump that cannot exist, and the
|
||||
# reason was only on the gaps page, which the card did not link.
|
||||
lines.append(
|
||||
f"- Not collectable: {_escape_tags(str(f['unsourceable']))} "
|
||||
@@ -2844,7 +2844,7 @@ def _plain_value(value: object) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "yes" if value else "no"
|
||||
if value is None:
|
||||
# A YAML null states that there is none; Python's `None` read as a gap.
|
||||
# A YAML null states that there is none, but Python's `None` read as a gap.
|
||||
return "none"
|
||||
if isinstance(value, (list, tuple)):
|
||||
return ", ".join(_plain_value(item) for item in value)
|
||||
@@ -3021,7 +3021,7 @@ def generate_emulator_page(
|
||||
source_raw = profile.get("source", "")
|
||||
source = str(source_raw) if not isinstance(source_raw, dict) else ""
|
||||
# A key set to null says there is none (the project is its own upstream,
|
||||
# it publishes no version); an absent key says nobody wrote it down. The
|
||||
# it publishes no version), and an absent key says nobody wrote it down. The
|
||||
# page printed `None` for the first, which read as the second.
|
||||
upstream_raw = profile.get("upstream", "")
|
||||
upstream_none = "upstream" in profile and upstream_raw is None
|
||||
@@ -3100,7 +3100,7 @@ def generate_emulator_page(
|
||||
if revision:
|
||||
lines.append(f"| Revision | {revision} |")
|
||||
lines.append(f"| Profiled | {profiled} |")
|
||||
# The YAML is the reusable document this site renders; the JSON is the
|
||||
# The YAML is the reusable document this site renders, and the JSON is the
|
||||
# same profile for a program.
|
||||
lines.append(
|
||||
f"| Profile | [YAML]({REPO_URL}/blob/main/emulators/{name}.yml), "
|
||||
@@ -3288,7 +3288,7 @@ def _render_problem_files(platform_problems: list, settled: list | None = None)
|
||||
"",
|
||||
"A platform can declare one path twice with different hashes. The "
|
||||
"path holds the file one declaration accepts, so it is counted once, "
|
||||
"as held; the other declaration cannot be met at the same time.",
|
||||
"as held. The other declaration cannot be met at the same time.",
|
||||
"",
|
||||
"| File | Platform | Status | Path held for system |",
|
||||
"|------|----------|--------|--------------|",
|
||||
@@ -3443,7 +3443,7 @@ def generate_gap_analysis(
|
||||
"required and optional files alike since both ship. The first column "
|
||||
"is the BIOS list the platform publishes. The second counts files its "
|
||||
"emulators load that this list never mentions, found by reading their "
|
||||
"source code, and it is routinely several times larger; a short "
|
||||
"source code, and it is routinely several times larger. A short "
|
||||
"fraction is flagged and named in the sections below. That second "
|
||||
"number is a floor, not a ceiling: an emulator that accepts any file "
|
||||
"handed to it names none in its code, so nothing there can be counted. "
|
||||
@@ -3520,7 +3520,7 @@ def generate_gap_analysis(
|
||||
"Platform file lists are scraped as-is from each upstream project. "
|
||||
"Emulator profiles are read from source code. The two do not always "
|
||||
"agree, and this table counts the differences. Packs follow the "
|
||||
"platform contract; these numbers show where that contract diverges "
|
||||
"platform contract, and these numbers show where that contract diverges "
|
||||
"from what the code loads.",
|
||||
"",
|
||||
"| Platform | Missing from list | Phantom | Hash conflict | Required status |",
|
||||
@@ -4003,7 +4003,7 @@ def generate_wiki_data_model(db: dict, profiles: dict) -> str:
|
||||
"",
|
||||
"`provenance` maps each catalog that lists the file to the DAT and entry "
|
||||
"it was matched against. It is present only when the file matches a "
|
||||
"snapshot under `provenance/`; the join runs by SHA1 first, then by "
|
||||
"snapshot under `provenance/`. The join runs by SHA1 first, then by "
|
||||
"MD5 + size. See [dump provenance](../provenance.md).",
|
||||
"",
|
||||
"### Indexes",
|
||||
@@ -4020,20 +4020,20 @@ def generate_wiki_data_model(db: dict, profiles: dict) -> str:
|
||||
"",
|
||||
"`resolve_local_file` tries these steps in order:",
|
||||
"",
|
||||
"1. SHA1 exact match; every other declared hash must agree with the record",
|
||||
"1. SHA1 exact match: every other declared hash must agree with the record",
|
||||
"2. SHA256 exact match, with the same all-declarations-must-agree rule",
|
||||
"3. CRC32 plus declared size, only when no stronger hash is present",
|
||||
"4. MD5 direct lookup (including explicitly supported truncated MD5 values)",
|
||||
"5. Path suffix lookup for regional variants; with hashes it is accepted only if those hashes match",
|
||||
"5. Path suffix lookup for regional variants. With hashes it is accepted only if those hashes match",
|
||||
"6. Name and alias lookup only when no content hash was declared",
|
||||
"7. Candidate inspection for composite ZIP MD5 or direct MD5; a named candidate with the wrong content returns `hash_mismatch`",
|
||||
"7. Candidate inspection for composite ZIP MD5 or direct MD5. A named candidate with the wrong content returns `hash_mismatch`",
|
||||
"8. `zipped_file` content match via the inner-ROM MD5 index",
|
||||
"9. MAME clone fallback, only for declarations without a content hash",
|
||||
"10. Data-directory scan; declared hashes are computed over the candidate before it is accepted",
|
||||
"10. Data-directory scan: declared hashes are computed over the candidate before it is accepted",
|
||||
"11. Agnostic size/path fallback, only for declarations without a content hash",
|
||||
"",
|
||||
"A filename or destination can never override a declared hash. The first "
|
||||
"evidence-compatible match wins; otherwise the resolver reports a mismatch "
|
||||
"evidence-compatible match wins, otherwise the resolver reports a mismatch "
|
||||
"or absence. Steps and their return codes are described in "
|
||||
"[verification modes](verification-modes.md#file-resolution-chain).",
|
||||
"",
|
||||
@@ -4075,7 +4075,7 @@ def generate_wiki_data_model(db: dict, profiles: dict) -> str:
|
||||
"",
|
||||
"The website publishes versioned JSON, CSV and SQLite metadata generated "
|
||||
"from these same structures. Start with the [Data & API](../data.md) "
|
||||
"catalog; each downloadable artifact carries a SHA256 in "
|
||||
"catalog. Each downloadable artifact carries a SHA256 in "
|
||||
"`api/v1/catalog.json`.",
|
||||
"",
|
||||
]
|
||||
@@ -4247,15 +4247,15 @@ everything or to choose systems, cores and regions, and `--system`, `--core`,
|
||||
[installer reference](wiki/installer.md#choosing-what-to-install)). From a
|
||||
clone of the repository, `python scripts/generate_pack.py --platform retroarch
|
||||
--region us` builds the same narrowing as a pack and `--required-only` the bare
|
||||
minimum each core needs to start; `--help` lists every way to build your own.
|
||||
minimum each core needs to start, and `--help` lists every way to build your own.
|
||||
|
||||
---
|
||||
|
||||
## After extraction
|
||||
|
||||
Launch a game and use the emulator's firmware status screen where available.
|
||||
Most supported frontends find files in the documented directory automatically;
|
||||
standalone emulators may still need their BIOS path configured.
|
||||
Most supported frontends find files in the documented directory automatically,
|
||||
but standalone emulators may still need their BIOS path configured.
|
||||
|
||||
If a game still asks for a missing file, check the
|
||||
[platforms section](platforms/index.md) for the full file list, or the
|
||||
@@ -4385,7 +4385,7 @@ def main() -> None:
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
# Two runs over one docs/ delete each other's pages and scratch
|
||||
# files in the sweep; the second is refused instead.
|
||||
# files in the sweep, so the second is refused instead.
|
||||
with artifact_lock(args.docs_dir):
|
||||
_generate(args)
|
||||
except ArtifactLockBusy as exc:
|
||||
@@ -4725,7 +4725,7 @@ plugins:
|
||||
- search
|
||||
# Link rot fails the build: deploy-site.yml runs `mkdocs build --strict`.
|
||||
# omitted_files stays at its default (info) so a stale local docs/ does not
|
||||
# break a local build; CI regenerates docs/ from scratch anyway.
|
||||
# break a local build. CI regenerates docs/ from scratch anyway.
|
||||
validation:
|
||||
absolute_links: warn
|
||||
unrecognized_links: warn
|
||||
|
||||
@@ -81,7 +81,7 @@ def main(argv: list[str] | None = None) -> None:
|
||||
profiles = load_emulator_profiles(args.emulators_dir)
|
||||
|
||||
# Load database (optional)
|
||||
# The default may be absent; a --db that was named must exist.
|
||||
# The default may be absent, but a --db that was named must exist.
|
||||
db: dict | None = None
|
||||
if args.db != DEFAULT_DB_FILE or os.path.exists(args.db):
|
||||
db = load_database(args.db)
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
"""Content digests.
|
||||
|
||||
SHA-1 is the collection's primary key; the rest are what the various
|
||||
SHA-1 is the collection's primary key. The rest are what the various
|
||||
frontends check against."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -15,7 +15,7 @@ from pathlib import Path
|
||||
_ALL_ALGORITHMS = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"})
|
||||
|
||||
# (path, inode, size, mtime_ns, ctime_ns) -> the digests read so far.
|
||||
# verify --all hashed GB_dmg.bin 13 times, once per platform and entry; the
|
||||
# verify --all hashed GB_dmg.bin 13 times, once per platform and entry. The
|
||||
# bytes are still read from disk, once per file and change.
|
||||
_hash_cache: dict[tuple, dict[str, str]] = {}
|
||||
|
||||
@@ -118,7 +118,7 @@ def parse_hash_list(raw: str | list | int | None) -> list[str]:
|
||||
"""Normalize any hash field into its accepted values, lowercase.
|
||||
|
||||
A field holds one value, a YAML list, or one comma-joined string
|
||||
(Recalbox md5, mesence's four sha256 for one entry); a crc32 may carry
|
||||
(Recalbox md5, mesence's four sha256 for one entry), and a crc32 may carry
|
||||
a 0x prefix. Read one way here, the resolver, the gap report and
|
||||
profile_sync accept the same values.
|
||||
"""
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ nothing in the pipeline could see it, because on paper both names resolved.
|
||||
|
||||
This walks the tree rather than the index, groups by content, and reports a
|
||||
group whose paths sit under different manufacturers. A dump legitimately
|
||||
shared by two systems of the same maker is common; one shared across makers
|
||||
shared by two systems of the same maker is common, one shared across makers
|
||||
almost never is.
|
||||
"""
|
||||
|
||||
|
||||
@@ -96,8 +96,8 @@ def published_asset_name(name: str) -> str:
|
||||
def asset_candidates(name: str, registered: Iterable[str]) -> list[str]:
|
||||
"""Assets that may hold *name*, a registered path or a bare file name.
|
||||
|
||||
A bare name shared by several registered paths has one asset per path;
|
||||
the caller's hash tells them apart.
|
||||
A bare name shared by several registered paths has one asset per path,
|
||||
and the caller's hash tells them apart.
|
||||
"""
|
||||
registered = list(registered)
|
||||
if "/" in name:
|
||||
@@ -127,7 +127,7 @@ def fetch_large_file(
|
||||
) -> str | None:
|
||||
"""Return a verified cached large file, downloading it only when allowed.
|
||||
|
||||
*name* is a registered bios/ path or a bare file name; asset_names()
|
||||
*name* is a registered bios/ path or a bare file name. asset_names()
|
||||
turns it into the asset to fetch, and a bare name shared by several
|
||||
registered paths tries each of their assets until one verifies.
|
||||
"""
|
||||
@@ -255,7 +255,7 @@ def _replaced_on_release(name: str, cached: str, offline: bool) -> bool:
|
||||
The release lists a sha256 per asset: a dump rebuilt at the same size
|
||||
(the eight 251658304-byte DSi NANDs) differs only there, and judged by
|
||||
size the stale copy was kept for good. Without a digest the sizes are
|
||||
compared, as check_release_assets does. No download either way;
|
||||
compared, as check_release_assets does. No download either way:
|
||||
offline or unreadable, the copy is assumed current.
|
||||
"""
|
||||
if offline:
|
||||
@@ -275,7 +275,7 @@ _digests_cache: dict[str, str] | None = None
|
||||
|
||||
|
||||
def _served_digests() -> dict[str, str]:
|
||||
"""sha256 of each asset the release serves, by asset name; empty when
|
||||
"""sha256 of each asset the release serves, by asset name, empty when
|
||||
the API cannot be read."""
|
||||
global _digests_cache
|
||||
if _digests_cache is not None:
|
||||
|
||||
@@ -14,7 +14,7 @@ declaration can sit behind one, as `/* Naomi */ GAME( 1998, naomi, ...)` does.
|
||||
|
||||
A set no machine macro declares is not an error. Device ROMs take the
|
||||
shortname of their DEFINE_DEVICE_TYPE and some archives are bare ROM_START
|
||||
blocks; a ref there cites something this check cannot judge, and says so.
|
||||
blocks. A ref there cites something this check cannot judge, and says so.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -126,7 +126,7 @@ def main() -> None:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--offline", action="store_true",
|
||||
help="read only what the cache holds; no network access",
|
||||
help="read only what the cache holds, no network access",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--write", action="store_true",
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
The sitemap gave all 664 pages the build date as lastmod, so the field told
|
||||
a crawler nothing. A page whose content carries its own date (an emulator
|
||||
page: the day its profile was last read against the source) states it in
|
||||
its front matter; the rest get no lastmod rather than an inexact one.
|
||||
its front matter. The rest get no lastmod rather than an inexact one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -25,7 +25,7 @@ from __future__ import annotations
|
||||
DEFAULT_MODE = "existence"
|
||||
|
||||
#: Every verification mode a platform YAML may declare. Mirrors the enum in
|
||||
#: schemas/platform.schema.json; tests/test_audit_regressions.py pins them
|
||||
#: schemas/platform.schema.json, and tests/test_audit_regressions.py pins them
|
||||
#: together so a new mode cannot be added to one without the other.
|
||||
MODES = ("existence", "md5", "sha1")
|
||||
|
||||
|
||||
@@ -228,7 +228,7 @@ def unplaceable_extras(config: dict, undeclared: list[dict], profiles: dict) ->
|
||||
"""Keys of the report entries a slug-based platform cannot place.
|
||||
|
||||
RomM files every BIOS under the slug of its system and knows no slug
|
||||
for DOS or for the arcade sets MAME 2003 reads; the builder skipped
|
||||
for DOS or for the arcade sets MAME 2003 reads. The builder skipped
|
||||
those entries and the report counted them in the pack.
|
||||
"""
|
||||
from common import _norm_system_id
|
||||
@@ -283,7 +283,7 @@ def _agnostic_scan_extras(
|
||||
# filename walks into whatever tree that name happens to hit: rom1.bin
|
||||
# is a PS2 ROM and a Roland SC-55 ROM, GameIndex.yaml belongs to four
|
||||
# PS2 profiles and to an Android package. One match is not evidence
|
||||
# of a directory; two files of the same profile agreeing is.
|
||||
# of a directory, two files of the same profile agreeing is.
|
||||
resolved_dirs: dict[int, str] = {}
|
||||
named_only: set[int] = set()
|
||||
agnostic_votes: dict[str, int] = {}
|
||||
@@ -467,7 +467,7 @@ def expand_agnostic_entries(
|
||||
in its BIOS folder, which is what the platform pack's scan carries. The
|
||||
emulator pack and verify --emulator looked the placeholder up by name,
|
||||
found nothing and shipped LRPS2 without a PS2 BIOS. The same scan
|
||||
replaces the placeholders here; with nothing found they stay, a gap.
|
||||
replaces the placeholders here. With nothing found they stay, a gap.
|
||||
"""
|
||||
if profile.get("bios_mode") != "agnostic" and not any(f.get("agnostic") for f in files):
|
||||
return files
|
||||
@@ -861,7 +861,7 @@ def _collect_emulator_extras(
|
||||
if fname not in covered_names or f.get("unsourceable"):
|
||||
continue
|
||||
# A member of an archive is read inside it, never as a loose
|
||||
# file; the archive itself is placed by the passes above.
|
||||
# file. The archive itself is placed by the passes above.
|
||||
if f.get("archive"):
|
||||
continue
|
||||
# No path: the core reads it from nowhere a pack reaches
|
||||
@@ -963,12 +963,12 @@ def _report_extra(u: dict, name: str, dest: str) -> dict:
|
||||
if u.get("members"):
|
||||
extra["members"] = u["members"]
|
||||
# The archive layout the core opens ranks same-named copies by what
|
||||
# they hold; without it the first copy indexed won.
|
||||
# they hold. Without it the first copy indexed won.
|
||||
if u.get("contents"):
|
||||
extra["contents"] = u["contents"]
|
||||
# Keep every reproducible identity constraint carried by the source
|
||||
# profile. Some profiles expose CRC32/SHA-256 only through their
|
||||
# validation block; dropping those fields here would let a same-named
|
||||
# validation block, and dropping those fields here would let a same-named
|
||||
# but different payload enter a generated pack.
|
||||
expected = u.get("expected") or {}
|
||||
for identity in PROFILE_IDENTITY_FIELDS:
|
||||
@@ -983,7 +983,7 @@ def _report_extra(u: dict, name: str, dest: str) -> dict:
|
||||
def _take(
|
||||
extra: dict, full_dest: str, seen_dests: set[str], taken: dict[str, dict] | None
|
||||
) -> bool:
|
||||
"""Whether an extra takes its destination; a taken one records the claim."""
|
||||
"""Whether an extra takes its destination. A taken one records the claim."""
|
||||
if full_dest in seen_dests:
|
||||
if taken is not None:
|
||||
_another_claim(taken.get(full_dest), extra)
|
||||
@@ -998,7 +998,7 @@ def _another_claim(extra: dict | None, claim: dict) -> None:
|
||||
"""Record a core that reads a destination another core's extra took.
|
||||
|
||||
The pack ships one file per destination, so the first claimant's extra
|
||||
stands; the others are kept on it so the manifest can name every core
|
||||
stands. The others are kept on it so the manifest can name every core
|
||||
the file serves. A pack built for a target without the first core packs
|
||||
the next one's copy, and an installer narrowing by core must keep it.
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Finding the bytes a pack entry names.
|
||||
|
||||
Resolution proper lives in common; this layer adds what only a build
|
||||
Resolution proper lives in common. This layer adds what only a build
|
||||
needs: storage tiers, release assets for files too large to commit,
|
||||
and the offline switch that forbids reaching for the network."""
|
||||
|
||||
@@ -51,7 +51,7 @@ def _verified_asset(
|
||||
"""The release asset of that name, if it is a copy the entry accepts.
|
||||
|
||||
The store is asked once per accepted sha1 (or md5), since it keeps one
|
||||
revision under a name; what it returns is held against every hash the
|
||||
revision under a name. What it returns is held against every hash the
|
||||
entry declares, sha256 and crc32 included.
|
||||
"""
|
||||
attempts = (
|
||||
@@ -125,7 +125,7 @@ def lookup_hashes(
|
||||
emulators_dir: str,
|
||||
platforms_dir: str,
|
||||
) -> int:
|
||||
"""Print diagnostic info for each hash; return how many were not found."""
|
||||
"""Print diagnostic info for each hash and return how many were not found."""
|
||||
missing = 0
|
||||
files_db = db.get("files", {})
|
||||
by_md5 = db.get("indexes", {}).get("by_md5", {})
|
||||
@@ -216,8 +216,8 @@ def resolve_file(
|
||||
# missing OR has a hash mismatch (wrong variant on disk)
|
||||
name = file_entry.get("name", "")
|
||||
if file_entry.get("type") == "directory" or name.endswith("/"):
|
||||
# A tree the collection does not hold is missing, as verify says;
|
||||
# the asset store holds files, not trees.
|
||||
# A tree the collection does not hold is missing, as verify says,
|
||||
# since the asset store holds files, not trees.
|
||||
return (path, status) if path else (None, "not_found")
|
||||
declared = _declared_hashes(file_entry)
|
||||
if file_entry.get("unsourceable") and not any(declared.values()):
|
||||
|
||||
+4
-4
@@ -288,7 +288,7 @@ def _truth_steps(args: argparse.Namespace, results: dict) -> bool:
|
||||
all_ok = True
|
||||
# Step 2c: Generate truth YAMLs
|
||||
# A targeted run writes each platform's model under the target as that
|
||||
# platform files it; the diff is told the target and reads the same place.
|
||||
# platform files it, and the diff is told the target and reads the same place.
|
||||
truth_dir = Path(args.output_dir) / "truth"
|
||||
|
||||
if args.with_truth or args.with_export:
|
||||
@@ -547,7 +547,7 @@ def main() -> None:
|
||||
all_ok = all_ok and ok
|
||||
|
||||
# Step 3b: Destinations both layers claim, and how each was settled. The
|
||||
# ones the pack settles by itself must stay at zero cost; the rest name an
|
||||
# ones the pack settles by itself must stay at zero cost. The rest name an
|
||||
# upstream declaration no build can repair, so this reports and never gates.
|
||||
# A crash is not a report: without --strict slots.py exits 0 on every
|
||||
# conflict, so a non-zero code means it never finished reading.
|
||||
@@ -658,7 +658,7 @@ def main() -> None:
|
||||
print("\n--- 4c/8 generate target manifests: SKIPPED (--skip-packs) ---")
|
||||
results["generate_target_manifests"] = SKIPPED
|
||||
|
||||
# Step 5: Consistency check. verify describes the full pack; a run that
|
||||
# Step 5: Consistency check. verify describes the full pack, and a run that
|
||||
# built only a reduced one (--source platform or truth) has nothing to
|
||||
# compare it with, and says so instead of failing on an empty parse.
|
||||
full_pack_built = args.source == "full" or args.all_variants
|
||||
@@ -675,7 +675,7 @@ def main() -> None:
|
||||
|
||||
# Step 6: Pack integrity (extract + hash verification)
|
||||
if not args.skip_packs and not full_pack_built:
|
||||
# The check reads the full pack's name; a reduced pack holds fewer
|
||||
# The check reads the full pack's name, while a reduced pack holds fewer
|
||||
# files than the platform declares by design.
|
||||
print("\n--- 6/8 pack integrity: SKIPPED (no full pack built) ---")
|
||||
results["pack_integrity"] = SKIPPED
|
||||
|
||||
+26
-26
@@ -66,7 +66,7 @@ WIDEN_STEPS = (0, 3, 6, 12, 25, 50)
|
||||
# The fallback mapping is a full sequence diff, and it only runs once exact
|
||||
# anchoring has failed, which is rare. openbor.c is the largest file any
|
||||
# profile cites, 55k lines against 57k, and diffing the pair takes nine
|
||||
# seconds; the ceiling sits above it so that one resolves, and still stops a
|
||||
# seconds. The ceiling sits above it so that one resolves, and still stops a
|
||||
# pathological pair from stalling a sweep of every profile.
|
||||
MAX_MATCH_LINES = 60000
|
||||
|
||||
@@ -217,7 +217,7 @@ def strip_repo_word(
|
||||
if head.lower() in known and tail and "/" not in head and "." not in head:
|
||||
tail = tail.strip()
|
||||
# `ArcadeFlashWeb v1.0.2 flash/flash.html` names the release the file
|
||||
# was read from; the version is not part of the path either.
|
||||
# was read from. The version is not part of the path either.
|
||||
version, _, rest = tail.partition(" ")
|
||||
if rest and _VERSION_WORD_RE.fullmatch(version):
|
||||
return rest.strip()
|
||||
@@ -229,7 +229,7 @@ def narrow_by_cited(matches: list[str], cited_dirs: Iterable[str]) -> list[str]:
|
||||
"""Keep the candidates under the deepest directory the profile cites.
|
||||
|
||||
A bare `version.h` matches BasiliskII/src/include/version.h and
|
||||
SheepShaver/src/include/version.h in cebix/macemu; the profile's own
|
||||
SheepShaver/src/include/version.h in cebix/macemu. The profile's own
|
||||
source_refs all live under SheepShaver/src, which is the tree the prose
|
||||
meant. Candidates under no cited directory are kept only when none is.
|
||||
"""
|
||||
@@ -271,7 +271,7 @@ def _anchor_tokens(entry: dict) -> list[str]:
|
||||
|
||||
Both the content values and the set name are searched, because profiles
|
||||
cite two different shapes. FBNeo and Hypseus refs point at the ROM table,
|
||||
where the CRC32s are; MAME refs point at the machine declaration, which
|
||||
where the CRC32s are, while MAME refs point at the machine declaration, which
|
||||
carries the set name and no hash at all.
|
||||
"""
|
||||
tokens = collect_tokens(entry)
|
||||
@@ -328,7 +328,7 @@ class Citation:
|
||||
|
||||
A structured citation lives under a source_ref key and keeps its entry
|
||||
for the values it declares. A prose citation is a path:line run found
|
||||
inside any other string scalar; its spans locate the path and every
|
||||
inside any other string scalar. Its spans locate the path and every
|
||||
range token inside the scalar line, so a recale can move the numbers
|
||||
and leave the sentence around them alone.
|
||||
"""
|
||||
@@ -382,7 +382,7 @@ def _walk_document(
|
||||
|
||||
|
||||
# What separates one citation from the next in prose. lynx-emu writes
|
||||
# "...:112; mednafen src/lynx/system.cpp:86-93": without the semicolon the
|
||||
# `...:112; mednafen src/lynx/system.cpp:86-93`: without the semicolon the
|
||||
# project word read as prose and the citation was charged to the fork. A
|
||||
# sentence end separates them the same way.
|
||||
_PROSE_LIST_DELIMITERS = ",;:(."
|
||||
@@ -421,7 +421,7 @@ def resolve_project_word(path: str, trees: Iterable[list[str] | None]) -> str |
|
||||
device/pif/bootrom_hle.c`: plugins vendored in the declared repository
|
||||
under mupen64plus-video-rice/ and mupen64plus-core/. A citation is in
|
||||
reach when exactly one file ends with the path below a directory naming
|
||||
the project; the first tree that answers decides.
|
||||
the project. The first tree that answers decides.
|
||||
"""
|
||||
word, _, rest = path.partition(" ")
|
||||
rest = rest.strip().lstrip("/")
|
||||
@@ -460,7 +460,7 @@ def _prose_runs(
|
||||
# A literal block keeps its line breaks, so a citation wrapped after
|
||||
# `file:` or after a range's comma lost the ranges on the next line
|
||||
# (gearcoleco `libretro.cpp:` then four ranges). The next line is
|
||||
# read as the run's continuation; a run reaching into it is wrapped.
|
||||
# read as the run's continuation, and a run reaching into it is wrapped.
|
||||
line = own.rstrip()
|
||||
boundary = len(line)
|
||||
following = lines[number + 1].lstrip() if number + 1 < len(lines) else ""
|
||||
@@ -941,7 +941,7 @@ def anchor_part(
|
||||
|
||||
pin_lines = fetch(PIN, part.path, part.start, tokens)
|
||||
if pin_lines is None and path != part.path:
|
||||
# A ref may cite a bare filename the tree search resolved at HEAD; the
|
||||
# A ref may cite a bare filename the tree search resolved at HEAD. The
|
||||
# same resolved path usually holds at the pin. A genuine rename keeps
|
||||
# the old path at the pin, which is why that one is tried first.
|
||||
pin_lines = fetch(PIN, path, part.start, tokens)
|
||||
@@ -949,7 +949,7 @@ def anchor_part(
|
||||
# Present at HEAD, absent at the pin, and the cited range fits the
|
||||
# HEAD file: the ref was written against HEAD while source_commit
|
||||
# still names an older revision. Saying "missing" sends the reader
|
||||
# hunting for a move that never happened; the fix is the pin.
|
||||
# hunting for a move that never happened: the fix is the pin.
|
||||
reason = "pin revision missing"
|
||||
if _written_against_head(part, head_lines):
|
||||
reason = "written against HEAD, pin names an older revision"
|
||||
@@ -1255,7 +1255,7 @@ def branch_in_url(url: str) -> str | None:
|
||||
def branch_candidates(url: str) -> list[str]:
|
||||
"""Each prefix of a /tree/ tail, shortest first, that may be the branch.
|
||||
|
||||
A branch name can hold a slash, so the tail is not split blindly; the
|
||||
A branch name can hold a slash, so the tail is not split blindly. The
|
||||
first prefix the forge resolves is the branch, the rest a directory.
|
||||
Taken whole, `master/tests/cruzes` resolved to nothing and the profile
|
||||
was skipped as having no resolvable pin.
|
||||
@@ -1311,7 +1311,7 @@ def verify_at_pin(
|
||||
part, "UNCHECKED", None, None, None, [],
|
||||
"entry declares no value to look for",
|
||||
)
|
||||
# An archive ref cites the line declaring the set; its members follow, one
|
||||
# An archive ref cites the line declaring the set. Its members follow, one
|
||||
# per line, so the window reaches forward as far as the entry has members.
|
||||
reach = SELF_CHECK_CONTEXT + len(tokens)
|
||||
lo = max(0, part.start - 1 - SELF_CHECK_CONTEXT)
|
||||
@@ -1427,7 +1427,7 @@ def _resolve_project_words(
|
||||
) -> list:
|
||||
"""Point `project file` parts at the declared tree that vendors them.
|
||||
|
||||
tree_order() yields the trees to search, pin before HEAD; it is only
|
||||
tree_order() yields the trees to search, pin before HEAD. It is only
|
||||
called for a part that needs it, since reading a HEAD tree costs a
|
||||
request. Parts no tree carries stay external.
|
||||
"""
|
||||
@@ -1587,7 +1587,7 @@ def build_report(
|
||||
|
||||
The pin comes before HEAD because that is the revision the ref was
|
||||
written against. `mame` cites files that both mamedev/mame and the
|
||||
libretro fork carry today, but only the upstream pin holds them; the
|
||||
libretro fork carry today, but only the upstream pin holds them. The
|
||||
fork's pin is an older tree where they lived elsewhere.
|
||||
"""
|
||||
candidates = [path]
|
||||
@@ -1825,7 +1825,7 @@ def build_report(
|
||||
A ref is only broken when it fails everywhere. `mame` cites files that
|
||||
both mamedev/mame and its libretro fork hold, with the same delimited
|
||||
token on the cited line, so no single attribution can be right for all
|
||||
of them; taking the best outcome across repositories settles it.
|
||||
of them. Taking the best outcome across repositories settles it.
|
||||
"""
|
||||
owner, _actual = resolve_path(part.path, part.start, tuple(tokens))
|
||||
carrying = [
|
||||
@@ -2670,7 +2670,7 @@ def pending_recale(
|
||||
moves = _prose_moves(entry, movable)
|
||||
if not moves:
|
||||
# Either nothing has to move, or the run cannot be written
|
||||
# without guessing; the second case still blocks the pin.
|
||||
# without guessing, and the second case still blocks the pin.
|
||||
if any(
|
||||
part.status in movable
|
||||
and part.start is not None
|
||||
@@ -2766,7 +2766,7 @@ def pins_changed(before: dict, after: dict) -> list[str]:
|
||||
def cites_primary(report: ProfileReport) -> bool:
|
||||
"""Whether a ref resolved in the repository source_commit pins.
|
||||
|
||||
A part attributed to another declared repository carries its slug; one
|
||||
A part attributed to another declared repository carries its slug, and one
|
||||
read in the primary carries none. A profile citing nothing proves
|
||||
nothing either way and is not held back.
|
||||
"""
|
||||
@@ -2790,7 +2790,7 @@ def _set_pin(text: str, document: dict, field: str, value: str, after: str) -> s
|
||||
|
||||
|
||||
def _git_history(path: Path) -> list[str] | None:
|
||||
"""Commits touching the profile, newest first; None when git cannot say.
|
||||
"""Commits touching the profile, newest first, or None when git cannot say.
|
||||
|
||||
An untracked profile has no history. A git that fails (no repository,
|
||||
dubious ownership, git missing) has not answered, and reading that as
|
||||
@@ -2897,7 +2897,7 @@ def _resolve_at(
|
||||
) -> tuple[str, object]:
|
||||
"""("found", (path, lines)) or ("unclear"|"missing", reason).
|
||||
|
||||
Prose often cites a bare filename; the tree at the revision resolves it
|
||||
Prose often cites a bare filename. The tree at the revision resolves it
|
||||
when exactly one path carries that name. An unreadable or truncated
|
||||
tree, or several paths carrying the name, proves nothing: those come
|
||||
back "unclear", never "missing", because a verdict of absence taken on
|
||||
@@ -2984,7 +2984,7 @@ def _writing_pairs(
|
||||
|
||||
Each reading lists (repo, writing pin, current pin) per declared
|
||||
repository. A repository whose text lived under several pins gives one
|
||||
reading per pin; the caller settles them when they all agree.
|
||||
reading per pin, and the caller settles them when they all agree.
|
||||
|
||||
`prose_written_at` names the pin a reader confirmed the prose against.
|
||||
Among several candidate pins it is the answer: fixnes's doOpenFDSBIOS
|
||||
@@ -3314,7 +3314,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--offline", action="store_true",
|
||||
help="read only what the cache holds; no network access",
|
||||
help="read only what the cache holds, no network access",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cache-dir", default=DEFAULT_CACHE,
|
||||
@@ -3428,7 +3428,7 @@ def _apply_writes(
|
||||
if not (args.rebase_refs or args.bump_commit) or _stale_prose_blocks(name, path, args):
|
||||
return
|
||||
with tempfile.TemporaryDirectory() as scratch:
|
||||
# Always work on a copy. A dry run reports what it left there; a real
|
||||
# Always work on a copy. A dry run reports what it left there, a real
|
||||
# run promotes it. A recale rewrites the refs for HEAD, so the pin
|
||||
# has to follow or neither moves: a profile whose refs describe one
|
||||
# revision and whose pin names another is exactly what the
|
||||
@@ -3445,7 +3445,7 @@ def _apply_writes(
|
||||
if args.rebase_refs and applied and not moved:
|
||||
print(
|
||||
f"{name}: refs recaled but the pin will not follow, so nothing "
|
||||
"was written; the prose or an annotated ref has to move first",
|
||||
"was written: the prose or an annotated ref has to move first",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return
|
||||
@@ -3523,7 +3523,7 @@ def _print_extras(
|
||||
) -> None:
|
||||
"""Optional per-profile sections beyond the ref report.
|
||||
|
||||
The ref report mutes a forge that refuses; the extras read the same
|
||||
The ref report mutes a forge that refuses, but the extras read the same
|
||||
forge again, so a refusal here is reported on the profile's own line
|
||||
instead of ending a pass over every profile. A quota signal still
|
||||
stops everything.
|
||||
@@ -3688,7 +3688,7 @@ def _refuse_flag_combinations(args: argparse.Namespace) -> None:
|
||||
) if on
|
||||
]
|
||||
if machine and extras:
|
||||
# The extra sections only exist in the text report; the other outputs
|
||||
# The extra sections only exist in the text report. The other outputs
|
||||
# returned before reaching them and printed as if never asked.
|
||||
print(
|
||||
f"{', '.join(machine)} does not carry {', '.join(extras)}",
|
||||
@@ -3775,7 +3775,7 @@ def main() -> None:
|
||||
try:
|
||||
_apply_writes(args, name, profile, report)
|
||||
except YamlWriteError as exc:
|
||||
# The guard already left the file untouched; keep going.
|
||||
# The guard already left the file untouched, so keep going.
|
||||
print(f"{name}: write refused: {exc}", file=sys.stderr)
|
||||
failed.add(name)
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ def archive_members(db: dict) -> dict[tuple[str, int], list[tuple[str, str]]]:
|
||||
def _held_in_archive(entry: dict, members: dict) -> bool:
|
||||
"""Whether a catalog entry is a member of one of the collection's ZIPs.
|
||||
|
||||
crc32 and size only nominate candidates; a declared sha1 must match the
|
||||
crc32 and size only nominate candidates. A declared sha1 must match the
|
||||
member's bytes.
|
||||
"""
|
||||
crc = str(entry.get("crc32") or "").lower()
|
||||
@@ -99,7 +99,7 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict
|
||||
|
||||
A DAT counts as covered when the collection holds at least one of
|
||||
its entries. Missing entries from covered DATs are acquisition
|
||||
targets; missing entries from DATs the collection does not cover at
|
||||
targets. Missing entries from DATs the collection does not cover at
|
||||
all are out of scope and only counted. No-Intro tags every non-game
|
||||
dump "[BIOS]", including digital title distribution, so without this
|
||||
split the target list is swamped by content the project never ships.
|
||||
@@ -178,7 +178,7 @@ def main() -> int:
|
||||
"--derive",
|
||||
action="store_true",
|
||||
help="Find missing entries a held dump contains (halves, word splits, "
|
||||
"byte swaps, concatenations); reads the collection, about 35 s",
|
||||
"byte swaps, concatenations). Reads the collection, about 35 s",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Full report as JSON")
|
||||
parser.add_argument(
|
||||
@@ -189,7 +189,7 @@ def main() -> int:
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.write_derivable and not args.derive:
|
||||
parser.error("--write-derivable records what --derive finds; pass both")
|
||||
parser.error("--write-derivable records what --derive finds: pass both")
|
||||
|
||||
db = load_database(args.db)
|
||||
snapshots = load_provenance_snapshots(args.provenance_dir)
|
||||
|
||||
@@ -68,7 +68,7 @@ def hold_data_dirs(stack: contextlib.ExitStack, registry: dict) -> None:
|
||||
"""Hold every cached data directory shared for the life of *stack*.
|
||||
|
||||
A pack resolves names against a tree, then copies the file minutes
|
||||
later; a refresh swapping the tree in between shipped bytes nobody
|
||||
later. A refresh swapping the tree in between shipped bytes nobody
|
||||
verified, or died on a file the new tree no longer has. A refresh waits
|
||||
for the holders. Never called before a refresh in the same process:
|
||||
the exclusive lock would wait on this one.
|
||||
@@ -445,8 +445,8 @@ def _refresh_entry(
|
||||
|
||||
if remote_tag is None:
|
||||
# Read before the download. A release published in between then
|
||||
# leaves an older tag on newer content, which the next run refreshes;
|
||||
# read after, it left a newer tag on older content, trusted for good.
|
||||
# leaves an older tag on newer content, which the next run refreshes.
|
||||
# Read after, it left a newer tag on older content, trusted for good.
|
||||
if source_type == "zip":
|
||||
remote_tag = _get_remote_etag(source_url)
|
||||
else:
|
||||
|
||||
@@ -55,7 +55,7 @@ PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
|
||||
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
|
||||
MANUAL_AREAS = ("coreinfo", "ci", "profiles")
|
||||
|
||||
JOB_TIMEOUT = 1800 # 30 minutes per refresher; scrapers rarely exceed 10
|
||||
JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -81,7 +81,7 @@ class JobResult:
|
||||
def _load_platform_registry() -> dict[str, dict]:
|
||||
"""Return the platforms map from _registry.yml.
|
||||
|
||||
pyyaml is already a project dependency; the registry is small so we read
|
||||
pyyaml is already a project dependency, and the registry is small so we read
|
||||
it once at startup rather than parsing each dispatch.
|
||||
"""
|
||||
import yaml
|
||||
@@ -122,7 +122,7 @@ def plan_jobs(
|
||||
|
||||
Surfaced-only entries carry stale state that only a human can resolve
|
||||
(new .info file, manual DAT download, PyPI bump). Ignored entries are
|
||||
anything not STALE (OK, SKIPPED, UNKNOWN, ERROR); ERROR is also surfaced
|
||||
anything not STALE (OK, SKIPPED, UNKNOWN, ERROR). ERROR is also surfaced
|
||||
so the final report mentions it.
|
||||
"""
|
||||
jobs: list[Job] = []
|
||||
@@ -219,7 +219,7 @@ def _command_for(
|
||||
]
|
||||
|
||||
if area == "targets":
|
||||
# "<platform> cores" rows list buildbot names without a profile; no
|
||||
# "<platform> cores" rows list buildbot names without a profile. No
|
||||
# scraper fixes that, a human writes `cores:` or _overrides.yml.
|
||||
if subject.endswith(" cores"):
|
||||
return None
|
||||
@@ -237,7 +237,7 @@ def _command_for(
|
||||
|
||||
if area == "data":
|
||||
if subject == "_data_dirs.yml":
|
||||
# The registry file itself cannot be refreshed; its entries can.
|
||||
# The registry file itself cannot be refreshed, but its entries can.
|
||||
return None
|
||||
return [
|
||||
sys.executable,
|
||||
@@ -276,7 +276,7 @@ def _command_for(
|
||||
def _github_token_env() -> dict[str, str]:
|
||||
"""Hand `gh auth token` to the subprocess environment.
|
||||
|
||||
emudeck and retropie target scrapers hit the GitHub API; without a token
|
||||
emudeck and retropie target scrapers hit the GitHub API. Without a token
|
||||
the unauthenticated rate limit (60/h) is burnt within a few platforms.
|
||||
"""
|
||||
env = os.environ.copy()
|
||||
@@ -396,7 +396,7 @@ def main() -> int:
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="list what would be refreshed; do not run any command",
|
||||
help="list what would be refreshed without running any command",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--jobs",
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
"""Region vocabulary and ordered-priority selection for pack generation.
|
||||
|
||||
A file's region: value is the region the emulator code selects it for, not the
|
||||
region of the dump. Signal standard has no field of its own; it survives only
|
||||
region of the dump. Signal standard has no field of its own. It survives only
|
||||
inside slugs whose hardware register fuses the two axes (Saturn SMPC areas).
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Cross-check profile region: values against dump-catalog provenance.
|
||||
|
||||
Advisory only. The emulator source decides what a region: value should be; this
|
||||
Advisory only. The emulator source decides what a region: value should be. This
|
||||
reports where an independent catalogue disagrees, so a hand-written value gets a
|
||||
second opinion before it is trusted.
|
||||
|
||||
|
||||
@@ -391,7 +391,7 @@ def missing_report(
|
||||
"md5": target,
|
||||
"recipes_tried": sorted(usable),
|
||||
"reason": (
|
||||
"no recipe reproduces this MD5; the pinned archive is "
|
||||
"no recipe reproduces this MD5: the pinned archive is "
|
||||
"not TorrentZip or follows a recipe not documented here"
|
||||
if usable
|
||||
else "ROMs for every documented recipe are missing"
|
||||
|
||||
@@ -102,7 +102,7 @@ def yaml_load(stream: str | TextIO) -> Any:
|
||||
def yaml_load_upstream(stream: str | TextIO) -> Any:
|
||||
"""Parse a third party's YAML the way its own PyYAML code reads it.
|
||||
|
||||
Our files refuse a repeated key; a platform's file is data we transcribe,
|
||||
Our files refuse a repeated key, but a platform's file is data we transcribe,
|
||||
and its code keeps the last value. Batocera 43.1's configgen-defaults.yml
|
||||
names rtcw's core twice, and refusing it stopped the rescrape.
|
||||
"""
|
||||
|
||||
@@ -76,7 +76,7 @@ def merge_mame_profile(
|
||||
updated_bios.append(entry)
|
||||
matched_names.add(set_name)
|
||||
elif add_new:
|
||||
# New BIOS set — only added to the main profile
|
||||
# New BIOS set, only added to the main profile
|
||||
entry = {
|
||||
"name": f"{set_name}.zip",
|
||||
"required": True,
|
||||
@@ -88,7 +88,7 @@ def merge_mame_profile(
|
||||
updated_bios.append(entry)
|
||||
|
||||
# Entries not matched by the scraper stay untouched
|
||||
# (computer ROMs, device ROMs, etc. — outside BIOS root set scope)
|
||||
# (computer ROMs, device ROMs, etc., outside BIOS root set scope)
|
||||
for set_name, entry in existing_by_name.items():
|
||||
if set_name not in matched_names:
|
||||
updated_bios.append(entry)
|
||||
@@ -296,7 +296,7 @@ def _diff_fbneo(
|
||||
}
|
||||
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
|
||||
|
||||
def _load_yaml(path: str) -> dict[str, Any]:
|
||||
@@ -399,7 +399,7 @@ def _patch_core_version(text: str, version: str) -> str:
|
||||
|
||||
|
||||
def _patch_scalar(text: str, key: str, value: str) -> str:
|
||||
"""Replace a top-level scalar in-place, quoted; absent value, no change."""
|
||||
"""Replace a top-level scalar in-place, quoted. An absent value changes nothing."""
|
||||
if not value:
|
||||
return text
|
||||
return re.sub(
|
||||
@@ -600,7 +600,7 @@ def _patch_block_values(block: str, entry: dict) -> str:
|
||||
match = _FIELD_LINE.match(line)
|
||||
if match and match["key"] == key:
|
||||
if match["value"].strip().strip('"') != str(value):
|
||||
# A size comment spells the old value; it goes with it.
|
||||
# A size comment spells the old value, so it goes with it.
|
||||
lines[i] = f" {key}: {rendered}"
|
||||
break
|
||||
else:
|
||||
@@ -619,7 +619,7 @@ def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None:
|
||||
FBNeo profiles have individual ROM entries with archive: field.
|
||||
Patches core_version, the content values of existing entries that
|
||||
changed, and appends new ROM entries. Advancing core_version alone left
|
||||
a profile claiming a release whose crc32 it did not carry; comments and
|
||||
a profile claiming a release whose crc32 it did not carry. Comments and
|
||||
curated source_refs stay as written.
|
||||
"""
|
||||
p = Path(path)
|
||||
|
||||
@@ -390,7 +390,7 @@ def scraper_cli(
|
||||
config["systems"][sys_id]["files"].append(entry)
|
||||
# Merge into existing YAML: preserve fields the scraper doesn't generate
|
||||
# (data_directories, case_insensitive_fs, manually added metadata).
|
||||
# The scraper replaces systems + files; everything else is preserved.
|
||||
# The scraper replaces systems + files, everything else is preserved.
|
||||
output_path = Path(args.output)
|
||||
if output_path.exists():
|
||||
with open(output_path) as f:
|
||||
|
||||
@@ -95,7 +95,7 @@ def es_systems_emulators(text: str) -> tuple[set[str], set[str]]:
|
||||
"""Every core es_systems.yml offers, and those that run standalone.
|
||||
|
||||
ES lists, per system, each emulator a user may pick and the cores under
|
||||
it; configgen-defaults.yml only names the one picked by default, so
|
||||
it. configgen-defaults.yml only names the one picked by default, so
|
||||
reading it alone left DOSBox Staging, melonDS DS and nine other built
|
||||
cores outside the pack.
|
||||
"""
|
||||
@@ -111,7 +111,7 @@ def es_systems_emulators(text: str) -> tuple[set[str], set[str]]:
|
||||
for emulator, entries in emulators.items():
|
||||
if not isinstance(entries, dict):
|
||||
continue
|
||||
# A standalone emulator is its own core; what it lists under it
|
||||
# A standalone emulator is its own core. What it lists under it
|
||||
# are machines or builds, and the non-cores among them are named
|
||||
# in the overrides. Keys whose value is not a mapping
|
||||
# (archs_include) describe the entry, not a core.
|
||||
|
||||
@@ -12,8 +12,8 @@ BizHawk declares firmware in FirmwareDatabase.cs using four patterns:
|
||||
FirmwareAndOption(sha1, size, sys, id, ...) - combined one-liner
|
||||
|
||||
Variable assignments (var x = File(...)) let Option() reference files
|
||||
by name. Multiple options per firmware slot are ranked by status;
|
||||
the Ideal non-bad option is selected as canonical.
|
||||
by name. Multiple options per firmware slot are ranked by status,
|
||||
and the Ideal non-bad option is selected as canonical.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -127,7 +127,7 @@ BIZHAWK_CORES = [
|
||||
"handy",
|
||||
"quicknes",
|
||||
"genesis_plus_gx",
|
||||
# Ares64 is BizHawk's N64 core only; its firmware (the 64DD IPL) is in
|
||||
# Ares64 is BizHawk's N64 core only. Its firmware (the 64DD IPL) is in
|
||||
# FirmwareDatabase.cs. The ares profile's thirty systems are not what
|
||||
# BizHawk runs ares for.
|
||||
"mupen64plus_next",
|
||||
@@ -173,7 +173,7 @@ def parse_firmware_database(
|
||||
"""
|
||||
source = _strip_comments(source)
|
||||
|
||||
# ── Pass 1: collect File() definitions ────────────────────────
|
||||
# Pass 1: collect File() definitions
|
||||
files_by_hash: dict[str, dict] = {}
|
||||
var_to_hash: dict[str, str] = {}
|
||||
|
||||
@@ -210,7 +210,7 @@ def parse_firmware_database(
|
||||
if var_name:
|
||||
var_to_hash[var_name] = key
|
||||
|
||||
# ── Pass 2: collect firmware slots and options ────────────────
|
||||
# Pass 2: collect firmware slots and options
|
||||
|
||||
# FirmwareAndOption one-liner
|
||||
fao_re = re.compile(
|
||||
|
||||
@@ -27,7 +27,7 @@ SOURCE_URL = (
|
||||
)
|
||||
|
||||
# Libretro cores that expect BIOS files in a subdirectory of system/.
|
||||
# System.dat lists filenames flat; the scraper prepends the prefix.
|
||||
# System.dat lists filenames flat, and the scraper prepends the prefix.
|
||||
# ref: each core's libretro.c or equivalent -see platforms/README.md
|
||||
CORE_SUBDIR_MAP = {
|
||||
"nec-pc-98": "np2kai", # libretro-np2kai/sdl/libretro.c
|
||||
|
||||
@@ -39,7 +39,7 @@ _REPO_URL = "https://github.com/mamedev/mame.git"
|
||||
_STALE_HOURS = 24
|
||||
|
||||
|
||||
# ── Cache ────────────────────────────────────────────────────────────
|
||||
# Cache
|
||||
|
||||
|
||||
def _load_cache() -> dict[str, Any] | None:
|
||||
@@ -74,7 +74,7 @@ def _write_cache(data: dict[str, Any]) -> None:
|
||||
log.info("cache written to %s", _CACHE_PATH)
|
||||
|
||||
|
||||
# ── Git operations ───────────────────────────────────────────────────
|
||||
# Git operations
|
||||
|
||||
|
||||
def _run_git(
|
||||
@@ -92,7 +92,7 @@ def _run_git(
|
||||
def _sparse_clone(tag: str) -> None:
|
||||
"""Clone the release the data will be labelled with, not master.
|
||||
|
||||
The version comes from the latest release; cloning master gave line
|
||||
The version comes from the latest release. Cloning master gave line
|
||||
numbers and hashes of a later revision under that release's label, and
|
||||
the profiles cited lines no declared revision holds.
|
||||
"""
|
||||
@@ -183,7 +183,7 @@ def _cleanup() -> None:
|
||||
shutil.rmtree(_CLONE_DIR)
|
||||
|
||||
|
||||
# ── Profile discovery ────────────────────────────────────────────────
|
||||
# Profile discovery
|
||||
|
||||
|
||||
_MAME_TREES = frozenset({
|
||||
@@ -223,7 +223,7 @@ def _find_mame_profiles() -> list[Path]:
|
||||
return profiles
|
||||
|
||||
|
||||
# ── Diff formatting ──────────────────────────────────────────────────
|
||||
# Diff formatting
|
||||
|
||||
|
||||
def _format_diff(
|
||||
@@ -268,7 +268,7 @@ def _format_diff(
|
||||
return lines
|
||||
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────
|
||||
# Main
|
||||
|
||||
|
||||
def _fetch_hashes(force: bool) -> dict[str, Any]:
|
||||
@@ -355,10 +355,10 @@ def release_profile(
|
||||
) -> dict[str, Any]:
|
||||
"""A profile for one MAME release a platform builds, from its own source.
|
||||
|
||||
Recalbox ships libretro/mame twice, built at 0.258 and at 0.278; the
|
||||
Recalbox ships libretro/mame twice, built at 0.258 and at 0.278. The
|
||||
BIOS root sets and their ROMs change between releases, so the current
|
||||
MAME profile does not describe them. Each set keeps the system and the
|
||||
required flag the current profile gives it; the contents and the line
|
||||
required flag the current profile gives it, while the contents and the line
|
||||
are the release's own.
|
||||
"""
|
||||
version = release.removeprefix("mame")
|
||||
|
||||
@@ -9,8 +9,8 @@ MiSTer verification logic (MiSTer-devel/Downloader_MiSTer):
|
||||
- file_system.py:705-712, hash_file() is a plain MD5 of the file bytes
|
||||
- jobs/process_db_index_worker.py:216, an installed file is up to date when
|
||||
its MD5 equals the description hash
|
||||
- jobs/process_db_index_worker.py:242, integrity checks compare MD5 only;
|
||||
the declared size is logged on failure but never gates the result
|
||||
- jobs/process_db_index_worker.py:242, integrity checks compare MD5 only,
|
||||
and the declared size is logged on failure but never gates the result
|
||||
- jobs/fetch_file_worker.py:61-65, a freshly downloaded file is rejected on
|
||||
MD5 mismatch
|
||||
|
||||
@@ -51,7 +51,7 @@ SOURCE_URL = (
|
||||
|
||||
DB_MEMBER = "bios_db.json"
|
||||
|
||||
# Files live under games/ on the SD card; base_destination carries that prefix
|
||||
# Files live under games/ on the SD card, and base_destination carries that prefix
|
||||
DB_ROOT = "games/"
|
||||
|
||||
# MiSTer core folder -> (retrobios system ID, display name, core repository)
|
||||
|
||||
@@ -108,7 +108,7 @@ SYSTEMS_TREE_URL = (
|
||||
)
|
||||
SYSTEMS_DIR = "package/recalbox-romfs2/systems"
|
||||
# One raw read per system.ini made about 150 requests and GitLab throttled
|
||||
# the scrape past half an hour; GraphQL returns a batch of blobs at once.
|
||||
# the scrape past half an hour. GraphQL returns a batch of blobs at once.
|
||||
GRAPHQL_URL = "https://gitlab.com/api/graphql"
|
||||
BLOBS_QUERY = (
|
||||
"query($paths: [String!]!, $ref: String!) { project(fullPath: \"recalbox/recalbox\") "
|
||||
|
||||
@@ -456,7 +456,7 @@ class Scraper(BaseScraper):
|
||||
"verification_mode": "md5",
|
||||
"cores": cores,
|
||||
# Each component is one emulator RetroDECK ships and launches on
|
||||
# its own; retroarch is the one that runs cores.
|
||||
# its own, and retroarch is the one that runs cores.
|
||||
"standalone_cores": [c for c in cores if c != "retroarch"],
|
||||
"systems": systems,
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ SOURCE_TEMPLATE = (
|
||||
"https://raw.githubusercontent.com/ROCKNIX/distribution/{ref}/"
|
||||
"projects/ROCKNIX/packages/rocknix/sources/scripts/rocknix-systems"
|
||||
)
|
||||
# The branch a test or a manual run can still name; a scrape reads the tag
|
||||
# The branch a test or a manual run can still name. A scrape reads the tag
|
||||
# of the release its YAML is labelled with.
|
||||
SOURCE_URL = SOURCE_TEMPLATE.format(ref="next")
|
||||
STANDALONE_TREE_URL = (
|
||||
@@ -53,7 +53,7 @@ def standalone_emulators(package_names: list[str]) -> list[str]:
|
||||
"""The standalone emulators ROCKNIX packages, by the names profiles use.
|
||||
|
||||
A package is named after its emulator with an ``-sa`` suffix
|
||||
(``duckstation-sa``), once without its hyphen (``hatarisa``); both
|
||||
(``duckstation-sa``), once without its hyphen (``hatarisa``), and both
|
||||
spellings are kept so a profile listing either is found.
|
||||
"""
|
||||
names: set[str] = set()
|
||||
|
||||
@@ -120,7 +120,7 @@ SLUG_MAP: dict[str, str] = {
|
||||
# Mirrors are per file, not per core: a slug sharing a core with another reads
|
||||
# only the firmware that core needs for its own system. The Neo Geo slugs run
|
||||
# fbneo, as arcade does (index.ts:410-418), and fbneo boots every Neo Geo
|
||||
# game from neogeo.zip; the fixture gives it to arcade alone. None mirrors
|
||||
# game from neogeo.zip, but the fixture gives it to arcade alone. None mirrors
|
||||
# every file of the slug.
|
||||
FIRMWARE_MIRRORS: dict[str, dict[str, tuple[str, ...] | None]] = {
|
||||
"tg16": {"turbografx-cd": None},
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
A DAT states, per set, the members an emulator version expects: name, size
|
||||
and CRC32. That is a recipe, and TorrentZip turns a recipe plus the ROM bytes
|
||||
into the exact archive. Profiles document a few hundred sets by hand; a DAT
|
||||
into the exact archive. Profiles document a few hundred sets by hand, while a DAT
|
||||
documents every set of one version at once.
|
||||
|
||||
MAME ships its ``-listxml`` output as a release asset and FBNeo keeps its DATs
|
||||
@@ -463,7 +463,7 @@ def main() -> int:
|
||||
nargs="?",
|
||||
const="latest",
|
||||
help="download upstream instead of using --pack "
|
||||
"(a MAME tag such as mame0289; bare, the newest release)",
|
||||
"(a MAME tag such as mame0289, or bare for the newest release)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cache-dir", default=".cache/dats",
|
||||
|
||||
@@ -45,7 +45,7 @@ class ShellSubsetError(RuntimeError):
|
||||
|
||||
|
||||
def _condition(text: str, env: dict[str, str]) -> bool | None:
|
||||
"""`[ "${A}" = "x" -o "${B}" != "y" ]`, `-n`, `-z`; None when a clause
|
||||
"""`[ "${A}" = "x" -o "${B}" != "y" ]`, `-n`, `-z`. None when a clause
|
||||
tests a variable nothing has set and no other clause is true."""
|
||||
results: list[bool | None] = []
|
||||
for clause in re.split(r"\s+-o\s+", text):
|
||||
@@ -72,12 +72,12 @@ def _condition(text: str, env: dict[str, str]) -> bool | None:
|
||||
|
||||
|
||||
# The variables this scraper reads. Set inside a case statement, which the
|
||||
# evaluator does not follow, they stop the scrape; anything else is skipped.
|
||||
# evaluator does not follow, they stop the scrape. Anything else is skipped.
|
||||
WATCHED = frozenset({"OPENGLES", "LIBRETRO_CORES", "EXCLUDE_LIBRETRO_CORES"})
|
||||
|
||||
|
||||
def _expand(value: str, env: dict[str, str]) -> str:
|
||||
"""${NAME} and $NAME from what is set so far; unset expands to nothing."""
|
||||
"""${NAME} and $NAME from what is set so far. Unset expands to nothing."""
|
||||
return re.sub(
|
||||
r"\$\{(\w+)\}|\$(\w+)",
|
||||
lambda m: env.get(m.group(1) or m.group(2), ""),
|
||||
@@ -182,7 +182,7 @@ def device_cores(cores_mk: str, env: dict[str, str]) -> list[str]:
|
||||
|
||||
def device_arch(project: str, options: str) -> str:
|
||||
"""The architecture Lakka builds a device for: aarch64 where its options
|
||||
list it, else arm; the Generic images are x86_64."""
|
||||
list it, else arm. The Generic images are x86_64."""
|
||||
if project == "Generic":
|
||||
return "x86_64"
|
||||
return "aarch64" if re.search(r"^\s*aarch64\)", options, re.MULTILINE) else "arm"
|
||||
|
||||
@@ -148,7 +148,7 @@ class Scraper(BaseTargetScraper):
|
||||
print(f" warning: no rp_module_id in {filename}", file=sys.stderr)
|
||||
continue
|
||||
# A libretro package is lr-<core> with the buildbot name hyphenated
|
||||
# (lr-beetle-psx -> beetle_psx); a standalone package keeps the id
|
||||
# (lr-beetle-psx -> beetle_psx), and a standalone package keeps the id
|
||||
# the platform list carries (dosbox-staging).
|
||||
core_name = module_id
|
||||
if core_name.startswith("lr-"):
|
||||
|
||||
@@ -91,7 +91,7 @@ def core_names(libretro: list[str], standalone: list[str]) -> list[str]:
|
||||
"""Package names as the profiles' cores: lists spell them.
|
||||
|
||||
A libretro package is the core's name with `-lr` (`beetle-psx-lr`,
|
||||
`neocd_lr`), spelled with underscores as the buildbot names it; a
|
||||
`neocd_lr`), spelled with underscores as the buildbot names it, and a
|
||||
standalone package drops its `-sa` (`duckstation-sa`, `hatarisa`).
|
||||
"""
|
||||
names = {re.sub(r"[-_]lr$", "", package).replace("-", "_") for package in libretro}
|
||||
@@ -101,7 +101,7 @@ def core_names(libretro: list[str], standalone: list[str]) -> list[str]:
|
||||
|
||||
# A package that builds nothing itself and pulls other cores in. idtech-lr
|
||||
# installs ecwolf, prboom, tyrquake and vitaquake2 (vitaquake3 and boom3 on
|
||||
# x86_64); no profile could name it.
|
||||
# x86_64). No profile could name it.
|
||||
METAPACKAGES = {"idtech-lr": "projects/ROCKNIX/packages/emulators/libretro/idtech-lr/package.mk"}
|
||||
_IF_RE = re.compile(r'^if \[ "\$\{(\w+)\}" = "?([\w-]+)"? \]; then\n(.*?)^fi$',
|
||||
re.MULTILINE | re.DOTALL)
|
||||
@@ -111,7 +111,7 @@ def metapackage_cores(package_mk: str, arch: str) -> list[str]:
|
||||
"""The -lr packages a metapackage depends on for one architecture.
|
||||
|
||||
OPENGLES_SUPPORT is yes on every ROCKNIX device (config/graphic, from
|
||||
OPENGLES="mesa" in projects/ROCKNIX/options); TARGET_ARCH is the device's.
|
||||
OPENGLES="mesa" in projects/ROCKNIX/options), and TARGET_ARCH is the device's.
|
||||
"""
|
||||
found: list[str] = []
|
||||
for variable, value, body in _IF_RE.findall(package_mk):
|
||||
|
||||
+10
-4
@@ -13,7 +13,7 @@ SYSTEM_ICON_BASE = "https://raw.githubusercontent.com/libretro/retroarch-assets/
|
||||
ICON_CACHE_PATH = Path(".cache") / "system_icons.json"
|
||||
|
||||
# Icon names confirmed to exist upstream. A name absent from this map has not
|
||||
# been checked yet; a name mapped to False has no icon and gets none rendered,
|
||||
# been checked yet. A name mapped to False has no icon and gets none rendered,
|
||||
# because a heading with a broken image reads worse than a heading without one.
|
||||
_icon_available: dict[str, bool] = {}
|
||||
|
||||
@@ -136,6 +136,12 @@ def _system_link(sys_id: str, prefix: str = "") -> str:
|
||||
return f"[{sys_id}]({prefix}systems/{slug}.md#{anchor})"
|
||||
return sys_id
|
||||
|
||||
def _field_text(key: object, value: object) -> str:
|
||||
"""A scalar of a structured block. A source reference reads as code."""
|
||||
text = str(value)
|
||||
return f"`{text}`" if key == "source_ref" else text
|
||||
|
||||
|
||||
def _render_yaml_value(lines: list[str], val: object, indent: int = 4) -> None:
|
||||
"""Render any YAML value as indented markdown."""
|
||||
pad = " " * indent
|
||||
@@ -151,7 +157,7 @@ def _render_yaml_value(lines: list[str], val: object, indent: int = 4) -> None:
|
||||
for item in v:
|
||||
if isinstance(item, dict):
|
||||
parts = [
|
||||
f"{ik}: {iv}"
|
||||
f"{ik}: {_field_text(ik, iv)}"
|
||||
for ik, iv in item.items()
|
||||
if not isinstance(iv, (dict, list))
|
||||
]
|
||||
@@ -164,12 +170,12 @@ def _render_yaml_value(lines: list[str], val: object, indent: int = 4) -> None:
|
||||
sv = str(v)
|
||||
if len(sv) > 200:
|
||||
sv = sv[:200] + "..."
|
||||
lines.append(f"{pad}- **{k}:** {sv}")
|
||||
lines.append(f"{pad}- **{k}:** {_field_text(k, sv)}")
|
||||
elif isinstance(val, list):
|
||||
for item in val:
|
||||
if isinstance(item, dict):
|
||||
parts = [
|
||||
f"{ik}: {iv}"
|
||||
f"{ik}: {_field_text(ik, iv)}"
|
||||
for ik, iv in item.items()
|
||||
if not isinstance(iv, (dict, list))
|
||||
]
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
"""One file per slot: keep the BIOS a core would actually load.
|
||||
|
||||
A slot is a (system, region) pair holding several interchangeable BIOS. Region
|
||||
filtering narrows a pack to one territory; a system can still ship three US
|
||||
filtering narrows a pack to one territory, but a system can still ship three US
|
||||
PlayStation BIOS that differ only by revision, which is what leaves the choice
|
||||
open in the frontend.
|
||||
|
||||
@@ -112,7 +112,7 @@ def resolve_slot_drops(
|
||||
"""Destinations to skip, and the groups no declared order can decide.
|
||||
|
||||
Returns (drops, undecidable). A group is decided only when every candidate
|
||||
declares a priority and exactly one holds the lowest; anything else keeps
|
||||
declares a priority and exactly one holds the lowest. Anything else keeps
|
||||
every candidate. ``held`` names the destinations the collection holds: a
|
||||
candidate it lacks ships nothing, so it neither wins nor displaces one.
|
||||
"""
|
||||
|
||||
+9
-9
@@ -9,7 +9,7 @@ that names the right file for a slot could never win and never even be heard.
|
||||
|
||||
The index is keyed by destination AND by name, the name carrying the union of
|
||||
what every same-named entry claims. Dolphin declares three IPL.bin separated
|
||||
only by their path, so a name-only key merges them; a path-only key answers
|
||||
only by their path, so a name-only key merges them, and a path-only key answers
|
||||
nothing when a candidate is known by name alone. Ambiguity resolves to the
|
||||
union and is never discarded, the same rule region.py already applies.
|
||||
"""
|
||||
@@ -476,7 +476,7 @@ def pack_overrides(
|
||||
"""Full pack destination -> the file a platform pack serves there instead.
|
||||
|
||||
Where a source-verified profile contradicts the scraped baseline on one
|
||||
destination, the pack answers to the platform it is built for; in
|
||||
destination, the pack answers to the platform it is built for. In
|
||||
existence mode the frontend never reads the bytes, so serving the
|
||||
emulator's file satisfies both. The ZIP builder, the install manifest and
|
||||
verify all read this: two of them deciding alone gave the one-line
|
||||
@@ -508,7 +508,7 @@ def slot_choices(
|
||||
"""Destination -> (core, the file it has served there) in decision order.
|
||||
|
||||
A pack built for a set of cores serves the first choice whose core is in
|
||||
it, None standing for the platform's own answer; no core in it leaves
|
||||
it, None standing for the platform's own answer. No core in it leaves
|
||||
the platform's answer too. The order is arbitrate's: a core agreeing with
|
||||
the platform settles the slot whatever else is present, then the
|
||||
strongest proof decides. Read for one core set by pack_overrides and
|
||||
@@ -562,7 +562,7 @@ def apply_override(
|
||||
|
||||
|
||||
def _strongest(claims: list[Claim]) -> Claim:
|
||||
"""The profile claim with the strongest proof; the first emulator breaks ties.
|
||||
"""The profile claim with the strongest proof, the first emulator breaking ties.
|
||||
|
||||
yaps2 pins its GameIndex.yaml by sha1 while armsx2 and lrps2 only match a
|
||||
path: alphabetical order served armsx2's copy to all three.
|
||||
@@ -575,13 +575,13 @@ def arbitrate(conflict: Conflict, mode: str, addressee: str = "platform") -> Dec
|
||||
|
||||
A pack answers to whoever asked for it. A platform pack must leave the
|
||||
frontend's own check green, because a user reading red concludes the pack
|
||||
is broken; a pack built for one emulator answers to that emulator. The
|
||||
is broken. A pack built for one emulator answers to that emulator. The
|
||||
other layer is served as well whenever the destination allows it.
|
||||
|
||||
In ``existence`` mode the frontend only looks for a path, so the emulator's
|
||||
file satisfies both sides at once and there is nothing to trade away. In a
|
||||
content-checking mode the two answers cannot share one path, and the pack's
|
||||
addressee decides; the loss is reported rather than absorbed, because the
|
||||
addressee decides. The loss is reported rather than absorbed, because the
|
||||
cause is an upstream declaration that needs fixing at its source.
|
||||
"""
|
||||
profile = _strongest(conflict.profile_claims)
|
||||
@@ -606,7 +606,7 @@ def format_decision(decision: Decision) -> str:
|
||||
if decision.serves_both:
|
||||
winner = decision.winner.local_path
|
||||
# A claim that only matched a path is answered by whatever file sits
|
||||
# there; only one that pins content can be left wanting.
|
||||
# there. Only one that pins content can be left wanting.
|
||||
pinned_elsewhere = _pinned_elsewhere(conflict.profile_claims, winner)
|
||||
served = sorted({
|
||||
c.emulator for c in conflict.profile_claims if c.emulator
|
||||
@@ -614,7 +614,7 @@ def format_decision(decision: Decision) -> str:
|
||||
others = sorted(pinned_elsewhere)
|
||||
line = (
|
||||
f"{conflict.destination}: serve {winner} "
|
||||
f"({', '.join(served) or 'profile'}); the frontend only "
|
||||
f"({', '.join(served) or 'profile'}): the frontend only "
|
||||
"checks the path, so both are satisfied"
|
||||
)
|
||||
if others:
|
||||
@@ -659,7 +659,7 @@ def find_collisions(
|
||||
One path holds one file, so whichever declaration the builder reaches
|
||||
first decides in silence. Declaring an archive several times to name its
|
||||
inner ROMs is the documented zipped_file pattern and resolves to one
|
||||
archive; two declarations landing on two files is a contradiction the
|
||||
archive. Two declarations landing on two files is a contradiction the
|
||||
upstream list carries. RetroDECK aims a PC-88 disk subsystem ROM and a
|
||||
CoCo disk ROM at one bios/disk.rom.
|
||||
"""
|
||||
|
||||
@@ -225,7 +225,7 @@ def main() -> int:
|
||||
parser.add_argument("target", type=Path, help="a pack, or a directory of packs")
|
||||
parser.add_argument(
|
||||
"--max-size", type=_size, default=ASSET_LIMIT, metavar="SIZE",
|
||||
help="a part stays under this many bytes (K, M, G suffixes; default 2G)",
|
||||
help="a part stays under this many bytes (K, M, G suffixes, default 2G)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
# The packs are rewritten in place by generate_pack under this lock: a
|
||||
|
||||
@@ -100,7 +100,7 @@ def identify_romset(
|
||||
"""Find which recipe rebuilds to a target archive MD5.
|
||||
|
||||
recipes maps a label (e.g. a MAME version) to a list of
|
||||
(member name, CRC32 hex) pairs; atoms maps CRC32 hex to ROM bytes.
|
||||
(member name, CRC32 hex) pairs, and atoms maps CRC32 hex to ROM bytes.
|
||||
Returns the matching label, or None when no recipe reproduces the hash.
|
||||
"""
|
||||
target = target_md5.lower()
|
||||
|
||||
+6
-6
@@ -51,7 +51,7 @@ _HOSTS: dict[str, tuple[str, str, str]] = {
|
||||
"https://codeberg.org/api/v1",
|
||||
"https://codeberg.org",
|
||||
),
|
||||
# The Ryujinx repository left GitHub in October 2024; this Gitea pull
|
||||
# The Ryujinx repository left GitHub in October 2024. This Gitea pull
|
||||
# mirror kept its history up to 1.1.1402.
|
||||
"git.ngram.ca": (
|
||||
"forgejo",
|
||||
@@ -192,7 +192,7 @@ def content_url(repo: Repo, sha: str, path: str) -> str:
|
||||
|
||||
Forgejo and Gitea hosts put their web routes, /raw/commit included,
|
||||
behind a proof-of-work page (git.ngram.ca answers 200 with an HTML
|
||||
challenge); the API raw route is not challenged. raw_url stays the
|
||||
challenge), while the API raw route is not challenged. raw_url stays the
|
||||
link a reader opens.
|
||||
"""
|
||||
if repo.family != "forgejo":
|
||||
@@ -236,7 +236,7 @@ def _host_is_unresolvable(exc: BaseException) -> bool:
|
||||
"""Whether a connection failure is the name itself, not the network.
|
||||
|
||||
URLError carries the cause in `reason`, and a wrapped one carries it in
|
||||
`__cause__`; the walk is bounded because either chain can be cyclic.
|
||||
`__cause__`. The walk is bounded because either chain can be cyclic.
|
||||
"""
|
||||
seen: BaseException | None = exc
|
||||
for _ in range(8):
|
||||
@@ -271,7 +271,7 @@ def _fetch(url: str, accept_json: bool = False) -> bytes | None:
|
||||
|
||||
A pass over every profile issues thousands of requests, so a dropped
|
||||
connection or a transient 5xx is a certainty rather than an accident.
|
||||
Those are retried; a definitive answer from the forge is not.
|
||||
Those are retried, a definitive answer from the forge is not.
|
||||
"""
|
||||
failure: UpstreamError | None = None
|
||||
for attempt in range(RETRIES):
|
||||
@@ -376,7 +376,7 @@ def _api(
|
||||
) -> object | None:
|
||||
"""Cached API call. The cache stores absence as well as payloads.
|
||||
|
||||
``max_age`` bounds the trust in a cached answer, in seconds; a stale
|
||||
``max_age`` bounds the trust in a cached answer, in seconds. A stale
|
||||
answer is refreshed unless the run is offline, when it still serves.
|
||||
``absent_max_age`` bounds a cached absence alone, for a resource that
|
||||
can appear but, once there, does not change.
|
||||
@@ -668,7 +668,7 @@ MAX_TREE_PAGES = 50
|
||||
def _gitlab_tree(
|
||||
repo: Repo, sha: str, cache_dir: str, offline: bool
|
||||
) -> tuple[list[str], bool]:
|
||||
"""GitLab serves its tree in pages; each page is cached on its own URL.
|
||||
"""GitLab serves its tree in pages, each page cached on its own URL.
|
||||
|
||||
A page that cannot be read mid-walk reports the listing as truncated:
|
||||
the paths already collected are real, but their absence proves nothing.
|
||||
|
||||
+16
-16
@@ -86,7 +86,7 @@ class ValidationResult:
|
||||
return all(s != "FAIL" for s, _ in self.checks)
|
||||
|
||||
def to_markdown(self) -> str:
|
||||
status = "✅" if self.passed else "❌"
|
||||
status = "PASS" if self.passed else "FAIL"
|
||||
lines = [f"### {status} `{self.filename}`"]
|
||||
lines.append("")
|
||||
lines.append(f"- **Path**: `{self.filepath}`")
|
||||
@@ -98,13 +98,13 @@ class ValidationResult:
|
||||
|
||||
for status_str, message in self.checks:
|
||||
if status_str == "PASS":
|
||||
lines.append(f"- ✅ {message}")
|
||||
lines.append(f"- Pass: {message}")
|
||||
elif status_str == "FAIL":
|
||||
lines.append(f"- ❌ {message}")
|
||||
lines.append(f"- Fail: {message}")
|
||||
elif status_str == "WARN":
|
||||
lines.append(f"- ⚠️ {message}")
|
||||
lines.append(f"- Warning: {message}")
|
||||
else:
|
||||
lines.append(f"- ℹ️ {message}")
|
||||
lines.append(f"- Info: {message}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -306,7 +306,7 @@ def validate_file(
|
||||
if expected and (len(parts) < 4 or parts[1] not in makers):
|
||||
# Depth alone passed a PS1 BIOS filed under bios/Nintendo/Game Boy/.
|
||||
result.add_warning(
|
||||
"Placed under " + (here or "the bios/ root") + "; the other files of "
|
||||
"Placed under " + (here or "the bios/ root") + ", while the other files of "
|
||||
"its system are in " + " or ".join(f"`{f}`" for f in sorted(expected))
|
||||
)
|
||||
elif len(parts) >= 4:
|
||||
@@ -316,12 +316,12 @@ def validate_file(
|
||||
elif expected_folders:
|
||||
# The contributor asks before forking: say where it goes.
|
||||
result.add_warning(
|
||||
"File is not under bios/ yet; its system's files are in "
|
||||
"File is not under bios/ yet. Its system's files are in "
|
||||
+ " or ".join(f"`{f}`" for f in expected_folders)
|
||||
)
|
||||
else:
|
||||
result.add_warning(
|
||||
"File is not under bios/ yet; place it in bios/Manufacturer/Console/"
|
||||
"File is not under bios/ yet. Place it in bios/Manufacturer/Console/"
|
||||
)
|
||||
|
||||
if name_known and not sha1_known and not md5_known:
|
||||
@@ -371,7 +371,7 @@ def get_changed_files() -> list[str]:
|
||||
changed.extend(found or [])
|
||||
if not compared:
|
||||
print(
|
||||
"no upstream/ or origin/ main line to compare against; reading the "
|
||||
"no upstream/ or origin/ main line to compare against, reading the "
|
||||
"staged, modified and untracked files only",
|
||||
file=sys.stderr,
|
||||
)
|
||||
@@ -389,7 +389,7 @@ def _files_to_validate(
|
||||
if args.changed and args.files:
|
||||
# Named files were dropped for the git diff, and the run reported
|
||||
# "No changed BIOS files detected" about files it never looked at.
|
||||
parser.error("--changed reads the git diff; name files or pass --changed, not both")
|
||||
parser.error("--changed reads the git diff: name files or pass --changed, not both")
|
||||
if not args.changed:
|
||||
if not args.files:
|
||||
parser.error("No files specified. Use --changed or provide file paths.")
|
||||
@@ -408,7 +408,7 @@ def _load_database_if_any(path: str) -> dict | None:
|
||||
"""The database, optional only where the default is meant.
|
||||
|
||||
A mistyped --db read as "no database" and every file passed its
|
||||
duplicate and placement checks unchecked; a damaged one did the same
|
||||
duplicate and placement checks unchecked. A damaged one did the same
|
||||
after a warning.
|
||||
"""
|
||||
if path == DEFAULT_DB and not os.path.exists(path):
|
||||
@@ -473,7 +473,7 @@ def main() -> None:
|
||||
print(json.dumps(output, indent=2))
|
||||
elif args.markdown:
|
||||
lines = ["## BIOS Validation Report", ""]
|
||||
status = "✅ All checks passed" if all_passed else "❌ Some checks failed"
|
||||
status = "all checks passed" if all_passed else "some checks failed"
|
||||
lines.append(f"**Status**: {status}")
|
||||
lines.append("")
|
||||
|
||||
@@ -491,13 +491,13 @@ def main() -> None:
|
||||
print(f" Size: {r.size:,}")
|
||||
for s, m in r.checks:
|
||||
marker = (
|
||||
"✓"
|
||||
"ok"
|
||||
if s == "PASS"
|
||||
else "✗"
|
||||
else "fail"
|
||||
if s == "FAIL"
|
||||
else "!"
|
||||
else "warn"
|
||||
if s == "WARN"
|
||||
else "i"
|
||||
else "info"
|
||||
)
|
||||
print(f" [{marker}] {m}")
|
||||
|
||||
|
||||
@@ -134,7 +134,7 @@ def _validate_pack_manifests(dist: Path) -> list[str]:
|
||||
return _scan_pack_manifests(dist)
|
||||
except ArtifactLockBusy:
|
||||
print(
|
||||
f"note: {dist.name} is being written; skipping pack manifests",
|
||||
f"note: {dist.name} is being written, skipping pack manifests",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return []
|
||||
@@ -340,7 +340,7 @@ def _semantic_install_checks(path: Path, manifest: dict) -> list[str]:
|
||||
def unchecked_formats() -> list[str]:
|
||||
"""Formats the schemas declare that this jsonschema install cannot check.
|
||||
|
||||
date-time and uri need the format extras; without them the ten format
|
||||
date-time and uri need the format extras. Without them the ten format
|
||||
constraints of the published schemas were accepted unread.
|
||||
"""
|
||||
used = {
|
||||
|
||||
@@ -89,7 +89,7 @@ def _entry_rule(emu_name: str, f: dict, checks: set[str]) -> dict:
|
||||
def warned_checks(f: dict, checks: set[str]) -> set[str]:
|
||||
"""The checks of an entry the code warns about instead of refusing.
|
||||
|
||||
`on_mismatch: warn` covers every check; a map names them one by one, as
|
||||
`on_mismatch: warn` covers every check, while a map names them one by one, as
|
||||
Azahar refuses a SecureInfo_A of the wrong size and only logs a bad
|
||||
signature.
|
||||
"""
|
||||
@@ -340,7 +340,7 @@ def check_file_validation(
|
||||
them must accept it through one of its own entries for the name, the
|
||||
way each runs its own check. Without, the checks of every core naming
|
||||
the file are merged. A check declared `on_mismatch: warn` rejects
|
||||
nothing, since the core runs with the file; ``strict`` counts it, to
|
||||
nothing, since the core runs with the file, while ``strict`` counts it, to
|
||||
prefer a file that passes it too.
|
||||
|
||||
Returns None if all checks pass or no validation applies.
|
||||
@@ -786,7 +786,7 @@ def validated_choice(
|
||||
"""The file a platform destination ships, and the disagreement if any.
|
||||
|
||||
A frontend's own check can pass while an emulator it runs rejects the
|
||||
file; a held variant both accept replaces it. The pack and the install
|
||||
file. A held variant both accept replaces it. The pack and the install
|
||||
manifest both read this, so they name the same file. A destination
|
||||
another emulator declares more precisely is that emulator's: ZEsarUX's
|
||||
48 KB cpc6128.rom check has no say over ep128emu/roms/cpc6128.rom.
|
||||
@@ -816,8 +816,8 @@ def validated_choice(
|
||||
|
||||
if not check(local_path, True):
|
||||
return local_path, None
|
||||
# A held file every check passes, warn-level ones included, comes first;
|
||||
# then the file itself when only a warning fails (the core runs it);
|
||||
# A held file every check passes, warn-level ones included, comes first,
|
||||
# then the file itself when only a warning fails (the core runs it),
|
||||
# then a held file the core at least runs.
|
||||
better = variant(True)
|
||||
if better:
|
||||
@@ -839,6 +839,6 @@ def existence_discrepancy(file_entry: dict, local_path: str, platform_display: s
|
||||
declared = file_entry.get("md5", "") or file_entry.get("sha1", "")
|
||||
actual = compute_hashes(local_path)
|
||||
return (
|
||||
f"packed per {platform_display} existence check; declared hash {declared}, "
|
||||
f"packed per {platform_display} existence check: declared hash {declared}, "
|
||||
f"file md5 {actual['md5']} sha1 {actual['sha1']}"
|
||||
)
|
||||
+12
-12
@@ -153,7 +153,7 @@ def verify_entry_existence(
|
||||
return {"name": name, "status": Status.MISSING, "required": required}
|
||||
result = {"name": name, "status": Status.OK, "required": required}
|
||||
if resolve_status == "hash_mismatch":
|
||||
# The frontend reads no bytes, so the file stays; the declared hash
|
||||
# The frontend reads no bytes, so the file stays, and the declared hash
|
||||
# it contradicts is reported, as the pack does.
|
||||
result["discrepancy"] = existence_discrepancy(
|
||||
file_entry, local_path, platform_display
|
||||
@@ -267,7 +267,7 @@ def verify_entry_md5(
|
||||
**base,
|
||||
"status": Status.UNTESTED,
|
||||
"path": local_path,
|
||||
"reason": f"expected {md5_list[0][:12]}… got {actual_md5[:12]}…",
|
||||
"reason": f"expected {md5_list[0][:12]}... got {actual_md5[:12]}...",
|
||||
}
|
||||
|
||||
|
||||
@@ -296,7 +296,7 @@ def verify_entry_sha1(
|
||||
**base,
|
||||
"status": Status.UNTESTED,
|
||||
"path": local_path,
|
||||
"reason": f"expected {expected_sha1[:12]}… got {actual_sha1[:12]}…",
|
||||
"reason": f"expected {expected_sha1[:12]}... got {actual_sha1[:12]}...",
|
||||
}
|
||||
|
||||
|
||||
@@ -628,7 +628,7 @@ def find_undeclared_files(
|
||||
# A copy contradicting the declared hash counts as held
|
||||
# only where the builder would ship it. Under existence
|
||||
# the frontend never opens the file, so the pack carries
|
||||
# it and reports the divergence; under a digest mode the
|
||||
# it and reports the divergence. Under a digest mode the
|
||||
# frontend would reject it, the builder leaves it out, and
|
||||
# calling it available here would describe a pack that
|
||||
# does not contain it.
|
||||
@@ -794,7 +794,7 @@ def _mark_hidden(
|
||||
|
||||
Returns the full destinations the builder fills with a core's file.
|
||||
``report`` is the gap report the caller already computed with the
|
||||
builder's arguments; without it the builder's pass computes its own.
|
||||
builder's arguments. Without it the builder's pass computes its own.
|
||||
"""
|
||||
from packextras import hidden_extras
|
||||
|
||||
@@ -868,8 +868,8 @@ def _resolve_declaration(
|
||||
def _twin_unmet(result: dict, held_by: str | None) -> bool:
|
||||
"""Mark a declaration unmet at a path another declaration holds.
|
||||
|
||||
The path is answered by the winner's file, which the pack counts once;
|
||||
this declaration's failure is reported, never folded into the
|
||||
The path is answered by the winner's file, which the pack counts once.
|
||||
This declaration's failure is reported, never folded into the
|
||||
destination's status.
|
||||
"""
|
||||
if held_by is None or result["status"] == Status.OK:
|
||||
@@ -895,7 +895,7 @@ def _resolve_for_platform(
|
||||
) -> tuple[str | None, str]:
|
||||
"""Resolve one platform entry the way the pack builder does.
|
||||
|
||||
A slot the arbitration decided wins over the resolver; a frontend that
|
||||
A slot the arbitration decided wins over the resolver, and a frontend that
|
||||
never reads the bytes takes the filename-free core's own image.
|
||||
"""
|
||||
override = slot_overrides.get(
|
||||
@@ -932,7 +932,7 @@ def verify_platform(
|
||||
mode = normalize_mode(config.get("verification_mode"))
|
||||
platform = config.get("platform", "unknown")
|
||||
|
||||
# The builder settles a destination claimed by both layers; this must read
|
||||
# The builder settles a destination claimed by both layers, and this must read
|
||||
# the same decision, or the two tools describe different packs.
|
||||
has_zipped = any(
|
||||
fe.get("zipped_file")
|
||||
@@ -1588,7 +1588,7 @@ def verify_emulator(
|
||||
dest_to_name: dict[str, str] = {}
|
||||
data_dir_notices: list[str] = []
|
||||
|
||||
# The emulator pack withdraws these; the report withdraws the same.
|
||||
# The emulator pack withdraws these, and the report withdraws the same.
|
||||
region_drops: set[str] = set()
|
||||
if regions:
|
||||
from packextras import emulator_region_drops
|
||||
@@ -2114,7 +2114,7 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
|
||||
def _refuse_listing_narrowings(
|
||||
args: argparse.Namespace, parser: argparse.ArgumentParser
|
||||
) -> None:
|
||||
"""A listing mode reads --platform at most; anything else is refused.
|
||||
"""A listing mode reads --platform at most, anything else is refused.
|
||||
|
||||
Printing past a narrowing flag lets the user believe it applied
|
||||
(generate_pack applies the same table).
|
||||
@@ -2378,7 +2378,7 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
def _json_details(details: list[dict], verbose: bool) -> list[dict]:
|
||||
"""The entries a JSON report lists: what needs attention, or every one.
|
||||
|
||||
--verbose shows each entry with its ground truth in the text report; it
|
||||
--verbose shows each entry with its ground truth in the text report. It
|
||||
used to be accepted beside --json and dropped, the output byte for byte
|
||||
the same, so a script asking for the passing entries got nothing.
|
||||
"""
|
||||
|
||||
+5
-5
@@ -223,7 +223,7 @@ MAX_ZIP_MEMBERS = 100_000
|
||||
MAX_ZIP_MEMBER_SIZE = 8 * 1024 * 1024 * 1024
|
||||
|
||||
# The largest generated pack is already ~5 GB uncompressed and the collection
|
||||
# only grows; this bounds a malicious archive without capping a real one.
|
||||
# only grows. This bounds a malicious archive without capping a real one.
|
||||
MAX_ZIP_TOTAL_SIZE = 64 * 1024 * 1024 * 1024
|
||||
|
||||
# DEFLATE cannot exceed roughly 1,032:1, so this rejects a declared ratio no
|
||||
@@ -249,14 +249,14 @@ def safe_extract_zip(
|
||||
"""
|
||||
dest = os.path.realpath(dest_dir)
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
# Reading the umask sets it; it is put back at once.
|
||||
# Reading the umask sets it, so it is put back at once.
|
||||
umask = os.umask(0)
|
||||
os.umask(umask)
|
||||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||||
members = zf.infolist()
|
||||
if len(members) > max_members:
|
||||
raise ValueError(
|
||||
f"ZIP has {len(members)} members; limit is {max_members}"
|
||||
f"ZIP has {len(members)} members, limit is {max_members}"
|
||||
)
|
||||
|
||||
declared_total = 0
|
||||
@@ -292,7 +292,7 @@ def safe_extract_zip(
|
||||
declared_total += member.file_size
|
||||
if declared_total > max_total_size:
|
||||
raise ValueError(
|
||||
f"ZIP expands to {declared_total} bytes; limit is {max_total_size}"
|
||||
f"ZIP expands to {declared_total} bytes, limit is {max_total_size}"
|
||||
)
|
||||
if member.file_size and member.compress_type in _BOUNDED_RATIO_METHODS:
|
||||
if member.compress_size == 0:
|
||||
@@ -331,7 +331,7 @@ def safe_extract_zip(
|
||||
f"ZIP member size mismatch for {name}: "
|
||||
f"{actual_size} != {member.file_size}"
|
||||
)
|
||||
# The member's own mode, else what a plain open() gives; the
|
||||
# The member's own mode, else what a plain open() gives. The
|
||||
# temporary file is private and would keep 0600 otherwise.
|
||||
os.chmod(tmp_path, ((mode & 0o777) or 0o666) & ~umask)
|
||||
os.replace(tmp_path, target)
|
||||
|
||||
Reference in new issue
Block a user