mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-10 13:33:24 -05:00
refactor: split this round's fixes into small helpers
This commit is contained in:
1 parent
70d6f431e3
commit
f3d01aeeeb
15 files changed
+218
-157
No files matched your search
+31
-22
@@ -102,20 +102,25 @@ def _build_supplemental_index(
|
|||||||
if dpath.is_dir() and not dpath.name.startswith("."):
|
if dpath.is_dir() and not dpath.name.startswith("."):
|
||||||
names.add(dpath.name + "/")
|
names.add(dpath.name + "/")
|
||||||
names.add(dpath.name.lower() + "/")
|
names.add(dpath.name.lower() + "/")
|
||||||
import zipfile
|
names |= _zip_member_names(bios_path)
|
||||||
|
return names
|
||||||
|
|
||||||
for zpath in bios_path.rglob("*.zip"):
|
|
||||||
try:
|
def _zip_member_names(bios_path: Path) -> set[str]:
|
||||||
with zipfile.ZipFile(zpath) as zf:
|
"""Basenames of the files inside every ZIP under bios/, both cases."""
|
||||||
for member in zf.namelist():
|
import zipfile
|
||||||
if not member.endswith("/"):
|
|
||||||
basename = (
|
names: set[str] = set()
|
||||||
member.rsplit("/", 1)[-1] if "/" in member else member
|
for zpath in bios_path.rglob("*.zip"):
|
||||||
)
|
try:
|
||||||
names.add(basename)
|
with zipfile.ZipFile(zpath) as zf:
|
||||||
names.add(basename.lower())
|
members = [m for m in zf.namelist() if not m.endswith("/")]
|
||||||
except (zipfile.BadZipFile, OSError):
|
except (zipfile.BadZipFile, OSError) as exc:
|
||||||
pass
|
print(f" WARNING: unreadable archive {zpath}: {exc}", file=sys.stderr)
|
||||||
|
continue
|
||||||
|
for member in members:
|
||||||
|
basename = member.rsplit("/", 1)[-1]
|
||||||
|
names.update((basename, basename.lower()))
|
||||||
return names
|
return names
|
||||||
|
|
||||||
|
|
||||||
@@ -177,18 +182,22 @@ def _resolve_source(
|
|||||||
by_name_lower[canonical.lower()]
|
by_name_lower[canonical.lower()]
|
||||||
):
|
):
|
||||||
return "bios"
|
return "bios"
|
||||||
# data/ supplemental index: a directory entry looks for a directory, a
|
if data_names and _data_hit(fname, basename, file_entry, data_names):
|
||||||
# file entry for a file.
|
return "data"
|
||||||
if data_names:
|
|
||||||
is_directory = fname.endswith("/") or (file_entry or {}).get("type") == "directory"
|
|
||||||
looked_up = [fname, key] + ([basename, basename.lower()] if basename else [])
|
|
||||||
if is_directory:
|
|
||||||
looked_up = [name.rstrip("/") + "/" for name in looked_up]
|
|
||||||
if any(name in data_names for name in looked_up):
|
|
||||||
return "data"
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _data_hit(
|
||||||
|
fname: str, basename: str | None, file_entry: dict | None, data_names: set[str]
|
||||||
|
) -> bool:
|
||||||
|
"""Whether data/ or a ZIP holds the name: a directory entry looks for a
|
||||||
|
directory, a file entry for a file."""
|
||||||
|
looked_up = [fname, fname.lower()] + ([basename, basename.lower()] if basename else [])
|
||||||
|
if fname.endswith("/") or (file_entry or {}).get("type") == "directory":
|
||||||
|
looked_up = [name.rstrip("/") + "/" for name in looked_up]
|
||||||
|
return any(name in data_names for name in looked_up)
|
||||||
|
|
||||||
|
|
||||||
def entry_source(f: dict, index: dict) -> str | None:
|
def entry_source(f: dict, index: dict) -> str | None:
|
||||||
"""Where the collection holds a profile entry, or None.
|
"""Where the collection holds a profile entry, or None.
|
||||||
|
|
||||||
|
|||||||
@@ -38,6 +38,10 @@ def _is_placeholder(name: str) -> bool:
|
|||||||
return "<" in name or ">" in name or "*" in name
|
return "<" in name or ">" in name or "*" in name
|
||||||
|
|
||||||
|
|
||||||
|
def _field_sets(entry: dict | None) -> dict[str, set[str]]:
|
||||||
|
return {f: set(_hash_values(entry or {}, f)) for f in HASH_FIELDS}
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class NativeFile:
|
class NativeFile:
|
||||||
"""One file as the platform will read it, after correction."""
|
"""One file as the platform will read it, after correction."""
|
||||||
@@ -67,8 +71,7 @@ class NativeFile:
|
|||||||
truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and
|
truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and
|
||||||
reading it as a contradiction emptied the md5 the frontend checks.
|
reading it as a contradiction emptied the md5 the frontend checks.
|
||||||
"""
|
"""
|
||||||
truth = {f: set(_hash_values(self.truth or {}, f)) for f in HASH_FIELDS}
|
truth, plat = _field_sets(self.truth), _field_sets(self.platform)
|
||||||
plat = {f: set(_hash_values(self.platform or {}, f)) for f in HASH_FIELDS}
|
|
||||||
if not any(truth.values()):
|
if not any(truth.values()):
|
||||||
return "platform"
|
return "platform"
|
||||||
if not any(plat.values()):
|
if not any(plat.values()):
|
||||||
@@ -76,10 +79,12 @@ class NativeFile:
|
|||||||
shared = [f for f in HASH_FIELDS if truth[f] and plat[f]]
|
shared = [f for f in HASH_FIELDS if truth[f] and plat[f]]
|
||||||
if any(truth[f] & plat[f] for f in shared):
|
if any(truth[f] & plat[f] for f in shared):
|
||||||
return "both"
|
return "both"
|
||||||
|
return "truth" if shared or self._sizes_differ() else "both"
|
||||||
|
|
||||||
|
def _sizes_differ(self) -> bool:
|
||||||
t_size = (self.truth or {}).get("size")
|
t_size = (self.truth or {}).get("size")
|
||||||
p_size = (self.platform or {}).get("size")
|
p_size = (self.platform or {}).get("size")
|
||||||
sizes_differ = isinstance(t_size, int) and isinstance(p_size, int) and t_size != p_size
|
return isinstance(t_size, int) and isinstance(p_size, int) and t_size != p_size
|
||||||
return "truth" if shared or sizes_differ else "both"
|
|
||||||
|
|
||||||
def hashes(self, field_name: str) -> list[str]:
|
def hashes(self, field_name: str) -> list[str]:
|
||||||
"""Accepted values for a hash, truth first when it has an opinion.
|
"""Accepted values for a hash, truth first when it has an opinion.
|
||||||
|
|||||||
@@ -95,6 +95,14 @@ class Exporter(BaseExporter):
|
|||||||
return False
|
return False
|
||||||
return super().states(fe, field_name)
|
return super().states(fe, field_name)
|
||||||
|
|
||||||
|
def _expected_md5s(
|
||||||
|
self, systems: dict[str, NativeSystem], name: str, system_id: str
|
||||||
|
) -> list[str]:
|
||||||
|
"""What a check must now hold; a refused rewrite keeps EmuDeck's own array."""
|
||||||
|
if name in getattr(self, "_withdrawn", {}):
|
||||||
|
return []
|
||||||
|
return self._md5s(systems, system_id)
|
||||||
|
|
||||||
def _refused_systems(self) -> set[str]:
|
def _refused_systems(self) -> set[str]:
|
||||||
withdrawn = getattr(self, "_withdrawn", {})
|
withdrawn = getattr(self, "_withdrawn", {})
|
||||||
return {FUNCTION_HASH_MAP[name] for name in withdrawn if name in FUNCTION_HASH_MAP}
|
return {FUNCTION_HASH_MAP[name] for name in withdrawn if name in FUNCTION_HASH_MAP}
|
||||||
@@ -200,9 +208,8 @@ class Exporter(BaseExporter):
|
|||||||
if name not in defined:
|
if name not in defined:
|
||||||
issues.append(f"check absent from the output: {name}")
|
issues.append(f"check absent from the output: {name}")
|
||||||
continue
|
continue
|
||||||
md5s = self._md5s(systems, system_id)
|
md5s = self._expected_md5s(systems, name, system_id)
|
||||||
if not md5s or name in getattr(self, "_withdrawn", {}):
|
if not md5s:
|
||||||
# A refused rewrite keeps EmuDeck's own array, by design.
|
|
||||||
continue
|
continue
|
||||||
body = next(
|
body = next(
|
||||||
content[start:end]
|
content[start:end]
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ class Exporter(BaseExporter):
|
|||||||
"""
|
"""
|
||||||
return native_id in SLUG_MAP
|
return native_id in SLUG_MAP
|
||||||
|
|
||||||
def writable(self, fe: NativeFile, require: str = "") -> bool:
|
def writable(self, fe: NativeFile, require: str = "") -> bool: # noqa: ARG002
|
||||||
"""What RomM already ships stays; the conditions gate additions.
|
"""What RomM already ships stays; the conditions gate additions.
|
||||||
|
|
||||||
An entry of theirs that could never verify is still theirs, and the
|
An entry of theirs that could never verify is still theirs, and the
|
||||||
|
|||||||
+19
-12
@@ -2456,6 +2456,24 @@ def _refuse_for_all_variants(args, parser) -> None:
|
|||||||
parser.error(f"{flag} is incompatible with --verify-packs")
|
parser.error(f"{flag} is incompatible with --verify-packs")
|
||||||
|
|
||||||
|
|
||||||
|
def _refuse_refresh_data(args, parser) -> None:
|
||||||
|
"""Only the platform pack path refreshes the data directories.
|
||||||
|
|
||||||
|
Elsewhere the cache on disk was packed as it stood, and --offline wins
|
||||||
|
over the refresh without a word.
|
||||||
|
"""
|
||||||
|
if not args.refresh_data:
|
||||||
|
return
|
||||||
|
for flag, on in (
|
||||||
|
("--offline", args.offline),
|
||||||
|
("--emulator", args.emulator),
|
||||||
|
("--system", args.system and not (args.platform or args.all)),
|
||||||
|
("--from-md5", args.from_md5 or args.from_md5_file),
|
||||||
|
):
|
||||||
|
if on:
|
||||||
|
parser.error(f"--refresh-data is incompatible with {flag}")
|
||||||
|
|
||||||
|
|
||||||
def _refuse_unapplied_flags(args, parser) -> None:
|
def _refuse_unapplied_flags(args, parser) -> None:
|
||||||
"""Refuse every flag the requested mode would not apply.
|
"""Refuse every flag the requested mode would not apply.
|
||||||
|
|
||||||
@@ -2475,18 +2493,7 @@ def _refuse_unapplied_flags(args, parser) -> None:
|
|||||||
parser.error("--region is incompatible with --manifest-targets")
|
parser.error("--region is incompatible with --manifest-targets")
|
||||||
if args.one_per_slot and args.manifest_targets:
|
if args.one_per_slot and args.manifest_targets:
|
||||||
parser.error("--one-per-slot is incompatible with --manifest-targets")
|
parser.error("--one-per-slot is incompatible with --manifest-targets")
|
||||||
# Only the platform pack path refreshes the data directories; elsewhere
|
_refuse_refresh_data(args, parser)
|
||||||
# the cache on disk was packed as it stood.
|
|
||||||
if args.refresh_data:
|
|
||||||
if args.offline:
|
|
||||||
parser.error("--refresh-data is incompatible with --offline")
|
|
||||||
for flag, on in (
|
|
||||||
("--emulator", args.emulator),
|
|
||||||
("--system", args.system and not (args.platform or args.all)),
|
|
||||||
("--from-md5", args.from_md5 or args.from_md5_file),
|
|
||||||
):
|
|
||||||
if on:
|
|
||||||
parser.error(f"--refresh-data is incompatible with {flag}")
|
|
||||||
|
|
||||||
_refuse_for_all_variants(args, parser)
|
_refuse_for_all_variants(args, parser)
|
||||||
mode = next(
|
mode = next(
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ from common import (
|
|||||||
load_database,
|
load_database,
|
||||||
load_emulator_profiles,
|
load_emulator_profiles,
|
||||||
load_provenance_snapshots,
|
load_provenance_snapshots,
|
||||||
parse_md5_list,
|
|
||||||
require_yaml,
|
require_yaml,
|
||||||
resolve_platform_cores,
|
resolve_platform_cores,
|
||||||
unique_emulator_profiles,
|
unique_emulator_profiles,
|
||||||
@@ -52,7 +51,7 @@ from nativemode import reads_file_contents
|
|||||||
|
|
||||||
|
|
||||||
yaml = require_yaml()
|
yaml = require_yaml()
|
||||||
from generate_readme import compute_coverage, release_totals
|
from generate_readme import compute_coverage, contributing_text, release_totals
|
||||||
from profile_sync import source_ref_values, split_source_ref
|
from profile_sync import source_ref_values, split_source_ref
|
||||||
from provenance_report import build_report
|
from provenance_report import build_report
|
||||||
import release_record
|
import release_record
|
||||||
@@ -3108,8 +3107,6 @@ def generate_cross_reference(
|
|||||||
|
|
||||||
|
|
||||||
def generate_contributing() -> str:
|
def generate_contributing() -> str:
|
||||||
from generate_readme import contributing_text
|
|
||||||
|
|
||||||
return contributing_text(
|
return contributing_text(
|
||||||
lambda page: f"{page}.md", "# Contributing - RetroBIOS", "on this site"
|
lambda page: f"{page}.md", "# Contributing - RetroBIOS", "on this site"
|
||||||
)
|
)
|
||||||
|
|||||||
+76
-48
@@ -2676,17 +2676,60 @@ def _writing_pairs(
|
|||||||
written = _writing_pins(revisions, intro_sha, f"{pin_field}_commit")
|
written = _writing_pins(revisions, intro_sha, f"{pin_field}_commit")
|
||||||
if len(written) > 1:
|
if len(written) > 1:
|
||||||
ambiguous.append(f"{pin_field}_commit {' -> '.join(written)}")
|
ambiguous.append(f"{pin_field}_commit {' -> '.join(written)}")
|
||||||
moved = [(repo, pin, current) for pin in written if pin != current]
|
# The current pin among the candidates reads as "no move" (None) for
|
||||||
# The current pin among the candidates reads as "no move" for that
|
# that repository, which the other candidates must then agree with.
|
||||||
# repository, which the other candidates must then agree with.
|
options.append([
|
||||||
options.append(moved + ([None] if current in written else []))
|
(repo, pin, current) if pin != current else None for pin in written
|
||||||
|
])
|
||||||
readings = [
|
readings = [
|
||||||
[pair for pair in combo if pair is not None]
|
[pair for pair in combo if pair is not None]
|
||||||
for combo in itertools.product(*options)
|
for combo in itertools.product(*options)
|
||||||
] if options else [[]]
|
]
|
||||||
return readings, ambiguous
|
return readings, ambiguous
|
||||||
|
|
||||||
|
|
||||||
|
def _realign_citation(
|
||||||
|
citation: Citation,
|
||||||
|
readings: list[list],
|
||||||
|
ambiguous: list[str],
|
||||||
|
cache_dir: str,
|
||||||
|
offline: bool,
|
||||||
|
) -> tuple[dict[int, tuple[int, int, str | None]], list[str]]:
|
||||||
|
"""The moves one prose run needs, and what blocks them.
|
||||||
|
|
||||||
|
Several pins under one text: the history cannot say which the text
|
||||||
|
describes, but when the cited range lands in the same place from every
|
||||||
|
one of them the question does not matter. When the readings disagree,
|
||||||
|
moving from the wrong one rewrites a correct citation onto someone
|
||||||
|
else's code, with nothing to show it happened.
|
||||||
|
"""
|
||||||
|
moves: dict[int, tuple[int, int, str | None]] = {}
|
||||||
|
blocked: list[str] = []
|
||||||
|
for index, part in enumerate(citation.parts):
|
||||||
|
outcomes = {
|
||||||
|
repr(result): result
|
||||||
|
for result in (
|
||||||
|
_realign_part(part, pairs, cache_dir, offline) if pairs else None
|
||||||
|
for pairs in readings
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if len(outcomes) > 1:
|
||||||
|
blocked.extend(
|
||||||
|
f"pin moved under this text: {item}; the readings disagree"
|
||||||
|
for item in ambiguous
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
outcome = next(iter(outcomes.values()))
|
||||||
|
if outcome is None:
|
||||||
|
continue
|
||||||
|
state, payload = outcome
|
||||||
|
if state == "ok":
|
||||||
|
moves[index] = (payload[0], payload[1], None)
|
||||||
|
else:
|
||||||
|
blocked.append(payload)
|
||||||
|
return moves, blocked
|
||||||
|
|
||||||
|
|
||||||
def realign_prose(
|
def realign_prose(
|
||||||
path: Path, cache_dir: str, offline: bool = False, dry_run: bool = False
|
path: Path, cache_dir: str, offline: bool = False, dry_run: bool = False
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
@@ -2748,34 +2791,7 @@ def realign_prose(
|
|||||||
readings, ambiguous = _writing_pairs(document, repos, revisions, intro_sha)
|
readings, ambiguous = _writing_pairs(document, repos, revisions, intro_sha)
|
||||||
if not any(readings):
|
if not any(readings):
|
||||||
continue
|
continue
|
||||||
moves: dict[int, tuple[int, int, str | None]] = {}
|
moves, blocked = _realign_citation(citation, readings, ambiguous, cache_dir, offline)
|
||||||
blocked: list[str] = []
|
|
||||||
for index, part in enumerate(citation.parts):
|
|
||||||
# Several pins under one text: the history cannot say which the
|
|
||||||
# text describes, but when the cited range lands in the same
|
|
||||||
# place from every one of them the question does not matter.
|
|
||||||
results = [
|
|
||||||
_realign_part(part, pairs, cache_dir, offline) if pairs else None
|
|
||||||
for pairs in readings
|
|
||||||
]
|
|
||||||
outcomes = {repr(result): result for result in results}
|
|
||||||
if len(outcomes) > 1:
|
|
||||||
# Moving from the wrong one rewrites a correct citation onto
|
|
||||||
# someone else's code, with nothing to show it happened.
|
|
||||||
blocked.extend(
|
|
||||||
f"pin moved under this text: {item}; the readings disagree"
|
|
||||||
for item in ambiguous
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
outcome = next(iter(outcomes.values()))
|
|
||||||
if outcome is None:
|
|
||||||
continue
|
|
||||||
state, payload = outcome
|
|
||||||
if state == "ok":
|
|
||||||
start, end = payload
|
|
||||||
moves[index] = (start, end, None)
|
|
||||||
else:
|
|
||||||
blocked.append(payload)
|
|
||||||
if blocked:
|
if blocked:
|
||||||
# Half a run must not move: the untouched ranges would read as
|
# Half a run must not move: the untouched ranges would read as
|
||||||
# already realigned when they were never even located.
|
# already realigned when they were never even located.
|
||||||
@@ -2911,21 +2927,13 @@ def _check_quota(count: int, offline: bool) -> None:
|
|||||||
raise SystemExit(1)
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
|
||||||
def _apply_writes(args, name: str, profile: dict, report: ProfileReport) -> None:
|
def _stale_prose_blocks(name: str, path: Path, args) -> bool:
|
||||||
path = Path(args.emulators_dir) / f"{name}.yml"
|
"""Whether prose written under an older pin has to move first.
|
||||||
if not path.is_file():
|
|
||||||
return
|
Anchored from the current pin, a recale or a bump follows someone else's
|
||||||
if args.backfill_commits and report.pin and not profile.get("source_commit"):
|
code (boom3 FileSystem.cpp:2125, written at b810234e, an unrelated
|
||||||
if args.dry_run:
|
OpenFileRead at 132dfddb). It moves first, from the pin it was written at.
|
||||||
print(f"{name}: would write source_commit {report.pin[:7]}")
|
"""
|
||||||
elif backfill_commit(path, report.pin):
|
|
||||||
print(f"{name}: source_commit {report.pin[:7]}")
|
|
||||||
if not (args.rebase_refs or args.bump_commit):
|
|
||||||
return
|
|
||||||
# Prose written under an older pin describes that revision: anchored from
|
|
||||||
# the current pin, a recale or a bump follows someone else's code
|
|
||||||
# (boom3 FileSystem.cpp:2125, written at b810234e, an unrelated
|
|
||||||
# OpenFileRead at 132dfddb). It moves first, from the pin it was written at.
|
|
||||||
stale = realign_prose(path, args.cache_dir, args.offline, dry_run=True)
|
stale = realign_prose(path, args.cache_dir, args.offline, dry_run=True)
|
||||||
if stale:
|
if stale:
|
||||||
print(
|
print(
|
||||||
@@ -2933,6 +2941,26 @@ def _apply_writes(args, name: str, profile: dict, report: ProfileReport) -> None
|
|||||||
"run --realign-prose or read them before the pin moves",
|
"run --realign-prose or read them before the pin moves",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
|
return bool(stale)
|
||||||
|
|
||||||
|
|
||||||
|
def _backfill(args, name: str, profile: dict, report: ProfileReport, path: Path) -> None:
|
||||||
|
"""Write the pin a profile without source_commit was written at."""
|
||||||
|
if not report.pin or profile.get("source_commit"):
|
||||||
|
return
|
||||||
|
if args.dry_run:
|
||||||
|
print(f"{name}: would write source_commit {report.pin[:7]}")
|
||||||
|
elif backfill_commit(path, report.pin):
|
||||||
|
print(f"{name}: source_commit {report.pin[:7]}")
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_writes(args, name: str, profile: dict, report: ProfileReport) -> None:
|
||||||
|
path = Path(args.emulators_dir) / f"{name}.yml"
|
||||||
|
if not path.is_file():
|
||||||
|
return
|
||||||
|
if args.backfill_commits:
|
||||||
|
_backfill(args, name, profile, report, path)
|
||||||
|
if not (args.rebase_refs or args.bump_commit) or _stale_prose_blocks(name, path, args):
|
||||||
return
|
return
|
||||||
with tempfile.TemporaryDirectory() as scratch:
|
with tempfile.TemporaryDirectory() as scratch:
|
||||||
# Always work on a copy. A dry run reports what it left there; a real
|
# Always work on a copy. A dry run reports what it left there; a real
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -101,27 +102,7 @@ def _extract_version() -> tuple[str, str]:
|
|||||||
version = _version_from_resource_h()
|
version = _version_from_resource_h()
|
||||||
# Last resort: use GitHub API for latest real release tag
|
# Last resort: use GitHub API for latest real release tag
|
||||||
if version == "unknown":
|
if version == "unknown":
|
||||||
try:
|
version = _version_from_api()
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
req = urllib.request.Request(
|
|
||||||
"https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10",
|
|
||||||
headers={"User-Agent": "retrobios-scraper/1.0"},
|
|
||||||
)
|
|
||||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
||||||
import json as json_mod
|
|
||||||
|
|
||||||
tags = json_mod.loads(resp.read())
|
|
||||||
for t in tags:
|
|
||||||
if t["name"] != "latest" and t["name"].startswith("v"):
|
|
||||||
version = t["name"]
|
|
||||||
break
|
|
||||||
except (urllib.error.URLError, OSError) as exc:
|
|
||||||
raise RuntimeError(f"cannot determine the FBNeo version: {exc}") from exc
|
|
||||||
if version == "unknown":
|
|
||||||
# Written as core_version otherwise, and cached for a day.
|
|
||||||
raise RuntimeError("cannot determine the FBNeo version: no tag, resource.h or release")
|
|
||||||
|
|
||||||
sha_result = subprocess.run(
|
sha_result = subprocess.run(
|
||||||
["git", "rev-parse", "HEAD"],
|
["git", "rev-parse", "HEAD"],
|
||||||
@@ -135,6 +116,31 @@ def _extract_version() -> tuple[str, str]:
|
|||||||
return version, commit
|
return version, commit
|
||||||
|
|
||||||
|
|
||||||
|
def _version_from_api() -> str:
|
||||||
|
"""The latest real release tag, or an error: "unknown" was written as
|
||||||
|
core_version and cached for a day."""
|
||||||
|
import json as json_mod
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
headers = {"User-Agent": "retrobios-scraper/1.0"}
|
||||||
|
token = os.environ.get("GITHUB_TOKEN")
|
||||||
|
if token:
|
||||||
|
headers["Authorization"] = f"Bearer {token}"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
"https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10", headers=headers
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||||
|
tags = json_mod.loads(resp.read())
|
||||||
|
except (urllib.error.URLError, OSError, ValueError) as exc:
|
||||||
|
raise RuntimeError(f"cannot determine the FBNeo version: {exc}") from exc
|
||||||
|
for tag in tags:
|
||||||
|
if tag["name"] != "latest" and tag["name"].startswith("v"):
|
||||||
|
return tag["name"]
|
||||||
|
raise RuntimeError("cannot determine the FBNeo version: no tag, resource.h or release")
|
||||||
|
|
||||||
|
|
||||||
def _version_from_resource_h() -> str:
|
def _version_from_resource_h() -> str:
|
||||||
"""Fallback: parse VER_FULL_VERSION_STR from resource.h."""
|
"""Fallback: parse VER_FULL_VERSION_STR from resource.h."""
|
||||||
resource_h = CLONE_DIR / "src" / "burner" / "resource.h"
|
resource_h = CLONE_DIR / "src" / "burner" / "resource.h"
|
||||||
|
|||||||
@@ -268,6 +268,14 @@ def unchecked_formats() -> list[str]:
|
|||||||
return sorted(used - set(FormatChecker().checkers))
|
return sorted(used - set(FormatChecker().checkers))
|
||||||
|
|
||||||
|
|
||||||
|
def _format_errors() -> list[str]:
|
||||||
|
"""jsonschema skips a format it has no checker for: say so, and fail."""
|
||||||
|
return [
|
||||||
|
f"no checker for format {name}: install \"jsonschema[format-nongpl]\""
|
||||||
|
for name in unchecked_formats()
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@@ -277,17 +285,7 @@ def main() -> int:
|
|||||||
)
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
unchecked = unchecked_formats()
|
errors: list[str] = _format_errors()
|
||||||
if unchecked:
|
|
||||||
# jsonschema skips a format it has no checker for, silently.
|
|
||||||
print(
|
|
||||||
f"ERROR no checker for format(s) {', '.join(unchecked)}: "
|
|
||||||
'install "jsonschema[format-nongpl]"',
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
errors: list[str] = []
|
|
||||||
errors.extend(
|
errors.extend(
|
||||||
_validate_yaml_directory(ROOT / "emulators", "emulator.schema.json")
|
_validate_yaml_directory(ROOT / "emulators", "emulator.schema.json")
|
||||||
)
|
)
|
||||||
|
|||||||
+22
-13
@@ -82,6 +82,9 @@ from validation import (
|
|||||||
|
|
||||||
DEFAULT_DB = "database.json"
|
DEFAULT_DB = "database.json"
|
||||||
DEFAULT_PLATFORMS_DIR = "platforms"
|
DEFAULT_PLATFORMS_DIR = "platforms"
|
||||||
|
# The repository's platforms wherever the script runs from, for library calls
|
||||||
|
# that pass no directory.
|
||||||
|
_REPO_PLATFORMS = os.path.join(os.path.dirname(__file__), "..", "platforms")
|
||||||
DEFAULT_EMULATORS_DIR = "emulators"
|
DEFAULT_EMULATORS_DIR = "emulators"
|
||||||
|
|
||||||
|
|
||||||
@@ -1258,7 +1261,7 @@ def verify_emulator(
|
|||||||
db: dict,
|
db: dict,
|
||||||
standalone: bool = False,
|
standalone: bool = False,
|
||||||
regions: list[str] | None = None,
|
regions: list[str] | None = None,
|
||||||
platforms_dir: str | None = None,
|
platforms_dir: str = _REPO_PLATFORMS,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Verify files for specific emulator profiles.
|
"""Verify files for specific emulator profiles.
|
||||||
|
|
||||||
@@ -1276,9 +1279,7 @@ def verify_emulator(
|
|||||||
# Build validation index from selected profiles only
|
# Build validation index from selected profiles only
|
||||||
selected_profiles = {n: p for n, p in selected}
|
selected_profiles = {n: p for n, p in selected}
|
||||||
validation_index = _build_validation_index(selected_profiles)
|
validation_index = _build_validation_index(selected_profiles)
|
||||||
data_registry = load_data_dir_registry(
|
data_registry = load_data_dir_registry(platforms_dir)
|
||||||
platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms")
|
|
||||||
)
|
|
||||||
|
|
||||||
details = []
|
details = []
|
||||||
file_status: dict[str, str] = {}
|
file_status: dict[str, str] = {}
|
||||||
@@ -1556,7 +1557,7 @@ def verify_system(
|
|||||||
db: dict,
|
db: dict,
|
||||||
standalone: bool = False,
|
standalone: bool = False,
|
||||||
regions: list[str] | None = None,
|
regions: list[str] | None = None,
|
||||||
platforms_dir: str | None = None,
|
platforms_dir: str = _REPO_PLATFORMS,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Verify files for all emulators supporting given system IDs."""
|
"""Verify files for all emulators supporting given system IDs."""
|
||||||
profiles = load_emulator_profiles(emulators_dir)
|
profiles = load_emulator_profiles(emulators_dir)
|
||||||
@@ -1729,6 +1730,21 @@ def _refuse_listing_narrowings(
|
|||||||
parser.error(f"{flag} is incompatible with {listing}")
|
parser.error(f"{flag} is incompatible with {listing}")
|
||||||
|
|
||||||
|
|
||||||
|
def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
||||||
|
"""A flag the chosen mode does not read is refused, never ignored."""
|
||||||
|
for refused, message in (
|
||||||
|
(args.standalone and not (args.emulator or args.system),
|
||||||
|
"--standalone requires --emulator or --system"),
|
||||||
|
(args.include_archived and not args.all, "--include-archived requires --all"),
|
||||||
|
(args.target and not (args.platform or args.all),
|
||||||
|
"--target requires --platform or --all"),
|
||||||
|
(args.target and (args.emulator or args.system),
|
||||||
|
"--target is incompatible with --emulator and --system"),
|
||||||
|
):
|
||||||
|
if refused:
|
||||||
|
parser.error(message)
|
||||||
|
|
||||||
|
|
||||||
def _run_listing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
def _run_listing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
||||||
if args.list_emulators:
|
if args.list_emulators:
|
||||||
list_emulator_profiles(args.emulators_dir)
|
list_emulator_profiles(args.emulators_dir)
|
||||||
@@ -1811,14 +1827,7 @@ def main():
|
|||||||
parser.error(
|
parser.error(
|
||||||
"--platform, --all, --emulator, and --system are mutually exclusive"
|
"--platform, --all, --emulator, and --system are mutually exclusive"
|
||||||
)
|
)
|
||||||
if args.standalone and not (args.emulator or args.system):
|
_refuse_mode_flags(args, parser)
|
||||||
parser.error("--standalone requires --emulator or --system")
|
|
||||||
if args.include_archived and not args.all:
|
|
||||||
parser.error("--include-archived requires --all")
|
|
||||||
if args.target and not (args.platform or args.all):
|
|
||||||
parser.error("--target requires --platform or --all")
|
|
||||||
if args.target and (args.emulator or args.system):
|
|
||||||
parser.error("--target is incompatible with --emulator and --system")
|
|
||||||
|
|
||||||
with open(args.db) as f:
|
with open(args.db) as f:
|
||||||
db = json.load(f)
|
db = json.load(f)
|
||||||
|
|||||||
@@ -576,7 +576,7 @@ class SchemaFormatsAreChecked(unittest.TestCase):
|
|||||||
|
|
||||||
@unittest.skipUnless(HAS_JSONSCHEMA, "validating a schema needs jsonschema")
|
@unittest.skipUnless(HAS_JSONSCHEMA, "validating a schema needs jsonschema")
|
||||||
def test_a_missing_checker_is_named(self):
|
def test_a_missing_checker_is_named(self):
|
||||||
import validate_schemas
|
import validate_schemas # noqa: PLC0415
|
||||||
|
|
||||||
empty = mock.Mock(checkers={})
|
empty = mock.Mock(checkers={})
|
||||||
with mock.patch.object(validate_schemas, "FormatChecker", return_value=empty):
|
with mock.patch.object(validate_schemas, "FormatChecker", return_value=empty):
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import re
|
|||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||||
@@ -37,11 +38,9 @@ class CoverageReadsTheGivenProfiles(unittest.TestCase):
|
|||||||
platform coverage, which read ./emulators whatever was given."""
|
platform coverage, which read ./emulators whatever was given."""
|
||||||
|
|
||||||
def test_the_directory_reaches_verify(self):
|
def test_the_directory_reaches_verify(self):
|
||||||
from unittest import mock # noqa: PLC0415
|
|
||||||
|
|
||||||
seen: list[str] = []
|
seen: list[str] = []
|
||||||
|
|
||||||
def fake_verify(config, db, emulators_dir, **_kwargs):
|
def fake_verify(_config, _db, emulators_dir, **_kwargs):
|
||||||
seen.append(emulators_dir)
|
seen.append(emulators_dir)
|
||||||
return {"status_counts": {}, "total_files": 0, "undeclared_files": [], "details": []}
|
return {"status_counts": {}, "total_files": 0, "undeclared_files": [], "details": []}
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,10 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
|
|||||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||||
|
|
||||||
from exporter.baseline import NativeFile, NativeSystem, build_native_model # noqa: E402
|
from exporter.baseline import NativeFile, NativeSystem, build_native_model # noqa: E402
|
||||||
|
from exporter.bizhawk_exporter import Exporter as BizHawk # noqa: E402
|
||||||
|
from exporter.misterfpga_exporter import Exporter as Mister # noqa: E402
|
||||||
|
from exporter.romm_exporter import Exporter as Romm # noqa: E402
|
||||||
|
from scraper.emudeck_scraper import FUNCTION_HASH_MAP # noqa: E402
|
||||||
from exporter.emudeck_exporter import Exporter as EmuDeck # noqa: E402
|
from exporter.emudeck_exporter import Exporter as EmuDeck # noqa: E402
|
||||||
from exporter.retrobat_exporter import Exporter as RetroBat # noqa: E402
|
from exporter.retrobat_exporter import Exporter as RetroBat # noqa: E402
|
||||||
from exporter.retrodeck_exporter import Exporter as RetroDeck # noqa: E402
|
from exporter.retrodeck_exporter import Exporter as RetroDeck # noqa: E402
|
||||||
@@ -130,8 +134,6 @@ class EmuDeckWritesItsCorrections(unittest.TestCase):
|
|||||||
SCRIPT = "checkPS1BIOS(){\n local hashes=(%s)\n}\n"
|
SCRIPT = "checkPS1BIOS(){\n local hashes=(%s)\n}\n"
|
||||||
|
|
||||||
def _systems(self, corrected: NativeFile, other: NativeFile) -> dict:
|
def _systems(self, corrected: NativeFile, other: NativeFile) -> dict:
|
||||||
from scraper.emudeck_scraper import FUNCTION_HASH_MAP
|
|
||||||
|
|
||||||
system_id = FUNCTION_HASH_MAP["checkPS1BIOS"]
|
system_id = FUNCTION_HASH_MAP["checkPS1BIOS"]
|
||||||
corrected.native_system = other.native_system = system_id
|
corrected.native_system = other.native_system = system_id
|
||||||
return {system_id: NativeSystem(system_id, files=[corrected, other])}
|
return {system_id: NativeSystem(system_id, files=[corrected, other])}
|
||||||
@@ -168,8 +170,6 @@ class EmuDeckWritesItsCorrections(unittest.TestCase):
|
|||||||
|
|
||||||
class BizHawkCountsWhatItWrites(unittest.TestCase):
|
class BizHawkCountsWhatItWrites(unittest.TestCase):
|
||||||
def test_an_ambiguous_name_is_not_counted(self):
|
def test_an_ambiguous_name_is_not_counted(self):
|
||||||
from exporter.bizhawk_exporter import Exporter as BizHawk
|
|
||||||
|
|
||||||
sha_a, sha_b, sha_c = "a" * 40, "b" * 40, "c" * 40
|
sha_a, sha_b, sha_c = "a" * 40, "b" * 40, "c" * 40
|
||||||
first = NativeFile("bios.bin", "bios.bin", "S1", platform={"sha1": sha_a},
|
first = NativeFile("bios.bin", "bios.bin", "S1", platform={"sha1": sha_a},
|
||||||
truth={"sha1": sha_b}, corrections=["sha1"])
|
truth={"sha1": sha_b}, corrections=["sha1"])
|
||||||
@@ -183,8 +183,6 @@ class BizHawkCountsWhatItWrites(unittest.TestCase):
|
|||||||
|
|
||||||
class MisterCountsWhatItWrites(unittest.TestCase):
|
class MisterCountsWhatItWrites(unittest.TestCase):
|
||||||
def test_a_path_the_database_lacks_is_not_counted(self):
|
def test_a_path_the_database_lacks_is_not_counted(self):
|
||||||
from exporter.misterfpga_exporter import Exporter as Mister
|
|
||||||
|
|
||||||
held = NativeFile("boot.rom", "NES/boot.rom", "nes", platform={"md5": A},
|
held = NativeFile("boot.rom", "NES/boot.rom", "nes", platform={"md5": A},
|
||||||
truth={"md5": B}, corrections=["md5"])
|
truth={"md5": B}, corrections=["md5"])
|
||||||
absent = NativeFile("boot.rom", "SNES/boot.rom", "snes", platform={"md5": A},
|
absent = NativeFile("boot.rom", "SNES/boot.rom", "snes", platform={"md5": A},
|
||||||
@@ -260,8 +258,6 @@ class OneFileOneIdentity(unittest.TestCase):
|
|||||||
class RommKeepsItsOwnKeys(unittest.TestCase):
|
class RommKeepsItsOwnKeys(unittest.TestCase):
|
||||||
def test_a_same_named_addition_does_not_replace_their_entry(self):
|
def test_a_same_named_addition_does_not_replace_their_entry(self):
|
||||||
"""fbneo's 480-byte boot.bin overwrote RomM's Dreamcast boot.bin."""
|
"""fbneo's 480-byte boot.bin overwrote RomM's Dreamcast boot.bin."""
|
||||||
from exporter.romm_exporter import Exporter as Romm
|
|
||||||
|
|
||||||
theirs = NativeFile("boot.bin", "dc/boot.bin", "dc",
|
theirs = NativeFile("boot.bin", "dc/boot.bin", "dc",
|
||||||
platform={"size": 2097152, "md5": A})
|
platform={"size": 2097152, "md5": A})
|
||||||
ours = NativeFile("boot.bin", "fbneo/boot.bin", "dc",
|
ours = NativeFile("boot.bin", "fbneo/boot.bin", "dc",
|
||||||
|
|||||||
@@ -2558,7 +2558,9 @@ class TestBumpCommit(unittest.TestCase):
|
|||||||
text = self.path.read_text(encoding="utf-8")
|
text = self.path.read_text(encoding="utf-8")
|
||||||
self.path.write_text(
|
self.path.write_text(
|
||||||
"source_commit:\n standalone: \"aaa\"\n libretro: \"bbb\"\n"
|
"source_commit:\n standalone: \"aaa\"\n libretro: \"bbb\"\n"
|
||||||
+ "\n".join(l for l in text.splitlines() if not l.startswith("source_commit"))
|
+ "\n".join(
|
||||||
|
line for line in text.splitlines() if not line.startswith("source_commit")
|
||||||
|
)
|
||||||
+ "\n",
|
+ "\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -9,7 +9,9 @@ from pathlib import Path
|
|||||||
|
|
||||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
||||||
|
|
||||||
import region
|
import common # noqa: E402
|
||||||
|
import region # noqa: E402
|
||||||
|
from packextras import emulator_region_drops # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
class TestVocabulary(unittest.TestCase):
|
class TestVocabulary(unittest.TestCase):
|
||||||
@@ -647,10 +649,6 @@ class RegionalRequestsKeepWhatTheCodeLoads(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def test_each_request_keeps_its_file(self):
|
def test_each_request_keeps_its_file(self):
|
||||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
|
||||||
import common
|
|
||||||
from packextras import emulator_region_drops
|
|
||||||
|
|
||||||
profiles = common.load_emulator_profiles(
|
profiles = common.load_emulator_profiles(
|
||||||
os.path.join(os.path.dirname(__file__), "..", "emulators")
|
os.path.join(os.path.dirname(__file__), "..", "emulators")
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in new issue
Block a user