mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-06 10:03:49 -06:00
chore: merge tooling fixes, fourth pass
This commit is contained in:
commit
e491ff1bc6
45 files changed
+801
-153
No files matched your search
@@ -22,6 +22,10 @@ the most useful contribution.
|
||||
hash differs from the one already held under its name goes to that folder's
|
||||
`.variants/`.
|
||||
|
||||
A ZIP that no list declares as a whole is read file by file, and the report
|
||||
says which of them the collection already holds: upload the files
|
||||
themselves, unzipped.
|
||||
|
||||
From a clone, the file goes to `bios/Manufacturer/Console/filename`, and
|
||||
`python scripts/validate_pr.py path/to/file` names that folder for a file still
|
||||
outside `bios/`, and `--place` copies it there under its declared name. Given a
|
||||
|
||||
@@ -83,6 +83,7 @@
|
||||
"galaksija",
|
||||
"gam4980",
|
||||
"gambatte",
|
||||
"gametank",
|
||||
"gearboy",
|
||||
"gearcoleco",
|
||||
"geargrafx",
|
||||
@@ -346,6 +347,7 @@
|
||||
"galaksija",
|
||||
"gam4980",
|
||||
"gambatte",
|
||||
"gametank",
|
||||
"gearboy",
|
||||
"gearcoleco",
|
||||
"geargrafx",
|
||||
|
||||
@@ -121,13 +121,6 @@ retrodeck:
|
||||
# includes the frontend name alongside the actual cores.
|
||||
remove_cores: [retroarch]
|
||||
|
||||
emudeck:
|
||||
targets:
|
||||
_default:
|
||||
# EmuDeck's EmuScripts tree carries a template.sh placeholder used
|
||||
# to scaffold new installers. It is not an emulator.
|
||||
remove_cores: [template]
|
||||
|
||||
lakka:
|
||||
targets:
|
||||
generic:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
platform: emudeck
|
||||
source: https://github.com/dragoonDorise/EmuDeck
|
||||
scraped_at: '2026-10-17T18:43:57Z'
|
||||
source: https://github.com/dragoonDorise/EmuDeck/tree/dddb30229adbbd54d5ff7283cfd70e11c4322c26
|
||||
scraped_at: '2026-11-06T07:10:22Z'
|
||||
targets:
|
||||
steamos:
|
||||
architecture: x86_64
|
||||
@@ -88,6 +88,7 @@ targets:
|
||||
- galaksija
|
||||
- gam4980
|
||||
- gambatte
|
||||
- gametank
|
||||
- gearboy
|
||||
- gearcoleco
|
||||
- geargrafx
|
||||
@@ -352,6 +353,7 @@ targets:
|
||||
- galaksija
|
||||
- gam4980
|
||||
- gambatte
|
||||
- gametank
|
||||
- gearboy
|
||||
- gearcoleco
|
||||
- geargrafx
|
||||
@@ -487,7 +489,6 @@ targets:
|
||||
- supermodel
|
||||
- swanstation
|
||||
- tamalibretro
|
||||
- template
|
||||
- tgbdual
|
||||
- theodore
|
||||
- thepowdertoy
|
||||
|
||||
@@ -19,6 +19,7 @@ import re
|
||||
_TIMESTAMP_PATTERNS = [
|
||||
re.compile(r'"generated_at":\s*"[^"]*"'), # database.json
|
||||
re.compile(r'"imported_at":\s*"[^"]*"'), # provenance snapshots
|
||||
re.compile(r'"fetched":\s*"[^"]*"'), # games/catalogs/
|
||||
re.compile(r"\*Auto-generated on [^*]*\*"), # README.md
|
||||
re.compile(r"\*Generated on [^*]*\*"), # docs site pages
|
||||
# The decorated pages carry the same stamp again as a rendered element,
|
||||
@@ -208,6 +209,10 @@ def _strip_timestamps(text: str) -> str:
|
||||
class ArtifactLockBusy(RuntimeError):
|
||||
"""Raised when another process already holds the artifact directory."""
|
||||
|
||||
|
||||
class ArtifactDirectoryMissing(FileNotFoundError):
|
||||
"""A reader named a directory that does not exist."""
|
||||
|
||||
# A run that holds artifact directories for its whole duration names them
|
||||
# here, os.pathsep apart, for the steps it spawns: they would otherwise be
|
||||
# refused by it. A pipeline holds the repository root and its --output-dir.
|
||||
@@ -227,6 +232,14 @@ def artifact_lock(directory: str, exclusive: bool = True) -> Iterator[None]:
|
||||
Writers take the lock exclusively, readers share it. On platforms
|
||||
without flock the lock is a no-op.
|
||||
"""
|
||||
# A reader creating the directory it was pointed at left a typo behind
|
||||
# as a directory holding a .lock, and the next run with the same typo
|
||||
# no longer said the path did not exist.
|
||||
if not exclusive and not os.path.isdir(directory):
|
||||
raise ArtifactDirectoryMissing(
|
||||
f"{directory}: no such directory, pass the --output-dir the packs "
|
||||
"were written to (default dist/)"
|
||||
)
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError:
|
||||
|
||||
@@ -91,8 +91,19 @@ def sha1_of(path: Path) -> str:
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
class GitFailed(RuntimeError):
|
||||
"""A git command failed: its empty output says nothing about the file."""
|
||||
|
||||
|
||||
def git(root: Path, *args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
|
||||
"""Run git in root. A failure raises: an unreadable index read as an empty
|
||||
status made a session's working change look unmodified, and --apply wrote
|
||||
HEAD's bytes over it."""
|
||||
done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
|
||||
if done.returncode:
|
||||
message = done.stderr.decode(errors="replace").strip()
|
||||
raise GitFailed(f"git {' '.join(args)} exited {done.returncode}: {message}")
|
||||
return done
|
||||
|
||||
|
||||
def head_blob(root: Path, path: str) -> str:
|
||||
@@ -197,7 +208,11 @@ def main() -> int:
|
||||
from common import load_database
|
||||
|
||||
root = Path.cwd()
|
||||
report = check(root, load_database(args.db), args.apply)
|
||||
try:
|
||||
report = check(root, load_database(args.db), args.apply)
|
||||
except GitFailed as exc:
|
||||
print(f"ERROR: {exc}. Nothing was restored past this point.", file=sys.stderr)
|
||||
return 2
|
||||
if args.json:
|
||||
print(json.dumps(report.as_dict(), indent=2))
|
||||
else:
|
||||
|
||||
+20
-17
@@ -32,7 +32,6 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
@@ -51,6 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
import export_native # noqa: E402
|
||||
import pin_installer # noqa: E402
|
||||
import refresh_data_dirs # noqa: E402
|
||||
import upstream # noqa: E402
|
||||
from common import ( # noqa: E402
|
||||
@@ -83,7 +83,6 @@ PYPI_URL = "https://pypi.org/pypi/{name}/json"
|
||||
NPM_URL = "https://registry.npmjs.org/{name}/latest"
|
||||
# The site's npm dependencies, pinned to exact versions and built by Deploy Site.
|
||||
SITE_PACKAGE = REPO_ROOT / "website" / "package.json"
|
||||
INSTALLERS = ("install.sh", "install.ps1")
|
||||
|
||||
SHOWN_ITEMS = 4 # items named in a diff summary before ", ..."
|
||||
SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..."
|
||||
@@ -509,7 +508,10 @@ def check_native(
|
||||
) -> list[Finding]:
|
||||
exporters = discover_exporters()
|
||||
index = cache_dir / export_native.SOURCES_INDEX
|
||||
recorded = export_native.load_sources(cache_dir)
|
||||
try:
|
||||
recorded = export_native.load_sources(cache_dir)
|
||||
except export_native.SourcesIndexUnreadable as exc:
|
||||
return [Finding("native", index.name, ERROR, detail=str(exc))]
|
||||
findings: list[Finding] = []
|
||||
for name in list_registered_platforms(str(platforms_dir), include_archived=True):
|
||||
exporter_class = exporters.get(name)
|
||||
@@ -1462,18 +1464,16 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]:
|
||||
|
||||
def _check_installer_pins() -> Finding:
|
||||
"""The bootstraps pin install.py by SHA-256, so a stale pin refuses every install."""
|
||||
digest = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
|
||||
stale = []
|
||||
for name in INSTALLERS:
|
||||
text = (REPO_ROOT / name).read_text(encoding="utf-8", errors="replace")
|
||||
if digest not in text:
|
||||
stale.append(name)
|
||||
stale = pin_installer.stale_pins(REPO_ROOT)
|
||||
return Finding(
|
||||
"ci",
|
||||
"install.py pin",
|
||||
STALE if stale else OK,
|
||||
local=digest[:12],
|
||||
detail=("pin outdated in " + ", ".join(stale)) if stale else "install.sh and install.ps1 match",
|
||||
local=pin_installer.installer_sha256(REPO_ROOT)[:12],
|
||||
detail=(
|
||||
f"pin outdated in {', '.join(stale)}, {pin_installer.COMMAND} writes it"
|
||||
if stale else "install.sh and install.ps1 match"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -1757,15 +1757,18 @@ def render(findings: list[Finding]) -> str:
|
||||
# Columns are padded, never cut: the profile_sync row says in its
|
||||
# local and remote values what to re-read, and lost the end of it.
|
||||
local_width = min(max((len(f.local) for f in rows), default=0), 44)
|
||||
remote_width = min(max((len(f.remote) for f in rows), default=0), 44)
|
||||
# The arrow and the remote value, or as many blanks where a row has none.
|
||||
remote_span = 4 + remote_width if remote_width else 0
|
||||
for f in rows:
|
||||
cols = f" {f.status:7} {f.subject:{width}}"
|
||||
if f.remote:
|
||||
cols += f" {f.local:{local_width}} -> {f.remote}"
|
||||
elif f.local or (f.detail and local_width):
|
||||
# An arrow pointing at nothing read as a lost value. An empty
|
||||
# local value still takes its column, or the detail starts
|
||||
# where the other rows show theirs.
|
||||
cols += f" {f.local:{local_width}}"
|
||||
cols += f" {f.local:{local_width}} -> {f.remote:{remote_width}}"
|
||||
elif f.local or (f.detail and (local_width or remote_span)):
|
||||
# An arrow pointing at nothing read as a lost value. Empty
|
||||
# local and remote values still take their columns, or the
|
||||
# detail starts where the other rows show theirs.
|
||||
cols += f" {f.local:{local_width}}" + " " * remote_span
|
||||
if f.detail:
|
||||
cols += f" {f.detail}"
|
||||
lines.append(cols.rstrip())
|
||||
|
||||
+2
-1
@@ -3587,6 +3587,7 @@ from artifacts import ( # noqa: E402,F401
|
||||
copy_file_atomic,
|
||||
plain_file_mode,
|
||||
set_plain_mode,
|
||||
ArtifactDirectoryMissing,
|
||||
ArtifactLockBusy,
|
||||
artifact_lock,
|
||||
hold_artifact_lock,
|
||||
@@ -3634,7 +3635,7 @@ def run_cli(main: Callable[[], int | None]) -> None:
|
||||
try:
|
||||
code = main()
|
||||
sys.stdout.flush()
|
||||
except (InputMissing, ArtifactLockBusy) as exc:
|
||||
except (InputMissing, ArtifactLockBusy, ArtifactDirectoryMissing) as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except BrokenPipeError:
|
||||
|
||||
@@ -36,6 +36,7 @@ from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_tex
|
||||
import yaml
|
||||
|
||||
from common import (
|
||||
InputMissing,
|
||||
build_zip_contents_index,
|
||||
platform_arg,
|
||||
list_registered_platforms,
|
||||
@@ -62,13 +63,27 @@ _USER_AGENT = "retrobios-exporter/1.0"
|
||||
_MAX_BYTES = 64 * 1024 * 1024
|
||||
|
||||
|
||||
class SourcesIndexUnreadable(InputMissing):
|
||||
"""The cache's URL index exists and does not read as JSON."""
|
||||
|
||||
|
||||
def _load_sources(index: Path | None) -> dict[str, str]:
|
||||
"""The URL each cached file was fetched from.
|
||||
|
||||
An absent index records nothing. An index that does not read raises:
|
||||
read as empty it let every cached original pass for current, and the
|
||||
next download rewrote it with that one entry.
|
||||
"""
|
||||
if index is None or not index.is_file():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(index.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise SourcesIndexUnreadable(
|
||||
f"{index} does not read ({exc}): remove it, then"
|
||||
" python scripts/export_native.py --all --refresh-cache"
|
||||
" downloads every original again and records its URL"
|
||||
) from exc
|
||||
|
||||
|
||||
def source_key(destination: Path, index: Path | None) -> str:
|
||||
|
||||
@@ -941,6 +941,10 @@ the most useful contribution.
|
||||
hash differs from the one already held under its name goes to that folder's
|
||||
`.variants/`.
|
||||
|
||||
A ZIP that no list declares as a whole is read file by file, and the report
|
||||
says which of them the collection already holds: upload the files
|
||||
themselves, unzipped.
|
||||
|
||||
From a clone, the file goes to `bios/Manufacturer/Console/filename`, and
|
||||
`python scripts/validate_pr.py path/to/file` names that folder for a file still
|
||||
outside `bios/`, and `--place` copies it there under its declared name. Given a
|
||||
|
||||
+15
-12
@@ -14,13 +14,13 @@ import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import (
|
||||
build_target_cores_cache,
|
||||
platform_arg,
|
||||
list_registered_platforms,
|
||||
load_data_dir_registry,
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
load_target_config,
|
||||
require_directory,
|
||||
require_platform,
|
||||
require_yaml,
|
||||
@@ -97,6 +97,19 @@ def main(argv: list[str] | None = None) -> None:
|
||||
require_platform(args.platform, args.platforms_dir)
|
||||
platforms = [args.platform]
|
||||
|
||||
# A named platform without that target is refused, as verify and
|
||||
# generate_pack refuse it: skipped, it printed Done. and exited 0 with
|
||||
# no model written. --all leaves such platforms out and says so.
|
||||
target_cores_cache: dict[str, set[str] | None] = {}
|
||||
if args.target:
|
||||
try:
|
||||
target_cores_cache, platforms = build_target_cores_cache(
|
||||
platforms, args.target, args.platforms_dir, is_all=args.all
|
||||
)
|
||||
except (FileNotFoundError, ValueError) as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# A target-narrowed model is a different artifact: written under the
|
||||
# platform's own name it overwrote the full one, and diff_truth then
|
||||
# compared a narrowed model against the whole scrape. Its directory is
|
||||
@@ -109,17 +122,7 @@ def main(argv: list[str] | None = None) -> None:
|
||||
output_dir, [name], args.target, args.platforms_dir
|
||||
)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
target_cores: set[str] | None = None
|
||||
if args.target:
|
||||
try:
|
||||
target_cores = load_target_config(
|
||||
name,
|
||||
args.target,
|
||||
args.platforms_dir,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
print(f" {name}: no target config, skipped")
|
||||
continue
|
||||
target_cores = target_cores_cache.get(name)
|
||||
|
||||
# Load platform config (with inheritance) and registry entry
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write the SHA-256 of install.py into the bootstraps that pin it.
|
||||
|
||||
install.sh and install.ps1 refuse an installer whose hash is not the one
|
||||
they carry, so every change to install.py moves both pins in the same
|
||||
commit. --check writes nothing and exits 1 naming each stale pin.
|
||||
|
||||
Usage:
|
||||
python scripts/pin_installer.py
|
||||
python scripts/pin_installer.py --check
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
INSTALLER = "install.py"
|
||||
# Each bootstrap and its one pin line, the value between the two groups.
|
||||
PINS: dict[str, re.Pattern[str]] = {
|
||||
"install.sh": re.compile(r'^(DEFAULT_INSTALL_SHA256=")[0-9a-f]{64}(")', re.M),
|
||||
"install.ps1": re.compile(r'^(\s*\$defaultInstallSha256 = ")[0-9a-f]{64}(")', re.M),
|
||||
}
|
||||
COMMAND = "python scripts/pin_installer.py"
|
||||
|
||||
|
||||
def installer_sha256(root: Path = ROOT) -> str:
|
||||
return hashlib.sha256((root / INSTALLER).read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def stale_pins(root: Path = ROOT) -> list[str]:
|
||||
"""The bootstraps whose pin is not the SHA-256 of install.py."""
|
||||
digest = installer_sha256(root)
|
||||
stale = []
|
||||
for name, pin in PINS.items():
|
||||
found = pin.search((root / name).read_bytes().decode("utf-8"))
|
||||
if found is None:
|
||||
raise ValueError(f"{name}: no installer pin line")
|
||||
if found.group(0) != f"{found.group(1)}{digest}{found.group(2)}":
|
||||
stale.append(name)
|
||||
return stale
|
||||
|
||||
|
||||
def write_pins(root: Path = ROOT) -> list[str]:
|
||||
"""Rewrite each stale pin. Returns the bootstraps written."""
|
||||
digest = installer_sha256(root)
|
||||
written = []
|
||||
for name in stale_pins(root):
|
||||
path = root / name
|
||||
text = path.read_bytes().decode("utf-8")
|
||||
new, count = PINS[name].subn(rf"\g<1>{digest}\g<2>", text)
|
||||
if count != 1:
|
||||
raise ValueError(f"{name}: {count} installer pin lines, expected 1")
|
||||
path.write_bytes(new.encode("utf-8"))
|
||||
written.append(name)
|
||||
return written
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--check", action="store_true",
|
||||
help="write nothing, exit 1 if a pin is stale")
|
||||
args = parser.parse_args()
|
||||
if args.check:
|
||||
stale = stale_pins()
|
||||
for name in stale:
|
||||
print(f"{name}: pin is not the SHA-256 of {INSTALLER}, {COMMAND} writes it",
|
||||
file=sys.stderr)
|
||||
return 1 if stale else 0
|
||||
for name in write_pins():
|
||||
print(f"{name}: pinned {installer_sha256()}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from common import run_cli
|
||||
|
||||
run_cli(main)
|
||||
@@ -71,6 +71,24 @@ def cache_lock(
|
||||
return _announced_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared, cache_dir)
|
||||
|
||||
|
||||
def _wait_message(lock_path: Path, shared: bool, cache_dir: Path) -> str:
|
||||
"""Who to wait for, and, for a refresh, how not to.
|
||||
|
||||
The bare "another run holds it" named neither the lock nor its holder,
|
||||
and the build waited with no limit.
|
||||
"""
|
||||
message = (
|
||||
f"waiting for {cache_dir}: another run holds {lock_path} "
|
||||
f"(lsof {lock_path} names it)"
|
||||
)
|
||||
if not shared:
|
||||
message += (
|
||||
". generate_pack.py --no-refresh-data, or --offline there and in"
|
||||
" pipeline.py, builds from the cache as it is without waiting"
|
||||
)
|
||||
return message
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[None]:
|
||||
"""file_lock, saying so when it has to wait.
|
||||
@@ -90,7 +108,7 @@ def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[
|
||||
try:
|
||||
fcntl.flock(handle, mode | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
log.warning("waiting for %s: another run holds it", cache_dir)
|
||||
log.warning(_wait_message(lock_path, shared, cache_dir))
|
||||
fcntl.flock(handle, mode)
|
||||
try:
|
||||
yield
|
||||
|
||||
@@ -34,7 +34,10 @@ has ended, and its export is rewritten even when no row named it.
|
||||
|
||||
Surfaced only (no refresh): data rows that are not a _data_dirs.yml key
|
||||
(the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs,
|
||||
CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs
|
||||
CI pins (PyPI, pinned actions, while scripts/pin_installer.py writes the
|
||||
install.py SHA-256 pin), profile_sync, rom-folder
|
||||
pins (moved by hand, then python -m scripts.scraper.romfolders --setup NAME
|
||||
reads them again). Each needs
|
||||
a human read (new .info file, manual DAT download, PyPI version bump).
|
||||
|
||||
Usage:
|
||||
@@ -65,7 +68,9 @@ PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
|
||||
DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
|
||||
|
||||
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
|
||||
MANUAL_AREAS = ("coreinfo", "ci", "profiles")
|
||||
# A rom-folder pin is moved by hand in platforms/romfolders/_sources/,
|
||||
# then `python -m scripts.scraper.romfolders --setup NAME` reads it again.
|
||||
MANUAL_AREAS = ("romfolders", "coreinfo", "ci", "profiles")
|
||||
|
||||
JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10
|
||||
|
||||
@@ -397,6 +402,10 @@ def _command_for(
|
||||
"--force",
|
||||
]
|
||||
|
||||
if area == "ci" and subject == "install.py pin":
|
||||
# The pin is the hash of the committed install.py, nothing to judge.
|
||||
return [sys.executable, "scripts/pin_installer.py"]
|
||||
|
||||
if area == "catalogs":
|
||||
if subject == "mame recipes":
|
||||
return [
|
||||
@@ -563,7 +572,8 @@ def main() -> int:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument(
|
||||
"--only",
|
||||
help="comma-separated areas to check: " + ",".join(AUTO_AREAS),
|
||||
help="comma-separated areas to check: " + ",".join(AUTO_AREAS)
|
||||
+ " (refreshed), " + ",".join(MANUAL_AREAS) + " (listed for a human)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
@@ -579,7 +589,7 @@ def main() -> int:
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="machine-readable summary (jobs, surfaced, ignored)",
|
||||
help="machine-readable summary (jobs, surfaced, ignored), the plan with --dry-run",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--findings",
|
||||
@@ -618,6 +628,22 @@ def main() -> int:
|
||||
registry = _load_platform_registry()
|
||||
jobs, surfaced, ignored = plan_jobs(findings, registry)
|
||||
|
||||
if args.dry_run and args.json:
|
||||
print(json.dumps({
|
||||
"planned": [
|
||||
{
|
||||
"area": job.area,
|
||||
"subject": job.subject,
|
||||
"command": job.command,
|
||||
"after": sorted(job.after),
|
||||
"then": [list(command) for command in job.then],
|
||||
}
|
||||
for job in sorted(jobs, key=lambda j: (j.area, j.subject))
|
||||
],
|
||||
"surfaced": surfaced,
|
||||
"ignored_count": len(ignored),
|
||||
}, indent=2))
|
||||
return 0
|
||||
if args.dry_run:
|
||||
print(f"\n[plan] {len(jobs)} refreshable, {len(surfaced)} manual, "
|
||||
f"{len(ignored)} already clean/skipped")
|
||||
|
||||
@@ -49,7 +49,7 @@ from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from ..artifacts import write_text_atomic
|
||||
from ..artifacts import write_if_changed
|
||||
from ..common import load_emulator_profiles, run_cli
|
||||
from .romset_dat_importer import (
|
||||
LISTXML_CHUNK,
|
||||
@@ -1114,14 +1114,6 @@ def dump(doc: dict) -> str:
|
||||
return json.dumps(doc, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def write_if_changed(path: Path, text: str) -> bool:
|
||||
if path.is_file() and path.read_text(encoding="utf-8") == text:
|
||||
return False
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_text_atomic(str(path), text)
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
|
||||
which = parser.add_mutually_exclusive_group(required=True)
|
||||
@@ -1197,7 +1189,8 @@ def main() -> int:
|
||||
continue
|
||||
if not args.dry_run:
|
||||
target = args.output_dir / f"{profile}.json"
|
||||
if write_if_changed(target, dump(doc)):
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if write_if_changed(str(target), dump(doc)):
|
||||
print(f" wrote {target}")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
@@ -46,14 +46,18 @@ STORAGE = "${storagePath}"
|
||||
FLATPAK_DATA = "${HOME}/.var/app/"
|
||||
|
||||
# EmuScripts file names that differ from the emulator profile they set up.
|
||||
# The platform and target scrapers both read them here.
|
||||
SCRIPT_EMULATORS = {
|
||||
"pcsx2qt": "pcsx2",
|
||||
"rpcs3legacy": "rpcs3",
|
||||
"cemuproton": "cemu",
|
||||
"rmg": "mupen64plus_next",
|
||||
}
|
||||
# The main RetroArch config script installs no emulator.
|
||||
NOT_EMULATORS = {"retroarch_maincfg"}
|
||||
# The main RetroArch config script installs no emulator, and emudeck-we's
|
||||
# emuDeckTemplate.ps1 is the skeleton new installers are copied from.
|
||||
NOT_EMULATORS = {"retroarch_maincfg", "template"}
|
||||
# EmuDeck's SteamOS installers are shell scripts, emudeck-we's PowerShell.
|
||||
SCRIPT_SUFFIXES = (".sh", ".ps1")
|
||||
|
||||
# What the emulator does with a directory written into each key.
|
||||
DIRECTORY_KINDS = {
|
||||
@@ -66,9 +70,10 @@ DIRECTORY_KINDS = {
|
||||
|
||||
def emulator_of_script(file_name: str) -> str | None:
|
||||
"""The emulator an EmuScripts file sets up, None for a helper script."""
|
||||
if not file_name.endswith(".sh"):
|
||||
stem, dot, suffix = file_name.rpartition(".")
|
||||
if not dot or f".{suffix}" not in SCRIPT_SUFFIXES:
|
||||
return None
|
||||
key = re.sub(r"^emuDeck", "", file_name[: -len(".sh")], flags=re.IGNORECASE).lower()
|
||||
key = re.sub(r"^emuDeck", "", stem, flags=re.IGNORECASE).lower()
|
||||
if not key or key in NOT_EMULATORS:
|
||||
return None
|
||||
return SCRIPT_EMULATORS.get(key, key)
|
||||
|
||||
@@ -469,9 +469,10 @@ class Scraper(BaseScraper):
|
||||
|
||||
version = fetch_github_latest_version("dragoonDorise/EmuDeck")
|
||||
|
||||
cores = self._fetch_installed_emulators()
|
||||
revision = launcher_revision(self.checkbios_url)
|
||||
cores = self._fetch_installed_emulators(revision)
|
||||
standalone = [core for core in cores if core != "retroarch"]
|
||||
tree = RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url))
|
||||
tree = RepoTree.github(EMUDECK_REPO, revision)
|
||||
libretro = set(installed_libretro_cores(tree.read(RETROARCH_SCRIPT)))
|
||||
|
||||
return {
|
||||
@@ -501,16 +502,16 @@ class Scraper(BaseScraper):
|
||||
"launchers": extract_launchers(tree),
|
||||
}
|
||||
|
||||
def _fetch_installed_emulators(self) -> list[str]:
|
||||
"""Fetch the list of emulators installed by EmuDeck from EmuScripts.
|
||||
|
||||
Returns core names normalized to match emulator profile keys.
|
||||
def _fetch_installed_emulators(self, revision: str) -> list[str]:
|
||||
"""The emulators EmuDeck's EmuScripts install at the pinned revision,
|
||||
named as their profiles are: the launcher layer and the SteamOS
|
||||
target are read at that revision too.
|
||||
"""
|
||||
import json
|
||||
|
||||
api_url = (
|
||||
"https://api.github.com/repos/dragoonDorise/EmuDeck/"
|
||||
"contents/functions/EmuScripts"
|
||||
f"contents/functions/EmuScripts?ref={revision}"
|
||||
)
|
||||
try:
|
||||
req = urllib.request.Request(api_url, headers=github_headers())
|
||||
|
||||
@@ -38,7 +38,7 @@ _SCRIPTS_DIR = str(Path(__file__).resolve().parents[1])
|
||||
if _SCRIPTS_DIR not in sys.path:
|
||||
sys.path.insert(0, _SCRIPTS_DIR)
|
||||
|
||||
from artifacts import copy_stream_atomic, write_text_atomic # noqa: E402
|
||||
from artifacts import copy_stream_atomic, write_if_changed, write_text_atomic # noqa: E402
|
||||
from common import parse_untrusted_xml # noqa: E402
|
||||
from region import REGIONS, WORLD # noqa: E402
|
||||
from region_audit import NOINTRO # noqa: E402
|
||||
@@ -275,14 +275,6 @@ def dump_catalog(doc: dict) -> str:
|
||||
return f'{text},"games":[\n{games}\n]}}\n'
|
||||
|
||||
|
||||
def write_if_changed(path: Path, text: str) -> bool:
|
||||
if path.is_file() and path.read_text(encoding="utf-8") == text:
|
||||
return False
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_text_atomic(str(path), text)
|
||||
return True
|
||||
|
||||
|
||||
# Fetching ---------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -503,7 +495,9 @@ def main() -> int:
|
||||
failures.append(f"{doc['source']}/{doc['system']}: {cov['games']} games, "
|
||||
f"floor {floor_of(doc)}")
|
||||
if not args.dry_run:
|
||||
write_if_changed(args.output_dir / doc["source"] / f"{doc['system']}.json", text)
|
||||
target = args.output_dir / doc["source"] / f"{doc['system']}.json"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_if_changed(str(target), text)
|
||||
print(f"{len(docs)} catalogs, {raw_total} bytes raw, {gz_total} bytes gzip")
|
||||
for failure in failures:
|
||||
print(f"FAIL {failure}", file=sys.stderr)
|
||||
|
||||
@@ -12,7 +12,6 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
@@ -22,6 +21,7 @@ import yaml
|
||||
|
||||
from common import github_token, yaml_load
|
||||
|
||||
from ..emudeck_launchers import SCRIPT_SUFFIXES, emulator_of_script
|
||||
from . import BaseTargetScraper
|
||||
|
||||
PLATFORM_NAME = "emudeck"
|
||||
@@ -33,24 +33,6 @@ WINDOWS_API = (
|
||||
"https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts"
|
||||
)
|
||||
|
||||
# Map EmuDeck script names to emulator profile keys
|
||||
# Script naming: emuDeckDolphin.sh -> dolphin
|
||||
# Some need explicit mapping when names differ
|
||||
_NAME_OVERRIDES: dict[str, str] = {
|
||||
"pcsx2qt": "pcsx2",
|
||||
"rpcs3legacy": "rpcs3",
|
||||
"cemuproton": "cemu",
|
||||
"rmg": "mupen64plus_next",
|
||||
"bigpemu": "bigpemu",
|
||||
"eden": "eden",
|
||||
"suyu": "suyu",
|
||||
"ares": "ares",
|
||||
}
|
||||
|
||||
# Scripts that are not emulators (config helpers, etc.)
|
||||
_SKIP = {"retroarch_maincfg", "retroarch"}
|
||||
|
||||
|
||||
def _fetch(url: str) -> str:
|
||||
headers = {"User-Agent": "retrobios-scraper/1.0"}
|
||||
token = github_token()
|
||||
@@ -69,26 +51,13 @@ def _list_emuscripts(api_url: str) -> list[str]:
|
||||
"""List emulator script filenames from GitHub API."""
|
||||
entries = json.loads(_fetch(api_url))
|
||||
names = [
|
||||
e["name"] for e in entries if e.get("name", "").endswith((".sh", ".ps1"))
|
||||
e["name"] for e in entries if e.get("name", "").endswith(SCRIPT_SUFFIXES)
|
||||
]
|
||||
if not names:
|
||||
raise RuntimeError(f"no emulator scripts listed at {api_url}")
|
||||
return names
|
||||
|
||||
|
||||
def _script_to_core(filename: str) -> str | None:
|
||||
"""Convert EmuScripts filename to core profile key."""
|
||||
# Strip extension and emuDeck prefix
|
||||
name = re.sub(r"\.(sh|ps1)$", "", filename, flags=re.IGNORECASE)
|
||||
name = re.sub(r"^emuDeck", "", name, flags=re.IGNORECASE)
|
||||
if not name:
|
||||
return None
|
||||
key = name.lower()
|
||||
if key in _SKIP:
|
||||
return None
|
||||
return _NAME_OVERRIDES.get(key, key)
|
||||
|
||||
|
||||
class Scraper(BaseTargetScraper):
|
||||
"""Fetches emulator lists for EmuDeck SteamOS and Windows targets."""
|
||||
|
||||
@@ -114,14 +83,13 @@ class Scraper(BaseTargetScraper):
|
||||
seen: set[str] = set()
|
||||
has_retroarch = False
|
||||
for script in scripts:
|
||||
core = _script_to_core(script)
|
||||
if core and core not in seen:
|
||||
core = emulator_of_script(script)
|
||||
# RetroArch stands for its libretro cores, added below.
|
||||
if core == "retroarch":
|
||||
has_retroarch = True
|
||||
elif core and core not in seen:
|
||||
seen.add(core)
|
||||
cores.append(core)
|
||||
# Detect RetroArch presence (provides all libretro cores)
|
||||
name = re.sub(r"\.(sh|ps1)$", "", script, flags=re.IGNORECASE)
|
||||
if name.lower() in ("emudeckretroarch", "retroarch_maincfg"):
|
||||
has_retroarch = True
|
||||
|
||||
standalone_count = len(cores)
|
||||
# EmuDeck ships RetroArch = all its libretro cores are available
|
||||
|
||||
+49
-11
@@ -677,12 +677,21 @@ class Collision:
|
||||
resolved: list[str]
|
||||
# Each declaration with the file it resolved to, in the platform's order.
|
||||
declarations: list[tuple[dict, str | None]] = field(default_factory=list)
|
||||
# The system of each declaration, in the same order.
|
||||
systems: list[str] = field(default_factory=list)
|
||||
# The file the pack ships at the destination, as the builder decides it.
|
||||
shipped: str | None = None
|
||||
|
||||
|
||||
def _collision_json(collision: Collision) -> dict:
|
||||
return {
|
||||
"destination": collision.destination,
|
||||
"resolved": list(collision.resolved),
|
||||
"shipped": collision.shipped,
|
||||
"declarations": [
|
||||
{"system": system, "name": entry.get("name", ""), "file": local}
|
||||
for system, (entry, local) in zip(collision.systems, collision.declarations)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -702,19 +711,27 @@ def find_collisions(
|
||||
CoCo disk ROM at one bios/disk.rom.
|
||||
"""
|
||||
fold = bool(config.get("case_insensitive_fs", False))
|
||||
by_dest: dict[str, list[dict]] = {}
|
||||
for system in (config.get("systems") or {}).values():
|
||||
systems = config.get("systems") or {}
|
||||
# The builder lets a hash-constrained declaration that resolves by
|
||||
# content take the path, else the first it reaches walking the systems
|
||||
# sorted: the same choice says which file ships.
|
||||
preferred = set(
|
||||
preferred_entries(systems, db, "bios", "", False, zip_contents, True).values()
|
||||
)
|
||||
by_dest: dict[str, list[tuple[str, dict]]] = {}
|
||||
for sys_id, system in systems.items():
|
||||
for entry in system.get("files") or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
dest = entry.get("destination") or entry.get("name") or ""
|
||||
if dest:
|
||||
by_dest.setdefault(_normalize(dest, fold), []).append(entry)
|
||||
by_dest.setdefault(_normalize(dest, fold), []).append((sys_id, entry))
|
||||
|
||||
collisions = []
|
||||
for key, entries in sorted(by_dest.items()):
|
||||
if len(entries) < 2:
|
||||
for key, claims in sorted(by_dest.items()):
|
||||
if len(claims) < 2:
|
||||
continue
|
||||
entries = [entry for _sys_id, entry in claims]
|
||||
resolved = []
|
||||
declarations = []
|
||||
for entry in entries:
|
||||
@@ -734,8 +751,22 @@ def find_collisions(
|
||||
# also names the first one's file.
|
||||
if any(_accepted_by_all(path, entries, db) for path in resolved):
|
||||
continue
|
||||
reached = sorted(
|
||||
(sys_id, local)
|
||||
for (sys_id, _entry), (_e, local) in zip(claims, declarations) if local
|
||||
)
|
||||
shipped = next(
|
||||
(local for entry, local in declarations if local and id(entry) in preferred),
|
||||
reached[0][1],
|
||||
)
|
||||
collisions.append(
|
||||
Collision(destination=key, resolved=resolved, declarations=declarations)
|
||||
Collision(
|
||||
destination=key,
|
||||
resolved=resolved,
|
||||
declarations=declarations,
|
||||
systems=[sys_id for sys_id, _entry in claims],
|
||||
shipped=shipped,
|
||||
)
|
||||
)
|
||||
return collisions
|
||||
|
||||
@@ -863,11 +894,18 @@ def _same_file_family(paths: list[str]) -> bool:
|
||||
|
||||
|
||||
def format_collision(collision: Collision) -> str:
|
||||
"""One line naming the destination and the files fighting for it."""
|
||||
return (
|
||||
f"{collision.destination}: declared for "
|
||||
+ " and for ".join(collision.resolved)
|
||||
)
|
||||
"""One line naming the destination, which system declares which file,
|
||||
and the one the pack ships there."""
|
||||
owners: dict[str, list[str]] = {}
|
||||
for system, (_entry, local) in zip(collision.systems, collision.declarations):
|
||||
if local and system not in owners.setdefault(local, []):
|
||||
owners[local].append(system)
|
||||
parts = [
|
||||
f"{path} ({', '.join(owners[path])}) "
|
||||
+ ("packed" if path == collision.shipped else "left out")
|
||||
for path in collision.resolved
|
||||
]
|
||||
return f"{collision.destination}: " + ", ".join(parts)
|
||||
|
||||
|
||||
def format_conflict(conflict: Conflict) -> str:
|
||||
|
||||
@@ -338,6 +338,10 @@ def main() -> int:
|
||||
# The packs are rewritten in place by generate_pack under this lock: a
|
||||
# build running beside the split mixes its bytes into the pack being
|
||||
# read, or writes a pack that the split then deletes.
|
||||
if not args.target.exists():
|
||||
print(f"Error: {args.target}: no such pack or directory, pass a pack or the "
|
||||
"--output-dir generate_pack wrote (default dist/)", file=sys.stderr)
|
||||
return 1
|
||||
directory = args.target if args.target.is_dir() else args.target.parent
|
||||
try:
|
||||
with artifact_lock(str(directory)):
|
||||
|
||||
+74
-6
@@ -18,6 +18,7 @@ Outputs a structured report suitable for PR comments.
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -25,6 +26,7 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
@@ -335,6 +337,53 @@ def load_profile_hashes(
|
||||
return known
|
||||
|
||||
|
||||
def _archive_members(
|
||||
filepath: str, db: dict | None, platform_hashes: dict, profile_hashes: dict
|
||||
) -> list[str] | None:
|
||||
"""What each file of a ZIP is, None when filepath is not a ZIP.
|
||||
|
||||
Read only for an archive no list declares as a whole: MAME and FBNeo
|
||||
sets are ZIP BIOS files in their own right.
|
||||
"""
|
||||
if not zipfile.is_zipfile(filepath):
|
||||
return None
|
||||
held = (db or {}).get("files", {})
|
||||
described = []
|
||||
try:
|
||||
with zipfile.ZipFile(filepath) as archive:
|
||||
for info in archive.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
if info.file_size > MAX_FILE_SIZE:
|
||||
described.append(f"`{info.filename}` too large to read here")
|
||||
continue
|
||||
sha1, md5 = hashlib.sha1(), hashlib.md5()
|
||||
with archive.open(info) as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
||||
sha1.update(chunk)
|
||||
md5.update(chunk)
|
||||
sha1_hex, md5_hex = sha1.hexdigest(), md5.hexdigest()
|
||||
crc = f"{info.CRC:08x}"
|
||||
if sha1_hex in held:
|
||||
state = f"already held at `{held[sha1_hex]['path']}`"
|
||||
elif (
|
||||
sha1_hex in platform_hashes.get("sha1", set())
|
||||
or md5_hex in platform_hashes.get("md5", set())
|
||||
or (crc, info.file_size) in platform_hashes.get("crc32", set())
|
||||
or profile_hashes.get("sha1", {}).get(sha1_hex)
|
||||
or profile_hashes.get("md5", {}).get(md5_hex)
|
||||
or profile_hashes.get("crc32", {}).get((crc, info.file_size))
|
||||
):
|
||||
state = "declared and not held"
|
||||
else:
|
||||
state = "not recognised"
|
||||
described.append(f"`{info.filename}` {state}")
|
||||
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
|
||||
# An encrypted member raises RuntimeError.
|
||||
described.append(f"unreadable ({exc})")
|
||||
return described
|
||||
|
||||
|
||||
def validate_file(
|
||||
filepath: str,
|
||||
db: dict | None,
|
||||
@@ -446,6 +495,8 @@ def validate_file(
|
||||
or profile_hashes.get("crc32", {}).get((result.crc32, None))
|
||||
)
|
||||
|
||||
# The files of a ZIP no list declares whole, when the upload is one.
|
||||
members: list[str] | None = None
|
||||
if sha1_known:
|
||||
result.add_check(True, "SHA1 matches known platform requirement")
|
||||
elif md5_known:
|
||||
@@ -466,11 +517,26 @@ def validate_file(
|
||||
)
|
||||
else:
|
||||
result.outcome = "unknown"
|
||||
result.add_warning(
|
||||
"Not recognised: no platform list or emulator profile declares this "
|
||||
f"file. Say in the {'issue' if too_large else 'pull request'} which "
|
||||
"emulator reads it, for which system, and where the dump comes from"
|
||||
members = None if too_large else _archive_members(
|
||||
filepath, db, platform_hashes, profile_hashes
|
||||
)
|
||||
if members is not None:
|
||||
# A folder zipped to upload many files at once was hashed as one
|
||||
# file and its contributor asked where that dump came from.
|
||||
shown = "; ".join(members[:10]) + (
|
||||
f"; and {len(members) - 10} more" if len(members) > 10 else ""
|
||||
)
|
||||
result.add_warning(
|
||||
"A ZIP no platform list or emulator profile declares as a whole: "
|
||||
"add the files it holds, unzipped, not the archive. Inside: "
|
||||
+ (shown or "nothing")
|
||||
)
|
||||
else:
|
||||
result.add_warning(
|
||||
"Not recognised: no platform list or emulator profile declares this "
|
||||
f"file. Say in the {'issue' if too_large else 'pull request'} which "
|
||||
"emulator reads it, for which system, and where the dump comes from"
|
||||
)
|
||||
|
||||
normalized = os.path.normpath(relative)
|
||||
# A platform's declaration first, then a profile's: most files the gaps
|
||||
@@ -503,7 +569,8 @@ def validate_file(
|
||||
result.add_check(True, f"Correct placement: bios/{parts[1]}/{parts[2]}/")
|
||||
else:
|
||||
result.add_warning("File should be in bios/Manufacturer/Console/ structure")
|
||||
elif result.outcome == "held":
|
||||
elif result.outcome == "held" or members is not None:
|
||||
# Nothing to place: a copy, or an archive whose files go in unzipped.
|
||||
pass
|
||||
elif expected_folders:
|
||||
# The contributor asks before forking: say where it goes.
|
||||
@@ -847,7 +914,8 @@ def main() -> None:
|
||||
{
|
||||
"file": r.filepath,
|
||||
"passed": r.passed,
|
||||
"outcome": r.outcome or "new",
|
||||
# A file that failed, unread or refused, is not a new one.
|
||||
"outcome": r.outcome or ("new" if r.passed else "failed"),
|
||||
"destination": r.destination,
|
||||
"sha1": r.sha1,
|
||||
"md5": r.md5,
|
||||
|
||||
@@ -14,7 +14,15 @@ from pathlib import Path, PurePosixPath
|
||||
import yaml
|
||||
|
||||
from common import source_yaml_paths, yaml_load
|
||||
from jsonschema import Draft202012Validator, FormatChecker
|
||||
|
||||
try:
|
||||
from jsonschema import Draft202012Validator, FormatChecker
|
||||
except ImportError:
|
||||
# Tests import this module and skip on the ImportError, the command
|
||||
# names the install pipeline.py names for the same step.
|
||||
if __name__ != "__main__":
|
||||
raise
|
||||
sys.exit('Error: jsonschema required (pip install "jsonschema[format-nongpl]")')
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCHEMAS = ROOT / "schemas"
|
||||
|
||||
+3
-3
@@ -2988,13 +2988,13 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
print(_overall_summary(group_results))
|
||||
|
||||
if args.destinations:
|
||||
held = {}
|
||||
destinations = {}
|
||||
for (result, names), (group, _rep) in zip(group_results, groups):
|
||||
for name, key in zip(names, group):
|
||||
held[name] = {
|
||||
destinations[name] = {
|
||||
"platform": key, "pack_destinations": result["pack_destinations"],
|
||||
}
|
||||
Path(args.destinations).write_text(json.dumps(held, indent=1, sort_keys=True))
|
||||
Path(args.destinations).write_text(json.dumps(destinations, indent=1, sort_keys=True))
|
||||
|
||||
if args.json:
|
||||
for r in all_results.values():
|
||||
|
||||
+11
-3
@@ -13,6 +13,7 @@ temporary directory instead of naming the missing parent.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("RETROBIOS_OFFLINE", "1")
|
||||
@@ -22,9 +23,16 @@ os.environ.setdefault(
|
||||
str(Path(__file__).resolve().parent.parent / "tmp" / "tests" / "probe.lock"),
|
||||
)
|
||||
|
||||
(Path(__file__).resolve().parent.parent / "tmp" / "tests").mkdir(
|
||||
parents=True, exist_ok=True
|
||||
)
|
||||
_SCRATCH = Path(__file__).resolve().parent.parent / "tmp" / "tests"
|
||||
_SCRATCH.mkdir(parents=True, exist_ok=True)
|
||||
# A TemporaryDirectory() without dir= landed in /tmp, a 4 GB tmpfs here: once
|
||||
# another run filled it, modules unrelated to any change failed with ENOSPC.
|
||||
# The default is the repository's tmp/, for this process and its children.
|
||||
os.environ["TMPDIR"] = str(_SCRATCH)
|
||||
tempfile.tempdir = str(_SCRATCH)
|
||||
# A scratch directory is not part of this checkout: git run inside one must
|
||||
# not walk up to the repository, as it could not from /tmp.
|
||||
os.environ["GIT_CEILING_DIRECTORIES"] = str(_SCRATCH)
|
||||
# The installer's cache (the games index, what reading a card found) is the
|
||||
# suite's own: a run never reads nor fills the one of the person running it.
|
||||
os.environ.setdefault(
|
||||
|
||||
@@ -61,6 +61,28 @@ class ArtifactLockTest(unittest.TestCase):
|
||||
with artifact_lock(self.dir, exclusive=False):
|
||||
pass
|
||||
|
||||
def test_a_reader_never_creates_the_directory(self):
|
||||
"""A mistyped --output-dir given to --verify-packs or release_record
|
||||
was created holding a .lock, and the next run with the same typo no
|
||||
longer said the path did not exist."""
|
||||
from common import ArtifactDirectoryMissing # noqa: PLC0415
|
||||
|
||||
missing = os.path.join(self.dir, "sub", "dsit")
|
||||
with self.assertRaises(ArtifactDirectoryMissing):
|
||||
with artifact_lock(missing, exclusive=False):
|
||||
pass
|
||||
self.assertFalse(os.path.exists(os.path.join(self.dir, "sub")))
|
||||
|
||||
def test_split_refuses_a_missing_target_before_locking(self):
|
||||
missing = Path(self.dir) / "sub" / "dsit"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "scripts/split_pack.py", str(missing)],
|
||||
cwd=REPO_ROOT, capture_output=True, text=True, check=False, timeout=60,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
self.assertIn("no such pack or directory", proc.stderr)
|
||||
self.assertFalse(missing.parent.exists())
|
||||
|
||||
def test_lock_is_released_on_exit(self):
|
||||
with artifact_lock(self.dir):
|
||||
pass
|
||||
|
||||
@@ -206,11 +206,16 @@ class AModuleRunAloneHasItsScratchDirectory(unittest.TestCase):
|
||||
)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-I", "-c",
|
||||
"import sys; sys.path.insert(0, '.'); import tests"],
|
||||
"import sys; sys.path.insert(0, '.'); import tests, tempfile, os;"
|
||||
" print(tempfile.gettempdir()); print(os.environ['TMPDIR'])"],
|
||||
cwd=clone, capture_output=True, text=True, timeout=60, check=False,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
self.assertTrue((clone / "tmp" / "tests").is_dir())
|
||||
# 264 TemporaryDirectory() calls without dir= wrote to the /tmp
|
||||
# tmpfs, and a full tmpfs failed modules no change touched.
|
||||
scratch = str((clone / "tmp" / "tests").resolve())
|
||||
self.assertEqual(proc.stdout.split(), [scratch, scratch])
|
||||
|
||||
|
||||
class ReleaseSigningRegressions(unittest.TestCase):
|
||||
@@ -2291,7 +2296,9 @@ class EveryManifestEntryIsFetchable(unittest.TestCase):
|
||||
else "gitignored" if repo_path in ignored
|
||||
else "untracked, never committed"
|
||||
)
|
||||
if bool(entry.get("release_asset")) != (state == "gitignored"):
|
||||
if state == "untracked, never committed" or (
|
||||
bool(entry.get("release_asset")) != (state == "gitignored")
|
||||
):
|
||||
wrong.append((entry["dest"], repo_path, state))
|
||||
self.assertEqual(wrong, [], f"{name} picks the wrong source: {wrong[:3]}")
|
||||
|
||||
@@ -2305,6 +2312,36 @@ class EveryManifestEntryIsFetchable(unittest.TestCase):
|
||||
self.skipTest("not a git checkout")
|
||||
self.assertEqual(sorted(untracked)[:20], [], f"{len(untracked)} untracked files under bios/")
|
||||
|
||||
def test_every_database_path_is_committed_or_a_release_asset(self):
|
||||
"""A git mv under bios/ committed without regenerating anything left
|
||||
database.json and the manifests agreeing on the old path: the tests
|
||||
above compare them with each other, so all passed, and the installer
|
||||
got a 404 (issue 78, with the database stale too)."""
|
||||
database = ROOT / "database.json"
|
||||
if not database.is_file():
|
||||
self.skipTest("database.json not generated")
|
||||
files = json.loads(database.read_text(encoding="utf-8"))["files"]
|
||||
paths = {
|
||||
path
|
||||
for entry in files.values()
|
||||
for path in [entry.get("path"), *(entry.get("paths") or [])]
|
||||
if path
|
||||
}
|
||||
tracked = _git_paths("ls-files", "-z")
|
||||
if tracked is None:
|
||||
self.skipTest("not a git checkout")
|
||||
outside = sorted(paths - tracked)
|
||||
ignored = _git_paths("check-ignore", "-z", "--no-index", "--stdin", stdin=outside)
|
||||
if ignored is None:
|
||||
self.skipTest("not a git checkout")
|
||||
stale = [path for path in outside if path not in ignored]
|
||||
self.assertEqual(
|
||||
stale[:20], [],
|
||||
f"{len(stale)} database paths git neither holds nor ignores: "
|
||||
"python scripts/generate_db.py --force --bios-dir bios --output database.json, "
|
||||
"then the manifests",
|
||||
)
|
||||
|
||||
|
||||
class ReleaseAssetCriterion(unittest.TestCase):
|
||||
"""What decides whether the installer fetches from the release is git, not size.
|
||||
|
||||
@@ -891,6 +891,16 @@ class BiosIntegrity(Scratch):
|
||||
self.assertEqual([(f.path, f.verdict) for f in report.findings], [(rel, bib.SKIPPED)])
|
||||
self.assertEqual((self.repo / rel).read_bytes(), b"\x02" * 80)
|
||||
|
||||
def test_an_unreadable_index_stops_before_any_restore(self):
|
||||
"""git status failed on a corrupt index, its empty stdout read as
|
||||
"unmodified", and --apply wrote HEAD's bytes over the session's."""
|
||||
rel = "bios/Sony/edited.bin"
|
||||
(self.repo / rel).write_bytes(b"\x02" * 80)
|
||||
(self.repo / ".git" / "index").write_bytes(b"garbage\n")
|
||||
with self.assertRaises(bib.GitFailed):
|
||||
bib.check(self.repo, self.db, apply=True)
|
||||
self.assertEqual((self.repo / rel).read_bytes(), b"\x02" * 80)
|
||||
|
||||
def test_a_mismatch_the_second_read_does_not_confirm_is_a_misread(self):
|
||||
reads = {}
|
||||
expected = {e["path"]: e["sha1"] for e in self.db["files"].values()}
|
||||
|
||||
@@ -35,6 +35,14 @@ class NativeCacheTests(unittest.TestCase):
|
||||
self.root.mkdir()
|
||||
self.wanted = {"list.txt": "https://example.invalid/tag-2/list.txt"}
|
||||
|
||||
def test_an_unreadable_index_is_an_error_not_unrecorded(self):
|
||||
"""Read as empty, it named "no URL recorded" for every original."""
|
||||
self.index.write_text("{", encoding="utf-8")
|
||||
findings = cf.check_native(REPO_ROOT / "platforms", self.cache, self.cache)
|
||||
self.assertEqual([(f.subject, f.status) for f in findings],
|
||||
[(self.index.name, cf.ERROR)])
|
||||
self.assertIn("--refresh-cache", findings[0].detail)
|
||||
|
||||
def _cached(self, mtime: float) -> None:
|
||||
path = self.root / "list.txt"
|
||||
path.write_text("x", encoding="utf-8")
|
||||
@@ -734,6 +742,19 @@ class RowsSayAllTheyStandFor(unittest.TestCase):
|
||||
rows = [line for line in text.splitlines() if line.startswith(" STALE")]
|
||||
self.assertEqual(rows[0].index("no longer"), rows[1].index("is not what"))
|
||||
|
||||
def test_a_remote_value_keeps_the_detail_column(self):
|
||||
"""A row with a remote value pushed its detail past the column the
|
||||
rows without one used, in [coreinfo] by some fifteen characters."""
|
||||
text = cf.render([
|
||||
cf.Finding("coreinfo", "anybor", cf.STALE, detail="no profile claims"),
|
||||
cf.Finding("coreinfo", "amiberry", cf.STALE, local="e5d1f0f9a143",
|
||||
remote="ae545d85c06f", detail=".info changed"),
|
||||
cf.Finding("coreinfo", "x", cf.STALE, local="abc", remote="d", detail="third"),
|
||||
])
|
||||
rows = [line for line in text.splitlines() if line.startswith(" STALE")]
|
||||
columns = {row.index(word) for row, word in zip(rows, ("no profile", ".info", "third"))}
|
||||
self.assertEqual(len(columns), 1, rows)
|
||||
|
||||
def test_json_carries_every_unresolved_name(self):
|
||||
missing = {f"core{i:02d}": [] for i in range(50)}
|
||||
(row,) = cf._unresolved_findings("retropie", missing)
|
||||
|
||||
@@ -1005,6 +1005,22 @@ class CachedOriginal(unittest.TestCase):
|
||||
self._upstream("v1.txt", "moved")
|
||||
self.assertEqual(fetch(url, target, self.index), b"one")
|
||||
|
||||
def test_an_unreadable_index_serves_nothing(self):
|
||||
"""A truncated index read as empty: the cached batocera-41 file was
|
||||
served for a 43.1 URL, and the next download erased every other
|
||||
platform's recorded URL."""
|
||||
import export_native
|
||||
|
||||
old = self._upstream("batocera-41.txt", "old")
|
||||
target = self.cache / "plat" / "list.txt"
|
||||
fetch(old, target, self.index)
|
||||
self.index.write_text('{"plat/list.txt": "' + old + '",', encoding="utf-8")
|
||||
new = self._upstream("batocera-43.txt", "new")
|
||||
with self.assertRaises(export_native.SourcesIndexUnreadable) as caught:
|
||||
fetch(new, target, self.index)
|
||||
self.assertIn("--refresh-cache", str(caught.exception))
|
||||
self.assertTrue(self.index.read_text(encoding="utf-8").endswith(","))
|
||||
|
||||
def test_refresh_reads_a_branch_url_again(self):
|
||||
"""A branch URL keeps its name while the platform moves."""
|
||||
url = self._upstream("branch.txt", "one")
|
||||
|
||||
@@ -102,6 +102,17 @@ class FixtureImport(unittest.TestCase):
|
||||
twin = second / path.relative_to(first)
|
||||
self.assertEqual(path.read_bytes(), twin.read_bytes(), path.name)
|
||||
|
||||
def test_a_new_fetch_date_alone_rewrites_nothing(self) -> None:
|
||||
"""A refresh rewrote every catalog for the day it ran, and the diff
|
||||
read before commit no longer showed which DAT had moved."""
|
||||
self.assertEqual(run_import("redump", self.out).returncode, 0)
|
||||
path = self.out / "redump" / "sega-megacd.json"
|
||||
doc = json.loads(path.read_text())
|
||||
doc["fetched"] = "2000-01-01"
|
||||
path.write_text(importer.dump_catalog(doc))
|
||||
self.assertEqual(run_import("redump", self.out).returncode, 0)
|
||||
self.assertEqual(json.loads(path.read_text())["fetched"], "2000-01-01")
|
||||
|
||||
def test_floor_refuses_a_short_dat(self) -> None:
|
||||
result = run_import("no-intro", self.out, "--floor", "--dry-run")
|
||||
self.assertEqual(result.returncode, 1)
|
||||
|
||||
+29
-2
@@ -36,6 +36,9 @@ _spec.loader.exec_module(install)
|
||||
sys.path.insert(0, str(REPO_ROOT / "tests"))
|
||||
from netguard import module_guard # noqa: E402
|
||||
|
||||
# What a stale pin's failure says: the fix is one command.
|
||||
PIN_ADVICE = "install.py changed: python scripts/pin_installer.py rewrites both pins"
|
||||
|
||||
# No connection off the loopback and no card of the host reaches a test:
|
||||
# the module fails at its end otherwise.
|
||||
setUpModule, tearDownModule = module_guard("test_install")
|
||||
@@ -2201,7 +2204,7 @@ class TestAvailablePlatforms(unittest.TestCase):
|
||||
match = re.search(r'\$defaultInstallSha256 = "([0-9a-f]{64})"', content)
|
||||
self.assertIsNotNone(match, "install.ps1 must embed the installer SHA256")
|
||||
expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
|
||||
self.assertEqual(match.group(1), expected)
|
||||
self.assertEqual(match.group(1), expected, PIN_ADVICE)
|
||||
|
||||
def test_powershell_wrapper_rejects_non_https_bootstrap(self):
|
||||
content = (REPO_ROOT / "install.ps1").read_text()
|
||||
@@ -2212,7 +2215,31 @@ class TestAvailablePlatforms(unittest.TestCase):
|
||||
match = re.search(r'DEFAULT_INSTALL_SHA256="([0-9a-f]{64})"', content)
|
||||
self.assertIsNotNone(match, "install.sh must embed the installer SHA256")
|
||||
expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
|
||||
self.assertEqual(match.group(1), expected)
|
||||
self.assertEqual(match.group(1), expected, PIN_ADVICE)
|
||||
|
||||
def test_pin_script_writes_both_pins(self):
|
||||
"""Every edit to install.py meant a sha256sum pasted by hand into two
|
||||
files with two syntaxes, and the failing test named no command."""
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
import pin_installer # noqa: PLC0415
|
||||
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory:
|
||||
root = Path(directory)
|
||||
for name in ("install.py", "install.sh", "install.ps1"):
|
||||
shutil.copy2(REPO_ROOT / name, root / name)
|
||||
(root / "install.py").write_text("# changed\n", encoding="utf-8")
|
||||
sh_mode = (root / "install.sh").stat().st_mode
|
||||
self.assertEqual(pin_installer.stale_pins(root), ["install.sh", "install.ps1"])
|
||||
self.assertEqual(pin_installer.write_pins(root), ["install.sh", "install.ps1"])
|
||||
self.assertEqual(pin_installer.stale_pins(root), [])
|
||||
digest = hashlib.sha256(b"# changed\n").hexdigest()
|
||||
for name in ("install.sh", "install.ps1"):
|
||||
before = (REPO_ROOT / name).read_text(encoding="utf-8").splitlines()
|
||||
after = (root / name).read_text(encoding="utf-8").splitlines()
|
||||
changed = [b for a, b in zip(before, after) if a != b]
|
||||
self.assertEqual(len(changed), 1, name)
|
||||
self.assertIn(digest, changed[0])
|
||||
self.assertEqual((root / "install.sh").stat().st_mode, sh_mode)
|
||||
|
||||
def test_shell_one_liner_downloads_verifies_and_forwards_arguments(self):
|
||||
scratch = REPO_ROOT / "tmp" / "tests"
|
||||
|
||||
@@ -373,6 +373,8 @@ class Helpers(unittest.TestCase):
|
||||
self.assertEqual(el.emulator_of_script("emuDeckXemu.sh"), "xemu")
|
||||
self.assertIsNone(el.emulator_of_script("RetroArch_maincfg.sh"))
|
||||
self.assertIsNone(el.emulator_of_script("README.md"))
|
||||
self.assertEqual(el.emulator_of_script("emuDeckPCSX2QT.ps1"), "pcsx2")
|
||||
self.assertIsNone(el.emulator_of_script("emuDeckTemplate.ps1"))
|
||||
|
||||
def test_the_revision_comes_from_the_pinned_url(self):
|
||||
url = (
|
||||
|
||||
@@ -118,6 +118,35 @@ class RefreshConcurrency(unittest.TestCase):
|
||||
self.assertTrue(finished, "the refresh waited for a reader")
|
||||
self.assertEqual(result, [False])
|
||||
|
||||
def test_a_waiting_refresh_names_the_lock_and_the_way_around(self):
|
||||
""""waiting for data/dolphin-sys: another run holds it" named
|
||||
neither the lock nor its holder, and the build waited unbounded."""
|
||||
import fcntl # noqa: PLC0415
|
||||
|
||||
self.cache.mkdir(parents=True)
|
||||
lock_path = self.cache.with_name(".pak.lock")
|
||||
entered = threading.Event()
|
||||
|
||||
def refresh() -> None:
|
||||
with rdd.cache_lock(self.cache):
|
||||
entered.set()
|
||||
|
||||
with open(lock_path, "a") as reader, self.assertLogs(rdd.log, "WARNING") as logs:
|
||||
fcntl.flock(reader, fcntl.LOCK_SH)
|
||||
worker = threading.Thread(target=refresh, daemon=True)
|
||||
worker.start()
|
||||
for _ in range(200):
|
||||
if logs.records:
|
||||
break
|
||||
worker.join(timeout=0.05)
|
||||
self.assertFalse(entered.is_set())
|
||||
fcntl.flock(reader, fcntl.LOCK_UN)
|
||||
worker.join(timeout=20)
|
||||
self.assertTrue(entered.is_set())
|
||||
message = logs.records[0].getMessage()
|
||||
self.assertIn(f"lsof {lock_path}", message)
|
||||
self.assertIn("--no-refresh-data", message)
|
||||
|
||||
def test_version_file_is_never_left_truncated(self):
|
||||
Path(self.versions).parent.mkdir(parents=True)
|
||||
Path(self.versions).write_text(json.dumps({"keep": {"sha": "1"}}))
|
||||
|
||||
@@ -162,6 +162,11 @@ class Planning(unittest.TestCase):
|
||||
self.assertEqual(jobs, [])
|
||||
self.assertEqual(len(surfaced), 3)
|
||||
|
||||
def test_the_installer_pin_is_written_by_its_script(self):
|
||||
jobs, surfaced, _ = self._plan(finding("ci", "install.py pin"))
|
||||
self.assertEqual([job.command[1:] for job in jobs], [["scripts/pin_installer.py"]])
|
||||
self.assertEqual(surfaced, [])
|
||||
|
||||
|
||||
class Running(unittest.TestCase):
|
||||
def test_a_child_starts_after_its_parent_ends(self):
|
||||
@@ -234,6 +239,38 @@ class SavedFindings(unittest.TestCase):
|
||||
findings = rs.load_findings(path, ("data", "catalogs"))
|
||||
self.assertEqual([f["subject"] for f in findings], ["dolphin-sys"])
|
||||
|
||||
def test_dry_run_json_prints_the_plan_as_json(self):
|
||||
"""--dry-run --json printed the text plan, and a caller parsing it
|
||||
as JSON failed on the first line."""
|
||||
import json # noqa: PLC0415
|
||||
import subprocess # noqa: PLC0415
|
||||
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory:
|
||||
path = Path(directory) / "freshness.json"
|
||||
path.write_text(json.dumps([
|
||||
finding("catalogs", "mame recipes"), finding("coreinfo", "anybor"),
|
||||
]), encoding="utf-8")
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "scripts/refresh_stale.py", "--findings", str(path),
|
||||
"--dry-run", "--json"],
|
||||
cwd=REPO_ROOT, capture_output=True, text=True, check=False, timeout=120,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
plan = json.loads(proc.stdout)
|
||||
self.assertEqual([(j["area"], j["subject"]) for j in plan["planned"]],
|
||||
[("catalogs", "mame recipes")])
|
||||
self.assertEqual([f["subject"] for f in plan["surfaced"]], ["anybor"])
|
||||
|
||||
def test_every_area_check_freshness_reports_is_known(self):
|
||||
"""romfolders was added to check_freshness and not here: its rows
|
||||
were dropped from a saved report without a word, and --only
|
||||
romfolders was refused."""
|
||||
import check_freshness # noqa: PLC0415
|
||||
|
||||
self.assertEqual(set(check_freshness.AREAS), set(rs.AUTO_AREAS + rs.MANUAL_AREAS))
|
||||
jobs, surfaced, _ = rs.plan_jobs([finding("romfolders", "muos")], REGISTRY, {})
|
||||
self.assertEqual((jobs, [f["subject"] for f in surfaced]), ([], ["muos"]))
|
||||
|
||||
def test_a_report_that_is_not_a_list_is_refused(self):
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory:
|
||||
path = Path(directory) / "freshness.json"
|
||||
|
||||
@@ -120,5 +120,31 @@ class DataRootIsKeyedBySystem(unittest.TestCase):
|
||||
self.assertTrue(self._errors({}))
|
||||
|
||||
|
||||
class WithoutJsonschema(unittest.TestCase):
|
||||
"""The documented command stopped on a bare ImportError traceback that
|
||||
named neither the install nor the format-nongpl extra."""
|
||||
|
||||
def test_the_command_names_the_install(self):
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
scratch = REPO_ROOT / "tmp" / "tests"
|
||||
scratch.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(dir=scratch) as directory:
|
||||
stub = Path(directory) / "jsonschema"
|
||||
stub.mkdir()
|
||||
(stub / "__init__.py").write_text(
|
||||
'raise ImportError("No module named jsonschema")\n', encoding="utf-8"
|
||||
)
|
||||
env = {**os.environ, "PYTHONPATH": directory}
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "scripts/validate_schemas.py", "--source-only"],
|
||||
capture_output=True, text=True, check=False, cwd=REPO_ROOT, env=env,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 1)
|
||||
self.assertNotIn("Traceback", proc.stderr)
|
||||
self.assertIn('pip install "jsonschema[format-nongpl]"', proc.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -724,7 +724,7 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase):
|
||||
with mock.patch("urllib.request.urlopen", self._refuse), self.assertRaises(
|
||||
RuntimeError
|
||||
):
|
||||
emudeck_scraper.Scraper()._fetch_installed_emulators()
|
||||
emudeck_scraper.Scraper(checkbios_url="pinned")._fetch_installed_emulators("0" * 40)
|
||||
|
||||
def test_libretro_core_metadata_raises(self):
|
||||
"""libretro dropped every system name and manufacturer when the
|
||||
|
||||
@@ -509,6 +509,23 @@ class TestSelfContradictingDestinations(unittest.TestCase):
|
||||
self.assertEqual(len(collisions), 1)
|
||||
self.assertEqual(len(collisions[0].resolved), 2)
|
||||
|
||||
def test_the_line_names_each_system_and_the_file_packed(self):
|
||||
"""The line named two files and neither the system that declares
|
||||
each nor the one the pack ships: the reader went to the YAML."""
|
||||
config = {"systems": {
|
||||
"zeta": {"files": [{"name": "IPL.bin", "destination": "disk.rom",
|
||||
"md5": "m" * 32}]},
|
||||
"alpha": {"files": [{"name": "IPL.bin", "destination": "disk.rom",
|
||||
"md5": "n" * 32}]},
|
||||
}}
|
||||
collision = slots.find_collisions(config, REGIONS_DB)[0]
|
||||
line = slots.format_collision(collision)
|
||||
left, packed = collision.resolved
|
||||
# The builder walks the systems sorted: alpha's declaration ships.
|
||||
self.assertIn(f"{packed} (alpha) packed", line)
|
||||
self.assertIn(f"{left} (zeta) left out", line)
|
||||
self.assertEqual(collision.shipped, packed)
|
||||
|
||||
def test_one_archive_named_by_its_inner_roms_is_not_a_collision(self):
|
||||
# The documented zipped_file pattern: several md5s, one archive.
|
||||
config = self._config(
|
||||
|
||||
@@ -541,6 +541,59 @@ class TargetsReadThePlatformRevision(unittest.TestCase):
|
||||
self.assertTrue(steamos)
|
||||
self.assertTrue(all(u.endswith(f"?ref={pin}") for u in steamos), steamos)
|
||||
|
||||
def test_emudeck_platform_lists_its_installers_at_the_pin(self):
|
||||
"""cores: was read on master while launchers: and the SteamOS target
|
||||
were read at the pin: the YAML described two revisions."""
|
||||
from scripts.scraper import emudeck_scraper
|
||||
|
||||
pin = "d" * 40
|
||||
seen: list[str] = []
|
||||
|
||||
class _Body:
|
||||
def __init__(self, url):
|
||||
seen.append(url)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return json.dumps([{"name": "emuDeckRMG.sh"}]).encode()
|
||||
|
||||
with mock.patch("urllib.request.urlopen", lambda req, **_kw: _Body(req.full_url)):
|
||||
cores = emudeck_scraper.Scraper(checkbios_url="pinned")._fetch_installed_emulators(pin)
|
||||
self.assertEqual(cores, ["mupen64plus_next"])
|
||||
self.assertEqual(len(seen), 1)
|
||||
self.assertTrue(seen[0].endswith(f"?ref={pin}"), seen)
|
||||
|
||||
|
||||
class EmuDeckScriptNamesHaveOneRule(unittest.TestCase):
|
||||
"""The target scraper kept its own copy of the EmuScripts-to-profile
|
||||
table: it published emudeck-we's template skeleton as a core, and a
|
||||
rename added to one copy only made --target filter that core out."""
|
||||
|
||||
def test_the_target_scraper_names_scripts_as_the_platform_does(self):
|
||||
from scripts.scraper.emudeck_launchers import emulator_of_script
|
||||
|
||||
listing = [
|
||||
"emuDeckRMG.sh",
|
||||
"emuDeckPCSX2QT.ps1",
|
||||
"emuDeckTemplate.ps1",
|
||||
"emuDeckRetroArch.ps1",
|
||||
"RetroArch_maincfg.sh",
|
||||
"emuDeckXemu.ps1",
|
||||
]
|
||||
scraper = emudeck_targets_scraper.Scraper(revision="0" * 40)
|
||||
with mock.patch.object(
|
||||
emudeck_targets_scraper, "_list_emuscripts", return_value=listing
|
||||
), mock.patch.object(scraper, "_load_retroarch_cores", return_value=["mgba"]):
|
||||
cores = scraper._fetch_cores_for_target("unused", "Windows")
|
||||
standalone = {emulator_of_script(name) for name in listing} - {None, "retroarch"}
|
||||
self.assertEqual(standalone, {"mupen64plus_next", "pcsx2", "xemu"})
|
||||
self.assertEqual(cores, sorted(standalone | {"mgba"}))
|
||||
|
||||
|
||||
class HashScrapersRefuseAnUnknownVersion(unittest.TestCase):
|
||||
"""mame_hash_scraper wrote core_version "unknown" after a failed API call
|
||||
|
||||
@@ -60,6 +60,22 @@ class ATargetedModelIsItsOwnArtifact(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
def test_a_named_platform_without_the_target_is_refused(self):
|
||||
"""bizhawk --target switch printed "skipped", then Done., exit 0,
|
||||
and wrote no model: the diff after it failed instead."""
|
||||
repo = pathlib.Path(__file__).resolve().parent.parent
|
||||
with tempfile.TemporaryDirectory(dir=str(repo / "tmp")) as directory:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "scripts/generate_truth.py", "--platform", "bizhawk",
|
||||
"--target", "switch", "--output-dir", directory],
|
||||
capture_output=True, check=False, text=True, cwd=str(repo), timeout=400,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
|
||||
self.assertIn("bizhawk has no hardware targets", proc.stderr)
|
||||
self.assertNotIn("Done.", proc.stdout)
|
||||
self.assertEqual(list(pathlib.Path(directory).iterdir()), [])
|
||||
|
||||
|
||||
class ADiffThatComparedNothingFails(unittest.TestCase):
|
||||
"""diff_truth --all skipped every platform without a model and exited 0."""
|
||||
|
||||
|
||||
@@ -269,6 +269,29 @@ class HashRecognition(_Fixture):
|
||||
self.assertIn("no platform list or emulator profile declares this file", markdown)
|
||||
self.assertIn("which emulator reads it", markdown)
|
||||
|
||||
def test_a_zip_of_bios_files_is_read_member_by_member(self):
|
||||
"""A card's BIOS folder zipped to upload it was hashed as one file,
|
||||
and its contributor was asked where that dump came from."""
|
||||
import hashlib # noqa: PLC0415
|
||||
import zipfile # noqa: PLC0415
|
||||
|
||||
held, declared = b"HELD BIOS", b"DECLARED BIOS"
|
||||
with zipfile.ZipFile(self.tmp / "mybios.zip", "w") as archive:
|
||||
archive.writestr("scph5501.bin", held)
|
||||
archive.writestr("sub/declared.bin", declared)
|
||||
archive.writestr("other.bin", b"noise")
|
||||
db = {"files": {hashlib.sha1(held).hexdigest(): {"path": "bios/Sony/PS/scph5501.bin"}}}
|
||||
known = {**EMPTY_HASHES, "md5": {hashlib.md5(declared).hexdigest()}}
|
||||
result = validate_pr.validate_file("mybios.zip", db, known)
|
||||
markdown = result.to_markdown()
|
||||
self.assertEqual(result.outcome, "unknown")
|
||||
self.assertIn("unzipped, not the archive", markdown)
|
||||
self.assertIn("`scph5501.bin` already held at `bios/Sony/PS/scph5501.bin`", markdown)
|
||||
self.assertIn("`sub/declared.bin` declared and not held", markdown)
|
||||
self.assertIn("`other.bin` not recognised", markdown)
|
||||
self.assertNotIn("where the dump comes from", markdown)
|
||||
self.assertNotIn("Place it in", markdown)
|
||||
|
||||
def test_a_duplicate_of_a_known_file_is_reported(self):
|
||||
"""A copy of a held file passed, and was sent to the folder that
|
||||
already holds it."""
|
||||
@@ -522,6 +545,15 @@ class FlagsAreAppliedOrRefused(unittest.TestCase):
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn("not allowed with", result.stderr)
|
||||
|
||||
def test_a_failed_file_is_not_reported_new(self):
|
||||
"""A path that does not exist came out as "outcome": "new", and a
|
||||
reader listing new contributions announced a BIOS nobody sent."""
|
||||
import json # noqa: PLC0415
|
||||
|
||||
result = self._run("--json", "bios/nope.bin")
|
||||
report = json.loads(result.stdout)
|
||||
self.assertEqual([(r["passed"], r["outcome"]) for r in report], [(False, "failed")])
|
||||
|
||||
|
||||
|
||||
class AFileOutsideBiosIsToldWhereToGo(unittest.TestCase):
|
||||
|
||||
@@ -173,6 +173,23 @@ class TheDestinationsFlagNeedsAPack(unittest.TestCase):
|
||||
self.assertEqual(result.returncode, 2, result.stdout[-300:])
|
||||
self.assertIn("--destinations", result.stderr)
|
||||
|
||||
def test_the_lock_stack_is_not_rebound(self):
|
||||
"""The --destinations branch reused `held`, the ExitStack that holds
|
||||
the data directory locks, for its dict: a later enter_context in
|
||||
that mode raised AttributeError."""
|
||||
import ast
|
||||
|
||||
tree = ast.parse((REPO_ROOT / "scripts" / "verify.py").read_text(encoding="utf-8"))
|
||||
body = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_main")
|
||||
stack = body.args.args[0].arg
|
||||
rebound = [
|
||||
node.lineno for node in ast.walk(body)
|
||||
if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign))
|
||||
for target in (node.targets if isinstance(node, ast.Assign) else [node.target])
|
||||
if isinstance(target, ast.Name) and target.id == stack
|
||||
]
|
||||
self.assertEqual(rebound, [])
|
||||
|
||||
|
||||
class EveryGapCountedIsNamed(unittest.TestCase):
|
||||
"""verify --platform batocera --verbose counted 3 optional core files and
|
||||
|
||||
+3
-1
@@ -1702,7 +1702,9 @@ a real 3.8 interpreter, which CI installs beside 3.12 so the check cannot skip.
|
||||
The build scripts need 3.9 and are not bound by this.
|
||||
|
||||
Every change to `install.py` recomputes the SHA-256 pinned in `install.sh` and
|
||||
`install.ps1` in the same commit. The tests compare the pins with the file.
|
||||
`install.ps1` in the same commit: `python scripts/pin_installer.py` writes both
|
||||
(`--check` writes nothing and exits 1 on a stale pin). The tests compare the
|
||||
pins with the file and name that command when they differ.
|
||||
|
||||
The tables of options, events, exit codes, keys and card markers, the line
|
||||
menu and the run samples on this page sit between `generated` markers and are
|
||||
|
||||
@@ -9,9 +9,11 @@ The tests hold what users and platforms rely on: a pack holds what its
|
||||
platform checks, the site states what the code does, the installer writes only
|
||||
where it should.
|
||||
|
||||
No test needs network access. Most build synthetic fixtures under the
|
||||
repository's `tmp/`, and modules that read real artifacts skip when those are
|
||||
absent. The test runner prints the count.
|
||||
No test needs network access. Most build synthetic fixtures, and every
|
||||
temporary directory lands under the repository's `tmp/tests/`:
|
||||
`tests/__init__.py` points `tempfile` and `TMPDIR` there, never at the `/tmp`
|
||||
tmpfs. Modules that read real artifacts skip when those are absent. The test
|
||||
runner prints the count.
|
||||
|
||||
## Running tests
|
||||
|
||||
|
||||
+5
-1
@@ -760,7 +760,10 @@ committed.
|
||||
|
||||
Rows that need a decision are listed and never run: a buildbot core with no
|
||||
profile, a new `.info` file, the Redump, No-Intro and TOSEC packs, the CI
|
||||
pins, and `profile_sync`. A failed scraper leaves its cached original
|
||||
pins, `profile_sync`, and a rom-folder pin (moved by hand in
|
||||
`platforms/romfolders/_sources/`, then `python -m scripts.scraper.romfolders
|
||||
--setup NAME` reads it again). `--dry-run --json` prints the plan as JSON.
|
||||
A failed scraper leaves its cached original
|
||||
untouched. Each job writes its output to `tmp/refresh_stale/`, and a GitHub
|
||||
token is taken from `GITHUB_TOKEN` or from `gh auth token`.
|
||||
|
||||
@@ -1343,6 +1346,7 @@ before a release (wiki/release-process.md). Its lines stay in
|
||||
| `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs |
|
||||
| `generate_event_schema.py` | Write `schemas/install-events.schema.json` from `EVENT_KEYS` in `install.py` (`--check` exits 1 when the file is stale, which `validate_schemas.py` also reports) |
|
||||
| `generate_installer_doc.py` | Rewrite the generated blocks of `wiki/installer.md` from `install.py`: the tables (options, events, exit codes, keys per route, card markers), the line menu, and the run samples `install.py` prints in a sandbox against the committed `install/*.json`, so a new list rewrites the page (`--check` exits 1 when a block is stale, `--page FILE` writes another copy of the page). Pipeline step 7 |
|
||||
| `pin_installer.py` | Write the SHA-256 of `install.py` into the pins of `install.sh` and `install.ps1`, which refuse any other installer (`--check` writes nothing and exits 1 when a pin is stale). `check_freshness.py` reports a stale pin and `refresh_stale.py` runs this |
|
||||
| `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline). A download is accepted only when every cryptographic hash the entry declares (sha256, sha1, md5) matches. crc32 and size find a file the database already holds, never accept a download. An entry with no cryptographic hash is reported and not searched |
|
||||
| `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones) |
|
||||
| `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) |
|
||||
|
||||
Reference in new issue
Block a user