chore: merge tooling fixes, fourth pass

This commit is contained in:
Abdessamad Derraz committed 2026-11-06 09:50:16 +01:00
commit e491ff1bc6
45 files changed
+801 -153

No files matched your search

+4
View File
@@ -22,6 +22,10 @@ the most useful contribution.
hash differs from the one already held under its name goes to that folder's
`.variants/`.
A ZIP that no list declares as a whole is read file by file, and the report
says which of them the collection already holds: upload the files
themselves, unzipped.
From a clone, the file goes to `bios/Manufacturer/Console/filename`, and
`python scripts/validate_pr.py path/to/file` names that folder for a file still
outside `bios/`, and `--place` copies it there under its declared name. Given a
+2
View File
@@ -83,6 +83,7 @@
"galaksija",
"gam4980",
"gambatte",
"gametank",
"gearboy",
"gearcoleco",
"geargrafx",
@@ -346,6 +347,7 @@
"galaksija",
"gam4980",
"gambatte",
"gametank",
"gearboy",
"gearcoleco",
"geargrafx",
-7
View File
@@ -121,13 +121,6 @@ retrodeck:
# includes the frontend name alongside the actual cores.
remove_cores: [retroarch]
emudeck:
targets:
_default:
# EmuDeck's EmuScripts tree carries a template.sh placeholder used
# to scaffold new installers. It is not an emulator.
remove_cores: [template]
lakka:
targets:
generic:
+4 -3
View File
@@ -1,6 +1,6 @@
platform: emudeck
source: https://github.com/dragoonDorise/EmuDeck
scraped_at: '2026-10-17T18:43:57Z'
source: https://github.com/dragoonDorise/EmuDeck/tree/dddb30229adbbd54d5ff7283cfd70e11c4322c26
scraped_at: '2026-11-06T07:10:22Z'
targets:
steamos:
architecture: x86_64
@@ -88,6 +88,7 @@ targets:
- galaksija
- gam4980
- gambatte
- gametank
- gearboy
- gearcoleco
- geargrafx
@@ -352,6 +353,7 @@ targets:
- galaksija
- gam4980
- gambatte
- gametank
- gearboy
- gearcoleco
- geargrafx
@@ -487,7 +489,6 @@ targets:
- supermodel
- swanstation
- tamalibretro
- template
- tgbdual
- theodore
- thepowdertoy
+13
View File
@@ -19,6 +19,7 @@ import re
_TIMESTAMP_PATTERNS = [
re.compile(r'"generated_at":\s*"[^"]*"'), # database.json
re.compile(r'"imported_at":\s*"[^"]*"'), # provenance snapshots
re.compile(r'"fetched":\s*"[^"]*"'), # games/catalogs/
re.compile(r"\*Auto-generated on [^*]*\*"), # README.md
re.compile(r"\*Generated on [^*]*\*"), # docs site pages
# The decorated pages carry the same stamp again as a rendered element,
@@ -208,6 +209,10 @@ def _strip_timestamps(text: str) -> str:
class ArtifactLockBusy(RuntimeError):
"""Raised when another process already holds the artifact directory."""
class ArtifactDirectoryMissing(FileNotFoundError):
"""A reader named a directory that does not exist."""
# A run that holds artifact directories for its whole duration names them
# here, os.pathsep apart, for the steps it spawns: they would otherwise be
# refused by it. A pipeline holds the repository root and its --output-dir.
@@ -227,6 +232,14 @@ def artifact_lock(directory: str, exclusive: bool = True) -> Iterator[None]:
Writers take the lock exclusively, readers share it. On platforms
without flock the lock is a no-op.
"""
# A reader creating the directory it was pointed at left a typo behind
# as a directory holding a .lock, and the next run with the same typo
# no longer said the path did not exist.
if not exclusive and not os.path.isdir(directory):
raise ArtifactDirectoryMissing(
f"{directory}: no such directory, pass the --output-dir the packs "
"were written to (default dist/)"
)
try:
import fcntl
except ImportError:
+17 -2
View File
@@ -91,8 +91,19 @@ def sha1_of(path: Path) -> str:
return digest.hexdigest()
class GitFailed(RuntimeError):
"""A git command failed: its empty output says nothing about the file."""
def git(root: Path, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
"""Run git in root. A failure raises: an unreadable index read as an empty
status made a session's working change look unmodified, and --apply wrote
HEAD's bytes over it."""
done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
if done.returncode:
message = done.stderr.decode(errors="replace").strip()
raise GitFailed(f"git {' '.join(args)} exited {done.returncode}: {message}")
return done
def head_blob(root: Path, path: str) -> str:
@@ -197,7 +208,11 @@ def main() -> int:
from common import load_database
root = Path.cwd()
report = check(root, load_database(args.db), args.apply)
try:
report = check(root, load_database(args.db), args.apply)
except GitFailed as exc:
print(f"ERROR: {exc}. Nothing was restored past this point.", file=sys.stderr)
return 2
if args.json:
print(json.dumps(report.as_dict(), indent=2))
else:
+20 -17
View File
@@ -32,7 +32,6 @@ from __future__ import annotations
import argparse
import contextlib
import hashlib
import io
import json
import re
@@ -51,6 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import export_native # noqa: E402
import pin_installer # noqa: E402
import refresh_data_dirs # noqa: E402
import upstream # noqa: E402
from common import ( # noqa: E402
@@ -83,7 +83,6 @@ PYPI_URL = "https://pypi.org/pypi/{name}/json"
NPM_URL = "https://registry.npmjs.org/{name}/latest"
# The site's npm dependencies, pinned to exact versions and built by Deploy Site.
SITE_PACKAGE = REPO_ROOT / "website" / "package.json"
INSTALLERS = ("install.sh", "install.ps1")
SHOWN_ITEMS = 4 # items named in a diff summary before ", ..."
SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..."
@@ -509,7 +508,10 @@ def check_native(
) -> list[Finding]:
exporters = discover_exporters()
index = cache_dir / export_native.SOURCES_INDEX
recorded = export_native.load_sources(cache_dir)
try:
recorded = export_native.load_sources(cache_dir)
except export_native.SourcesIndexUnreadable as exc:
return [Finding("native", index.name, ERROR, detail=str(exc))]
findings: list[Finding] = []
for name in list_registered_platforms(str(platforms_dir), include_archived=True):
exporter_class = exporters.get(name)
@@ -1462,18 +1464,16 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]:
def _check_installer_pins() -> Finding:
"""The bootstraps pin install.py by SHA-256, so a stale pin refuses every install."""
digest = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
stale = []
for name in INSTALLERS:
text = (REPO_ROOT / name).read_text(encoding="utf-8", errors="replace")
if digest not in text:
stale.append(name)
stale = pin_installer.stale_pins(REPO_ROOT)
return Finding(
"ci",
"install.py pin",
STALE if stale else OK,
local=digest[:12],
detail=("pin outdated in " + ", ".join(stale)) if stale else "install.sh and install.ps1 match",
local=pin_installer.installer_sha256(REPO_ROOT)[:12],
detail=(
f"pin outdated in {', '.join(stale)}, {pin_installer.COMMAND} writes it"
if stale else "install.sh and install.ps1 match"
),
)
@@ -1757,15 +1757,18 @@ def render(findings: list[Finding]) -> str:
# Columns are padded, never cut: the profile_sync row says in its
# local and remote values what to re-read, and lost the end of it.
local_width = min(max((len(f.local) for f in rows), default=0), 44)
remote_width = min(max((len(f.remote) for f in rows), default=0), 44)
# The arrow and the remote value, or as many blanks where a row has none.
remote_span = 4 + remote_width if remote_width else 0
for f in rows:
cols = f" {f.status:7} {f.subject:{width}}"
if f.remote:
cols += f" {f.local:{local_width}} -> {f.remote}"
elif f.local or (f.detail and local_width):
# An arrow pointing at nothing read as a lost value. An empty
# local value still takes its column, or the detail starts
# where the other rows show theirs.
cols += f" {f.local:{local_width}}"
cols += f" {f.local:{local_width}} -> {f.remote:{remote_width}}"
elif f.local or (f.detail and (local_width or remote_span)):
# An arrow pointing at nothing read as a lost value. Empty
# local and remote values still take their columns, or the
# detail starts where the other rows show theirs.
cols += f" {f.local:{local_width}}" + " " * remote_span
if f.detail:
cols += f" {f.detail}"
lines.append(cols.rstrip())
+2 -1
View File
@@ -3587,6 +3587,7 @@ from artifacts import ( # noqa: E402,F401
copy_file_atomic,
plain_file_mode,
set_plain_mode,
ArtifactDirectoryMissing,
ArtifactLockBusy,
artifact_lock,
hold_artifact_lock,
@@ -3634,7 +3635,7 @@ def run_cli(main: Callable[[], int | None]) -> None:
try:
code = main()
sys.stdout.flush()
except (InputMissing, ArtifactLockBusy) as exc:
except (InputMissing, ArtifactLockBusy, ArtifactDirectoryMissing) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
sys.exit(1)
except BrokenPipeError:
+17 -2
View File
@@ -36,6 +36,7 @@ from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_tex
import yaml
from common import (
InputMissing,
build_zip_contents_index,
platform_arg,
list_registered_platforms,
@@ -62,13 +63,27 @@ _USER_AGENT = "retrobios-exporter/1.0"
_MAX_BYTES = 64 * 1024 * 1024
class SourcesIndexUnreadable(InputMissing):
"""The cache's URL index exists and does not read as JSON."""
def _load_sources(index: Path | None) -> dict[str, str]:
"""The URL each cached file was fetched from.
An absent index records nothing. An index that does not read raises:
read as empty it let every cached original pass for current, and the
next download rewrote it with that one entry.
"""
if index is None or not index.is_file():
return {}
try:
return json.loads(index.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
except (OSError, json.JSONDecodeError) as exc:
raise SourcesIndexUnreadable(
f"{index} does not read ({exc}): remove it, then"
" python scripts/export_native.py --all --refresh-cache"
" downloads every original again and records its URL"
) from exc
def source_key(destination: Path, index: Path | None) -> str:
+4
View File
@@ -941,6 +941,10 @@ the most useful contribution.
hash differs from the one already held under its name goes to that folder's
`.variants/`.
A ZIP that no list declares as a whole is read file by file, and the report
says which of them the collection already holds: upload the files
themselves, unzipped.
From a clone, the file goes to `bios/Manufacturer/Console/filename`, and
`python scripts/validate_pr.py path/to/file` names that folder for a file still
outside `bios/`, and `--place` copies it there under its declared name. Given a
+15 -12
View File
@@ -14,13 +14,13 @@ import sys
sys.path.insert(0, os.path.dirname(__file__))
from common import (
build_target_cores_cache,
platform_arg,
list_registered_platforms,
load_data_dir_registry,
load_database,
load_emulator_profiles,
load_platform_config,
load_target_config,
require_directory,
require_platform,
require_yaml,
@@ -97,6 +97,19 @@ def main(argv: list[str] | None = None) -> None:
require_platform(args.platform, args.platforms_dir)
platforms = [args.platform]
# A named platform without that target is refused, as verify and
# generate_pack refuse it: skipped, it printed Done. and exited 0 with
# no model written. --all leaves such platforms out and says so.
target_cores_cache: dict[str, set[str] | None] = {}
if args.target:
try:
target_cores_cache, platforms = build_target_cores_cache(
platforms, args.target, args.platforms_dir, is_all=args.all
)
except (FileNotFoundError, ValueError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
sys.exit(1)
# A target-narrowed model is a different artifact: written under the
# platform's own name it overwrote the full one, and diff_truth then
# compared a narrowed model against the whole scrape. Its directory is
@@ -109,17 +122,7 @@ def main(argv: list[str] | None = None) -> None:
output_dir, [name], args.target, args.platforms_dir
)
os.makedirs(output_dir, exist_ok=True)
target_cores: set[str] | None = None
if args.target:
try:
target_cores = load_target_config(
name,
args.target,
args.platforms_dir,
)
except FileNotFoundError:
print(f" {name}: no target config, skipped")
continue
target_cores = target_cores_cache.get(name)
# Load platform config (with inheritance) and registry entry
try:
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env python3
"""Write the SHA-256 of install.py into the bootstraps that pin it.
install.sh and install.ps1 refuse an installer whose hash is not the one
they carry, so every change to install.py moves both pins in the same
commit. --check writes nothing and exits 1 naming each stale pin.
Usage:
python scripts/pin_installer.py
python scripts/pin_installer.py --check
"""
from __future__ import annotations
import argparse
import hashlib
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
INSTALLER = "install.py"
# Each bootstrap and its one pin line, the value between the two groups.
PINS: dict[str, re.Pattern[str]] = {
"install.sh": re.compile(r'^(DEFAULT_INSTALL_SHA256=")[0-9a-f]{64}(")', re.M),
"install.ps1": re.compile(r'^(\s*\$defaultInstallSha256 = ")[0-9a-f]{64}(")', re.M),
}
COMMAND = "python scripts/pin_installer.py"
def installer_sha256(root: Path = ROOT) -> str:
return hashlib.sha256((root / INSTALLER).read_bytes()).hexdigest()
def stale_pins(root: Path = ROOT) -> list[str]:
"""The bootstraps whose pin is not the SHA-256 of install.py."""
digest = installer_sha256(root)
stale = []
for name, pin in PINS.items():
found = pin.search((root / name).read_bytes().decode("utf-8"))
if found is None:
raise ValueError(f"{name}: no installer pin line")
if found.group(0) != f"{found.group(1)}{digest}{found.group(2)}":
stale.append(name)
return stale
def write_pins(root: Path = ROOT) -> list[str]:
"""Rewrite each stale pin. Returns the bootstraps written."""
digest = installer_sha256(root)
written = []
for name in stale_pins(root):
path = root / name
text = path.read_bytes().decode("utf-8")
new, count = PINS[name].subn(rf"\g<1>{digest}\g<2>", text)
if count != 1:
raise ValueError(f"{name}: {count} installer pin lines, expected 1")
path.write_bytes(new.encode("utf-8"))
written.append(name)
return written
def main() -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
parser.add_argument("--check", action="store_true",
help="write nothing, exit 1 if a pin is stale")
args = parser.parse_args()
if args.check:
stale = stale_pins()
for name in stale:
print(f"{name}: pin is not the SHA-256 of {INSTALLER}, {COMMAND} writes it",
file=sys.stderr)
return 1 if stale else 0
for name in write_pins():
print(f"{name}: pinned {installer_sha256()}")
return 0
if __name__ == "__main__":
from common import run_cli
run_cli(main)
+19 -1
View File
@@ -71,6 +71,24 @@ def cache_lock(
return _announced_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared, cache_dir)
def _wait_message(lock_path: Path, shared: bool, cache_dir: Path) -> str:
"""Who to wait for, and, for a refresh, how not to.
The bare "another run holds it" named neither the lock nor its holder,
and the build waited with no limit.
"""
message = (
f"waiting for {cache_dir}: another run holds {lock_path} "
f"(lsof {lock_path} names it)"
)
if not shared:
message += (
". generate_pack.py --no-refresh-data, or --offline there and in"
" pipeline.py, builds from the cache as it is without waiting"
)
return message
@contextlib.contextmanager
def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[None]:
"""file_lock, saying so when it has to wait.
@@ -90,7 +108,7 @@ def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[
try:
fcntl.flock(handle, mode | fcntl.LOCK_NB)
except BlockingIOError:
log.warning("waiting for %s: another run holds it", cache_dir)
log.warning(_wait_message(lock_path, shared, cache_dir))
fcntl.flock(handle, mode)
try:
yield
+30 -4
View File
@@ -34,7 +34,10 @@ has ended, and its export is rewritten even when no row named it.
Surfaced only (no refresh): data rows that are not a _data_dirs.yml key
(the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs,
CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs
CI pins (PyPI, pinned actions, while scripts/pin_installer.py writes the
install.py SHA-256 pin), profile_sync, rom-folder
pins (moved by hand, then python -m scripts.scraper.romfolders --setup NAME
reads them again). Each needs
a human read (new .info file, manual DAT download, PyPI version bump).
Usage:
@@ -65,7 +68,9 @@ PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
MANUAL_AREAS = ("coreinfo", "ci", "profiles")
# A rom-folder pin is moved by hand in platforms/romfolders/_sources/,
# then `python -m scripts.scraper.romfolders --setup NAME` reads it again.
MANUAL_AREAS = ("romfolders", "coreinfo", "ci", "profiles")
JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10
@@ -397,6 +402,10 @@ def _command_for(
"--force",
]
if area == "ci" and subject == "install.py pin":
# The pin is the hash of the committed install.py, nothing to judge.
return [sys.executable, "scripts/pin_installer.py"]
if area == "catalogs":
if subject == "mame recipes":
return [
@@ -563,7 +572,8 @@ def main() -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
parser.add_argument(
"--only",
help="comma-separated areas to check: " + ",".join(AUTO_AREAS),
help="comma-separated areas to check: " + ",".join(AUTO_AREAS)
+ " (refreshed), " + ",".join(MANUAL_AREAS) + " (listed for a human)",
)
parser.add_argument(
"--dry-run",
@@ -579,7 +589,7 @@ def main() -> int:
parser.add_argument(
"--json",
action="store_true",
help="machine-readable summary (jobs, surfaced, ignored)",
help="machine-readable summary (jobs, surfaced, ignored), the plan with --dry-run",
)
parser.add_argument(
"--findings",
@@ -618,6 +628,22 @@ def main() -> int:
registry = _load_platform_registry()
jobs, surfaced, ignored = plan_jobs(findings, registry)
if args.dry_run and args.json:
print(json.dumps({
"planned": [
{
"area": job.area,
"subject": job.subject,
"command": job.command,
"after": sorted(job.after),
"then": [list(command) for command in job.then],
}
for job in sorted(jobs, key=lambda j: (j.area, j.subject))
],
"surfaced": surfaced,
"ignored_count": len(ignored),
}, indent=2))
return 0
if args.dry_run:
print(f"\n[plan] {len(jobs)} refreshable, {len(surfaced)} manual, "
f"{len(ignored)} already clean/skipped")
+3 -10
View File
@@ -49,7 +49,7 @@ from collections.abc import Callable, Iterator
from dataclasses import dataclass, field
from pathlib import Path
from ..artifacts import write_text_atomic
from ..artifacts import write_if_changed
from ..common import load_emulator_profiles, run_cli
from .romset_dat_importer import (
LISTXML_CHUNK,
@@ -1114,14 +1114,6 @@ def dump(doc: dict) -> str:
return json.dumps(doc, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
def write_if_changed(path: Path, text: str) -> bool:
if path.is_file() and path.read_text(encoding="utf-8") == text:
return False
path.parent.mkdir(parents=True, exist_ok=True)
write_text_atomic(str(path), text)
return True
def main() -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
which = parser.add_mutually_exclusive_group(required=True)
@@ -1197,7 +1189,8 @@ def main() -> int:
continue
if not args.dry_run:
target = args.output_dir / f"{profile}.json"
if write_if_changed(target, dump(doc)):
target.parent.mkdir(parents=True, exist_ok=True)
if write_if_changed(str(target), dump(doc)):
print(f" wrote {target}")
return 1 if failed else 0
+9 -4
View File
@@ -46,14 +46,18 @@ STORAGE = "${storagePath}"
FLATPAK_DATA = "${HOME}/.var/app/"
# EmuScripts file names that differ from the emulator profile they set up.
# The platform and target scrapers both read them here.
SCRIPT_EMULATORS = {
"pcsx2qt": "pcsx2",
"rpcs3legacy": "rpcs3",
"cemuproton": "cemu",
"rmg": "mupen64plus_next",
}
# The main RetroArch config script installs no emulator.
NOT_EMULATORS = {"retroarch_maincfg"}
# The main RetroArch config script installs no emulator, and emudeck-we's
# emuDeckTemplate.ps1 is the skeleton new installers are copied from.
NOT_EMULATORS = {"retroarch_maincfg", "template"}
# EmuDeck's SteamOS installers are shell scripts, emudeck-we's PowerShell.
SCRIPT_SUFFIXES = (".sh", ".ps1")
# What the emulator does with a directory written into each key.
DIRECTORY_KINDS = {
@@ -66,9 +70,10 @@ DIRECTORY_KINDS = {
def emulator_of_script(file_name: str) -> str | None:
"""The emulator an EmuScripts file sets up, None for a helper script."""
if not file_name.endswith(".sh"):
stem, dot, suffix = file_name.rpartition(".")
if not dot or f".{suffix}" not in SCRIPT_SUFFIXES:
return None
key = re.sub(r"^emuDeck", "", file_name[: -len(".sh")], flags=re.IGNORECASE).lower()
key = re.sub(r"^emuDeck", "", stem, flags=re.IGNORECASE).lower()
if not key or key in NOT_EMULATORS:
return None
return SCRIPT_EMULATORS.get(key, key)
+8 -7
View File
@@ -469,9 +469,10 @@ class Scraper(BaseScraper):
version = fetch_github_latest_version("dragoonDorise/EmuDeck")
cores = self._fetch_installed_emulators()
revision = launcher_revision(self.checkbios_url)
cores = self._fetch_installed_emulators(revision)
standalone = [core for core in cores if core != "retroarch"]
tree = RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url))
tree = RepoTree.github(EMUDECK_REPO, revision)
libretro = set(installed_libretro_cores(tree.read(RETROARCH_SCRIPT)))
return {
@@ -501,16 +502,16 @@ class Scraper(BaseScraper):
"launchers": extract_launchers(tree),
}
def _fetch_installed_emulators(self) -> list[str]:
"""Fetch the list of emulators installed by EmuDeck from EmuScripts.
Returns core names normalized to match emulator profile keys.
def _fetch_installed_emulators(self, revision: str) -> list[str]:
"""The emulators EmuDeck's EmuScripts install at the pinned revision,
named as their profiles are: the launcher layer and the SteamOS
target are read at that revision too.
"""
import json
api_url = (
"https://api.github.com/repos/dragoonDorise/EmuDeck/"
"contents/functions/EmuScripts"
f"contents/functions/EmuScripts?ref={revision}"
)
try:
req = urllib.request.Request(api_url, headers=github_headers())
+4 -10
View File
@@ -38,7 +38,7 @@ _SCRIPTS_DIR = str(Path(__file__).resolve().parents[1])
if _SCRIPTS_DIR not in sys.path:
sys.path.insert(0, _SCRIPTS_DIR)
from artifacts import copy_stream_atomic, write_text_atomic # noqa: E402
from artifacts import copy_stream_atomic, write_if_changed, write_text_atomic # noqa: E402
from common import parse_untrusted_xml # noqa: E402
from region import REGIONS, WORLD # noqa: E402
from region_audit import NOINTRO # noqa: E402
@@ -275,14 +275,6 @@ def dump_catalog(doc: dict) -> str:
return f'{text},"games":[\n{games}\n]}}\n'
def write_if_changed(path: Path, text: str) -> bool:
if path.is_file() and path.read_text(encoding="utf-8") == text:
return False
path.parent.mkdir(parents=True, exist_ok=True)
write_text_atomic(str(path), text)
return True
# Fetching ---------------------------------------------------------------
@@ -503,7 +495,9 @@ def main() -> int:
failures.append(f"{doc['source']}/{doc['system']}: {cov['games']} games, "
f"floor {floor_of(doc)}")
if not args.dry_run:
write_if_changed(args.output_dir / doc["source"] / f"{doc['system']}.json", text)
target = args.output_dir / doc["source"] / f"{doc['system']}.json"
target.parent.mkdir(parents=True, exist_ok=True)
write_if_changed(str(target), text)
print(f"{len(docs)} catalogs, {raw_total} bytes raw, {gz_total} bytes gzip")
for failure in failures:
print(f"FAIL {failure}", file=sys.stderr)
@@ -12,7 +12,6 @@ from __future__ import annotations
import argparse
import json
import re
import sys
import urllib.error
import urllib.request
@@ -22,6 +21,7 @@ import yaml
from common import github_token, yaml_load
from ..emudeck_launchers import SCRIPT_SUFFIXES, emulator_of_script
from . import BaseTargetScraper
PLATFORM_NAME = "emudeck"
@@ -33,24 +33,6 @@ WINDOWS_API = (
"https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts"
)
# Map EmuDeck script names to emulator profile keys
# Script naming: emuDeckDolphin.sh -> dolphin
# Some need explicit mapping when names differ
_NAME_OVERRIDES: dict[str, str] = {
"pcsx2qt": "pcsx2",
"rpcs3legacy": "rpcs3",
"cemuproton": "cemu",
"rmg": "mupen64plus_next",
"bigpemu": "bigpemu",
"eden": "eden",
"suyu": "suyu",
"ares": "ares",
}
# Scripts that are not emulators (config helpers, etc.)
_SKIP = {"retroarch_maincfg", "retroarch"}
def _fetch(url: str) -> str:
headers = {"User-Agent": "retrobios-scraper/1.0"}
token = github_token()
@@ -69,26 +51,13 @@ def _list_emuscripts(api_url: str) -> list[str]:
"""List emulator script filenames from GitHub API."""
entries = json.loads(_fetch(api_url))
names = [
e["name"] for e in entries if e.get("name", "").endswith((".sh", ".ps1"))
e["name"] for e in entries if e.get("name", "").endswith(SCRIPT_SUFFIXES)
]
if not names:
raise RuntimeError(f"no emulator scripts listed at {api_url}")
return names
def _script_to_core(filename: str) -> str | None:
"""Convert EmuScripts filename to core profile key."""
# Strip extension and emuDeck prefix
name = re.sub(r"\.(sh|ps1)$", "", filename, flags=re.IGNORECASE)
name = re.sub(r"^emuDeck", "", name, flags=re.IGNORECASE)
if not name:
return None
key = name.lower()
if key in _SKIP:
return None
return _NAME_OVERRIDES.get(key, key)
class Scraper(BaseTargetScraper):
"""Fetches emulator lists for EmuDeck SteamOS and Windows targets."""
@@ -114,14 +83,13 @@ class Scraper(BaseTargetScraper):
seen: set[str] = set()
has_retroarch = False
for script in scripts:
core = _script_to_core(script)
if core and core not in seen:
core = emulator_of_script(script)
# RetroArch stands for its libretro cores, added below.
if core == "retroarch":
has_retroarch = True
elif core and core not in seen:
seen.add(core)
cores.append(core)
# Detect RetroArch presence (provides all libretro cores)
name = re.sub(r"\.(sh|ps1)$", "", script, flags=re.IGNORECASE)
if name.lower() in ("emudeckretroarch", "retroarch_maincfg"):
has_retroarch = True
standalone_count = len(cores)
# EmuDeck ships RetroArch = all its libretro cores are available
+49 -11
View File
@@ -677,12 +677,21 @@ class Collision:
resolved: list[str]
# Each declaration with the file it resolved to, in the platform's order.
declarations: list[tuple[dict, str | None]] = field(default_factory=list)
# The system of each declaration, in the same order.
systems: list[str] = field(default_factory=list)
# The file the pack ships at the destination, as the builder decides it.
shipped: str | None = None
def _collision_json(collision: Collision) -> dict:
return {
"destination": collision.destination,
"resolved": list(collision.resolved),
"shipped": collision.shipped,
"declarations": [
{"system": system, "name": entry.get("name", ""), "file": local}
for system, (entry, local) in zip(collision.systems, collision.declarations)
],
}
@@ -702,19 +711,27 @@ def find_collisions(
CoCo disk ROM at one bios/disk.rom.
"""
fold = bool(config.get("case_insensitive_fs", False))
by_dest: dict[str, list[dict]] = {}
for system in (config.get("systems") or {}).values():
systems = config.get("systems") or {}
# The builder lets a hash-constrained declaration that resolves by
# content take the path, else the first it reaches walking the systems
# sorted: the same choice says which file ships.
preferred = set(
preferred_entries(systems, db, "bios", "", False, zip_contents, True).values()
)
by_dest: dict[str, list[tuple[str, dict]]] = {}
for sys_id, system in systems.items():
for entry in system.get("files") or []:
if not isinstance(entry, dict):
continue
dest = entry.get("destination") or entry.get("name") or ""
if dest:
by_dest.setdefault(_normalize(dest, fold), []).append(entry)
by_dest.setdefault(_normalize(dest, fold), []).append((sys_id, entry))
collisions = []
for key, entries in sorted(by_dest.items()):
if len(entries) < 2:
for key, claims in sorted(by_dest.items()):
if len(claims) < 2:
continue
entries = [entry for _sys_id, entry in claims]
resolved = []
declarations = []
for entry in entries:
@@ -734,8 +751,22 @@ def find_collisions(
# also names the first one's file.
if any(_accepted_by_all(path, entries, db) for path in resolved):
continue
reached = sorted(
(sys_id, local)
for (sys_id, _entry), (_e, local) in zip(claims, declarations) if local
)
shipped = next(
(local for entry, local in declarations if local and id(entry) in preferred),
reached[0][1],
)
collisions.append(
Collision(destination=key, resolved=resolved, declarations=declarations)
Collision(
destination=key,
resolved=resolved,
declarations=declarations,
systems=[sys_id for sys_id, _entry in claims],
shipped=shipped,
)
)
return collisions
@@ -863,11 +894,18 @@ def _same_file_family(paths: list[str]) -> bool:
def format_collision(collision: Collision) -> str:
"""One line naming the destination and the files fighting for it."""
return (
f"{collision.destination}: declared for "
+ " and for ".join(collision.resolved)
)
"""One line naming the destination, which system declares which file,
and the one the pack ships there."""
owners: dict[str, list[str]] = {}
for system, (_entry, local) in zip(collision.systems, collision.declarations):
if local and system not in owners.setdefault(local, []):
owners[local].append(system)
parts = [
f"{path} ({', '.join(owners[path])}) "
+ ("packed" if path == collision.shipped else "left out")
for path in collision.resolved
]
return f"{collision.destination}: " + ", ".join(parts)
def format_conflict(conflict: Conflict) -> str:
+4
View File
@@ -338,6 +338,10 @@ def main() -> int:
# The packs are rewritten in place by generate_pack under this lock: a
# build running beside the split mixes its bytes into the pack being
# read, or writes a pack that the split then deletes.
if not args.target.exists():
print(f"Error: {args.target}: no such pack or directory, pass a pack or the "
"--output-dir generate_pack wrote (default dist/)", file=sys.stderr)
return 1
directory = args.target if args.target.is_dir() else args.target.parent
try:
with artifact_lock(str(directory)):
+74 -6
View File
@@ -18,6 +18,7 @@ Outputs a structured report suitable for PR comments.
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
@@ -25,6 +26,7 @@ import shutil
import subprocess
import sys
import urllib.parse
import zipfile
from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__))
@@ -335,6 +337,53 @@ def load_profile_hashes(
return known
def _archive_members(
filepath: str, db: dict | None, platform_hashes: dict, profile_hashes: dict
) -> list[str] | None:
"""What each file of a ZIP is, None when filepath is not a ZIP.
Read only for an archive no list declares as a whole: MAME and FBNeo
sets are ZIP BIOS files in their own right.
"""
if not zipfile.is_zipfile(filepath):
return None
held = (db or {}).get("files", {})
described = []
try:
with zipfile.ZipFile(filepath) as archive:
for info in archive.infolist():
if info.is_dir():
continue
if info.file_size > MAX_FILE_SIZE:
described.append(f"`{info.filename}` too large to read here")
continue
sha1, md5 = hashlib.sha1(), hashlib.md5()
with archive.open(info) as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
sha1.update(chunk)
md5.update(chunk)
sha1_hex, md5_hex = sha1.hexdigest(), md5.hexdigest()
crc = f"{info.CRC:08x}"
if sha1_hex in held:
state = f"already held at `{held[sha1_hex]['path']}`"
elif (
sha1_hex in platform_hashes.get("sha1", set())
or md5_hex in platform_hashes.get("md5", set())
or (crc, info.file_size) in platform_hashes.get("crc32", set())
or profile_hashes.get("sha1", {}).get(sha1_hex)
or profile_hashes.get("md5", {}).get(md5_hex)
or profile_hashes.get("crc32", {}).get((crc, info.file_size))
):
state = "declared and not held"
else:
state = "not recognised"
described.append(f"`{info.filename}` {state}")
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
# An encrypted member raises RuntimeError.
described.append(f"unreadable ({exc})")
return described
def validate_file(
filepath: str,
db: dict | None,
@@ -446,6 +495,8 @@ def validate_file(
or profile_hashes.get("crc32", {}).get((result.crc32, None))
)
# The files of a ZIP no list declares whole, when the upload is one.
members: list[str] | None = None
if sha1_known:
result.add_check(True, "SHA1 matches known platform requirement")
elif md5_known:
@@ -466,11 +517,26 @@ def validate_file(
)
else:
result.outcome = "unknown"
result.add_warning(
"Not recognised: no platform list or emulator profile declares this "
f"file. Say in the {'issue' if too_large else 'pull request'} which "
"emulator reads it, for which system, and where the dump comes from"
members = None if too_large else _archive_members(
filepath, db, platform_hashes, profile_hashes
)
if members is not None:
# A folder zipped to upload many files at once was hashed as one
# file and its contributor asked where that dump came from.
shown = "; ".join(members[:10]) + (
f"; and {len(members) - 10} more" if len(members) > 10 else ""
)
result.add_warning(
"A ZIP no platform list or emulator profile declares as a whole: "
"add the files it holds, unzipped, not the archive. Inside: "
+ (shown or "nothing")
)
else:
result.add_warning(
"Not recognised: no platform list or emulator profile declares this "
f"file. Say in the {'issue' if too_large else 'pull request'} which "
"emulator reads it, for which system, and where the dump comes from"
)
normalized = os.path.normpath(relative)
# A platform's declaration first, then a profile's: most files the gaps
@@ -503,7 +569,8 @@ def validate_file(
result.add_check(True, f"Correct placement: bios/{parts[1]}/{parts[2]}/")
else:
result.add_warning("File should be in bios/Manufacturer/Console/ structure")
elif result.outcome == "held":
elif result.outcome == "held" or members is not None:
# Nothing to place: a copy, or an archive whose files go in unzipped.
pass
elif expected_folders:
# The contributor asks before forking: say where it goes.
@@ -847,7 +914,8 @@ def main() -> None:
{
"file": r.filepath,
"passed": r.passed,
"outcome": r.outcome or "new",
# A file that failed, unread or refused, is not a new one.
"outcome": r.outcome or ("new" if r.passed else "failed"),
"destination": r.destination,
"sha1": r.sha1,
"md5": r.md5,
+9 -1
View File
@@ -14,7 +14,15 @@ from pathlib import Path, PurePosixPath
import yaml
from common import source_yaml_paths, yaml_load
from jsonschema import Draft202012Validator, FormatChecker
try:
from jsonschema import Draft202012Validator, FormatChecker
except ImportError:
# Tests import this module and skip on the ImportError, the command
# names the install pipeline.py names for the same step.
if __name__ != "__main__":
raise
sys.exit('Error: jsonschema required (pip install "jsonschema[format-nongpl]")')
ROOT = Path(__file__).resolve().parent.parent
SCHEMAS = ROOT / "schemas"
+3 -3
View File
@@ -2988,13 +2988,13 @@ def _main(held: contextlib.ExitStack) -> None:
print(_overall_summary(group_results))
if args.destinations:
held = {}
destinations = {}
for (result, names), (group, _rep) in zip(group_results, groups):
for name, key in zip(names, group):
held[name] = {
destinations[name] = {
"platform": key, "pack_destinations": result["pack_destinations"],
}
Path(args.destinations).write_text(json.dumps(held, indent=1, sort_keys=True))
Path(args.destinations).write_text(json.dumps(destinations, indent=1, sort_keys=True))
if args.json:
for r in all_results.values():
+11 -3
View File
@@ -13,6 +13,7 @@ temporary directory instead of naming the missing parent.
from __future__ import annotations
import os
import tempfile
from pathlib import Path
os.environ.setdefault("RETROBIOS_OFFLINE", "1")
@@ -22,9 +23,16 @@ os.environ.setdefault(
str(Path(__file__).resolve().parent.parent / "tmp" / "tests" / "probe.lock"),
)
(Path(__file__).resolve().parent.parent / "tmp" / "tests").mkdir(
parents=True, exist_ok=True
)
_SCRATCH = Path(__file__).resolve().parent.parent / "tmp" / "tests"
_SCRATCH.mkdir(parents=True, exist_ok=True)
# A TemporaryDirectory() without dir= landed in /tmp, a 4 GB tmpfs here: once
# another run filled it, modules unrelated to any change failed with ENOSPC.
# The default is the repository's tmp/, for this process and its children.
os.environ["TMPDIR"] = str(_SCRATCH)
tempfile.tempdir = str(_SCRATCH)
# A scratch directory is not part of this checkout: git run inside one must
# not walk up to the repository, as it could not from /tmp.
os.environ["GIT_CEILING_DIRECTORIES"] = str(_SCRATCH)
# The installer's cache (the games index, what reading a card found) is the
# suite's own: a run never reads nor fills the one of the person running it.
os.environ.setdefault(
+22
View File
@@ -61,6 +61,28 @@ class ArtifactLockTest(unittest.TestCase):
with artifact_lock(self.dir, exclusive=False):
pass
def test_a_reader_never_creates_the_directory(self):
"""A mistyped --output-dir given to --verify-packs or release_record
was created holding a .lock, and the next run with the same typo no
longer said the path did not exist."""
from common import ArtifactDirectoryMissing # noqa: PLC0415
missing = os.path.join(self.dir, "sub", "dsit")
with self.assertRaises(ArtifactDirectoryMissing):
with artifact_lock(missing, exclusive=False):
pass
self.assertFalse(os.path.exists(os.path.join(self.dir, "sub")))
def test_split_refuses_a_missing_target_before_locking(self):
missing = Path(self.dir) / "sub" / "dsit"
proc = subprocess.run(
[sys.executable, "scripts/split_pack.py", str(missing)],
cwd=REPO_ROOT, capture_output=True, text=True, check=False, timeout=60,
)
self.assertEqual(proc.returncode, 1)
self.assertIn("no such pack or directory", proc.stderr)
self.assertFalse(missing.parent.exists())
def test_lock_is_released_on_exit(self):
with artifact_lock(self.dir):
pass
+39 -2
View File
@@ -206,11 +206,16 @@ class AModuleRunAloneHasItsScratchDirectory(unittest.TestCase):
)
proc = subprocess.run(
[sys.executable, "-I", "-c",
"import sys; sys.path.insert(0, '.'); import tests"],
"import sys; sys.path.insert(0, '.'); import tests, tempfile, os;"
" print(tempfile.gettempdir()); print(os.environ['TMPDIR'])"],
cwd=clone, capture_output=True, text=True, timeout=60, check=False,
)
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertTrue((clone / "tmp" / "tests").is_dir())
# 264 TemporaryDirectory() calls without dir= wrote to the /tmp
# tmpfs, and a full tmpfs failed modules no change touched.
scratch = str((clone / "tmp" / "tests").resolve())
self.assertEqual(proc.stdout.split(), [scratch, scratch])
class ReleaseSigningRegressions(unittest.TestCase):
@@ -2291,7 +2296,9 @@ class EveryManifestEntryIsFetchable(unittest.TestCase):
else "gitignored" if repo_path in ignored
else "untracked, never committed"
)
if bool(entry.get("release_asset")) != (state == "gitignored"):
if state == "untracked, never committed" or (
bool(entry.get("release_asset")) != (state == "gitignored")
):
wrong.append((entry["dest"], repo_path, state))
self.assertEqual(wrong, [], f"{name} picks the wrong source: {wrong[:3]}")
@@ -2305,6 +2312,36 @@ class EveryManifestEntryIsFetchable(unittest.TestCase):
self.skipTest("not a git checkout")
self.assertEqual(sorted(untracked)[:20], [], f"{len(untracked)} untracked files under bios/")
def test_every_database_path_is_committed_or_a_release_asset(self):
"""A git mv under bios/ committed without regenerating anything left
database.json and the manifests agreeing on the old path: the tests
above compare them with each other, so all passed, and the installer
got a 404 (issue 78, with the database stale too)."""
database = ROOT / "database.json"
if not database.is_file():
self.skipTest("database.json not generated")
files = json.loads(database.read_text(encoding="utf-8"))["files"]
paths = {
path
for entry in files.values()
for path in [entry.get("path"), *(entry.get("paths") or [])]
if path
}
tracked = _git_paths("ls-files", "-z")
if tracked is None:
self.skipTest("not a git checkout")
outside = sorted(paths - tracked)
ignored = _git_paths("check-ignore", "-z", "--no-index", "--stdin", stdin=outside)
if ignored is None:
self.skipTest("not a git checkout")
stale = [path for path in outside if path not in ignored]
self.assertEqual(
stale[:20], [],
f"{len(stale)} database paths git neither holds nor ignores: "
"python scripts/generate_db.py --force --bios-dir bios --output database.json, "
"then the manifests",
)
class ReleaseAssetCriterion(unittest.TestCase):
"""What decides whether the installer fetches from the release is git, not size.
+10
View File
@@ -891,6 +891,16 @@ class BiosIntegrity(Scratch):
self.assertEqual([(f.path, f.verdict) for f in report.findings], [(rel, bib.SKIPPED)])
self.assertEqual((self.repo / rel).read_bytes(), b"\x02" * 80)
def test_an_unreadable_index_stops_before_any_restore(self):
"""git status failed on a corrupt index, its empty stdout read as
"unmodified", and --apply wrote HEAD's bytes over the session's."""
rel = "bios/Sony/edited.bin"
(self.repo / rel).write_bytes(b"\x02" * 80)
(self.repo / ".git" / "index").write_bytes(b"garbage\n")
with self.assertRaises(bib.GitFailed):
bib.check(self.repo, self.db, apply=True)
self.assertEqual((self.repo / rel).read_bytes(), b"\x02" * 80)
def test_a_mismatch_the_second_read_does_not_confirm_is_a_misread(self):
reads = {}
expected = {e["path"]: e["sha1"] for e in self.db["files"].values()}
+21
View File
@@ -35,6 +35,14 @@ class NativeCacheTests(unittest.TestCase):
self.root.mkdir()
self.wanted = {"list.txt": "https://example.invalid/tag-2/list.txt"}
def test_an_unreadable_index_is_an_error_not_unrecorded(self):
"""Read as empty, it named "no URL recorded" for every original."""
self.index.write_text("{", encoding="utf-8")
findings = cf.check_native(REPO_ROOT / "platforms", self.cache, self.cache)
self.assertEqual([(f.subject, f.status) for f in findings],
[(self.index.name, cf.ERROR)])
self.assertIn("--refresh-cache", findings[0].detail)
def _cached(self, mtime: float) -> None:
path = self.root / "list.txt"
path.write_text("x", encoding="utf-8")
@@ -734,6 +742,19 @@ class RowsSayAllTheyStandFor(unittest.TestCase):
rows = [line for line in text.splitlines() if line.startswith(" STALE")]
self.assertEqual(rows[0].index("no longer"), rows[1].index("is not what"))
def test_a_remote_value_keeps_the_detail_column(self):
"""A row with a remote value pushed its detail past the column the
rows without one used, in [coreinfo] by some fifteen characters."""
text = cf.render([
cf.Finding("coreinfo", "anybor", cf.STALE, detail="no profile claims"),
cf.Finding("coreinfo", "amiberry", cf.STALE, local="e5d1f0f9a143",
remote="ae545d85c06f", detail=".info changed"),
cf.Finding("coreinfo", "x", cf.STALE, local="abc", remote="d", detail="third"),
])
rows = [line for line in text.splitlines() if line.startswith(" STALE")]
columns = {row.index(word) for row, word in zip(rows, ("no profile", ".info", "third"))}
self.assertEqual(len(columns), 1, rows)
def test_json_carries_every_unresolved_name(self):
missing = {f"core{i:02d}": [] for i in range(50)}
(row,) = cf._unresolved_findings("retropie", missing)
+16
View File
@@ -1005,6 +1005,22 @@ class CachedOriginal(unittest.TestCase):
self._upstream("v1.txt", "moved")
self.assertEqual(fetch(url, target, self.index), b"one")
def test_an_unreadable_index_serves_nothing(self):
"""A truncated index read as empty: the cached batocera-41 file was
served for a 43.1 URL, and the next download erased every other
platform's recorded URL."""
import export_native
old = self._upstream("batocera-41.txt", "old")
target = self.cache / "plat" / "list.txt"
fetch(old, target, self.index)
self.index.write_text('{"plat/list.txt": "' + old + '",', encoding="utf-8")
new = self._upstream("batocera-43.txt", "new")
with self.assertRaises(export_native.SourcesIndexUnreadable) as caught:
fetch(new, target, self.index)
self.assertIn("--refresh-cache", str(caught.exception))
self.assertTrue(self.index.read_text(encoding="utf-8").endswith(","))
def test_refresh_reads_a_branch_url_again(self):
"""A branch URL keeps its name while the platform moves."""
url = self._upstream("branch.txt", "one")
+11
View File
@@ -102,6 +102,17 @@ class FixtureImport(unittest.TestCase):
twin = second / path.relative_to(first)
self.assertEqual(path.read_bytes(), twin.read_bytes(), path.name)
def test_a_new_fetch_date_alone_rewrites_nothing(self) -> None:
"""A refresh rewrote every catalog for the day it ran, and the diff
read before commit no longer showed which DAT had moved."""
self.assertEqual(run_import("redump", self.out).returncode, 0)
path = self.out / "redump" / "sega-megacd.json"
doc = json.loads(path.read_text())
doc["fetched"] = "2000-01-01"
path.write_text(importer.dump_catalog(doc))
self.assertEqual(run_import("redump", self.out).returncode, 0)
self.assertEqual(json.loads(path.read_text())["fetched"], "2000-01-01")
def test_floor_refuses_a_short_dat(self) -> None:
result = run_import("no-intro", self.out, "--floor", "--dry-run")
self.assertEqual(result.returncode, 1)
+29 -2
View File
@@ -36,6 +36,9 @@ _spec.loader.exec_module(install)
sys.path.insert(0, str(REPO_ROOT / "tests"))
from netguard import module_guard # noqa: E402
# What a stale pin's failure says: the fix is one command.
PIN_ADVICE = "install.py changed: python scripts/pin_installer.py rewrites both pins"
# No connection off the loopback and no card of the host reaches a test:
# the module fails at its end otherwise.
setUpModule, tearDownModule = module_guard("test_install")
@@ -2201,7 +2204,7 @@ class TestAvailablePlatforms(unittest.TestCase):
match = re.search(r'\$defaultInstallSha256 = "([0-9a-f]{64})"', content)
self.assertIsNotNone(match, "install.ps1 must embed the installer SHA256")
expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
self.assertEqual(match.group(1), expected)
self.assertEqual(match.group(1), expected, PIN_ADVICE)
def test_powershell_wrapper_rejects_non_https_bootstrap(self):
content = (REPO_ROOT / "install.ps1").read_text()
@@ -2212,7 +2215,31 @@ class TestAvailablePlatforms(unittest.TestCase):
match = re.search(r'DEFAULT_INSTALL_SHA256="([0-9a-f]{64})"', content)
self.assertIsNotNone(match, "install.sh must embed the installer SHA256")
expected = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
self.assertEqual(match.group(1), expected)
self.assertEqual(match.group(1), expected, PIN_ADVICE)
def test_pin_script_writes_both_pins(self):
"""Every edit to install.py meant a sha256sum pasted by hand into two
files with two syntaxes, and the failing test named no command."""
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import pin_installer # noqa: PLC0415
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory:
root = Path(directory)
for name in ("install.py", "install.sh", "install.ps1"):
shutil.copy2(REPO_ROOT / name, root / name)
(root / "install.py").write_text("# changed\n", encoding="utf-8")
sh_mode = (root / "install.sh").stat().st_mode
self.assertEqual(pin_installer.stale_pins(root), ["install.sh", "install.ps1"])
self.assertEqual(pin_installer.write_pins(root), ["install.sh", "install.ps1"])
self.assertEqual(pin_installer.stale_pins(root), [])
digest = hashlib.sha256(b"# changed\n").hexdigest()
for name in ("install.sh", "install.ps1"):
before = (REPO_ROOT / name).read_text(encoding="utf-8").splitlines()
after = (root / name).read_text(encoding="utf-8").splitlines()
changed = [b for a, b in zip(before, after) if a != b]
self.assertEqual(len(changed), 1, name)
self.assertIn(digest, changed[0])
self.assertEqual((root / "install.sh").stat().st_mode, sh_mode)
def test_shell_one_liner_downloads_verifies_and_forwards_arguments(self):
scratch = REPO_ROOT / "tmp" / "tests"
+2
View File
@@ -373,6 +373,8 @@ class Helpers(unittest.TestCase):
self.assertEqual(el.emulator_of_script("emuDeckXemu.sh"), "xemu")
self.assertIsNone(el.emulator_of_script("RetroArch_maincfg.sh"))
self.assertIsNone(el.emulator_of_script("README.md"))
self.assertEqual(el.emulator_of_script("emuDeckPCSX2QT.ps1"), "pcsx2")
self.assertIsNone(el.emulator_of_script("emuDeckTemplate.ps1"))
def test_the_revision_comes_from_the_pinned_url(self):
url = (
+29
View File
@@ -118,6 +118,35 @@ class RefreshConcurrency(unittest.TestCase):
self.assertTrue(finished, "the refresh waited for a reader")
self.assertEqual(result, [False])
def test_a_waiting_refresh_names_the_lock_and_the_way_around(self):
""""waiting for data/dolphin-sys: another run holds it" named
neither the lock nor its holder, and the build waited unbounded."""
import fcntl # noqa: PLC0415
self.cache.mkdir(parents=True)
lock_path = self.cache.with_name(".pak.lock")
entered = threading.Event()
def refresh() -> None:
with rdd.cache_lock(self.cache):
entered.set()
with open(lock_path, "a") as reader, self.assertLogs(rdd.log, "WARNING") as logs:
fcntl.flock(reader, fcntl.LOCK_SH)
worker = threading.Thread(target=refresh, daemon=True)
worker.start()
for _ in range(200):
if logs.records:
break
worker.join(timeout=0.05)
self.assertFalse(entered.is_set())
fcntl.flock(reader, fcntl.LOCK_UN)
worker.join(timeout=20)
self.assertTrue(entered.is_set())
message = logs.records[0].getMessage()
self.assertIn(f"lsof {lock_path}", message)
self.assertIn("--no-refresh-data", message)
def test_version_file_is_never_left_truncated(self):
Path(self.versions).parent.mkdir(parents=True)
Path(self.versions).write_text(json.dumps({"keep": {"sha": "1"}}))
+37
View File
@@ -162,6 +162,11 @@ class Planning(unittest.TestCase):
self.assertEqual(jobs, [])
self.assertEqual(len(surfaced), 3)
def test_the_installer_pin_is_written_by_its_script(self):
jobs, surfaced, _ = self._plan(finding("ci", "install.py pin"))
self.assertEqual([job.command[1:] for job in jobs], [["scripts/pin_installer.py"]])
self.assertEqual(surfaced, [])
class Running(unittest.TestCase):
def test_a_child_starts_after_its_parent_ends(self):
@@ -234,6 +239,38 @@ class SavedFindings(unittest.TestCase):
findings = rs.load_findings(path, ("data", "catalogs"))
self.assertEqual([f["subject"] for f in findings], ["dolphin-sys"])
def test_dry_run_json_prints_the_plan_as_json(self):
"""--dry-run --json printed the text plan, and a caller parsing it
as JSON failed on the first line."""
import json # noqa: PLC0415
import subprocess # noqa: PLC0415
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory:
path = Path(directory) / "freshness.json"
path.write_text(json.dumps([
finding("catalogs", "mame recipes"), finding("coreinfo", "anybor"),
]), encoding="utf-8")
proc = subprocess.run(
[sys.executable, "scripts/refresh_stale.py", "--findings", str(path),
"--dry-run", "--json"],
cwd=REPO_ROOT, capture_output=True, text=True, check=False, timeout=120,
)
self.assertEqual(proc.returncode, 0, proc.stderr)
plan = json.loads(proc.stdout)
self.assertEqual([(j["area"], j["subject"]) for j in plan["planned"]],
[("catalogs", "mame recipes")])
self.assertEqual([f["subject"] for f in plan["surfaced"]], ["anybor"])
def test_every_area_check_freshness_reports_is_known(self):
"""romfolders was added to check_freshness and not here: its rows
were dropped from a saved report without a word, and --only
romfolders was refused."""
import check_freshness # noqa: PLC0415
self.assertEqual(set(check_freshness.AREAS), set(rs.AUTO_AREAS + rs.MANUAL_AREAS))
jobs, surfaced, _ = rs.plan_jobs([finding("romfolders", "muos")], REGISTRY, {})
self.assertEqual((jobs, [f["subject"] for f in surfaced]), ([], ["muos"]))
def test_a_report_that_is_not_a_list_is_refused(self):
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as directory:
path = Path(directory) / "freshness.json"
+26
View File
@@ -120,5 +120,31 @@ class DataRootIsKeyedBySystem(unittest.TestCase):
self.assertTrue(self._errors({}))
class WithoutJsonschema(unittest.TestCase):
"""The documented command stopped on a bare ImportError traceback that
named neither the install nor the format-nongpl extra."""
def test_the_command_names_the_install(self):
import os
import subprocess
scratch = REPO_ROOT / "tmp" / "tests"
scratch.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=scratch) as directory:
stub = Path(directory) / "jsonschema"
stub.mkdir()
(stub / "__init__.py").write_text(
'raise ImportError("No module named jsonschema")\n', encoding="utf-8"
)
env = {**os.environ, "PYTHONPATH": directory}
proc = subprocess.run(
[sys.executable, "scripts/validate_schemas.py", "--source-only"],
capture_output=True, text=True, check=False, cwd=REPO_ROOT, env=env,
)
self.assertEqual(proc.returncode, 1)
self.assertNotIn("Traceback", proc.stderr)
self.assertIn('pip install "jsonschema[format-nongpl]"', proc.stderr)
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -724,7 +724,7 @@ class UnreadableReleaseStopsTheScrape(unittest.TestCase):
with mock.patch("urllib.request.urlopen", self._refuse), self.assertRaises(
RuntimeError
):
emudeck_scraper.Scraper()._fetch_installed_emulators()
emudeck_scraper.Scraper(checkbios_url="pinned")._fetch_installed_emulators("0" * 40)
def test_libretro_core_metadata_raises(self):
"""libretro dropped every system name and manufacturer when the
+17
View File
@@ -509,6 +509,23 @@ class TestSelfContradictingDestinations(unittest.TestCase):
self.assertEqual(len(collisions), 1)
self.assertEqual(len(collisions[0].resolved), 2)
def test_the_line_names_each_system_and_the_file_packed(self):
"""The line named two files and neither the system that declares
each nor the one the pack ships: the reader went to the YAML."""
config = {"systems": {
"zeta": {"files": [{"name": "IPL.bin", "destination": "disk.rom",
"md5": "m" * 32}]},
"alpha": {"files": [{"name": "IPL.bin", "destination": "disk.rom",
"md5": "n" * 32}]},
}}
collision = slots.find_collisions(config, REGIONS_DB)[0]
line = slots.format_collision(collision)
left, packed = collision.resolved
# The builder walks the systems sorted: alpha's declaration ships.
self.assertIn(f"{packed} (alpha) packed", line)
self.assertIn(f"{left} (zeta) left out", line)
self.assertEqual(collision.shipped, packed)
def test_one_archive_named_by_its_inner_roms_is_not_a_collision(self):
# The documented zipped_file pattern: several md5s, one archive.
config = self._config(
+53
View File
@@ -541,6 +541,59 @@ class TargetsReadThePlatformRevision(unittest.TestCase):
self.assertTrue(steamos)
self.assertTrue(all(u.endswith(f"?ref={pin}") for u in steamos), steamos)
def test_emudeck_platform_lists_its_installers_at_the_pin(self):
"""cores: was read on master while launchers: and the SteamOS target
were read at the pin: the YAML described two revisions."""
from scripts.scraper import emudeck_scraper
pin = "d" * 40
seen: list[str] = []
class _Body:
def __init__(self, url):
seen.append(url)
def __enter__(self):
return self
def __exit__(self, *_exc):
return False
def read(self):
return json.dumps([{"name": "emuDeckRMG.sh"}]).encode()
with mock.patch("urllib.request.urlopen", lambda req, **_kw: _Body(req.full_url)):
cores = emudeck_scraper.Scraper(checkbios_url="pinned")._fetch_installed_emulators(pin)
self.assertEqual(cores, ["mupen64plus_next"])
self.assertEqual(len(seen), 1)
self.assertTrue(seen[0].endswith(f"?ref={pin}"), seen)
class EmuDeckScriptNamesHaveOneRule(unittest.TestCase):
"""The target scraper kept its own copy of the EmuScripts-to-profile
table: it published emudeck-we's template skeleton as a core, and a
rename added to one copy only made --target filter that core out."""
def test_the_target_scraper_names_scripts_as_the_platform_does(self):
from scripts.scraper.emudeck_launchers import emulator_of_script
listing = [
"emuDeckRMG.sh",
"emuDeckPCSX2QT.ps1",
"emuDeckTemplate.ps1",
"emuDeckRetroArch.ps1",
"RetroArch_maincfg.sh",
"emuDeckXemu.ps1",
]
scraper = emudeck_targets_scraper.Scraper(revision="0" * 40)
with mock.patch.object(
emudeck_targets_scraper, "_list_emuscripts", return_value=listing
), mock.patch.object(scraper, "_load_retroarch_cores", return_value=["mgba"]):
cores = scraper._fetch_cores_for_target("unused", "Windows")
standalone = {emulator_of_script(name) for name in listing} - {None, "retroarch"}
self.assertEqual(standalone, {"mupen64plus_next", "pcsx2", "xemu"})
self.assertEqual(cores, sorted(standalone | {"mgba"}))
class HashScrapersRefuseAnUnknownVersion(unittest.TestCase):
"""mame_hash_scraper wrote core_version "unknown" after a failed API call
+16
View File
@@ -60,6 +60,22 @@ class ATargetedModelIsItsOwnArtifact(unittest.TestCase):
)
def test_a_named_platform_without_the_target_is_refused(self):
"""bizhawk --target switch printed "skipped", then Done., exit 0,
and wrote no model: the diff after it failed instead."""
repo = pathlib.Path(__file__).resolve().parent.parent
with tempfile.TemporaryDirectory(dir=str(repo / "tmp")) as directory:
proc = subprocess.run(
[sys.executable, "scripts/generate_truth.py", "--platform", "bizhawk",
"--target", "switch", "--output-dir", directory],
capture_output=True, check=False, text=True, cwd=str(repo), timeout=400,
)
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
self.assertIn("bizhawk has no hardware targets", proc.stderr)
self.assertNotIn("Done.", proc.stdout)
self.assertEqual(list(pathlib.Path(directory).iterdir()), [])
class ADiffThatComparedNothingFails(unittest.TestCase):
"""diff_truth --all skipped every platform without a model and exited 0."""
+32
View File
@@ -269,6 +269,29 @@ class HashRecognition(_Fixture):
self.assertIn("no platform list or emulator profile declares this file", markdown)
self.assertIn("which emulator reads it", markdown)
def test_a_zip_of_bios_files_is_read_member_by_member(self):
"""A card's BIOS folder zipped to upload it was hashed as one file,
and its contributor was asked where that dump came from."""
import hashlib # noqa: PLC0415
import zipfile # noqa: PLC0415
held, declared = b"HELD BIOS", b"DECLARED BIOS"
with zipfile.ZipFile(self.tmp / "mybios.zip", "w") as archive:
archive.writestr("scph5501.bin", held)
archive.writestr("sub/declared.bin", declared)
archive.writestr("other.bin", b"noise")
db = {"files": {hashlib.sha1(held).hexdigest(): {"path": "bios/Sony/PS/scph5501.bin"}}}
known = {**EMPTY_HASHES, "md5": {hashlib.md5(declared).hexdigest()}}
result = validate_pr.validate_file("mybios.zip", db, known)
markdown = result.to_markdown()
self.assertEqual(result.outcome, "unknown")
self.assertIn("unzipped, not the archive", markdown)
self.assertIn("`scph5501.bin` already held at `bios/Sony/PS/scph5501.bin`", markdown)
self.assertIn("`sub/declared.bin` declared and not held", markdown)
self.assertIn("`other.bin` not recognised", markdown)
self.assertNotIn("where the dump comes from", markdown)
self.assertNotIn("Place it in", markdown)
def test_a_duplicate_of_a_known_file_is_reported(self):
"""A copy of a held file passed, and was sent to the folder that
already holds it."""
@@ -522,6 +545,15 @@ class FlagsAreAppliedOrRefused(unittest.TestCase):
self.assertEqual(result.returncode, 2)
self.assertIn("not allowed with", result.stderr)
def test_a_failed_file_is_not_reported_new(self):
"""A path that does not exist came out as "outcome": "new", and a
reader listing new contributions announced a BIOS nobody sent."""
import json # noqa: PLC0415
result = self._run("--json", "bios/nope.bin")
report = json.loads(result.stdout)
self.assertEqual([(r["passed"], r["outcome"]) for r in report], [(False, "failed")])
class AFileOutsideBiosIsToldWhereToGo(unittest.TestCase):
+17
View File
@@ -173,6 +173,23 @@ class TheDestinationsFlagNeedsAPack(unittest.TestCase):
self.assertEqual(result.returncode, 2, result.stdout[-300:])
self.assertIn("--destinations", result.stderr)
def test_the_lock_stack_is_not_rebound(self):
"""The --destinations branch reused `held`, the ExitStack that holds
the data directory locks, for its dict: a later enter_context in
that mode raised AttributeError."""
import ast
tree = ast.parse((REPO_ROOT / "scripts" / "verify.py").read_text(encoding="utf-8"))
body = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_main")
stack = body.args.args[0].arg
rebound = [
node.lineno for node in ast.walk(body)
if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign))
for target in (node.targets if isinstance(node, ast.Assign) else [node.target])
if isinstance(target, ast.Name) and target.id == stack
]
self.assertEqual(rebound, [])
class EveryGapCountedIsNamed(unittest.TestCase):
"""verify --platform batocera --verbose counted 3 optional core files and
+3 -1
View File
@@ -1702,7 +1702,9 @@ a real 3.8 interpreter, which CI installs beside 3.12 so the check cannot skip.
The build scripts need 3.9 and are not bound by this.
Every change to `install.py` recomputes the SHA-256 pinned in `install.sh` and
`install.ps1` in the same commit. The tests compare the pins with the file.
`install.ps1` in the same commit: `python scripts/pin_installer.py` writes both
(`--check` writes nothing and exits 1 on a stale pin). The tests compare the
pins with the file and name that command when they differ.
The tables of options, events, exit codes, keys and card markers, the line
menu and the run samples on this page sit between `generated` markers and are
+5 -3
View File
@@ -9,9 +9,11 @@ The tests hold what users and platforms rely on: a pack holds what its
platform checks, the site states what the code does, the installer writes only
where it should.
No test needs network access. Most build synthetic fixtures under the
repository's `tmp/`, and modules that read real artifacts skip when those are
absent. The test runner prints the count.
No test needs network access. Most build synthetic fixtures, and every
temporary directory lands under the repository's `tmp/tests/`:
`tests/__init__.py` points `tempfile` and `TMPDIR` there, never at the `/tmp`
tmpfs. Modules that read real artifacts skip when those are absent. The test
runner prints the count.
## Running tests
+5 -1
View File
@@ -760,7 +760,10 @@ committed.
Rows that need a decision are listed and never run: a buildbot core with no
profile, a new `.info` file, the Redump, No-Intro and TOSEC packs, the CI
pins, and `profile_sync`. A failed scraper leaves its cached original
pins, `profile_sync`, and a rom-folder pin (moved by hand in
`platforms/romfolders/_sources/`, then `python -m scripts.scraper.romfolders
--setup NAME` reads it again). `--dry-run --json` prints the plan as JSON.
A failed scraper leaves its cached original
untouched. Each job writes its output to `tmp/refresh_stale/`, and a GitHub
token is taken from `GITHUB_TOKEN` or from `gh auth token`.
@@ -1343,6 +1346,7 @@ before a release (wiki/release-process.md). Its lines stay in
| `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs |
| `generate_event_schema.py` | Write `schemas/install-events.schema.json` from `EVENT_KEYS` in `install.py` (`--check` exits 1 when the file is stale, which `validate_schemas.py` also reports) |
| `generate_installer_doc.py` | Rewrite the generated blocks of `wiki/installer.md` from `install.py`: the tables (options, events, exit codes, keys per route, card markers), the line menu, and the run samples `install.py` prints in a sandbox against the committed `install/*.json`, so a new list rewrites the page (`--check` exits 1 when a block is stale, `--page FILE` writes another copy of the page). Pipeline step 7 |
| `pin_installer.py` | Write the SHA-256 of `install.py` into the pins of `install.sh` and `install.ps1`, which refuse any other installer (`--check` writes nothing and exits 1 when a pin is stale). `check_freshness.py` reports a stale pin and `refresh_stale.py` runs this |
| `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline). A download is accepted only when every cryptographic hash the entry declares (sha256, sha1, md5) matches. crc32 and size find a file the database already holds, never accept a download. An entry with no cryptographic hash is reported and not searched |
| `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones) |
| `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) |