mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-25 05:09:03 -05:00
chore: merge tooling follow-up
# Conflicts: # tests/test_audit_regressions.py
This commit is contained in:
commit
6c2204a7a8
27 files changed
+618
-84
No files matched your search
@@ -50,6 +50,25 @@ def write_if_changed(
|
||||
return True
|
||||
|
||||
|
||||
def plain_file_mode() -> int:
|
||||
"""The mode open() gives a new file: 0o666 less the process umask."""
|
||||
# Reading the umask sets it, so it is put back at once.
|
||||
umask = os.umask(0)
|
||||
os.umask(umask)
|
||||
return 0o666 & ~umask
|
||||
|
||||
|
||||
def set_plain_mode(path: str | os.PathLike) -> None:
|
||||
"""Give a scratch file the mode a plain open() would have.
|
||||
|
||||
mkstemp creates its file 0600 so no other user reads it while it is
|
||||
written. Renamed into place, that mode stayed: database.json, the
|
||||
manifests, the packs and the recipe snapshots came out unreadable to
|
||||
every other account on the machine and to a web server serving them.
|
||||
"""
|
||||
os.chmod(path, plain_file_mode())
|
||||
|
||||
|
||||
def write_text_atomic(path: str, content: str) -> None:
|
||||
"""Write a whole file or nothing.
|
||||
|
||||
@@ -65,6 +84,7 @@ def write_text_atomic(path: str, content: str) -> None:
|
||||
try:
|
||||
with os.fdopen(handle, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, path)
|
||||
except BaseException:
|
||||
with contextlib.suppress(OSError):
|
||||
@@ -81,6 +101,7 @@ def write_bytes_atomic(path: str, content: bytes) -> None:
|
||||
try:
|
||||
with os.fdopen(handle, "wb") as f:
|
||||
f.write(content)
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, path)
|
||||
except BaseException:
|
||||
with contextlib.suppress(OSError):
|
||||
|
||||
@@ -1253,8 +1253,6 @@ def check_profiles(run: bool, offline: bool) -> list[Finding]:
|
||||
"--all",
|
||||
"--json",
|
||||
"--check-version",
|
||||
"--detect-new-files",
|
||||
"--watch-hashes",
|
||||
]
|
||||
if offline:
|
||||
cmd.append("--offline")
|
||||
|
||||
@@ -2805,6 +2805,8 @@ from artifacts import ( # noqa: E402,F401
|
||||
write_if_changed,
|
||||
write_text_atomic,
|
||||
copy_file_atomic,
|
||||
plain_file_mode,
|
||||
set_plain_mode,
|
||||
ArtifactLockBusy,
|
||||
artifact_lock,
|
||||
hold_artifact_lock,
|
||||
|
||||
@@ -27,6 +27,7 @@ from common import (
|
||||
list_registered_platforms,
|
||||
load_provenance_snapshots,
|
||||
require_yaml,
|
||||
set_plain_mode,
|
||||
source_yaml_paths,
|
||||
variant_base_name,
|
||||
write_if_changed,
|
||||
@@ -309,6 +310,7 @@ def _restore(cached: str, path: str) -> None:
|
||||
os.close(fd)
|
||||
try:
|
||||
shutil.copy2(cached, scratch)
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, path)
|
||||
finally:
|
||||
if os.path.exists(scratch):
|
||||
|
||||
@@ -14,7 +14,7 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
|
||||
from artifacts import _build_timestamp
|
||||
from artifacts import _build_timestamp, set_plain_mode
|
||||
import argparse
|
||||
import contextlib
|
||||
import functools
|
||||
@@ -242,6 +242,7 @@ def _writing_pack(zip_path: str) -> Iterator[zipfile.ZipFile]:
|
||||
try:
|
||||
with zipfile.ZipFile(scratch, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
yield zf
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, zip_path)
|
||||
except BaseException:
|
||||
Path(scratch).unlink(missing_ok=True)
|
||||
@@ -4617,6 +4618,7 @@ def inject_manifest(zip_path: str, manifest: dict) -> None:
|
||||
continue
|
||||
dst.writestr(item, src.read(item.filename))
|
||||
_write_generated_member(dst, "manifest.json", manifest_json)
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, zip_path)
|
||||
except BaseException:
|
||||
Path(scratch).unlink(missing_ok=True)
|
||||
|
||||
@@ -754,10 +754,11 @@ Full walkthrough: [profiling guide]({link("wiki/profiling")}).
|
||||
The checks CI runs on a pull request (Python 3.12 or newer):
|
||||
|
||||
```bash
|
||||
pip install pyyaml "jsonschema[format-nongpl]" markdown
|
||||
pip install pyyaml "jsonschema[format-nongpl]" markdown \\
|
||||
"mkdocs-material>=9.7.5,<10" "pymdown-extensions>=10.14"
|
||||
python scripts/validate_pr.py --changed # the BIOS files the branch adds
|
||||
python scripts/validate_schemas.py --source-only # profiles and platform files
|
||||
python scripts/profile_sync.py --emulator NAME --strict # each profile the branch adds
|
||||
python scripts/profile_sync.py --changed --strict # each profile the branch adds or edits
|
||||
python scripts/refresh_data_dirs.py # data directories the pack-building tests read
|
||||
python -m unittest discover -s tests -t .
|
||||
```
|
||||
|
||||
@@ -379,7 +379,6 @@ def generate_home(
|
||||
]
|
||||
)
|
||||
|
||||
# Emulator classification breakdown
|
||||
lines.extend(
|
||||
[
|
||||
"## Emulator profiles",
|
||||
@@ -413,7 +412,6 @@ def generate_home(
|
||||
]
|
||||
)
|
||||
|
||||
# Quick links
|
||||
lines.extend(
|
||||
[
|
||||
"---",
|
||||
@@ -1958,7 +1956,6 @@ def generate_platform_page(
|
||||
if fname:
|
||||
config_files.setdefault((sys_id, fname), []).append(fe)
|
||||
|
||||
# Group details by system
|
||||
by_system: dict[str, list] = {}
|
||||
for d in cov["details"]:
|
||||
sys_id = d.get("system", "unknown")
|
||||
@@ -2655,7 +2652,6 @@ def generate_system_page(
|
||||
icon_md = system_icon_markdown(manufacturer, console_name)
|
||||
lines.append(f"## {icon_md}{console_name}")
|
||||
lines.append("")
|
||||
# Separate main files from variants
|
||||
main_files = [f for f in files if "/.variants/" not in f["path"]]
|
||||
variant_files = [f for f in files if "/.variants/" in f["path"]]
|
||||
|
||||
@@ -2690,7 +2686,6 @@ def generate_emulators_index(profiles: dict) -> str:
|
||||
}
|
||||
aliases = {k: v for k, v in profiles.items() if v.get("type") == "alias"}
|
||||
|
||||
# Group by classification
|
||||
by_class: dict[str, list[tuple[str, dict]]] = {}
|
||||
for name in sorted(unique.keys()):
|
||||
p = unique[name]
|
||||
@@ -4323,7 +4318,6 @@ def _render_xref_by_platform(
|
||||
sys_str += f" +{len(systems) - 3}"
|
||||
|
||||
file_count = len(files)
|
||||
# Count mode divergences
|
||||
libretro_only = sum(1 for f in files if f.get("mode") == "libretro")
|
||||
standalone_only = sum(1 for f in files if f.get("mode") == "standalone")
|
||||
file_str = str(file_count)
|
||||
@@ -4372,7 +4366,6 @@ def _render_xref_by_upstream(
|
||||
]
|
||||
)
|
||||
|
||||
# Group profiles by upstream
|
||||
by_upstream: dict[str, list[str]] = {}
|
||||
for emu_name, p in sorted(unique.items()):
|
||||
# A null upstream is the project's own source: grouping it under
|
||||
@@ -4382,7 +4375,6 @@ def _render_xref_by_upstream(
|
||||
if up_str:
|
||||
by_upstream.setdefault(up_str, []).append(emu_name)
|
||||
|
||||
# Build platform membership per core
|
||||
platform_membership: dict[str, set[str]] = {}
|
||||
for pname, cov in coverages.items():
|
||||
for key in resolve_platform_cores(cov["config"], unique):
|
||||
@@ -4679,8 +4671,7 @@ Without them, the emulator either refuses to start the game or runs it with
|
||||
reduced accuracy. RetroBIOS maps those requirements to source-traced emulator
|
||||
profiles and verifies local content against the evidence each platform exposes.
|
||||
|
||||
This page picks the right pack for a setup. For BIOS directory paths per
|
||||
platform, verification, and the CLI, see
|
||||
BIOS directory paths per platform, verification and the CLI are in
|
||||
[Getting started](wiki/getting-started.md).
|
||||
|
||||
## Quick install
|
||||
@@ -4866,7 +4857,6 @@ def generate_mkdocs_nav(
|
||||
k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test")
|
||||
}
|
||||
|
||||
# Group emulators by classification for nav
|
||||
by_class: dict[str, list[tuple[str, str]]] = {}
|
||||
for name in sorted(unique_profiles.keys()):
|
||||
p = unique_profiles[name]
|
||||
@@ -4874,7 +4864,6 @@ def generate_mkdocs_nav(
|
||||
display = p.get("emulator", name)
|
||||
by_class.setdefault(cls, []).append((display, f"emulators/{name}.md"))
|
||||
|
||||
# Classification display names
|
||||
cls_labels = {
|
||||
"pure_libretro": "Pure libretro",
|
||||
"official_port": "Official ports",
|
||||
@@ -4992,7 +4981,6 @@ def _generate(args: argparse.Namespace) -> None:
|
||||
for d in GENERATED_DIRS:
|
||||
(docs / d).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Copy stylesheet if source exists
|
||||
css_src = Path("docs_assets") / "extra.css"
|
||||
css_dest = docs / "stylesheets" / "extra.css"
|
||||
if css_src.exists():
|
||||
@@ -5009,7 +4997,6 @@ def _generate(args: argparse.Namespace) -> None:
|
||||
# the sitemap instead (overrides/main.html).
|
||||
(docs / "robots.txt").unlink(missing_ok=True)
|
||||
|
||||
# Copy branding assets
|
||||
images_dest = docs / "assets" / "images"
|
||||
images_dest.mkdir(parents=True, exist_ok=True)
|
||||
assets_src = Path(".github") / "assets"
|
||||
@@ -5056,7 +5043,6 @@ def _generate(args: argparse.Namespace) -> None:
|
||||
f" {len(profiles)} profiles ({unique_count} unique, {len(profiles) - unique_count} aliases)"
|
||||
)
|
||||
|
||||
# Build cross-reference indexes
|
||||
consumers = _build_file_consumers(db, coverages, profiles)
|
||||
emulator_files = _build_emulator_file_index(profiles)
|
||||
|
||||
|
||||
@@ -72,12 +72,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
args = parse_args(argv)
|
||||
|
||||
# Load registry
|
||||
registry_path = os.path.join(args.platforms_dir, "_registry.yml")
|
||||
with open(registry_path) as f:
|
||||
registry = (yaml_load(f) or {}).get("platforms", {})
|
||||
|
||||
# Load emulator profiles
|
||||
profiles = load_emulator_profiles(args.emulators_dir)
|
||||
|
||||
# Load database (optional)
|
||||
@@ -86,7 +84,6 @@ def main(argv: list[str] | None = None) -> None:
|
||||
if args.db != DEFAULT_DB_FILE or os.path.exists(args.db):
|
||||
db = load_database(args.db)
|
||||
|
||||
# Determine platforms
|
||||
if args.all:
|
||||
platforms = list_registered_platforms(
|
||||
args.platforms_dir,
|
||||
@@ -108,7 +105,6 @@ def main(argv: list[str] | None = None) -> None:
|
||||
output_dir, [name], args.target, args.platforms_dir
|
||||
)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
# Resolve target cores
|
||||
target_cores: set[str] | None = None
|
||||
if args.target:
|
||||
try:
|
||||
|
||||
@@ -19,6 +19,7 @@ from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
from artifacts import set_plain_mode
|
||||
from hashing import compute_hashes
|
||||
|
||||
|
||||
@@ -219,6 +220,7 @@ def _fetch_asset(
|
||||
compute_hashes(tmp_path), expected_sha1, expected_md5
|
||||
):
|
||||
return None
|
||||
set_plain_mode(tmp_path)
|
||||
os.replace(tmp_path, cached)
|
||||
return cached
|
||||
finally:
|
||||
|
||||
+128
-24
@@ -939,6 +939,10 @@ class ProfileReport:
|
||||
# Repositories whose pin is no ancestor of the branch the profile
|
||||
# tracks: history was rewritten, or the pin lies on another branch.
|
||||
orphaned: list[str] = field(default_factory=list)
|
||||
# Repositories the forge answers under another owner/name: renamed or
|
||||
# transferred. The redirect is followed in silence, so the move is named
|
||||
# and counted as review work, since a renamed project is re-profiled.
|
||||
moved_repos: list[str] = field(default_factory=list)
|
||||
# A profile off any forge names the page announcing its releases
|
||||
# (version_source): the version read there, beside the one declared.
|
||||
declared_version: str | None = None
|
||||
@@ -950,6 +954,10 @@ class ProfileReport:
|
||||
# review, or the triage read "0 to review" on a profile whose pin could
|
||||
# not move (blastem, xenia-edge, lindbergh-loader).
|
||||
stale_prose: list[str] | None = None
|
||||
# --check-version on a forge: core_version against the latest tag and
|
||||
# release, kept on the report so every output carries it.
|
||||
version: VersionReport | None = None
|
||||
version_warning: str | None = None
|
||||
|
||||
# Skips that say the profile cannot be checked as written: a repository
|
||||
# that no longer answers, no pin to read, a death the forge contradicts.
|
||||
@@ -972,6 +980,7 @@ class ProfileReport:
|
||||
sum(counts.get(status, 0) for status in REVIEW_STATUSES)
|
||||
+ len(self.orphaned)
|
||||
+ len(self.stale_prose or ())
|
||||
+ len(self.moved_repos)
|
||||
)
|
||||
|
||||
|
||||
@@ -1328,6 +1337,25 @@ def _path_roles(refs: list) -> dict[str, set[str]]:
|
||||
return roles
|
||||
|
||||
|
||||
def moved_repositories(
|
||||
views: list[RepoView], cache_dir: str, offline: bool
|
||||
) -> list[str]:
|
||||
"""Declared repositories the forge now answers under another name."""
|
||||
moved = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for view in views:
|
||||
key = (view.repo.host, view.repo.slug)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
current = upstream.current_slug(view.repo, cache_dir, offline)
|
||||
if current and current.lower() != view.repo.slug.lower():
|
||||
moved.append(
|
||||
f"{view.repo.slug} answers as {current} (renamed or transferred)"
|
||||
)
|
||||
return moved
|
||||
|
||||
|
||||
def build_report(
|
||||
name: str, profile: dict, cache_dir: str, offline: bool = False,
|
||||
text: str | None = None,
|
||||
@@ -1447,6 +1475,7 @@ def build_report(
|
||||
report.upstream_pin = view.pin
|
||||
report.upstream_head = view.head
|
||||
report.upstream_origin = view.origin
|
||||
report.moved_repos = moved_repositories(views, cache_dir, offline)
|
||||
# A profile can hold its pin on purpose, documenting a build that is that
|
||||
# revision. Comparing it to HEAD says nothing and recaling would repoint
|
||||
# its refs at code the build never had, so it is judged like a frozen tag:
|
||||
@@ -1969,6 +1998,8 @@ def format_report(report: ProfileReport, changed_only: bool = False) -> str:
|
||||
lines.append(f" not read (unsupported host): {url}")
|
||||
for line in report.orphaned:
|
||||
lines.append(f" orphaned pin: {line}")
|
||||
for line in report.moved_repos:
|
||||
lines.append(f" moved repository: {line}")
|
||||
if report.declared_version or report.page_version:
|
||||
lines.append(
|
||||
f" version page: declared {report.declared_version or '-'}, "
|
||||
@@ -1977,6 +2008,15 @@ def format_report(report: ProfileReport, changed_only: bool = False) -> str:
|
||||
if report.page_version and report.page_version == report.no_genuine_copy
|
||||
else "")
|
||||
)
|
||||
if report.version:
|
||||
version = report.version
|
||||
lines.append(
|
||||
f" version: declared {version.declared}, "
|
||||
f"latest tag {version.latest_tag}, "
|
||||
f"latest release {version.latest_release} ({version.release_date})"
|
||||
)
|
||||
if report.version_warning:
|
||||
lines.append(f" {report.version_warning}")
|
||||
if report.skipped:
|
||||
lines.append(f" skipped: {report.skipped}")
|
||||
return "\n".join(lines)
|
||||
@@ -3346,6 +3386,20 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--emulator", help="a single profile")
|
||||
group.add_argument("--all", action="store_true", help="every profile")
|
||||
group.add_argument(
|
||||
"--changed", action="store_true",
|
||||
help="the profiles the branch adds or edits against --base, "
|
||||
"uncommitted ones included",
|
||||
)
|
||||
group.add_argument(
|
||||
"--added", action="store_true",
|
||||
help="the profiles the branch adds against --base, as CI checks them",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base", default="origin/main",
|
||||
help="revision --changed and --added compare with, through its merge "
|
||||
"base with HEAD (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--emulators-dir", default="emulators",
|
||||
help="emulator profile directory (default: %(default)s)",
|
||||
@@ -3460,8 +3514,51 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
return parser
|
||||
|
||||
|
||||
def _git_lines(args: list[str]) -> list[str]:
|
||||
result = subprocess.run(
|
||||
["git", *args], capture_output=True, text=True, check=False
|
||||
)
|
||||
if result.returncode:
|
||||
detail = result.stderr.strip() or f"exit {result.returncode}"
|
||||
raise SystemExit(f"ERROR: git {' '.join(args)}: {detail}")
|
||||
return [line for line in result.stdout.splitlines() if line]
|
||||
|
||||
|
||||
def branch_profiles(emulators_dir: str, base: str, added_only: bool) -> list[str]:
|
||||
"""Names of the profiles the branch adds (or edits) against ``base``.
|
||||
|
||||
The diff starts at the merge base, so commits main gained since the
|
||||
branch point are not counted, and ends at the working tree, so a
|
||||
profile not yet committed is checked too. A rename reads as an addition.
|
||||
"""
|
||||
merge_base = _git_lines(["merge-base", base, "HEAD"])[0]
|
||||
paths = _git_lines([
|
||||
"diff", "--name-only", "--no-renames",
|
||||
"--diff-filter=A" if added_only else "--diff-filter=AM",
|
||||
merge_base, "--", emulators_dir,
|
||||
])
|
||||
paths += _git_lines([
|
||||
"ls-files", "--others", "--exclude-standard", "--full-name",
|
||||
"--", emulators_dir,
|
||||
])
|
||||
top = Path(_git_lines(["rev-parse", "--show-toplevel"])[0])
|
||||
root = Path(emulators_dir).resolve()
|
||||
return sorted({
|
||||
full.stem
|
||||
for path in paths
|
||||
if (full := (top / path).resolve()).suffix == ".yml" and full.parent == root
|
||||
})
|
||||
|
||||
|
||||
def select_profiles(profiles: dict, args: argparse.Namespace) -> dict:
|
||||
"""Profiles matching the selection flags."""
|
||||
if args.changed or args.added:
|
||||
names = branch_profiles(args.emulators_dir, args.base, args.added)
|
||||
selected = {name: profiles[name] for name in names if name in profiles}
|
||||
if not selected:
|
||||
verb = "adds" if args.added else "adds or edits"
|
||||
print(f"the branch {verb} no profile against {args.base}", file=sys.stderr)
|
||||
return selected
|
||||
if args.emulator:
|
||||
if args.emulator not in profiles:
|
||||
# Suggested from what the listing it names shows: counted over the
|
||||
@@ -3515,9 +3612,9 @@ def _check_quota(count: int, offline: bool) -> None:
|
||||
"""Refuse a run the anonymous quota cannot carry to completion."""
|
||||
if offline or github_token():
|
||||
return
|
||||
if count * 4 > ANON_QUOTA:
|
||||
if count * 5 > ANON_QUOTA:
|
||||
print(
|
||||
f"{count} profiles need roughly {count * 4} API calls, the anonymous "
|
||||
f"{count} profiles need roughly {count * 5} API calls, the anonymous "
|
||||
f"quota is {ANON_QUOTA} per hour. Log in with `gh auth login`, set "
|
||||
"GITHUB_TOKEN, or use --offline.",
|
||||
file=sys.stderr,
|
||||
@@ -3651,23 +3748,6 @@ def _cited_paths(report: ProfileReport) -> list[str]:
|
||||
)
|
||||
|
||||
|
||||
def _print_version(
|
||||
args: argparse.Namespace, profile: dict, report: ProfileReport,
|
||||
repo: upstream.Repo,
|
||||
) -> None:
|
||||
version = check_version(profile, repo, args.cache_dir, args.offline)
|
||||
if version is None:
|
||||
return
|
||||
print(
|
||||
f" version: declared {version.declared}, "
|
||||
f"latest tag {version.latest_tag}, "
|
||||
f"latest release {version.latest_release} ({version.release_date})"
|
||||
)
|
||||
warning = version_warning(version, report.pin)
|
||||
if warning:
|
||||
print(f" {warning}")
|
||||
|
||||
|
||||
def _print_detection(
|
||||
args: argparse.Namespace, profile: dict, report: ProfileReport,
|
||||
repo: upstream.Repo,
|
||||
@@ -3701,6 +3781,27 @@ def _print_detection(
|
||||
print(f" new {kind}: {path} {value}")
|
||||
|
||||
|
||||
def _attach_version(
|
||||
args: argparse.Namespace, profile: dict, report: ProfileReport
|
||||
) -> None:
|
||||
"""Record core_version against the forge's latest tag and release.
|
||||
|
||||
Kept on the report rather than printed, so --json carries it as the text
|
||||
report does. A forge that refuses is named on the profile's own line.
|
||||
"""
|
||||
repo = select_repo(profile) if report.head else None
|
||||
if repo is None:
|
||||
return
|
||||
try:
|
||||
report.version = check_version(profile, repo, args.cache_dir, args.offline)
|
||||
except upstream.RateLimitError:
|
||||
raise
|
||||
except upstream.UpstreamError as exc:
|
||||
print(f"{report.name}: version check skipped: {exc}", file=sys.stderr)
|
||||
return
|
||||
report.version_warning = version_warning(report.version, report.pin)
|
||||
|
||||
|
||||
def _print_extras(
|
||||
args: argparse.Namespace, profile: dict, report: ProfileReport
|
||||
) -> None:
|
||||
@@ -3712,8 +3813,7 @@ def _print_extras(
|
||||
stops everything.
|
||||
"""
|
||||
wants = (
|
||||
args.check_version
|
||||
or args.detect_new_files
|
||||
args.detect_new_files
|
||||
or args.watch_hashes
|
||||
or args.full_diff
|
||||
or args.tree_diff
|
||||
@@ -3735,8 +3835,6 @@ def _print_extras_from(
|
||||
args: argparse.Namespace, profile: dict, report: ProfileReport,
|
||||
repo: upstream.Repo,
|
||||
) -> None:
|
||||
if args.check_version:
|
||||
_print_version(args, profile, report, repo)
|
||||
if args.detect_new_files or args.watch_hashes:
|
||||
_print_detection(args, profile, report, repo)
|
||||
if args.full_diff:
|
||||
@@ -3883,9 +3981,13 @@ def _refuse_flag_combinations(args: argparse.Namespace) -> None:
|
||||
("--triage", args.triage),
|
||||
) if on
|
||||
]
|
||||
# The version check lives on the report, so the outputs built from the
|
||||
# report (text, --json, --markdown) carry it. A URL list and a ranking
|
||||
# do not.
|
||||
lists_only = args.fetch_plan or args.triage
|
||||
extras = [
|
||||
flag for flag, on in (
|
||||
("--check-version", args.check_version),
|
||||
("--check-version", args.check_version and lists_only),
|
||||
("--detect-new-files", args.detect_new_files),
|
||||
("--watch-hashes", args.watch_hashes),
|
||||
("--full-diff", args.full_diff),
|
||||
@@ -3982,6 +4084,8 @@ def main() -> None:
|
||||
report.page_version = page_version(profile, args.cache_dir, args.offline)
|
||||
report.no_genuine_copy = str(
|
||||
profile["version_source"].get("no_genuine_copy") or "") or None
|
||||
if args.check_version:
|
||||
_attach_version(args, profile, report)
|
||||
reports.append(report)
|
||||
if writes:
|
||||
try:
|
||||
|
||||
@@ -25,7 +25,7 @@ import urllib.request
|
||||
import zipfile
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from artifacts import file_lock as _file_lock
|
||||
from artifacts import file_lock as _file_lock, set_plain_mode
|
||||
from common import github_token, platform_arg, require_platform, yaml_load
|
||||
|
||||
try:
|
||||
@@ -133,6 +133,7 @@ def _save_versions(
|
||||
with os.fdopen(handle, "w") as f:
|
||||
json.dump(versions, f, indent=2, sort_keys=True)
|
||||
f.write("\n")
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, path)
|
||||
except BaseException:
|
||||
with contextlib.suppress(OSError):
|
||||
|
||||
+38
-2
@@ -200,6 +200,36 @@ def _same_file(files: list[dict], file_entry: dict, emu_name: str) -> dict | Non
|
||||
)
|
||||
|
||||
|
||||
def _require_by_slot(files: list[dict]) -> None:
|
||||
"""A core that requires a name an entry answers for requires the entry.
|
||||
|
||||
Merged by name, one core's alternatives land in several entries. Ymir
|
||||
reads sega_101.bin, mpr-17933.bin or saturn_bios.bin, so the sega_101.bin
|
||||
entry carries those aliases. kronos, yabause and yabasanshiro open
|
||||
saturn_bios.bin first and sega_101.bin as a fallback: they sat in that
|
||||
entry's _cores, out of its _required_by, while their saturn_bios.bin entry
|
||||
had them requiring it, and a platform line paired through the alias read
|
||||
them as optional. A core already among the entry's readers that requires
|
||||
one of its aliases, as another entry of the system, is one of its
|
||||
requirers. No reader is added.
|
||||
"""
|
||||
required_by_name: dict[str, set[str]] = {}
|
||||
for entry in files:
|
||||
required_by_name.setdefault(entry["name"].lower(), set()).update(
|
||||
entry.get("_required_by", ())
|
||||
)
|
||||
for entry in files:
|
||||
# Only the aliases: entries sharing the entry's own name are other
|
||||
# files that name already kept apart (pak0.pak under each game).
|
||||
names = {str(alias).lower() for alias in entry.get("aliases") or ()}
|
||||
names.discard(entry["name"].lower())
|
||||
requirers = set().union(*(required_by_name.get(n, set()) for n in names))
|
||||
added = (requirers & entry.get("_cores", set())) - entry.get("_required_by", set())
|
||||
if added:
|
||||
entry["_required_by"] = entry.get("_required_by", set()) | added
|
||||
entry["required"] = True
|
||||
|
||||
|
||||
def _brought(file_entry: dict, emu_name: str) -> set[str]:
|
||||
"""The core and build that need the file brought, empty when the
|
||||
emulator installs it. An entry read without layer (a test, a caller of
|
||||
@@ -666,6 +696,9 @@ def generate_platform_truth(
|
||||
)
|
||||
sys_cov["unprofiled"].add(emu_name)
|
||||
|
||||
for sys_data in systems.values():
|
||||
_require_by_slot(sys_data.get("files", []))
|
||||
|
||||
# Drop files with no exploitable data AFTER all cores have contributed.
|
||||
# A file declared by one core without hash/size/path may be enriched by
|
||||
# another core that has the same entry with data, so the filter must run
|
||||
@@ -735,10 +768,13 @@ def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None:
|
||||
|
||||
A value shared on any field settles it: the truth holds every revision a
|
||||
core accepts, and a profile may give one revision's sha1 without its md5.
|
||||
A hash the code does not check contradicts nothing (checked_hashes).
|
||||
A hash the code does not check contradicts nothing (checked_hashes), and
|
||||
a platform md5 that is a composite of a zip's members is no container
|
||||
md5 (comparable), the rule pair_entries applies.
|
||||
"""
|
||||
from exporter.baseline import checked_hashes
|
||||
from exporter.baseline import checked_hashes, comparable
|
||||
|
||||
s_entry = comparable(s_entry)
|
||||
if _hash_set(t_entry) & _hash_set(s_entry):
|
||||
return None
|
||||
t_entry = checked_hashes(t_entry) or {}
|
||||
|
||||
@@ -21,6 +21,8 @@ from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from artifacts import set_plain_mode
|
||||
|
||||
USER_AGENT = "retrobios-profile-sync/1.0"
|
||||
# Where fetched upstream files live, shared by every reader of this cache.
|
||||
CACHE_DIR = ".cache/upstream"
|
||||
@@ -46,6 +48,12 @@ _HOSTS: dict[str, tuple[str, str, str]] = {
|
||||
"https://git.libretro.com/api/v4",
|
||||
"https://git.libretro.com",
|
||||
),
|
||||
# EDuke32 and its forks (eduke32, fury) are developed on this GitLab.
|
||||
"voidpoint.io": (
|
||||
"gitlab",
|
||||
"https://voidpoint.io/api/v4",
|
||||
"https://voidpoint.io",
|
||||
),
|
||||
"codeberg.org": (
|
||||
"forgejo",
|
||||
"https://codeberg.org/api/v1",
|
||||
@@ -337,6 +345,7 @@ def write_cache(target: Path, text: str) -> None:
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(text)
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, target)
|
||||
except OSError:
|
||||
Path(scratch).unlink(missing_ok=True)
|
||||
@@ -455,6 +464,25 @@ def _first_sha(payload: object) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def current_slug(repo: Repo, cache_dir: str, offline: bool = False) -> str | None:
|
||||
"""The owner/name the forge answers for a repository today.
|
||||
|
||||
A renamed or transferred repository keeps answering under its old name:
|
||||
GitHub redirects (AloysHF/NicaiEmu answers as AloysHF/CoolbarEmu), GitLab
|
||||
and Gitea do the same. Every later call follows the redirect, so only the
|
||||
name in the payload tells the project moved.
|
||||
"""
|
||||
if repo.family == "gitlab":
|
||||
url, key = f"{repo.api_base}/projects/{_project(repo)}", "path_with_namespace"
|
||||
else:
|
||||
url, key = f"{repo.api_base}/repos/{repo.slug}", "full_name"
|
||||
payload = _api(url, cache_dir, offline, MOVING_TTL)
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
value = payload.get(key)
|
||||
return str(value) if value else None
|
||||
|
||||
|
||||
def resolve_head(
|
||||
repo: Repo, cache_dir: str, offline: bool = False, branch: str | None = None
|
||||
) -> str | None:
|
||||
|
||||
@@ -147,7 +147,6 @@ class Status:
|
||||
MISSING = "missing"
|
||||
|
||||
|
||||
# Severity for per-file required/optional distinction
|
||||
class Severity:
|
||||
CRITICAL = "critical" # required file missing or bad hash (Recalbox RED)
|
||||
WARNING = "warning" # optional missing or hash mismatch (Recalbox YELLOW)
|
||||
@@ -381,8 +380,6 @@ def compute_severity(
|
||||
return Severity.OK
|
||||
|
||||
|
||||
# ZIP content index
|
||||
|
||||
# Cross-reference: undeclared files used by cores
|
||||
|
||||
|
||||
@@ -419,7 +416,6 @@ def _name_in_index(
|
||||
basename = name.rsplit("/", 1)[-1] if "/" in name else name
|
||||
if basename != name and basename in by_name:
|
||||
return True
|
||||
# Case-insensitive by_name lookup
|
||||
if by_name_lower:
|
||||
key = name.lower()
|
||||
if key in by_name_lower:
|
||||
@@ -513,7 +509,6 @@ def find_undeclared_files(
|
||||
config.get("verification_mode")
|
||||
)
|
||||
|
||||
# Collect data_directory refs
|
||||
declared_dd: set[str] = set()
|
||||
for sys_id, system in config.get("systems", {}).items():
|
||||
for dd in system.get("data_directories", []):
|
||||
@@ -607,7 +602,6 @@ def find_undeclared_files(
|
||||
archive = f.get("archive")
|
||||
seen_files.add(seen_key)
|
||||
|
||||
# Archived files are grouped by archive
|
||||
if archive:
|
||||
archive_key = (
|
||||
archive,
|
||||
@@ -731,7 +725,6 @@ def find_undeclared_files(
|
||||
undeclared = [u for u in undeclared if id(u) not in settled]
|
||||
undeclared.extend(added)
|
||||
|
||||
# Append grouped archive entries
|
||||
for entry in sorted(archive_entries.values(), key=lambda e: e["name"]):
|
||||
undeclared.append(entry)
|
||||
|
||||
@@ -771,7 +764,6 @@ def find_exclusion_notes(
|
||||
|
||||
emu_display = profile.get("emulator", emu_name)
|
||||
|
||||
# Launcher excluded entirely
|
||||
if profile.get("type") == "launcher":
|
||||
notes.append(
|
||||
{
|
||||
@@ -1135,7 +1127,6 @@ def verify_platform(
|
||||
validation_index = _build_validation_index(platform_profiles)
|
||||
validation_owners = destination_owners(platform_profiles)
|
||||
|
||||
# Filter systems by target
|
||||
if not target_cores:
|
||||
plat_cores = None
|
||||
verify_systems = filter_systems_by_target(
|
||||
@@ -1145,9 +1136,7 @@ def verify_platform(
|
||||
platform_cores=plat_cores,
|
||||
)
|
||||
|
||||
# Per-entry results
|
||||
details = []
|
||||
# Per-destination aggregation
|
||||
file_status: dict[str, str] = {}
|
||||
file_required: dict[str, bool] = {}
|
||||
file_severity: dict[str, str] = {}
|
||||
@@ -1401,7 +1390,6 @@ def verify_platform(
|
||||
config, emulators_dir, emu_profiles, target_cores=target_cores
|
||||
)
|
||||
|
||||
# Ground truth coverage
|
||||
gt_filenames = set(validation_index)
|
||||
profiled_names: set[str] = set()
|
||||
for profile in profiles.values():
|
||||
@@ -1714,7 +1702,6 @@ def print_platform_result(
|
||||
ok_count = c[Severity.OK]
|
||||
problems = total - ok_count
|
||||
|
||||
# Summary line
|
||||
if not reads_file_contents(mode):
|
||||
if problems:
|
||||
missing = c.get(Severity.WARNING, 0) + c.get(Severity.CRITICAL, 0)
|
||||
@@ -2378,7 +2365,6 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
|
||||
f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)"
|
||||
)
|
||||
|
||||
# Ground truth coverage footer
|
||||
gt_cov = result.get("ground_truth_coverage")
|
||||
if gt_cov and gt_cov["total"] > 0:
|
||||
pct = round(gt_cov["with_validation"] * 100 / gt_cov["total"])
|
||||
@@ -2548,7 +2534,6 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
_run_listing(args, parser)
|
||||
return
|
||||
|
||||
# Mutual exclusion
|
||||
modes = sum(1 for x in (args.platform, args.all, args.emulator, args.system) if x)
|
||||
if modes == 0:
|
||||
parser.error("Specify --platform, --all, --emulator, or --system")
|
||||
|
||||
@@ -14,6 +14,7 @@ import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
from artifacts import set_plain_mode
|
||||
from hashing import md5sum
|
||||
|
||||
|
||||
@@ -141,6 +142,7 @@ def _write_member_cache(path: str, data: dict[str, list[list]]) -> None:
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, separators=(",", ":"))
|
||||
set_plain_mode(scratch)
|
||||
os.replace(scratch, path)
|
||||
except OSError as exc:
|
||||
print(f"warning: could not write {path}: {exc}", file=sys.stderr)
|
||||
|
||||
Reference in new issue
Block a user