mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-24 12:49:03 -05:00
fix: make maintainer tool output actionable
--emulator accepts a buildbot core name from a profile's cores:, and a mistyped target gets the closest names instead of every target. diff_truth shows destinations and reports unpaired entries as evidence, not removals. The verify summary counts discrepancies, a shared destination names both systems, a truth hash conflict names both cores, and pipeline steps carry their place in the real run instead of N/8. Every GitHub API caller takes the token from github_token(), which falls back to gh auth token.
This commit is contained in:
1 parent
6ee138a819
commit
7095aa61e7
29 files changed
+764
-154
No files matched your search
@@ -1241,7 +1241,12 @@ def render(findings: list[Finding]) -> str:
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--only", help="comma-separated areas: " + ",".join(AREAS))
|
||||
# Joined with ", " so argparse wraps between names: one comma-joined
|
||||
# token was cut inside "coreinfo".
|
||||
parser.add_argument(
|
||||
"--only", metavar="AREAS",
|
||||
help="comma-separated areas, among: " + ", ".join(AREAS),
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="machine-readable output")
|
||||
parser.add_argument(
|
||||
"--profiles", action="store_true",
|
||||
|
||||
+76
-13
@@ -442,21 +442,72 @@ def list_registered_platforms(
|
||||
SHORT_LIST = 12
|
||||
|
||||
|
||||
def suggest_names(typed: list[str], available: list[str], listing_flag: str) -> str:
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _gh_cli_token() -> str:
|
||||
"""What `gh auth token` prints, or "" when gh is absent or logged out."""
|
||||
import logging # noqa: PLC0415
|
||||
import subprocess # noqa: PLC0415
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["gh", "auth", "token"], capture_output=True, text=True,
|
||||
timeout=10, check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
log.debug("gh is not installed, GitHub API calls go anonymous")
|
||||
return ""
|
||||
except (subprocess.TimeoutExpired, OSError) as exc:
|
||||
log.warning("gh auth token did not answer: %s", exc)
|
||||
return ""
|
||||
if proc.returncode != 0:
|
||||
log.debug("gh auth token: %s", (proc.stderr or "").strip())
|
||||
return ""
|
||||
return (proc.stdout or "").strip()
|
||||
|
||||
|
||||
def github_token() -> str:
|
||||
"""The token for api.github.com: GITHUB_TOKEN, GH_TOKEN, then gh's own.
|
||||
|
||||
A maintainer logged in to gh had to export GITHUB_TOKEN by hand before
|
||||
profile_sync, the scrapers or an online pipeline, which refused the run
|
||||
or burnt the anonymous quota of 60 calls an hour. The token gh returns
|
||||
is put in the environment, so the subprocesses a run starts and the
|
||||
readers of os.environ get it too. RETROBIOS_OFFLINE runs make no API
|
||||
call and never ask gh.
|
||||
"""
|
||||
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or ""
|
||||
if token or os.environ.get("RETROBIOS_OFFLINE"):
|
||||
return token
|
||||
token = _gh_cli_token()
|
||||
if token:
|
||||
os.environ["GITHUB_TOKEN"] = token
|
||||
return token
|
||||
|
||||
|
||||
def suggest_names(
|
||||
typed: list[str],
|
||||
available: list[str],
|
||||
listing_flag: str,
|
||||
listed: int | None = None,
|
||||
) -> str:
|
||||
"""The closest known names to what was typed, and how to see them all.
|
||||
|
||||
The hints showed the first ten or twenty names of the alphabet behind a
|
||||
"..." whatever was typed: sony-playstaton listed 3do to apple-iie.
|
||||
`listed` is how many entries the listing shows when `available` also
|
||||
holds aliases of them.
|
||||
"""
|
||||
import difflib # noqa: PLC0415
|
||||
|
||||
total = len(available) if listed is None else listed
|
||||
close = sorted({m for name in typed
|
||||
for m in difflib.get_close_matches(name, available, n=3, cutoff=0.6)})
|
||||
if close:
|
||||
return f"Did you mean: {', '.join(close)}? {listing_flag} lists all {len(available)}."
|
||||
return f"Did you mean: {', '.join(close)}? {listing_flag} lists all {total}."
|
||||
if len(available) <= SHORT_LIST:
|
||||
return f"Known: {', '.join(available)}."
|
||||
return f"{listing_flag} lists all {len(available)}."
|
||||
return f"{listing_flag} lists all {total}."
|
||||
|
||||
|
||||
def platform_arg(value: str) -> str:
|
||||
@@ -603,16 +654,14 @@ def load_target_config(
|
||||
|
||||
canonical = alias_index.get(target)
|
||||
if canonical is None:
|
||||
available = sorted(targets.keys())
|
||||
aliases = []
|
||||
for tname, ovr in overrides.items():
|
||||
for a in ovr.get("aliases", []):
|
||||
aliases.append(f"{a} -> {tname}")
|
||||
msg = f"Unknown target '{target}' for platform '{platform_name}'.\n"
|
||||
msg += f"Available targets: {', '.join(available)}"
|
||||
if aliases:
|
||||
msg += f"\nAliases: {', '.join(sorted(aliases))}"
|
||||
raise ValueError(msg)
|
||||
# A typo got the 30 targets and 18 aliases in two lines to read
|
||||
# through, where a platform or emulator name gets the closest ones.
|
||||
raise ValueError(
|
||||
f"Unknown target '{target}' for platform '{platform_name}'.\n"
|
||||
+ suggest_names(
|
||||
[target], sorted(alias_index), "--list-targets", len(targets)
|
||||
)
|
||||
)
|
||||
|
||||
declared = (targets[canonical] or {}).get("cores")
|
||||
if declared is None:
|
||||
@@ -1728,6 +1777,20 @@ def select_emulator_profiles(
|
||||
"""
|
||||
selected: list[tuple[str, dict]] = []
|
||||
for name in profile_names:
|
||||
if name not in all_profiles:
|
||||
# A buildbot name (pcfx, advmame) lives in a profile's cores:,
|
||||
# and the spelling hints sent pcfx to two PlayStation cores.
|
||||
claimants = sorted(
|
||||
key for key in upstream_profile_index(all_profiles).get(name, ())
|
||||
if all_profiles[key].get("type") not in ("launcher", "test")
|
||||
)
|
||||
if len(claimants) == 1:
|
||||
name = claimants[0]
|
||||
elif claimants:
|
||||
raise ProfileSelectionError(
|
||||
f"{name} is a core name of {', '.join(claimants)}: "
|
||||
f"use --emulator with one of them"
|
||||
)
|
||||
if name not in all_profiles:
|
||||
available = sorted(
|
||||
key
|
||||
|
||||
@@ -855,16 +855,17 @@ def main() -> None:
|
||||
require_platform(args.platform, args.platforms_dir)
|
||||
|
||||
profiles = load_emulator_profiles(args.emulators_dir)
|
||||
if args.emulator:
|
||||
emulator = args.emulator
|
||||
if emulator:
|
||||
# A typo answered "No emulator profiles found" with exit 0.
|
||||
try:
|
||||
select_emulator_profiles(
|
||||
((emulator, _),) = select_emulator_profiles(
|
||||
[args.emulator],
|
||||
load_emulator_profiles(args.emulators_dir, skip_aliases=False),
|
||||
)
|
||||
except ProfileSelectionError as exc:
|
||||
raise SystemExit(f"ERROR: {exc}") from None
|
||||
profiles = {k: v for k, v in profiles.items() if k == args.emulator}
|
||||
profiles = {k: v for k, v in profiles.items() if k == emulator}
|
||||
|
||||
if args.target and not args.platform:
|
||||
parser.error("--target requires --platform")
|
||||
@@ -897,14 +898,14 @@ def main() -> None:
|
||||
))
|
||||
|
||||
if not profiles:
|
||||
if args.emulator and args.target and set(args.emulator.split(",")) & run_anywhere:
|
||||
if emulator and args.target and emulator in run_anywhere:
|
||||
raise SystemExit(
|
||||
f"ERROR: {args.emulator} is not built for {args.platform}'s "
|
||||
f"ERROR: {emulator} is not built for {args.platform}'s "
|
||||
f"{args.target} target; {args.platform} runs it elsewhere "
|
||||
f"(drop --target to see its files)"
|
||||
)
|
||||
if args.emulator:
|
||||
raise SystemExit(f"ERROR: {args.emulator} is not run by {args.platform}")
|
||||
if emulator:
|
||||
raise SystemExit(f"ERROR: {emulator} is not run by {args.platform}")
|
||||
print(f"{args.platform or 'The tree'} runs no profiled emulator.", file=sys.stderr)
|
||||
return
|
||||
|
||||
@@ -921,8 +922,8 @@ def main() -> None:
|
||||
if args.json:
|
||||
print(json.dumps(report, indent=2))
|
||||
else:
|
||||
if args.emulator:
|
||||
scope = args.emulator
|
||||
if emulator:
|
||||
scope = emulator
|
||||
elif args.platform:
|
||||
scope = f"the emulators {args.platform} runs"
|
||||
else:
|
||||
|
||||
+27
-13
@@ -39,6 +39,11 @@ def _load_truth(truth_dir: str, platform: str) -> dict | None:
|
||||
return yaml_load(f) or {}
|
||||
|
||||
|
||||
def _where(entry: dict) -> str:
|
||||
"""Where a diffed entry sits: one name can be declared at several paths."""
|
||||
return entry.get("destination") or entry["name"]
|
||||
|
||||
|
||||
def _format_terminal(report: dict) -> str:
|
||||
lines: list[str] = []
|
||||
platform = report.get("platform", "unknown")
|
||||
@@ -56,7 +61,7 @@ def _format_terminal(report: dict) -> str:
|
||||
if s["total_missing"]:
|
||||
totals.append(f"{s['total_missing']} missing")
|
||||
if s["total_extra_phantom"]:
|
||||
totals.append(f"{s['total_extra_phantom']} phantom")
|
||||
totals.append(f"{s['total_extra_phantom']} unpaired")
|
||||
if s["total_extra_unprofiled"]:
|
||||
totals.append(f"{s['total_extra_unprofiled']} unprofiled")
|
||||
if s["total_hash_mismatch"]:
|
||||
@@ -73,7 +78,7 @@ def _format_terminal(report: dict) -> str:
|
||||
if div.get("missing"):
|
||||
labels.append(f"MISSING:{len(div['missing'])}")
|
||||
if div.get("extra_phantom"):
|
||||
labels.append(f"PHANTOM:{len(div['extra_phantom'])}")
|
||||
labels.append(f"UNPAIRED:{len(div['extra_phantom'])}")
|
||||
if div.get("extra_unprofiled"):
|
||||
labels.append(f"UNPROF:{len(div['extra_unprofiled'])}")
|
||||
if div.get("hash_mismatch"):
|
||||
@@ -84,19 +89,19 @@ def _format_terminal(report: dict) -> str:
|
||||
|
||||
for m in div.get("missing", []):
|
||||
cores = ", ".join(m.get("cores", []))
|
||||
lines.append(f" + {m['name']} [{cores}]")
|
||||
lines.append(f" + {_where(m)} [{cores}]")
|
||||
for h in div.get("hash_mismatch", []):
|
||||
ht = h["hash_type"]
|
||||
lines.append(
|
||||
f" ~ {h['name']} {ht}: {h[f'truth_{ht}']} != {h[f'scraped_{ht}']}"
|
||||
f" ~ {_where(h)} {ht}: {h[f'truth_{ht}']} != {h[f'scraped_{ht}']}"
|
||||
)
|
||||
for p in div.get("extra_phantom", []):
|
||||
lines.append(f" - {p['name']} (phantom)")
|
||||
lines.append(f" - {_where(p)} (no profile entry of this system matches it)")
|
||||
for u in div.get("extra_unprofiled", []):
|
||||
lines.append(f" ? {u['name']} (unprofiled)")
|
||||
lines.append(f" ? {_where(u)} (unprofiled cores)")
|
||||
for r in div.get("required_mismatch", []):
|
||||
lines.append(
|
||||
f" ! {r['name']} required: {r['truth_required']} != {r['scraped_required']}"
|
||||
f" ! {_where(r)} required: {r['truth_required']} != {r['scraped_required']}"
|
||||
)
|
||||
|
||||
uncovered = report.get("uncovered_systems", [])
|
||||
@@ -121,7 +126,7 @@ def _format_markdown(report: dict) -> str:
|
||||
)
|
||||
lines.append(
|
||||
f"**{s['total_missing']}** missing | "
|
||||
f"**{s['total_extra_phantom']}** phantom | "
|
||||
f"**{s['total_extra_phantom']}** unpaired | "
|
||||
f"**{s['total_extra_unprofiled']}** unprofiled | "
|
||||
f"**{s['total_hash_mismatch']}** hash | "
|
||||
f"**{s['total_required_mismatch']}** required"
|
||||
@@ -135,19 +140,28 @@ def _format_markdown(report: dict) -> str:
|
||||
refs = ""
|
||||
if m.get("source_refs"):
|
||||
refs = " " + " ".join(f"`{r}`" for r in m["source_refs"])
|
||||
lines.append(f"- **Add** `{m['name']}`{refs}")
|
||||
lines.append(f"- **Add** `{_where(m)}`{refs}")
|
||||
for h in div.get("hash_mismatch", []):
|
||||
ht = h["hash_type"]
|
||||
lines.append(
|
||||
f"- **Fix hash** `{h['name']}` {ht}: `{h[f'truth_{ht}']}` != `{h[f'scraped_{ht}']}`"
|
||||
f"- **Fix hash** `{_where(h)}` {ht}: `{h[f'truth_{ht}']}` != `{h[f'scraped_{ht}']}`"
|
||||
)
|
||||
# Evidence, not a request: an entry leaves a platform's file only on
|
||||
# proof (export_native's removal_candidates). A core can read a
|
||||
# file its profile does not pair with this declaration.
|
||||
for p in div.get("extra_phantom", []):
|
||||
lines.append(f"- **Remove** `{p['name']}` (phantom)")
|
||||
lines.append(
|
||||
f"- **Unpaired** `{_where(p)}`: no profile entry of this system matches it "
|
||||
"by path, name or hash"
|
||||
)
|
||||
for u in div.get("extra_unprofiled", []):
|
||||
lines.append(f"- **Check** `{u['name']}` (unprofiled cores)")
|
||||
lines.append(
|
||||
f"- **Unpaired** `{_where(u)}`: some cores of this platform "
|
||||
"have no profile yet"
|
||||
)
|
||||
for r in div.get("required_mismatch", []):
|
||||
lines.append(
|
||||
f"- **Fix required** `{r['name']}`: truth={r['truth_required']}, scraped={r['scraped_required']}"
|
||||
f"- **Fix required** `{_where(r)}`: truth={r['truth_required']}, scraped={r['scraped_required']}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
|
||||
@@ -3087,8 +3087,10 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
)
|
||||
return
|
||||
if args.list:
|
||||
# Archived platforms build like the others but have no release pack.
|
||||
active = set(list_registered_platforms(args.platforms_dir))
|
||||
for p in list_platforms(args.platforms_dir):
|
||||
print(p)
|
||||
print(p if p in active else f"{p} (archived)")
|
||||
return
|
||||
if args.list_emulators:
|
||||
list_emulator_profiles(args.emulators_dir)
|
||||
|
||||
@@ -22,6 +22,7 @@ sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import (
|
||||
compute_composition,
|
||||
count_catalog_matched,
|
||||
github_token,
|
||||
list_registered_platforms,
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
@@ -286,7 +287,7 @@ def fetch_contributors() -> list[dict]:
|
||||
|
||||
url = f"https://api.github.com/repos/{REPO}/contributors"
|
||||
headers = {"User-Agent": "retrobios-readme/1.0"}
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
try:
|
||||
|
||||
@@ -285,8 +285,11 @@ def _served_digests() -> dict[str, str]:
|
||||
f"{LARGE_FILES_RELEASE}"
|
||||
)
|
||||
headers = {"User-Agent": "retrobios/1.0", "Accept": "application/vnd.github+json"}
|
||||
if os.environ.get("GITHUB_TOKEN"):
|
||||
headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}"
|
||||
from common import github_token # noqa: PLC0415 (common imports this module)
|
||||
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
digests: dict[str, str] = {}
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
|
||||
@@ -7,7 +7,7 @@ Respects the `status` field in _registry.yml:
|
||||
|
||||
Usage:
|
||||
python scripts/list_platforms.py # Active platforms (for CI)
|
||||
python scripts/list_platforms.py --all # All platforms including archived
|
||||
python scripts/list_platforms.py --include-archived # archived ones too
|
||||
python scripts/list_platforms.py >> "$GITHUB_OUTPUT"
|
||||
"""
|
||||
|
||||
@@ -31,10 +31,15 @@ def list_platforms(include_archived: bool = False) -> list[str]:
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description="List available platforms")
|
||||
parser.add_argument("--all", action="store_true", help="Include archived platforms")
|
||||
# The name verify, generate_pack, slots and pipeline use. --all is the
|
||||
# older spelling, still read so existing commands keep working.
|
||||
parser.add_argument(
|
||||
"--include-archived", "--all", dest="include_archived", action="store_true",
|
||||
help="include archived platforms",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
platforms = list_platforms(include_archived=args.all)
|
||||
platforms = list_platforms(include_archived=args.include_archived)
|
||||
|
||||
if not platforms:
|
||||
print("No platform configs found", file=sys.stderr)
|
||||
|
||||
+115
-47
@@ -61,6 +61,7 @@ sys.path.insert(0, str(Path(__file__).parent))
|
||||
from common import (
|
||||
ArtifactLockBusy,
|
||||
artifact_lock,
|
||||
github_token,
|
||||
hold_artifact_lock,
|
||||
load_data_dir_registry,
|
||||
)
|
||||
@@ -92,6 +93,62 @@ class StepResults(dict):
|
||||
# Steps that report without failing the run.
|
||||
NON_BLOCKING = frozenset({"check_buildbot"})
|
||||
|
||||
# Every step in run order, by its results key. The position and the total
|
||||
# printed during the run come from this list: hand-written "N/8" labels
|
||||
# counted 8 steps in a run of 24, and the eighth came before 8b. The summary
|
||||
# prints the same labels the run printed, so a FAILED row is found in the
|
||||
# log by its number.
|
||||
STEP_LABELS: dict[str, str] = {
|
||||
"validate_schemas": "0 validate schemas",
|
||||
"generate_db": "1 generate database",
|
||||
"provenance": "1b provenance report",
|
||||
"romset_recipes": "1c romset recipes",
|
||||
"refresh_data": "2 refresh data directories",
|
||||
"mame_hashes": "2a refresh MAME hashes",
|
||||
"fbneo_hashes": "2a2 refresh FBNeo hashes",
|
||||
"check_buildbot": "2b check buildbot system",
|
||||
"check_release_assets": "2b2 check release assets",
|
||||
"generate_truth": "2c generate truth",
|
||||
"diff_truth": "2d diff truth",
|
||||
"export_native": "2e export native",
|
||||
"verify": "3 verify all platforms",
|
||||
"identity": "3a file identity",
|
||||
"slots": "3b slot arbitration",
|
||||
"generate_packs": "4 generate packs",
|
||||
"generate_manifests": "4b generate install manifests",
|
||||
"generate_target_manifests": "4c generate target manifests",
|
||||
"bundled_only": "4d copies only an emulator install provides",
|
||||
"consistency": "5 consistency check",
|
||||
"pack_integrity": "6 pack integrity",
|
||||
"generate_readme": "7 generate readme",
|
||||
"generate_site": "8 generate site",
|
||||
"render_site": "8b render site",
|
||||
}
|
||||
_KEY_BY_LABEL = {label: key for key, label in STEP_LABELS.items()}
|
||||
|
||||
|
||||
def step_number(key: str) -> str:
|
||||
"""The number a step's progress lines carry ("2b2", "4d")."""
|
||||
return STEP_LABELS[key].split(" ", 1)[0]
|
||||
|
||||
|
||||
def step_title(label: str) -> str:
|
||||
"""What the run prints for a label: its name and its place in the run.
|
||||
|
||||
A second command of one step ("2c generate truth (export model)") is
|
||||
printed as written.
|
||||
"""
|
||||
key = _KEY_BY_LABEL.get(label)
|
||||
if key is None:
|
||||
return label
|
||||
return f"{label} [{list(STEP_LABELS).index(key) + 1}/{len(STEP_LABELS)}]"
|
||||
|
||||
|
||||
def skip_notice(key: str, reason: str = "") -> None:
|
||||
"""The line a step that does not run prints in place of its output."""
|
||||
because = f" ({reason})" if reason else ""
|
||||
print(f"\n--- {step_title(STEP_LABELS[key])}: SKIPPED{because} ---")
|
||||
|
||||
def run(cmd: list[str], label: str) -> tuple[bool, str]:
|
||||
"""Run a command, echoing its output as it comes. Returns (success, output).
|
||||
|
||||
@@ -99,7 +156,8 @@ def run(cmd: list[str], label: str) -> tuple[bool, str]:
|
||||
after stdout: a twenty-minute pack build showed nothing, and a warning
|
||||
landed far from the line it was about. One stream, read line by line.
|
||||
"""
|
||||
print(f"\n--- {label} ---", flush=True)
|
||||
title = step_title(label)
|
||||
print(f"\n--- {title} ---", flush=True)
|
||||
start = time.monotonic()
|
||||
repo_root = str(Path(__file__).resolve().parent.parent)
|
||||
lines: list[str] = []
|
||||
@@ -114,7 +172,7 @@ def run(cmd: list[str], label: str) -> tuple[bool, str]:
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
ok = returncode == 0
|
||||
print(f"--- {label}: {'OK' if ok else 'FAILED'} ({elapsed:.1f}s) ---")
|
||||
print(f"--- {title}: {'OK' if ok else 'FAILED'} ({elapsed:.1f}s) ---")
|
||||
return ok, "".join(lines)
|
||||
|
||||
|
||||
@@ -248,7 +306,7 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
|
||||
verify_extras = parse_verify_extras(verify_output)
|
||||
pack_extras = parse_pack_extras(pack_output)
|
||||
|
||||
print("\n--- 5/8 consistency check ---")
|
||||
print(f"\n--- {step_title(STEP_LABELS['consistency'])} ---")
|
||||
all_ok = True
|
||||
if not v or not p:
|
||||
# Nothing parsed is nothing compared: a changed output format would
|
||||
@@ -358,7 +416,7 @@ def _truth_steps(args: argparse.Namespace, results: dict) -> bool:
|
||||
]
|
||||
if args.target:
|
||||
truth_cmd.extend(["--target", args.target])
|
||||
ok, _ = run(truth_cmd, "2c generate truth")
|
||||
ok, _ = run(truth_cmd, STEP_LABELS["generate_truth"])
|
||||
# A native file is one per platform, never per target: the export
|
||||
# corrects it against the full model, which a targeted run does not
|
||||
# write.
|
||||
@@ -375,7 +433,7 @@ def _truth_steps(args: argparse.Namespace, results: dict) -> bool:
|
||||
diff_cmd.extend(["--truth-dir", str(truth_dir)])
|
||||
if args.target:
|
||||
diff_cmd.extend(["--target", args.target])
|
||||
ok, _ = run(diff_cmd, "2d diff truth")
|
||||
ok, _ = run(diff_cmd, STEP_LABELS["diff_truth"])
|
||||
results["diff_truth"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
@@ -397,7 +455,7 @@ def _truth_steps(args: argparse.Namespace, results: dict) -> bool:
|
||||
# file has to be in the cache already.
|
||||
if not args.offline:
|
||||
export_cmd.append("--fetch")
|
||||
ok, _ = run(export_cmd, "2e export native")
|
||||
ok, _ = run(export_cmd, STEP_LABELS["export_native"])
|
||||
results["export_native"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
@@ -455,6 +513,11 @@ def main() -> None:
|
||||
)
|
||||
args = parser.parse_args()
|
||||
_refuse_unapplied_flags(parser, args)
|
||||
if not args.offline:
|
||||
# Read once and put in the environment every step inherits: the
|
||||
# scrapers and refresh_data_dirs went anonymous unless the user had
|
||||
# exported GITHUB_TOKEN by hand, though gh was logged in.
|
||||
github_token()
|
||||
|
||||
# A second run on the same output directory is refused before any work:
|
||||
# the database rebuild alone takes minutes, and the reader holding the
|
||||
@@ -478,12 +541,12 @@ def main() -> None:
|
||||
# `cores: [81]` unquoted passed the whole pipeline and failed CI on push.
|
||||
# jsonschema is tooling only, so its absence skips the step.
|
||||
if importlib.util.find_spec("jsonschema") is None:
|
||||
print("\n--- 0/8 validate schemas: SKIPPED (jsonschema not installed) ---")
|
||||
skip_notice("validate_schemas", "jsonschema not installed")
|
||||
results["validate_schemas"] = SKIPPED
|
||||
else:
|
||||
ok, _ = run(
|
||||
[sys.executable, "scripts/validate_schemas.py", "--source-only"],
|
||||
"0/8 validate schemas",
|
||||
STEP_LABELS["validate_schemas"],
|
||||
)
|
||||
results["validate_schemas"] = ok
|
||||
all_ok = all_ok and ok
|
||||
@@ -499,7 +562,7 @@ def main() -> None:
|
||||
"--output",
|
||||
"database.json",
|
||||
],
|
||||
"1/8 generate database",
|
||||
STEP_LABELS["generate_db"],
|
||||
)
|
||||
results["generate_db"] = ok
|
||||
if not ok:
|
||||
@@ -512,7 +575,7 @@ def main() -> None:
|
||||
ok, _ = run(
|
||||
[sys.executable, "scripts/provenance_report.py", "--derive",
|
||||
"--write-derivable", "derivable.json"],
|
||||
"1b provenance report",
|
||||
STEP_LABELS["provenance"],
|
||||
)
|
||||
results["provenance"] = ok
|
||||
all_ok = all_ok and ok
|
||||
@@ -522,7 +585,7 @@ def main() -> None:
|
||||
# Read-only: writing reconstructions is an explicit --write invocation.
|
||||
ok, out = run(
|
||||
[sys.executable, "scripts/romset_recipes.py"],
|
||||
"1c romset recipes",
|
||||
STEP_LABELS["romset_recipes"],
|
||||
)
|
||||
results["romset_recipes"] = ok
|
||||
all_ok = all_ok and ok
|
||||
@@ -531,47 +594,49 @@ def main() -> None:
|
||||
if not args.offline:
|
||||
ok, out = run(
|
||||
[sys.executable, "scripts/refresh_data_dirs.py"],
|
||||
"2/8 refresh data directories",
|
||||
STEP_LABELS["refresh_data"],
|
||||
)
|
||||
results["refresh_data"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 2/8 refresh data directories: SKIPPED (--offline) ---")
|
||||
skip_notice("refresh_data", "--offline")
|
||||
results["refresh_data"] = SKIPPED
|
||||
|
||||
# Step 2a: Refresh MAME BIOS hashes
|
||||
if not args.offline:
|
||||
ok, _ = run(
|
||||
[sys.executable, "-m", "scripts.scraper.mame_hash_scraper"],
|
||||
"2a refresh MAME hashes",
|
||||
STEP_LABELS["mame_hashes"],
|
||||
)
|
||||
results["mame_hashes"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 2a refresh MAME hashes: SKIPPED (--offline) ---")
|
||||
skip_notice("mame_hashes", "--offline")
|
||||
results["mame_hashes"] = SKIPPED
|
||||
|
||||
# Step 2a2: Refresh FBNeo BIOS hashes
|
||||
if not args.offline:
|
||||
ok, _ = run(
|
||||
[sys.executable, "-m", "scripts.scraper.fbneo_hash_scraper"],
|
||||
"2a2 refresh FBNeo hashes",
|
||||
STEP_LABELS["fbneo_hashes"],
|
||||
)
|
||||
results["fbneo_hashes"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 2a2 refresh FBNeo hashes: SKIPPED (--offline) ---")
|
||||
skip_notice("fbneo_hashes", "--offline")
|
||||
results["fbneo_hashes"] = SKIPPED
|
||||
|
||||
# Step 2b: Check buildbot system directory (non-blocking)
|
||||
if args.check_buildbot and not args.offline:
|
||||
ok, _ = run(
|
||||
[sys.executable, "scripts/check_buildbot_system.py"],
|
||||
"2b check buildbot system",
|
||||
STEP_LABELS["check_buildbot"],
|
||||
)
|
||||
results["check_buildbot"] = ok
|
||||
elif args.check_buildbot:
|
||||
print("\n--- 2b check buildbot system: SKIPPED (--offline) ---")
|
||||
else:
|
||||
if args.check_buildbot:
|
||||
skip_notice("check_buildbot", "--offline")
|
||||
results["check_buildbot"] = SKIPPED
|
||||
|
||||
# Step 2b2: The release must serve the bytes the manifests describe. The
|
||||
# installer compares Content-Length with the manifest size, so an asset
|
||||
@@ -579,12 +644,12 @@ def main() -> None:
|
||||
if not args.offline:
|
||||
ok, _ = run(
|
||||
[sys.executable, "scripts/check_release_assets.py"],
|
||||
"2b2 check release assets",
|
||||
STEP_LABELS["check_release_assets"],
|
||||
)
|
||||
results["check_release_assets"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 2b2 check release assets: SKIPPED (--offline) ---")
|
||||
skip_notice("check_release_assets", "--offline")
|
||||
results["check_release_assets"] = SKIPPED
|
||||
|
||||
all_ok = _truth_steps(args, results) and all_ok
|
||||
@@ -603,7 +668,7 @@ def main() -> None:
|
||||
verify_cmd.append("--include-archived")
|
||||
if args.target:
|
||||
verify_cmd.extend(["--target", args.target])
|
||||
ok, verify_output = run(verify_cmd, "3/8 verify all platforms")
|
||||
ok, verify_output = run(verify_cmd, STEP_LABELS["verify"])
|
||||
results["verify"] = ok
|
||||
all_ok = all_ok and ok
|
||||
|
||||
@@ -612,7 +677,7 @@ def main() -> None:
|
||||
# cannot see a file copied under another machine's name.
|
||||
ok, _ = run(
|
||||
[sys.executable, "scripts/identity.py", "--strict"],
|
||||
"3a/8 file identity",
|
||||
STEP_LABELS["identity"],
|
||||
)
|
||||
results["identity"] = ok
|
||||
all_ok = all_ok and ok
|
||||
@@ -629,7 +694,7 @@ def main() -> None:
|
||||
slots_cmd.append("--include-archived")
|
||||
if args.target:
|
||||
slots_cmd.extend(["--target", args.target])
|
||||
ok, _ = run(slots_cmd, "3b/8 slot arbitration")
|
||||
ok, _ = run(slots_cmd, "3b slot arbitration")
|
||||
results["slots"] = ok
|
||||
all_ok &= ok
|
||||
|
||||
@@ -679,11 +744,11 @@ def main() -> None:
|
||||
pack_cmd.extend(["--source", args.source])
|
||||
if args.all_variants:
|
||||
pack_cmd.append("--all-variants")
|
||||
ok, pack_output = run(pack_cmd, "4/8 generate packs")
|
||||
ok, pack_output = run(pack_cmd, STEP_LABELS["generate_packs"])
|
||||
results["generate_packs"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 4/8 generate packs: SKIPPED (--skip-packs) ---")
|
||||
skip_notice("generate_packs", "--skip-packs")
|
||||
results["generate_packs"] = SKIPPED
|
||||
|
||||
# Step 4b: Generate install manifests
|
||||
@@ -706,11 +771,11 @@ def main() -> None:
|
||||
manifest_cmd.append("--no-refresh-data")
|
||||
if args.target:
|
||||
manifest_cmd.extend(["--target", args.target])
|
||||
ok, _ = run(manifest_cmd, "4b/8 generate install manifests")
|
||||
ok, _ = run(manifest_cmd, STEP_LABELS["generate_manifests"])
|
||||
results["generate_manifests"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 4b/8 generate install manifests: SKIPPED (--skip-packs) ---")
|
||||
skip_notice("generate_manifests", "--skip-packs")
|
||||
results["generate_manifests"] = SKIPPED
|
||||
|
||||
# Step 4c: Generate target manifests
|
||||
@@ -722,11 +787,11 @@ def main() -> None:
|
||||
"--output-dir",
|
||||
"install/targets",
|
||||
]
|
||||
ok, _ = run(target_cmd, "4c/8 generate target manifests")
|
||||
ok, _ = run(target_cmd, STEP_LABELS["generate_target_manifests"])
|
||||
results["generate_target_manifests"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 4c/8 generate target manifests: SKIPPED (--skip-packs) ---")
|
||||
skip_notice("generate_target_manifests", "--skip-packs")
|
||||
results["generate_target_manifests"] = SKIPPED
|
||||
|
||||
# Step 4d: A collected copy that only an emulator's own install provides.
|
||||
@@ -735,7 +800,7 @@ def main() -> None:
|
||||
# what an installation ships.
|
||||
ok, _ = run(
|
||||
[sys.executable, "scripts/bundled_only.py", "--strict"],
|
||||
"4d/8 copies only an emulator install provides",
|
||||
STEP_LABELS["bundled_only"],
|
||||
)
|
||||
results["bundled_only"] = ok
|
||||
all_ok &= ok
|
||||
@@ -745,21 +810,21 @@ def main() -> None:
|
||||
# compare it with, and says so instead of failing on an empty parse.
|
||||
full_pack_built = args.source == "full" or args.all_variants
|
||||
if pack_output and verify_output and not full_pack_built:
|
||||
print("\n--- 5/8 consistency check: SKIPPED (no full pack built) ---")
|
||||
skip_notice("consistency", "no full pack built")
|
||||
results["consistency"] = SKIPPED
|
||||
elif pack_output and verify_output:
|
||||
ok = check_consistency(verify_output, pack_output)
|
||||
results["consistency"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 5/8 consistency check: SKIPPED ---")
|
||||
skip_notice("consistency")
|
||||
results["consistency"] = SKIPPED
|
||||
|
||||
# Step 6: Pack integrity (extract + hash verification)
|
||||
if not args.skip_packs and not full_pack_built:
|
||||
# The check reads the full pack's name, while a reduced pack holds fewer
|
||||
# files than the platform declares by design.
|
||||
print("\n--- 6/8 pack integrity: SKIPPED (no full pack built) ---")
|
||||
skip_notice("pack_integrity", "no full pack built")
|
||||
results["pack_integrity"] = SKIPPED
|
||||
elif not args.skip_packs:
|
||||
integrity_cmd = [
|
||||
@@ -777,11 +842,11 @@ def main() -> None:
|
||||
# the target removed as missing from a pack that was exactly right.
|
||||
if args.target:
|
||||
integrity_cmd.extend(["--target", args.target])
|
||||
ok, _ = run(integrity_cmd, "6/8 pack integrity")
|
||||
ok, _ = run(integrity_cmd, STEP_LABELS["pack_integrity"])
|
||||
results["pack_integrity"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 6/8 pack integrity: SKIPPED (--skip-packs) ---")
|
||||
skip_notice("pack_integrity", "--skip-packs")
|
||||
results["pack_integrity"] = SKIPPED
|
||||
|
||||
# Step 7: Generate README
|
||||
@@ -795,36 +860,36 @@ def main() -> None:
|
||||
"--platforms-dir",
|
||||
"platforms",
|
||||
],
|
||||
"7/8 generate readme",
|
||||
STEP_LABELS["generate_readme"],
|
||||
)
|
||||
results["generate_readme"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 7/8 generate readme: SKIPPED (--skip-docs) ---")
|
||||
skip_notice("generate_readme", "--skip-docs")
|
||||
results["generate_readme"] = SKIPPED
|
||||
|
||||
# Step 8: Generate site pages
|
||||
if not args.skip_docs:
|
||||
ok, _ = run(
|
||||
[sys.executable, "scripts/generate_site.py"],
|
||||
"8/8 generate site",
|
||||
STEP_LABELS["generate_site"],
|
||||
)
|
||||
results["generate_site"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
print("\n--- 8/8 generate site: SKIPPED (--skip-docs) ---")
|
||||
skip_notice("generate_site", "--skip-docs")
|
||||
results["generate_site"] = SKIPPED
|
||||
|
||||
# Step 8b: the render the site deploy runs. Generating the pages without
|
||||
# rendering them let 20 pages with duplicate ids through to CI.
|
||||
if args.skip_docs:
|
||||
print("\n--- 8b render site: SKIPPED (--skip-docs) ---")
|
||||
skip_notice("render_site", "--skip-docs")
|
||||
results["render_site"] = SKIPPED
|
||||
elif shutil.which("mkdocs") is None:
|
||||
print("\n--- 8b render site: SKIPPED (mkdocs not installed) ---")
|
||||
skip_notice("render_site", "mkdocs not installed")
|
||||
results["render_site"] = SKIPPED
|
||||
else:
|
||||
ok, _ = run(["mkdocs", "build", "--strict"], "8b render site")
|
||||
ok, _ = run(["mkdocs", "build", "--strict"], STEP_LABELS["render_site"])
|
||||
if ok:
|
||||
ok, _ = run([sys.executable, "scripts/validate_site.py"], "8b validate site")
|
||||
results["render_site"] = ok
|
||||
@@ -835,8 +900,11 @@ def main() -> None:
|
||||
print(f"\n{'=' * 60}")
|
||||
warned = []
|
||||
for step, outcome in results.items():
|
||||
# The number finds the step's lines in the log, the key is the
|
||||
# name a caller reading the results knows it by.
|
||||
number = step_number(step)
|
||||
if outcome is SKIPPED:
|
||||
print(f" {step:.<40} SKIPPED")
|
||||
print(f" {number:<4} {step:.<40} SKIPPED")
|
||||
continue
|
||||
label = "OK" if outcome else "FAILED"
|
||||
if not outcome and step in NON_BLOCKING:
|
||||
@@ -844,8 +912,8 @@ def main() -> None:
|
||||
# contradiction: the step reports, it does not stop the run.
|
||||
label = "WARN"
|
||||
warned.append(step)
|
||||
print(f" {step:.<40} {label:<7}{results.seconds[step]:>8.1f}s")
|
||||
print(f" {'total':.<40} {'':<7}{total_elapsed:>8.1f}s")
|
||||
print(f" {number:<4} {step:.<40} {label:<7}{results.seconds[step]:>8.1f}s")
|
||||
print(f" {'':<4} {'total':.<40} {'':<7}{total_elapsed:>8.1f}s")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Pipeline {'COMPLETE' if all_ok else 'FINISHED WITH ERRORS'}")
|
||||
if warned:
|
||||
|
||||
@@ -30,7 +30,7 @@ import yaml
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
import upstream
|
||||
from common import load_emulator_profiles, suggest_names
|
||||
from common import github_token, load_emulator_profiles, suggest_names
|
||||
from hashing import parse_hash_list
|
||||
from sourceref import ( # noqa: F401 (re-exported for callers and tests) # noqa: F401 (the prose grammar, re-exported)
|
||||
_ANNOTATION_RE,
|
||||
@@ -3372,12 +3372,13 @@ def select_profiles(profiles: dict, args: argparse.Namespace) -> dict:
|
||||
|
||||
def _check_quota(count: int, offline: bool) -> None:
|
||||
"""Refuse a run the anonymous quota cannot carry to completion."""
|
||||
if offline or os.environ.get("GITHUB_TOKEN"):
|
||||
if offline or github_token():
|
||||
return
|
||||
if count * 4 > ANON_QUOTA:
|
||||
print(
|
||||
f"{count} profiles need roughly {count * 4} API calls, the anonymous "
|
||||
f"quota is {ANON_QUOTA} per hour. Set GITHUB_TOKEN or use --offline.",
|
||||
f"quota is {ANON_QUOTA} per hour. Log in with `gh auth login`, set "
|
||||
"GITHUB_TOKEN, or use --offline.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
@@ -25,7 +25,7 @@ import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from artifacts import file_lock as _file_lock
|
||||
from common import platform_arg, require_platform, yaml_load
|
||||
from common import github_token, platform_arg, require_platform, yaml_load
|
||||
|
||||
try:
|
||||
import yaml
|
||||
@@ -159,7 +159,7 @@ def _api_request(url: str) -> dict:
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
||||
token = github_token()
|
||||
if token and "github" in url:
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
|
||||
|
||||
@@ -289,17 +289,11 @@ def _github_token_env() -> dict[str, str]:
|
||||
emudeck and retropie target scrapers hit the GitHub API. Without a token
|
||||
the unauthenticated rate limit (60/h) is burnt within a few platforms.
|
||||
"""
|
||||
from common import github_token # noqa: PLC0415
|
||||
|
||||
env = os.environ.copy()
|
||||
if env.get("GITHUB_TOKEN"):
|
||||
return env
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["gh", "auth", "token"], capture_output=True, text=True, timeout=10, check=False
|
||||
)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return env
|
||||
token = (proc.stdout or "").strip()
|
||||
if proc.returncode == 0 and token:
|
||||
token = github_token()
|
||||
if token:
|
||||
env["GITHUB_TOKEN"] = token
|
||||
return env
|
||||
|
||||
|
||||
@@ -467,7 +467,11 @@ def _resolve_modes(args: argparse.Namespace) -> None:
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__)
|
||||
parser = argparse.ArgumentParser(
|
||||
allow_abbrev=False,
|
||||
description=__doc__.replace("``", ""),
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--identify", action="store_true", help="label local archives")
|
||||
parser.add_argument(
|
||||
"--missing", action="store_true", help="attempt pinned archives we lack"
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
@@ -16,7 +15,7 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, NoReturn
|
||||
|
||||
from common import write_text_atomic, yaml_load
|
||||
from common import github_token, write_text_atomic, yaml_load
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -273,7 +272,8 @@ def github_headers() -> dict[str, str]:
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
# GITHUB_TOKEN, else the token gh is logged in with.
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
@@ -338,10 +338,11 @@ def _exit_on_scrape_error(error: Exception) -> NoReturn:
|
||||
"""Print what failed and what to try, then exit 1 without a traceback."""
|
||||
print(f"Error: {error}", file=sys.stderr)
|
||||
refused = re.search(r"\b(403|429)\b|rate limit", str(error), re.I)
|
||||
if refused and not os.environ.get("GITHUB_TOKEN"):
|
||||
if refused and not github_token():
|
||||
# A gh login is read on its own, so the hint is the login itself.
|
||||
print(
|
||||
"Anonymous GitHub API calls are limited to 60 an hour. "
|
||||
"Run again with GITHUB_TOKEN=$(gh auth token) set.",
|
||||
"Log in with `gh auth login`, or set GITHUB_TOKEN, and run again.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -20,7 +20,7 @@ from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from common import source_yaml_paths, write_text_atomic, yaml_load
|
||||
from common import github_token, source_yaml_paths, write_text_atomic, yaml_load
|
||||
|
||||
from scripts.scraper._hash_merge import compute_diff, merge_fbneo_profile
|
||||
from scripts.scraper.fbneo_parser import parse_fbneo_source_tree
|
||||
@@ -125,7 +125,7 @@ def _version_from_api() -> str:
|
||||
import urllib.request
|
||||
|
||||
headers = {"User-Agent": "retrobios-scraper/1.0"}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
req = urllib.request.Request(
|
||||
|
||||
@@ -22,7 +22,7 @@ from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from common import source_yaml_paths, write_text_atomic, yaml_load
|
||||
from common import github_token, source_yaml_paths, write_text_atomic, yaml_load
|
||||
|
||||
import upstream
|
||||
|
||||
@@ -144,7 +144,7 @@ def _get_release_tag() -> str:
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
url = "https://api.github.com/repos/mamedev/mame/releases/latest"
|
||||
|
||||
@@ -28,7 +28,6 @@ Verification logic (api_data_processing.sh:289-405):
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
@@ -256,7 +255,9 @@ class Scraper(BaseScraper):
|
||||
|
||||
def _fetch_remote_manifests(self) -> list[tuple[str, dict]]:
|
||||
"""Fetch component manifests via GitHub API."""
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
from common import github_token # noqa: PLC0415
|
||||
|
||||
token = github_token()
|
||||
headers = {"User-Agent": "retrobios-scraper/1.0"}
|
||||
if token:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
|
||||
@@ -9,7 +9,6 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
@@ -18,7 +17,7 @@ from datetime import datetime, timezone
|
||||
|
||||
import yaml
|
||||
|
||||
from common import yaml_load
|
||||
from common import github_token, yaml_load
|
||||
|
||||
from . import BaseTargetScraper
|
||||
|
||||
@@ -51,7 +50,7 @@ _SKIP = {"retroarch_maincfg", "retroarch"}
|
||||
|
||||
def _fetch(url: str) -> str:
|
||||
headers = {"User-Agent": "retrobios-scraper/1.0"}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
token = github_token()
|
||||
if token and "api.github.com" in url:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
try:
|
||||
|
||||
@@ -10,7 +10,6 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
@@ -67,7 +66,9 @@ _MODULE_FLAGS_RE = re.compile(r'rp_module_flags\s*=\s*["\']([^"\']*)["\']')
|
||||
|
||||
def _fetch(url: str, accept: str = "text/plain") -> str:
|
||||
headers = {"User-Agent": "retrobios-scraper/1.0", "Accept": accept}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
from common import github_token # noqa: PLC0415
|
||||
|
||||
token = github_token()
|
||||
if token and "api.github.com" in url:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
try:
|
||||
|
||||
+20
-3
@@ -290,6 +290,7 @@ def _merge_hashes(
|
||||
continue
|
||||
if not ours:
|
||||
existing[h] = theirs
|
||||
existing.setdefault("_hash_by", {})[h] = emu_name
|
||||
continue
|
||||
t_list = theirs if isinstance(theirs, list) else [theirs]
|
||||
o_list = ours if isinstance(ours, list) else [ours]
|
||||
@@ -305,9 +306,13 @@ def _merge_hashes(
|
||||
key = (file_entry["name"], h, str(ours), str(theirs), emu_name)
|
||||
if key not in _CONFLICTS_SHOWN:
|
||||
_CONFLICTS_SHOWN.add(key)
|
||||
# Both cores and which value stays: naming only the second
|
||||
# core sent the reader through ten profiles for the first.
|
||||
first = existing.get("_hash_by", {}).get(h, "an earlier core")
|
||||
print(
|
||||
f"WARNING: hash conflict for {file_entry['name']} "
|
||||
f"({h}: {ours} vs {theirs}, core {emu_name})",
|
||||
f"WARNING: hash conflict for {file_entry['name']}: "
|
||||
f"{h} {ours} ({first}) kept, {theirs} ({emu_name}) "
|
||||
"left out of the model",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
@@ -355,6 +360,9 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
|
||||
entry["_source_refs"] = {_serialize_source_ref(sr)} if sr is not None else set()
|
||||
if db:
|
||||
_enrich_hashes(entry, db)
|
||||
# Which core gave each hash, for the conflict warning. Dropped before
|
||||
# the model is written.
|
||||
entry["_hash_by"] = {h: emu_name for h in _HASH_FIELDS if entry.get(h)}
|
||||
return entry
|
||||
|
||||
|
||||
@@ -685,6 +693,7 @@ def generate_platform_truth(
|
||||
# Convert sets to sorted lists for serialization
|
||||
for sys_id, sys_data in systems.items():
|
||||
for fe in sys_data.get("files", []):
|
||||
fe.pop("_hash_by", None)
|
||||
fe["_cores"] = sorted(fe.get("_cores", set()))
|
||||
fe["_required_by"] = sorted(fe.get("_required_by", set()))
|
||||
fe["_named_by"] = sorted(fe.get("_named_by", set()))
|
||||
@@ -796,6 +805,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
missing.append(
|
||||
{
|
||||
"name": t_entry["name"],
|
||||
"destination": t_entry.get("path") or t_entry["name"],
|
||||
"cores": list(t_entry.get("_cores", [])),
|
||||
"source_refs": list(t_entry.get("_source_refs", [])),
|
||||
}
|
||||
@@ -804,6 +814,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
s_entry = scraped_files[s_position]
|
||||
mismatch = _hash_mismatch(t_entry, s_entry)
|
||||
if mismatch:
|
||||
mismatch["destination"] = s_entry.get("destination") or s_entry["name"]
|
||||
hash_mismatch.append(mismatch)
|
||||
t_req = t_entry.get("required")
|
||||
s_req = s_entry.get("required")
|
||||
@@ -811,6 +822,7 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
required_mismatch.append(
|
||||
{
|
||||
"name": s_entry["name"],
|
||||
"destination": s_entry.get("destination") or s_entry["name"],
|
||||
"truth_required": t_req,
|
||||
"scraped_required": s_req,
|
||||
}
|
||||
@@ -835,7 +847,12 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
if key in seen_extra:
|
||||
continue
|
||||
seen_extra.add(key)
|
||||
entry = {"name": s_entry["name"]}
|
||||
# The destination tells apart two declarations of one name: the
|
||||
# report listed six `hiscore.dat` to add beside one to drop.
|
||||
entry = {
|
||||
"name": s_entry["name"],
|
||||
"destination": s_entry.get("destination") or s_entry["name"],
|
||||
}
|
||||
if has_unprofiled:
|
||||
extra_unprofiled.append(entry)
|
||||
else:
|
||||
|
||||
+7
-3
@@ -206,9 +206,13 @@ def _headers(url: str, accept_json: bool = False) -> dict[str, str]:
|
||||
headers = {"User-Agent": USER_AGENT}
|
||||
if accept_json:
|
||||
headers["Accept"] = "application/json"
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
if token and urllib.parse.urlsplit(url).netloc in GITHUB_HOSTS:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
if urllib.parse.urlsplit(url).netloc in GITHUB_HOSTS:
|
||||
from common import github_token # noqa: PLC0415
|
||||
|
||||
# GITHUB_TOKEN, else the one gh is logged in with.
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
return headers
|
||||
|
||||
|
||||
|
||||
@@ -280,7 +280,11 @@ def validate_site(site: Path, config_path: Path) -> list[str]:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__)
|
||||
parser = argparse.ArgumentParser(
|
||||
allow_abbrev=False,
|
||||
description=__doc__.replace("``", ""),
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--site-dir", default="site",
|
||||
help="the built site (default: %(default)s)",
|
||||
|
||||
+52
-4
@@ -937,17 +937,34 @@ def _resolve_declaration(
|
||||
return local_path, "" if held_by else resolve_status, held_by
|
||||
|
||||
|
||||
def _twin_unmet(result: dict, held_by: str | None) -> bool:
|
||||
def _twin_unmet(
|
||||
result: dict,
|
||||
held_by: str | None,
|
||||
file_entry: dict | None = None,
|
||||
sys_id: str = "",
|
||||
destination: str = "",
|
||||
) -> bool:
|
||||
"""Mark a declaration unmet at a path another declaration holds.
|
||||
|
||||
The path is answered by the winner's file, which the pack counts once.
|
||||
This declaration's failure is reported, never folded into the
|
||||
destination's status.
|
||||
destination's status. The message names the path, both systems and
|
||||
what this declaration expects: "is not met at the path coco holds"
|
||||
named neither the path nor the file wanted there.
|
||||
"""
|
||||
if held_by is None or result["status"] == Status.OK:
|
||||
return False
|
||||
entry = file_entry or {}
|
||||
expects = next(
|
||||
(f" ({h} {str(entry[h]).split(',')[0]})"
|
||||
for h in ("sha1", "md5", "crc32") if entry.get(h)),
|
||||
"",
|
||||
)
|
||||
owner = sys_id or "this"
|
||||
result["discrepancy"] = (
|
||||
f"{result.get('name', '')} is not met at the path {held_by} holds"
|
||||
f"{destination or result.get('name', '')} is declared for {owner}"
|
||||
f" and for {held_by}. The {owner} declaration{expects} is not met at"
|
||||
f" the path, which holds the {held_by} file"
|
||||
)
|
||||
# Read by reports that list entries: the destination itself is held, and
|
||||
# a count by destination does not include this one.
|
||||
@@ -1180,7 +1197,10 @@ def verify_platform(
|
||||
local_path,
|
||||
)
|
||||
shipped_keys.add(_dest_key(file_entry, sys_id))
|
||||
if _twin_unmet(result, held_by):
|
||||
if _twin_unmet(
|
||||
result, held_by, file_entry, sys_id,
|
||||
f"{base_dest}/{destination}" if base_dest else destination,
|
||||
):
|
||||
continue
|
||||
|
||||
# Aggregate by destination, keyed as the builder keys its pack.
|
||||
@@ -1387,6 +1407,26 @@ def _print_ground_truth(gt: list[dict], verbose: bool) -> None:
|
||||
print(f" {_format_ground_truth_aggregate(gt)}")
|
||||
|
||||
|
||||
def _discrepancy_keys(details: list[dict]) -> set[str]:
|
||||
"""The DISCREPANCY lines _print_detail_entries prints for these details.
|
||||
|
||||
A file already reported MISSING or UNTESTED on its own ground is not
|
||||
printed again as a discrepancy, and is not counted as one either.
|
||||
"""
|
||||
shown = {
|
||||
f"{d['system']}/{d['name']}"
|
||||
for d in details
|
||||
if d["status"] == Status.MISSING
|
||||
or (d["status"] == Status.UNTESTED and not d.get("discrepancy"))
|
||||
}
|
||||
return {
|
||||
key
|
||||
for d in details
|
||||
if d.get("discrepancy")
|
||||
and (key := f"{d['system']}/{d['name']}") not in shown
|
||||
}
|
||||
|
||||
|
||||
def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) -> None:
|
||||
"""Print UNTESTED, MISSING, and DISCREPANCY entries from verification details."""
|
||||
for d in details:
|
||||
@@ -2562,6 +2602,14 @@ def _overall_summary(group_results: list[tuple[dict, list[str]]]) -> str:
|
||||
for r, group in group_results
|
||||
}
|
||||
parts.append(f"{sum(counts.values())} {label}{where(counts)}")
|
||||
# Shipped files whose bytes contradict a declaration or a core's own
|
||||
# check: the body printed 19 of them under a last line that read clean.
|
||||
counts = {
|
||||
" / ".join(group): len(_discrepancy_keys(r.get("details", [])))
|
||||
for r, group in group_results
|
||||
}
|
||||
total = sum(counts.values())
|
||||
parts.append(f"{total} {'discrepancy' if total == 1 else 'discrepancies'}{where(counts)}")
|
||||
lines = [f"{platforms} platforms: {', '.join(parts)}"]
|
||||
|
||||
missing = {
|
||||
|
||||
Reference in new issue
Block a user