kmoody/libretro
1
0
Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-04-13 12:22:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: lint and format entire codebase

Browse Source 
Run ruff check --fix: remove unused imports (F401), fix f-strings
without placeholders (F541), remove unused variables (F841), fix
duplicate dict key (F601).

Run isort --profile black: normalize import ordering across all files.

Run ruff format: apply consistent formatting (black-compatible) to
all 58 Python files.

3 intentional E402 remain (imports after require_yaml() must execute
after yaml is available).
This commit is contained in:
Abdessamad Derraz
2026-04-01 13:17:55 +02:00
parent a2d30557e4
commit 0a272dc4e9
56 changed files with 5115 additions and 2679 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
scripts/auto_fetch.py
View File

@@ -21,12 +21,17 @@ import json
import os import os
import subprocess import subprocess
import sys import sys
import urllib.request
import urllib.error import urllib.error
import urllib.request
from pathlib import Path from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from common import list_registered_platforms, load_database, load_platform_config, require_yaml from common import (
list_registered_platforms,
load_database,
load_platform_config,
require_yaml,
)
yaml = require_yaml() yaml = require_yaml()
@@ -83,14 +88,16 @@ def find_missing(config: dict, db: dict) -> list[dict]:
found = any(m in by_md5 for m in md5_list) found = any(m in by_md5 for m in md5_list)
if not found: if not found:
missing.append({ missing.append(
"name": name, {
"system": sys_id, "name": name,
"sha1": sha1, "system": sys_id,
"md5": md5, "sha1": sha1,
"size": file_entry.get("size"), "md5": md5,
"destination": file_entry.get("destination", name), "size": file_entry.get("size"),
}) "destination": file_entry.get("destination", name),
}
)
return missing return missing
@@ -139,14 +146,16 @@ def step2_scan_branches(entry: dict) -> bytes | None:
try: try:
subprocess.run( subprocess.run(
["git", "rev-parse", "--verify", ref], ["git", "rev-parse", "--verify", ref],
capture_output=True, check=True, capture_output=True,
check=True,
) )
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
continue continue
result = subprocess.run( result = subprocess.run(
["git", "ls-tree", "-r", "--name-only", ref], ["git", "ls-tree", "-r", "--name-only", ref],
capture_output=True, text=True, capture_output=True,
text=True,
) )
for filepath in result.stdout.strip().split("\n"): for filepath in result.stdout.strip().split("\n"):
@@ -154,7 +163,8 @@ def step2_scan_branches(entry: dict) -> bytes | None:
try: try:
blob = subprocess.run( blob = subprocess.run(
["git", "show", f"{ref}:{filepath}"], ["git", "show", f"{ref}:{filepath}"],
capture_output=True, check=True, capture_output=True,
check=True,
) )
if verify_content(blob.stdout, entry): if verify_content(blob.stdout, entry):
return blob.stdout return blob.stdout
@@ -172,7 +182,9 @@ def step3_search_public_repos(entry: dict) -> bytes | None:
for url_template in PUBLIC_REPOS: for url_template in PUBLIC_REPOS:
url = url_template.format(name=name) url = url_template.format(name=name)
try: try:
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-fetch/1.0"}) req = urllib.request.Request(
url, headers={"User-Agent": "retrobios-fetch/1.0"}
)
with urllib.request.urlopen(req, timeout=30) as resp: with urllib.request.urlopen(req, timeout=30) as resp:
data = _read_limited(resp) data = _read_limited(resp)
if data is None: if data is None:
@@ -185,7 +197,9 @@ def step3_search_public_repos(entry: dict) -> bytes | None:
if "/" in destination: if "/" in destination:
url = url_template.format(name=destination) url = url_template.format(name=destination)
try: try:
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-fetch/1.0"}) req = urllib.request.Request(
url, headers={"User-Agent": "retrobios-fetch/1.0"}
)
with urllib.request.urlopen(req, timeout=30) as resp: with urllib.request.urlopen(req, timeout=30) as resp:
data = _read_limited(resp) data = _read_limited(resp)
if data is None: if data is None:
@@ -206,7 +220,9 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
for path in [name, f"system/{name}", f"bios/{name}"]: for path in [name, f"system/{name}", f"bios/{name}"]:
url = f"https://archive.org/download/{collection_id}/{path}" url = f"https://archive.org/download/{collection_id}/{path}"
try: try:
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-fetch/1.0"}) req = urllib.request.Request(
url, headers={"User-Agent": "retrobios-fetch/1.0"}
)
with urllib.request.urlopen(req, timeout=30) as resp: with urllib.request.urlopen(req, timeout=30) as resp:
data = _read_limited(resp) data = _read_limited(resp)
if data is None: if data is None:
@@ -221,12 +237,13 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
return None return None
search_url = ( search_url = (
f"https://archive.org/advancedsearch.php?" f"https://archive.org/advancedsearch.php?q=sha1:{sha1}&output=json&rows=1"
f"q=sha1:{sha1}&output=json&rows=1"
) )
try: try:
req = urllib.request.Request(search_url, headers={"User-Agent": "retrobios-fetch/1.0"}) req = urllib.request.Request(
search_url, headers={"User-Agent": "retrobios-fetch/1.0"}
)
with urllib.request.urlopen(req, timeout=30) as resp: with urllib.request.urlopen(req, timeout=30) as resp:
result = json.loads(resp.read()) result = json.loads(resp.read())
docs = result.get("response", {}).get("docs", []) docs = result.get("response", {}).get("docs", [])
@@ -235,7 +252,9 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
if identifier: if identifier:
dl_url = f"https://archive.org/download/{identifier}/{name}" dl_url = f"https://archive.org/download/{identifier}/{name}"
try: try:
req2 = urllib.request.Request(dl_url, headers={"User-Agent": "retrobios-fetch/1.0"}) req2 = urllib.request.Request(
dl_url, headers={"User-Agent": "retrobios-fetch/1.0"}
)
with urllib.request.urlopen(req2, timeout=30) as resp2: with urllib.request.urlopen(req2, timeout=30) as resp2:
data = _read_limited(resp2) data = _read_limited(resp2)
if data is not None and verify_content(data, entry): if data is not None and verify_content(data, entry):
@@ -297,7 +316,7 @@ def fetch_missing(
continue continue
if dry_run: if dry_run:
print(f" [DRY RUN] Would search branches, repos, archive.org") print(" [DRY RUN] Would search branches, repos, archive.org")
still_missing.append(entry) still_missing.append(entry)
stats["not_found"] += 1 stats["not_found"] += 1
continue continue
@@ -323,7 +342,7 @@ def fetch_missing(
stats["found"] += 1 stats["found"] += 1
continue continue
print(f" [5] Not found - needs community contribution") print(" [5] Not found - needs community contribution")
still_missing.append(entry) still_missing.append(entry)
stats["not_found"] += 1 stats["not_found"] += 1
@@ -345,16 +364,20 @@ def generate_issue_body(missing: list[dict], platform: str) -> str:
for entry in missing: for entry in missing:
sha1 = entry.get("sha1") or "N/A" sha1 = entry.get("sha1") or "N/A"
md5 = entry.get("md5") or "N/A" md5 = entry.get("md5") or "N/A"
lines.append(f"| `{entry['name']}` | {entry['system']} | `{sha1[:12]}...` | `{md5[:12]}...` |") lines.append(
f"| `{entry['name']}` | {entry['system']} | `{sha1[:12]}...` | `{md5[:12]}...` |"
)
lines.extend([ lines.extend(
"", [
"### How to Contribute", "",
"", "### How to Contribute",
"1. Fork this repository", "",
"2. Add the BIOS file to `bios/Manufacturer/Console/`", "1. Fork this repository",
"3. Create a Pull Request - checksums are verified automatically", "2. Add the BIOS file to `bios/Manufacturer/Console/`",
]) "3. Create a Pull Request - checksums are verified automatically",
]
)
return "\n".join(lines) return "\n".join(lines)
@@ -363,11 +386,15 @@ def main():
parser = argparse.ArgumentParser(description="Auto-fetch missing BIOS files") parser = argparse.ArgumentParser(description="Auto-fetch missing BIOS files")
parser.add_argument("--platform", "-p", help="Platform to check") parser.add_argument("--platform", "-p", help="Platform to check")
parser.add_argument("--all", action="store_true", help="Check all platforms") parser.add_argument("--all", action="store_true", help="Check all platforms")
parser.add_argument("--dry-run", action="store_true", help="Don't download, just report") parser.add_argument(
"--dry-run", action="store_true", help="Don't download, just report"
)
parser.add_argument("--db", default=DEFAULT_DB) parser.add_argument("--db", default=DEFAULT_DB)
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR) parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR) parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR)
parser.add_argument("--create-issues", action="store_true", help="Output GitHub Issue bodies") parser.add_argument(
"--create-issues", action="store_true", help="Output GitHub Issue bodies"
)
args = parser.parse_args() args = parser.parse_args()
if not os.path.exists(args.db): if not os.path.exists(args.db):
@@ -378,7 +405,8 @@ def main():
if args.all: if args.all:
platforms = list_registered_platforms( platforms = list_registered_platforms(
args.platforms_dir, include_archived=True, args.platforms_dir,
include_archived=True,
) )
elif args.platform: elif args.platform:
platforms = [args.platform] platforms = [args.platform]
@@ -389,19 +417,19 @@ def main():
all_still_missing = {} all_still_missing = {}
for platform in sorted(platforms): for platform in sorted(platforms):
print(f"\n{'='*60}") print(f"\n{'=' * 60}")
print(f"Platform: {platform}") print(f"Platform: {platform}")
print(f"{'='*60}") print(f"{'=' * 60}")
try: try:
config = load_platform_config(platform, args.platforms_dir) config = load_platform_config(platform, args.platforms_dir)
except FileNotFoundError: except FileNotFoundError:
print(f" Config not found, skipping") print(" Config not found, skipping")
continue continue
missing = find_missing(config, db) missing = find_missing(config, db)
if not missing: if not missing:
print(f" All BIOS files present!") print(" All BIOS files present!")
continue continue
print(f" {len(missing)} missing files") print(f" {len(missing)} missing files")
@@ -414,9 +442,9 @@ def main():
print(f"\n Results: {stats['found']} found, {stats['not_found']} not found") print(f"\n Results: {stats['found']} found, {stats['not_found']} not found")
if args.create_issues and all_still_missing: if args.create_issues and all_still_missing:
print(f"\n{'='*60}") print(f"\n{'=' * 60}")
print("GitHub Issue Bodies") print("GitHub Issue Bodies")
print(f"{'='*60}") print(f"{'=' * 60}")
for platform, missing in all_still_missing.items(): for platform, missing in all_still_missing.items():
print(f"\n--- Issue for {platform} ---\n") print(f"\n--- Issue for {platform} ---\n")
print(generate_issue_body(missing, platform)) print(generate_issue_body(missing, platform))

47
scripts/check_buildbot_system.py
View File

@@ -9,6 +9,7 @@ Usage:
python scripts/check_buildbot_system.py --update python scripts/check_buildbot_system.py --update
python scripts/check_buildbot_system.py --json python scripts/check_buildbot_system.py --json
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse
@@ -36,10 +37,14 @@ def fetch_index() -> set[str]:
"""Fetch .index from buildbot, return set of ZIP filenames.""" """Fetch .index from buildbot, return set of ZIP filenames."""
req = urllib.request.Request(INDEX_URL, headers={"User-Agent": USER_AGENT}) req = urllib.request.Request(INDEX_URL, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp: with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
return {line.strip() for line in resp.read().decode().splitlines() if line.strip()} return {
line.strip() for line in resp.read().decode().splitlines() if line.strip()
}
def load_tracked_entries(registry_path: str = DEFAULT_REGISTRY) -> dict[str, tuple[str, str]]: def load_tracked_entries(
registry_path: str = DEFAULT_REGISTRY,
) -> dict[str, tuple[str, str]]:
"""Load buildbot entries from _data_dirs.yml. """Load buildbot entries from _data_dirs.yml.
Returns {decoded_zip_name: (key, source_url)}. Returns {decoded_zip_name: (key, source_url)}.
@@ -64,8 +69,9 @@ def load_tracked_entries(registry_path: str = DEFAULT_REGISTRY) -> dict[str, tup
def get_remote_etag(url: str) -> str | None: def get_remote_etag(url: str) -> str | None:
"""HEAD request to get ETag.""" """HEAD request to get ETag."""
try: try:
req = urllib.request.Request(url, method="HEAD", req = urllib.request.Request(
headers={"User-Agent": USER_AGENT}) url, method="HEAD", headers={"User-Agent": USER_AGENT}
)
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp: with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
return resp.headers.get("ETag") or resp.headers.get("Last-Modified") or "" return resp.headers.get("ETag") or resp.headers.get("Last-Modified") or ""
except (urllib.error.URLError, OSError): except (urllib.error.URLError, OSError):
@@ -114,8 +120,15 @@ def check(registry_path: str = DEFAULT_REGISTRY) -> dict:
status = "OK" status = "OK"
else: else:
status = "UPDATED" status = "UPDATED"
results.append({"zip": z, "status": status, "key": key, results.append(
"stored_etag": stored, "remote_etag": remote or ""}) {
"zip": z,
"status": status,
"key": key,
"stored_etag": stored,
"remote_etag": remote or "",
}
)
return {"entries": results} return {"entries": results}
@@ -144,8 +157,13 @@ def update_changed(report: dict) -> None:
if e["status"] == "UPDATED" and e.get("key"): if e["status"] == "UPDATED" and e.get("key"):
log.info("refreshing %s ...", e["key"]) log.info("refreshing %s ...", e["key"])
subprocess.run( subprocess.run(
[sys.executable, "scripts/refresh_data_dirs.py", [
"--force", "--key", e["key"]], sys.executable,
"scripts/refresh_data_dirs.py",
"--force",
"--key",
e["key"],
],
check=False, check=False,
) )
@@ -155,10 +173,15 @@ def main() -> None:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Check buildbot system directory for changes", description="Check buildbot system directory for changes",
) )
parser.add_argument("--update", action="store_true", parser.add_argument(
help="Auto-refresh changed entries") "--update", action="store_true", help="Auto-refresh changed entries"
parser.add_argument("--json", action="store_true", dest="json_output", )
help="Machine-readable JSON output") parser.add_argument(
"--json",
action="store_true",
dest="json_output",
help="Machine-readable JSON output",
)
parser.add_argument("--registry", default=DEFAULT_REGISTRY) parser.add_argument("--registry", default=DEFAULT_REGISTRY)
args = parser.parse_args() args = parser.parse_args()

135
scripts/common.py
View File

@@ -26,9 +26,11 @@ def require_yaml():
"""Import and return yaml, exiting if PyYAML is not installed.""" """Import and return yaml, exiting if PyYAML is not installed."""
try: try:
import yaml as _yaml import yaml as _yaml
return _yaml return _yaml
except ImportError: except ImportError:
import sys import sys
print("Error: PyYAML required (pip install pyyaml)", file=sys.stderr) print("Error: PyYAML required (pip install pyyaml)", file=sys.stderr)
sys.exit(1) sys.exit(1)
@@ -154,12 +156,17 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
if "inherits" in config: if "inherits" in config:
parent = load_platform_config(config["inherits"], platforms_dir) parent = load_platform_config(config["inherits"], platforms_dir)
merged = {**parent} merged = {**parent}
merged.update({k: v for k, v in config.items() if k not in ("inherits", "overrides")}) merged.update(
{k: v for k, v in config.items() if k not in ("inherits", "overrides")}
)
if "overrides" in config and "systems" in config["overrides"]: if "overrides" in config and "systems" in config["overrides"]:
merged.setdefault("systems", {}) merged.setdefault("systems", {})
for sys_id, override in config["overrides"]["systems"].items(): for sys_id, override in config["overrides"]["systems"].items():
if sys_id in merged["systems"]: if sys_id in merged["systems"]:
merged["systems"][sys_id] = {**merged["systems"][sys_id], **override} merged["systems"][sys_id] = {
**merged["systems"][sys_id],
**override,
}
else: else:
merged["systems"][sys_id] = override merged["systems"][sys_id] = override
config = merged config = merged
@@ -346,12 +353,14 @@ def list_available_targets(
result = [] result = []
for tname, tdata in sorted(data.get("targets", {}).items()): for tname, tdata in sorted(data.get("targets", {}).items()):
aliases = overrides.get(tname, {}).get("aliases", []) aliases = overrides.get(tname, {}).get("aliases", [])
result.append({ result.append(
"name": tname, {
"architecture": tdata.get("architecture", ""), "name": tname,
"core_count": len(tdata.get("cores", [])), "architecture": tdata.get("architecture", ""),
"aliases": aliases, "core_count": len(tdata.get("cores", [])),
}) "aliases": aliases,
}
)
return result return result
@@ -398,7 +407,9 @@ def resolve_local_file(
if hint_base and hint_base not in names_to_try: if hint_base and hint_base not in names_to_try:
names_to_try.append(hint_base) names_to_try.append(hint_base)
md5_list = [m.strip().lower() for m in md5_raw.split(",") if m.strip()] if md5_raw else [] md5_list = (
[m.strip().lower() for m in md5_raw.split(",") if m.strip()] if md5_raw else []
)
files_db = db.get("files", {}) files_db = db.get("files", {})
by_md5 = db.get("indexes", {}).get("by_md5", {}) by_md5 = db.get("indexes", {}).get("by_md5", {})
by_name = db.get("indexes", {}).get("by_name", {}) by_name = db.get("indexes", {}).get("by_name", {})
@@ -480,7 +491,9 @@ def resolve_local_file(
if candidates: if candidates:
if zipped_file: if zipped_file:
candidates = [(p, m) for p, m in candidates if ".zip" in os.path.basename(p)] candidates = [
(p, m) for p, m in candidates if ".zip" in os.path.basename(p)
]
if md5_set: if md5_set:
for path, db_md5 in candidates: for path, db_md5 in candidates:
if ".zip" in os.path.basename(path): if ".zip" in os.path.basename(path):
@@ -530,7 +543,11 @@ def resolve_local_file(
if canonical and canonical != name: if canonical and canonical != name:
canonical_entry = {"name": canonical} canonical_entry = {"name": canonical}
result = resolve_local_file( result = resolve_local_file(
canonical_entry, db, zip_contents, dest_hint, _depth=_depth + 1, canonical_entry,
db,
zip_contents,
dest_hint,
_depth=_depth + 1,
data_dir_registry=data_dir_registry, data_dir_registry=data_dir_registry,
) )
if result[0]: if result[0]:
@@ -643,9 +660,7 @@ def build_zip_contents_index(db: dict, max_entry_size: int = 512 * 1024 * 1024)
if path.endswith(".zip") and os.path.exists(path): if path.endswith(".zip") and os.path.exists(path):
zip_entries.append((path, sha1)) zip_entries.append((path, sha1))
fingerprint = frozenset( fingerprint = frozenset((path, os.path.getmtime(path)) for path, _ in zip_entries)
(path, os.path.getmtime(path)) for path, _ in zip_entries
)
if _zip_contents_cache is not None and _zip_contents_cache[0] == fingerprint: if _zip_contents_cache is not None and _zip_contents_cache[0] == fingerprint:
return _zip_contents_cache[1] return _zip_contents_cache[1]
@@ -672,7 +687,8 @@ _emulator_profiles_cache: dict[tuple[str, bool], dict[str, dict]] = {}
def load_emulator_profiles( def load_emulator_profiles(
emulators_dir: str, skip_aliases: bool = True, emulators_dir: str,
skip_aliases: bool = True,
) -> dict[str, dict]: ) -> dict[str, dict]:
"""Load all emulator YAML profiles from a directory (cached).""" """Load all emulator YAML profiles from a directory (cached)."""
cache_key = (os.path.realpath(emulators_dir), skip_aliases) cache_key = (os.path.realpath(emulators_dir), skip_aliases)
@@ -701,7 +717,8 @@ def load_emulator_profiles(
def group_identical_platforms( def group_identical_platforms(
platforms: list[str], platforms_dir: str, platforms: list[str],
platforms_dir: str,
target_cores_cache: dict[str, set[str] | None] | None = None, target_cores_cache: dict[str, set[str] | None] | None = None,
) -> list[tuple[list[str], str]]: ) -> list[tuple[list[str], str]]:
"""Group platforms that produce identical packs (same files + base_destination). """Group platforms that produce identical packs (same files + base_destination).
@@ -744,7 +761,9 @@ def group_identical_platforms(
fp = hashlib.sha1(f"{fp}|{tc_str}".encode()).hexdigest() fp = hashlib.sha1(f"{fp}|{tc_str}".encode()).hexdigest()
fingerprints.setdefault(fp, []).append(platform) fingerprints.setdefault(fp, []).append(platform)
# Prefer the root platform (no inherits) as representative # Prefer the root platform (no inherits) as representative
if fp not in representatives or (not inherits[platform] and inherits.get(representatives[fp], False)): if fp not in representatives or (
not inherits[platform] and inherits.get(representatives[fp], False)
):
representatives[fp] = platform representatives[fp] = platform
result = [] result = []
@@ -756,7 +775,8 @@ def group_identical_platforms(
def resolve_platform_cores( def resolve_platform_cores(
config: dict, profiles: dict[str, dict], config: dict,
profiles: dict[str, dict],
target_cores: set[str] | None = None, target_cores: set[str] | None = None,
) -> set[str]: ) -> set[str]:
"""Resolve which emulator profiles are relevant for a platform. """Resolve which emulator profiles are relevant for a platform.
@@ -773,9 +793,9 @@ def resolve_platform_cores(
if cores_config == "all_libretro": if cores_config == "all_libretro":
result = { result = {
name for name, p in profiles.items() name
if "libretro" in p.get("type", "") for name, p in profiles.items()
and p.get("type") != "alias" if "libretro" in p.get("type", "") and p.get("type") != "alias"
} }
elif isinstance(cores_config, list): elif isinstance(cores_config, list):
core_set = {str(c) for c in cores_config} core_set = {str(c) for c in cores_config}
@@ -786,25 +806,22 @@ def resolve_platform_cores(
core_to_profile[name] = name core_to_profile[name] = name
for core_name in p.get("cores", []): for core_name in p.get("cores", []):
core_to_profile[str(core_name)] = name core_to_profile[str(core_name)] = name
result = { result = {core_to_profile[c] for c in core_set if c in core_to_profile}
core_to_profile[c]
for c in core_set
if c in core_to_profile
}
# Support "all_libretro" as a list element: combines all libretro # Support "all_libretro" as a list element: combines all libretro
# profiles with explicitly listed standalone cores (e.g. RetroDECK # profiles with explicitly listed standalone cores (e.g. RetroDECK
# ships RetroArch + standalone emulators) # ships RetroArch + standalone emulators)
if "all_libretro" in core_set or "retroarch" in core_set: if "all_libretro" in core_set or "retroarch" in core_set:
result |= { result |= {
name for name, p in profiles.items() name
if "libretro" in p.get("type", "") for name, p in profiles.items()
and p.get("type") != "alias" if "libretro" in p.get("type", "") and p.get("type") != "alias"
} }
else: else:
# Fallback: system ID intersection with normalization # Fallback: system ID intersection with normalization
norm_plat_systems = {_norm_system_id(s) for s in config.get("systems", {})} norm_plat_systems = {_norm_system_id(s) for s in config.get("systems", {})}
result = { result = {
name for name, p in profiles.items() name
for name, p in profiles.items()
if {_norm_system_id(s) for s in p.get("systems", [])} & norm_plat_systems if {_norm_system_id(s) for s in p.get("systems", [])} & norm_plat_systems
and p.get("type") != "alias" and p.get("type") != "alias"
} }
@@ -826,11 +843,34 @@ def resolve_platform_cores(
MANUFACTURER_PREFIXES = ( MANUFACTURER_PREFIXES = (
"acorn-", "apple-", "microsoft-", "nintendo-", "sony-", "sega-", "acorn-",
"snk-", "panasonic-", "nec-", "epoch-", "mattel-", "fairchild-", "apple-",
"hartung-", "tiger-", "magnavox-", "philips-", "bandai-", "casio-", "microsoft-",
"coleco-", "commodore-", "sharp-", "sinclair-", "atari-", "sammy-", "nintendo-",
"gce-", "interton-", "texas-instruments-", "videoton-", "sony-",
"sega-",
"snk-",
"panasonic-",
"nec-",
"epoch-",
"mattel-",
"fairchild-",
"hartung-",
"tiger-",
"magnavox-",
"philips-",
"bandai-",
"casio-",
"coleco-",
"commodore-",
"sharp-",
"sinclair-",
"atari-",
"sammy-",
"gce-",
"interton-",
"texas-instruments-",
"videoton-",
) )
@@ -877,7 +917,7 @@ def _norm_system_id(sid: str) -> str:
s = SYSTEM_ALIASES.get(s, s) s = SYSTEM_ALIASES.get(s, s)
for prefix in MANUFACTURER_PREFIXES: for prefix in MANUFACTURER_PREFIXES:
if s.startswith(prefix): if s.startswith(prefix):
s = s[len(prefix):] s = s[len(prefix) :]
break break
return s.replace("-", "") return s.replace("-", "")
@@ -984,9 +1024,9 @@ def expand_platform_declared_names(config: dict, db: dict) -> set[str]:
import re import re
_TIMESTAMP_PATTERNS = [ _TIMESTAMP_PATTERNS = [
re.compile(r'"generated_at":\s*"[^"]*"'), # database.json re.compile(r'"generated_at":\s*"[^"]*"'), # database.json
re.compile(r'\*Auto-generated on [^*]*\*'), # README.md re.compile(r"\*Auto-generated on [^*]*\*"), # README.md
re.compile(r'\*Generated on [^*]*\*'), # docs site pages re.compile(r"\*Generated on [^*]*\*"), # docs site pages
] ]
@@ -1023,8 +1063,12 @@ LARGE_FILES_REPO = "Abdess/retrobios"
LARGE_FILES_CACHE = ".cache/large" LARGE_FILES_CACHE = ".cache/large"
def fetch_large_file(name: str, dest_dir: str = LARGE_FILES_CACHE, def fetch_large_file(
expected_sha1: str = "", expected_md5: str = "") -> str | None: name: str,
dest_dir: str = LARGE_FILES_CACHE,
expected_sha1: str = "",
expected_md5: str = "",
) -> str | None:
"""Download a large file from the 'large-files' GitHub release if not cached.""" """Download a large file from the 'large-files' GitHub release if not cached."""
cached = os.path.join(dest_dir, name) cached = os.path.join(dest_dir, name)
if os.path.exists(cached): if os.path.exists(cached):
@@ -1033,7 +1077,9 @@ def fetch_large_file(name: str, dest_dir: str = LARGE_FILES_CACHE,
if expected_sha1 and hashes["sha1"].lower() != expected_sha1.lower(): if expected_sha1 and hashes["sha1"].lower() != expected_sha1.lower():
os.unlink(cached) os.unlink(cached)
elif expected_md5: elif expected_md5:
md5_list = [m.strip().lower() for m in expected_md5.split(",") if m.strip()] md5_list = [
m.strip().lower() for m in expected_md5.split(",") if m.strip()
]
if hashes["md5"].lower() not in md5_list: if hashes["md5"].lower() not in md5_list:
os.unlink(cached) os.unlink(cached)
else: else:
@@ -1122,8 +1168,9 @@ def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None:
file_count = len(systems[sys_id].get("files", [])) file_count = len(systems[sys_id].get("files", []))
mfr = systems[sys_id].get("manufacturer", "") mfr = systems[sys_id].get("manufacturer", "")
mfr_display = f" [{mfr.split('|')[0]}]" if mfr else "" mfr_display = f" [{mfr.split('|')[0]}]" if mfr else ""
print(f" {sys_id:35s} ({file_count} file{'s' if file_count != 1 else ''}){mfr_display}") print(
f" {sys_id:35s} ({file_count} file{'s' if file_count != 1 else ''}){mfr_display}"
)
def build_target_cores_cache( def build_target_cores_cache(

55
scripts/cross_reference.py
View File

@@ -19,7 +19,13 @@ import sys
from pathlib import Path from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__)) sys.path.insert(0, os.path.dirname(__file__))
from common import list_registered_platforms, load_database, load_emulator_profiles, load_platform_config, require_yaml from common import (
list_registered_platforms,
load_database,
load_emulator_profiles,
load_platform_config,
require_yaml,
)
yaml = require_yaml() yaml = require_yaml()
@@ -28,11 +34,15 @@ DEFAULT_PLATFORMS_DIR = "platforms"
DEFAULT_DB = "database.json" DEFAULT_DB = "database.json"
def load_platform_files(platforms_dir: str) -> tuple[dict[str, set[str]], dict[str, set[str]]]: def load_platform_files(
platforms_dir: str,
) -> tuple[dict[str, set[str]], dict[str, set[str]]]:
"""Load all platform configs and collect declared filenames + data_directories per system.""" """Load all platform configs and collect declared filenames + data_directories per system."""
declared = {} declared = {}
platform_data_dirs = {} platform_data_dirs = {}
for platform_name in list_registered_platforms(platforms_dir, include_archived=True): for platform_name in list_registered_platforms(
platforms_dir, include_archived=True
):
config = load_platform_config(platform_name, platforms_dir) config = load_platform_config(platform_name, platforms_dir)
for sys_id, system in config.get("systems", {}).items(): for sys_id, system in config.get("systems", {}).items():
for fe in system.get("files", []): for fe in system.get("files", []):
@@ -46,8 +56,9 @@ def load_platform_files(platforms_dir: str) -> tuple[dict[str, set[str]], dict[s
return declared, platform_data_dirs return declared, platform_data_dirs
def _build_supplemental_index(data_root: str = "data", def _build_supplemental_index(
bios_root: str = "bios") -> set[str]: data_root: str = "data", bios_root: str = "bios"
) -> set[str]:
"""Build a set of filenames and directory names in data/ and inside bios/ ZIPs.""" """Build a set of filenames and directory names in data/ and inside bios/ ZIPs."""
names: set[str] = set() names: set[str] = set()
root_path = Path(data_root) root_path = Path(data_root)
@@ -76,12 +87,15 @@ def _build_supplemental_index(data_root: str = "data",
names.add(dpath.name + "/") names.add(dpath.name + "/")
names.add(dpath.name.lower() + "/") names.add(dpath.name.lower() + "/")
import zipfile import zipfile
for zpath in bios_path.rglob("*.zip"): for zpath in bios_path.rglob("*.zip"):
try: try:
with zipfile.ZipFile(zpath) as zf: with zipfile.ZipFile(zpath) as zf:
for member in zf.namelist(): for member in zf.namelist():
if not member.endswith("/"): if not member.endswith("/"):
basename = member.rsplit("/", 1)[-1] if "/" in member else member basename = (
member.rsplit("/", 1)[-1] if "/" in member else member
)
names.add(basename) names.add(basename)
names.add(basename.lower()) names.add(basename.lower())
except (zipfile.BadZipFile, OSError): except (zipfile.BadZipFile, OSError):
@@ -89,8 +103,12 @@ def _build_supplemental_index(data_root: str = "data",
return names return names
def _find_in_repo(fname: str, by_name: dict[str, list], by_name_lower: dict[str, str], def _find_in_repo(
data_names: set[str] | None = None) -> bool: fname: str,
by_name: dict[str, list],
by_name_lower: dict[str, str],
data_names: set[str] | None = None,
) -> bool:
if fname in by_name: if fname in by_name:
return True return True
# For directory entries or paths, extract the meaningful basename # For directory entries or paths, extract the meaningful basename
@@ -170,7 +188,9 @@ def cross_reference(
if not in_repo: if not in_repo:
path_field = f.get("path", "") path_field = f.get("path", "")
if path_field and path_field != fname: if path_field and path_field != fname:
in_repo = _find_in_repo(path_field, by_name, by_name_lower, data_names) in_repo = _find_in_repo(
path_field, by_name, by_name_lower, data_names
)
# Try MD5 hash match (handles files that exist under different names) # Try MD5 hash match (handles files that exist under different names)
if not in_repo: if not in_repo:
md5_raw = f.get("md5", "") md5_raw = f.get("md5", "")
@@ -231,9 +251,11 @@ def print_report(report: dict) -> None:
status = f"{data['gap_in_repo']} in repo, {data['gap_missing']} missing" status = f"{data['gap_in_repo']} in repo, {data['gap_missing']} missing"
print(f"\n{data['emulator']} ({', '.join(data['systems'])})") print(f"\n{data['emulator']} ({', '.join(data['systems'])})")
print(f" {data['total_files']} files in profile, " print(
f"{data['platform_covered']} declared by platforms, " f" {data['total_files']} files in profile, "
f"{gaps} undeclared") f"{data['platform_covered']} declared by platforms, "
f"{gaps} undeclared"
)
if gaps > 0: if gaps > 0:
print(f" Gaps: {status}") print(f" Gaps: {status}")
@@ -259,7 +281,9 @@ def main():
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR) parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
parser.add_argument("--db", default=DEFAULT_DB) parser.add_argument("--db", default=DEFAULT_DB)
parser.add_argument("--emulator", "-e", help="Analyze single emulator") parser.add_argument("--emulator", "-e", help="Analyze single emulator")
parser.add_argument("--platform", "-p", help="Platform name (required for --target)") parser.add_argument(
"--platform", "-p", help="Platform name (required for --target)"
)
parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)") parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)")
parser.add_argument("--json", action="store_true", help="JSON output") parser.add_argument("--json", action="store_true", help="JSON output")
args = parser.parse_args() args = parser.parse_args()
@@ -272,7 +296,10 @@ def main():
if not args.platform: if not args.platform:
parser.error("--target requires --platform") parser.error("--target requires --platform")
from common import load_target_config, resolve_platform_cores from common import load_target_config, resolve_platform_cores
target_cores = load_target_config(args.platform, args.target, args.platforms_dir)
target_cores = load_target_config(
args.platform, args.target, args.platforms_dir
)
config = load_platform_config(args.platform, args.platforms_dir) config = load_platform_config(args.platform, args.platforms_dir)
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores) relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
profiles = {k: v for k, v in profiles.items() if k in relevant} profiles = {k: v for k, v in profiles.items() if k in relevant}

75
scripts/crypto_verify.py
View File

@@ -14,6 +14,7 @@ Source refs:
Azahar src/core/hw/rsa/rsa.cpp Azahar src/core/hw/rsa/rsa.cpp
Azahar src/core/file_sys/otp.cpp Azahar src/core/file_sys/otp.cpp
""" """
from __future__ import annotations from __future__ import annotations
import hashlib import hashlib
@@ -22,9 +23,9 @@ import subprocess
from collections.abc import Callable from collections.abc import Callable
from pathlib import Path from pathlib import Path
# Key file parsing (keys.txt / aes_keys.txt format) # Key file parsing (keys.txt / aes_keys.txt format)
def parse_keys_file(path: str | Path) -> dict[str, dict[str, bytes]]: def parse_keys_file(path: str | Path) -> dict[str, dict[str, bytes]]:
"""Parse a 3DS keys file with :AES, :RSA, :ECC sections. """Parse a 3DS keys file with :AES, :RSA, :ECC sections.
@@ -67,6 +68,7 @@ def find_keys_file(bios_dir: str | Path) -> Path | None:
# Pure Python RSA-2048 PKCS1v15 SHA256 verification (zero dependencies) # Pure Python RSA-2048 PKCS1v15 SHA256 verification (zero dependencies)
def _rsa_verify_pkcs1v15_sha256( def _rsa_verify_pkcs1v15_sha256(
message: bytes, message: bytes,
signature: bytes, signature: bytes,
@@ -98,14 +100,29 @@ def _rsa_verify_pkcs1v15_sha256(
# PKCS#1 v1.5 signature encoding: 0x00 0x01 [0xFF padding] 0x00 [DigestInfo] # PKCS#1 v1.5 signature encoding: 0x00 0x01 [0xFF padding] 0x00 [DigestInfo]
# DigestInfo for SHA-256: # DigestInfo for SHA-256:
# SEQUENCE { SEQUENCE { OID sha256, NULL }, OCTET STRING hash } # SEQUENCE { SEQUENCE { OID sha256, NULL }, OCTET STRING hash }
digest_info_prefix = bytes([ digest_info_prefix = bytes(
0x30, 0x31, # SEQUENCE (49 bytes) [
0x30, 0x0D, # SEQUENCE (13 bytes) 0x30,
0x06, 0x09, # OID (9 bytes) 0x31, # SEQUENCE (49 bytes)
0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, # sha256 0x30,
0x05, 0x00, # NULL 0x0D, # SEQUENCE (13 bytes)
0x04, 0x20, # OCTET STRING (32 bytes) 0x06,
]) 0x09, # OID (9 bytes)
0x60,
0x86,
0x48,
0x01,
0x65,
0x03,
0x04,
0x02,
0x01, # sha256
0x05,
0x00, # NULL
0x04,
0x20, # OCTET STRING (32 bytes)
]
)
sha256_hash = hashlib.sha256(message).digest() sha256_hash = hashlib.sha256(message).digest()
expected_digest_info = digest_info_prefix + sha256_hash expected_digest_info = digest_info_prefix + sha256_hash
@@ -122,11 +139,13 @@ def _rsa_verify_pkcs1v15_sha256(
# AES-128-CBC decryption (with fallback) # AES-128-CBC decryption (with fallback)
def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes: def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
"""Decrypt AES-128-CBC without padding.""" """Decrypt AES-128-CBC without padding."""
# Try cryptography library first # Try cryptography library first
try: try:
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
decryptor = cipher.decryptor() decryptor = cipher.decryptor()
return decryptor.update(data) + decryptor.finalize() return decryptor.update(data) + decryptor.finalize()
@@ -136,6 +155,7 @@ def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
# Try pycryptodome # Try pycryptodome
try: try:
from Crypto.Cipher import AES # type: ignore[import-untyped] from Crypto.Cipher import AES # type: ignore[import-untyped]
cipher = AES.new(key, AES.MODE_CBC, iv) cipher = AES.new(key, AES.MODE_CBC, iv)
return cipher.decrypt(data) return cipher.decrypt(data)
except ImportError: except ImportError:
@@ -145,8 +165,15 @@ def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
try: try:
result = subprocess.run( result = subprocess.run(
[ [
"openssl", "enc", "-aes-128-cbc", "-d", "openssl",
"-K", key.hex(), "-iv", iv.hex(), "-nopad", "enc",
"-aes-128-cbc",
"-d",
"-K",
key.hex(),
"-iv",
iv.hex(),
"-nopad",
], ],
input=data, input=data,
capture_output=True, capture_output=True,
@@ -162,6 +189,7 @@ def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
# File verification functions # File verification functions
def verify_secure_info_a( def verify_secure_info_a(
filepath: str | Path, filepath: str | Path,
keys: dict[str, dict[str, bytes]], keys: dict[str, dict[str, bytes]],
@@ -204,7 +232,10 @@ def verify_secure_info_a(
continue continue
modified_body = bytes([test_region]) + body[1:] modified_body = bytes([test_region]) + body[1:]
if _rsa_verify_pkcs1v15_sha256(modified_body, signature, modulus, exponent): if _rsa_verify_pkcs1v15_sha256(modified_body, signature, modulus, exponent):
return False, f"signature invalid (region changed from {test_region} to {region_byte})" return (
False,
f"signature invalid (region changed from {test_region} to {region_byte})",
)
return False, "signature invalid" return False, "signature invalid"
@@ -307,7 +338,7 @@ def verify_otp(
Returns (valid, reason_string). Returns (valid, reason_string).
""" """
from sect233r1 import ecdsa_verify_sha256, _ec_mul, _Gx, _Gy, _N from sect233r1 import _N, _ec_mul, _Gx, _Gy, ecdsa_verify_sha256
data = bytearray(Path(filepath).read_bytes()) data = bytearray(Path(filepath).read_bytes())
@@ -322,7 +353,10 @@ def verify_otp(
magic = struct.unpack_from("<I", data, 0)[0] magic = struct.unpack_from("<I", data, 0)[0]
if magic != 0xDEADB00F: if magic != 0xDEADB00F:
if not otp_key or not otp_iv: if not otp_key or not otp_iv:
return False, "encrypted OTP but missing AES keys (otpKey/otpIV) in keys file" return (
False,
"encrypted OTP but missing AES keys (otpKey/otpIV) in keys file",
)
try: try:
data = bytearray(_aes_128_cbc_decrypt(bytes(data), otp_key, otp_iv)) data = bytearray(_aes_128_cbc_decrypt(bytes(data), otp_key, otp_iv))
except RuntimeError as e: except RuntimeError as e:
@@ -343,7 +377,10 @@ def verify_otp(
ecc_keys = keys.get("ECC", {}) ecc_keys = keys.get("ECC", {})
root_public_xy = ecc_keys.get("rootPublicXY") root_public_xy = ecc_keys.get("rootPublicXY")
if not root_public_xy or len(root_public_xy) != 60: if not root_public_xy or len(root_public_xy) != 60:
return True, "decrypted, magic valid, SHA-256 valid (ECC skipped: no rootPublicXY)" return (
True,
"decrypted, magic valid, SHA-256 valid (ECC skipped: no rootPublicXY)",
)
# Extract CTCert fields from OTP body # Extract CTCert fields from OTP body
device_id = struct.unpack_from("<I", data, 0x04)[0] device_id = struct.unpack_from("<I", data, 0x04)[0]
@@ -368,9 +405,7 @@ def verify_otp(
pub_point = _ec_mul(priv_key_int, (_Gx, _Gy)) pub_point = _ec_mul(priv_key_int, (_Gx, _Gy))
if pub_point is None: if pub_point is None:
return False, "ECC cert: derived public key is point at infinity" return False, "ECC cert: derived public key is point at infinity"
pub_key_xy = ( pub_key_xy = pub_point[0].to_bytes(30, "big") + pub_point[1].to_bytes(30, "big")
pub_point[0].to_bytes(30, "big") + pub_point[1].to_bytes(30, "big")
)
# Build certificate body (what was signed) # Build certificate body (what was signed)
# Issuer: "Nintendo CA - G3_NintendoCTR2prod" or "...dev" # Issuer: "Nintendo CA - G3_NintendoCTR2prod" or "...dev"
@@ -379,12 +414,12 @@ def verify_otp(
issuer_str = b"Nintendo CA - G3_NintendoCTR2prod" issuer_str = b"Nintendo CA - G3_NintendoCTR2prod"
else: else:
issuer_str = b"Nintendo CA - G3_NintendoCTR2dev" issuer_str = b"Nintendo CA - G3_NintendoCTR2dev"
issuer[:len(issuer_str)] = issuer_str issuer[: len(issuer_str)] = issuer_str
# Name: "CT{device_id:08X}-{system_type:02X}" # Name: "CT{device_id:08X}-{system_type:02X}"
name = bytearray(0x40) name = bytearray(0x40)
name_str = f"CT{device_id:08X}-{system_type:02X}".encode() name_str = f"CT{device_id:08X}-{system_type:02X}".encode()
name[:len(name_str)] = name_str name[: len(name_str)] = name_str
# Key type = 2 (ECC), big-endian u32 # Key type = 2 (ECC), big-endian u32
key_type = struct.pack(">I", 2) key_type = struct.pack(">I", 2)

32
scripts/dedup.py
View File

@@ -17,6 +17,7 @@ Two types of deduplication:
After dedup, run generate_db.py --force to rebuild database indexes. After dedup, run generate_db.py --force to rebuild database indexes.
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse
@@ -110,13 +111,10 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
unique_names = sorted(by_name.keys()) unique_names = sorted(by_name.keys())
if len(unique_names) > 1: if len(unique_names) > 1:
# Check if these are all in MAME/Arcade dirs AND all ZIPs # Check if these are all in MAME/Arcade dirs AND all ZIPs
all_mame_zip = ( all_mame_zip = all(
all( any(_is_mame_dir(p) for p in name_paths)
any(_is_mame_dir(p) for p in name_paths) for name_paths in by_name.values()
for name_paths in by_name.values() ) and all(n.endswith(".zip") for n in unique_names)
)
and all(n.endswith(".zip") for n in unique_names)
)
if all_mame_zip: if all_mame_zip:
# MAME device clones: different ZIP names, same ROM content # MAME device clones: different ZIP names, same ROM content
# Keep one canonical, remove clones, record in clone map # Keep one canonical, remove clones, record in clone map
@@ -202,7 +200,9 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
prefix = "Would remove" if dry_run else "Removed" prefix = "Would remove" if dry_run else "Removed"
print(f"\n{prefix}: {total_removed} files") print(f"\n{prefix}: {total_removed} files")
print(f"Space {'to save' if dry_run else 'saved'}: {total_saved / 1024 / 1024:.1f} MB") print(
f"Space {'to save' if dry_run else 'saved'}: {total_saved / 1024 / 1024:.1f} MB"
)
if not dry_run and empty_cleaned: if not dry_run and empty_cleaned:
print(f"Cleaned {empty_cleaned} empty directories") print(f"Cleaned {empty_cleaned} empty directories")
@@ -211,21 +211,27 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
clone_path = "_mame_clones.json" clone_path = "_mame_clones.json"
if dry_run: if dry_run:
print(f"\nWould write MAME clone map: {clone_path}") print(f"\nWould write MAME clone map: {clone_path}")
print(f" {len(mame_clones)} canonical ZIPs with " print(
f"{sum(len(v['clones']) for v in mame_clones.values())} clones") f" {len(mame_clones)} canonical ZIPs with "
f"{sum(len(v['clones']) for v in mame_clones.values())} clones"
)
else: else:
with open(clone_path, "w") as f: with open(clone_path, "w") as f:
json.dump(mame_clones, f, indent=2, sort_keys=True) json.dump(mame_clones, f, indent=2, sort_keys=True)
print(f"\nWrote MAME clone map: {clone_path}") print(f"\nWrote MAME clone map: {clone_path}")
print(f" {len(mame_clones)} canonical ZIPs with " print(
f"{sum(len(v['clones']) for v in mame_clones.values())} clones") f" {len(mame_clones)} canonical ZIPs with "
f"{sum(len(v['clones']) for v in mame_clones.values())} clones"
)
return results return results
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser(description="Deduplicate bios/ directory") parser = argparse.ArgumentParser(description="Deduplicate bios/ directory")
parser.add_argument("--dry-run", action="store_true", help="Preview without deleting") parser.add_argument(
"--dry-run", action="store_true", help="Preview without deleting"
)
parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR) parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR)
args = parser.parse_args() args = parser.parse_args()

24
scripts/deterministic_zip.py
View File

@@ -22,10 +22,10 @@ Usage:
] ]
build_deterministic_zip("neogeo.zip", recipe, atom_store) build_deterministic_zip("neogeo.zip", recipe, atom_store)
""" """
from __future__ import annotations from __future__ import annotations
import hashlib import hashlib
import struct
import zipfile import zipfile
import zlib import zlib
from io import BytesIO from io import BytesIO
@@ -63,7 +63,9 @@ def build_deterministic_zip(
# Sort by filename for deterministic order # Sort by filename for deterministic order
sorted_recipe = sorted(recipe, key=lambda r: r["name"]) sorted_recipe = sorted(recipe, key=lambda r: r["name"])
with zipfile.ZipFile(str(output_path), "w", compression, compresslevel=_COMPRESS_LEVEL) as zf: with zipfile.ZipFile(
str(output_path), "w", compression, compresslevel=_COMPRESS_LEVEL
) as zf:
for entry in sorted_recipe: for entry in sorted_recipe:
name = entry["name"] name = entry["name"]
expected_crc = entry.get("crc32", "").lower() expected_crc = entry.get("crc32", "").lower()
@@ -127,12 +129,14 @@ def extract_atoms_with_names(zip_path: str | Path) -> list[dict]:
continue continue
data = zf.read(info.filename) data = zf.read(info.filename)
crc = format(zlib.crc32(data) & 0xFFFFFFFF, "08x") crc = format(zlib.crc32(data) & 0xFFFFFFFF, "08x")
result.append({ result.append(
"name": info.filename, {
"crc32": crc, "name": info.filename,
"size": len(data), "crc32": crc,
"data": data, "size": len(data),
}) "data": data,
}
)
return result return result
@@ -154,7 +158,9 @@ def verify_zip_determinism(zip_path: str | Path) -> tuple[bool, str, str]:
# Rebuild to memory # Rebuild to memory
buf = BytesIO() buf = BytesIO()
sorted_recipe = sorted(recipe, key=lambda r: r["name"]) sorted_recipe = sorted(recipe, key=lambda r: r["name"])
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED, compresslevel=_COMPRESS_LEVEL) as zf: with zipfile.ZipFile(
buf, "w", zipfile.ZIP_DEFLATED, compresslevel=_COMPRESS_LEVEL
) as zf:
for entry in sorted_recipe: for entry in sorted_recipe:
info = zipfile.ZipInfo(filename=entry["name"], date_time=_FIXED_DATE_TIME) info = zipfile.ZipInfo(filename=entry["name"], date_time=_FIXED_DATE_TIME)
info.compress_type = zipfile.ZIP_DEFLATED info.compress_type = zipfile.ZIP_DEFLATED

37
scripts/diff_truth.py
View File

@@ -78,13 +78,17 @@ def _format_terminal(report: dict) -> str:
lines.append(f" + {m['name']} [{cores}]") lines.append(f" + {m['name']} [{cores}]")
for h in div.get("hash_mismatch", []): for h in div.get("hash_mismatch", []):
ht = h["hash_type"] ht = h["hash_type"]
lines.append(f" ~ {h['name']} {ht}: {h[f'truth_{ht}']} != {h[f'scraped_{ht}']}") lines.append(
f" ~ {h['name']} {ht}: {h[f'truth_{ht}']} != {h[f'scraped_{ht}']}"
)
for p in div.get("extra_phantom", []): for p in div.get("extra_phantom", []):
lines.append(f" - {p['name']} (phantom)") lines.append(f" - {p['name']} (phantom)")
for u in div.get("extra_unprofiled", []): for u in div.get("extra_unprofiled", []):
lines.append(f" ? {u['name']} (unprofiled)") lines.append(f" ? {u['name']} (unprofiled)")
for r in div.get("required_mismatch", []): for r in div.get("required_mismatch", []):
lines.append(f" ! {r['name']} required: {r['truth_required']} != {r['scraped_required']}") lines.append(
f" ! {r['name']} required: {r['truth_required']} != {r['scraped_required']}"
)
uncovered = report.get("uncovered_systems", []) uncovered = report.get("uncovered_systems", [])
if uncovered: if uncovered:
@@ -125,13 +129,17 @@ def _format_markdown(report: dict) -> str:
lines.append(f"- **Add** `{m['name']}`{refs}") lines.append(f"- **Add** `{m['name']}`{refs}")
for h in div.get("hash_mismatch", []): for h in div.get("hash_mismatch", []):
ht = h["hash_type"] ht = h["hash_type"]
lines.append(f"- **Fix hash** `{h['name']}` {ht}: `{h[f'truth_{ht}']}` != `{h[f'scraped_{ht}']}`") lines.append(
f"- **Fix hash** `{h['name']}` {ht}: `{h[f'truth_{ht}']}` != `{h[f'scraped_{ht}']}`"
)
for p in div.get("extra_phantom", []): for p in div.get("extra_phantom", []):
lines.append(f"- **Remove** `{p['name']}` (phantom)") lines.append(f"- **Remove** `{p['name']}` (phantom)")
for u in div.get("extra_unprofiled", []): for u in div.get("extra_unprofiled", []):
lines.append(f"- **Check** `{u['name']}` (unprofiled cores)") lines.append(f"- **Check** `{u['name']}` (unprofiled cores)")
for r in div.get("required_mismatch", []): for r in div.get("required_mismatch", []):
lines.append(f"- **Fix required** `{r['name']}`: truth={r['truth_required']}, scraped={r['scraped_required']}") lines.append(
f"- **Fix required** `{r['name']}`: truth={r['truth_required']}, scraped={r['scraped_required']}"
)
lines.append("") lines.append("")
uncovered = report.get("uncovered_systems", []) uncovered = report.get("uncovered_systems", [])
@@ -148,17 +156,25 @@ def _format_markdown(report: dict) -> str:
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser(description="Compare scraped vs truth YAMLs") parser = argparse.ArgumentParser(description="Compare scraped vs truth YAMLs")
group = parser.add_mutually_exclusive_group(required=True) group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--all", action="store_true", help="diff all registered platforms") group.add_argument(
"--all", action="store_true", help="diff all registered platforms"
)
group.add_argument("--platform", help="diff a single platform") group.add_argument("--platform", help="diff a single platform")
parser.add_argument("--json", action="store_true", dest="json_output", help="JSON output") parser.add_argument(
parser.add_argument("--format", choices=["terminal", "markdown"], default="terminal") "--json", action="store_true", dest="json_output", help="JSON output"
)
parser.add_argument(
"--format", choices=["terminal", "markdown"], default="terminal"
)
parser.add_argument("--truth-dir", default="dist/truth") parser.add_argument("--truth-dir", default="dist/truth")
parser.add_argument("--platforms-dir", default="platforms") parser.add_argument("--platforms-dir", default="platforms")
parser.add_argument("--include-archived", action="store_true") parser.add_argument("--include-archived", action="store_true")
args = parser.parse_args() args = parser.parse_args()
if args.all: if args.all:
platforms = list_registered_platforms(args.platforms_dir, include_archived=args.include_archived) platforms = list_registered_platforms(
args.platforms_dir, include_archived=args.include_archived
)
else: else:
platforms = [args.platform] platforms = [args.platform]
@@ -169,7 +185,10 @@ def main() -> None:
truth = _load_truth(args.truth_dir, platform) truth = _load_truth(args.truth_dir, platform)
if truth is None: if truth is None:
if not args.json_output: if not args.json_output:
print(f"skip {platform}: no truth YAML in {args.truth_dir}/", file=sys.stderr) print(
f"skip {platform}: no truth YAML in {args.truth_dir}/",
file=sys.stderr,
)
continue continue
try: try:

42
scripts/download.py
View File

@@ -16,8 +16,8 @@ import argparse
import json import json
import os import os
import sys import sys
import urllib.request
import urllib.error import urllib.error
import urllib.request
import zipfile import zipfile
from pathlib import Path from pathlib import Path
@@ -31,10 +31,13 @@ REPO = "Abdess/retrobios"
def get_latest_release() -> dict: def get_latest_release() -> dict:
"""Fetch latest release info from GitHub API.""" """Fetch latest release info from GitHub API."""
url = f"{GITHUB_API}/repos/{REPO}/releases/latest" url = f"{GITHUB_API}/repos/{REPO}/releases/latest"
req = urllib.request.Request(url, headers={ req = urllib.request.Request(
"User-Agent": "retrobios-downloader/1.0", url,
"Accept": "application/vnd.github.v3+json", headers={
}) "User-Agent": "retrobios-downloader/1.0",
"Accept": "application/vnd.github.v3+json",
},
)
try: try:
with urllib.request.urlopen(req, timeout=30) as resp: with urllib.request.urlopen(req, timeout=30) as resp:
@@ -71,7 +74,9 @@ def find_asset(release: dict, platform: str) -> dict | None:
def download_file(url: str, dest: str, expected_size: int = 0): def download_file(url: str, dest: str, expected_size: int = 0):
"""Download a file with progress indication.""" """Download a file with progress indication."""
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-downloader/1.0"}) req = urllib.request.Request(
url, headers={"User-Agent": "retrobios-downloader/1.0"}
)
with urllib.request.urlopen(req, timeout=300) as resp: with urllib.request.urlopen(req, timeout=300) as resp:
total = int(resp.headers.get("Content-Length", expected_size)) total = int(resp.headers.get("Content-Length", expected_size))
@@ -88,7 +93,11 @@ def download_file(url: str, dest: str, expected_size: int = 0):
if total > 0: if total > 0:
pct = downloaded * 100 // total pct = downloaded * 100 // total
bar = "=" * (pct // 2) + " " * (50 - pct // 2) bar = "=" * (pct // 2) + " " * (50 - pct // 2)
print(f"\r [{bar}] {pct}% ({downloaded:,}/{total:,})", end="", flush=True) print(
f"\r [{bar}] {pct}% ({downloaded:,}/{total:,})",
end="",
flush=True,
)
print() print()
@@ -114,11 +123,14 @@ def verify_files(platform: str, dest_dir: str, release: dict):
return return
import tempfile import tempfile
tmp = tempfile.NamedTemporaryFile(suffix=".json", delete=False) tmp = tempfile.NamedTemporaryFile(suffix=".json", delete=False)
tmp.close() tmp.close()
try: try:
download_file(db_asset["browser_download_url"], tmp.name, db_asset.get("size", 0)) download_file(
db_asset["browser_download_url"], tmp.name, db_asset.get("size", 0)
)
with open(tmp.name) as f: with open(tmp.name) as f:
db = json.load(f) db = json.load(f)
finally: finally:
@@ -142,7 +154,9 @@ def verify_files(platform: str, dest_dir: str, release: dict):
break break
else: else:
mismatched += 1 mismatched += 1
print(f" MISMATCH: {name} (expected {sha1[:12]}..., got {local_sha1[:12]}...)") print(
f" MISMATCH: {name} (expected {sha1[:12]}..., got {local_sha1[:12]}...)"
)
found = True found = True
break break
@@ -166,7 +180,7 @@ def show_info(platform: str, release: dict):
print(f" Platform: {platform}") print(f" Platform: {platform}")
print(f" File: {asset['name']}") print(f" File: {asset['name']}")
print(f" Size: {asset['size']:,} bytes ({asset['size'] / (1024*1024):.1f} MB)") print(f" Size: {asset['size']:,} bytes ({asset['size'] / (1024 * 1024):.1f} MB)")
print(f" Downloads: {asset.get('download_count', 'N/A')}") print(f" Downloads: {asset.get('download_count', 'N/A')}")
print(f" Updated: {asset.get('updated_at', 'N/A')}") print(f" Updated: {asset.get('updated_at', 'N/A')}")
@@ -200,7 +214,12 @@ Examples:
print(f" - {p}") print(f" - {p}")
else: else:
print("No platform packs found in latest release") print("No platform packs found in latest release")
except (urllib.error.URLError, urllib.error.HTTPError, OSError, json.JSONDecodeError) as e: except (
urllib.error.URLError,
urllib.error.HTTPError,
OSError,
json.JSONDecodeError,
) as e:
print(f"Error: {e}") print(f"Error: {e}")
return return
@@ -233,6 +252,7 @@ Examples:
sys.exit(1) sys.exit(1)
import tempfile import tempfile
fd, zip_path = tempfile.mkstemp(suffix=".zip") fd, zip_path = tempfile.mkstemp(suffix=".zip")
os.close(fd) os.close(fd)

20
scripts/export_native.py
View File

@@ -9,11 +9,9 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.path.insert(0, str(Path(__file__).resolve().parent))
import yaml import yaml
from common import list_registered_platforms, load_platform_config from common import list_registered_platforms, load_platform_config
from exporter import discover_exporters from exporter import discover_exporters
OUTPUT_FILENAMES: dict[str, str] = { OUTPUT_FILENAMES: dict[str, str] = {
"retroarch": "System.dat", "retroarch": "System.dat",
"lakka": "System.dat", "lakka": "System.dat",
@@ -94,23 +92,31 @@ def main() -> None:
group.add_argument("--all", action="store_true", help="export all platforms") group.add_argument("--all", action="store_true", help="export all platforms")
group.add_argument("--platform", help="export a single platform") group.add_argument("--platform", help="export a single platform")
parser.add_argument( parser.add_argument(
"--output-dir", default="dist/upstream", help="output directory", "--output-dir",
default="dist/upstream",
help="output directory",
) )
parser.add_argument( parser.add_argument(
"--truth-dir", default="dist/truth", help="truth YAML directory", "--truth-dir",
default="dist/truth",
help="truth YAML directory",
) )
parser.add_argument( parser.add_argument(
"--platforms-dir", default="platforms", help="platform configs directory", "--platforms-dir",
default="platforms",
help="platform configs directory",
) )
parser.add_argument( parser.add_argument(
"--include-archived", action="store_true", "--include-archived",
action="store_true",
help="include archived platforms", help="include archived platforms",
) )
args = parser.parse_args() args = parser.parse_args()
if args.all: if args.all:
platforms = list_registered_platforms( platforms = list_registered_platforms(
args.platforms_dir, include_archived=args.include_archived, args.platforms_dir,
include_archived=args.include_archived,
) )
else: else:
platforms = [args.platform] platforms = [args.platform]

28
scripts/exporter/base_exporter.py
View File

@@ -38,7 +38,8 @@ class BaseExporter(ABC):
@staticmethod @staticmethod
def _display_name( def _display_name(
sys_id: str, scraped_sys: dict | None = None, sys_id: str,
scraped_sys: dict | None = None,
) -> str: ) -> str:
"""Get display name for a system from scraped data or slug.""" """Get display name for a system from scraped data or slug."""
if scraped_sys: if scraped_sys:
@@ -47,9 +48,28 @@ class BaseExporter(ABC):
return name return name
# Fallback: convert slug to display name with acronym handling # Fallback: convert slug to display name with acronym handling
_UPPER = { _UPPER = {
"3do", "cdi", "cpc", "cps1", "cps2", "cps3", "dos", "gba", "3do",
"gbc", "hle", "msx", "nes", "nds", "ngp", "psp", "psx", "cdi",
"sms", "snes", "stv", "tvc", "vb", "zx", "cpc",
"cps1",
"cps2",
"cps3",
"dos",
"gba",
"gbc",
"hle",
"msx",
"nes",
"nds",
"ngp",
"psp",
"psx",
"sms",
"snes",
"stv",
"tvc",
"vb",
"zx",
} }
parts = sys_id.replace("-", " ").split() parts = sys_id.replace("-", " ").split()
result = [] result = []

10
scripts/exporter/batocera_exporter.py
View File

@@ -11,8 +11,6 @@ from pathlib import Path
from .base_exporter import BaseExporter from .base_exporter import BaseExporter
class Exporter(BaseExporter): class Exporter(BaseExporter):
"""Export truth data to Batocera batocera-systems format.""" """Export truth data to Batocera batocera-systems format."""
@@ -44,7 +42,9 @@ class Exporter(BaseExporter):
continue continue
native_id = native_map.get(sys_id, sys_id) native_id = native_map.get(sys_id, sys_id)
scraped_sys = scraped_data.get("systems", {}).get(sys_id) if scraped_data else None scraped_sys = (
scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
)
display_name = self._display_name(sys_id, scraped_sys) display_name = self._display_name(sys_id, scraped_sys)
# Build md5 lookup from scraped data for this system # Build md5 lookup from scraped data for this system
@@ -74,9 +74,7 @@ class Exporter(BaseExporter):
# Original format requires md5 for every entry — skip without # Original format requires md5 for every entry — skip without
if not md5: if not md5:
continue continue
bios_parts.append( bios_parts.append(f'{{ "md5": "{md5}", "file": "bios/{dest}" }}')
f'{{ "md5": "{md5}", "file": "bios/{dest}" }}'
)
bios_str = ", ".join(bios_parts) bios_str = ", ".join(bios_parts)
line = ( line = (

7
scripts/exporter/emudeck_exporter.py
View File

@@ -156,7 +156,9 @@ class Exporter(BaseExporter):
continue continue
md5 = fe.get("md5", "") md5 = fe.get("md5", "")
if isinstance(md5, list): if isinstance(md5, list):
md5s.extend(m for m in md5 if m and re.fullmatch(r"[a-f0-9]{32}", m)) md5s.extend(
m for m in md5 if m and re.fullmatch(r"[a-f0-9]{32}", m)
)
elif md5 and re.fullmatch(r"[a-f0-9]{32}", md5): elif md5 and re.fullmatch(r"[a-f0-9]{32}", md5):
md5s.append(md5) md5s.append(md5)
if md5s: if md5s:
@@ -195,7 +197,8 @@ class Exporter(BaseExporter):
# Only flag if the system has usable data for the function type # Only flag if the system has usable data for the function type
if cfg["pattern"] == "md5": if cfg["pattern"] == "md5":
has_md5 = any( has_md5 = any(
fe.get("md5") and isinstance(fe.get("md5"), str) fe.get("md5")
and isinstance(fe.get("md5"), str)
and re.fullmatch(r"[a-f0-9]{32}", fe["md5"]) and re.fullmatch(r"[a-f0-9]{32}", fe["md5"])
for fe in sys_data["files"] for fe in sys_data["files"]
) )

17
scripts/exporter/recalbox_exporter.py
View File

@@ -15,8 +15,6 @@ from pathlib import Path
from .base_exporter import BaseExporter from .base_exporter import BaseExporter
class Exporter(BaseExporter): class Exporter(BaseExporter):
"""Export truth data to Recalbox es_bios.xml format.""" """Export truth data to Recalbox es_bios.xml format."""
@@ -51,7 +49,9 @@ class Exporter(BaseExporter):
continue continue
native_id = native_map.get(sys_id, sys_id) native_id = native_map.get(sys_id, sys_id)
scraped_sys = scraped_data.get("systems", {}).get(sys_id) if scraped_data else None scraped_sys = (
scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
)
display_name = self._display_name(sys_id, scraped_sys) display_name = self._display_name(sys_id, scraped_sys)
lines.append(f' <system fullname="{display_name}" platform="{native_id}">') lines.append(f' <system fullname="{display_name}" platform="{native_id}">')
@@ -85,7 +85,9 @@ class Exporter(BaseExporter):
# Build cores string from _cores # Build cores string from _cores
cores_list = fe.get("_cores", []) cores_list = fe.get("_cores", [])
core_str = ",".join(f"libretro/{c}" for c in cores_list) if cores_list else "" core_str = (
",".join(f"libretro/{c}" for c in cores_list) if cores_list else ""
)
attrs = [f'path="{path}"'] attrs = [f'path="{path}"']
if md5: if md5:
@@ -97,7 +99,7 @@ class Exporter(BaseExporter):
if core_str: if core_str:
attrs.append(f'core="{core_str}"') attrs.append(f'core="{core_str}"')
lines.append(f' <bios {" ".join(attrs)} />') lines.append(f" <bios {' '.join(attrs)} />")
lines.append(" </system>") lines.append(" </system>")
@@ -125,6 +127,9 @@ class Exporter(BaseExporter):
if name.startswith("_") or self._is_pattern(name): if name.startswith("_") or self._is_pattern(name):
continue continue
dest = self._dest(fe) dest = self._dest(fe)
if name.lower() not in exported_paths and dest.lower() not in exported_paths: if (
name.lower() not in exported_paths
and dest.lower() not in exported_paths
):
issues.append(f"missing: {name}") issues.append(f"missing: {name}")
return issues return issues

10
scripts/exporter/retrobat_exporter.py
View File

@@ -15,8 +15,6 @@ from pathlib import Path
from .base_exporter import BaseExporter from .base_exporter import BaseExporter
class Exporter(BaseExporter): class Exporter(BaseExporter):
"""Export truth data to RetroBat batocera-systems.json format.""" """Export truth data to RetroBat batocera-systems.json format."""
@@ -47,7 +45,9 @@ class Exporter(BaseExporter):
continue continue
native_id = native_map.get(sys_id, sys_id) native_id = native_map.get(sys_id, sys_id)
scraped_sys = scraped_data.get("systems", {}).get(sys_id) if scraped_data else None scraped_sys = (
scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
)
display_name = self._display_name(sys_id, scraped_sys) display_name = self._display_name(sys_id, scraped_sys)
bios_files: list[OrderedDict] = [] bios_files: list[OrderedDict] = []
@@ -70,7 +70,9 @@ class Exporter(BaseExporter):
if bios_files: if bios_files:
if native_id in output: if native_id in output:
existing_files = {e.get("file") for e in output[native_id]["biosFiles"]} existing_files = {
e.get("file") for e in output[native_id]["biosFiles"]
}
for entry in bios_files: for entry in bios_files:
if entry.get("file") not in existing_files: if entry.get("file") not in existing_files:
output[native_id]["biosFiles"].append(entry) output[native_id]["biosFiles"].append(entry)

4
scripts/exporter/retrodeck_exporter.py
View File

@@ -170,7 +170,9 @@ class Exporter(BaseExporter):
if native_id in manifest: if native_id in manifest:
# Merge into existing component (multiple truth systems # Merge into existing component (multiple truth systems
# may map to the same native ID) # may map to the same native ID)
existing_names = {e["filename"] for e in manifest[native_id]["bios"]} existing_names = {
e["filename"] for e in manifest[native_id]["bios"]
}
for entry in bios_entries: for entry in bios_entries:
if entry["filename"] not in existing_names: if entry["filename"] not in existing_names:
manifest[native_id]["bios"].append(entry) manifest[native_id]["bios"].append(entry)

22
scripts/exporter/systemdat_exporter.py
View File

@@ -58,16 +58,18 @@ class Exporter(BaseExporter):
] ]
if version: if version:
lines.append(f"\tversion {version}") lines.append(f"\tversion {version}")
lines.extend([ lines.extend(
'\tauthor "libretro"', [
'\thomepage "https://github.com/libretro/libretro-database/blob/master/dat/System.dat"', '\tauthor "libretro"',
'\turl "https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"', '\thomepage "https://github.com/libretro/libretro-database/blob/master/dat/System.dat"',
")", '\turl "https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"',
"", ")",
"game (", "",
'\tname "System"', "game (",
'\tcomment "System"', '\tname "System"',
]) '\tcomment "System"',
]
)
systems = truth_data.get("systems", {}) systems = truth_data.get("systems", {})
for sys_id in sorted(systems): for sys_id in sorted(systems):

52
scripts/generate_db.py
View File

@@ -44,7 +44,11 @@ def _canonical_name(filepath: Path) -> str:
if "/.variants/" in str(filepath) or "\\.variants\\" in str(filepath): if "/.variants/" in str(filepath) or "\\.variants\\" in str(filepath):
# naomi2.zip.da79eca4 -> naomi2.zip # naomi2.zip.da79eca4 -> naomi2.zip
parts = name.rsplit(".", 1) parts = name.rsplit(".", 1)
if len(parts) == 2 and len(parts[1]) == 8 and all(c in "0123456789abcdef" for c in parts[1]): if (
len(parts) == 2
and len(parts[1]) == 8
and all(c in "0123456789abcdef" for c in parts[1])
):
return parts[0] return parts[0]
return name return name
@@ -83,7 +87,9 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
if existing_is_variant and not is_variant: if existing_is_variant and not is_variant:
if sha1 not in aliases: if sha1 not in aliases:
aliases[sha1] = [] aliases[sha1] = []
aliases[sha1].append({"name": files[sha1]["name"], "path": files[sha1]["path"]}) aliases[sha1].append(
{"name": files[sha1]["name"], "path": files[sha1]["path"]}
)
files[sha1] = { files[sha1] = {
"path": rel_path, "path": rel_path,
"name": _canonical_name(filepath), "name": _canonical_name(filepath),
@@ -93,7 +99,9 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
else: else:
if sha1 not in aliases: if sha1 not in aliases:
aliases[sha1] = [] aliases[sha1] = []
aliases[sha1].append({"name": _canonical_name(filepath), "path": rel_path}) aliases[sha1].append(
{"name": _canonical_name(filepath), "path": rel_path}
)
else: else:
entry = { entry = {
"path": rel_path, "path": rel_path,
@@ -114,7 +122,9 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
# Non-variant file should be primary over .variants/ file # Non-variant file should be primary over .variants/ file
if sha1 not in aliases: if sha1 not in aliases:
aliases[sha1] = [] aliases[sha1] = []
aliases[sha1].append({"name": files[sha1]["name"], "path": files[sha1]["path"]}) aliases[sha1].append(
{"name": files[sha1]["name"], "path": files[sha1]["path"]}
)
files[sha1] = { files[sha1] = {
"path": rel_path, "path": rel_path,
"name": _canonical_name(filepath), "name": _canonical_name(filepath),
@@ -124,7 +134,9 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
else: else:
if sha1 not in aliases: if sha1 not in aliases:
aliases[sha1] = [] aliases[sha1] = []
aliases[sha1].append({"name": _canonical_name(filepath), "path": rel_path}) aliases[sha1].append(
{"name": _canonical_name(filepath), "path": rel_path}
)
else: else:
entry = { entry = {
"path": rel_path, "path": rel_path,
@@ -275,8 +287,12 @@ def _preserve_large_file_entries(files: dict, db_path: str) -> int:
def main(): def main():
parser = argparse.ArgumentParser(description="Generate multi-indexed BIOS database") parser = argparse.ArgumentParser(description="Generate multi-indexed BIOS database")
parser.add_argument("--force", action="store_true", help="Force rehash all files") parser.add_argument("--force", action="store_true", help="Force rehash all files")
parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR, help="BIOS directory path") parser.add_argument(
parser.add_argument("--output", "-o", default=DEFAULT_OUTPUT, help="Output JSON file") "--bios-dir", default=DEFAULT_BIOS_DIR, help="BIOS directory path"
)
parser.add_argument(
"--output", "-o", default=DEFAULT_OUTPUT, help="Output JSON file"
)
args = parser.parse_args() args = parser.parse_args()
bios_dir = Path(args.bios_dir) bios_dir = Path(args.bios_dir)
@@ -354,7 +370,10 @@ def _collect_all_aliases(files: dict) -> dict:
if platforms_dir.is_dir(): if platforms_dir.is_dir():
try: try:
import yaml import yaml
for platform_name in list_registered_platforms(str(platforms_dir), include_archived=True):
for platform_name in list_registered_platforms(
str(platforms_dir), include_archived=True
):
config_file = platforms_dir / f"{platform_name}.yml" config_file = platforms_dir / f"{platform_name}.yml"
try: try:
with open(config_file) as f: with open(config_file) as f:
@@ -383,6 +402,7 @@ def _collect_all_aliases(files: dict) -> dict:
try: try:
sys.path.insert(0, "scripts") sys.path.insert(0, "scripts")
from scraper.coreinfo_scraper import Scraper as CoreInfoScraper from scraper.coreinfo_scraper import Scraper as CoreInfoScraper
ci_reqs = CoreInfoScraper().fetch_requirements() ci_reqs = CoreInfoScraper().fetch_requirements()
for r in ci_reqs: for r in ci_reqs:
basename = r.name basename = r.name
@@ -400,6 +420,7 @@ def _collect_all_aliases(files: dict) -> dict:
if emulators_dir.is_dir(): if emulators_dir.is_dir():
try: try:
import yaml import yaml
for emu_file in emulators_dir.glob("*.yml"): for emu_file in emulators_dir.glob("*.yml"):
if emu_file.name.endswith(".old.yml"): if emu_file.name.endswith(".old.yml"):
continue continue
@@ -454,10 +475,17 @@ def _collect_all_aliases(files: dict) -> dict:
# ZX Spectrum # ZX Spectrum
["48.rom", "zx48.rom"], ["48.rom", "zx48.rom"],
# SquirrelJME - all JARs are the same # SquirrelJME - all JARs are the same
["squirreljme.sqc", "squirreljme.jar", "squirreljme-fast.jar", [
"squirreljme-slow.jar", "squirreljme-slow-test.jar", "squirreljme.sqc",
"squirreljme-0.3.0.jar", "squirreljme-0.3.0-fast.jar", "squirreljme.jar",
"squirreljme-0.3.0-slow.jar", "squirreljme-0.3.0-slow-test.jar"], "squirreljme-fast.jar",
"squirreljme-slow.jar",
"squirreljme-slow-test.jar",
"squirreljme-0.3.0.jar",
"squirreljme-0.3.0-fast.jar",
"squirreljme-0.3.0-slow.jar",
"squirreljme-0.3.0-slow-test.jar",
],
# Arcade - FBNeo spectrum # Arcade - FBNeo spectrum
["spectrum.zip", "fbneo/spectrum.zip", "spec48k.zip"], ["spectrum.zip", "fbneo/spectrum.zip", "spec48k.zip"],
] ]

1047
scripts/generate_pack.py
View File

File diff suppressed because it is too large Load Diff

304
scripts/generate_readme.py
View File

@@ -18,15 +18,29 @@ from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__)) sys.path.insert(0, os.path.dirname(__file__))
from common import list_registered_platforms, load_database, load_platform_config, write_if_changed from common import (
list_registered_platforms,
load_database,
load_platform_config,
write_if_changed,
)
from verify import verify_platform from verify import verify_platform
def compute_coverage(platform_name: str, platforms_dir: str, db: dict,
data_registry: dict | None = None, def compute_coverage(
supplemental_names: set[str] | None = None) -> dict: platform_name: str,
platforms_dir: str,
db: dict,
data_registry: dict | None = None,
supplemental_names: set[str] | None = None,
) -> dict:
config = load_platform_config(platform_name, platforms_dir) config = load_platform_config(platform_name, platforms_dir)
result = verify_platform(config, db, data_dir_registry=data_registry, result = verify_platform(
supplemental_names=supplemental_names) config,
db,
data_dir_registry=data_registry,
supplemental_names=supplemental_names,
)
sc = result.get("status_counts", {}) sc = result.get("status_counts", {})
ok = sc.get("ok", 0) ok = sc.get("ok", 0)
untested = sc.get("untested", 0) untested = sc.get("untested", 0)
@@ -55,8 +69,9 @@ REPO = "Abdess/retrobios"
def fetch_contributors() -> list[dict]: def fetch_contributors() -> list[dict]:
"""Fetch contributors from GitHub API, exclude bots.""" """Fetch contributors from GitHub API, exclude bots."""
import urllib.request
import urllib.error import urllib.error
import urllib.request
url = f"https://api.github.com/repos/{REPO}/contributors" url = f"https://api.github.com/repos/{REPO}/contributors"
headers = {"User-Agent": "retrobios-readme/1.0"} headers = {"User-Agent": "retrobios-readme/1.0"}
token = os.environ.get("GITHUB_TOKEN", "") token = os.environ.get("GITHUB_TOKEN", "")
@@ -68,7 +83,8 @@ def fetch_contributors() -> list[dict]:
data = json.loads(resp.read().decode()) data = json.loads(resp.read().decode())
owner = REPO.split("/")[0] owner = REPO.split("/")[0]
return [ return [
c for c in data c
for c in data
if not c.get("login", "").endswith("[bot]") if not c.get("login", "").endswith("[bot]")
and c.get("type") == "User" and c.get("type") == "User"
and c.get("login") != owner and c.get("login") != owner
@@ -87,21 +103,28 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
from common import load_data_dir_registry from common import load_data_dir_registry
from cross_reference import _build_supplemental_index from cross_reference import _build_supplemental_index
data_registry = load_data_dir_registry(platforms_dir) data_registry = load_data_dir_registry(platforms_dir)
suppl_names = _build_supplemental_index() suppl_names = _build_supplemental_index()
coverages = {} coverages = {}
for name in platform_names: for name in platform_names:
try: try:
coverages[name] = compute_coverage(name, platforms_dir, db, coverages[name] = compute_coverage(
data_registry, suppl_names) name, platforms_dir, db, data_registry, suppl_names
)
except FileNotFoundError: except FileNotFoundError:
pass pass
emulator_count = sum( emulator_count = (
1 for f in Path("emulators").glob("*.yml") sum(
if not f.name.endswith(".old.yml") 1
) if Path("emulators").exists() else 0 for f in Path("emulators").glob("*.yml")
if not f.name.endswith(".old.yml")
)
if Path("emulators").exists()
else 0
)
# Count systems from emulator profiles # Count systems from emulator profiles
system_ids: set[str] = set() system_ids: set[str] = set()
@@ -109,6 +132,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
if emu_dir.exists(): if emu_dir.exists():
try: try:
import yaml import yaml
for f in emu_dir.glob("*.yml"): for f in emu_dir.glob("*.yml"):
if f.name.endswith(".old.yml"): if f.name.endswith(".old.yml"):
continue continue
@@ -122,8 +146,12 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
"# RetroBIOS", "# RetroBIOS",
"", "",
f"Complete BIOS and firmware packs for " f"Complete BIOS and firmware packs for "
f"{', '.join(c['platform'] for c in sorted(coverages.values(), key=lambda x: x['platform'])[:-1])}" f"{', '.join(c['platform'] for c in sorted(coverages.values(), key=lambda x: x[
f", and {sorted(coverages.values(), key=lambda x: x['platform'])[-1]['platform']}.", 'platform'
])[:-1])}"
f", and {sorted(coverages.values(), key=lambda x: x[
'platform'
])[-1]['platform']}.",
"", "",
f"**{total_files:,}** verified files across **{len(system_ids)}** systems," f"**{total_files:,}** verified files across **{len(system_ids)}** systems,"
f" ready to extract into your emulator's BIOS directory.", f" ready to extract into your emulator's BIOS directory.",
@@ -170,48 +198,78 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
display = cov["platform"] display = cov["platform"]
path = extract_paths.get(display, "") path = extract_paths.get(display, "")
lines.append( lines.append(
f"| {display} | {cov['total']} | {path} | " f"| {display} | {cov['total']} | {path} | [Download]({RELEASE_URL}) |"
f"[Download]({RELEASE_URL}) |"
) )
lines.extend([ lines.extend(
"", [
"## What's included", "",
"", "## What's included",
"BIOS, firmware, and system files for consoles from Atari to PlayStation 3.", "",
f"Each file is checked against the emulator's source code to match what the" "BIOS, firmware, and system files for consoles from Atari to PlayStation 3.",
f" code actually loads at runtime.", "Each file is checked against the emulator's source code to match what the"
"", " code actually loads at runtime.",
f"- **{len(coverages)} platforms** supported with platform-specific verification", "",
f"- **{emulator_count} emulators** profiled from source (RetroArch cores + standalone)", f"- **{len(coverages)} platforms** supported with platform-specific verification",
f"- **{len(system_ids)} systems** covered (NES, SNES, PlayStation, Saturn, Dreamcast, ...)", f"- **{emulator_count} emulators** profiled from source (RetroArch cores + standalone)",
f"- **{total_files:,} files** verified with MD5, SHA1, CRC32 checksums", f"- **{len(system_ids)} systems** covered (NES, SNES, PlayStation, Saturn, Dreamcast, ...)",
f"- **{size_mb:.0f} MB** total collection size", f"- **{total_files:,} files** verified with MD5, SHA1, CRC32 checksums",
"", f"- **{size_mb:.0f} MB** total collection size",
"## Supported systems", "",
"", "## Supported systems",
]) "",
]
)
# Show well-known systems for SEO, link to full list # Show well-known systems for SEO, link to full list
well_known = [ well_known = [
"NES", "SNES", "Nintendo 64", "GameCube", "Wii", "Game Boy", "Game Boy Advance", "NES",
"Nintendo DS", "Nintendo 3DS", "Switch", "SNES",
"PlayStation", "PlayStation 2", "PlayStation 3", "PSP", "PS Vita", "Nintendo 64",
"Mega Drive", "Saturn", "Dreamcast", "Game Gear", "Master System", "GameCube",
"Neo Geo", "Atari 2600", "Atari 7800", "Atari Lynx", "Atari ST", "Wii",
"MSX", "PC Engine", "TurboGrafx-16", "ColecoVision", "Intellivision", "Game Boy",
"Commodore 64", "Amiga", "ZX Spectrum", "Arcade (MAME)", "Game Boy Advance",
"Nintendo DS",
"Nintendo 3DS",
"Switch",
"PlayStation",
"PlayStation 2",
"PlayStation 3",
"PSP",
"PS Vita",
"Mega Drive",
"Saturn",
"Dreamcast",
"Game Gear",
"Master System",
"Neo Geo",
"Atari 2600",
"Atari 7800",
"Atari Lynx",
"Atari ST",
"MSX",
"PC Engine",
"TurboGrafx-16",
"ColecoVision",
"Intellivision",
"Commodore 64",
"Amiga",
"ZX Spectrum",
"Arcade (MAME)",
] ]
lines.extend([ lines.extend(
", ".join(well_known) + f", and {len(system_ids) - len(well_known)}+ more.", [
"", ", ".join(well_known) + f", and {len(system_ids) - len(well_known)}+ more.",
f"Full list with per-file details: **[{SITE_URL}]({SITE_URL})**", "",
"", f"Full list with per-file details: **[{SITE_URL}]({SITE_URL})**",
"## Coverage", "",
"", "## Coverage",
"| Platform | Coverage | Verified | Untested | Missing |", "",
"|----------|----------|----------|----------|---------|", "| Platform | Coverage | Verified | Untested | Missing |",
]) "|----------|----------|----------|----------|---------|",
]
)
for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]):
pct = f"{cov['percentage']:.1f}%" pct = f"{cov['percentage']:.1f}%"
@@ -220,62 +278,66 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
f"{cov['verified']} | {cov['untested']} | {cov['missing']} |" f"{cov['verified']} | {cov['untested']} | {cov['missing']} |"
) )
lines.extend([ lines.extend(
"", [
"## Build your own pack", "",
"", "## Build your own pack",
"Clone the repo and generate packs for any platform, emulator, or system:", "",
"", "Clone the repo and generate packs for any platform, emulator, or system:",
"```bash", "",
"# Full platform pack", "```bash",
"python scripts/generate_pack.py --platform retroarch --output-dir dist/", "# Full platform pack",
"python scripts/generate_pack.py --platform batocera --output-dir dist/", "python scripts/generate_pack.py --platform retroarch --output-dir dist/",
"", "python scripts/generate_pack.py --platform batocera --output-dir dist/",
"# Single emulator or system", "",
"python scripts/generate_pack.py --emulator dolphin", "# Single emulator or system",
"python scripts/generate_pack.py --system sony-playstation-2", "python scripts/generate_pack.py --emulator dolphin",
"", "python scripts/generate_pack.py --system sony-playstation-2",
"# List available emulators and systems", "",
"python scripts/generate_pack.py --list-emulators", "# List available emulators and systems",
"python scripts/generate_pack.py --list-systems", "python scripts/generate_pack.py --list-emulators",
"", "python scripts/generate_pack.py --list-systems",
"# Verify your BIOS collection", "",
"python scripts/verify.py --all", "# Verify your BIOS collection",
"python scripts/verify.py --platform batocera", "python scripts/verify.py --all",
"python scripts/verify.py --emulator flycast", "python scripts/verify.py --platform batocera",
"python scripts/verify.py --platform retroarch --verbose # emulator ground truth", "python scripts/verify.py --emulator flycast",
"```", "python scripts/verify.py --platform retroarch --verbose # emulator ground truth",
"", "```",
f"Only dependency: Python 3 + `pyyaml`.", "",
"", "Only dependency: Python 3 + `pyyaml`.",
"## Documentation site", "",
"", "## Documentation site",
f"The [documentation site]({SITE_URL}) provides:", "",
"", f"The [documentation site]({SITE_URL}) provides:",
f"- **Per-platform pages** with file-by-file verification status and hashes", "",
f"- **Per-emulator profiles** with source code references for every file", "- **Per-platform pages** with file-by-file verification status and hashes",
f"- **Per-system pages** showing which emulators and platforms cover each console", "- **Per-emulator profiles** with source code references for every file",
f"- **Gap analysis** identifying missing files and undeclared core requirements", "- **Per-system pages** showing which emulators and platforms cover each console",
f"- **Cross-reference** mapping files across {len(coverages)} platforms and {emulator_count} emulators", "- **Gap analysis** identifying missing files and undeclared core requirements",
"", f"- **Cross-reference** mapping files across {len(coverages)} platforms and {emulator_count} emulators",
"## How it works", "",
"", "## How it works",
"Documentation and metadata can drift from what emulators actually load.", "",
"To keep packs accurate, each file is checked against the emulator's source code.", "Documentation and metadata can drift from what emulators actually load.",
"", "To keep packs accurate, each file is checked against the emulator's source code.",
"1. **Read emulator source code** - trace every file the code loads, its expected hash and size", "",
"2. **Cross-reference with platforms** - match against what each platform declares", "1. **Read emulator source code** - trace every file the code loads, its expected hash and size",
"3. **Build packs** - include baseline files plus what each platform's cores need", "2. **Cross-reference with platforms** - match against what each platform declares",
"4. **Verify** - run platform-native checks and emulator-level validation", "3. **Build packs** - include baseline files plus what each platform's cores need",
"", "4. **Verify** - run platform-native checks and emulator-level validation",
]) "",
]
)
contributors = fetch_contributors() contributors = fetch_contributors()
if contributors: if contributors:
lines.extend([ lines.extend(
"## Contributors", [
"", "## Contributors",
]) "",
]
)
for c in contributors: for c in contributors:
login = c["login"] login = c["login"]
avatar = c.get("avatar_url", "") avatar = c.get("avatar_url", "")
@@ -285,18 +347,20 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
) )
lines.append("") lines.append("")
lines.extend([ lines.extend(
"", [
"## Contributing", "",
"", "## Contributing",
"See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.", "",
"", "See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.",
"## License", "",
"", "## License",
"This repository provides BIOS files for personal backup and archival purposes.", "",
"", "This repository provides BIOS files for personal backup and archival purposes.",
f"*Auto-generated on {ts}*", "",
]) f"*Auto-generated on {ts}*",
]
)
return "\n".join(lines) + "\n" return "\n".join(lines) + "\n"
@@ -332,7 +396,11 @@ def main():
print(f"{status} ./README.md") print(f"{status} ./README.md")
contributing = generate_contributing() contributing = generate_contributing()
status = "Generated" if write_if_changed("CONTRIBUTING.md", contributing) else "Unchanged" status = (
"Generated"
if write_if_changed("CONTRIBUTING.md", contributing)
else "Unchanged"
)
print(f"{status} ./CONTRIBUTING.md") print(f"{status} ./CONTRIBUTING.md")

570
scripts/generate_site.py
View File

File diff suppressed because it is too large Load Diff

40
scripts/generate_truth.py
View File

@@ -39,20 +39,28 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
group.add_argument("--all", action="store_true", help="all registered platforms") group.add_argument("--all", action="store_true", help="all registered platforms")
group.add_argument("--platform", help="single platform name") group.add_argument("--platform", help="single platform name")
parser.add_argument( parser.add_argument(
"--output-dir", default=DEFAULT_OUTPUT_DIR, help="output directory", "--output-dir",
default=DEFAULT_OUTPUT_DIR,
help="output directory",
) )
parser.add_argument( parser.add_argument(
"--target", "-t", default=None, help="hardware target filter", "--target",
"-t",
default=None,
help="hardware target filter",
) )
parser.add_argument( parser.add_argument(
"--include-archived", action="store_true", "--include-archived",
action="store_true",
help="include archived platforms with --all", help="include archived platforms with --all",
) )
parser.add_argument( parser.add_argument(
"--platforms-dir", default=DEFAULT_PLATFORMS_DIR, "--platforms-dir",
default=DEFAULT_PLATFORMS_DIR,
) )
parser.add_argument( parser.add_argument(
"--emulators-dir", default=DEFAULT_EMULATORS_DIR, "--emulators-dir",
default=DEFAULT_EMULATORS_DIR,
) )
parser.add_argument("--db", default=DEFAULT_DB_FILE, help="database.json path") parser.add_argument("--db", default=DEFAULT_DB_FILE, help="database.json path")
return parser.parse_args(argv) return parser.parse_args(argv)
@@ -77,7 +85,8 @@ def main(argv: list[str] | None = None) -> None:
# Determine platforms # Determine platforms
if args.all: if args.all:
platforms = list_registered_platforms( platforms = list_registered_platforms(
args.platforms_dir, include_archived=args.include_archived, args.platforms_dir,
include_archived=args.include_archived,
) )
else: else:
platforms = [args.platform] platforms = [args.platform]
@@ -90,7 +99,9 @@ def main(argv: list[str] | None = None) -> None:
if args.target: if args.target:
try: try:
target_cores = load_target_config( target_cores = load_target_config(
name, args.target, args.platforms_dir, name,
args.target,
args.platforms_dir,
) )
except FileNotFoundError: except FileNotFoundError:
print(f" {name}: no target config, skipped") print(f" {name}: no target config, skipped")
@@ -105,15 +116,22 @@ def main(argv: list[str] | None = None) -> None:
registry_entry = registry.get(name, {}) registry_entry = registry.get(name, {})
result = generate_platform_truth( result = generate_platform_truth(
name, config, registry_entry, profiles, name,
db=db, target_cores=target_cores, config,
registry_entry,
profiles,
db=db,
target_cores=target_cores,
) )
out_path = os.path.join(args.output_dir, f"{name}.yml") out_path = os.path.join(args.output_dir, f"{name}.yml")
with open(out_path, "w") as f: with open(out_path, "w") as f:
yaml.dump( yaml.dump(
result, f, result,
default_flow_style=False, sort_keys=False, allow_unicode=True, f,
default_flow_style=False,
sort_keys=False,
allow_unicode=True,
) )
n_systems = len(result.get("systems", {})) n_systems = len(result.get("systems", {}))

103
scripts/migrate.py
View File

@@ -78,11 +78,9 @@ BIOS_FILE_MAP = {
"sanyotry.bin": ("3DO Company", "3DO"), "sanyotry.bin": ("3DO Company", "3DO"),
"3do_arcade_saot.bin": ("3DO Company", "3DO"), "3do_arcade_saot.bin": ("3DO Company", "3DO"),
"3dobios.zip": ("3DO Company", "3DO"), "3dobios.zip": ("3DO Company", "3DO"),
"cpc464.rom": ("Amstrad", "CPC"), "cpc464.rom": ("Amstrad", "CPC"),
"cpc664.rom": ("Amstrad", "CPC"), "cpc664.rom": ("Amstrad", "CPC"),
"cpc6128.rom": ("Amstrad", "CPC"), "cpc6128.rom": ("Amstrad", "CPC"),
"neogeo.zip": ("SNK", "Neo Geo"), "neogeo.zip": ("SNK", "Neo Geo"),
"pgm.zip": ("Arcade", "Arcade"), "pgm.zip": ("Arcade", "Arcade"),
"skns.zip": ("Arcade", "Arcade"), "skns.zip": ("Arcade", "Arcade"),
@@ -94,7 +92,6 @@ BIOS_FILE_MAP = {
"nmk004.zip": ("Arcade", "Arcade"), "nmk004.zip": ("Arcade", "Arcade"),
"ym2608.zip": ("Arcade", "Arcade"), "ym2608.zip": ("Arcade", "Arcade"),
"qsound.zip": ("Arcade", "Arcade"), "qsound.zip": ("Arcade", "Arcade"),
"ATARIBAS.ROM": ("Atari", "400-800"), "ATARIBAS.ROM": ("Atari", "400-800"),
"ATARIOSA.ROM": ("Atari", "400-800"), "ATARIOSA.ROM": ("Atari", "400-800"),
"ATARIOSB.ROM": ("Atari", "400-800"), "ATARIOSB.ROM": ("Atari", "400-800"),
@@ -106,10 +103,8 @@ BIOS_FILE_MAP = {
"7800 BIOS (E).rom": ("Atari", "7800"), "7800 BIOS (E).rom": ("Atari", "7800"),
"lynxboot.img": ("Atari", "Lynx"), "lynxboot.img": ("Atari", "Lynx"),
"tos.img": ("Atari", "ST"), "tos.img": ("Atari", "ST"),
"colecovision.rom": ("Coleco", "ColecoVision"), "colecovision.rom": ("Coleco", "ColecoVision"),
"coleco.rom": ("Coleco", "ColecoVision"), "coleco.rom": ("Coleco", "ColecoVision"),
"kick33180.A500": ("Commodore", "Amiga"), "kick33180.A500": ("Commodore", "Amiga"),
"kick34005.A500": ("Commodore", "Amiga"), "kick34005.A500": ("Commodore", "Amiga"),
"kick34005.CDTV": ("Commodore", "Amiga"), "kick34005.CDTV": ("Commodore", "Amiga"),
@@ -122,33 +117,26 @@ BIOS_FILE_MAP = {
"kick40063.A600": ("Commodore", "Amiga"), "kick40063.A600": ("Commodore", "Amiga"),
"kick40068.A1200": ("Commodore", "Amiga"), "kick40068.A1200": ("Commodore", "Amiga"),
"kick40068.A4000": ("Commodore", "Amiga"), "kick40068.A4000": ("Commodore", "Amiga"),
"sl31253.bin": ("Fairchild", "Channel F"), "sl31253.bin": ("Fairchild", "Channel F"),
"sl31254.bin": ("Fairchild", "Channel F"), "sl31254.bin": ("Fairchild", "Channel F"),
"sl90025.bin": ("Fairchild", "Channel F"), "sl90025.bin": ("Fairchild", "Channel F"),
"prboom.wad": ("Id Software", "Doom"), "prboom.wad": ("Id Software", "Doom"),
"ecwolf.pk3": ("Id Software", "Wolfenstein 3D"), "ecwolf.pk3": ("Id Software", "Wolfenstein 3D"),
"MacII.ROM": ("Apple", "Macintosh II"), "MacII.ROM": ("Apple", "Macintosh II"),
"MacIIx.ROM": ("Apple", "Macintosh II"), "MacIIx.ROM": ("Apple", "Macintosh II"),
"vMac.ROM": ("Apple", "Macintosh II"), "vMac.ROM": ("Apple", "Macintosh II"),
"o2rom.bin": ("Magnavox", "Odyssey2"), "o2rom.bin": ("Magnavox", "Odyssey2"),
"g7400.bin": ("Philips", "Videopac+"), "g7400.bin": ("Philips", "Videopac+"),
"jopac.bin": ("Philips", "Videopac+"), "jopac.bin": ("Philips", "Videopac+"),
"exec.bin": ("Mattel", "Intellivision"), "exec.bin": ("Mattel", "Intellivision"),
"grom.bin": ("Mattel", "Intellivision"), "grom.bin": ("Mattel", "Intellivision"),
"ECS.bin": ("Mattel", "Intellivision"), "ECS.bin": ("Mattel", "Intellivision"),
"IVOICE.BIN": ("Mattel", "Intellivision"), "IVOICE.BIN": ("Mattel", "Intellivision"),
"MSX.ROM": ("Microsoft", "MSX"), "MSX.ROM": ("Microsoft", "MSX"),
"MSX2.ROM": ("Microsoft", "MSX"), "MSX2.ROM": ("Microsoft", "MSX"),
"MSX2EXT.ROM": ("Microsoft", "MSX"), "MSX2EXT.ROM": ("Microsoft", "MSX"),
"MSX2P.ROM": ("Microsoft", "MSX"), "MSX2P.ROM": ("Microsoft", "MSX"),
"MSX2PEXT.ROM": ("Microsoft", "MSX"), "MSX2PEXT.ROM": ("Microsoft", "MSX"),
"syscard1.pce": ("NEC", "PC Engine"), "syscard1.pce": ("NEC", "PC Engine"),
"syscard2.pce": ("NEC", "PC Engine"), "syscard2.pce": ("NEC", "PC Engine"),
"syscard2u.pce": ("NEC", "PC Engine"), "syscard2u.pce": ("NEC", "PC Engine"),
@@ -156,7 +144,6 @@ BIOS_FILE_MAP = {
"syscard3u.pce": ("NEC", "PC Engine"), "syscard3u.pce": ("NEC", "PC Engine"),
"gexpress.pce": ("NEC", "PC Engine"), "gexpress.pce": ("NEC", "PC Engine"),
"pcfx.rom": ("NEC", "PC-FX"), "pcfx.rom": ("NEC", "PC-FX"),
"disksys.rom": ("Nintendo", "Famicom Disk System"), "disksys.rom": ("Nintendo", "Famicom Disk System"),
"gba_bios.bin": ("Nintendo", "Game Boy Advance"), "gba_bios.bin": ("Nintendo", "Game Boy Advance"),
"gb_bios.bin": ("Nintendo", "Game Boy"), "gb_bios.bin": ("Nintendo", "Game Boy"),
@@ -179,7 +166,6 @@ BIOS_FILE_MAP = {
"dsifirmware.bin": ("Nintendo", "Nintendo DS"), "dsifirmware.bin": ("Nintendo", "Nintendo DS"),
"bios.min": ("Nintendo", "Pokemon Mini"), "bios.min": ("Nintendo", "Pokemon Mini"),
"64DD_IPL.bin": ("Nintendo", "Nintendo 64DD"), "64DD_IPL.bin": ("Nintendo", "Nintendo 64DD"),
"dc_boot.bin": ("Sega", "Dreamcast"), "dc_boot.bin": ("Sega", "Dreamcast"),
"dc_flash.bin": ("Sega", "Dreamcast"), "dc_flash.bin": ("Sega", "Dreamcast"),
"bios.gg": ("Sega", "Game Gear"), "bios.gg": ("Sega", "Game Gear"),
@@ -196,7 +182,6 @@ BIOS_FILE_MAP = {
"saturn_bios.bin": ("Sega", "Saturn"), "saturn_bios.bin": ("Sega", "Saturn"),
"sega_101.bin": ("Sega", "Saturn"), "sega_101.bin": ("Sega", "Saturn"),
"stvbios.zip": ("Sega", "Saturn"), "stvbios.zip": ("Sega", "Saturn"),
"scph1001.bin": ("Sony", "PlayStation"), "scph1001.bin": ("Sony", "PlayStation"),
"SCPH1001.BIN": ("Sony", "PlayStation"), "SCPH1001.BIN": ("Sony", "PlayStation"),
"scph5500.bin": ("Sony", "PlayStation"), "scph5500.bin": ("Sony", "PlayStation"),
@@ -207,7 +192,6 @@ BIOS_FILE_MAP = {
"ps1_rom.bin": ("Sony", "PlayStation"), "ps1_rom.bin": ("Sony", "PlayStation"),
"psxonpsp660.bin": ("Sony", "PlayStation"), "psxonpsp660.bin": ("Sony", "PlayStation"),
"PSXONPSP660.BIN": ("Sony", "PlayStation Portable"), "PSXONPSP660.BIN": ("Sony", "PlayStation Portable"),
"scummvm.zip": ("ScummVM", "ScummVM"), "scummvm.zip": ("ScummVM", "ScummVM"),
"MT32_CONTROL.ROM": ("ScummVM", "ScummVM"), "MT32_CONTROL.ROM": ("ScummVM", "ScummVM"),
"MT32_PCM.ROM": ("ScummVM", "ScummVM"), "MT32_PCM.ROM": ("ScummVM", "ScummVM"),
@@ -254,8 +238,11 @@ SKIP_LARGE_ROM_DIRS = {"roms/"}
BRANCHES = ["RetroArch", "RetroPie", "Recalbox", "batocera", "Other"] BRANCHES = ["RetroArch", "RetroPie", "Recalbox", "batocera", "Other"]
SKIP_FILES = { SKIP_FILES = {
"README.md", ".gitignore", "desktop.ini", "README.md",
"telemetry_id", "citra_log.txt", ".gitignore",
"desktop.ini",
"telemetry_id",
"citra_log.txt",
} }
SKIP_EXTENSIONS = {".txt", ".log", ".pem", ".nvm", ".ctg", ".exe", ".bat", ".sh"} SKIP_EXTENSIONS = {".txt", ".log", ".pem", ".nvm", ".ctg", ".exe", ".bat", ".sh"}
@@ -279,17 +266,33 @@ def classify_file(filepath: str) -> tuple:
return None return None
clean = filepath clean = filepath
for prefix in ("bios/", "BIOS/", "roms/fba/", "roms/fbneo/", "roms/mame/", for prefix in (
"roms/mame-libretro/", "roms/neogeo/", "roms/naomi/", "bios/",
"roms/atomiswave/", "roms/macintosh/"): "BIOS/",
"roms/fba/",
"roms/fbneo/",
"roms/mame/",
"roms/mame-libretro/",
"roms/neogeo/",
"roms/naomi/",
"roms/atomiswave/",
"roms/macintosh/",
):
if clean.startswith(prefix): if clean.startswith(prefix):
clean = clean[len(prefix):] clean = clean[len(prefix) :]
break break
if filepath.startswith("roms/") and not any( if filepath.startswith("roms/") and not any(
filepath.startswith(p) for p in ( filepath.startswith(p)
"roms/fba/", "roms/fbneo/", "roms/mame/", "roms/mame-libretro/", for p in (
"roms/neogeo/", "roms/naomi/", "roms/atomiswave/", "roms/macintosh/" "roms/fba/",
"roms/fbneo/",
"roms/mame/",
"roms/mame-libretro/",
"roms/neogeo/",
"roms/naomi/",
"roms/atomiswave/",
"roms/macintosh/",
) )
): ):
return None return None
@@ -341,12 +344,12 @@ def get_subpath(filepath: str, manufacturer: str, console: str) -> str:
clean = filepath clean = filepath
for prefix in ("bios/", "BIOS/"): for prefix in ("bios/", "BIOS/"):
if clean.startswith(prefix): if clean.startswith(prefix):
clean = clean[len(prefix):] clean = clean[len(prefix) :]
break break
for prefix in PATH_PREFIX_MAP: for prefix in PATH_PREFIX_MAP:
if clean.startswith(prefix): if clean.startswith(prefix):
remaining = clean[len(prefix):] remaining = clean[len(prefix) :]
if "/" in remaining: if "/" in remaining:
return remaining return remaining
return remaining return remaining
@@ -363,16 +366,14 @@ def extract_from_branches(target: Path, dry_run: bool, existing_hashes: set) ->
try: try:
subprocess.run( subprocess.run(
["git", "rev-parse", "--verify", ref], ["git", "rev-parse", "--verify", ref], capture_output=True, check=True
capture_output=True, check=True
) )
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print(f" Branch {branch} not found, skipping") print(f" Branch {branch} not found, skipping")
continue continue
result = subprocess.run( result = subprocess.run(
["git", "ls-tree", "-r", "--name-only", ref], ["git", "ls-tree", "-r", "--name-only", ref], capture_output=True, text=True
capture_output=True, text=True
) )
files = result.stdout.strip().split("\n") files = result.stdout.strip().split("\n")
print(f"\n Branch '{branch}': {len(files)} files") print(f"\n Branch '{branch}': {len(files)} files")
@@ -391,7 +392,8 @@ def extract_from_branches(target: Path, dry_run: bool, existing_hashes: set) ->
try: try:
blob = subprocess.run( blob = subprocess.run(
["git", "show", f"{ref}:{filepath}"], ["git", "show", f"{ref}:{filepath}"],
capture_output=True, check=True capture_output=True,
check=True,
) )
content = blob.stdout content = blob.stdout
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
@@ -493,14 +495,20 @@ def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Migrate BIOS files to Manufacturer/Console structure" description="Migrate BIOS files to Manufacturer/Console structure"
) )
parser.add_argument("--dry-run", action="store_true", parser.add_argument(
help="Show what would be done without moving files") "--dry-run",
parser.add_argument("--source", default=".", action="store_true",
help="Source directory (repo root)") help="Show what would be done without moving files",
parser.add_argument("--target", default="bios", )
help="Target directory for organized BIOS files") parser.add_argument("--source", default=".", help="Source directory (repo root)")
parser.add_argument("--include-branches", action="store_true", parser.add_argument(
help="Also extract BIOS files from all remote branches") "--target", default="bios", help="Target directory for organized BIOS files"
)
parser.add_argument(
"--include-branches",
action="store_true",
help="Also extract BIOS files from all remote branches",
)
args = parser.parse_args() args = parser.parse_args()
source = Path(args.source) source = Path(args.source)
@@ -517,7 +525,9 @@ def main():
print() print()
print("=== Phase 1: Local files (libretro branch) ===") print("=== Phase 1: Local files (libretro branch) ===")
moved, skipped, errors, existing_hashes = migrate_local(source, target, args.dry_run) moved, skipped, errors, existing_hashes = migrate_local(
source, target, args.dry_run
)
action = "Would copy" if args.dry_run else "Copied" action = "Would copy" if args.dry_run else "Copied"
print(f"\n{action} {moved} files, skipped {skipped}") print(f"\n{action} {moved} files, skipped {skipped}")
@@ -529,8 +539,15 @@ def main():
if source.is_dir(): if source.is_dir():
known = set(SYSTEM_MAP.keys()) | { known = set(SYSTEM_MAP.keys()) | {
"bios", "scripts", "platforms", "schemas", ".github", ".cache", "bios",
".git", "README.md", ".gitignore", "scripts",
"platforms",
"schemas",
".github",
".cache",
".git",
"README.md",
".gitignore",
} }
for d in sorted(source.iterdir()): for d in sorted(source.iterdir()):
if d.name not in known and not d.name.startswith("."): if d.name not in known and not d.name.startswith("."):

145
scripts/pipeline.py
View File

@@ -19,10 +19,10 @@ Usage:
python scripts/pipeline.py --skip-docs # skip steps 8-9 python scripts/pipeline.py --skip-docs # skip steps 8-9
python scripts/pipeline.py --offline # skip step 2 python scripts/pipeline.py --offline # skip step 2
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import json
import subprocess import subprocess
import sys import sys
import time import time
@@ -54,6 +54,7 @@ def parse_verify_counts(output: str) -> dict[str, tuple[int, int]]:
Returns {group_label: (ok, total)}. Returns {group_label: (ok, total)}.
""" """
import re import re
counts = {} counts = {}
for line in output.splitlines(): for line in output.splitlines():
m = re.match(r"^(.+?):\s+(\d+)/(\d+)\s+(OK|present)", line) m = re.match(r"^(.+?):\s+(\d+)/(\d+)\s+(OK|present)", line)
@@ -71,6 +72,7 @@ def parse_pack_counts(output: str) -> dict[str, tuple[int, int]]:
Returns {pack_label: (ok, total)}. Returns {pack_label: (ok, total)}.
""" """
import re import re
counts = {} counts = {}
current_label = "" current_label = ""
for line in output.splitlines(): for line in output.splitlines():
@@ -84,7 +86,7 @@ def parse_pack_counts(output: str) -> dict[str, tuple[int, int]]:
base_m = re.search(r"\((\d+) baseline", line) base_m = re.search(r"\((\d+) baseline", line)
ok_m = re.search(r"(\d+)/(\d+) files OK", line) ok_m = re.search(r"(\d+)/(\d+) files OK", line)
if base_m and ok_m: if base_m and ok_m:
baseline = int(base_m.group(1)) int(base_m.group(1))
ok, total = int(ok_m.group(1)), int(ok_m.group(2)) ok, total = int(ok_m.group(1)), int(ok_m.group(2))
counts[current_label] = (ok, total) counts[current_label] = (ok, total)
elif ok_m: elif ok_m:
@@ -118,12 +120,18 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
print(f" {v_label}: MISMATCH total verify {v_total} != pack {p_total}") print(f" {v_label}: MISMATCH total verify {v_total} != pack {p_total}")
all_ok = False all_ok = False
elif p_ok < v_ok: elif p_ok < v_ok:
print(f" {v_label}: MISMATCH pack {p_ok} OK < verify {v_ok} OK (/{v_total})") print(
f" {v_label}: MISMATCH pack {p_ok} OK < verify {v_ok} OK (/{v_total})"
)
all_ok = False all_ok = False
elif p_ok == v_ok: elif p_ok == v_ok:
print(f" {v_label}: verify {v_ok}/{v_total} == pack {p_ok}/{p_total} OK") print(
f" {v_label}: verify {v_ok}/{v_total} == pack {p_ok}/{p_total} OK"
)
else: else:
print(f" {v_label}: verify {v_ok}/{v_total}, pack {p_ok}/{p_total} OK (pack resolves more)") print(
f" {v_label}: verify {v_ok}/{v_total}, pack {p_ok}/{p_total} OK (pack resolves more)"
)
else: else:
print(f" {v_label}: {v_ok}/{v_total} (no separate pack)") print(f" {v_label}: {v_ok}/{v_total} (no separate pack)")
@@ -134,26 +142,45 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
def main(): def main():
parser = argparse.ArgumentParser(description="Run the full retrobios pipeline") parser = argparse.ArgumentParser(description="Run the full retrobios pipeline")
parser.add_argument("--include-archived", action="store_true", parser.add_argument(
help="Include archived platforms") "--include-archived", action="store_true", help="Include archived platforms"
parser.add_argument("--skip-packs", action="store_true", )
help="Only regenerate DB and verify, skip pack generation") parser.add_argument(
parser.add_argument("--skip-docs", action="store_true", "--skip-packs",
help="Skip README and site generation") action="store_true",
parser.add_argument("--offline", action="store_true", help="Only regenerate DB and verify, skip pack generation",
help="Skip data directory refresh") )
parser.add_argument("--output-dir", default="dist", parser.add_argument(
help="Pack output directory (default: dist/)") "--skip-docs", action="store_true", help="Skip README and site generation"
)
parser.add_argument(
"--offline", action="store_true", help="Skip data directory refresh"
)
parser.add_argument(
"--output-dir", default="dist", help="Pack output directory (default: dist/)"
)
# --include-extras is now a no-op: core requirements are always included # --include-extras is now a no-op: core requirements are always included
parser.add_argument("--include-extras", action="store_true", parser.add_argument(
help="(no-op) Core requirements are always included") "--include-extras",
action="store_true",
help="(no-op) Core requirements are always included",
)
parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)") parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)")
parser.add_argument("--check-buildbot", action="store_true", parser.add_argument(
help="Check buildbot system directory for changes") "--check-buildbot",
parser.add_argument("--with-truth", action="store_true", action="store_true",
help="Generate truth YAMLs and diff against scraped") help="Check buildbot system directory for changes",
parser.add_argument("--with-export", action="store_true", )
help="Export native formats (implies --with-truth)") parser.add_argument(
"--with-truth",
action="store_true",
help="Generate truth YAMLs and diff against scraped",
)
parser.add_argument(
"--with-export",
action="store_true",
help="Export native formats (implies --with-truth)",
)
args = parser.parse_args() args = parser.parse_args()
results = {} results = {}
@@ -162,8 +189,15 @@ def main():
# Step 1: Generate database # Step 1: Generate database
ok, out = run( ok, out = run(
[sys.executable, "scripts/generate_db.py", "--force", [
"--bios-dir", "bios", "--output", "database.json"], sys.executable,
"scripts/generate_db.py",
"--force",
"--bios-dir",
"bios",
"--output",
"database.json",
],
"1/8 generate database", "1/8 generate database",
) )
results["generate_db"] = ok results["generate_db"] = ok
@@ -216,8 +250,13 @@ def main():
# Step 2c: Generate truth YAMLs # Step 2c: Generate truth YAMLs
if args.with_truth or args.with_export: if args.with_truth or args.with_export:
truth_cmd = [sys.executable, "scripts/generate_truth.py", "--all", truth_cmd = [
"--output-dir", str(Path(args.output_dir) / "truth")] sys.executable,
"scripts/generate_truth.py",
"--all",
"--output-dir",
str(Path(args.output_dir) / "truth"),
]
if args.include_archived: if args.include_archived:
truth_cmd.append("--include-archived") truth_cmd.append("--include-archived")
if args.target: if args.target:
@@ -242,9 +281,15 @@ def main():
# Step 2e: Export native formats # Step 2e: Export native formats
if args.with_export: if args.with_export:
export_cmd = [sys.executable, "scripts/export_native.py", "--all", export_cmd = [
"--output-dir", str(Path(args.output_dir) / "upstream"), sys.executable,
"--truth-dir", str(Path(args.output_dir) / "truth")] "scripts/export_native.py",
"--all",
"--output-dir",
str(Path(args.output_dir) / "upstream"),
"--truth-dir",
str(Path(args.output_dir) / "truth"),
]
if args.include_archived: if args.include_archived:
export_cmd.append("--include-archived") export_cmd.append("--include-archived")
ok, _ = run(export_cmd, "2e export native") ok, _ = run(export_cmd, "2e export native")
@@ -267,8 +312,11 @@ def main():
pack_output = "" pack_output = ""
if not args.skip_packs: if not args.skip_packs:
pack_cmd = [ pack_cmd = [
sys.executable, "scripts/generate_pack.py", "--all", sys.executable,
"--output-dir", args.output_dir, "scripts/generate_pack.py",
"--all",
"--output-dir",
args.output_dir,
] ]
if args.include_archived: if args.include_archived:
pack_cmd.append("--include-archived") pack_cmd.append("--include-archived")
@@ -288,8 +336,12 @@ def main():
# Step 4b: Generate install manifests # Step 4b: Generate install manifests
if not args.skip_packs: if not args.skip_packs:
manifest_cmd = [ manifest_cmd = [
sys.executable, "scripts/generate_pack.py", "--all", sys.executable,
"--manifest", "--output-dir", "install", "scripts/generate_pack.py",
"--all",
"--manifest",
"--output-dir",
"install",
] ]
if args.include_archived: if args.include_archived:
manifest_cmd.append("--include-archived") manifest_cmd.append("--include-archived")
@@ -307,8 +359,11 @@ def main():
# Step 4c: Generate target manifests # Step 4c: Generate target manifests
if not args.skip_packs: if not args.skip_packs:
target_cmd = [ target_cmd = [
sys.executable, "scripts/generate_pack.py", sys.executable,
"--manifest-targets", "--output-dir", "install/targets", "scripts/generate_pack.py",
"--manifest-targets",
"--output-dir",
"install/targets",
] ]
ok, _ = run(target_cmd, "4c/8 generate target manifests") ok, _ = run(target_cmd, "4c/8 generate target manifests")
results["generate_target_manifests"] = ok results["generate_target_manifests"] = ok
@@ -329,8 +384,12 @@ def main():
# Step 6: Pack integrity (extract + hash verification) # Step 6: Pack integrity (extract + hash verification)
if not args.skip_packs: if not args.skip_packs:
integrity_cmd = [ integrity_cmd = [
sys.executable, "scripts/generate_pack.py", "--all", sys.executable,
"--verify-packs", "--output-dir", args.output_dir, "scripts/generate_pack.py",
"--all",
"--verify-packs",
"--output-dir",
args.output_dir,
] ]
if args.include_archived: if args.include_archived:
integrity_cmd.append("--include-archived") integrity_cmd.append("--include-archived")
@@ -344,8 +403,14 @@ def main():
# Step 7: Generate README # Step 7: Generate README
if not args.skip_docs: if not args.skip_docs:
ok, _ = run( ok, _ = run(
[sys.executable, "scripts/generate_readme.py", [
"--db", "database.json", "--platforms-dir", "platforms"], sys.executable,
"scripts/generate_readme.py",
"--db",
"database.json",
"--platforms-dir",
"platforms",
],
"7/8 generate readme", "7/8 generate readme",
) )
results["generate_readme"] = ok results["generate_readme"] = ok

78
scripts/refresh_data_dirs.py
View File

@@ -57,7 +57,9 @@ def _load_versions(versions_path: str = VERSIONS_FILE) -> dict[str, dict]:
return json.load(f) return json.load(f)
def _save_versions(versions: dict[str, dict], versions_path: str = VERSIONS_FILE) -> None: def _save_versions(
versions: dict[str, dict], versions_path: str = VERSIONS_FILE
) -> None:
path = Path(versions_path) path = Path(versions_path)
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w") as f: with open(path, "w") as f:
@@ -66,10 +68,13 @@ def _save_versions(versions: dict[str, dict], versions_path: str = VERSIONS_FILE
def _api_request(url: str) -> dict: def _api_request(url: str) -> dict:
req = urllib.request.Request(url, headers={ req = urllib.request.Request(
"User-Agent": USER_AGENT, url,
"Accept": "application/json", headers={
}) "User-Agent": USER_AGENT,
"Accept": "application/json",
},
)
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
if token and "github" in url: if token and "github" in url:
req.add_header("Authorization", f"token {token}") req.add_header("Authorization", f"token {token}")
@@ -111,7 +116,9 @@ def get_remote_sha(source_url: str, version: str) -> str | None:
data = _api_request(url) data = _api_request(url)
return data["commit"]["id"] return data["commit"]["id"]
except (urllib.error.URLError, KeyError, OSError) as exc: except (urllib.error.URLError, KeyError, OSError) as exc:
log.warning("failed to fetch remote SHA for %s/%s@%s: %s", owner, repo, version, exc) log.warning(
"failed to fetch remote SHA for %s/%s@%s: %s", owner, repo, version, exc
)
return None return None
@@ -167,7 +174,7 @@ def _download_and_extract(
if not member.name.startswith(prefix) and member.name != source_path: if not member.name.startswith(prefix) and member.name != source_path:
continue continue
rel = member.name[len(prefix):] rel = member.name[len(prefix) :]
if not rel: if not rel:
continue continue
@@ -285,8 +292,9 @@ def _download_and_extract_zip(
def _get_remote_etag(source_url: str) -> str | None: def _get_remote_etag(source_url: str) -> str | None:
"""HEAD request to get ETag or Last-Modified for freshness check.""" """HEAD request to get ETag or Last-Modified for freshness check."""
try: try:
req = urllib.request.Request(source_url, method="HEAD", req = urllib.request.Request(
headers={"User-Agent": USER_AGENT}) source_url, method="HEAD", headers={"User-Agent": USER_AGENT}
)
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp: with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
return resp.headers.get("ETag") or resp.headers.get("Last-Modified") or "" return resp.headers.get("ETag") or resp.headers.get("Last-Modified") or ""
except (urllib.error.URLError, OSError): except (urllib.error.URLError, OSError):
@@ -333,17 +341,31 @@ def refresh_entry(
return False return False
if dry_run: if dry_run:
log.info("[%s] would refresh (type: %s, cached: %s)", key, source_type, cached_tag or "none") log.info(
"[%s] would refresh (type: %s, cached: %s)",
key,
source_type,
cached_tag or "none",
)
return True return True
try: try:
if source_type == "zip": if source_type == "zip":
strip = entry.get("strip_components", 0) strip = entry.get("strip_components", 0)
file_count = _download_and_extract_zip(source_url, local_cache, exclude, strip) file_count = _download_and_extract_zip(
source_url, local_cache, exclude, strip
)
else: else:
source_path = entry["source_path"].format(version=version) source_path = entry["source_path"].format(version=version)
file_count = _download_and_extract(source_url, source_path, local_cache, exclude) file_count = _download_and_extract(
except (urllib.error.URLError, OSError, tarfile.TarError, zipfile.BadZipFile) as exc: source_url, source_path, local_cache, exclude
)
except (
urllib.error.URLError,
OSError,
tarfile.TarError,
zipfile.BadZipFile,
) as exc:
log.warning("[%s] download failed: %s", key, exc) log.warning("[%s] download failed: %s", key, exc)
return False return False
@@ -380,18 +402,30 @@ def refresh_all(
if platform and allowed and platform not in allowed: if platform and allowed and platform not in allowed:
continue continue
results[key] = refresh_entry( results[key] = refresh_entry(
key, entry, force=force, dry_run=dry_run, versions_path=versions_path, key,
entry,
force=force,
dry_run=dry_run,
versions_path=versions_path,
) )
return results return results
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser(description="Refresh cached data directories from upstream") parser = argparse.ArgumentParser(
description="Refresh cached data directories from upstream"
)
parser.add_argument("--key", help="Refresh only this entry") parser.add_argument("--key", help="Refresh only this entry")
parser.add_argument("--force", action="store_true", help="Re-download even if up to date") parser.add_argument(
parser.add_argument("--dry-run", action="store_true", help="Preview without downloading") "--force", action="store_true", help="Re-download even if up to date"
)
parser.add_argument(
"--dry-run", action="store_true", help="Preview without downloading"
)
parser.add_argument("--platform", help="Only refresh entries for this platform") parser.add_argument("--platform", help="Only refresh entries for this platform")
parser.add_argument("--registry", default=DEFAULT_REGISTRY, help="Path to _data_dirs.yml") parser.add_argument(
"--registry", default=DEFAULT_REGISTRY, help="Path to _data_dirs.yml"
)
args = parser.parse_args() args = parser.parse_args()
logging.basicConfig( logging.basicConfig(
@@ -405,9 +439,13 @@ def main() -> None:
if args.key not in registry: if args.key not in registry:
log.error("unknown key: %s (available: %s)", args.key, ", ".join(registry)) log.error("unknown key: %s (available: %s)", args.key, ", ".join(registry))
raise SystemExit(1) raise SystemExit(1)
refresh_entry(args.key, registry[args.key], force=args.force, dry_run=args.dry_run) refresh_entry(
args.key, registry[args.key], force=args.force, dry_run=args.dry_run
)
else: else:
refresh_all(registry, force=args.force, dry_run=args.dry_run, platform=args.platform) refresh_all(
registry, force=args.force, dry_run=args.dry_run, platform=args.platform
)
if __name__ == "__main__": if __name__ == "__main__":

292
scripts/scraper/_hash_merge.py
View File

@@ -34,40 +34,40 @@ def merge_mame_profile(
profile = _load_yaml(profile_path) profile = _load_yaml(profile_path)
hashes = _load_json(hashes_path) hashes = _load_json(hashes_path)
profile['core_version'] = hashes.get('version', profile.get('core_version')) profile["core_version"] = hashes.get("version", profile.get("core_version"))
files = profile.get('files', []) files = profile.get("files", [])
bios_zip, non_bios = _split_files(files, lambda f: f.get('category') == 'bios_zip') bios_zip, non_bios = _split_files(files, lambda f: f.get("category") == "bios_zip")
existing_by_name: dict[str, dict] = {} existing_by_name: dict[str, dict] = {}
for entry in bios_zip: for entry in bios_zip:
key = _zip_name_to_set(entry['name']) key = _zip_name_to_set(entry["name"])
existing_by_name[key] = entry existing_by_name[key] = entry
updated_bios: list[dict] = [] updated_bios: list[dict] = []
matched_names: set[str] = set() matched_names: set[str] = set()
for set_name, set_data in hashes.get('bios_sets', {}).items(): for set_name, set_data in hashes.get("bios_sets", {}).items():
contents = _build_contents(set_data.get('roms', [])) contents = _build_contents(set_data.get("roms", []))
source_ref = _build_source_ref(set_data) source_ref = _build_source_ref(set_data)
if set_name in existing_by_name: if set_name in existing_by_name:
# Update existing entry: preserve manual fields, update contents # Update existing entry: preserve manual fields, update contents
entry = existing_by_name[set_name].copy() entry = existing_by_name[set_name].copy()
entry['contents'] = contents entry["contents"] = contents
if source_ref: if source_ref:
entry['source_ref'] = source_ref entry["source_ref"] = source_ref
updated_bios.append(entry) updated_bios.append(entry)
matched_names.add(set_name) matched_names.add(set_name)
elif add_new: elif add_new:
# New BIOS set — only added to the main profile # New BIOS set — only added to the main profile
entry = { entry = {
'name': f'{set_name}.zip', "name": f"{set_name}.zip",
'required': True, "required": True,
'category': 'bios_zip', "category": "bios_zip",
'system': None, "system": None,
'source_ref': source_ref, "source_ref": source_ref,
'contents': contents, "contents": contents,
} }
updated_bios.append(entry) updated_bios.append(entry)
@@ -77,7 +77,7 @@ def merge_mame_profile(
if set_name not in matched_names: if set_name not in matched_names:
updated_bios.append(entry) updated_bios.append(entry)
profile['files'] = non_bios + updated_bios profile["files"] = non_bios + updated_bios
if write: if write:
_backup_and_write(profile_path, profile) _backup_and_write(profile_path, profile)
@@ -102,49 +102,49 @@ def merge_fbneo_profile(
profile = _load_yaml(profile_path) profile = _load_yaml(profile_path)
hashes = _load_json(hashes_path) hashes = _load_json(hashes_path)
profile['core_version'] = hashes.get('version', profile.get('core_version')) profile["core_version"] = hashes.get("version", profile.get("core_version"))
files = profile.get('files', []) files = profile.get("files", [])
archive_files, non_archive = _split_files(files, lambda f: 'archive' in f) archive_files, non_archive = _split_files(files, lambda f: "archive" in f)
existing_by_key: dict[tuple[str, str], dict] = {} existing_by_key: dict[tuple[str, str], dict] = {}
for entry in archive_files: for entry in archive_files:
key = (entry['archive'], entry['name']) key = (entry["archive"], entry["name"])
existing_by_key[key] = entry existing_by_key[key] = entry
merged: list[dict] = [] merged: list[dict] = []
matched_keys: set[tuple[str, str]] = set() matched_keys: set[tuple[str, str]] = set()
for set_name, set_data in hashes.get('bios_sets', {}).items(): for set_name, set_data in hashes.get("bios_sets", {}).items():
archive_name = f'{set_name}.zip' archive_name = f"{set_name}.zip"
source_ref = _build_source_ref(set_data) source_ref = _build_source_ref(set_data)
for rom in set_data.get('roms', []): for rom in set_data.get("roms", []):
rom_name = rom['name'] rom_name = rom["name"]
key = (archive_name, rom_name) key = (archive_name, rom_name)
if key in existing_by_key: if key in existing_by_key:
entry = existing_by_key[key].copy() entry = existing_by_key[key].copy()
entry['size'] = rom['size'] entry["size"] = rom["size"]
entry['crc32'] = rom['crc32'] entry["crc32"] = rom["crc32"]
if rom.get('sha1'): if rom.get("sha1"):
entry['sha1'] = rom['sha1'] entry["sha1"] = rom["sha1"]
if source_ref: if source_ref:
entry['source_ref'] = source_ref entry["source_ref"] = source_ref
merged.append(entry) merged.append(entry)
matched_keys.add(key) matched_keys.add(key)
elif add_new: elif add_new:
entry = { entry = {
'name': rom_name, "name": rom_name,
'archive': archive_name, "archive": archive_name,
'required': True, "required": True,
'size': rom['size'], "size": rom["size"],
'crc32': rom['crc32'], "crc32": rom["crc32"],
} }
if rom.get('sha1'): if rom.get("sha1"):
entry['sha1'] = rom['sha1'] entry["sha1"] = rom["sha1"]
if source_ref: if source_ref:
entry['source_ref'] = source_ref entry["source_ref"] = source_ref
merged.append(entry) merged.append(entry)
# Entries not matched stay untouched # Entries not matched stay untouched
@@ -152,7 +152,7 @@ def merge_fbneo_profile(
if key not in matched_keys: if key not in matched_keys:
merged.append(entry) merged.append(entry)
profile['files'] = non_archive + merged profile["files"] = non_archive + merged
if write: if write:
_backup_and_write_fbneo(profile_path, profile, hashes) _backup_and_write_fbneo(profile_path, profile, hashes)
@@ -163,7 +163,7 @@ def merge_fbneo_profile(
def compute_diff( def compute_diff(
profile_path: str, profile_path: str,
hashes_path: str, hashes_path: str,
mode: str = 'mame', mode: str = "mame",
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Compute diff between profile and hashes without writing. """Compute diff between profile and hashes without writing.
@@ -172,7 +172,7 @@ def compute_diff(
profile = _load_yaml(profile_path) profile = _load_yaml(profile_path)
hashes = _load_json(hashes_path) hashes = _load_json(hashes_path)
if mode == 'mame': if mode == "mame":
return _diff_mame(profile, hashes) return _diff_mame(profile, hashes)
return _diff_fbneo(profile, hashes) return _diff_fbneo(profile, hashes)
@@ -181,26 +181,26 @@ def _diff_mame(
profile: dict[str, Any], profile: dict[str, Any],
hashes: dict[str, Any], hashes: dict[str, Any],
) -> dict[str, Any]: ) -> dict[str, Any]:
files = profile.get('files', []) files = profile.get("files", [])
bios_zip, _ = _split_files(files, lambda f: f.get('category') == 'bios_zip') bios_zip, _ = _split_files(files, lambda f: f.get("category") == "bios_zip")
existing_by_name: dict[str, dict] = {} existing_by_name: dict[str, dict] = {}
for entry in bios_zip: for entry in bios_zip:
existing_by_name[_zip_name_to_set(entry['name'])] = entry existing_by_name[_zip_name_to_set(entry["name"])] = entry
added: list[str] = [] added: list[str] = []
updated: list[str] = [] updated: list[str] = []
unchanged = 0 unchanged = 0
bios_sets = hashes.get('bios_sets', {}) bios_sets = hashes.get("bios_sets", {})
for set_name, set_data in bios_sets.items(): for set_name, set_data in bios_sets.items():
if set_name not in existing_by_name: if set_name not in existing_by_name:
added.append(set_name) added.append(set_name)
continue continue
old_entry = existing_by_name[set_name] old_entry = existing_by_name[set_name]
new_contents = _build_contents(set_data.get('roms', [])) new_contents = _build_contents(set_data.get("roms", []))
old_contents = old_entry.get('contents', []) old_contents = old_entry.get("contents", [])
if _contents_differ(old_contents, new_contents): if _contents_differ(old_contents, new_contents):
updated.append(set_name) updated.append(set_name)
@@ -213,11 +213,11 @@ def _diff_mame(
) )
return { return {
'added': added, "added": added,
'updated': updated, "updated": updated,
'removed': [], "removed": [],
'unchanged': unchanged, "unchanged": unchanged,
'out_of_scope': out_of_scope, "out_of_scope": out_of_scope,
} }
@@ -225,24 +225,24 @@ def _diff_fbneo(
profile: dict[str, Any], profile: dict[str, Any],
hashes: dict[str, Any], hashes: dict[str, Any],
) -> dict[str, Any]: ) -> dict[str, Any]:
files = profile.get('files', []) files = profile.get("files", [])
archive_files, _ = _split_files(files, lambda f: 'archive' in f) archive_files, _ = _split_files(files, lambda f: "archive" in f)
existing_by_key: dict[tuple[str, str], dict] = {} existing_by_key: dict[tuple[str, str], dict] = {}
for entry in archive_files: for entry in archive_files:
existing_by_key[(entry['archive'], entry['name'])] = entry existing_by_key[(entry["archive"], entry["name"])] = entry
added: list[str] = [] added: list[str] = []
updated: list[str] = [] updated: list[str] = []
unchanged = 0 unchanged = 0
seen_keys: set[tuple[str, str]] = set() seen_keys: set[tuple[str, str]] = set()
bios_sets = hashes.get('bios_sets', {}) bios_sets = hashes.get("bios_sets", {})
for set_name, set_data in bios_sets.items(): for set_name, set_data in bios_sets.items():
archive_name = f'{set_name}.zip' archive_name = f"{set_name}.zip"
for rom in set_data.get('roms', []): for rom in set_data.get("roms", []):
key = (archive_name, rom['name']) key = (archive_name, rom["name"])
seen_keys.add(key) seen_keys.add(key)
label = f"{archive_name}:{rom['name']}" label = f"{archive_name}:{rom['name']}"
@@ -251,7 +251,9 @@ def _diff_fbneo(
continue continue
old = existing_by_key[key] old = existing_by_key[key]
if old.get('crc32') != rom.get('crc32') or old.get('size') != rom.get('size'): if old.get("crc32") != rom.get("crc32") or old.get("size") != rom.get(
"size"
):
updated.append(label) updated.append(label)
else: else:
unchanged += 1 unchanged += 1
@@ -259,11 +261,11 @@ def _diff_fbneo(
out_of_scope = sum(1 for k in existing_by_key if k not in seen_keys) out_of_scope = sum(1 for k in existing_by_key if k not in seen_keys)
return { return {
'added': added, "added": added,
'updated': updated, "updated": updated,
'removed': [], "removed": [],
'unchanged': unchanged, "unchanged": unchanged,
'out_of_scope': out_of_scope, "out_of_scope": out_of_scope,
} }
@@ -271,12 +273,12 @@ def _diff_fbneo(
def _load_yaml(path: str) -> dict[str, Any]: def _load_yaml(path: str) -> dict[str, Any]:
with open(path, encoding='utf-8') as f: with open(path, encoding="utf-8") as f:
return yaml.safe_load(f) or {} return yaml.safe_load(f) or {}
def _load_json(path: str) -> dict[str, Any]: def _load_json(path: str) -> dict[str, Any]:
with open(path, encoding='utf-8') as f: with open(path, encoding="utf-8") as f:
return json.load(f) return json.load(f)
@@ -295,7 +297,7 @@ def _split_files(
def _zip_name_to_set(name: str) -> str: def _zip_name_to_set(name: str) -> str:
if name.endswith('.zip'): if name.endswith(".zip"):
return name[:-4] return name[:-4]
return name return name
@@ -304,42 +306,42 @@ def _build_contents(roms: list[dict]) -> list[dict]:
contents: list[dict] = [] contents: list[dict] = []
for rom in roms: for rom in roms:
entry: dict[str, Any] = { entry: dict[str, Any] = {
'name': rom['name'], "name": rom["name"],
'size': rom['size'], "size": rom["size"],
'crc32': rom['crc32'], "crc32": rom["crc32"],
} }
if rom.get('sha1'): if rom.get("sha1"):
entry['sha1'] = rom['sha1'] entry["sha1"] = rom["sha1"]
desc = rom.get('bios_description') or rom.get('bios_label') or '' desc = rom.get("bios_description") or rom.get("bios_label") or ""
if desc: if desc:
entry['description'] = desc entry["description"] = desc
if rom.get('bad_dump'): if rom.get("bad_dump"):
entry['bad_dump'] = True entry["bad_dump"] = True
contents.append(entry) contents.append(entry)
return contents return contents
def _build_source_ref(set_data: dict) -> str: def _build_source_ref(set_data: dict) -> str:
source_file = set_data.get('source_file', '') source_file = set_data.get("source_file", "")
source_line = set_data.get('source_line') source_line = set_data.get("source_line")
if source_file and source_line is not None: if source_file and source_line is not None:
return f'{source_file}:{source_line}' return f"{source_file}:{source_line}"
return source_file return source_file
def _contents_differ(old: list[dict], new: list[dict]) -> bool: def _contents_differ(old: list[dict], new: list[dict]) -> bool:
if len(old) != len(new): if len(old) != len(new):
return True return True
old_by_name = {c['name']: c for c in old} old_by_name = {c["name"]: c for c in old}
for entry in new: for entry in new:
prev = old_by_name.get(entry['name']) prev = old_by_name.get(entry["name"])
if prev is None: if prev is None:
return True return True
if prev.get('crc32') != entry.get('crc32'): if prev.get("crc32") != entry.get("crc32"):
return True return True
if prev.get('size') != entry.get('size'): if prev.get("size") != entry.get("size"):
return True return True
if prev.get('sha1') != entry.get('sha1'): if prev.get("sha1") != entry.get("sha1"):
return True return True
return False return False
@@ -352,15 +354,15 @@ def _backup_and_write(path: str, data: dict) -> None:
(core_version, contents, source_ref), and appends new entries. (core_version, contents, source_ref), and appends new entries.
""" """
p = Path(path) p = Path(path)
backup = p.with_suffix('.old.yml') backup = p.with_suffix(".old.yml")
shutil.copy2(p, backup) shutil.copy2(p, backup)
original = p.read_text(encoding='utf-8') original = p.read_text(encoding="utf-8")
patched = _patch_core_version(original, data.get('core_version', '')) patched = _patch_core_version(original, data.get("core_version", ""))
patched = _patch_bios_entries(patched, data.get('files', [])) patched = _patch_bios_entries(patched, data.get("files", []))
patched = _append_new_entries(patched, data.get('files', []), original) patched = _append_new_entries(patched, data.get("files", []), original)
p.write_text(patched, encoding='utf-8') p.write_text(patched, encoding="utf-8")
def _patch_core_version(text: str, version: str) -> str: def _patch_core_version(text: str, version: str) -> str:
@@ -368,8 +370,9 @@ def _patch_core_version(text: str, version: str) -> str:
if not version: if not version:
return text return text
import re import re
return re.sub( return re.sub(
r'^(core_version:\s*).*$', r"^(core_version:\s*).*$",
rf'\g<1>"{version}"', rf'\g<1>"{version}"',
text, text,
count=1, count=1,
@@ -390,18 +393,18 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
# Build a lookup of what to patch # Build a lookup of what to patch
patches: dict[str, dict] = {} patches: dict[str, dict] = {}
for fe in files: for fe in files:
if fe.get('category') != 'bios_zip': if fe.get("category") != "bios_zip":
continue continue
patches[fe['name']] = fe patches[fe["name"]] = fe
if not patches: if not patches:
return text return text
lines = text.split('\n') lines = text.split("\n")
# Find all entry start positions (line indices) # Find all entry start positions (line indices)
entry_starts: list[tuple[int, str]] = [] entry_starts: list[tuple[int, str]] = []
for i, line in enumerate(lines): for i, line in enumerate(lines):
m = re.match(r'^ - name:\s*(.+?)\s*$', line) m = re.match(r"^ - name:\s*(.+?)\s*$", line)
if m: if m:
entry_starts.append((i, m.group(1).strip('"').strip("'"))) entry_starts.append((i, m.group(1).strip('"').strip("'")))
@@ -412,8 +415,8 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
continue continue
fe = patches[entry_name] fe = patches[entry_name]
contents = fe.get('contents', []) contents = fe.get("contents", [])
source_ref = fe.get('source_ref', '') source_ref = fe.get("source_ref", "")
# Find the last "owned" line of this entry # Find the last "owned" line of this entry
# Owned = indented with 4+ spaces (field lines of this entry) # Owned = indented with 4+ spaces (field lines of this entry)
@@ -422,11 +425,11 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
stripped = lines[j].strip() stripped = lines[j].strip()
if not stripped: if not stripped:
break # blank line = end of entry break # blank line = end of entry
if stripped.startswith('#'): if stripped.startswith("#"):
break # comment = belongs to next entry break # comment = belongs to next entry
if re.match(r'^ - ', lines[j]): if re.match(r"^ - ", lines[j]):
break # next list item break # next list item
if re.match(r'^ ', lines[j]) or re.match(r'^ \w', lines[j]): if re.match(r"^ ", lines[j]) or re.match(r"^ \w", lines[j]):
last_owned = j last_owned = j
else: else:
break break
@@ -435,7 +438,7 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
if source_ref: if source_ref:
found_sr = False found_sr = False
for j in range(start_line + 1, last_owned + 1): for j in range(start_line + 1, last_owned + 1):
if re.match(r'^ source_ref:', lines[j]): if re.match(r"^ source_ref:", lines[j]):
lines[j] = f' source_ref: "{source_ref}"' lines[j] = f' source_ref: "{source_ref}"'
found_sr = True found_sr = True
break break
@@ -447,10 +450,10 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
contents_start = None contents_start = None
contents_end = None contents_end = None
for j in range(start_line + 1, last_owned + 1): for j in range(start_line + 1, last_owned + 1):
if re.match(r'^ contents:', lines[j]): if re.match(r"^ contents:", lines[j]):
contents_start = j contents_start = j
elif contents_start is not None: elif contents_start is not None:
if re.match(r'^ ', lines[j]): if re.match(r"^ ", lines[j]):
contents_end = j contents_end = j
else: else:
break break
@@ -458,29 +461,29 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
contents_end = contents_start contents_end = contents_start
if contents_start is not None: if contents_start is not None:
del lines[contents_start:contents_end + 1] del lines[contents_start : contents_end + 1]
last_owned -= (contents_end - contents_start + 1) last_owned -= contents_end - contents_start + 1
# Insert new contents after last owned line # Insert new contents after last owned line
if contents: if contents:
new_lines = _format_contents(contents).split('\n') new_lines = _format_contents(contents).split("\n")
for k, cl in enumerate(new_lines): for k, cl in enumerate(new_lines):
lines.insert(last_owned + 1 + k, cl) lines.insert(last_owned + 1 + k, cl)
return '\n'.join(lines) return "\n".join(lines)
def _append_new_entries(text: str, files: list[dict], original: str) -> str: def _append_new_entries(text: str, files: list[dict], original: str) -> str:
"""Append new bios_zip entries (system=None) that aren't in the original.""" """Append new bios_zip entries (system=None) that aren't in the original."""
# Parse original to get existing entry names (more reliable than text search) # Parse original to get existing entry names (more reliable than text search)
existing_data = yaml.safe_load(original) or {} existing_data = yaml.safe_load(original) or {}
existing_names = {f['name'] for f in existing_data.get('files', [])} existing_names = {f["name"] for f in existing_data.get("files", [])}
new_entries = [] new_entries = []
for fe in files: for fe in files:
if fe.get('category') != 'bios_zip' or fe.get('system') is not None: if fe.get("category") != "bios_zip" or fe.get("system") is not None:
continue continue
if fe['name'] in existing_names: if fe["name"] in existing_names:
continue continue
new_entries.append(fe) new_entries.append(fe)
@@ -489,36 +492,36 @@ def _append_new_entries(text: str, files: list[dict], original: str) -> str:
lines = [] lines = []
for fe in new_entries: for fe in new_entries:
lines.append(f'\n - name: {fe["name"]}') lines.append(f"\n - name: {fe['name']}")
lines.append(f' required: {str(fe["required"]).lower()}') lines.append(f" required: {str(fe['required']).lower()}")
lines.append(f' category: bios_zip') lines.append(" category: bios_zip")
if fe.get('source_ref'): if fe.get("source_ref"):
lines.append(f' source_ref: "{fe["source_ref"]}"') lines.append(f' source_ref: "{fe["source_ref"]}"')
if fe.get('contents'): if fe.get("contents"):
lines.append(_format_contents(fe['contents'])) lines.append(_format_contents(fe["contents"]))
if lines: if lines:
text = text.rstrip('\n') + '\n' + '\n'.join(lines) + '\n' text = text.rstrip("\n") + "\n" + "\n".join(lines) + "\n"
return text return text
def _format_contents(contents: list[dict]) -> str: def _format_contents(contents: list[dict]) -> str:
"""Format a contents list as YAML text.""" """Format a contents list as YAML text."""
lines = [' contents:'] lines = [" contents:"]
for rom in contents: for rom in contents:
lines.append(f' - name: {rom["name"]}') lines.append(f" - name: {rom['name']}")
if rom.get('description'): if rom.get("description"):
lines.append(f' description: {rom["description"]}') lines.append(f" description: {rom['description']}")
if rom.get('size'): if rom.get("size"):
lines.append(f' size: {rom["size"]}') lines.append(f" size: {rom['size']}")
if rom.get('crc32'): if rom.get("crc32"):
lines.append(f' crc32: "{rom["crc32"]}"') lines.append(f' crc32: "{rom["crc32"]}"')
if rom.get('sha1'): if rom.get("sha1"):
lines.append(f' sha1: "{rom["sha1"]}"') lines.append(f' sha1: "{rom["sha1"]}"')
if rom.get('bad_dump'): if rom.get("bad_dump"):
lines.append(f' bad_dump: true') lines.append(" bad_dump: true")
return '\n'.join(lines) return "\n".join(lines)
def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None: def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None:
@@ -529,37 +532,38 @@ def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None:
Existing entries are left untouched (CRC32 changes are rare). Existing entries are left untouched (CRC32 changes are rare).
""" """
p = Path(path) p = Path(path)
backup = p.with_suffix('.old.yml') backup = p.with_suffix(".old.yml")
shutil.copy2(p, backup) shutil.copy2(p, backup)
original = p.read_text(encoding='utf-8') original = p.read_text(encoding="utf-8")
patched = _patch_core_version(original, data.get('core_version', '')) patched = _patch_core_version(original, data.get("core_version", ""))
# Identify new ROM entries by comparing parsed data keys, not text search # Identify new ROM entries by comparing parsed data keys, not text search
existing_data = yaml.safe_load(original) or {} existing_data = yaml.safe_load(original) or {}
existing_keys = { existing_keys = {
(f['archive'], f['name']) (f["archive"], f["name"])
for f in existing_data.get('files', []) for f in existing_data.get("files", [])
if f.get('archive') if f.get("archive")
} }
new_roms = [ new_roms = [
f for f in data.get('files', []) f
if f.get('archive') and (f['archive'], f['name']) not in existing_keys for f in data.get("files", [])
if f.get("archive") and (f["archive"], f["name"]) not in existing_keys
] ]
if new_roms: if new_roms:
lines = [] lines = []
for fe in new_roms: for fe in new_roms:
lines.append(f' - name: "{fe["name"]}"') lines.append(f' - name: "{fe["name"]}"')
lines.append(f' archive: {fe["archive"]}') lines.append(f" archive: {fe['archive']}")
lines.append(f' required: {str(fe.get("required", True)).lower()}') lines.append(f" required: {str(fe.get('required', True)).lower()}")
if fe.get('size'): if fe.get("size"):
lines.append(f' size: {fe["size"]}') lines.append(f" size: {fe['size']}")
if fe.get('crc32'): if fe.get("crc32"):
lines.append(f' crc32: "{fe["crc32"]}"') lines.append(f' crc32: "{fe["crc32"]}"')
if fe.get('source_ref'): if fe.get("source_ref"):
lines.append(f' source_ref: "{fe["source_ref"]}"') lines.append(f' source_ref: "{fe["source_ref"]}"')
lines.append('') lines.append("")
patched = patched.rstrip('\n') + '\n\n' + '\n'.join(lines) patched = patched.rstrip("\n") + "\n\n" + "\n".join(lines)
p.write_text(patched, encoding='utf-8') p.write_text(patched, encoding="utf-8")

122
scripts/scraper/base_scraper.py
View File

@@ -4,8 +4,8 @@ from __future__ import annotations
import json import json
import sys import sys
import urllib.request
import urllib.error import urllib.error
import urllib.request
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
@@ -14,6 +14,7 @@ from pathlib import Path
@dataclass @dataclass
class BiosRequirement: class BiosRequirement:
"""A single BIOS file requirement from a platform source.""" """A single BIOS file requirement from a platform source."""
name: str name: str
system: str system: str
sha1: str | None = None sha1: str | None = None
@@ -29,9 +30,12 @@ class BiosRequirement:
@dataclass @dataclass
class ChangeSet: class ChangeSet:
"""Differences between scraped requirements and current config.""" """Differences between scraped requirements and current config."""
added: list[BiosRequirement] = field(default_factory=list) added: list[BiosRequirement] = field(default_factory=list)
removed: list[BiosRequirement] = field(default_factory=list) removed: list[BiosRequirement] = field(default_factory=list)
modified: list[tuple[BiosRequirement, BiosRequirement]] = field(default_factory=list) modified: list[tuple[BiosRequirement, BiosRequirement]] = field(
default_factory=list
)
@property @property
def has_changes(self) -> bool: def has_changes(self) -> bool:
@@ -80,7 +84,9 @@ class BaseScraper(ABC):
if not self.url: if not self.url:
raise ValueError("No source URL configured") raise ValueError("No source URL configured")
try: try:
req = urllib.request.Request(self.url, headers={"User-Agent": "retrobios-scraper/1.0"}) req = urllib.request.Request(
self.url, headers={"User-Agent": "retrobios-scraper/1.0"}
)
with urllib.request.urlopen(req, timeout=30) as resp: with urllib.request.urlopen(req, timeout=30) as resp:
self._raw_data = _read_limited(resp).decode("utf-8") self._raw_data = _read_limited(resp).decode("utf-8")
return self._raw_data return self._raw_data
@@ -113,35 +119,49 @@ class BaseScraper(ABC):
changes.added.append(req) changes.added.append(req)
else: else:
existing_file = existing[key] existing_file = existing[key]
if req.sha1 and existing_file.get("sha1") and req.sha1 != existing_file["sha1"]: if (
changes.modified.append(( req.sha1
BiosRequirement( and existing_file.get("sha1")
name=existing_file["name"], and req.sha1 != existing_file["sha1"]
system=key[0], ):
sha1=existing_file.get("sha1"), changes.modified.append(
md5=existing_file.get("md5"), (
), BiosRequirement(
req, name=existing_file["name"],
)) system=key[0],
elif req.md5 and existing_file.get("md5") and req.md5 != existing_file["md5"]: sha1=existing_file.get("sha1"),
changes.modified.append(( md5=existing_file.get("md5"),
BiosRequirement( ),
name=existing_file["name"], req,
system=key[0], )
md5=existing_file.get("md5"), )
), elif (
req, req.md5
)) and existing_file.get("md5")
and req.md5 != existing_file["md5"]
):
changes.modified.append(
(
BiosRequirement(
name=existing_file["name"],
system=key[0],
md5=existing_file.get("md5"),
),
req,
)
)
for key in existing: for key in existing:
if key not in scraped_map: if key not in scraped_map:
f = existing[key] f = existing[key]
changes.removed.append(BiosRequirement( changes.removed.append(
name=f["name"], BiosRequirement(
system=key[0], name=f["name"],
sha1=f.get("sha1"), system=key[0],
md5=f.get("md5"), sha1=f.get("sha1"),
)) md5=f.get("md5"),
)
)
return changes return changes
@@ -163,10 +183,13 @@ def fetch_github_latest_version(repo: str) -> str | None:
"""Fetch the latest release version tag from a GitHub repo.""" """Fetch the latest release version tag from a GitHub repo."""
url = f"https://api.github.com/repos/{repo}/releases/latest" url = f"https://api.github.com/repos/{repo}/releases/latest"
try: try:
req = urllib.request.Request(url, headers={ req = urllib.request.Request(
"User-Agent": "retrobios-scraper/1.0", url,
"Accept": "application/vnd.github.v3+json", headers={
}) "User-Agent": "retrobios-scraper/1.0",
"Accept": "application/vnd.github.v3+json",
},
)
with urllib.request.urlopen(req, timeout=15) as resp: with urllib.request.urlopen(req, timeout=15) as resp:
data = json.loads(resp.read()) data = json.loads(resp.read())
return data.get("tag_name", "") return data.get("tag_name", "")
@@ -174,7 +197,9 @@ def fetch_github_latest_version(repo: str) -> str | None:
return None return None
def scraper_cli(scraper_class: type, description: str = "Scrape BIOS requirements") -> None: def scraper_cli(
scraper_class: type, description: str = "Scrape BIOS requirements"
) -> None:
"""Shared CLI entry point for all scrapers. Eliminates main() boilerplate.""" """Shared CLI entry point for all scrapers. Eliminates main() boilerplate."""
import argparse import argparse
@@ -203,13 +228,23 @@ def scraper_cli(scraper_class: type, description: str = "Scrape BIOS requirement
return return
if args.json: if args.json:
data = [{"name": r.name, "system": r.system, "sha1": r.sha1, "md5": r.md5, data = [
"size": r.size, "required": r.required} for r in reqs] {
"name": r.name,
"system": r.system,
"sha1": r.sha1,
"md5": r.md5,
"size": r.size,
"required": r.required,
}
for r in reqs
]
print(json.dumps(data, indent=2)) print(json.dumps(data, indent=2))
return return
if args.output: if args.output:
import yaml import yaml
# Use scraper's generate_platform_yaml() if available (includes # Use scraper's generate_platform_yaml() if available (includes
# platform metadata, cores list, standalone_cores, etc.) # platform metadata, cores list, standalone_cores, etc.)
if hasattr(scraper, "generate_platform_yaml"): if hasattr(scraper, "generate_platform_yaml"):
@@ -224,7 +259,11 @@ def scraper_cli(scraper_class: type, description: str = "Scrape BIOS requirement
if req.native_id: if req.native_id:
sys_entry["native_id"] = req.native_id sys_entry["native_id"] = req.native_id
config["systems"][sys_id] = sys_entry config["systems"][sys_id] = sys_entry
entry = {"name": req.name, "destination": req.destination or req.name, "required": req.required} entry = {
"name": req.name,
"destination": req.destination or req.name,
"required": req.required,
}
if req.sha1: if req.sha1:
entry["sha1"] = req.sha1 entry["sha1"] = req.sha1
if req.md5: if req.md5:
@@ -265,10 +304,13 @@ def fetch_github_latest_tag(repo: str, prefix: str = "") -> str | None:
"""Fetch the most recent matching tag from a GitHub repo.""" """Fetch the most recent matching tag from a GitHub repo."""
url = f"https://api.github.com/repos/{repo}/tags?per_page=50" url = f"https://api.github.com/repos/{repo}/tags?per_page=50"
try: try:
req = urllib.request.Request(url, headers={ req = urllib.request.Request(
"User-Agent": "retrobios-scraper/1.0", url,
"Accept": "application/vnd.github.v3+json", headers={
}) "User-Agent": "retrobios-scraper/1.0",
"Accept": "application/vnd.github.v3+json",
},
)
with urllib.request.urlopen(req, timeout=15) as resp: with urllib.request.urlopen(req, timeout=15) as resp:
tags = json.loads(resp.read()) tags = json.loads(resp.read())
for tag in tags: for tag in tags:

50
scripts/scraper/batocera_scraper.py
View File

@@ -12,8 +12,8 @@ import ast
import json import json
import re import re
import sys import sys
import urllib.request
import urllib.error import urllib.error
import urllib.request
from pathlib import Path from pathlib import Path
import yaml import yaml
@@ -102,7 +102,6 @@ SYSTEM_SLUG_MAP = {
"dragon64": "dragon64", "dragon64": "dragon64",
"mc10": "mc10", "mc10": "mc10",
"msx2+": "microsoft-msx", "msx2+": "microsoft-msx",
"msxturbor": "microsoft-msx",
"spectravideo": "spectravideo", "spectravideo": "spectravideo",
"tvc": "videoton-tvc", "tvc": "videoton-tvc",
"enterprise": "enterprise-64-128", "enterprise": "enterprise-64-128",
@@ -116,7 +115,7 @@ SYSTEM_SLUG_MAP = {
} }
_MD5_RE = re.compile(r'^[a-fA-F0-9]+$') _MD5_RE = re.compile(r"^[a-fA-F0-9]+$")
def _load_md5_index() -> dict[str, str]: def _load_md5_index() -> dict[str, str]:
@@ -183,11 +182,11 @@ class Scraper(BaseScraper):
def _extract_systems_dict(self, raw: str) -> dict: def _extract_systems_dict(self, raw: str) -> dict:
"""Extract and parse the 'systems' dict from the Python source via ast.literal_eval.""" """Extract and parse the 'systems' dict from the Python source via ast.literal_eval."""
match = re.search(r'^systems\s*=\s*\{', raw, re.MULTILINE) match = re.search(r"^systems\s*=\s*\{", raw, re.MULTILINE)
if not match: if not match:
raise ValueError("Could not find 'systems = {' in batocera-systems") raise ValueError("Could not find 'systems = {' in batocera-systems")
start = match.start() + raw[match.start():].index("{") start = match.start() + raw[match.start() :].index("{")
depth = 0 depth = 0
i = start i = start
in_str = False in_str = False
@@ -195,7 +194,7 @@ class Scraper(BaseScraper):
while i < len(raw): while i < len(raw):
ch = raw[i] ch = raw[i]
if in_str: if in_str:
if ch == '\\': if ch == "\\":
i += 2 i += 2
continue continue
if ch == str_ch: if ch == str_ch:
@@ -214,7 +213,7 @@ class Scraper(BaseScraper):
i += 1 i += 1
i += 1 i += 1
dict_str = raw[start:i + 1] dict_str = raw[start : i + 1]
lines = [] lines = []
for line in dict_str.split("\n"): for line in dict_str.split("\n"):
@@ -224,7 +223,7 @@ class Scraper(BaseScraper):
j = 0 j = 0
while j < len(line): while j < len(line):
ch = line[j] ch = line[j]
if ch == '\\' and j + 1 < len(line): if ch == "\\" and j + 1 < len(line):
clean.append(ch) clean.append(ch)
clean.append(line[j + 1]) clean.append(line[j + 1])
j += 2 j += 2
@@ -246,8 +245,8 @@ class Scraper(BaseScraper):
clean_dict_str = "\n".join(lines) clean_dict_str = "\n".join(lines)
# OrderedDict({...}) -> just the inner dict literal # OrderedDict({...}) -> just the inner dict literal
clean_dict_str = re.sub(r'OrderedDict\(\s*\{', '{', clean_dict_str) clean_dict_str = re.sub(r"OrderedDict\(\s*\{", "{", clean_dict_str)
clean_dict_str = re.sub(r'\}\s*\)', '}', clean_dict_str) clean_dict_str = re.sub(r"\}\s*\)", "}", clean_dict_str)
try: try:
return ast.literal_eval(clean_dict_str) return ast.literal_eval(clean_dict_str)
@@ -279,22 +278,24 @@ class Scraper(BaseScraper):
name = file_path.split("/")[-1] if "/" in file_path else file_path name = file_path.split("/")[-1] if "/" in file_path else file_path
requirements.append(BiosRequirement( requirements.append(
name=name, BiosRequirement(
system=system_slug, name=name,
md5=md5 or None, system=system_slug,
destination=file_path, md5=md5 or None,
required=True, destination=file_path,
zipped_file=zipped_file or None, required=True,
native_id=sys_key, zipped_file=zipped_file or None,
)) native_id=sys_key,
)
)
return requirements return requirements
def validate_format(self, raw_data: str) -> bool: def validate_format(self, raw_data: str) -> bool:
"""Validate batocera-systems format.""" """Validate batocera-systems format."""
has_systems = "systems" in raw_data and "biosFiles" in raw_data has_systems = "systems" in raw_data and "biosFiles" in raw_data
has_dict = re.search(r'^systems\s*=\s*\{', raw_data, re.MULTILINE) is not None has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
has_md5 = '"md5"' in raw_data has_md5 = '"md5"' in raw_data
has_file = '"file"' in raw_data has_file = '"file"' in raw_data
return has_systems and has_dict and has_md5 and has_file return has_systems and has_dict and has_md5 and has_file
@@ -336,7 +337,9 @@ class Scraper(BaseScraper):
systems[req.system]["files"].append(entry) systems[req.system]["files"].append(entry)
tag = fetch_github_latest_tag("batocera-linux/batocera.linux", prefix="batocera-") tag = fetch_github_latest_tag(
"batocera-linux/batocera.linux", prefix="batocera-"
)
batocera_version = "" batocera_version = ""
if tag: if tag:
num = tag.removeprefix("batocera-") num = tag.removeprefix("batocera-")
@@ -344,7 +347,9 @@ class Scraper(BaseScraper):
batocera_version = num batocera_version = num
if not batocera_version: if not batocera_version:
# Preserve existing version when fetch fails (offline mode) # Preserve existing version when fetch fails (offline mode)
existing = Path(__file__).resolve().parents[2] / "platforms" / "batocera.yml" existing = (
Path(__file__).resolve().parents[2] / "platforms" / "batocera.yml"
)
if existing.exists(): if existing.exists():
with open(existing) as f: with open(existing) as f:
old = yaml.safe_load(f) or {} old = yaml.safe_load(f) or {}
@@ -369,6 +374,7 @@ class Scraper(BaseScraper):
def main(): def main():
from scripts.scraper.base_scraper import scraper_cli from scripts.scraper.base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape batocera BIOS requirements") scraper_cli(Scraper, "Scrape batocera BIOS requirements")

102
scripts/scraper/bizhawk_scraper.py
View File

@@ -19,7 +19,6 @@ the Ideal non-bad option is selected as canonical.
from __future__ import annotations from __future__ import annotations
import re import re
import sys
try: try:
from .base_scraper import ( from .base_scraper import (
@@ -108,12 +107,33 @@ SYSTEM_ID_MAP: dict[str, str] = {
# Cores that overlap with BizHawk's system coverage # Cores that overlap with BizHawk's system coverage
BIZHAWK_CORES = [ BIZHAWK_CORES = [
"gambatte", "mgba", "sameboy", "melonds", "snes9x", "bsnes", "gambatte",
"beetle_psx", "beetle_saturn", "beetle_pce", "beetle_pcfx", "mgba",
"beetle_wswan", "beetle_vb", "beetle_ngp", "opera", "stella", "sameboy",
"picodrive", "ppsspp", "handy", "quicknes", "genesis_plus_gx", "melonds",
"ares", "mupen64plus_next", "puae", "prboom", "virtualjaguar", "snes9x",
"vice_x64", "mame", "bsnes",
"beetle_psx",
"beetle_saturn",
"beetle_pce",
"beetle_pcfx",
"beetle_wswan",
"beetle_vb",
"beetle_ngp",
"opera",
"stella",
"picodrive",
"ppsspp",
"handy",
"quicknes",
"genesis_plus_gx",
"ares",
"mupen64plus_next",
"puae",
"prboom",
"virtualjaguar",
"vice_x64",
"mame",
] ]
@@ -137,9 +157,7 @@ def _safe_arithmetic(expr: str) -> int:
def _strip_comments(source: str) -> str: def _strip_comments(source: str) -> str:
"""Remove block comments and #if false blocks.""" """Remove block comments and #if false blocks."""
source = re.sub(r"/\*.*?\*/", "", source, flags=re.DOTALL) source = re.sub(r"/\*.*?\*/", "", source, flags=re.DOTALL)
source = re.sub( source = re.sub(r"#if\s+false\b.*?#endif", "", source, flags=re.DOTALL)
r"#if\s+false\b.*?#endif", "", source, flags=re.DOTALL
)
return source return source
@@ -158,14 +176,14 @@ def parse_firmware_database(
var_to_hash: dict[str, str] = {} var_to_hash: dict[str, str] = {}
file_re = re.compile( file_re = re.compile(
r'(?:var\s+(\w+)\s*=\s*)?' r"(?:var\s+(\w+)\s*=\s*)?"
r'File\(\s*' r"File\(\s*"
r'(?:"([A-Fa-f0-9]+)"|SHA1Checksum\.Dummy)\s*,\s*' r'(?:"([A-Fa-f0-9]+)"|SHA1Checksum\.Dummy)\s*,\s*'
r'([^,]+?)\s*,\s*' r"([^,]+?)\s*,\s*"
r'"([^"]+)"\s*,\s*' r'"([^"]+)"\s*,\s*'
r'"([^"]*)"' r'"([^"]*)"'
r'(?:\s*,\s*isBad:\s*(true|false))?' r"(?:\s*,\s*isBad:\s*(true|false))?"
r'\s*\)' r"\s*\)"
) )
for m in file_re.finditer(source): for m in file_re.finditer(source):
@@ -194,15 +212,15 @@ def parse_firmware_database(
# FirmwareAndOption one-liner # FirmwareAndOption one-liner
fao_re = re.compile( fao_re = re.compile(
r'FirmwareAndOption\(\s*' r"FirmwareAndOption\(\s*"
r'(?:"([A-Fa-f0-9]+)"|SHA1Checksum\.Dummy)\s*,\s*' r'(?:"([A-Fa-f0-9]+)"|SHA1Checksum\.Dummy)\s*,\s*'
r'([^,]+?)\s*,\s*' r"([^,]+?)\s*,\s*"
r'"([^"]+)"\s*,\s*' r'"([^"]+)"\s*,\s*'
r'"([^"]+)"\s*,\s*' r'"([^"]+)"\s*,\s*'
r'"([^"]+)"\s*,\s*' r'"([^"]+)"\s*,\s*'
r'"([^"]*)"' r'"([^"]*)"'
r'(?:\s*,\s*FirmwareOptionStatus\.(\w+))?' r"(?:\s*,\s*FirmwareOptionStatus\.(\w+))?"
r'\s*\)' r"\s*\)"
) )
# Firmware(system, id, desc) # Firmware(system, id, desc)
@@ -213,10 +231,10 @@ def parse_firmware_database(
# Option(system, id, in varref|File(...), status?) # Option(system, id, in varref|File(...), status?)
option_re = re.compile( option_re = re.compile(
r'Option\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*' r'Option\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*'
r'(?:in\s+(\w+)' r"(?:in\s+(\w+)"
r'|File\(\s*"([A-Fa-f0-9]+)"\s*,\s*([^,]+?)\s*,\s*"([^"]+)"\s*,\s*"([^"]*)"\s*\))' r'|File\(\s*"([A-Fa-f0-9]+)"\s*,\s*([^,]+?)\s*,\s*"([^"]+)"\s*,\s*"([^"]*)"\s*\))'
r'(?:\s*,\s*FirmwareOptionStatus\.(\w+))?' r"(?:\s*,\s*FirmwareOptionStatus\.(\w+))?"
r'\s*\)' r"\s*\)"
) )
# Collect firmware slots # Collect firmware slots
@@ -269,15 +287,17 @@ def parse_firmware_database(
desc = m.group(6) desc = m.group(6)
status = m.group(7) or "Acceptable" status = m.group(7) or "Acceptable"
records.append({ records.append(
"system": system, {
"firmware_id": fw_id, "system": system,
"sha1": sha1, "firmware_id": fw_id,
"name": name, "sha1": sha1,
"size": _safe_arithmetic(size_expr), "name": name,
"description": desc, "size": _safe_arithmetic(size_expr),
"status": status, "description": desc,
}) "status": status,
}
)
# Build records from Firmware+Option pairs, picking best option # Build records from Firmware+Option pairs, picking best option
for (system, fw_id), options in slot_options.items(): for (system, fw_id), options in slot_options.items():
@@ -291,15 +311,17 @@ def parse_firmware_database(
viable.sort(key=lambda x: STATUS_RANK.get(x[1], 2), reverse=True) viable.sort(key=lambda x: STATUS_RANK.get(x[1], 2), reverse=True)
best_file, best_status = viable[0] best_file, best_status = viable[0]
records.append({ records.append(
"system": system, {
"firmware_id": fw_id, "system": system,
"sha1": best_file["sha1"], "firmware_id": fw_id,
"name": best_file["name"], "sha1": best_file["sha1"],
"size": best_file["size"], "name": best_file["name"],
"description": best_file.get("description", desc), "size": best_file["size"],
"status": best_status, "description": best_file.get("description", desc),
}) "status": best_status,
}
)
return records, files_by_hash return records, files_by_hash

73
scripts/scraper/coreinfo_scraper.py
View File

@@ -13,19 +13,24 @@ Complements libretro_scraper (System.dat) with:
from __future__ import annotations from __future__ import annotations
import json
import re import re
import sys import sys
import urllib.request
import urllib.error import urllib.error
import json import urllib.request
try: try:
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
except ImportError: except ImportError:
# Allow running directly: python scripts/scraper/coreinfo_scraper.py # Allow running directly: python scripts/scraper/coreinfo_scraper.py
import os import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from scraper.base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version from scraper.base_scraper import (
BaseScraper,
BiosRequirement,
fetch_github_latest_version,
)
PLATFORM_NAME = "libretro_coreinfo" PLATFORM_NAME = "libretro_coreinfo"
@@ -168,11 +173,13 @@ def _extract_firmware(info: dict) -> list[dict]:
if _is_native_lib(path): if _is_native_lib(path):
continue continue
firmware.append({ firmware.append(
"path": path, {
"desc": desc, "path": path,
"optional": opt.lower() == "true", "desc": desc,
}) "optional": opt.lower() == "true",
}
)
return firmware return firmware
@@ -182,7 +189,7 @@ def _extract_md5_from_notes(info: dict) -> dict[str, str]:
notes = info.get("notes", "") notes = info.get("notes", "")
md5_map = {} md5_map = {}
for match in re.finditer(r'\(!\)\s+(.+?)\s+\(md5\):\s+([a-f0-9]{32})', notes): for match in re.finditer(r"\(!\)\s+(.+?)\s+\(md5\):\s+([a-f0-9]{32})", notes):
filename = match.group(1).strip() filename = match.group(1).strip()
md5 = match.group(2) md5 = match.group(2)
md5_map[filename] = md5 md5_map[filename] = md5
@@ -202,15 +209,19 @@ class Scraper(BaseScraper):
# Use the tree API to get all files at once # Use the tree API to get all files at once
url = f"{GITHUB_API}/git/trees/master?recursive=1" url = f"{GITHUB_API}/git/trees/master?recursive=1"
try: try:
req = urllib.request.Request(url, headers={ req = urllib.request.Request(
"User-Agent": "retrobios-scraper/1.0", url,
"Accept": "application/vnd.github.v3+json", headers={
}) "User-Agent": "retrobios-scraper/1.0",
"Accept": "application/vnd.github.v3+json",
},
)
with urllib.request.urlopen(req, timeout=30) as resp: with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read()) data = json.loads(resp.read())
return [ return [
item["path"] for item in data.get("tree", []) item["path"]
for item in data.get("tree", [])
if item["path"].endswith("_libretro.info") if item["path"].endswith("_libretro.info")
] ]
except (urllib.error.URLError, json.JSONDecodeError) as e: except (urllib.error.URLError, json.JSONDecodeError) as e:
@@ -220,7 +231,9 @@ class Scraper(BaseScraper):
"""Fetch and parse a single .info file.""" """Fetch and parse a single .info file."""
url = f"{RAW_BASE}/{filename}" url = f"{RAW_BASE}/{filename}"
try: try:
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"}) req = urllib.request.Request(
url, headers={"User-Agent": "retrobios-scraper/1.0"}
)
with urllib.request.urlopen(req, timeout=15) as resp: with urllib.request.urlopen(req, timeout=15) as resp:
content = resp.read().decode("utf-8") content = resp.read().decode("utf-8")
return _parse_info_file(content) return _parse_info_file(content)
@@ -253,17 +266,25 @@ class Scraper(BaseScraper):
basename = path.split("/")[-1] if "/" in path else path basename = path.split("/")[-1] if "/" in path else path
# Full path when basename is generic to avoid SGB1.sfc/program.rom vs SGB2.sfc/program.rom collisions # Full path when basename is generic to avoid SGB1.sfc/program.rom vs SGB2.sfc/program.rom collisions
GENERIC_NAMES = {"program.rom", "data.rom", "boot.rom", "bios.bin", "firmware.bin"} GENERIC_NAMES = {
"program.rom",
"data.rom",
"boot.rom",
"bios.bin",
"firmware.bin",
}
name = path if basename.lower() in GENERIC_NAMES else basename name = path if basename.lower() in GENERIC_NAMES else basename
md5 = md5_map.get(basename) md5 = md5_map.get(basename)
requirements.append(BiosRequirement( requirements.append(
name=name, BiosRequirement(
system=system, name=name,
md5=md5, system=system,
destination=path, md5=md5,
required=not fw["optional"], destination=path,
)) required=not fw["optional"],
)
)
return requirements return requirements
@@ -281,7 +302,9 @@ def main():
"""CLI entry point.""" """CLI entry point."""
import argparse import argparse
parser = argparse.ArgumentParser(description="Scrape libretro-core-info firmware requirements") parser = argparse.ArgumentParser(
description="Scrape libretro-core-info firmware requirements"
)
parser.add_argument("--dry-run", action="store_true") parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--compare-db", help="Compare against database.json") parser.add_argument("--compare-db", help="Compare against database.json")
args = parser.parse_args() args = parser.parse_args()
@@ -296,6 +319,7 @@ def main():
if args.compare_db: if args.compare_db:
import json as _json import json as _json
with open(args.compare_db) as f: with open(args.compare_db) as f:
db = _json.load(f) db = _json.load(f)
@@ -320,6 +344,7 @@ def main():
return return
from collections import defaultdict from collections import defaultdict
by_system = defaultdict(list) by_system = defaultdict(list)
for r in reqs: for r in reqs:
by_system[r.system].append(r) by_system[r.system].append(r)

21
scripts/scraper/dat_parser.py
View File

@@ -10,13 +10,13 @@ Parses files like libretro's System.dat which uses the format:
from __future__ import annotations from __future__ import annotations
import re
from dataclasses import dataclass from dataclasses import dataclass
@dataclass @dataclass
class DatRom: class DatRom:
"""A ROM entry from a DAT file.""" """A ROM entry from a DAT file."""
name: str name: str
size: int size: int
crc32: str crc32: str
@@ -28,6 +28,7 @@ class DatRom:
@dataclass @dataclass
class DatMetadata: class DatMetadata:
"""Metadata from a DAT file header.""" """Metadata from a DAT file header."""
name: str = "" name: str = ""
version: str = "" version: str = ""
description: str = "" description: str = ""
@@ -53,7 +54,10 @@ def parse_dat(content: str) -> list[DatRom]:
if stripped.startswith("comment "): if stripped.startswith("comment "):
value = stripped[8:].strip().strip('"') value = stripped[8:].strip().strip('"')
if value in ("System", "System, firmware, and BIOS files used by libretro cores."): if value in (
"System",
"System, firmware, and BIOS files used by libretro cores.",
):
continue continue
current_system = value current_system = value
@@ -78,9 +82,16 @@ def parse_dat_metadata(content: str) -> DatMetadata:
if in_header and stripped == ")": if in_header and stripped == ")":
break break
if in_header: if in_header:
for field in ("name", "version", "description", "author", "homepage", "url"): for field in (
"name",
"version",
"description",
"author",
"homepage",
"url",
):
if stripped.startswith(f"{field} "): if stripped.startswith(f"{field} "):
value = stripped[len(field) + 1:].strip().strip('"') value = stripped[len(field) + 1 :].strip().strip('"')
setattr(meta, field, value) setattr(meta, field, value)
return meta return meta
@@ -94,7 +105,7 @@ def _parse_rom_line(line: str, system: str) -> DatRom | None:
if start == -1 or end == -1 or end <= start: if start == -1 or end == -1 or end <= start:
return None return None
content = line[start + 1:end].strip() content = line[start + 1 : end].strip()
fields = {} fields = {}
i = 0 i = 0

130
scripts/scraper/emudeck_scraper.py
View File

@@ -14,9 +14,8 @@ from __future__ import annotations
import csv import csv
import io import io
import re import re
import sys
import urllib.request
import urllib.error import urllib.error
import urllib.request
try: try:
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
@@ -31,8 +30,7 @@ CHECKBIOS_URL = (
) )
CSV_BASE_URL = ( CSV_BASE_URL = (
"https://raw.githubusercontent.com/EmuDeck/emudeck.github.io/" "https://raw.githubusercontent.com/EmuDeck/emudeck.github.io/main/docs/tables"
"main/docs/tables"
) )
CSV_SHEETS = [ CSV_SHEETS = [
@@ -117,10 +115,22 @@ KNOWN_BIOS_FILES = {
{"name": "scph5502.bin", "destination": "scph5502.bin", "region": "EU"}, {"name": "scph5502.bin", "destination": "scph5502.bin", "region": "EU"},
], ],
"sony-playstation-2": [ "sony-playstation-2": [
{"name": "SCPH-70004_BIOS_V12_EUR_200.BIN", "destination": "SCPH-70004_BIOS_V12_EUR_200.BIN"}, {
{"name": "SCPH-70004_BIOS_V12_EUR_200.EROM", "destination": "SCPH-70004_BIOS_V12_EUR_200.EROM"}, "name": "SCPH-70004_BIOS_V12_EUR_200.BIN",
{"name": "SCPH-70004_BIOS_V12_EUR_200.ROM1", "destination": "SCPH-70004_BIOS_V12_EUR_200.ROM1"}, "destination": "SCPH-70004_BIOS_V12_EUR_200.BIN",
{"name": "SCPH-70004_BIOS_V12_EUR_200.ROM2", "destination": "SCPH-70004_BIOS_V12_EUR_200.ROM2"}, },
{
"name": "SCPH-70004_BIOS_V12_EUR_200.EROM",
"destination": "SCPH-70004_BIOS_V12_EUR_200.EROM",
},
{
"name": "SCPH-70004_BIOS_V12_EUR_200.ROM1",
"destination": "SCPH-70004_BIOS_V12_EUR_200.ROM1",
},
{
"name": "SCPH-70004_BIOS_V12_EUR_200.ROM2",
"destination": "SCPH-70004_BIOS_V12_EUR_200.ROM2",
},
], ],
"sega-mega-cd": [ "sega-mega-cd": [
{"name": "bios_CD_E.bin", "destination": "bios_CD_E.bin", "region": "EU"}, {"name": "bios_CD_E.bin", "destination": "bios_CD_E.bin", "region": "EU"},
@@ -157,17 +167,17 @@ KNOWN_BIOS_FILES = {
} }
_RE_ARRAY = re.compile( _RE_ARRAY = re.compile(
r'(?:local\s+)?(\w+)=\(\s*((?:[0-9a-fA-F]+\s*)+)\)', r"(?:local\s+)?(\w+)=\(\s*((?:[0-9a-fA-F]+\s*)+)\)",
re.MULTILINE, re.MULTILINE,
) )
_RE_FUNC = re.compile( _RE_FUNC = re.compile(
r'function\s+(check\w+Bios)\s*\(\)', r"function\s+(check\w+Bios)\s*\(\)",
re.MULTILINE, re.MULTILINE,
) )
_RE_LOCAL_HASHES = re.compile( _RE_LOCAL_HASHES = re.compile(
r'local\s+hashes=\(\s*((?:[0-9a-fA-F]+\s*)+)\)', r"local\s+hashes=\(\s*((?:[0-9a-fA-F]+\s*)+)\)",
re.MULTILINE, re.MULTILINE,
) )
@@ -184,7 +194,9 @@ def _fetch_url(url: str) -> str:
class Scraper(BaseScraper): class Scraper(BaseScraper):
"""Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets.""" """Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets."""
def __init__(self, checkbios_url: str = CHECKBIOS_URL, csv_base_url: str = CSV_BASE_URL): def __init__(
self, checkbios_url: str = CHECKBIOS_URL, csv_base_url: str = CSV_BASE_URL
):
super().__init__(url=checkbios_url) super().__init__(url=checkbios_url)
self.checkbios_url = checkbios_url self.checkbios_url = checkbios_url
self.csv_base_url = csv_base_url self.csv_base_url = csv_base_url
@@ -241,12 +253,12 @@ class Scraper(BaseScraper):
@staticmethod @staticmethod
def _clean_markdown(text: str) -> str: def _clean_markdown(text: str) -> str:
"""Strip markdown/HTML artifacts from CSV fields.""" """Strip markdown/HTML artifacts from CSV fields."""
text = re.sub(r'\*\*', '', text) # bold text = re.sub(r"\*\*", "", text) # bold
text = re.sub(r':material-[^:]+:\{[^}]*\}', '', text) # mkdocs material icons text = re.sub(r":material-[^:]+:\{[^}]*\}", "", text) # mkdocs material icons
text = re.sub(r':material-[^:]+:', '', text) text = re.sub(r":material-[^:]+:", "", text)
text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text) # [text](url) -> text text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text) # [text](url) -> text
text = re.sub(r'<br\s*/?>', ' ', text) # <br/> text = re.sub(r"<br\s*/?>", " ", text) # <br/>
text = re.sub(r'<[^>]+>', '', text) # remaining HTML text = re.sub(r"<[^>]+>", "", text) # remaining HTML
return text.strip() return text.strip()
def _parse_csv_bios(self, csv_text: str) -> list[dict]: def _parse_csv_bios(self, csv_text: str) -> list[dict]:
@@ -274,28 +286,32 @@ class Scraper(BaseScraper):
system_col = self._clean_markdown((row[key] or "")) system_col = self._clean_markdown((row[key] or ""))
break break
slug = None slug = None
for part in re.split(r'[`\s/]+', folder_col): for part in re.split(r"[`\s/]+", folder_col):
part = part.strip().strip('`').lower() part = part.strip().strip("`").lower()
if part and part in SYSTEM_SLUG_MAP: if part and part in SYSTEM_SLUG_MAP:
slug = SYSTEM_SLUG_MAP[part] slug = SYSTEM_SLUG_MAP[part]
break break
if not slug: if not slug:
clean = re.sub(r'[^a-z0-9\-]', '', folder_col.strip().strip('`').lower()) clean = re.sub(
r"[^a-z0-9\-]", "", folder_col.strip().strip("`").lower()
)
slug = clean if clean else "unknown" slug = clean if clean else "unknown"
entries.append({ entries.append(
"system": slug, {
"system_name": system_col, "system": slug,
"bios_raw": bios_col, "system_name": system_col,
}) "bios_raw": bios_col,
}
)
return entries return entries
def _extract_filenames_from_bios_field(self, bios_raw: str) -> list[dict]: def _extract_filenames_from_bios_field(self, bios_raw: str) -> list[dict]:
"""Extract individual BIOS filenames from a CSV BIOS field.""" """Extract individual BIOS filenames from a CSV BIOS field."""
results = [] results = []
bios_raw = re.sub(r'<br\s*/?>', ' ', bios_raw) bios_raw = re.sub(r"<br\s*/?>", " ", bios_raw)
bios_raw = bios_raw.replace('`', '') bios_raw = bios_raw.replace("`", "")
patterns = re.findall( patterns = re.findall(
r'[\w\-./]+\.(?:bin|rom|zip|BIN|ROM|ZIP|EROM|ROM1|ROM2|n64|txt|keys)', r"[\w\-./]+\.(?:bin|rom|zip|BIN|ROM|ZIP|EROM|ROM1|ROM2|n64|txt|keys)",
bios_raw, bios_raw,
) )
for p in patterns: for p in patterns:
@@ -324,21 +340,25 @@ class Scraper(BaseScraper):
if key in seen: if key in seen:
continue continue
seen.add(key) seen.add(key)
requirements.append(BiosRequirement( requirements.append(
name=f["name"], BiosRequirement(
system=system, name=f["name"],
destination=f.get("destination", f["name"]), system=system,
required=True, destination=f.get("destination", f["name"]),
)) required=True,
)
)
for md5 in system_hashes: for md5 in system_hashes:
requirements.append(BiosRequirement( requirements.append(
name=f"{system}:{md5}", BiosRequirement(
system=system, name=f"{system}:{md5}",
md5=md5, system=system,
destination="", md5=md5,
required=True, destination="",
)) required=True,
)
)
for sheet in CSV_SHEETS: for sheet in CSV_SHEETS:
csv_text = self._fetch_csv(sheet) csv_text = self._fetch_csv(sheet)
@@ -353,19 +373,21 @@ class Scraper(BaseScraper):
seen.add(key) seen.add(key)
if system in KNOWN_BIOS_FILES: if system in KNOWN_BIOS_FILES:
continue continue
requirements.append(BiosRequirement( requirements.append(
name=f["name"], BiosRequirement(
system=system, name=f["name"],
destination=f.get("destination", f["name"]), system=system,
required=True, destination=f.get("destination", f["name"]),
)) required=True,
)
)
return requirements return requirements
def validate_format(self, raw_data: str) -> bool: def validate_format(self, raw_data: str) -> bool:
has_ps = "PSBios=" in raw_data or "PSBios =" in raw_data has_ps = "PSBios=" in raw_data or "PSBios =" in raw_data
has_func = "checkPS1BIOS" in raw_data or "checkPS2BIOS" in raw_data has_func = "checkPS1BIOS" in raw_data or "checkPS2BIOS" in raw_data
has_md5 = re.search(r'[0-9a-f]{32}', raw_data) is not None has_md5 = re.search(r"[0-9a-f]{32}", raw_data) is not None
return has_ps and has_func and has_md5 return has_ps and has_func and has_md5
def generate_platform_yaml(self) -> dict: def generate_platform_yaml(self) -> dict:
@@ -419,14 +441,17 @@ class Scraper(BaseScraper):
"contents/functions/EmuScripts" "contents/functions/EmuScripts"
) )
name_overrides = { name_overrides = {
"pcsx2qt": "pcsx2", "rpcs3legacy": "rpcs3", "pcsx2qt": "pcsx2",
"cemuproton": "cemu", "rmg": "mupen64plus_next", "rpcs3legacy": "rpcs3",
"cemuproton": "cemu",
"rmg": "mupen64plus_next",
} }
skip = {"retroarch_maincfg", "retroarch"} skip = {"retroarch_maincfg", "retroarch"}
try: try:
req = urllib.request.Request( req = urllib.request.Request(
api_url, headers={"User-Agent": "retrobios-scraper/1.0"}, api_url,
headers={"User-Agent": "retrobios-scraper/1.0"},
) )
data = json.loads(urllib.request.urlopen(req, timeout=30).read()) data = json.loads(urllib.request.urlopen(req, timeout=30).read())
except (urllib.error.URLError, OSError): except (urllib.error.URLError, OSError):
@@ -454,6 +479,7 @@ class Scraper(BaseScraper):
def main(): def main():
from scripts.scraper.base_scraper import scraper_cli from scripts.scraper.base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape emudeck BIOS requirements") scraper_cli(Scraper, "Scrape emudeck BIOS requirements")

202
scripts/scraper/fbneo_hash_scraper.py
View File

@@ -13,22 +13,22 @@ import logging
import shutil import shutil
import subprocess import subprocess
import sys import sys
from datetime import datetime, timezone, timedelta from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
import yaml import yaml
from scripts.scraper.fbneo_parser import parse_fbneo_source_tree
from scripts.scraper._hash_merge import compute_diff, merge_fbneo_profile from scripts.scraper._hash_merge import compute_diff, merge_fbneo_profile
from scripts.scraper.fbneo_parser import parse_fbneo_source_tree
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
REPO_URL = 'https://github.com/finalburnneo/FBNeo.git' REPO_URL = "https://github.com/finalburnneo/FBNeo.git"
REPO_ROOT = Path(__file__).resolve().parent.parent.parent REPO_ROOT = Path(__file__).resolve().parent.parent.parent
CLONE_DIR = REPO_ROOT / 'tmp' / 'fbneo' CLONE_DIR = REPO_ROOT / "tmp" / "fbneo"
CACHE_PATH = REPO_ROOT / 'data' / 'fbneo-hashes.json' CACHE_PATH = REPO_ROOT / "data" / "fbneo-hashes.json"
EMULATORS_DIR = REPO_ROOT / 'emulators' EMULATORS_DIR = REPO_ROOT / "emulators"
STALE_HOURS = 24 STALE_HOURS = 24
@@ -37,8 +37,8 @@ def _is_cache_fresh() -> bool:
if not CACHE_PATH.exists(): if not CACHE_PATH.exists():
return False return False
try: try:
data = json.loads(CACHE_PATH.read_text(encoding='utf-8')) data = json.loads(CACHE_PATH.read_text(encoding="utf-8"))
fetched_at = datetime.fromisoformat(data['fetched_at']) fetched_at = datetime.fromisoformat(data["fetched_at"])
return datetime.now(timezone.utc) - fetched_at < timedelta(hours=STALE_HOURS) return datetime.now(timezone.utc) - fetched_at < timedelta(hours=STALE_HOURS)
except (json.JSONDecodeError, KeyError, ValueError): except (json.JSONDecodeError, KeyError, ValueError):
return False return False
@@ -53,8 +53,14 @@ def _sparse_clone() -> None:
subprocess.run( subprocess.run(
[ [
'git', 'clone', '--depth', '1', '--filter=blob:none', "git",
'--sparse', REPO_URL, str(CLONE_DIR), "clone",
"--depth",
"1",
"--filter=blob:none",
"--sparse",
REPO_URL,
str(CLONE_DIR),
], ],
check=True, check=True,
capture_output=True, capture_output=True,
@@ -62,7 +68,7 @@ def _sparse_clone() -> None:
) )
subprocess.run( subprocess.run(
['git', 'sparse-checkout', 'set', 'src/burn/drv', 'src/burner/resource.h'], ["git", "sparse-checkout", "set", "src/burn/drv", "src/burner/resource.h"],
cwd=CLONE_DIR, cwd=CLONE_DIR,
check=True, check=True,
capture_output=True, capture_output=True,
@@ -76,42 +82,44 @@ def _extract_version() -> tuple[str, str]:
Returns (version, commit_sha). Falls back to resource.h if no tag. Returns (version, commit_sha). Falls back to resource.h if no tag.
""" """
result = subprocess.run( result = subprocess.run(
['git', 'describe', '--tags', '--abbrev=0'], ["git", "describe", "--tags", "--abbrev=0"],
cwd=CLONE_DIR, cwd=CLONE_DIR,
capture_output=True, capture_output=True,
text=True, text=True,
) )
# Prefer real version tags over pseudo-tags like "latest" # Prefer real version tags over pseudo-tags like "latest"
version = 'unknown' version = "unknown"
if result.returncode == 0: if result.returncode == 0:
tag = result.stdout.strip() tag = result.stdout.strip()
if tag and tag != 'latest': if tag and tag != "latest":
version = tag version = tag
# Fallback: resource.h # Fallback: resource.h
if version == 'unknown': if version == "unknown":
version = _version_from_resource_h() version = _version_from_resource_h()
# Last resort: use GitHub API for latest real release tag # Last resort: use GitHub API for latest real release tag
if version == 'unknown': if version == "unknown":
try: try:
import urllib.request
import urllib.error import urllib.error
import urllib.request
req = urllib.request.Request( req = urllib.request.Request(
'https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10', "https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10",
headers={'User-Agent': 'retrobios-scraper/1.0'}, headers={"User-Agent": "retrobios-scraper/1.0"},
) )
with urllib.request.urlopen(req, timeout=10) as resp: with urllib.request.urlopen(req, timeout=10) as resp:
import json as json_mod import json as json_mod
tags = json_mod.loads(resp.read()) tags = json_mod.loads(resp.read())
for t in tags: for t in tags:
if t['name'] != 'latest' and t['name'].startswith('v'): if t["name"] != "latest" and t["name"].startswith("v"):
version = t['name'] version = t["name"]
break break
except (urllib.error.URLError, OSError): except (urllib.error.URLError, OSError):
pass pass
sha_result = subprocess.run( sha_result = subprocess.run(
['git', 'rev-parse', 'HEAD'], ["git", "rev-parse", "HEAD"],
cwd=CLONE_DIR, cwd=CLONE_DIR,
capture_output=True, capture_output=True,
text=True, text=True,
@@ -124,17 +132,17 @@ def _extract_version() -> tuple[str, str]:
def _version_from_resource_h() -> str: def _version_from_resource_h() -> str:
"""Fallback: parse VER_FULL_VERSION_STR from resource.h.""" """Fallback: parse VER_FULL_VERSION_STR from resource.h."""
resource_h = CLONE_DIR / 'src' / 'burner' / 'resource.h' resource_h = CLONE_DIR / "src" / "burner" / "resource.h"
if not resource_h.exists(): if not resource_h.exists():
return 'unknown' return "unknown"
text = resource_h.read_text(encoding='utf-8', errors='replace') text = resource_h.read_text(encoding="utf-8", errors="replace")
for line in text.splitlines(): for line in text.splitlines():
if 'VER_FULL_VERSION_STR' in line: if "VER_FULL_VERSION_STR" in line:
parts = line.split('"') parts = line.split('"')
if len(parts) >= 2: if len(parts) >= 2:
return parts[1] return parts[1]
return 'unknown' return "unknown"
def _cleanup() -> None: def _cleanup() -> None:
@@ -146,33 +154,33 @@ def _cleanup() -> None:
def fetch_and_cache(force: bool = False) -> dict[str, Any]: def fetch_and_cache(force: bool = False) -> dict[str, Any]:
"""Clone, parse, and write JSON cache. Returns the cache dict.""" """Clone, parse, and write JSON cache. Returns the cache dict."""
if not force and _is_cache_fresh(): if not force and _is_cache_fresh():
log.info('cache fresh, skipping clone (use --force to override)') log.info("cache fresh, skipping clone (use --force to override)")
return json.loads(CACHE_PATH.read_text(encoding='utf-8')) return json.loads(CACHE_PATH.read_text(encoding="utf-8"))
try: try:
log.info('sparse cloning %s', REPO_URL) log.info("sparse cloning %s", REPO_URL)
_sparse_clone() _sparse_clone()
log.info('extracting version') log.info("extracting version")
version, commit = _extract_version() version, commit = _extract_version()
log.info('parsing source tree') log.info("parsing source tree")
bios_sets = parse_fbneo_source_tree(str(CLONE_DIR)) bios_sets = parse_fbneo_source_tree(str(CLONE_DIR))
cache: dict[str, Any] = { cache: dict[str, Any] = {
'source': 'finalburnneo/FBNeo', "source": "finalburnneo/FBNeo",
'version': version, "version": version,
'commit': commit, "commit": commit,
'fetched_at': datetime.now(timezone.utc).isoformat(), "fetched_at": datetime.now(timezone.utc).isoformat(),
'bios_sets': bios_sets, "bios_sets": bios_sets,
} }
CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
CACHE_PATH.write_text( CACHE_PATH.write_text(
json.dumps(cache, indent=2, ensure_ascii=False) + '\n', json.dumps(cache, indent=2, ensure_ascii=False) + "\n",
encoding='utf-8', encoding="utf-8",
) )
log.info('wrote %d BIOS sets to %s', len(bios_sets), CACHE_PATH) log.info("wrote %d BIOS sets to %s", len(bios_sets), CACHE_PATH)
return cache return cache
finally: finally:
@@ -182,48 +190,50 @@ def fetch_and_cache(force: bool = False) -> dict[str, Any]:
def _find_fbneo_profiles() -> list[Path]: def _find_fbneo_profiles() -> list[Path]:
"""Find emulator profiles whose upstream references finalburnneo/FBNeo.""" """Find emulator profiles whose upstream references finalburnneo/FBNeo."""
profiles: list[Path] = [] profiles: list[Path] = []
for path in sorted(EMULATORS_DIR.glob('*.yml')): for path in sorted(EMULATORS_DIR.glob("*.yml")):
if path.name.endswith('.old.yml'): if path.name.endswith(".old.yml"):
continue continue
try: try:
data = yaml.safe_load(path.read_text(encoding='utf-8')) data = yaml.safe_load(path.read_text(encoding="utf-8"))
except (yaml.YAMLError, OSError): except (yaml.YAMLError, OSError):
continue continue
if not data or not isinstance(data, dict): if not data or not isinstance(data, dict):
continue continue
upstream = data.get('upstream', '') upstream = data.get("upstream", "")
if isinstance(upstream, str) and 'finalburnneo/fbneo' in upstream.lower(): if isinstance(upstream, str) and "finalburnneo/fbneo" in upstream.lower():
profiles.append(path) profiles.append(path)
return profiles return profiles
def _format_diff(profile_name: str, diff: dict[str, Any], show_added: bool = True) -> str: def _format_diff(
profile_name: str, diff: dict[str, Any], show_added: bool = True
) -> str:
"""Format diff for a single profile.""" """Format diff for a single profile."""
lines: list[str] = [] lines: list[str] = []
lines.append(f' {profile_name}:') lines.append(f" {profile_name}:")
added = diff.get('added', []) added = diff.get("added", [])
updated = diff.get('updated', []) updated = diff.get("updated", [])
oos = diff.get('out_of_scope', 0) oos = diff.get("out_of_scope", 0)
if not added and not updated: if not added and not updated:
lines.append(' no changes') lines.append(" no changes")
if oos: if oos:
lines.append(f' . {oos} out of scope') lines.append(f" . {oos} out of scope")
return '\n'.join(lines) return "\n".join(lines)
if show_added: if show_added:
for label in added: for label in added:
lines.append(f' + {label}') lines.append(f" + {label}")
elif added: elif added:
lines.append(f' + {len(added)} new ROMs available (main profile only)') lines.append(f" + {len(added)} new ROMs available (main profile only)")
for label in updated: for label in updated:
lines.append(f' ~ {label}') lines.append(f" ~ {label}")
lines.append(f' = {diff["unchanged"]} unchanged') lines.append(f" = {diff['unchanged']} unchanged")
if oos: if oos:
lines.append(f' . {oos} out of scope') lines.append(f" . {oos} out of scope")
return '\n'.join(lines) return "\n".join(lines)
def run( def run(
@@ -234,82 +244,84 @@ def run(
"""Main entry point for the scraper.""" """Main entry point for the scraper."""
cache = fetch_and_cache(force=force) cache = fetch_and_cache(force=force)
version = cache.get('version', 'unknown') version = cache.get("version", "unknown")
commit = cache.get('commit', '?')[:12] commit = cache.get("commit", "?")[:12]
bios_sets = cache.get('bios_sets', {}) bios_sets = cache.get("bios_sets", {})
profiles = _find_fbneo_profiles() profiles = _find_fbneo_profiles()
if json_output: if json_output:
result: dict[str, Any] = { result: dict[str, Any] = {
'source': cache.get('source'), "source": cache.get("source"),
'version': version, "version": version,
'commit': cache.get('commit'), "commit": cache.get("commit"),
'bios_set_count': len(bios_sets), "bios_set_count": len(bios_sets),
'profiles': {}, "profiles": {},
} }
for path in profiles: for path in profiles:
diff = compute_diff(str(path), str(CACHE_PATH), mode='fbneo') diff = compute_diff(str(path), str(CACHE_PATH), mode="fbneo")
result['profiles'][path.stem] = diff result["profiles"][path.stem] = diff
print(json.dumps(result, indent=2)) print(json.dumps(result, indent=2))
return 0 return 0
header = ( header = (
f'fbneo-hashes: {len(bios_sets)} BIOS sets ' f"fbneo-hashes: {len(bios_sets)} BIOS sets "
f'from finalburnneo/FBNeo @ {version} ({commit})' f"from finalburnneo/FBNeo @ {version} ({commit})"
) )
print(header) print(header)
print() print()
if not profiles: if not profiles:
print(' no matching emulator profiles found') print(" no matching emulator profiles found")
return 0 return 0
for path in profiles: for path in profiles:
is_main = path.name == 'fbneo.yml' is_main = path.name == "fbneo.yml"
diff = compute_diff(str(path), str(CACHE_PATH), mode='fbneo') diff = compute_diff(str(path), str(CACHE_PATH), mode="fbneo")
print(_format_diff(path.stem, diff, show_added=is_main)) print(_format_diff(path.stem, diff, show_added=is_main))
effective_added = diff['added'] if is_main else [] effective_added = diff["added"] if is_main else []
if not dry_run and (effective_added or diff['updated']): if not dry_run and (effective_added or diff["updated"]):
merge_fbneo_profile(str(path), str(CACHE_PATH), write=True, add_new=is_main) merge_fbneo_profile(str(path), str(CACHE_PATH), write=True, add_new=is_main)
log.info('merged changes into %s', path.name) log.info("merged changes into %s", path.name)
return 0 return 0
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='Scrape FBNeo BIOS set hashes from upstream source', description="Scrape FBNeo BIOS set hashes from upstream source",
) )
parser.add_argument( parser.add_argument(
'--dry-run', "--dry-run",
action='store_true', action="store_true",
help='show diff without writing changes', help="show diff without writing changes",
) )
parser.add_argument( parser.add_argument(
'--force', "--force",
action='store_true', action="store_true",
help='force re-clone even if cache is fresh', help="force re-clone even if cache is fresh",
) )
parser.add_argument( parser.add_argument(
'--json', "--json",
action='store_true', action="store_true",
dest='json_output', dest="json_output",
help='output diff as JSON', help="output diff as JSON",
) )
args = parser.parse_args() args = parser.parse_args()
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format='%(name)s: %(message)s', format="%(name)s: %(message)s",
) )
sys.exit(run( sys.exit(
dry_run=args.dry_run, run(
force=args.force, dry_run=args.dry_run,
json_output=args.json_output, force=args.force,
)) json_output=args.json_output,
)
)
if __name__ == '__main__': if __name__ == "__main__":
main() main()

43
scripts/scraper/fbneo_parser.py
View File

@@ -11,18 +11,17 @@ import os
import re import re
from pathlib import Path from pathlib import Path
_ROM_ENTRY_RE = re.compile( _ROM_ENTRY_RE = re.compile(
r'\{\s*"([^"]+)"\s*,\s*(0x[\da-fA-F]+)\s*,\s*(0x[\da-fA-F]+)\s*,\s*([^}]+)\}', r'\{\s*"([^"]+)"\s*,\s*(0x[\da-fA-F]+)\s*,\s*(0x[\da-fA-F]+)\s*,\s*([^}]+)\}',
) )
_BURN_DRIVER_RE = re.compile( _BURN_DRIVER_RE = re.compile(
r'struct\s+BurnDriver\s+BurnDrv(\w+)\s*=\s*\{(.*?)\};', r"struct\s+BurnDriver\s+BurnDrv(\w+)\s*=\s*\{(.*?)\};",
re.DOTALL, re.DOTALL,
) )
_ROM_DESC_RE = re.compile( _ROM_DESC_RE = re.compile(
r'static\s+struct\s+BurnRomInfo\s+(\w+)RomDesc\s*\[\s*\]\s*=\s*\{(.*?)\};', r"static\s+struct\s+BurnRomInfo\s+(\w+)RomDesc\s*\[\s*\]\s*=\s*\{(.*?)\};",
re.DOTALL, re.DOTALL,
) )
@@ -37,7 +36,7 @@ def find_bios_sets(source: str, filename: str) -> dict[str, dict]:
for match in _BURN_DRIVER_RE.finditer(source): for match in _BURN_DRIVER_RE.finditer(source):
body = match.group(2) body = match.group(2)
if 'BDF_BOARDROM' not in body: if "BDF_BOARDROM" not in body:
continue continue
# Set name is the first quoted string in the struct body # Set name is the first quoted string in the struct body
@@ -46,11 +45,11 @@ def find_bios_sets(source: str, filename: str) -> dict[str, dict]:
continue continue
set_name = name_match.group(1) set_name = name_match.group(1)
line_num = source[:match.start()].count('\n') + 1 line_num = source[: match.start()].count("\n") + 1
results[set_name] = { results[set_name] = {
'source_file': filename, "source_file": filename,
'source_line': line_num, "source_line": line_num,
} }
return results return results
@@ -63,9 +62,9 @@ def parse_rom_info(source: str, set_name: str) -> list[dict]:
Sentinel entries (empty name) are skipped. Sentinel entries (empty name) are skipped.
""" """
pattern = re.compile( pattern = re.compile(
r'static\s+struct\s+BurnRomInfo\s+' r"static\s+struct\s+BurnRomInfo\s+"
+ re.escape(set_name) + re.escape(set_name)
+ r'RomDesc\s*\[\s*\]\s*=\s*\{(.*?)\};', + r"RomDesc\s*\[\s*\]\s*=\s*\{(.*?)\};",
re.DOTALL, re.DOTALL,
) )
match = pattern.search(source) match = pattern.search(source)
@@ -80,13 +79,15 @@ def parse_rom_info(source: str, set_name: str) -> list[dict]:
if not name: if not name:
continue continue
size = int(entry.group(2), 16) size = int(entry.group(2), 16)
crc32 = format(int(entry.group(3), 16), '08x') crc32 = format(int(entry.group(3), 16), "08x")
roms.append({ roms.append(
'name': name, {
'size': size, "name": name,
'crc32': crc32, "size": size,
}) "crc32": crc32,
}
)
return roms return roms
@@ -100,7 +101,7 @@ def parse_fbneo_source_tree(base_path: str) -> dict[str, dict]:
Returns a dict mapping set name to: Returns a dict mapping set name to:
{source_file, source_line, roms: [{name, size, crc32}, ...]} {source_file, source_line, roms: [{name, size, crc32}, ...]}
""" """
drv_path = Path(base_path) / 'src' / 'burn' / 'drv' drv_path = Path(base_path) / "src" / "burn" / "drv"
if not drv_path.is_dir(): if not drv_path.is_dir():
return {} return {}
@@ -108,20 +109,20 @@ def parse_fbneo_source_tree(base_path: str) -> dict[str, dict]:
for root, _dirs, files in os.walk(drv_path): for root, _dirs, files in os.walk(drv_path):
for fname in files: for fname in files:
if not fname.endswith('.cpp'): if not fname.endswith(".cpp"):
continue continue
filepath = Path(root) / fname filepath = Path(root) / fname
source = filepath.read_text(encoding='utf-8', errors='replace') source = filepath.read_text(encoding="utf-8", errors="replace")
rel_path = str(filepath.relative_to(base_path)) rel_path = str(filepath.relative_to(base_path))
bios_sets = find_bios_sets(source, rel_path) bios_sets = find_bios_sets(source, rel_path)
for set_name, meta in bios_sets.items(): for set_name, meta in bios_sets.items():
roms = parse_rom_info(source, set_name) roms = parse_rom_info(source, set_name)
results[set_name] = { results[set_name] = {
'source_file': meta['source_file'], "source_file": meta["source_file"],
'source_line': meta['source_line'], "source_line": meta["source_line"],
'roms': roms, "roms": roms,
} }
return results return results

180
scripts/scraper/libretro_scraper.py
View File

@@ -8,9 +8,8 @@ Hash: SHA1 primary
from __future__ import annotations from __future__ import annotations
import sys
import urllib.request
import urllib.error import urllib.error
import urllib.request
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format
@@ -18,18 +17,17 @@ from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format
PLATFORM_NAME = "libretro" PLATFORM_NAME = "libretro"
SOURCE_URL = ( SOURCE_URL = (
"https://raw.githubusercontent.com/libretro/libretro-database/" "https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"
"master/dat/System.dat"
) )
# Libretro cores that expect BIOS files in a subdirectory of system/. # Libretro cores that expect BIOS files in a subdirectory of system/.
# System.dat lists filenames flat; the scraper prepends the prefix. # System.dat lists filenames flat; the scraper prepends the prefix.
# ref: each core's libretro.c or equivalent -see platforms/README.md # ref: each core's libretro.c or equivalent -see platforms/README.md
CORE_SUBDIR_MAP = { CORE_SUBDIR_MAP = {
"nec-pc-98": "np2kai", # libretro-np2kai/sdl/libretro.c "nec-pc-98": "np2kai", # libretro-np2kai/sdl/libretro.c
"sharp-x68000": "keropi", # px68k/libretro/libretro.c "sharp-x68000": "keropi", # px68k/libretro/libretro.c
"sega-dreamcast": "dc", # flycast/shell/libretro/libretro.cpp "sega-dreamcast": "dc", # flycast/shell/libretro/libretro.cpp
"sega-dreamcast-arcade": "dc", # flycast -same subfolder "sega-dreamcast-arcade": "dc", # flycast -same subfolder
} }
SYSTEM_SLUG_MAP = { SYSTEM_SLUG_MAP = {
@@ -100,7 +98,6 @@ class Scraper(BaseScraper):
def __init__(self, url: str = SOURCE_URL): def __init__(self, url: str = SOURCE_URL):
super().__init__(url=url) super().__init__(url=url)
def fetch_requirements(self) -> list[BiosRequirement]: def fetch_requirements(self) -> list[BiosRequirement]:
"""Parse System.dat and return BIOS requirements.""" """Parse System.dat and return BIOS requirements."""
raw = self._fetch_raw() raw = self._fetch_raw()
@@ -113,7 +110,9 @@ class Scraper(BaseScraper):
for rom in roms: for rom in roms:
native_system = rom.system native_system = rom.system
system_slug = SYSTEM_SLUG_MAP.get(native_system, native_system.lower().replace(" ", "-")) system_slug = SYSTEM_SLUG_MAP.get(
native_system, native_system.lower().replace(" ", "-")
)
destination = rom.name destination = rom.name
name = rom.name.split("/")[-1] if "/" in rom.name else rom.name name = rom.name.split("/")[-1] if "/" in rom.name else rom.name
@@ -122,17 +121,19 @@ class Scraper(BaseScraper):
if subdir and not destination.startswith(subdir + "/"): if subdir and not destination.startswith(subdir + "/"):
destination = f"{subdir}/{destination}" destination = f"{subdir}/{destination}"
requirements.append(BiosRequirement( requirements.append(
name=name, BiosRequirement(
system=system_slug, name=name,
sha1=rom.sha1 or None, system=system_slug,
md5=rom.md5 or None, sha1=rom.sha1 or None,
crc32=rom.crc32 or None, md5=rom.md5 or None,
size=rom.size or None, crc32=rom.crc32 or None,
destination=destination, size=rom.size or None,
required=True, destination=destination,
native_id=native_system, required=True,
)) native_id=native_system,
)
)
return requirements return requirements
@@ -158,17 +159,22 @@ class Scraper(BaseScraper):
"""Fetch per-core metadata from libretro-core-info .info files.""" """Fetch per-core metadata from libretro-core-info .info files."""
metadata = {} metadata = {}
try: try:
url = f"https://api.github.com/repos/libretro/libretro-core-info/git/trees/master?recursive=1" url = "https://api.github.com/repos/libretro/libretro-core-info/git/trees/master?recursive=1"
req = urllib.request.Request(url, headers={ req = urllib.request.Request(
"User-Agent": "retrobios-scraper/1.0", url,
"Accept": "application/vnd.github.v3+json", headers={
}) "User-Agent": "retrobios-scraper/1.0",
"Accept": "application/vnd.github.v3+json",
},
)
with urllib.request.urlopen(req, timeout=30) as resp: with urllib.request.urlopen(req, timeout=30) as resp:
import json import json
tree = json.loads(resp.read()) tree = json.loads(resp.read())
info_files = [ info_files = [
item["path"] for item in tree.get("tree", []) item["path"]
for item in tree.get("tree", [])
if item["path"].endswith("_libretro.info") if item["path"].endswith("_libretro.info")
] ]
@@ -176,7 +182,9 @@ class Scraper(BaseScraper):
core_name = filename.replace("_libretro.info", "") core_name = filename.replace("_libretro.info", "")
try: try:
info_url = f"https://raw.githubusercontent.com/libretro/libretro-core-info/master/{filename}" info_url = f"https://raw.githubusercontent.com/libretro/libretro-core-info/master/{filename}"
req = urllib.request.Request(info_url, headers={"User-Agent": "retrobios-scraper/1.0"}) req = urllib.request.Request(
info_url, headers={"User-Agent": "retrobios-scraper/1.0"}
)
with urllib.request.urlopen(req, timeout=10) as resp: with urllib.request.urlopen(req, timeout=10) as resp:
content = resp.read().decode("utf-8") content = resp.read().decode("utf-8")
@@ -194,10 +202,11 @@ class Scraper(BaseScraper):
system_name = info.get("systemname", "") system_name = info.get("systemname", "")
manufacturer = info.get("manufacturer", "") manufacturer = info.get("manufacturer", "")
display_name = info.get("display_name", "") display_name = info.get("display_name", "")
categories = info.get("categories", "") info.get("categories", "")
# Map core to our system slug via firmware paths # Map core to our system slug via firmware paths
from .coreinfo_scraper import CORE_SYSTEM_MAP from .coreinfo_scraper import CORE_SYSTEM_MAP
system_slug = CORE_SYSTEM_MAP.get(core_name) system_slug = CORE_SYSTEM_MAP.get(core_name)
if not system_slug: if not system_slug:
continue continue
@@ -267,7 +276,11 @@ class Scraper(BaseScraper):
# ref: Vircon32/libretro.c -virtual console, single BIOS # ref: Vircon32/libretro.c -virtual console, single BIOS
"vircon32": { "vircon32": {
"files": [ "files": [
{"name": "Vircon32Bios.v32", "destination": "Vircon32Bios.v32", "required": True}, {
"name": "Vircon32Bios.v32",
"destination": "Vircon32Bios.v32",
"required": True,
},
], ],
"core": "vircon32", "core": "vircon32",
"manufacturer": "Vircon", "manufacturer": "Vircon",
@@ -276,7 +289,11 @@ class Scraper(BaseScraper):
# ref: xrick/src/sysvid.c, xrick/src/data.c -game data archive # ref: xrick/src/sysvid.c, xrick/src/data.c -game data archive
"xrick": { "xrick": {
"files": [ "files": [
{"name": "data.zip", "destination": "xrick/data.zip", "required": True}, {
"name": "data.zip",
"destination": "xrick/data.zip",
"required": True,
},
], ],
"core": "xrick", "core": "xrick",
"manufacturer": "Other", "manufacturer": "Other",
@@ -318,27 +335,51 @@ class Scraper(BaseScraper):
# segasp.zip for Sega System SP (Flycast) # segasp.zip for Sega System SP (Flycast)
if "sega-dreamcast-arcade" in systems: if "sega-dreamcast-arcade" in systems:
existing = {f["name"] for f in systems["sega-dreamcast-arcade"].get("files", [])} existing = {
f["name"] for f in systems["sega-dreamcast-arcade"].get("files", [])
}
if "segasp.zip" not in existing: if "segasp.zip" not in existing:
systems["sega-dreamcast-arcade"]["files"].append({ systems["sega-dreamcast-arcade"]["files"].append(
"name": "segasp.zip", {
"destination": "dc/segasp.zip", "name": "segasp.zip",
"required": True, "destination": "dc/segasp.zip",
}) "required": True,
}
)
# Extra files missing from System.dat for specific systems. # Extra files missing from System.dat for specific systems.
# Each traced to the core's source code. # Each traced to the core's source code.
EXTRA_SYSTEM_FILES = { EXTRA_SYSTEM_FILES = {
# melonDS DS DSi mode -ref: JesseTG/melonds-ds/src/libretro.cpp # melonDS DS DSi mode -ref: JesseTG/melonds-ds/src/libretro.cpp
"nintendo-ds": [ "nintendo-ds": [
{"name": "dsi_bios7.bin", "destination": "dsi_bios7.bin", "required": True}, {
{"name": "dsi_bios9.bin", "destination": "dsi_bios9.bin", "required": True}, "name": "dsi_bios7.bin",
{"name": "dsi_firmware.bin", "destination": "dsi_firmware.bin", "required": True}, "destination": "dsi_bios7.bin",
{"name": "dsi_nand.bin", "destination": "dsi_nand.bin", "required": True}, "required": True,
},
{
"name": "dsi_bios9.bin",
"destination": "dsi_bios9.bin",
"required": True,
},
{
"name": "dsi_firmware.bin",
"destination": "dsi_firmware.bin",
"required": True,
},
{
"name": "dsi_nand.bin",
"destination": "dsi_nand.bin",
"required": True,
},
], ],
# bsnes SGB naming -ref: bsnes/target-libretro/libretro.cpp # bsnes SGB naming -ref: bsnes/target-libretro/libretro.cpp
"nintendo-sgb": [ "nintendo-sgb": [
{"name": "sgb.boot.rom", "destination": "sgb.boot.rom", "required": False}, {
"name": "sgb.boot.rom",
"destination": "sgb.boot.rom",
"required": False,
},
], ],
# JollyCV -ref: jollycv/libretro.c # JollyCV -ref: jollycv/libretro.c
"coleco-colecovision": [ "coleco-colecovision": [
@@ -348,12 +389,20 @@ class Scraper(BaseScraper):
], ],
# Kronos ST-V -ref: libretro-kronos/libretro/libretro.c # Kronos ST-V -ref: libretro-kronos/libretro/libretro.c
"sega-saturn": [ "sega-saturn": [
{"name": "stvbios.zip", "destination": "kronos/stvbios.zip", "required": True}, {
"name": "stvbios.zip",
"destination": "kronos/stvbios.zip",
"required": True,
},
], ],
# PCSX ReARMed / Beetle PSX alt BIOS -ref: pcsx_rearmed/libpcsxcore/misc.c # PCSX ReARMed / Beetle PSX alt BIOS -ref: pcsx_rearmed/libpcsxcore/misc.c
# docs say PSXONPSP660.bin (uppercase) but core accepts any case # docs say PSXONPSP660.bin (uppercase) but core accepts any case
"sony-playstation": [ "sony-playstation": [
{"name": "psxonpsp660.bin", "destination": "psxonpsp660.bin", "required": False}, {
"name": "psxonpsp660.bin",
"destination": "psxonpsp660.bin",
"required": False,
},
], ],
# Dolphin GC -ref: DolphinLibretro/Boot.cpp:72-73, # Dolphin GC -ref: DolphinLibretro/Boot.cpp:72-73,
# BootManager.cpp:200-217, CommonPaths.h:139 GC_IPL="IPL.bin" # BootManager.cpp:200-217, CommonPaths.h:139 GC_IPL="IPL.bin"
@@ -361,15 +410,43 @@ class Scraper(BaseScraper):
# System.dat gc-ntsc-*.bin names are NOT what Dolphin loads. # System.dat gc-ntsc-*.bin names are NOT what Dolphin loads.
# We add the correct Dolphin paths for BIOS + essential firmware. # We add the correct Dolphin paths for BIOS + essential firmware.
"nintendo-gamecube": [ "nintendo-gamecube": [
{"name": "gc-ntsc-12.bin", "destination": "dolphin-emu/Sys/GC/USA/IPL.bin", "required": False}, {
{"name": "gc-pal-12.bin", "destination": "dolphin-emu/Sys/GC/EUR/IPL.bin", "required": False}, "name": "gc-ntsc-12.bin",
{"name": "gc-ntsc-12.bin", "destination": "dolphin-emu/Sys/GC/JAP/IPL.bin", "required": False}, "destination": "dolphin-emu/Sys/GC/USA/IPL.bin",
"required": False,
},
{
"name": "gc-pal-12.bin",
"destination": "dolphin-emu/Sys/GC/EUR/IPL.bin",
"required": False,
},
{
"name": "gc-ntsc-12.bin",
"destination": "dolphin-emu/Sys/GC/JAP/IPL.bin",
"required": False,
},
# DSP firmware -ref: Source/Core/Core/HW/DSPLLE/DSPHost.cpp # DSP firmware -ref: Source/Core/Core/HW/DSPLLE/DSPHost.cpp
{"name": "dsp_coef.bin", "destination": "dolphin-emu/Sys/GC/dsp_coef.bin", "required": True}, {
{"name": "dsp_rom.bin", "destination": "dolphin-emu/Sys/GC/dsp_rom.bin", "required": True}, "name": "dsp_coef.bin",
"destination": "dolphin-emu/Sys/GC/dsp_coef.bin",
"required": True,
},
{
"name": "dsp_rom.bin",
"destination": "dolphin-emu/Sys/GC/dsp_rom.bin",
"required": True,
},
# Fonts -ref: Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp # Fonts -ref: Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp
{"name": "font_western.bin", "destination": "dolphin-emu/Sys/GC/font_western.bin", "required": False}, {
{"name": "font_japanese.bin", "destination": "dolphin-emu/Sys/GC/font_japanese.bin", "required": False}, "name": "font_western.bin",
"destination": "dolphin-emu/Sys/GC/font_western.bin",
"required": False,
},
{
"name": "font_japanese.bin",
"destination": "dolphin-emu/Sys/GC/font_japanese.bin",
"required": False,
},
], ],
# minivmac casing -ref: minivmac/src/MYOSGLUE.c # minivmac casing -ref: minivmac/src/MYOSGLUE.c
# doc says MacII.rom, repo has MacII.ROM -both work on case-insensitive FS # doc says MacII.rom, repo has MacII.ROM -both work on case-insensitive FS
@@ -455,6 +532,7 @@ class Scraper(BaseScraper):
def main(): def main():
from scripts.scraper.base_scraper import scraper_cli from scripts.scraper.base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape libretro BIOS requirements") scraper_cli(Scraper, "Scrape libretro BIOS requirements")

186
scripts/scraper/mame_hash_scraper.py
View File

@@ -21,16 +21,16 @@ from typing import Any
import yaml import yaml
from .mame_parser import parse_mame_source_tree
from ._hash_merge import compute_diff, merge_mame_profile from ._hash_merge import compute_diff, merge_mame_profile
from .mame_parser import parse_mame_source_tree
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
_ROOT = Path(__file__).resolve().parents[2] _ROOT = Path(__file__).resolve().parents[2]
_CACHE_PATH = _ROOT / 'data' / 'mame-hashes.json' _CACHE_PATH = _ROOT / "data" / "mame-hashes.json"
_CLONE_DIR = _ROOT / 'tmp' / 'mame' _CLONE_DIR = _ROOT / "tmp" / "mame"
_EMULATORS_DIR = _ROOT / 'emulators' _EMULATORS_DIR = _ROOT / "emulators"
_REPO_URL = 'https://github.com/mamedev/mame.git' _REPO_URL = "https://github.com/mamedev/mame.git"
_STALE_HOURS = 24 _STALE_HOURS = 24
@@ -41,7 +41,7 @@ def _load_cache() -> dict[str, Any] | None:
if not _CACHE_PATH.exists(): if not _CACHE_PATH.exists():
return None return None
try: try:
with open(_CACHE_PATH, encoding='utf-8') as f: with open(_CACHE_PATH, encoding="utf-8") as f:
return json.load(f) return json.load(f)
except (json.JSONDecodeError, OSError): except (json.JSONDecodeError, OSError):
return None return None
@@ -50,7 +50,7 @@ def _load_cache() -> dict[str, Any] | None:
def _is_stale(cache: dict[str, Any] | None) -> bool: def _is_stale(cache: dict[str, Any] | None) -> bool:
if cache is None: if cache is None:
return True return True
fetched_at = cache.get('fetched_at') fetched_at = cache.get("fetched_at")
if not fetched_at: if not fetched_at:
return True return True
try: try:
@@ -63,17 +63,19 @@ def _is_stale(cache: dict[str, Any] | None) -> bool:
def _write_cache(data: dict[str, Any]) -> None: def _write_cache(data: dict[str, Any]) -> None:
_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) _CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
with open(_CACHE_PATH, 'w', encoding='utf-8') as f: with open(_CACHE_PATH, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False) json.dump(data, f, indent=2, ensure_ascii=False)
log.info('cache written to %s', _CACHE_PATH) log.info("cache written to %s", _CACHE_PATH)
# ── Git operations ─────────────────────────────────────────────────── # ── Git operations ───────────────────────────────────────────────────
def _run_git(args: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]: def _run_git(
args: list[str], cwd: Path | None = None
) -> subprocess.CompletedProcess[str]:
return subprocess.run( return subprocess.run(
['git', *args], ["git", *args],
cwd=cwd, cwd=cwd,
check=True, check=True,
capture_output=True, capture_output=True,
@@ -86,17 +88,20 @@ def _sparse_clone() -> None:
shutil.rmtree(_CLONE_DIR) shutil.rmtree(_CLONE_DIR)
_CLONE_DIR.parent.mkdir(parents=True, exist_ok=True) _CLONE_DIR.parent.mkdir(parents=True, exist_ok=True)
log.info('sparse cloning mamedev/mame into %s', _CLONE_DIR) log.info("sparse cloning mamedev/mame into %s", _CLONE_DIR)
_run_git([
'clone',
'--depth', '1',
'--filter=blob:none',
'--sparse',
_REPO_URL,
str(_CLONE_DIR),
])
_run_git( _run_git(
['sparse-checkout', 'set', 'src/mame', 'src/devices'], [
"clone",
"--depth",
"1",
"--filter=blob:none",
"--sparse",
_REPO_URL,
str(_CLONE_DIR),
]
)
_run_git(
["sparse-checkout", "set", "src/mame", "src/devices"],
cwd=_CLONE_DIR, cwd=_CLONE_DIR,
) )
@@ -106,41 +111,41 @@ def _get_version() -> str:
# Use GitHub API to get the latest release tag. # Use GitHub API to get the latest release tag.
try: try:
req = urllib.request.Request( req = urllib.request.Request(
'https://api.github.com/repos/mamedev/mame/releases/latest', "https://api.github.com/repos/mamedev/mame/releases/latest",
headers={'User-Agent': 'retrobios-scraper/1.0', headers={
'Accept': 'application/vnd.github.v3+json'}, "User-Agent": "retrobios-scraper/1.0",
"Accept": "application/vnd.github.v3+json",
},
) )
with urllib.request.urlopen(req, timeout=10) as resp: with urllib.request.urlopen(req, timeout=10) as resp:
data = json.loads(resp.read()) data = json.loads(resp.read())
tag = data.get('tag_name', '') tag = data.get("tag_name", "")
if tag: if tag:
return _parse_version_tag(tag) return _parse_version_tag(tag)
except (urllib.error.URLError, json.JSONDecodeError, OSError): except (urllib.error.URLError, json.JSONDecodeError, OSError):
pass pass
return 'unknown' return "unknown"
def _parse_version_tag(tag: str) -> str: def _parse_version_tag(tag: str) -> str:
prefix = 'mame' prefix = "mame"
raw = tag.removeprefix(prefix) if tag.startswith(prefix) else tag raw = tag.removeprefix(prefix) if tag.startswith(prefix) else tag
if raw.isdigit() and len(raw) >= 4: if raw.isdigit() and len(raw) >= 4:
return f'{raw[0]}.{raw[1:]}' return f"{raw[0]}.{raw[1:]}"
return raw return raw
def _get_commit() -> str: def _get_commit() -> str:
try: try:
result = _run_git(['rev-parse', 'HEAD'], cwd=_CLONE_DIR) result = _run_git(["rev-parse", "HEAD"], cwd=_CLONE_DIR)
return result.stdout.strip() return result.stdout.strip()
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
return '' return ""
def _cleanup() -> None: def _cleanup() -> None:
if _CLONE_DIR.exists(): if _CLONE_DIR.exists():
log.info('cleaning up %s', _CLONE_DIR) log.info("cleaning up %s", _CLONE_DIR)
shutil.rmtree(_CLONE_DIR) shutil.rmtree(_CLONE_DIR)
@@ -149,18 +154,21 @@ def _cleanup() -> None:
def _find_mame_profiles() -> list[Path]: def _find_mame_profiles() -> list[Path]:
profiles: list[Path] = [] profiles: list[Path] = []
for path in sorted(_EMULATORS_DIR.glob('*.yml')): for path in sorted(_EMULATORS_DIR.glob("*.yml")):
if path.name.endswith('.old.yml'): if path.name.endswith(".old.yml"):
continue continue
try: try:
with open(path, encoding='utf-8') as f: with open(path, encoding="utf-8") as f:
data = yaml.safe_load(f) data = yaml.safe_load(f)
if not isinstance(data, dict): if not isinstance(data, dict):
continue continue
upstream = data.get('upstream', '') upstream = data.get("upstream", "")
# Only match profiles tracking current MAME (not frozen snapshots # Only match profiles tracking current MAME (not frozen snapshots
# which have upstream like "mamedev/mame/tree/mame0139") # which have upstream like "mamedev/mame/tree/mame0139")
if isinstance(upstream, str) and upstream.rstrip('/') == 'https://github.com/mamedev/mame': if (
isinstance(upstream, str)
and upstream.rstrip("/") == "https://github.com/mamedev/mame"
):
profiles.append(path) profiles.append(path)
except (yaml.YAMLError, OSError): except (yaml.YAMLError, OSError):
continue continue
@@ -179,36 +187,36 @@ def _format_diff(
lines: list[str] = [] lines: list[str] = []
name = profile_path.stem name = profile_path.stem
added = diff.get('added', []) added = diff.get("added", [])
updated = diff.get('updated', []) updated = diff.get("updated", [])
removed = diff.get('removed', []) removed = diff.get("removed", [])
unchanged = diff.get('unchanged', 0) unchanged = diff.get("unchanged", 0)
if not added and not updated and not removed: if not added and not updated and not removed:
lines.append(f' {name}:') lines.append(f" {name}:")
lines.append(' no changes') lines.append(" no changes")
return lines return lines
lines.append(f' {name}:') lines.append(f" {name}:")
if show_added: if show_added:
bios_sets = hashes.get('bios_sets', {}) bios_sets = hashes.get("bios_sets", {})
for set_name in added: for set_name in added:
rom_count = len(bios_sets.get(set_name, {}).get('roms', [])) rom_count = len(bios_sets.get(set_name, {}).get("roms", []))
source_file = bios_sets.get(set_name, {}).get('source_file', '') source_file = bios_sets.get(set_name, {}).get("source_file", "")
source_line = bios_sets.get(set_name, {}).get('source_line', '') source_line = bios_sets.get(set_name, {}).get("source_line", "")
ref = f'{source_file}:{source_line}' if source_file else '' ref = f"{source_file}:{source_line}" if source_file else ""
lines.append(f' + {set_name}.zip ({ref}, {rom_count} ROMs)') lines.append(f" + {set_name}.zip ({ref}, {rom_count} ROMs)")
elif added: elif added:
lines.append(f' + {len(added)} new sets available (main profile only)') lines.append(f" + {len(added)} new sets available (main profile only)")
for set_name in updated: for set_name in updated:
lines.append(f' ~ {set_name}.zip (contents changed)') lines.append(f" ~ {set_name}.zip (contents changed)")
oos = diff.get('out_of_scope', 0) oos = diff.get("out_of_scope", 0)
lines.append(f' = {unchanged} unchanged') lines.append(f" = {unchanged} unchanged")
if oos: if oos:
lines.append(f' . {oos} out of scope (not BIOS root sets)') lines.append(f" . {oos} out of scope (not BIOS root sets)")
return lines return lines
@@ -218,7 +226,7 @@ def _format_diff(
def _fetch_hashes(force: bool) -> dict[str, Any]: def _fetch_hashes(force: bool) -> dict[str, Any]:
cache = _load_cache() cache = _load_cache()
if not force and not _is_stale(cache): if not force and not _is_stale(cache):
log.info('using cached data from %s', cache.get('fetched_at', '')) log.info("using cached data from %s", cache.get("fetched_at", ""))
return cache # type: ignore[return-value] return cache # type: ignore[return-value]
try: try:
@@ -228,11 +236,11 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
commit = _get_commit() commit = _get_commit()
data: dict[str, Any] = { data: dict[str, Any] = {
'source': 'mamedev/mame', "source": "mamedev/mame",
'version': version, "version": version,
'commit': commit, "commit": commit,
'fetched_at': datetime.now(timezone.utc).isoformat(), "fetched_at": datetime.now(timezone.utc).isoformat(),
'bios_sets': bios_sets, "bios_sets": bios_sets,
} }
_write_cache(data) _write_cache(data)
return data return data
@@ -243,34 +251,36 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
def _run(args: argparse.Namespace) -> None: def _run(args: argparse.Namespace) -> None:
hashes = _fetch_hashes(args.force) hashes = _fetch_hashes(args.force)
total_sets = len(hashes.get('bios_sets', {})) total_sets = len(hashes.get("bios_sets", {}))
version = hashes.get('version', 'unknown') version = hashes.get("version", "unknown")
commit = hashes.get('commit', '')[:12] commit = hashes.get("commit", "")[:12]
if args.json: if args.json:
json.dump(hashes, sys.stdout, indent=2, ensure_ascii=False) json.dump(hashes, sys.stdout, indent=2, ensure_ascii=False)
sys.stdout.write('\n') sys.stdout.write("\n")
return return
print(f'mame-hashes: {total_sets} BIOS root sets from mamedev/mame' print(
f' @ {version} ({commit})') f"mame-hashes: {total_sets} BIOS root sets from mamedev/mame"
f" @ {version} ({commit})"
)
print() print()
profiles = _find_mame_profiles() profiles = _find_mame_profiles()
if not profiles: if not profiles:
print(' no profiles with mamedev/mame upstream found') print(" no profiles with mamedev/mame upstream found")
return return
for profile_path in profiles: for profile_path in profiles:
is_main = profile_path.name == 'mame.yml' is_main = profile_path.name == "mame.yml"
diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode='mame') diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode="mame")
lines = _format_diff(profile_path, diff, hashes, show_added=is_main) lines = _format_diff(profile_path, diff, hashes, show_added=is_main)
for line in lines: for line in lines:
print(line) print(line)
if not args.dry_run: if not args.dry_run:
updated = diff.get('updated', []) updated = diff.get("updated", [])
added = diff.get('added', []) if is_main else [] added = diff.get("added", []) if is_main else []
if added or updated: if added or updated:
merge_mame_profile( merge_mame_profile(
str(profile_path), str(profile_path),
@@ -278,32 +288,32 @@ def _run(args: argparse.Namespace) -> None:
write=True, write=True,
add_new=is_main, add_new=is_main,
) )
log.info('merged into %s', profile_path.name) log.info("merged into %s", profile_path.name)
print() print()
if args.dry_run: if args.dry_run:
print('(dry run, no files modified)') print("(dry run, no files modified)")
def build_parser() -> argparse.ArgumentParser: def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog='mame_hash_scraper', prog="mame_hash_scraper",
description='Fetch MAME BIOS hashes from source and merge into profiles.', description="Fetch MAME BIOS hashes from source and merge into profiles.",
) )
parser.add_argument( parser.add_argument(
'--dry-run', "--dry-run",
action='store_true', action="store_true",
help='show diff only, do not modify profiles', help="show diff only, do not modify profiles",
) )
parser.add_argument( parser.add_argument(
'--json', "--json",
action='store_true', action="store_true",
help='output raw JSON to stdout', help="output raw JSON to stdout",
) )
parser.add_argument( parser.add_argument(
'--force', "--force",
action='store_true', action="store_true",
help='re-fetch even if cache is fresh', help="re-fetch even if cache is fresh",
) )
return parser return parser
@@ -311,12 +321,12 @@ def build_parser() -> argparse.ArgumentParser:
def main() -> None: def main() -> None:
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format='%(levelname)s: %(message)s', format="%(levelname)s: %(message)s",
) )
parser = build_parser() parser = build_parser()
args = parser.parse_args() args = parser.parse_args()
_run(args) _run(args)
if __name__ == '__main__': if __name__ == "__main__":
main() main()

149
scripts/scraper/mame_parser.py
View File

@@ -14,27 +14,27 @@ from pathlib import Path
# Macros that declare a machine entry # Macros that declare a machine entry
_MACHINE_MACROS = re.compile( _MACHINE_MACROS = re.compile(
r'\b(GAME|SYST|COMP|CONS)\s*\(', r"\b(GAME|SYST|COMP|CONS)\s*\(",
re.MULTILINE, re.MULTILINE,
) )
# ROM block boundaries # ROM block boundaries
_ROM_START = re.compile(r'ROM_START\s*\(\s*(\w+)\s*\)') _ROM_START = re.compile(r"ROM_START\s*\(\s*(\w+)\s*\)")
_ROM_END = re.compile(r'ROM_END') _ROM_END = re.compile(r"ROM_END")
# ROM_REGION variants: ROM_REGION, ROM_REGION16_BE, ROM_REGION16_LE, ROM_REGION32_LE, etc. # ROM_REGION variants: ROM_REGION, ROM_REGION16_BE, ROM_REGION16_LE, ROM_REGION32_LE, etc.
_ROM_REGION = re.compile( _ROM_REGION = re.compile(
r'ROM_REGION\w*\s*\(' r"ROM_REGION\w*\s*\("
r'\s*(0x[\da-fA-F]+|\d+)\s*,' # size r"\s*(0x[\da-fA-F]+|\d+)\s*," # size
r'\s*"([^"]+)"\s*,', # tag r'\s*"([^"]+)"\s*,', # tag
) )
# ROM_SYSTEM_BIOS( index, label, description ) # ROM_SYSTEM_BIOS( index, label, description )
_ROM_SYSTEM_BIOS = re.compile( _ROM_SYSTEM_BIOS = re.compile(
r'ROM_SYSTEM_BIOS\s*\(' r"ROM_SYSTEM_BIOS\s*\("
r'\s*(\d+)\s*,' # index r"\s*(\d+)\s*," # index
r'\s*"([^"]+)"\s*,' # label r'\s*"([^"]+)"\s*,' # label
r'\s*"([^"]+)"\s*\)', # description r'\s*"([^"]+)"\s*\)', # description
) )
# All ROM_LOAD variants including custom BIOS macros. # All ROM_LOAD variants including custom BIOS macros.
@@ -44,23 +44,23 @@ _ROM_SYSTEM_BIOS = re.compile(
# The key pattern: any macro containing "ROM_LOAD" or "ROMX_LOAD" in its name, # The key pattern: any macro containing "ROM_LOAD" or "ROMX_LOAD" in its name,
# with the first quoted string being the ROM filename. # with the first quoted string being the ROM filename.
_ROM_LOAD = re.compile( _ROM_LOAD = re.compile(
r'\b\w*ROMX?_LOAD\w*\s*\(' r"\b\w*ROMX?_LOAD\w*\s*\("
r'[^"]*' # skip any args before the filename (e.g., bios index) r'[^"]*' # skip any args before the filename (e.g., bios index)
r'"([^"]+)"\s*,' # name (first quoted string) r'"([^"]+)"\s*,' # name (first quoted string)
r'\s*(0x[\da-fA-F]+|\d+)\s*,' # offset r"\s*(0x[\da-fA-F]+|\d+)\s*," # offset
r'\s*(0x[\da-fA-F]+|\d+)\s*,', # size r"\s*(0x[\da-fA-F]+|\d+)\s*,", # size
) )
# CRC32 and SHA1 within a ROM_LOAD line # CRC32 and SHA1 within a ROM_LOAD line
_CRC_SHA = re.compile( _CRC_SHA = re.compile(
r'CRC\s*\(\s*([0-9a-fA-F]+)\s*\)' r"CRC\s*\(\s*([0-9a-fA-F]+)\s*\)"
r'\s+' r"\s+"
r'SHA1\s*\(\s*([0-9a-fA-F]+)\s*\)', r"SHA1\s*\(\s*([0-9a-fA-F]+)\s*\)",
) )
_NO_DUMP = re.compile(r'\bNO_DUMP\b') _NO_DUMP = re.compile(r"\bNO_DUMP\b")
_BAD_DUMP = re.compile(r'\bBAD_DUMP\b') _BAD_DUMP = re.compile(r"\bBAD_DUMP\b")
_ROM_BIOS = re.compile(r'ROM_BIOS\s*\(\s*(\d+)\s*\)') _ROM_BIOS = re.compile(r"ROM_BIOS\s*\(\s*(\d+)\s*\)")
def find_bios_root_sets(source: str, filename: str) -> dict[str, dict]: def find_bios_root_sets(source: str, filename: str) -> dict[str, dict]:
@@ -77,8 +77,8 @@ def find_bios_root_sets(source: str, filename: str) -> dict[str, dict]:
if block_end == -1: if block_end == -1:
continue continue
block = source[start:block_end + 1] block = source[start : block_end + 1]
if 'MACHINE_IS_BIOS_ROOT' not in block: if "MACHINE_IS_BIOS_ROOT" not in block:
continue continue
# Extract set name: first arg after the opening paren # Extract set name: first arg after the opening paren
@@ -97,11 +97,11 @@ def find_bios_root_sets(source: str, filename: str) -> dict[str, dict]:
continue continue
set_name = args[1].strip() set_name = args[1].strip()
line_no = source[:match.start()].count('\n') + 1 line_no = source[: match.start()].count("\n") + 1
results[set_name] = { results[set_name] = {
'source_file': filename, "source_file": filename,
'source_line': line_no, "source_line": line_no,
} }
return results return results
@@ -115,7 +115,7 @@ def parse_rom_block(source: str, set_name: str) -> list[dict]:
extracts all ROM entries. Skips NO_DUMP, flags BAD_DUMP. extracts all ROM entries. Skips NO_DUMP, flags BAD_DUMP.
""" """
pattern = re.compile( pattern = re.compile(
r'ROM_START\s*\(\s*' + re.escape(set_name) + r'\s*\)', r"ROM_START\s*\(\s*" + re.escape(set_name) + r"\s*\)",
) )
start_match = pattern.search(source) start_match = pattern.search(source)
if not start_match: if not start_match:
@@ -125,7 +125,7 @@ def parse_rom_block(source: str, set_name: str) -> list[dict]:
if not end_match: if not end_match:
return [] return []
block = source[start_match.end():end_match.start()] block = source[start_match.end() : end_match.start()]
# Pre-expand macros: find #define macros in the file that contain # Pre-expand macros: find #define macros in the file that contain
# ROM_LOAD/ROM_REGION/ROM_SYSTEM_BIOS calls, then expand their # ROM_LOAD/ROM_REGION/ROM_SYSTEM_BIOS calls, then expand their
@@ -144,26 +144,26 @@ def parse_mame_source_tree(base_path: str) -> dict[str, dict]:
results: dict[str, dict] = {} results: dict[str, dict] = {}
root = Path(base_path) root = Path(base_path)
search_dirs = [root / 'src' / 'mame', root / 'src' / 'devices'] search_dirs = [root / "src" / "mame", root / "src" / "devices"]
for search_dir in search_dirs: for search_dir in search_dirs:
if not search_dir.is_dir(): if not search_dir.is_dir():
continue continue
for dirpath, _dirnames, filenames in os.walk(search_dir): for dirpath, _dirnames, filenames in os.walk(search_dir):
for fname in filenames: for fname in filenames:
if not fname.endswith(('.cpp', '.c', '.h', '.hxx')): if not fname.endswith((".cpp", ".c", ".h", ".hxx")):
continue continue
filepath = Path(dirpath) / fname filepath = Path(dirpath) / fname
rel_path = str(filepath.relative_to(root)) rel_path = str(filepath.relative_to(root))
content = filepath.read_text(encoding='utf-8', errors='replace') content = filepath.read_text(encoding="utf-8", errors="replace")
bios_sets = find_bios_root_sets(content, rel_path) bios_sets = find_bios_root_sets(content, rel_path)
for set_name, info in bios_sets.items(): for set_name, info in bios_sets.items():
roms = parse_rom_block(content, set_name) roms = parse_rom_block(content, set_name)
results[set_name] = { results[set_name] = {
'source_file': info['source_file'], "source_file": info["source_file"],
'source_line': info['source_line'], "source_line": info["source_line"],
'roms': roms, "roms": roms,
} }
return results return results
@@ -171,13 +171,20 @@ def parse_mame_source_tree(base_path: str) -> dict[str, dict]:
# Regex for #define macros that span multiple lines (backslash continuation) # Regex for #define macros that span multiple lines (backslash continuation)
_DEFINE_RE = re.compile( _DEFINE_RE = re.compile(
r'^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s*((?:.*\\\n)*.*)', r"^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s*((?:.*\\\n)*.*)",
re.MULTILINE, re.MULTILINE,
) )
# ROM-related tokens that indicate a macro is relevant for expansion # ROM-related tokens that indicate a macro is relevant for expansion
_ROM_TOKENS = {'ROM_LOAD', 'ROMX_LOAD', 'ROM_REGION', 'ROM_SYSTEM_BIOS', _ROM_TOKENS = {
'ROM_FILL', 'ROM_COPY', 'ROM_RELOAD'} "ROM_LOAD",
"ROMX_LOAD",
"ROM_REGION",
"ROM_SYSTEM_BIOS",
"ROM_FILL",
"ROM_COPY",
"ROM_RELOAD",
}
def _collect_rom_macros(source: str) -> dict[str, str]: def _collect_rom_macros(source: str) -> dict[str, str]:
@@ -193,14 +200,14 @@ def _collect_rom_macros(source: str) -> dict[str, str]:
name = m.group(1) name = m.group(1)
body = m.group(2) body = m.group(2)
# Join backslash-continued lines # Join backslash-continued lines
body = body.replace('\\\n', ' ') body = body.replace("\\\n", " ")
# Only keep macros that contain ROM-related tokens # Only keep macros that contain ROM-related tokens
if not any(tok in body for tok in _ROM_TOKENS): if not any(tok in body for tok in _ROM_TOKENS):
continue continue
# Skip wrapper macros: if the body contains ROMX_LOAD/ROM_LOAD # Skip wrapper macros: if the body contains ROMX_LOAD/ROM_LOAD
# with unquoted args (formal parameters), it's a wrapper. # with unquoted args (formal parameters), it's a wrapper.
# These are already recognized by the _ROM_LOAD regex directly. # These are already recognized by the _ROM_LOAD regex directly.
if re.search(r'ROMX?_LOAD\s*\(\s*\w+\s*,\s*\w+\s*,', body): if re.search(r"ROMX?_LOAD\s*\(\s*\w+\s*,\s*\w+\s*,", body):
continue continue
macros[name] = body macros[name] = body
return macros return macros
@@ -223,7 +230,7 @@ def _expand_macros(block: str, macros: dict[str, str], depth: int = 5) -> str:
iterations += 1 iterations += 1
for name, body in macros.items(): for name, body in macros.items():
# Match macro invocation: NAME or NAME(args) # Match macro invocation: NAME or NAME(args)
pattern = re.compile(r'\b' + re.escape(name) + r'(?:\s*\([^)]*\))?') pattern = re.compile(r"\b" + re.escape(name) + r"(?:\s*\([^)]*\))?")
if pattern.search(block): if pattern.search(block):
block = pattern.sub(body, block) block = pattern.sub(body, block)
changed = True changed = True
@@ -237,9 +244,9 @@ def _find_closing_paren(source: str, start: int) -> int:
i = start i = start
while i < len(source): while i < len(source):
ch = source[i] ch = source[i]
if ch == '(': if ch == "(":
depth += 1 depth += 1
elif ch == ')': elif ch == ")":
depth -= 1 depth -= 1
if depth == 0: if depth == 0:
return i return i
@@ -268,24 +275,24 @@ def _split_macro_args(inner: str) -> list[str]:
i += 1 i += 1
if i < len(inner): if i < len(inner):
current.append(inner[i]) current.append(inner[i])
elif ch == '(': elif ch == "(":
depth += 1 depth += 1
current.append(ch) current.append(ch)
elif ch == ')': elif ch == ")":
if depth == 0: if depth == 0:
args.append(''.join(current)) args.append("".join(current))
break break
depth -= 1 depth -= 1
current.append(ch) current.append(ch)
elif ch == ',' and depth == 0: elif ch == "," and depth == 0:
args.append(''.join(current)) args.append("".join(current))
current = [] current = []
else: else:
current.append(ch) current.append(ch)
i += 1 i += 1
if current: if current:
remaining = ''.join(current).strip() remaining = "".join(current).strip()
if remaining: if remaining:
args.append(remaining) args.append(remaining)
@@ -300,15 +307,15 @@ def _parse_rom_entries(block: str) -> list[dict]:
Processes matches in order of appearance to track region and BIOS context. Processes matches in order of appearance to track region and BIOS context.
""" """
roms: list[dict] = [] roms: list[dict] = []
current_region = '' current_region = ""
bios_labels: dict[int, tuple[str, str]] = {} bios_labels: dict[int, tuple[str, str]] = {}
# Build a combined pattern that matches all interesting tokens # Build a combined pattern that matches all interesting tokens
# and process them in order of occurrence # and process them in order of occurrence
token_patterns = [ token_patterns = [
('region', _ROM_REGION), ("region", _ROM_REGION),
('bios_label', _ROM_SYSTEM_BIOS), ("bios_label", _ROM_SYSTEM_BIOS),
('rom_load', _ROM_LOAD), ("rom_load", _ROM_LOAD),
] ]
# Collect all matches with their positions # Collect all matches with their positions
@@ -321,22 +328,22 @@ def _parse_rom_entries(block: str) -> list[dict]:
events.sort(key=lambda e: e[0]) events.sort(key=lambda e: e[0])
for _pos, tag, m in events: for _pos, tag, m in events:
if tag == 'region': if tag == "region":
current_region = m.group(2) current_region = m.group(2)
elif tag == 'bios_label': elif tag == "bios_label":
idx = int(m.group(1)) idx = int(m.group(1))
bios_labels[idx] = (m.group(2), m.group(3)) bios_labels[idx] = (m.group(2), m.group(3))
elif tag == 'rom_load': elif tag == "rom_load":
# Get the full macro call as context (find closing paren) # Get the full macro call as context (find closing paren)
context_start = m.start() context_start = m.start()
# Find the opening paren of the ROM_LOAD macro # Find the opening paren of the ROM_LOAD macro
paren_pos = block.find('(', context_start) paren_pos = block.find("(", context_start)
if paren_pos != -1: if paren_pos != -1:
close_pos = _find_closing_paren(block, paren_pos) close_pos = _find_closing_paren(block, paren_pos)
context_end = close_pos + 1 if close_pos != -1 else m.end() + 200 context_end = close_pos + 1 if close_pos != -1 else m.end() + 200
else: else:
context_end = m.end() + 200 context_end = m.end() + 200
context = block[context_start:min(context_end, len(block))] context = block[context_start : min(context_end, len(block))]
if _NO_DUMP.search(context): if _NO_DUMP.search(context):
continue continue
@@ -345,8 +352,8 @@ def _parse_rom_entries(block: str) -> list[dict]:
rom_size = _parse_int(m.group(3)) rom_size = _parse_int(m.group(3))
crc_sha_match = _CRC_SHA.search(context) crc_sha_match = _CRC_SHA.search(context)
crc32 = '' crc32 = ""
sha1 = '' sha1 = ""
if crc_sha_match: if crc_sha_match:
crc32 = crc_sha_match.group(1).lower() crc32 = crc_sha_match.group(1).lower()
sha1 = crc_sha_match.group(2).lower() sha1 = crc_sha_match.group(2).lower()
@@ -354,8 +361,8 @@ def _parse_rom_entries(block: str) -> list[dict]:
bad_dump = bool(_BAD_DUMP.search(context)) bad_dump = bool(_BAD_DUMP.search(context))
bios_index = None bios_index = None
bios_label = '' bios_label = ""
bios_description = '' bios_description = ""
bios_ref = _ROM_BIOS.search(context) bios_ref = _ROM_BIOS.search(context)
if bios_ref: if bios_ref:
bios_index = int(bios_ref.group(1)) bios_index = int(bios_ref.group(1))
@@ -363,18 +370,18 @@ def _parse_rom_entries(block: str) -> list[dict]:
bios_label, bios_description = bios_labels[bios_index] bios_label, bios_description = bios_labels[bios_index]
entry: dict = { entry: dict = {
'name': rom_name, "name": rom_name,
'size': rom_size, "size": rom_size,
'crc32': crc32, "crc32": crc32,
'sha1': sha1, "sha1": sha1,
'region': current_region, "region": current_region,
'bad_dump': bad_dump, "bad_dump": bad_dump,
} }
if bios_index is not None: if bios_index is not None:
entry['bios_index'] = bios_index entry["bios_index"] = bios_index
entry['bios_label'] = bios_label entry["bios_label"] = bios_label
entry['bios_description'] = bios_description entry["bios_description"] = bios_description
roms.append(entry) roms.append(entry)
@@ -384,6 +391,6 @@ def _parse_rom_entries(block: str) -> list[dict]:
def _parse_int(value: str) -> int: def _parse_int(value: str) -> int:
"""Parse an integer that may be hex (0x...) or decimal.""" """Parse an integer that may be hex (0x...) or decimal."""
value = value.strip() value = value.strip()
if value.startswith('0x') or value.startswith('0X'): if value.startswith("0x") or value.startswith("0X"):
return int(value, 16) return int(value, 16)
return int(value) return int(value)

65
scripts/scraper/recalbox_scraper.py
View File

@@ -16,8 +16,6 @@ Recalbox verification logic:
from __future__ import annotations from __future__ import annotations
import sys import sys
import urllib.request
import urllib.error
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag
@@ -121,17 +119,19 @@ class Scraper(BaseScraper):
for bios_elem in system_elem.findall("bios"): for bios_elem in system_elem.findall("bios"):
paths_str = bios_elem.get("path", "") paths_str = bios_elem.get("path", "")
md5_str = bios_elem.get("md5", "") md5_str = bios_elem.get("md5", "")
core = bios_elem.get("core", "") bios_elem.get("core", "")
mandatory = bios_elem.get("mandatory", "true") != "false" mandatory = bios_elem.get("mandatory", "true") != "false"
hash_match_mandatory = bios_elem.get("hashMatchMandatory", "true") != "false" bios_elem.get("hashMatchMandatory", "true") != "false"
note = bios_elem.get("note", "") bios_elem.get("note", "")
paths = [p.strip() for p in paths_str.split("|") if p.strip()] paths = [p.strip() for p in paths_str.split("|") if p.strip()]
if not paths: if not paths:
continue continue
primary_path = paths[0] primary_path = paths[0]
name = primary_path.split("/")[-1] if "/" in primary_path else primary_path name = (
primary_path.split("/")[-1] if "/" in primary_path else primary_path
)
md5_list = [m.strip() for m in md5_str.split(",") if m.strip()] md5_list = [m.strip() for m in md5_str.split(",") if m.strip()]
all_md5 = ",".join(md5_list) if md5_list else None all_md5 = ",".join(md5_list) if md5_list else None
@@ -141,14 +141,16 @@ class Scraper(BaseScraper):
continue continue
seen.add(dedup_key) seen.add(dedup_key)
requirements.append(BiosRequirement( requirements.append(
name=name, BiosRequirement(
system=system_slug, name=name,
md5=all_md5, system=system_slug,
destination=primary_path, md5=all_md5,
required=mandatory, destination=primary_path,
native_id=platform, required=mandatory,
)) native_id=platform,
)
)
return requirements return requirements
@@ -168,7 +170,9 @@ class Scraper(BaseScraper):
md5_str = bios_elem.get("md5", "") md5_str = bios_elem.get("md5", "")
core = bios_elem.get("core", "") core = bios_elem.get("core", "")
mandatory = bios_elem.get("mandatory", "true") != "false" mandatory = bios_elem.get("mandatory", "true") != "false"
hash_match_mandatory = bios_elem.get("hashMatchMandatory", "true") != "false" hash_match_mandatory = (
bios_elem.get("hashMatchMandatory", "true") != "false"
)
note = bios_elem.get("note", "") note = bios_elem.get("note", "")
paths = [p.strip() for p in paths_str.split("|") if p.strip()] paths = [p.strip() for p in paths_str.split("|") if p.strip()]
@@ -179,17 +183,19 @@ class Scraper(BaseScraper):
name = paths[0].split("/")[-1] if "/" in paths[0] else paths[0] name = paths[0].split("/")[-1] if "/" in paths[0] else paths[0]
requirements.append({ requirements.append(
"name": name, {
"system": system_slug, "name": name,
"system_name": system_name, "system": system_slug,
"paths": paths, "system_name": system_name,
"md5_list": md5_list, "paths": paths,
"core": core, "md5_list": md5_list,
"mandatory": mandatory, "core": core,
"hash_match_mandatory": hash_match_mandatory, "mandatory": mandatory,
"note": note, "hash_match_mandatory": hash_match_mandatory,
}) "note": note,
}
)
return requirements return requirements
@@ -245,7 +251,9 @@ def main():
parser = argparse.ArgumentParser(description="Scrape Recalbox es_bios.xml") parser = argparse.ArgumentParser(description="Scrape Recalbox es_bios.xml")
parser.add_argument("--dry-run", action="store_true") parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--json", action="store_true") parser.add_argument("--json", action="store_true")
parser.add_argument("--full", action="store_true", help="Show full Recalbox-specific fields") parser.add_argument(
"--full", action="store_true", help="Show full Recalbox-specific fields"
)
parser.add_argument("--output", "-o") parser.add_argument("--output", "-o")
args = parser.parse_args() args = parser.parse_args()
@@ -264,6 +272,7 @@ def main():
if args.dry_run: if args.dry_run:
from collections import defaultdict from collections import defaultdict
by_system = defaultdict(list) by_system = defaultdict(list)
for r in reqs: for r in reqs:
by_system[r.system].append(r) by_system[r.system].append(r)
@@ -272,7 +281,7 @@ def main():
for f in files[:5]: for f in files[:5]:
print(f" {f.name} (md5={f.md5[:12] if f.md5 else 'N/A'}...)") print(f" {f.name} (md5={f.md5[:12] if f.md5 else 'N/A'}...)")
if len(files) > 5: if len(files) > 5:
print(f" ... +{len(files)-5} more") print(f" ... +{len(files) - 5} more")
print(f"\nTotal: {len(reqs)} BIOS files across {len(by_system)} systems") print(f"\nTotal: {len(reqs)} BIOS files across {len(by_system)} systems")
return return

21
scripts/scraper/retrobat_scraper.py
View File

@@ -9,9 +9,6 @@ Hash: MD5 primary
from __future__ import annotations from __future__ import annotations
import json import json
import sys
import urllib.request
import urllib.error
try: try:
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
@@ -43,7 +40,6 @@ class Scraper(BaseScraper):
super().__init__(url=url) super().__init__(url=url)
self._parsed: dict | None = None self._parsed: dict | None = None
def _parse_json(self) -> dict: def _parse_json(self) -> dict:
if self._parsed is not None: if self._parsed is not None:
return self._parsed return self._parsed
@@ -89,13 +85,15 @@ class Scraper(BaseScraper):
name = file_path.split("/")[-1] if "/" in file_path else file_path name = file_path.split("/")[-1] if "/" in file_path else file_path
requirements.append(BiosRequirement( requirements.append(
name=name, BiosRequirement(
system=SYSTEM_SLUG_MAP.get(sys_key, sys_key), name=name,
md5=md5 or None, system=SYSTEM_SLUG_MAP.get(sys_key, sys_key),
destination=file_path, md5=md5 or None,
required=True, destination=file_path,
)) required=True,
)
)
return requirements return requirements
@@ -170,6 +168,7 @@ class Scraper(BaseScraper):
def main(): def main():
from scripts.scraper.base_scraper import scraper_cli from scripts.scraper.base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape retrobat BIOS requirements") scraper_cli(Scraper, "Scrape retrobat BIOS requirements")

57
scripts/scraper/retrodeck_scraper.py
View File

@@ -29,8 +29,8 @@ import json
import os import os
import re import re
import sys import sys
import urllib.request
import urllib.error import urllib.error
import urllib.request
from pathlib import Path from pathlib import Path
try: try:
@@ -43,16 +43,16 @@ PLATFORM_NAME = "retrodeck"
COMPONENTS_REPO = "RetroDECK/components" COMPONENTS_REPO = "RetroDECK/components"
COMPONENTS_BRANCH = "main" COMPONENTS_BRANCH = "main"
COMPONENTS_API_URL = ( COMPONENTS_API_URL = (
f"https://api.github.com/repos/{COMPONENTS_REPO}" f"https://api.github.com/repos/{COMPONENTS_REPO}/git/trees/{COMPONENTS_BRANCH}"
f"/git/trees/{COMPONENTS_BRANCH}"
)
RAW_BASE = (
f"https://raw.githubusercontent.com/{COMPONENTS_REPO}"
f"/{COMPONENTS_BRANCH}"
) )
RAW_BASE = f"https://raw.githubusercontent.com/{COMPONENTS_REPO}/{COMPONENTS_BRANCH}"
SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"} SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"}
NON_EMULATOR_COMPONENTS = { NON_EMULATOR_COMPONENTS = {
"framework", "es-de", "steam-rom-manager", "flips", "portmaster", "framework",
"es-de",
"steam-rom-manager",
"flips",
"portmaster",
} }
# RetroDECK system ID -> retrobios slug. # RetroDECK system ID -> retrobios slug.
@@ -358,13 +358,20 @@ class Scraper(BaseScraper):
required_raw = entry.get("required", "") required_raw = entry.get("required", "")
required = bool(required_raw) and str(required_raw).lower() not in ( required = bool(required_raw) and str(required_raw).lower() not in (
"false", "no", "optional", "", "false",
"no",
"optional",
"",
) )
key = (system, filename.lower()) key = (system, filename.lower())
if key in seen: if key in seen:
existing = next( existing = next(
(r for r in requirements if (r.system, r.name.lower()) == key), (
r
for r in requirements
if (r.system, r.name.lower()) == key
),
None, None,
) )
if existing and md5 and existing.md5 and md5 != existing.md5: if existing and md5 and existing.md5 and md5 != existing.md5:
@@ -376,13 +383,15 @@ class Scraper(BaseScraper):
continue continue
seen.add(key) seen.add(key)
requirements.append(BiosRequirement( requirements.append(
name=filename, BiosRequirement(
system=system, name=filename,
destination=destination, system=system,
md5=md5, destination=destination,
required=required, md5=md5,
)) required=required,
)
)
return requirements return requirements
@@ -390,11 +399,14 @@ class Scraper(BaseScraper):
reqs = self.fetch_requirements() reqs = self.fetch_requirements()
manifests = self._get_manifests() manifests = self._get_manifests()
cores = sorted({ cores = sorted(
comp_name for comp_name, _ in manifests {
if comp_name not in SKIP_DIRS comp_name
and comp_name not in NON_EMULATOR_COMPONENTS for comp_name, _ in manifests
}) if comp_name not in SKIP_DIRS
and comp_name not in NON_EMULATOR_COMPONENTS
}
)
systems: dict[str, dict] = {} systems: dict[str, dict] = {}
for req in reqs: for req in reqs:
@@ -423,6 +435,7 @@ class Scraper(BaseScraper):
def main() -> None: def main() -> None:
from scraper.base_scraper import scraper_cli from scraper.base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape RetroDECK BIOS requirements") scraper_cli(Scraper, "Scrape RetroDECK BIOS requirements")

25
scripts/scraper/romm_scraper.py
View File

@@ -138,16 +138,18 @@ class Scraper(BaseScraper):
crc32 = (entry.get("crc") or "").strip() or None crc32 = (entry.get("crc") or "").strip() or None
size = int(entry["size"]) if entry.get("size") else None size = int(entry["size"]) if entry.get("size") else None
requirements.append(BiosRequirement( requirements.append(
name=filename, BiosRequirement(
system=system, name=filename,
sha1=sha1, system=system,
md5=md5, sha1=sha1,
crc32=crc32, md5=md5,
size=size, crc32=crc32,
destination=f"{igdb_slug}/{filename}", size=size,
required=True, destination=f"{igdb_slug}/{filename}",
)) required=True,
)
)
return requirements return requirements
@@ -164,7 +166,7 @@ class Scraper(BaseScraper):
for key in list(data.keys())[:5]: for key in list(data.keys())[:5]:
if ":" not in key: if ":" not in key:
return False return False
_, entry = key.split(":", 1), data[key] _, _entry = key.split(":", 1), data[key]
if not isinstance(data[key], dict): if not isinstance(data[key], dict):
return False return False
if "md5" not in data[key] and "sha1" not in data[key]: if "md5" not in data[key] and "sha1" not in data[key]:
@@ -217,6 +219,7 @@ class Scraper(BaseScraper):
def main(): def main():
from scripts.scraper.base_scraper import scraper_cli from scripts.scraper.base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape RomM BIOS requirements") scraper_cli(Scraper, "Scrape RomM BIOS requirements")

1
scripts/scraper/targets/__init__.py
View File

@@ -2,6 +2,7 @@
Auto-detects *_targets_scraper.py files and exposes their scrapers. Auto-detects *_targets_scraper.py files and exposes their scrapers.
""" """
from __future__ import annotations from __future__ import annotations
import importlib import importlib

48
scripts/scraper/targets/batocera_targets_scraper.py
View File

@@ -6,6 +6,7 @@ Sources (batocera-linux/batocera.linux):
- package/batocera/emulationstation/batocera-es-system/es_systems.yml - package/batocera/emulationstation/batocera-es-system/es_systems.yml
-- emulator requireAnyOf flag mapping -- emulator requireAnyOf flag mapping
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse
@@ -35,23 +36,23 @@ _HEADERS = {
"Accept": "application/vnd.github.v3+json", "Accept": "application/vnd.github.v3+json",
} }
_TARGET_FLAG_RE = re.compile(r'^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y', re.MULTILINE) _TARGET_FLAG_RE = re.compile(r"^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y", re.MULTILINE)
# Matches: select BR2_PACKAGE_FOO (optional: if CONDITION) # Matches: select BR2_PACKAGE_FOO (optional: if CONDITION)
# Condition may span multiple lines (backslash continuation) # Condition may span multiple lines (backslash continuation)
_SELECT_RE = re.compile( _SELECT_RE = re.compile(
r'^\s+select\s+(BR2_PACKAGE_\w+)' # package being selected r"^\s+select\s+(BR2_PACKAGE_\w+)" # package being selected
r'(?:\s+if\s+((?:[^\n]|\\\n)+?))?' # optional "if CONDITION" (may continue with \) r"(?:\s+if\s+((?:[^\n]|\\\n)+?))?" # optional "if CONDITION" (may continue with \)
r'(?:\s*#[^\n]*)?$', # optional trailing comment r"(?:\s*#[^\n]*)?$", # optional trailing comment
re.MULTILINE, re.MULTILINE,
) )
# Meta-flag definition: "if COND\n\tconfig DERIVED_FLAG\n\t...\nendif" # Meta-flag definition: "if COND\n\tconfig DERIVED_FLAG\n\t...\nendif"
_META_BLOCK_RE = re.compile( _META_BLOCK_RE = re.compile(
r'^if\s+((?:[^\n]|\\\n)+?)\n' # condition (may span lines via \) r"^if\s+((?:[^\n]|\\\n)+?)\n" # condition (may span lines via \)
r'(?:.*?\n)*?' # optional lines before the config r"(?:.*?\n)*?" # optional lines before the config
r'\s+config\s+(BR2_PACKAGE_\w+)' # derived flag name r"\s+config\s+(BR2_PACKAGE_\w+)" # derived flag name
r'.*?^endif', # end of block r".*?^endif", # end of block
re.MULTILINE | re.DOTALL, re.MULTILINE | re.DOTALL,
) )
@@ -80,7 +81,7 @@ def _fetch_json(url: str) -> list | dict | None:
def _normalise_condition(raw: str) -> str: def _normalise_condition(raw: str) -> str:
"""Strip backslash-continuations and collapse whitespace.""" """Strip backslash-continuations and collapse whitespace."""
return re.sub(r'\\\n\s*', ' ', raw).strip() return re.sub(r"\\\n\s*", " ", raw).strip()
def _tokenise(condition: str) -> list[str]: def _tokenise(condition: str) -> list[str]:
@@ -89,14 +90,16 @@ def _tokenise(condition: str) -> list[str]:
return token_re.findall(condition) return token_re.findall(condition)
def _check_condition(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: def _check_condition(
tokens: list[str], pos: int, active: frozenset[str]
) -> tuple[bool, int]:
"""Recursive descent check of a Kconfig boolean expression.""" """Recursive descent check of a Kconfig boolean expression."""
return _check_or(tokens, pos, active) return _check_or(tokens, pos, active)
def _check_or(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: def _check_or(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
left, pos = _check_and(tokens, pos, active) left, pos = _check_and(tokens, pos, active)
while pos < len(tokens) and tokens[pos] == '||': while pos < len(tokens) and tokens[pos] == "||":
pos += 1 pos += 1
right, pos = _check_and(tokens, pos, active) right, pos = _check_and(tokens, pos, active)
left = left or right left = left or right
@@ -105,7 +108,7 @@ def _check_or(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool
def _check_and(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: def _check_and(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
left, pos = _check_not(tokens, pos, active) left, pos = _check_not(tokens, pos, active)
while pos < len(tokens) and tokens[pos] == '&&': while pos < len(tokens) and tokens[pos] == "&&":
pos += 1 pos += 1
right, pos = _check_not(tokens, pos, active) right, pos = _check_not(tokens, pos, active)
left = left and right left = left and right
@@ -113,24 +116,26 @@ def _check_and(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[boo
def _check_not(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: def _check_not(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
if pos < len(tokens) and tokens[pos] == '!': if pos < len(tokens) and tokens[pos] == "!":
pos += 1 pos += 1
val, pos = _check_atom(tokens, pos, active) val, pos = _check_atom(tokens, pos, active)
return not val, pos return not val, pos
return _check_atom(tokens, pos, active) return _check_atom(tokens, pos, active)
def _check_atom(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]: def _check_atom(
tokens: list[str], pos: int, active: frozenset[str]
) -> tuple[bool, int]:
if pos >= len(tokens): if pos >= len(tokens):
return True, pos return True, pos
tok = tokens[pos] tok = tokens[pos]
if tok == '(': if tok == "(":
pos += 1 pos += 1
val, pos = _check_or(tokens, pos, active) val, pos = _check_or(tokens, pos, active)
if pos < len(tokens) and tokens[pos] == ')': if pos < len(tokens) and tokens[pos] == ")":
pos += 1 pos += 1
return val, pos return val, pos
if tok.startswith('BR2_'): if tok.startswith("BR2_"):
pos += 1 pos += 1
return tok in active, pos return tok in active, pos
if tok.startswith('"'): if tok.startswith('"'):
@@ -170,7 +175,9 @@ def _parse_meta_flags(text: str) -> list[tuple[str, str]]:
return results return results
def _expand_flags(primary_flag: str, meta_rules: list[tuple[str, str]]) -> frozenset[str]: def _expand_flags(
primary_flag: str, meta_rules: list[tuple[str, str]]
) -> frozenset[str]:
"""Given a board's primary flag, expand to all active derived flags. """Given a board's primary flag, expand to all active derived flags.
Iterates until stable (handles chained derivations like X86_64_ANY -> X86_ANY). Iterates until stable (handles chained derivations like X86_64_ANY -> X86_ANY).
@@ -194,7 +201,7 @@ def _parse_selects(text: str) -> list[tuple[str, str]]:
results: list[tuple[str, str]] = [] results: list[tuple[str, str]] = []
for m in _SELECT_RE.finditer(text): for m in _SELECT_RE.finditer(text):
pkg = m.group(1) pkg = m.group(1)
cond = _normalise_condition(m.group(2) or '') cond = _normalise_condition(m.group(2) or "")
results.append((pkg, cond)) results.append((pkg, cond))
return results return results
@@ -261,7 +268,8 @@ class Scraper(BaseTargetScraper):
if not data or not isinstance(data, list): if not data or not isinstance(data, list):
return [] return []
return [ return [
item["name"] for item in data item["name"]
for item in data
if isinstance(item, dict) if isinstance(item, dict)
and item.get("name", "").startswith("batocera-") and item.get("name", "").startswith("batocera-")
and item.get("name", "").endswith(".board") and item.get("name", "").endswith(".board")

33
scripts/scraper/targets/emudeck_targets_scraper.py
View File

@@ -4,6 +4,7 @@ Sources:
SteamOS: dragoonDorise/EmuDeck -functions/EmuScripts/*.sh SteamOS: dragoonDorise/EmuDeck -functions/EmuScripts/*.sh
Windows: EmuDeck/emudeck-we -functions/EmuScripts/*.ps1 Windows: EmuDeck/emudeck-we -functions/EmuScripts/*.ps1
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse
@@ -20,8 +21,12 @@ from . import BaseTargetScraper
PLATFORM_NAME = "emudeck" PLATFORM_NAME = "emudeck"
STEAMOS_API = "https://api.github.com/repos/dragoonDorise/EmuDeck/contents/functions/EmuScripts" STEAMOS_API = (
WINDOWS_API = "https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts" "https://api.github.com/repos/dragoonDorise/EmuDeck/contents/functions/EmuScripts"
)
WINDOWS_API = (
"https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts"
)
# Map EmuDeck script names to emulator profile keys # Map EmuDeck script names to emulator profile keys
# Script naming: emuDeckDolphin.sh -> dolphin # Script naming: emuDeckDolphin.sh -> dolphin
@@ -70,8 +75,8 @@ def _list_emuscripts(api_url: str) -> list[str]:
def _script_to_core(filename: str) -> str | None: def _script_to_core(filename: str) -> str | None:
"""Convert EmuScripts filename to core profile key.""" """Convert EmuScripts filename to core profile key."""
# Strip extension and emuDeck prefix # Strip extension and emuDeck prefix
name = re.sub(r'\.(sh|ps1)$', '', filename, flags=re.IGNORECASE) name = re.sub(r"\.(sh|ps1)$", "", filename, flags=re.IGNORECASE)
name = re.sub(r'^emuDeck', '', name, flags=re.IGNORECASE) name = re.sub(r"^emuDeck", "", name, flags=re.IGNORECASE)
if not name: if not name:
return None return None
key = name.lower() key = name.lower()
@@ -86,8 +91,9 @@ class Scraper(BaseTargetScraper):
def __init__(self, url: str = "https://github.com/dragoonDorise/EmuDeck"): def __init__(self, url: str = "https://github.com/dragoonDorise/EmuDeck"):
super().__init__(url=url) super().__init__(url=url)
def _fetch_cores_for_target(self, api_url: str, label: str, def _fetch_cores_for_target(
arch: str = "x86_64") -> list[str]: self, api_url: str, label: str, arch: str = "x86_64"
) -> list[str]:
print(f" fetching {label} EmuScripts...", file=sys.stderr) print(f" fetching {label} EmuScripts...", file=sys.stderr)
scripts = _list_emuscripts(api_url) scripts = _list_emuscripts(api_url)
cores: list[str] = [] cores: list[str] = []
@@ -99,7 +105,7 @@ class Scraper(BaseTargetScraper):
seen.add(core) seen.add(core)
cores.append(core) cores.append(core)
# Detect RetroArch presence (provides all libretro cores) # Detect RetroArch presence (provides all libretro cores)
name = re.sub(r'\.(sh|ps1)$', '', script, flags=re.IGNORECASE) name = re.sub(r"\.(sh|ps1)$", "", script, flags=re.IGNORECASE)
if name.lower() in ("emudeckretroarch", "retroarch_maincfg"): if name.lower() in ("emudeckretroarch", "retroarch_maincfg"):
has_retroarch = True has_retroarch = True
@@ -112,15 +118,18 @@ class Scraper(BaseTargetScraper):
seen.add(c) seen.add(c)
cores.append(c) cores.append(c)
print(f" {label}: {standalone_count} standalone + " print(
f"{len(cores) - standalone_count} via RetroArch = {len(cores)} total", f" {label}: {standalone_count} standalone + "
file=sys.stderr) f"{len(cores) - standalone_count} via RetroArch = {len(cores)} total",
file=sys.stderr,
)
return sorted(cores) return sorted(cores)
@staticmethod @staticmethod
def _load_retroarch_cores(arch: str) -> list[str]: def _load_retroarch_cores(arch: str) -> list[str]:
"""Load RetroArch target cores for given architecture.""" """Load RetroArch target cores for given architecture."""
import os import os
target_path = os.path.join("platforms", "targets", "retroarch.yml") target_path = os.path.join("platforms", "targets", "retroarch.yml")
if not os.path.exists(target_path): if not os.path.exists(target_path):
return [] return []
@@ -157,9 +166,7 @@ class Scraper(BaseTargetScraper):
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(description="Scrape EmuDeck emulator targets")
description="Scrape EmuDeck emulator targets"
)
parser.add_argument("--dry-run", action="store_true", help="Show target summary") parser.add_argument("--dry-run", action="store_true", help="Show target summary")
parser.add_argument("--output", "-o", help="Output YAML file") parser.add_argument("--output", "-o", help="Output YAML file")
args = parser.parse_args() args = parser.parse_args()

17
scripts/scraper/targets/retroarch_targets_scraper.py
View File

@@ -16,6 +16,7 @@ Buildbot structure varies by platform:
- ps2: playstation/ps2/latest/ -> *_libretro_ps2.elf.zip - ps2: playstation/ps2/latest/ -> *_libretro_ps2.elf.zip
- vita: bundles only (VPK) - no individual cores - vita: bundles only (VPK) - no individual cores
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse
@@ -64,7 +65,9 @@ RECIPE_TARGETS: list[tuple[str, str, str]] = [
("playstation/vita", "playstation-vita", "armv7"), ("playstation/vita", "playstation-vita", "armv7"),
] ]
RECIPE_BASE_URL = "https://raw.githubusercontent.com/libretro/libretro-super/master/recipes/" RECIPE_BASE_URL = (
"https://raw.githubusercontent.com/libretro/libretro-super/master/recipes/"
)
# Match any href containing _libretro followed by a platform-specific extension # Match any href containing _libretro followed by a platform-specific extension
# Covers: .so.zip, .dll.zip, .dylib.zip, .nro.zip, .dol.zip, .rpx.zip, # Covers: .so.zip, .dll.zip, .dylib.zip, .nro.zip, .dol.zip, .rpx.zip,
@@ -75,7 +78,7 @@ _HREF_RE = re.compile(
) )
# Extract core name: everything before _libretro # Extract core name: everything before _libretro
_CORE_NAME_RE = re.compile(r'^(.+?)_libretro') _CORE_NAME_RE = re.compile(r"^(.+?)_libretro")
class Scraper(BaseTargetScraper): class Scraper(BaseTargetScraper):
@@ -180,12 +183,16 @@ def main() -> None:
data = scraper.fetch_targets() data = scraper.fetch_targets()
total_cores = sum(len(t["cores"]) for t in data["targets"].values()) total_cores = sum(len(t["cores"]) for t in data["targets"].values())
print(f"\n{len(data['targets'])} targets, {total_cores} total core entries", print(
file=sys.stderr) f"\n{len(data['targets'])} targets, {total_cores} total core entries",
file=sys.stderr,
)
if args.dry_run: if args.dry_run:
for name, info in sorted(data["targets"].items()): for name, info in sorted(data["targets"].items()):
print(f" {name:30s} {info['architecture']:10s} {len(info['cores']):>4d} cores") print(
f" {name:30s} {info['architecture']:10s} {len(info['cores']):>4d} cores"
)
return return
if args.output: if args.output:

1
scripts/scraper/targets/retropie_targets_scraper.py
View File

@@ -4,6 +4,7 @@ Source: https://github.com/RetroPie/RetroPie-Setup/tree/master/scriptmodules/lib
Parses rp_module_id and rp_module_flags from each scriptmodule to determine Parses rp_module_id and rp_module_flags from each scriptmodule to determine
which platforms each core supports. which platforms each core supports.
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse

3
scripts/sect233r1.py
View File

@@ -9,6 +9,7 @@ Curve: sect233r1 (NIST B-233, SEC 2 v2)
Field: GF(2^233) with irreducible polynomial t^233 + t^74 + 1 Field: GF(2^233) with irreducible polynomial t^233 + t^74 + 1
Equation: y^2 + xy = x^3 + x^2 + b Equation: y^2 + xy = x^3 + x^2 + b
""" """
from __future__ import annotations from __future__ import annotations
import hashlib import hashlib
@@ -34,6 +35,7 @@ _H = 2
# GF(2^233) field arithmetic # GF(2^233) field arithmetic
def _gf_reduce(a: int) -> int: def _gf_reduce(a: int) -> int:
"""Reduce polynomial a modulo t^233 + t^74 + 1.""" """Reduce polynomial a modulo t^233 + t^74 + 1."""
while a.bit_length() > _M: while a.bit_length() > _M:
@@ -171,6 +173,7 @@ def _ec_mul(k: int, p: tuple[int, int] | None) -> tuple[int, int] | None:
# ECDSA-SHA256 verification # ECDSA-SHA256 verification
def _modinv(a: int, m: int) -> int: def _modinv(a: int, m: int) -> int:
"""Modular inverse of a modulo m (integers, not GF(2^m)).""" """Modular inverse of a modulo m (integers, not GF(2^m))."""
if a < 0: if a < 0:

108
scripts/truth.py
View File

@@ -13,7 +13,8 @@ from validation import filter_files_by_mode
def _determine_core_mode( def _determine_core_mode(
emu_name: str, profile: dict, emu_name: str,
profile: dict,
cores_config: str | list | None, cores_config: str | list | None,
standalone_set: set[str] | None, standalone_set: set[str] | None,
) -> str: ) -> str:
@@ -62,7 +63,10 @@ def _enrich_hashes(entry: dict, db: dict) -> None:
def _merge_file_into_system( def _merge_file_into_system(
system: dict, file_entry: dict, emu_name: str, db: dict | None, system: dict,
file_entry: dict,
emu_name: str,
db: dict | None,
) -> None: ) -> None:
"""Merge a file entry into a system's file list, deduplicating by name.""" """Merge a file entry into a system's file list, deduplicating by name."""
files = system.setdefault("files", []) files = system.setdefault("files", [])
@@ -100,9 +104,22 @@ def _merge_file_into_system(
entry: dict = {"name": file_entry["name"]} entry: dict = {"name": file_entry["name"]}
if file_entry.get("required") is not None: if file_entry.get("required") is not None:
entry["required"] = file_entry["required"] entry["required"] = file_entry["required"]
for field in ("sha1", "md5", "sha256", "crc32", "size", "path", for field in (
"description", "hle_fallback", "category", "note", "sha1",
"validation", "min_size", "max_size", "aliases"): "md5",
"sha256",
"crc32",
"size",
"path",
"description",
"hle_fallback",
"category",
"note",
"validation",
"min_size",
"max_size",
"aliases",
):
val = file_entry.get(field) val = file_entry.get(field)
if val is not None: if val is not None:
entry[field] = val entry[field] = val
@@ -206,7 +223,9 @@ def generate_platform_truth(
if mode == "both": if mode == "both":
filtered = raw_files filtered = raw_files
else: else:
filtered = filter_files_by_mode(raw_files, standalone=(mode == "standalone")) filtered = filter_files_by_mode(
raw_files, standalone=(mode == "standalone")
)
for fe in filtered: for fe in filtered:
profile_sid = fe.get("system", "") profile_sid = fe.get("system", "")
@@ -217,9 +236,13 @@ def generate_platform_truth(
system = systems.setdefault(sys_id, {}) system = systems.setdefault(sys_id, {})
_merge_file_into_system(system, fe, emu_name, db) _merge_file_into_system(system, fe, emu_name, db)
# Track core contribution per system # Track core contribution per system
sys_cov = system_cores.setdefault(sys_id, { sys_cov = system_cores.setdefault(
"profiled": set(), "unprofiled": set(), sys_id,
}) {
"profiled": set(),
"unprofiled": set(),
},
)
sys_cov["profiled"].add(emu_name) sys_cov["profiled"].add(emu_name)
# Ensure all systems of resolved cores have entries (even with 0 files). # Ensure all systems of resolved cores have entries (even with 0 files).
@@ -230,17 +253,25 @@ def generate_platform_truth(
for prof_sid in profile.get("systems", []): for prof_sid in profile.get("systems", []):
sys_id = _map_sys_id(prof_sid) sys_id = _map_sys_id(prof_sid)
systems.setdefault(sys_id, {}) systems.setdefault(sys_id, {})
sys_cov = system_cores.setdefault(sys_id, { sys_cov = system_cores.setdefault(
"profiled": set(), "unprofiled": set(), sys_id,
}) {
"profiled": set(),
"unprofiled": set(),
},
)
sys_cov["profiled"].add(emu_name) sys_cov["profiled"].add(emu_name)
# Track unprofiled cores per system based on profile system lists # Track unprofiled cores per system based on profile system lists
for emu_name in cores_unprofiled: for emu_name in cores_unprofiled:
for sys_id in systems: for sys_id in systems:
sys_cov = system_cores.setdefault(sys_id, { sys_cov = system_cores.setdefault(
"profiled": set(), "unprofiled": set(), sys_id,
}) {
"profiled": set(),
"unprofiled": set(),
},
)
sys_cov["unprofiled"].add(emu_name) sys_cov["unprofiled"].add(emu_name)
# Convert sets to sorted lists for serialization # Convert sets to sorted lists for serialization
@@ -269,6 +300,7 @@ def generate_platform_truth(
# Platform truth diffing # Platform truth diffing
def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict: def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
"""Compare files between truth and scraped for a single system.""" """Compare files between truth and scraped for a single system."""
# Build truth index: name.lower() -> entry, alias.lower() -> entry # Build truth index: name.lower() -> entry, alias.lower() -> entry
@@ -310,32 +342,38 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
t_set = {v.lower() for v in t_list} t_set = {v.lower() for v in t_list}
s_set = {v.lower() for v in s_list} s_set = {v.lower() for v in s_list}
if not t_set & s_set: if not t_set & s_set:
hash_mismatch.append({ hash_mismatch.append(
"name": s_entry["name"], {
"hash_type": h, "name": s_entry["name"],
f"truth_{h}": t_hash, "hash_type": h,
f"scraped_{h}": s_hash, f"truth_{h}": t_hash,
"truth_cores": list(t_entry.get("_cores", [])), f"scraped_{h}": s_hash,
}) "truth_cores": list(t_entry.get("_cores", [])),
}
)
break break
# Required mismatch # Required mismatch
t_req = t_entry.get("required") t_req = t_entry.get("required")
s_req = s_entry.get("required") s_req = s_entry.get("required")
if t_req is not None and s_req is not None and t_req != s_req: if t_req is not None and s_req is not None and t_req != s_req:
required_mismatch.append({ required_mismatch.append(
"name": s_entry["name"], {
"truth_required": t_req, "name": s_entry["name"],
"scraped_required": s_req, "truth_required": t_req,
}) "scraped_required": s_req,
}
)
# Collect unmatched files from both sides # Collect unmatched files from both sides
unmatched_truth = [ unmatched_truth = [
fe for fe in truth_sys.get("files", []) fe
for fe in truth_sys.get("files", [])
if fe["name"].lower() not in matched_truth_names if fe["name"].lower() not in matched_truth_names
] ]
unmatched_scraped = { unmatched_scraped = {
s_key: s_entry for s_key, s_entry in scraped_index.items() s_key: s_entry
for s_key, s_entry in scraped_index.items()
if s_key not in truth_index if s_key not in truth_index
} }
@@ -369,11 +407,13 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
# Truth files not matched (by name, alias, or hash) -> missing # Truth files not matched (by name, alias, or hash) -> missing
for fe in unmatched_truth: for fe in unmatched_truth:
if fe["name"].lower() not in rename_matched_truth: if fe["name"].lower() not in rename_matched_truth:
missing.append({ missing.append(
"name": fe["name"], {
"cores": list(fe.get("_cores", [])), "name": fe["name"],
"source_refs": list(fe.get("_source_refs", [])), "cores": list(fe.get("_cores", [])),
}) "source_refs": list(fe.get("_source_refs", [])),
}
)
# Scraped files not in truth -> extra # Scraped files not in truth -> extra
coverage = truth_sys.get("_coverage", {}) coverage = truth_sys.get("_coverage", {})

82
scripts/validate_pr.py
View File

@@ -36,8 +36,20 @@ DEFAULT_DB = "database.json"
DEFAULT_PLATFORMS_DIR = "platforms" DEFAULT_PLATFORMS_DIR = "platforms"
BLOCKED_EXTENSIONS = { BLOCKED_EXTENSIONS = {
".exe", ".bat", ".cmd", ".sh", ".ps1", ".vbs", ".js", ".exe",
".msi", ".dll", ".so", ".dylib", ".py", ".rb", ".pl", ".bat",
".cmd",
".sh",
".ps1",
".vbs",
".js",
".msi",
".dll",
".so",
".dylib",
".py",
".rb",
".pl",
} }
MAX_FILE_SIZE = 100 * 1024 * 1024 MAX_FILE_SIZE = 100 * 1024 * 1024
@@ -140,7 +152,10 @@ def validate_file(
result.add_check(False, f"Blocked file extension: {ext}") result.add_check(False, f"Blocked file extension: {ext}")
if result.size > MAX_FILE_SIZE: if result.size > MAX_FILE_SIZE:
result.add_check(False, f"File too large for embedded storage ({result.size:,} > {MAX_FILE_SIZE:,} bytes). Use storage: external in platform config.") result.add_check(
False,
f"File too large for embedded storage ({result.size:,} > {MAX_FILE_SIZE:,} bytes). Use storage: external in platform config.",
)
elif result.size == 0: elif result.size == 0:
result.add_check(False, "File is empty (0 bytes)") result.add_check(False, "File is empty (0 bytes)")
else: else:
@@ -149,7 +164,9 @@ def validate_file(
if db: if db:
if result.sha1 in db.get("files", {}): if result.sha1 in db.get("files", {}):
existing = db["files"][result.sha1] existing = db["files"][result.sha1]
result.add_warning(f"Duplicate: identical file already exists at `{existing['path']}`") result.add_warning(
f"Duplicate: identical file already exists at `{existing['path']}`"
)
else: else:
result.add_check(True, "Not a duplicate in database") result.add_check(True, "Not a duplicate in database")
@@ -162,9 +179,13 @@ def validate_file(
elif md5_known: elif md5_known:
result.add_check(True, "MD5 matches known platform requirement") result.add_check(True, "MD5 matches known platform requirement")
elif name_known: elif name_known:
result.add_warning("Filename matches a known requirement but hash differs - may be a variant") result.add_warning(
"Filename matches a known requirement but hash differs - may be a variant"
)
else: else:
result.add_warning("File not referenced in any platform config - needs manual review") result.add_warning(
"File not referenced in any platform config - needs manual review"
)
normalized = os.path.normpath(filepath) normalized = os.path.normpath(filepath)
if os.path.islink(filepath): if os.path.islink(filepath):
@@ -194,9 +215,15 @@ def get_changed_files() -> list[str]:
try: try:
result = subprocess.run( result = subprocess.run(
["git", "diff", "--name-only", f"origin/{base}...HEAD"], ["git", "diff", "--name-only", f"origin/{base}...HEAD"],
capture_output=True, text=True, check=True, capture_output=True,
text=True,
check=True,
) )
files = [f for f in result.stdout.strip().split("\n") if f.startswith("bios/")] files = [
f
for f in result.stdout.strip().split("\n")
if f.startswith("bios/")
]
if files: if files:
return files return files
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
@@ -206,7 +233,8 @@ def get_changed_files() -> list[str]:
result = subprocess.run( result = subprocess.run(
["git", "diff", "--cached", "--name-only"], ["git", "diff", "--cached", "--name-only"],
capture_output=True, text=True, capture_output=True,
text=True,
) )
return [f for f in result.stdout.strip().split("\n") if f.startswith("bios/") and f] return [f for f in result.stdout.strip().split("\n") if f.startswith("bios/") and f]
@@ -214,10 +242,14 @@ def get_changed_files() -> list[str]:
def main(): def main():
parser = argparse.ArgumentParser(description="Validate BIOS file contributions") parser = argparse.ArgumentParser(description="Validate BIOS file contributions")
parser.add_argument("files", nargs="*", help="Files to validate") parser.add_argument("files", nargs="*", help="Files to validate")
parser.add_argument("--changed", action="store_true", help="Auto-detect changed BIOS files") parser.add_argument(
"--changed", action="store_true", help="Auto-detect changed BIOS files"
)
parser.add_argument("--db", default=DEFAULT_DB, help="Path to database.json") parser.add_argument("--db", default=DEFAULT_DB, help="Path to database.json")
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR) parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
parser.add_argument("--markdown", action="store_true", help="Output as markdown (for PR comments)") parser.add_argument(
"--markdown", action="store_true", help="Output as markdown (for PR comments)"
)
parser.add_argument("--json", action="store_true", help="Output as JSON") parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args() args = parser.parse_args()
@@ -250,14 +282,16 @@ def main():
if args.json: if args.json:
output = [] output = []
for r in results: for r in results:
output.append({ output.append(
"file": r.filepath, {
"passed": r.passed, "file": r.filepath,
"sha1": r.sha1, "passed": r.passed,
"md5": r.md5, "sha1": r.sha1,
"size": r.size, "md5": r.md5,
"checks": [{"status": s, "message": m} for s, m in r.checks], "size": r.size,
}) "checks": [{"status": s, "message": m} for s, m in r.checks],
}
)
print(json.dumps(output, indent=2)) print(json.dumps(output, indent=2))
elif args.markdown: elif args.markdown:
lines = ["## BIOS Validation Report", ""] lines = ["## BIOS Validation Report", ""]
@@ -278,7 +312,15 @@ def main():
print(f" MD5: {r.md5}") print(f" MD5: {r.md5}")
print(f" Size: {r.size:,}") print(f" Size: {r.size:,}")
for s, m in r.checks: for s, m in r.checks:
marker = "✓" if s == "PASS" else "✗" if s == "FAIL" else "!" if s == "WARN" else "i" marker = (
"✓"
if s == "PASS"
else "✗"
if s == "FAIL"
else "!"
if s == "WARN"
else "i"
)
print(f" [{marker}] {m}") print(f" [{marker}] {m}")
if not all_passed: if not all_passed:

59
scripts/validation.py
View File

@@ -63,28 +63,37 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
continue continue
if fname not in index: if fname not in index:
index[fname] = { index[fname] = {
"checks": set(), "sizes": set(), "checks": set(),
"min_size": None, "max_size": None, "sizes": set(),
"crc32": set(), "md5": set(), "sha1": set(), "sha256": set(), "min_size": None,
"adler32": set(), "crypto_only": set(), "max_size": None,
"emulators": set(), "per_emulator": {}, "crc32": set(),
"md5": set(),
"sha1": set(),
"sha256": set(),
"adler32": set(),
"crypto_only": set(),
"emulators": set(),
"per_emulator": {},
} }
index[fname]["emulators"].add(emu_name) index[fname]["emulators"].add(emu_name)
index[fname]["checks"].update(checks) index[fname]["checks"].update(checks)
# Track non-reproducible crypto checks # Track non-reproducible crypto checks
index[fname]["crypto_only"].update( index[fname]["crypto_only"].update(c for c in checks if c in _CRYPTO_CHECKS)
c for c in checks if c in _CRYPTO_CHECKS
)
# Size checks # Size checks
if "size" in checks: if "size" in checks:
if f.get("size") is not None: if f.get("size") is not None:
index[fname]["sizes"].add(f["size"]) index[fname]["sizes"].add(f["size"])
if f.get("min_size") is not None: if f.get("min_size") is not None:
cur = index[fname]["min_size"] cur = index[fname]["min_size"]
index[fname]["min_size"] = min(cur, f["min_size"]) if cur is not None else f["min_size"] index[fname]["min_size"] = (
min(cur, f["min_size"]) if cur is not None else f["min_size"]
)
if f.get("max_size") is not None: if f.get("max_size") is not None:
cur = index[fname]["max_size"] cur = index[fname]["max_size"]
index[fname]["max_size"] = max(cur, f["max_size"]) if cur is not None else f["max_size"] index[fname]["max_size"] = (
max(cur, f["max_size"]) if cur is not None else f["max_size"]
)
# Hash checks -collect all accepted hashes as sets (multiple valid # Hash checks -collect all accepted hashes as sets (multiple valid
# versions of the same file, e.g. MT-32 ROM versions) # versions of the same file, e.g. MT-32 ROM versions)
if "crc32" in checks and f.get("crc32"): if "crc32" in checks and f.get("crc32"):
@@ -132,7 +141,9 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
if emu_name in pe: if emu_name in pe:
# Merge checks from multiple file entries for same emulator # Merge checks from multiple file entries for same emulator
existing = pe[emu_name] existing = pe[emu_name]
merged_checks = sorted(set(existing["checks"]) | set(pe_entry["checks"])) merged_checks = sorted(
set(existing["checks"]) | set(pe_entry["checks"])
)
existing["checks"] = merged_checks existing["checks"] = merged_checks
existing["expected"].update(pe_entry["expected"]) existing["expected"].update(pe_entry["expected"])
if pe_entry["source_ref"] and not existing["source_ref"]: if pe_entry["source_ref"] and not existing["source_ref"]:
@@ -160,17 +171,21 @@ def build_ground_truth(filename: str, validation_index: dict[str, dict]) -> list
result = [] result = []
for emu_name in sorted(entry["per_emulator"]): for emu_name in sorted(entry["per_emulator"]):
detail = entry["per_emulator"][emu_name] detail = entry["per_emulator"][emu_name]
result.append({ result.append(
"emulator": emu_name, {
"checks": detail["checks"], "emulator": emu_name,
"source_ref": detail.get("source_ref"), "checks": detail["checks"],
"expected": detail.get("expected", {}), "source_ref": detail.get("source_ref"),
}) "expected": detail.get("expected", {}),
}
)
return result return result
def check_file_validation( def check_file_validation(
local_path: str, filename: str, validation_index: dict[str, dict], local_path: str,
filename: str,
validation_index: dict[str, dict],
bios_dir: str = "bios", bios_dir: str = "bios",
) -> str | None: ) -> str | None:
"""Check emulator-level validation on a resolved file. """Check emulator-level validation on a resolved file.
@@ -199,10 +214,9 @@ def check_file_validation(
# Hash checks -compute once, reuse for all hash types. # Hash checks -compute once, reuse for all hash types.
# Each hash field is a set of accepted values (multiple valid ROM versions). # Each hash field is a set of accepted values (multiple valid ROM versions).
need_hashes = ( need_hashes = any(
any(h in checks and entry.get(h) for h in ("crc32", "md5", "sha1", "sha256")) h in checks and entry.get(h) for h in ("crc32", "md5", "sha1", "sha256")
or entry.get("adler32") ) or entry.get("adler32")
)
if need_hashes: if need_hashes:
hashes = compute_hashes(local_path) hashes = compute_hashes(local_path)
for hash_type in ("crc32", "md5", "sha1", "sha256"): for hash_type in ("crc32", "md5", "sha1", "sha256"):
@@ -218,6 +232,7 @@ def check_file_validation(
# Signature/crypto checks (3DS RSA, AES) # Signature/crypto checks (3DS RSA, AES)
if entry["crypto_only"]: if entry["crypto_only"]:
from crypto_verify import check_crypto_validation from crypto_verify import check_crypto_validation
crypto_reason = check_crypto_validation(local_path, filename, bios_dir) crypto_reason = check_crypto_validation(local_path, filename, bios_dir)
if crypto_reason: if crypto_reason:
return crypto_reason return crypto_reason

511
scripts/verify.py
View File

@@ -21,28 +21,41 @@ Usage:
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import hashlib
import json import json
import os import os
import sys import sys
import zipfile import zipfile
from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__)) sys.path.insert(0, os.path.dirname(__file__))
from common import ( from common import (
build_target_cores_cache, build_zip_contents_index, check_inside_zip, build_target_cores_cache,
compute_hashes, expand_platform_declared_names, filter_systems_by_target, build_zip_contents_index,
group_identical_platforms, list_emulator_profiles, list_system_ids, check_inside_zip,
load_data_dir_registry, load_emulator_profiles, load_platform_config, compute_hashes,
md5sum, md5_composite, require_yaml, resolve_local_file, expand_platform_declared_names,
filter_systems_by_target,
group_identical_platforms,
list_emulator_profiles,
list_system_ids,
load_data_dir_registry,
load_emulator_profiles,
load_platform_config,
md5_composite,
md5sum,
require_yaml,
resolve_local_file,
resolve_platform_cores, resolve_platform_cores,
) )
yaml = require_yaml() yaml = require_yaml()
from validation import ( from validation import (
_build_validation_index, _parse_validation, build_ground_truth, _build_validation_index,
check_file_validation, filter_files_by_mode, _parse_validation,
build_ground_truth,
check_file_validation,
filter_files_by_mode,
) )
DEFAULT_DB = "database.json" DEFAULT_DB = "database.json"
DEFAULT_PLATFORMS_DIR = "platforms" DEFAULT_PLATFORMS_DIR = "platforms"
DEFAULT_EMULATORS_DIR = "emulators" DEFAULT_EMULATORS_DIR = "emulators"
@@ -50,27 +63,36 @@ DEFAULT_EMULATORS_DIR = "emulators"
# Status model -aligned with Batocera BiosStatus (batocera-systems:967-969) # Status model -aligned with Batocera BiosStatus (batocera-systems:967-969)
class Status: class Status:
OK = "ok" OK = "ok"
UNTESTED = "untested" # file present, hash not confirmed UNTESTED = "untested" # file present, hash not confirmed
MISSING = "missing" MISSING = "missing"
# Severity for per-file required/optional distinction # Severity for per-file required/optional distinction
class Severity: class Severity:
CRITICAL = "critical" # required file missing or bad hash (Recalbox RED) CRITICAL = "critical" # required file missing or bad hash (Recalbox RED)
WARNING = "warning" # optional missing or hash mismatch (Recalbox YELLOW) WARNING = "warning" # optional missing or hash mismatch (Recalbox YELLOW)
INFO = "info" # optional missing on existence-only platform INFO = "info" # optional missing on existence-only platform
OK = "ok" # file verified OK = "ok" # file verified
_STATUS_ORDER = {Status.OK: 0, Status.UNTESTED: 1, Status.MISSING: 2} _STATUS_ORDER = {Status.OK: 0, Status.UNTESTED: 1, Status.MISSING: 2}
_SEVERITY_ORDER = {Severity.OK: 0, Severity.INFO: 1, Severity.WARNING: 2, Severity.CRITICAL: 3} _SEVERITY_ORDER = {
Severity.OK: 0,
Severity.INFO: 1,
Severity.WARNING: 2,
Severity.CRITICAL: 3,
}
# Verification functions # Verification functions
def verify_entry_existence( def verify_entry_existence(
file_entry: dict, local_path: str | None, file_entry: dict,
local_path: str | None,
validation_index: dict[str, dict] | None = None, validation_index: dict[str, dict] | None = None,
) -> dict: ) -> dict:
"""RetroArch verification: path_is_valid() -file exists = OK.""" """RetroArch verification: path_is_valid() -file exists = OK."""
@@ -120,13 +142,25 @@ def verify_entry_md5(
elif result != "not_in_zip": elif result != "not_in_zip":
found_in_zip = True found_in_zip = True
if had_error and not found_in_zip: if had_error and not found_in_zip:
return {**base, "status": Status.UNTESTED, "path": local_path, return {
"reason": f"{local_path} read error"} **base,
"status": Status.UNTESTED,
"path": local_path,
"reason": f"{local_path} read error",
}
if not found_in_zip: if not found_in_zip:
return {**base, "status": Status.UNTESTED, "path": local_path, return {
"reason": f"{zipped_file} not found inside ZIP"} **base,
return {**base, "status": Status.UNTESTED, "path": local_path, "status": Status.UNTESTED,
"reason": f"{zipped_file} MD5 mismatch inside ZIP"} "path": local_path,
"reason": f"{zipped_file} not found inside ZIP",
}
return {
**base,
"status": Status.UNTESTED,
"path": local_path,
"reason": f"{zipped_file} MD5 mismatch inside ZIP",
}
if not md5_list: if not md5_list:
return {**base, "status": Status.OK, "path": local_path} return {**base, "status": Status.OK, "path": local_path}
@@ -151,8 +185,12 @@ def verify_entry_md5(
except (zipfile.BadZipFile, OSError): except (zipfile.BadZipFile, OSError):
pass pass
return {**base, "status": Status.UNTESTED, "path": local_path, return {
"reason": f"expected {md5_list[0][:12]}… got {actual_md5[:12]}…"} **base,
"status": Status.UNTESTED,
"path": local_path,
"reason": f"expected {md5_list[0][:12]}… got {actual_md5[:12]}…",
}
def verify_entry_sha1( def verify_entry_sha1(
@@ -176,14 +214,22 @@ def verify_entry_sha1(
if actual_sha1 == expected_sha1.lower(): if actual_sha1 == expected_sha1.lower():
return {**base, "status": Status.OK, "path": local_path} return {**base, "status": Status.OK, "path": local_path}
return {**base, "status": Status.UNTESTED, "path": local_path, return {
"reason": f"expected {expected_sha1[:12]}… got {actual_sha1[:12]}…"} **base,
"status": Status.UNTESTED,
"path": local_path,
"reason": f"expected {expected_sha1[:12]}… got {actual_sha1[:12]}…",
}
# Severity mapping per platform # Severity mapping per platform
def compute_severity( def compute_severity(
status: str, required: bool, mode: str, hle_fallback: bool = False, status: str,
required: bool,
mode: str,
hle_fallback: bool = False,
) -> str: ) -> str:
"""Map (status, required, verification_mode, hle_fallback) -> severity. """Map (status, required, verification_mode, hle_fallback) -> severity.
@@ -235,8 +281,13 @@ def _build_expected(file_entry: dict, checks: list[str]) -> dict:
expected["adler32"] = adler_val expected["adler32"] = adler_val
return expected return expected
def _name_in_index(name: str, by_name: dict, by_path_suffix: dict | None = None,
data_names: set[str] | None = None) -> bool: def _name_in_index(
name: str,
by_name: dict,
by_path_suffix: dict | None = None,
data_names: set[str] | None = None,
) -> bool:
"""Check if a name is resolvable in the database indexes or data directories.""" """Check if a name is resolvable in the database indexes or data directories."""
if name in by_name: if name in by_name:
return True return True
@@ -248,7 +299,9 @@ def _name_in_index(name: str, by_name: dict, by_path_suffix: dict | None = None,
if data_names: if data_names:
if name in data_names or name.lower() in data_names: if name in data_names or name.lower() in data_names:
return True return True
if basename != name and (basename in data_names or basename.lower() in data_names): if basename != name and (
basename in data_names or basename.lower() in data_names
):
return True return True
return False return False
@@ -276,7 +329,11 @@ def find_undeclared_files(
by_name = db.get("indexes", {}).get("by_name", {}) by_name = db.get("indexes", {}).get("by_name", {})
by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {}) by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir) profiles = (
emu_profiles
if emu_profiles is not None
else load_emulator_profiles(emulators_dir)
)
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores) relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
standalone_set = set(str(c) for c in config.get("standalone_cores", [])) standalone_set = set(str(c) for c in config.get("standalone_cores", []))
@@ -340,7 +397,9 @@ def find_undeclared_files(
# Archived files are grouped by archive # Archived files are grouped by archive
if archive: if archive:
if archive not in archive_entries: if archive not in archive_entries:
in_repo = _name_in_index(archive, by_name, by_path_suffix, data_names) in_repo = _name_in_index(
archive, by_name, by_path_suffix, data_names
)
archive_entries[archive] = { archive_entries[archive] = {
"emulator": profile.get("emulator", emu_name), "emulator": profile.get("emulator", emu_name),
"name": archive, "name": archive,
@@ -377,19 +436,21 @@ def find_undeclared_files(
in_repo = _name_in_index(path_base, by_name, by_path_suffix, data_names) in_repo = _name_in_index(path_base, by_name, by_path_suffix, data_names)
checks = _parse_validation(f.get("validation")) checks = _parse_validation(f.get("validation"))
undeclared.append({ undeclared.append(
"emulator": profile.get("emulator", emu_name), {
"name": fname, "emulator": profile.get("emulator", emu_name),
"path": dest, "name": fname,
"required": f.get("required", False), "path": dest,
"hle_fallback": f.get("hle_fallback", False), "required": f.get("required", False),
"category": f.get("category", "bios"), "hle_fallback": f.get("hle_fallback", False),
"in_repo": in_repo, "category": f.get("category", "bios"),
"note": f.get("note", ""), "in_repo": in_repo,
"checks": sorted(checks) if checks else [], "note": f.get("note", ""),
"source_ref": f.get("source_ref"), "checks": sorted(checks) if checks else [],
"expected": _build_expected(f, checks), "source_ref": f.get("source_ref"),
}) "expected": _build_expected(f, checks),
}
)
# Append grouped archive entries # Append grouped archive entries
for entry in sorted(archive_entries.values(), key=lambda e: e["name"]): for entry in sorted(archive_entries.values(), key=lambda e: e["name"]):
@@ -399,7 +460,9 @@ def find_undeclared_files(
def find_exclusion_notes( def find_exclusion_notes(
config: dict, emulators_dir: str, emu_profiles: dict | None = None, config: dict,
emulators_dir: str,
emu_profiles: dict | None = None,
target_cores: set[str] | None = None, target_cores: set[str] | None = None,
) -> list[dict]: ) -> list[dict]:
"""Document why certain emulator files are intentionally excluded. """Document why certain emulator files are intentionally excluded.
@@ -410,7 +473,11 @@ def find_exclusion_notes(
- Frozen snapshots with files: [] (code doesn't load .info firmware) - Frozen snapshots with files: [] (code doesn't load .info firmware)
- Files covered by data_directories - Files covered by data_directories
""" """
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir) profiles = (
emu_profiles
if emu_profiles is not None
else load_emulator_profiles(emulators_dir)
)
platform_systems = set() platform_systems = set()
for sys_id in config.get("systems", {}): for sys_id in config.get("systems", {}):
platform_systems.add(sys_id) platform_systems.add(sys_id)
@@ -427,19 +494,27 @@ def find_exclusion_notes(
# Launcher excluded entirely # Launcher excluded entirely
if profile.get("type") == "launcher": if profile.get("type") == "launcher":
notes.append({ notes.append(
"emulator": emu_display, "reason": "launcher", {
"detail": profile.get("exclusion_note", "BIOS managed by standalone emulator"), "emulator": emu_display,
}) "reason": "launcher",
"detail": profile.get(
"exclusion_note", "BIOS managed by standalone emulator"
),
}
)
continue continue
# Profile-level exclusion note (frozen snapshots, etc.) # Profile-level exclusion note (frozen snapshots, etc.)
exclusion_note = profile.get("exclusion_note") exclusion_note = profile.get("exclusion_note")
if exclusion_note: if exclusion_note:
notes.append({ notes.append(
"emulator": emu_display, "reason": "exclusion_note", {
"detail": exclusion_note, "emulator": emu_display,
}) "reason": "exclusion_note",
"detail": exclusion_note,
}
)
continue continue
# Count standalone-only files -but only report as excluded if the # Count standalone-only files -but only report as excluded if the
@@ -449,22 +524,34 @@ def find_exclusion_notes(
standalone_set & {str(c) for c in profile.get("cores", [])} standalone_set & {str(c) for c in profile.get("cores", [])}
) )
if not is_standalone: if not is_standalone:
standalone_files = [f for f in profile.get("files", []) if f.get("mode") == "standalone"] standalone_files = [
f for f in profile.get("files", []) if f.get("mode") == "standalone"
]
if standalone_files: if standalone_files:
names = [f["name"] for f in standalone_files[:3]] names = [f["name"] for f in standalone_files[:3]]
more = f" +{len(standalone_files)-3}" if len(standalone_files) > 3 else "" more = (
notes.append({ f" +{len(standalone_files) - 3}"
"emulator": emu_display, "reason": "standalone_only", if len(standalone_files) > 3
"detail": f"{len(standalone_files)} files for standalone mode only ({', '.join(names)}{more})", else ""
}) )
notes.append(
{
"emulator": emu_display,
"reason": "standalone_only",
"detail": f"{len(standalone_files)} files for standalone mode only ({', '.join(names)}{more})",
}
)
return notes return notes
# Platform verification # Platform verification
def _find_best_variant( def _find_best_variant(
file_entry: dict, db: dict, current_path: str, file_entry: dict,
db: dict,
current_path: str,
validation_index: dict, validation_index: dict,
) -> str | None: ) -> str | None:
"""Search for a repo file that passes both platform MD5 and emulator validation.""" """Search for a repo file that passes both platform MD5 and emulator validation."""
@@ -473,7 +560,11 @@ def _find_best_variant(
return None return None
md5_expected = file_entry.get("md5", "") md5_expected = file_entry.get("md5", "")
md5_set = {m.strip().lower() for m in md5_expected.split(",") if m.strip()} if md5_expected else set() md5_set = (
{m.strip().lower() for m in md5_expected.split(",") if m.strip()}
if md5_expected
else set()
)
by_name = db.get("indexes", {}).get("by_name", {}) by_name = db.get("indexes", {}).get("by_name", {})
files_db = db.get("files", {}) files_db = db.get("files", {})
@@ -481,7 +572,11 @@ def _find_best_variant(
for sha1 in by_name.get(fname, []): for sha1 in by_name.get(fname, []):
candidate = files_db.get(sha1, {}) candidate = files_db.get(sha1, {})
path = candidate.get("path", "") path = candidate.get("path", "")
if not path or not os.path.exists(path) or os.path.realpath(path) == os.path.realpath(current_path): if (
not path
or not os.path.exists(path)
or os.path.realpath(path) == os.path.realpath(current_path)
):
continue continue
if md5_set and candidate.get("md5", "").lower() not in md5_set: if md5_set and candidate.get("md5", "").lower() not in md5_set:
continue continue
@@ -492,7 +587,8 @@ def _find_best_variant(
def verify_platform( def verify_platform(
config: dict, db: dict, config: dict,
db: dict,
emulators_dir: str = DEFAULT_EMULATORS_DIR, emulators_dir: str = DEFAULT_EMULATORS_DIR,
emu_profiles: dict | None = None, emu_profiles: dict | None = None,
target_cores: set[str] | None = None, target_cores: set[str] | None = None,
@@ -511,7 +607,11 @@ def verify_platform(
zip_contents = build_zip_contents_index(db) if has_zipped else {} zip_contents = build_zip_contents_index(db) if has_zipped else {}
# Build HLE + validation indexes from emulator profiles # Build HLE + validation indexes from emulator profiles
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir) profiles = (
emu_profiles
if emu_profiles is not None
else load_emulator_profiles(emulators_dir)
)
hle_index: dict[str, bool] = {} hle_index: dict[str, bool] = {}
for profile in profiles.values(): for profile in profiles.values():
for f in profile.get("files", []): for f in profile.get("files", []):
@@ -522,7 +622,9 @@ def verify_platform(
# Filter systems by target # Filter systems by target
plat_cores = resolve_platform_cores(config, profiles) if target_cores else None plat_cores = resolve_platform_cores(config, profiles) if target_cores else None
verify_systems = filter_systems_by_target( verify_systems = filter_systems_by_target(
config.get("systems", {}), profiles, target_cores, config.get("systems", {}),
profiles,
target_cores,
platform_cores=plat_cores, platform_cores=plat_cores,
) )
@@ -536,12 +638,16 @@ def verify_platform(
for sys_id, system in verify_systems.items(): for sys_id, system in verify_systems.items():
for file_entry in system.get("files", []): for file_entry in system.get("files", []):
local_path, resolve_status = resolve_local_file( local_path, resolve_status = resolve_local_file(
file_entry, db, zip_contents, file_entry,
db,
zip_contents,
data_dir_registry=data_dir_registry, data_dir_registry=data_dir_registry,
) )
if mode == "existence": if mode == "existence":
result = verify_entry_existence( result = verify_entry_existence(
file_entry, local_path, validation_index, file_entry,
local_path,
validation_index,
) )
elif mode == "sha1": elif mode == "sha1":
result = verify_entry_sha1(file_entry, local_path) result = verify_entry_sha1(file_entry, local_path)
@@ -555,16 +661,22 @@ def verify_platform(
reason = check_file_validation(local_path, fname, validation_index) reason = check_file_validation(local_path, fname, validation_index)
if reason: if reason:
better = _find_best_variant( better = _find_best_variant(
file_entry, db, local_path, validation_index, file_entry,
db,
local_path,
validation_index,
) )
if not better: if not better:
ventry = validation_index.get(fname, {}) ventry = validation_index.get(fname, {})
emus = ", ".join(ventry.get("emulators", [])) emus = ", ".join(ventry.get("emulators", []))
result["discrepancy"] = f"{platform} says OK but {emus} says {reason}" result["discrepancy"] = (
f"{platform} says OK but {emus} says {reason}"
)
result["system"] = sys_id result["system"] = sys_id
result["hle_fallback"] = hle_index.get(file_entry.get("name", ""), False) result["hle_fallback"] = hle_index.get(file_entry.get("name", ""), False)
result["ground_truth"] = build_ground_truth( result["ground_truth"] = build_ground_truth(
file_entry.get("name", ""), validation_index, file_entry.get("name", ""),
validation_index,
) )
details.append(result) details.append(result)
@@ -581,11 +693,18 @@ def verify_platform(
hle = hle_index.get(file_entry.get("name", ""), False) hle = hle_index.get(file_entry.get("name", ""), False)
sev = compute_severity(cur, required, mode, hle) sev = compute_severity(cur, required, mode, hle)
prev_sev = file_severity.get(dest) prev_sev = file_severity.get(dest)
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(prev_sev, 0): if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(
prev_sev, 0
):
file_severity[dest] = sev file_severity[dest] = sev
# Count by severity # Count by severity
counts = {Severity.OK: 0, Severity.INFO: 0, Severity.WARNING: 0, Severity.CRITICAL: 0} counts = {
Severity.OK: 0,
Severity.INFO: 0,
Severity.WARNING: 0,
Severity.CRITICAL: 0,
}
for s in file_severity.values(): for s in file_severity.values():
counts[s] = counts.get(s, 0) + 1 counts[s] = counts.get(s, 0) + 1
@@ -597,10 +716,19 @@ def verify_platform(
# Cross-reference undeclared files # Cross-reference undeclared files
if supplemental_names is None: if supplemental_names is None:
from cross_reference import _build_supplemental_index from cross_reference import _build_supplemental_index
supplemental_names = _build_supplemental_index() supplemental_names = _build_supplemental_index()
undeclared = find_undeclared_files(config, emulators_dir, db, emu_profiles, undeclared = find_undeclared_files(
target_cores=target_cores, data_names=supplemental_names) config,
exclusions = find_exclusion_notes(config, emulators_dir, emu_profiles, target_cores=target_cores) emulators_dir,
db,
emu_profiles,
target_cores=target_cores,
data_names=supplemental_names,
)
exclusions = find_exclusion_notes(
config, emulators_dir, emu_profiles, target_cores=target_cores
)
# Ground truth coverage # Ground truth coverage
gt_filenames = set(validation_index) gt_filenames = set(validation_index)
@@ -635,6 +763,7 @@ def verify_platform(
# Output # Output
def _format_ground_truth_aggregate(ground_truth: list[dict]) -> str: def _format_ground_truth_aggregate(ground_truth: list[dict]) -> str:
"""Format ground truth as a single aggregated line. """Format ground truth as a single aggregated line.
@@ -759,8 +888,16 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
bios_files = [u for u in undeclared if u.get("category", "bios") == "bios"] bios_files = [u for u in undeclared if u.get("category", "bios") == "bios"]
game_data = [u for u in undeclared if u.get("category", "bios") == "game_data"] game_data = [u for u in undeclared if u.get("category", "bios") == "game_data"]
req_not_in_repo = [u for u in bios_files if u["required"] and not u["in_repo"] and not u.get("hle_fallback")] req_not_in_repo = [
req_hle_not_in_repo = [u for u in bios_files if u["required"] and not u["in_repo"] and u.get("hle_fallback")] u
for u in bios_files
if u["required"] and not u["in_repo"] and not u.get("hle_fallback")
]
req_hle_not_in_repo = [
u
for u in bios_files
if u["required"] and not u["in_repo"] and u.get("hle_fallback")
]
req_in_repo = [u for u in bios_files if u["required"] and u["in_repo"]] req_in_repo = [u for u in bios_files if u["required"] and u["in_repo"]]
opt_in_repo = [u for u in bios_files if not u["required"] and u["in_repo"]] opt_in_repo = [u for u in bios_files if not u["required"] and u["in_repo"]]
opt_not_in_repo = [u for u in bios_files if not u["required"] and not u["in_repo"]] opt_not_in_repo = [u for u in bios_files if not u["required"] and not u["in_repo"]]
@@ -769,7 +906,9 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
core_missing_req = len(req_not_in_repo) + len(req_hle_not_in_repo) core_missing_req = len(req_not_in_repo) + len(req_hle_not_in_repo)
core_missing_opt = len(opt_not_in_repo) core_missing_opt = len(opt_not_in_repo)
print(f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, {core_missing_opt} optional missing") print(
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, {core_missing_opt} optional missing"
)
for u in req_not_in_repo: for u in req_not_in_repo:
_print_undeclared_entry(u, "MISSING (required)", verbose) _print_undeclared_entry(u, "MISSING (required)", verbose)
@@ -783,7 +922,9 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
print(f" Game data: {len(gd_present)} in pack, {len(gd_missing)} missing") print(f" Game data: {len(gd_present)} in pack, {len(gd_missing)} missing")
def print_platform_result(result: dict, group: list[str], verbose: bool = False) -> None: def print_platform_result(
result: dict, group: list[str], verbose: bool = False
) -> None:
mode = result["verification_mode"] mode = result["verification_mode"]
total = result["total_files"] total = result["total_files"]
c = result["severity_counts"] c = result["severity_counts"]
@@ -827,13 +968,16 @@ def print_platform_result(result: dict, group: list[str], verbose: bool = False)
gt_cov = result.get("ground_truth_coverage") gt_cov = result.get("ground_truth_coverage")
if gt_cov and gt_cov["total"] > 0: if gt_cov and gt_cov["total"] > 0:
pct = gt_cov["with_validation"] * 100 // gt_cov["total"] pct = gt_cov["with_validation"] * 100 // gt_cov["total"]
print(f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)") print(
f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)"
)
if gt_cov["platform_only"]: if gt_cov["platform_only"]:
print(f" {gt_cov['platform_only']} platform-only (no emulator profile)") print(f" {gt_cov['platform_only']} platform-only (no emulator profile)")
# Emulator/system mode verification # Emulator/system mode verification
def _effective_validation_label(details: list[dict], validation_index: dict) -> str: def _effective_validation_label(details: list[dict], validation_index: dict) -> str:
"""Determine the bracket label for the report. """Determine the bracket label for the report.
@@ -863,7 +1007,7 @@ def verify_emulator(
standalone: bool = False, standalone: bool = False,
) -> dict: ) -> dict:
"""Verify files for specific emulator profiles.""" """Verify files for specific emulator profiles."""
profiles = load_emulator_profiles(emulators_dir) load_emulator_profiles(emulators_dir)
zip_contents = build_zip_contents_index(db) zip_contents = build_zip_contents_index(db)
# Also load aliases for redirect messages # Also load aliases for redirect messages
@@ -873,26 +1017,35 @@ def verify_emulator(
selected: list[tuple[str, dict]] = [] selected: list[tuple[str, dict]] = []
for name in profile_names: for name in profile_names:
if name not in all_profiles: if name not in all_profiles:
available = sorted(k for k, v in all_profiles.items() available = sorted(
if v.get("type") not in ("alias", "test")) k
for k, v in all_profiles.items()
if v.get("type") not in ("alias", "test")
)
print(f"Error: emulator '{name}' not found", file=sys.stderr) print(f"Error: emulator '{name}' not found", file=sys.stderr)
print(f"Available: {', '.join(available[:10])}...", file=sys.stderr) print(f"Available: {', '.join(available[:10])}...", file=sys.stderr)
sys.exit(1) sys.exit(1)
p = all_profiles[name] p = all_profiles[name]
if p.get("type") == "alias": if p.get("type") == "alias":
alias_of = p.get("alias_of", "?") alias_of = p.get("alias_of", "?")
print(f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}", print(
file=sys.stderr) f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}",
file=sys.stderr,
)
sys.exit(1) sys.exit(1)
if p.get("type") == "launcher": if p.get("type") == "launcher":
print(f"Error: {name} is a launcher -use the emulator it launches", print(
file=sys.stderr) f"Error: {name} is a launcher -use the emulator it launches",
file=sys.stderr,
)
sys.exit(1) sys.exit(1)
# Check standalone capability # Check standalone capability
ptype = p.get("type", "libretro") ptype = p.get("type", "libretro")
if standalone and "standalone" not in ptype: if standalone and "standalone" not in ptype:
print(f"Error: {name} ({ptype}) does not support --standalone", print(
file=sys.stderr) f"Error: {name} ({ptype}) does not support --standalone",
file=sys.stderr,
)
sys.exit(1) sys.exit(1)
selected.append((name, p)) selected.append((name, p))
@@ -924,12 +1077,16 @@ def verify_emulator(
data_dir_notices.append(ref) data_dir_notices.append(ref)
if not files: if not files:
details.append({ details.append(
"name": f"({emu_name})", "status": Status.OK, {
"required": False, "system": "", "name": f"({emu_name})",
"note": f"No files needed for {profile.get('emulator', emu_name)}", "status": Status.OK,
"ground_truth": [], "required": False,
}) "system": "",
"note": f"No files needed for {profile.get('emulator', emu_name)}",
"ground_truth": [],
}
)
continue continue
# Verify archives as units (e.g., neogeo.zip, aes.zip) # Verify archives as units (e.g., neogeo.zip, aes.zip)
@@ -940,7 +1097,9 @@ def verify_emulator(
seen_archives.add(archive) seen_archives.add(archive)
archive_entry = {"name": archive} archive_entry = {"name": archive}
local_path, _ = resolve_local_file( local_path, _ = resolve_local_file(
archive_entry, db, zip_contents, archive_entry,
db,
zip_contents,
data_dir_registry=data_registry, data_dir_registry=data_registry,
) )
required = any( required = any(
@@ -948,11 +1107,18 @@ def verify_emulator(
for f in files for f in files
) )
if local_path: if local_path:
result = {"name": archive, "status": Status.OK, result = {
"required": required, "path": local_path} "name": archive,
"status": Status.OK,
"required": required,
"path": local_path,
}
else: else:
result = {"name": archive, "status": Status.MISSING, result = {
"required": required} "name": archive,
"status": Status.MISSING,
"required": required,
}
result["system"] = file_entry.get("system", "") result["system"] = file_entry.get("system", "")
result["hle_fallback"] = False result["hle_fallback"] = False
result["ground_truth"] = build_ground_truth(archive, validation_index) result["ground_truth"] = build_ground_truth(archive, validation_index)
@@ -961,11 +1127,15 @@ def verify_emulator(
dest_to_name[dest] = archive dest_to_name[dest] = archive
cur = result["status"] cur = result["status"]
prev = file_status.get(dest) prev = file_status.get(dest)
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0): if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(
prev, 0
):
file_status[dest] = cur file_status[dest] = cur
sev = compute_severity(cur, required, "existence", False) sev = compute_severity(cur, required, "existence", False)
prev_sev = file_severity.get(dest) prev_sev = file_severity.get(dest)
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(prev_sev, 0): if prev_sev is None or _SEVERITY_ORDER.get(
sev, 0
) > _SEVERITY_ORDER.get(prev_sev, 0):
file_severity[dest] = sev file_severity[dest] = sev
for file_entry in files: for file_entry in files:
@@ -975,7 +1145,10 @@ def verify_emulator(
dest_hint = file_entry.get("path", "") dest_hint = file_entry.get("path", "")
local_path, resolve_status = resolve_local_file( local_path, resolve_status = resolve_local_file(
file_entry, db, zip_contents, dest_hint=dest_hint, file_entry,
db,
zip_contents,
dest_hint=dest_hint,
data_dir_registry=data_registry, data_dir_registry=data_registry,
) )
name = file_entry.get("name", "") name = file_entry.get("name", "")
@@ -988,12 +1161,20 @@ def verify_emulator(
# Apply emulator validation # Apply emulator validation
reason = check_file_validation(local_path, name, validation_index) reason = check_file_validation(local_path, name, validation_index)
if reason: if reason:
result = {"name": name, "status": Status.UNTESTED, result = {
"required": required, "path": local_path, "name": name,
"reason": reason} "status": Status.UNTESTED,
"required": required,
"path": local_path,
"reason": reason,
}
else: else:
result = {"name": name, "status": Status.OK, result = {
"required": required, "path": local_path} "name": name,
"status": Status.OK,
"required": required,
"path": local_path,
}
result["system"] = file_entry.get("system", "") result["system"] = file_entry.get("system", "")
result["hle_fallback"] = hle result["hle_fallback"] = hle
@@ -1009,10 +1190,17 @@ def verify_emulator(
file_status[dest] = cur file_status[dest] = cur
sev = compute_severity(cur, required, "existence", hle) sev = compute_severity(cur, required, "existence", hle)
prev_sev = file_severity.get(dest) prev_sev = file_severity.get(dest)
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(prev_sev, 0): if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(
prev_sev, 0
):
file_severity[dest] = sev file_severity[dest] = sev
counts = {Severity.OK: 0, Severity.INFO: 0, Severity.WARNING: 0, Severity.CRITICAL: 0} counts = {
Severity.OK: 0,
Severity.INFO: 0,
Severity.WARNING: 0,
Severity.CRITICAL: 0,
}
for s in file_severity.values(): for s in file_severity.values():
counts[s] = counts.get(s, 0) + 1 counts[s] = counts.get(s, 0) + 1
status_counts: dict[str, int] = {} status_counts: dict[str, int] = {}
@@ -1067,13 +1255,19 @@ def verify_system(
for p in profiles.values(): for p in profiles.values():
all_systems.update(p.get("systems", [])) all_systems.update(p.get("systems", []))
if standalone: if standalone:
print(f"No standalone emulators found for system(s): {', '.join(system_ids)}", print(
file=sys.stderr) f"No standalone emulators found for system(s): {', '.join(system_ids)}",
file=sys.stderr,
)
else: else:
print(f"No emulators found for system(s): {', '.join(system_ids)}", print(
file=sys.stderr) f"No emulators found for system(s): {', '.join(system_ids)}",
print(f"Available systems: {', '.join(sorted(all_systems)[:20])}...", file=sys.stderr,
file=sys.stderr) )
print(
f"Available systems: {', '.join(sorted(all_systems)[:20])}...",
file=sys.stderr,
)
sys.exit(1) sys.exit(1)
return verify_emulator(matching, emulators_dir, db, standalone) return verify_emulator(matching, emulators_dir, db, standalone)
@@ -1147,13 +1341,17 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
print(f" {line}") print(f" {line}")
for ref in result.get("data_dir_notices", []): for ref in result.get("data_dir_notices", []):
print(f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)") print(
f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)"
)
# Ground truth coverage footer # Ground truth coverage footer
gt_cov = result.get("ground_truth_coverage") gt_cov = result.get("ground_truth_coverage")
if gt_cov and gt_cov["total"] > 0: if gt_cov and gt_cov["total"] > 0:
pct = gt_cov["with_validation"] * 100 // gt_cov["total"] pct = gt_cov["with_validation"] * 100 // gt_cov["total"]
print(f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)") print(
f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)"
)
if gt_cov["platform_only"]: if gt_cov["platform_only"]:
print(f" {gt_cov['platform_only']} platform-only (no emulator profile)") print(f" {gt_cov['platform_only']} platform-only (no emulator profile)")
@@ -1161,19 +1359,36 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
def main(): def main():
parser = argparse.ArgumentParser(description="Platform-native BIOS verification") parser = argparse.ArgumentParser(description="Platform-native BIOS verification")
parser.add_argument("--platform", "-p", help="Platform name") parser.add_argument("--platform", "-p", help="Platform name")
parser.add_argument("--all", action="store_true", help="Verify all active platforms") parser.add_argument(
parser.add_argument("--emulator", "-e", help="Emulator profile name(s), comma-separated") "--all", action="store_true", help="Verify all active platforms"
)
parser.add_argument(
"--emulator", "-e", help="Emulator profile name(s), comma-separated"
)
parser.add_argument("--system", "-s", help="System ID(s), comma-separated") parser.add_argument("--system", "-s", help="System ID(s), comma-separated")
parser.add_argument("--standalone", action="store_true", help="Use standalone mode") parser.add_argument("--standalone", action="store_true", help="Use standalone mode")
parser.add_argument("--list-emulators", action="store_true", help="List available emulators") parser.add_argument(
parser.add_argument("--list-systems", action="store_true", help="List available systems") "--list-emulators", action="store_true", help="List available emulators"
)
parser.add_argument(
"--list-systems", action="store_true", help="List available systems"
)
parser.add_argument("--include-archived", action="store_true") parser.add_argument("--include-archived", action="store_true")
parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)") parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)")
parser.add_argument("--list-targets", action="store_true", help="List available targets for the platform") parser.add_argument(
"--list-targets",
action="store_true",
help="List available targets for the platform",
)
parser.add_argument("--db", default=DEFAULT_DB) parser.add_argument("--db", default=DEFAULT_DB)
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR) parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
parser.add_argument("--emulators-dir", default=DEFAULT_EMULATORS_DIR) parser.add_argument("--emulators-dir", default=DEFAULT_EMULATORS_DIR)
parser.add_argument("--verbose", "-v", action="store_true", help="Show emulator ground truth details") parser.add_argument(
"--verbose",
"-v",
action="store_true",
help="Show emulator ground truth details",
)
parser.add_argument("--json", action="store_true", help="JSON output") parser.add_argument("--json", action="store_true", help="JSON output")
args = parser.parse_args() args = parser.parse_args()
@@ -1188,13 +1403,16 @@ def main():
if not args.platform: if not args.platform:
parser.error("--list-targets requires --platform") parser.error("--list-targets requires --platform")
from common import list_available_targets from common import list_available_targets
targets = list_available_targets(args.platform, args.platforms_dir) targets = list_available_targets(args.platform, args.platforms_dir)
if not targets: if not targets:
print(f"No targets configured for platform '{args.platform}'") print(f"No targets configured for platform '{args.platform}'")
return return
for t in targets: for t in targets:
aliases = f" (aliases: {', '.join(t['aliases'])})" if t['aliases'] else "" aliases = f" (aliases: {', '.join(t['aliases'])})" if t["aliases"] else ""
print(f" {t['name']:30s} {t['architecture']:10s} {t['core_count']:>4d} cores{aliases}") print(
f" {t['name']:30s} {t['architecture']:10s} {t['core_count']:>4d} cores{aliases}"
)
return return
# Mutual exclusion # Mutual exclusion
@@ -1202,7 +1420,9 @@ def main():
if modes == 0: if modes == 0:
parser.error("Specify --platform, --all, --emulator, or --system") parser.error("Specify --platform, --all, --emulator, or --system")
if modes > 1: if modes > 1:
parser.error("--platform, --all, --emulator, and --system are mutually exclusive") parser.error(
"--platform, --all, --emulator, and --system are mutually exclusive"
)
if args.standalone and not (args.emulator or args.system): if args.standalone and not (args.emulator or args.system):
parser.error("--standalone requires --emulator or --system") parser.error("--standalone requires --emulator or --system")
if args.target and not (args.platform or args.all): if args.target and not (args.platform or args.all):
@@ -1218,7 +1438,9 @@ def main():
names = [n.strip() for n in args.emulator.split(",") if n.strip()] names = [n.strip() for n in args.emulator.split(",") if n.strip()]
result = verify_emulator(names, args.emulators_dir, db, args.standalone) result = verify_emulator(names, args.emulators_dir, db, args.standalone)
if args.json: if args.json:
result["details"] = [d for d in result["details"] if d["status"] != Status.OK] result["details"] = [
d for d in result["details"] if d["status"] != Status.OK
]
print(json.dumps(result, indent=2)) print(json.dumps(result, indent=2))
else: else:
print_emulator_result(result, verbose=args.verbose) print_emulator_result(result, verbose=args.verbose)
@@ -1229,7 +1451,9 @@ def main():
system_ids = [s.strip() for s in args.system.split(",") if s.strip()] system_ids = [s.strip() for s in args.system.split(",") if s.strip()]
result = verify_system(system_ids, args.emulators_dir, db, args.standalone) result = verify_system(system_ids, args.emulators_dir, db, args.standalone)
if args.json: if args.json:
result["details"] = [d for d in result["details"] if d["status"] != Status.OK] result["details"] = [
d for d in result["details"] if d["status"] != Status.OK
]
print(json.dumps(result, indent=2)) print(json.dumps(result, indent=2))
else: else:
print_emulator_result(result, verbose=args.verbose) print_emulator_result(result, verbose=args.verbose)
@@ -1238,6 +1462,7 @@ def main():
# Platform mode (existing) # Platform mode (existing)
if args.all: if args.all:
from list_platforms import list_platforms as _list_platforms from list_platforms import list_platforms as _list_platforms
platforms = _list_platforms(include_archived=args.include_archived) platforms = _list_platforms(include_archived=args.include_archived)
elif args.platform: elif args.platform:
platforms = [args.platform] platforms = [args.platform]
@@ -1253,16 +1478,21 @@ def main():
if args.target: if args.target:
try: try:
target_cores_cache, platforms = build_target_cores_cache( target_cores_cache, platforms = build_target_cores_cache(
platforms, args.target, args.platforms_dir, is_all=args.all, platforms,
args.target,
args.platforms_dir,
is_all=args.all,
) )
except (FileNotFoundError, ValueError) as e: except (FileNotFoundError, ValueError) as e:
print(f"ERROR: {e}", file=sys.stderr) print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1) sys.exit(1)
# Group identical platforms (same function as generate_pack) # Group identical platforms (same function as generate_pack)
groups = group_identical_platforms(platforms, args.platforms_dir, groups = group_identical_platforms(
target_cores_cache if args.target else None) platforms, args.platforms_dir, target_cores_cache if args.target else None
)
from cross_reference import _build_supplemental_index from cross_reference import _build_supplemental_index
suppl_names = _build_supplemental_index() suppl_names = _build_supplemental_index()
all_results = {} all_results = {}
@@ -1271,11 +1501,18 @@ def main():
config = load_platform_config(representative, args.platforms_dir) config = load_platform_config(representative, args.platforms_dir)
tc = target_cores_cache.get(representative) if args.target else None tc = target_cores_cache.get(representative) if args.target else None
result = verify_platform( result = verify_platform(
config, db, args.emulators_dir, emu_profiles, config,
target_cores=tc, data_dir_registry=data_registry, db,
args.emulators_dir,
emu_profiles,
target_cores=tc,
data_dir_registry=data_registry,
supplemental_names=suppl_names, supplemental_names=suppl_names,
) )
names = [load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms] names = [
load_platform_config(p, args.platforms_dir).get("platform", p)
for p in group_platforms
]
group_results.append((result, names)) group_results.append((result, names))
for p in group_platforms: for p in group_platforms:
all_results[p] = result all_results[p] = result

1601
tests/test_e2e.py
View File

File diff suppressed because it is too large Load Diff

82
tests/test_fbneo_parser.py
View File

@@ -2,7 +2,6 @@
from __future__ import annotations from __future__ import annotations
import os
import tempfile import tempfile
import unittest import unittest
from pathlib import Path from pathlib import Path
@@ -84,87 +83,84 @@ struct BurnDriver BurnDrvmslug = {
class TestFindBiosSets(unittest.TestCase): class TestFindBiosSets(unittest.TestCase):
def test_detects_neogeo(self) -> None: def test_detects_neogeo(self) -> None:
result = find_bios_sets(NEOGEO_FIXTURE, 'd_neogeo.cpp') result = find_bios_sets(NEOGEO_FIXTURE, "d_neogeo.cpp")
self.assertIn('neogeo', result) self.assertIn("neogeo", result)
self.assertEqual(result['neogeo']['source_file'], 'd_neogeo.cpp') self.assertEqual(result["neogeo"]["source_file"], "d_neogeo.cpp")
def test_detects_pgm(self) -> None: def test_detects_pgm(self) -> None:
result = find_bios_sets(PGM_FIXTURE, 'd_pgm.cpp') result = find_bios_sets(PGM_FIXTURE, "d_pgm.cpp")
self.assertIn('pgm', result) self.assertIn("pgm", result)
self.assertEqual(result['pgm']['source_file'], 'd_pgm.cpp') self.assertEqual(result["pgm"]["source_file"], "d_pgm.cpp")
def test_ignores_non_bios(self) -> None: def test_ignores_non_bios(self) -> None:
result = find_bios_sets(NON_BIOS_FIXTURE, 'd_neogeo.cpp') result = find_bios_sets(NON_BIOS_FIXTURE, "d_neogeo.cpp")
self.assertEqual(result, {}) self.assertEqual(result, {})
def test_source_line_positive(self) -> None: def test_source_line_positive(self) -> None:
result = find_bios_sets(NEOGEO_FIXTURE, 'd_neogeo.cpp') result = find_bios_sets(NEOGEO_FIXTURE, "d_neogeo.cpp")
self.assertGreater(result['neogeo']['source_line'], 0) self.assertGreater(result["neogeo"]["source_line"], 0)
class TestParseRomInfo(unittest.TestCase): class TestParseRomInfo(unittest.TestCase):
def test_neogeo_rom_count(self) -> None: def test_neogeo_rom_count(self) -> None:
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
self.assertEqual(len(roms), 5) self.assertEqual(len(roms), 5)
def test_sentinel_skipped(self) -> None: def test_sentinel_skipped(self) -> None:
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
names = [r['name'] for r in roms] names = [r["name"] for r in roms]
self.assertNotIn('', names) self.assertNotIn("", names)
def test_crc32_lowercase_hex(self) -> None: def test_crc32_lowercase_hex(self) -> None:
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
first = roms[0] first = roms[0]
self.assertEqual(first['crc32'], '9036d879') self.assertEqual(first["crc32"], "9036d879")
self.assertRegex(first['crc32'], r'^[0-9a-f]{8}$') self.assertRegex(first["crc32"], r"^[0-9a-f]{8}$")
def test_no_sha1(self) -> None: def test_no_sha1(self) -> None:
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
for rom in roms: for rom in roms:
self.assertNotIn('sha1', rom) self.assertNotIn("sha1", rom)
def test_neogeo_first_rom(self) -> None: def test_neogeo_first_rom(self) -> None:
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
first = roms[0] first = roms[0]
self.assertEqual(first['name'], 'sp-s2.sp1') self.assertEqual(first["name"], "sp-s2.sp1")
self.assertEqual(first['size'], 0x020000) self.assertEqual(first["size"], 0x020000)
self.assertEqual(first['crc32'], '9036d879') self.assertEqual(first["crc32"], "9036d879")
def test_pgm_rom_count(self) -> None: def test_pgm_rom_count(self) -> None:
roms = parse_rom_info(PGM_FIXTURE, 'pgm') roms = parse_rom_info(PGM_FIXTURE, "pgm")
self.assertEqual(len(roms), 3) self.assertEqual(len(roms), 3)
def test_pgm_bios_entry(self) -> None: def test_pgm_bios_entry(self) -> None:
roms = parse_rom_info(PGM_FIXTURE, 'pgm') roms = parse_rom_info(PGM_FIXTURE, "pgm")
bios = roms[2] bios = roms[2]
self.assertEqual(bios['name'], 'pgm_p01s.rom') self.assertEqual(bios["name"], "pgm_p01s.rom")
self.assertEqual(bios['crc32'], 'e42b166e') self.assertEqual(bios["crc32"], "e42b166e")
def test_unknown_set_returns_empty(self) -> None: def test_unknown_set_returns_empty(self) -> None:
roms = parse_rom_info(NEOGEO_FIXTURE, 'nonexistent') roms = parse_rom_info(NEOGEO_FIXTURE, "nonexistent")
self.assertEqual(roms, []) self.assertEqual(roms, [])
class TestParseSourceTree(unittest.TestCase): class TestParseSourceTree(unittest.TestCase):
def test_walks_drv_directory(self) -> None: def test_walks_drv_directory(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
drv_dir = Path(tmpdir) / 'src' / 'burn' / 'drv' / 'neogeo' drv_dir = Path(tmpdir) / "src" / "burn" / "drv" / "neogeo"
drv_dir.mkdir(parents=True) drv_dir.mkdir(parents=True)
(drv_dir / 'd_neogeo.cpp').write_text(NEOGEO_FIXTURE) (drv_dir / "d_neogeo.cpp").write_text(NEOGEO_FIXTURE)
result = parse_fbneo_source_tree(tmpdir) result = parse_fbneo_source_tree(tmpdir)
self.assertIn('neogeo', result) self.assertIn("neogeo", result)
self.assertEqual(len(result['neogeo']['roms']), 5) self.assertEqual(len(result["neogeo"]["roms"]), 5)
def test_skips_non_cpp(self) -> None: def test_skips_non_cpp(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
drv_dir = Path(tmpdir) / 'src' / 'burn' / 'drv' drv_dir = Path(tmpdir) / "src" / "burn" / "drv"
drv_dir.mkdir(parents=True) drv_dir.mkdir(parents=True)
(drv_dir / 'd_neogeo.h').write_text(NEOGEO_FIXTURE) (drv_dir / "d_neogeo.h").write_text(NEOGEO_FIXTURE)
result = parse_fbneo_source_tree(tmpdir) result = parse_fbneo_source_tree(tmpdir)
self.assertEqual(result, {}) self.assertEqual(result, {})
@@ -175,16 +171,16 @@ class TestParseSourceTree(unittest.TestCase):
self.assertEqual(result, {}) self.assertEqual(result, {})
def test_multiple_sets(self) -> None: def test_multiple_sets(self) -> None:
combined = NEOGEO_FIXTURE + '\n' + PGM_FIXTURE combined = NEOGEO_FIXTURE + "\n" + PGM_FIXTURE
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
drv_dir = Path(tmpdir) / 'src' / 'burn' / 'drv' drv_dir = Path(tmpdir) / "src" / "burn" / "drv"
drv_dir.mkdir(parents=True) drv_dir.mkdir(parents=True)
(drv_dir / 'd_combined.cpp').write_text(combined) (drv_dir / "d_combined.cpp").write_text(combined)
result = parse_fbneo_source_tree(tmpdir) result = parse_fbneo_source_tree(tmpdir)
self.assertIn('neogeo', result) self.assertIn("neogeo", result)
self.assertIn('pgm', result) self.assertIn("pgm", result)
if __name__ == '__main__': if __name__ == "__main__":
unittest.main() unittest.main()

356
tests/test_hash_merge.py
View File

@@ -18,35 +18,35 @@ from scripts.scraper._hash_merge import (
def _write_yaml(path: Path, data: dict) -> str: def _write_yaml(path: Path, data: dict) -> str:
p = str(path) p = str(path)
with open(p, 'w', encoding='utf-8') as f: with open(p, "w", encoding="utf-8") as f:
yaml.dump(data, f, default_flow_style=False, sort_keys=False) yaml.dump(data, f, default_flow_style=False, sort_keys=False)
return p return p
def _write_json(path: Path, data: dict) -> str: def _write_json(path: Path, data: dict) -> str:
p = str(path) p = str(path)
with open(p, 'w', encoding='utf-8') as f: with open(p, "w", encoding="utf-8") as f:
json.dump(data, f) json.dump(data, f)
return p return p
def _make_mame_profile(**overrides: object) -> dict: def _make_mame_profile(**overrides: object) -> dict:
base = { base = {
'emulator': 'MAME', "emulator": "MAME",
'core_version': '0.285', "core_version": "0.285",
'files': [ "files": [
{ {
'name': 'neogeo.zip', "name": "neogeo.zip",
'required': True, "required": True,
'category': 'bios_zip', "category": "bios_zip",
'system': 'snk-neogeo-mvs', "system": "snk-neogeo-mvs",
'source_ref': 'src/mame/neogeo/neogeo.cpp:2400', "source_ref": "src/mame/neogeo/neogeo.cpp:2400",
'contents': [ "contents": [
{ {
'name': 'sp-s2.sp1', "name": "sp-s2.sp1",
'size': 131072, "size": 131072,
'crc32': 'oldcrc32', "crc32": "oldcrc32",
'description': 'Europe MVS (Ver. 2)', "description": "Europe MVS (Ver. 2)",
}, },
], ],
}, },
@@ -58,23 +58,23 @@ def _make_mame_profile(**overrides: object) -> dict:
def _make_mame_hashes(**overrides: object) -> dict: def _make_mame_hashes(**overrides: object) -> dict:
base = { base = {
'source': 'mamedev/mame', "source": "mamedev/mame",
'version': '0.286', "version": "0.286",
'commit': 'abc123', "commit": "abc123",
'fetched_at': '2026-03-30T12:00:00Z', "fetched_at": "2026-03-30T12:00:00Z",
'bios_sets': { "bios_sets": {
'neogeo': { "neogeo": {
'source_file': 'src/mame/neogeo/neogeo.cpp', "source_file": "src/mame/neogeo/neogeo.cpp",
'source_line': 2432, "source_line": 2432,
'roms': [ "roms": [
{ {
'name': 'sp-s2.sp1', "name": "sp-s2.sp1",
'size': 131072, "size": 131072,
'crc32': '9036d879', "crc32": "9036d879",
'sha1': '4f834c55', "sha1": "4f834c55",
'region': 'mainbios', "region": "mainbios",
'bios_label': 'euro', "bios_label": "euro",
'bios_description': 'Europe MVS (Ver. 2)', "bios_description": "Europe MVS (Ver. 2)",
}, },
], ],
}, },
@@ -86,21 +86,21 @@ def _make_mame_hashes(**overrides: object) -> dict:
def _make_fbneo_profile(**overrides: object) -> dict: def _make_fbneo_profile(**overrides: object) -> dict:
base = { base = {
'emulator': 'FinalBurn Neo', "emulator": "FinalBurn Neo",
'core_version': 'v1.0.0.02', "core_version": "v1.0.0.02",
'files': [ "files": [
{ {
'name': 'sp-s2.sp1', "name": "sp-s2.sp1",
'archive': 'neogeo.zip', "archive": "neogeo.zip",
'system': 'snk-neogeo-mvs', "system": "snk-neogeo-mvs",
'required': True, "required": True,
'size': 131072, "size": 131072,
'crc32': 'oldcrc32', "crc32": "oldcrc32",
'source_ref': 'src/burn/drv/neogeo/d_neogeo.cpp:1605', "source_ref": "src/burn/drv/neogeo/d_neogeo.cpp:1605",
}, },
{ {
'name': 'hiscore.dat', "name": "hiscore.dat",
'required': False, "required": False,
}, },
], ],
} }
@@ -110,20 +110,20 @@ def _make_fbneo_profile(**overrides: object) -> dict:
def _make_fbneo_hashes(**overrides: object) -> dict: def _make_fbneo_hashes(**overrides: object) -> dict:
base = { base = {
'source': 'finalburnneo/FBNeo', "source": "finalburnneo/FBNeo",
'version': 'v1.0.0.03', "version": "v1.0.0.03",
'commit': 'def456', "commit": "def456",
'fetched_at': '2026-03-30T12:00:00Z', "fetched_at": "2026-03-30T12:00:00Z",
'bios_sets': { "bios_sets": {
'neogeo': { "neogeo": {
'source_file': 'src/burn/drv/neogeo/d_neogeo.cpp', "source_file": "src/burn/drv/neogeo/d_neogeo.cpp",
'source_line': 1604, "source_line": 1604,
'roms': [ "roms": [
{ {
'name': 'sp-s2.sp1', "name": "sp-s2.sp1",
'size': 131072, "size": 131072,
'crc32': '9036d879', "crc32": "9036d879",
'sha1': 'aabbccdd', "sha1": "aabbccdd",
}, },
], ],
}, },
@@ -139,129 +139,129 @@ class TestMameMerge(unittest.TestCase):
def test_merge_updates_contents(self) -> None: def test_merge_updates_contents(self) -> None:
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile()) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
result = merge_mame_profile(profile_path, hashes_path) result = merge_mame_profile(profile_path, hashes_path)
bios_files = [f for f in result['files'] if f.get('category') == 'bios_zip'] bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"]
self.assertEqual(len(bios_files), 1) self.assertEqual(len(bios_files), 1)
contents = bios_files[0]['contents'] contents = bios_files[0]["contents"]
self.assertEqual(contents[0]['crc32'], '9036d879') self.assertEqual(contents[0]["crc32"], "9036d879")
self.assertEqual(contents[0]['sha1'], '4f834c55') self.assertEqual(contents[0]["sha1"], "4f834c55")
self.assertEqual(contents[0]['description'], 'Europe MVS (Ver. 2)') self.assertEqual(contents[0]["description"], "Europe MVS (Ver. 2)")
def test_merge_preserves_manual_fields(self) -> None: def test_merge_preserves_manual_fields(self) -> None:
profile = _make_mame_profile() profile = _make_mame_profile()
profile['files'][0]['note'] = 'manually curated note' profile["files"][0]["note"] = "manually curated note"
profile['files'][0]['system'] = 'snk-neogeo-mvs' profile["files"][0]["system"] = "snk-neogeo-mvs"
profile['files'][0]['required'] = False profile["files"][0]["required"] = False
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', profile) profile_path = _write_yaml(p / "mame.yml", profile)
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
result = merge_mame_profile(profile_path, hashes_path) result = merge_mame_profile(profile_path, hashes_path)
entry = [f for f in result['files'] if f.get('category') == 'bios_zip'][0] entry = [f for f in result["files"] if f.get("category") == "bios_zip"][0]
self.assertEqual(entry['note'], 'manually curated note') self.assertEqual(entry["note"], "manually curated note")
self.assertEqual(entry['system'], 'snk-neogeo-mvs') self.assertEqual(entry["system"], "snk-neogeo-mvs")
self.assertFalse(entry['required']) self.assertFalse(entry["required"])
def test_merge_adds_new_bios_set(self) -> None: def test_merge_adds_new_bios_set(self) -> None:
hashes = _make_mame_hashes() hashes = _make_mame_hashes()
hashes['bios_sets']['pgm'] = { hashes["bios_sets"]["pgm"] = {
'source_file': 'src/mame/igs/pgm.cpp', "source_file": "src/mame/igs/pgm.cpp",
'source_line': 5515, "source_line": 5515,
'roms': [ "roms": [
{'name': 'pgm_t01s.rom', 'size': 2097152, 'crc32': '1a7123a0'}, {"name": "pgm_t01s.rom", "size": 2097152, "crc32": "1a7123a0"},
], ],
} }
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile()) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
hashes_path = _write_json(p / 'hashes.json', hashes) hashes_path = _write_json(p / "hashes.json", hashes)
result = merge_mame_profile(profile_path, hashes_path) result = merge_mame_profile(profile_path, hashes_path)
bios_files = [f for f in result['files'] if f.get('category') == 'bios_zip'] bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"]
names = {f['name'] for f in bios_files} names = {f["name"] for f in bios_files}
self.assertIn('pgm.zip', names) self.assertIn("pgm.zip", names)
pgm = next(f for f in bios_files if f['name'] == 'pgm.zip') pgm = next(f for f in bios_files if f["name"] == "pgm.zip")
self.assertIsNone(pgm['system']) self.assertIsNone(pgm["system"])
self.assertTrue(pgm['required']) self.assertTrue(pgm["required"])
self.assertEqual(pgm['category'], 'bios_zip') self.assertEqual(pgm["category"], "bios_zip")
def test_merge_preserves_non_bios_files(self) -> None: def test_merge_preserves_non_bios_files(self) -> None:
profile = _make_mame_profile() profile = _make_mame_profile()
profile['files'].append({'name': 'hiscore.dat', 'required': False}) profile["files"].append({"name": "hiscore.dat", "required": False})
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', profile) profile_path = _write_yaml(p / "mame.yml", profile)
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
result = merge_mame_profile(profile_path, hashes_path) result = merge_mame_profile(profile_path, hashes_path)
non_bios = [f for f in result['files'] if f.get('category') != 'bios_zip'] non_bios = [f for f in result["files"] if f.get("category") != "bios_zip"]
self.assertEqual(len(non_bios), 1) self.assertEqual(len(non_bios), 1)
self.assertEqual(non_bios[0]['name'], 'hiscore.dat') self.assertEqual(non_bios[0]["name"], "hiscore.dat")
def test_merge_keeps_unmatched_bios_set(self) -> None: def test_merge_keeps_unmatched_bios_set(self) -> None:
"""Entries not in scraper scope stay untouched (no _upstream_removed).""" """Entries not in scraper scope stay untouched (no _upstream_removed)."""
hashes = _make_mame_hashes() hashes = _make_mame_hashes()
hashes['bios_sets'] = {} # nothing from scraper hashes["bios_sets"] = {} # nothing from scraper
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile()) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
hashes_path = _write_json(p / 'hashes.json', hashes) hashes_path = _write_json(p / "hashes.json", hashes)
result = merge_mame_profile(profile_path, hashes_path) result = merge_mame_profile(profile_path, hashes_path)
bios_files = [f for f in result['files'] if f.get('category') == 'bios_zip'] bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"]
self.assertEqual(len(bios_files), 1) self.assertEqual(len(bios_files), 1)
self.assertNotIn('_upstream_removed', bios_files[0]) self.assertNotIn("_upstream_removed", bios_files[0])
self.assertEqual(bios_files[0]['name'], 'neogeo.zip') self.assertEqual(bios_files[0]["name"], "neogeo.zip")
def test_merge_updates_core_version(self) -> None: def test_merge_updates_core_version(self) -> None:
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile()) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
result = merge_mame_profile(profile_path, hashes_path) result = merge_mame_profile(profile_path, hashes_path)
self.assertEqual(result['core_version'], '0.286') self.assertEqual(result["core_version"], "0.286")
def test_merge_backup_created(self) -> None: def test_merge_backup_created(self) -> None:
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile()) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
merge_mame_profile(profile_path, hashes_path, write=True) merge_mame_profile(profile_path, hashes_path, write=True)
backup = p / 'mame.old.yml' backup = p / "mame.old.yml"
self.assertTrue(backup.exists()) self.assertTrue(backup.exists())
with open(backup, encoding='utf-8') as f: with open(backup, encoding="utf-8") as f:
old = yaml.safe_load(f) old = yaml.safe_load(f)
self.assertEqual(old['core_version'], '0.285') self.assertEqual(old["core_version"], "0.285")
def test_merge_updates_source_ref(self) -> None: def test_merge_updates_source_ref(self) -> None:
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile()) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
result = merge_mame_profile(profile_path, hashes_path) result = merge_mame_profile(profile_path, hashes_path)
entry = [f for f in result['files'] if f.get('category') == 'bios_zip'][0] entry = [f for f in result["files"] if f.get("category") == "bios_zip"][0]
self.assertEqual(entry['source_ref'], 'src/mame/neogeo/neogeo.cpp:2432') self.assertEqual(entry["source_ref"], "src/mame/neogeo/neogeo.cpp:2432")
class TestFbneoMerge(unittest.TestCase): class TestFbneoMerge(unittest.TestCase):
@@ -270,74 +270,76 @@ class TestFbneoMerge(unittest.TestCase):
def test_merge_updates_rom_entries(self) -> None: def test_merge_updates_rom_entries(self) -> None:
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile()) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
hashes_path = _write_json(p / 'hashes.json', _make_fbneo_hashes()) hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes())
result = merge_fbneo_profile(profile_path, hashes_path) result = merge_fbneo_profile(profile_path, hashes_path)
archive_files = [f for f in result['files'] if 'archive' in f] archive_files = [f for f in result["files"] if "archive" in f]
self.assertEqual(len(archive_files), 1) self.assertEqual(len(archive_files), 1)
self.assertEqual(archive_files[0]['crc32'], '9036d879') self.assertEqual(archive_files[0]["crc32"], "9036d879")
self.assertEqual(archive_files[0]['system'], 'snk-neogeo-mvs') self.assertEqual(archive_files[0]["system"], "snk-neogeo-mvs")
def test_merge_adds_new_roms(self) -> None: def test_merge_adds_new_roms(self) -> None:
hashes = _make_fbneo_hashes() hashes = _make_fbneo_hashes()
hashes['bios_sets']['neogeo']['roms'].append({ hashes["bios_sets"]["neogeo"]["roms"].append(
'name': 'sp-s3.sp1', {
'size': 131072, "name": "sp-s3.sp1",
'crc32': '91b64be3', "size": 131072,
}) "crc32": "91b64be3",
}
)
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile()) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
hashes_path = _write_json(p / 'hashes.json', hashes) hashes_path = _write_json(p / "hashes.json", hashes)
result = merge_fbneo_profile(profile_path, hashes_path) result = merge_fbneo_profile(profile_path, hashes_path)
archive_files = [f for f in result['files'] if 'archive' in f] archive_files = [f for f in result["files"] if "archive" in f]
self.assertEqual(len(archive_files), 2) self.assertEqual(len(archive_files), 2)
new_rom = next(f for f in archive_files if f['name'] == 'sp-s3.sp1') new_rom = next(f for f in archive_files if f["name"] == "sp-s3.sp1")
self.assertEqual(new_rom['archive'], 'neogeo.zip') self.assertEqual(new_rom["archive"], "neogeo.zip")
self.assertTrue(new_rom['required']) self.assertTrue(new_rom["required"])
def test_merge_preserves_non_archive_files(self) -> None: def test_merge_preserves_non_archive_files(self) -> None:
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile()) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
hashes_path = _write_json(p / 'hashes.json', _make_fbneo_hashes()) hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes())
result = merge_fbneo_profile(profile_path, hashes_path) result = merge_fbneo_profile(profile_path, hashes_path)
non_archive = [f for f in result['files'] if 'archive' not in f] non_archive = [f for f in result["files"] if "archive" not in f]
self.assertEqual(len(non_archive), 1) self.assertEqual(len(non_archive), 1)
self.assertEqual(non_archive[0]['name'], 'hiscore.dat') self.assertEqual(non_archive[0]["name"], "hiscore.dat")
def test_merge_keeps_unmatched_roms(self) -> None: def test_merge_keeps_unmatched_roms(self) -> None:
"""Entries not in scraper scope stay untouched (no _upstream_removed).""" """Entries not in scraper scope stay untouched (no _upstream_removed)."""
hashes = _make_fbneo_hashes() hashes = _make_fbneo_hashes()
hashes['bios_sets'] = {} hashes["bios_sets"] = {}
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile()) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
hashes_path = _write_json(p / 'hashes.json', hashes) hashes_path = _write_json(p / "hashes.json", hashes)
result = merge_fbneo_profile(profile_path, hashes_path) result = merge_fbneo_profile(profile_path, hashes_path)
archive_files = [f for f in result['files'] if 'archive' in f] archive_files = [f for f in result["files"] if "archive" in f]
self.assertEqual(len(archive_files), 1) self.assertEqual(len(archive_files), 1)
self.assertNotIn('_upstream_removed', archive_files[0]) self.assertNotIn("_upstream_removed", archive_files[0])
def test_merge_updates_core_version(self) -> None: def test_merge_updates_core_version(self) -> None:
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile()) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
hashes_path = _write_json(p / 'hashes.json', _make_fbneo_hashes()) hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes())
result = merge_fbneo_profile(profile_path, hashes_path) result = merge_fbneo_profile(profile_path, hashes_path)
self.assertEqual(result['core_version'], 'v1.0.0.03') self.assertEqual(result["core_version"], "v1.0.0.03")
class TestDiff(unittest.TestCase): class TestDiff(unittest.TestCase):
@@ -345,79 +347,81 @@ class TestDiff(unittest.TestCase):
def test_diff_mame_detects_changes(self) -> None: def test_diff_mame_detects_changes(self) -> None:
hashes = _make_mame_hashes() hashes = _make_mame_hashes()
hashes['bios_sets']['pgm'] = { hashes["bios_sets"]["pgm"] = {
'source_file': 'src/mame/igs/pgm.cpp', "source_file": "src/mame/igs/pgm.cpp",
'source_line': 5515, "source_line": 5515,
'roms': [ "roms": [
{'name': 'pgm_t01s.rom', 'size': 2097152, 'crc32': '1a7123a0'}, {"name": "pgm_t01s.rom", "size": 2097152, "crc32": "1a7123a0"},
], ],
} }
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile()) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
hashes_path = _write_json(p / 'hashes.json', hashes) hashes_path = _write_json(p / "hashes.json", hashes)
diff = compute_diff(profile_path, hashes_path, mode='mame') diff = compute_diff(profile_path, hashes_path, mode="mame")
self.assertIn('pgm', diff['added']) self.assertIn("pgm", diff["added"])
self.assertIn('neogeo', diff['updated']) self.assertIn("neogeo", diff["updated"])
self.assertEqual(len(diff['removed']), 0) self.assertEqual(len(diff["removed"]), 0)
self.assertEqual(diff['unchanged'], 0) self.assertEqual(diff["unchanged"], 0)
def test_diff_mame_out_of_scope(self) -> None: def test_diff_mame_out_of_scope(self) -> None:
"""Items in profile but not in scraper output = out of scope, not removed.""" """Items in profile but not in scraper output = out of scope, not removed."""
hashes = _make_mame_hashes() hashes = _make_mame_hashes()
hashes['bios_sets'] = {} hashes["bios_sets"] = {}
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile()) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
hashes_path = _write_json(p / 'hashes.json', hashes) hashes_path = _write_json(p / "hashes.json", hashes)
diff = compute_diff(profile_path, hashes_path, mode='mame') diff = compute_diff(profile_path, hashes_path, mode="mame")
self.assertEqual(diff['removed'], []) self.assertEqual(diff["removed"], [])
self.assertEqual(diff['out_of_scope'], 1) self.assertEqual(diff["out_of_scope"], 1)
self.assertEqual(len(diff['added']), 0) self.assertEqual(len(diff["added"]), 0)
def test_diff_fbneo_detects_changes(self) -> None: def test_diff_fbneo_detects_changes(self) -> None:
hashes = _make_fbneo_hashes() hashes = _make_fbneo_hashes()
hashes['bios_sets']['neogeo']['roms'].append({ hashes["bios_sets"]["neogeo"]["roms"].append(
'name': 'sp-s3.sp1', {
'size': 131072, "name": "sp-s3.sp1",
'crc32': '91b64be3', "size": 131072,
}) "crc32": "91b64be3",
}
)
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile()) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
hashes_path = _write_json(p / 'hashes.json', hashes) hashes_path = _write_json(p / "hashes.json", hashes)
diff = compute_diff(profile_path, hashes_path, mode='fbneo') diff = compute_diff(profile_path, hashes_path, mode="fbneo")
self.assertIn('neogeo.zip:sp-s3.sp1', diff['added']) self.assertIn("neogeo.zip:sp-s3.sp1", diff["added"])
self.assertIn('neogeo.zip:sp-s2.sp1', diff['updated']) self.assertIn("neogeo.zip:sp-s2.sp1", diff["updated"])
self.assertEqual(len(diff['removed']), 0) self.assertEqual(len(diff["removed"]), 0)
def test_diff_fbneo_unchanged(self) -> None: def test_diff_fbneo_unchanged(self) -> None:
profile = _make_fbneo_profile() profile = _make_fbneo_profile()
profile['files'][0]['crc32'] = '9036d879' profile["files"][0]["crc32"] = "9036d879"
profile['files'][0]['size'] = 131072 profile["files"][0]["size"] = 131072
hashes = _make_fbneo_hashes() hashes = _make_fbneo_hashes()
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
p = Path(td) p = Path(td)
profile_path = _write_yaml(p / 'fbneo.yml', profile) profile_path = _write_yaml(p / "fbneo.yml", profile)
hashes_path = _write_json(p / 'hashes.json', hashes) hashes_path = _write_json(p / "hashes.json", hashes)
diff = compute_diff(profile_path, hashes_path, mode='fbneo') diff = compute_diff(profile_path, hashes_path, mode="fbneo")
self.assertEqual(diff['unchanged'], 1) self.assertEqual(diff["unchanged"], 1)
self.assertEqual(len(diff['added']), 0) self.assertEqual(len(diff["added"]), 0)
self.assertEqual(len(diff['updated']), 0) self.assertEqual(len(diff["updated"]), 0)
if __name__ == '__main__': if __name__ == "__main__":
unittest.main() unittest.main()

142
tests/test_mame_parser.py
View File

@@ -86,101 +86,101 @@ class TestFindBiosRootSets(unittest.TestCase):
"""Tests for find_bios_root_sets.""" """Tests for find_bios_root_sets."""
def test_detects_neogeo_from_game_macro(self) -> None: def test_detects_neogeo_from_game_macro(self) -> None:
result = find_bios_root_sets(NEOGEO_FIXTURE, 'src/mame/snk/neogeo.cpp') result = find_bios_root_sets(NEOGEO_FIXTURE, "src/mame/snk/neogeo.cpp")
self.assertIn('neogeo', result) self.assertIn("neogeo", result)
self.assertEqual(result['neogeo']['source_file'], 'src/mame/snk/neogeo.cpp') self.assertEqual(result["neogeo"]["source_file"], "src/mame/snk/neogeo.cpp")
self.assertIsInstance(result['neogeo']['source_line'], int) self.assertIsInstance(result["neogeo"]["source_line"], int)
def test_detects_from_comp_macro(self) -> None: def test_detects_from_comp_macro(self) -> None:
result = find_bios_root_sets(DEVICE_FIXTURE, 'src/mame/acorn/bbc.cpp') result = find_bios_root_sets(DEVICE_FIXTURE, "src/mame/acorn/bbc.cpp")
self.assertIn('bbcb', result) self.assertIn("bbcb", result)
def test_detects_from_cons_macro(self) -> None: def test_detects_from_cons_macro(self) -> None:
result = find_bios_root_sets(CONS_FIXTURE, 'src/mame/sega/megadriv.cpp') result = find_bios_root_sets(CONS_FIXTURE, "src/mame/sega/megadriv.cpp")
self.assertIn('megadriv', result) self.assertIn("megadriv", result)
def test_ignores_non_bios_games(self) -> None: def test_ignores_non_bios_games(self) -> None:
result = find_bios_root_sets(NON_BIOS_FIXTURE, 'src/mame/pacman/pacman.cpp') result = find_bios_root_sets(NON_BIOS_FIXTURE, "src/mame/pacman/pacman.cpp")
self.assertEqual(result, {}) self.assertEqual(result, {})
def test_detects_from_nodump_fixture(self) -> None: def test_detects_from_nodump_fixture(self) -> None:
result = find_bios_root_sets(NODUMP_FIXTURE, 'test.cpp') result = find_bios_root_sets(NODUMP_FIXTURE, "test.cpp")
self.assertIn('testnd', result) self.assertIn("testnd", result)
def test_detects_from_baddump_fixture(self) -> None: def test_detects_from_baddump_fixture(self) -> None:
result = find_bios_root_sets(BADDUMP_FIXTURE, 'test.cpp') result = find_bios_root_sets(BADDUMP_FIXTURE, "test.cpp")
self.assertIn('testbd', result) self.assertIn("testbd", result)
class TestParseRomBlock(unittest.TestCase): class TestParseRomBlock(unittest.TestCase):
"""Tests for parse_rom_block.""" """Tests for parse_rom_block."""
def test_extracts_rom_names(self) -> None: def test_extracts_rom_names(self) -> None:
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
names = [r['name'] for r in roms] names = [r["name"] for r in roms]
self.assertIn('sp-s2.sp1', names) self.assertIn("sp-s2.sp1", names)
self.assertIn('vs-bios.rom', names) self.assertIn("vs-bios.rom", names)
self.assertIn('sm1.sm1', names) self.assertIn("sm1.sm1", names)
def test_extracts_crc32_and_sha1(self) -> None: def test_extracts_crc32_and_sha1(self) -> None:
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
sp_s2 = next(r for r in roms if r['name'] == 'sp-s2.sp1') sp_s2 = next(r for r in roms if r["name"] == "sp-s2.sp1")
self.assertEqual(sp_s2['crc32'], '9036d879') self.assertEqual(sp_s2["crc32"], "9036d879")
self.assertEqual(sp_s2['sha1'], '4f5ed7105b7128794654ce82b51723e16e389543') self.assertEqual(sp_s2["sha1"], "4f5ed7105b7128794654ce82b51723e16e389543")
def test_extracts_size(self) -> None: def test_extracts_size(self) -> None:
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
sp_s2 = next(r for r in roms if r['name'] == 'sp-s2.sp1') sp_s2 = next(r for r in roms if r["name"] == "sp-s2.sp1")
self.assertEqual(sp_s2['size'], 0x020000) self.assertEqual(sp_s2["size"], 0x020000)
def test_extracts_bios_metadata(self) -> None: def test_extracts_bios_metadata(self) -> None:
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
sp_s2 = next(r for r in roms if r['name'] == 'sp-s2.sp1') sp_s2 = next(r for r in roms if r["name"] == "sp-s2.sp1")
self.assertEqual(sp_s2['bios_index'], 0) self.assertEqual(sp_s2["bios_index"], 0)
self.assertEqual(sp_s2['bios_label'], 'euro') self.assertEqual(sp_s2["bios_label"], "euro")
self.assertEqual(sp_s2['bios_description'], 'Europe MVS (Ver. 2)') self.assertEqual(sp_s2["bios_description"], "Europe MVS (Ver. 2)")
def test_non_bios_rom_has_no_bios_fields(self) -> None: def test_non_bios_rom_has_no_bios_fields(self) -> None:
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
sm1 = next(r for r in roms if r['name'] == 'sm1.sm1') sm1 = next(r for r in roms if r["name"] == "sm1.sm1")
self.assertNotIn('bios_index', sm1) self.assertNotIn("bios_index", sm1)
self.assertNotIn('bios_label', sm1) self.assertNotIn("bios_label", sm1)
def test_skips_no_dump(self) -> None: def test_skips_no_dump(self) -> None:
roms = parse_rom_block(NODUMP_FIXTURE, 'testnd') roms = parse_rom_block(NODUMP_FIXTURE, "testnd")
names = [r['name'] for r in roms] names = [r["name"] for r in roms]
self.assertIn('good.rom', names) self.assertIn("good.rom", names)
self.assertNotIn('missing.rom', names) self.assertNotIn("missing.rom", names)
def test_includes_bad_dump_with_flag(self) -> None: def test_includes_bad_dump_with_flag(self) -> None:
roms = parse_rom_block(BADDUMP_FIXTURE, 'testbd') roms = parse_rom_block(BADDUMP_FIXTURE, "testbd")
self.assertEqual(len(roms), 1) self.assertEqual(len(roms), 1)
self.assertEqual(roms[0]['name'], 'badrom.bin') self.assertEqual(roms[0]["name"], "badrom.bin")
self.assertTrue(roms[0]['bad_dump']) self.assertTrue(roms[0]["bad_dump"])
self.assertEqual(roms[0]['crc32'], 'deadbeef') self.assertEqual(roms[0]["crc32"], "deadbeef")
self.assertEqual(roms[0]['sha1'], '0123456789abcdef0123456789abcdef01234567') self.assertEqual(roms[0]["sha1"], "0123456789abcdef0123456789abcdef01234567")
def test_handles_rom_load16_word(self) -> None: def test_handles_rom_load16_word(self) -> None:
roms = parse_rom_block(CONS_FIXTURE, 'megadriv') roms = parse_rom_block(CONS_FIXTURE, "megadriv")
self.assertEqual(len(roms), 1) self.assertEqual(len(roms), 1)
self.assertEqual(roms[0]['name'], 'epr-6209.ic7') self.assertEqual(roms[0]["name"], "epr-6209.ic7")
self.assertEqual(roms[0]['crc32'], 'cafebabe') self.assertEqual(roms[0]["crc32"], "cafebabe")
def test_tracks_rom_region(self) -> None: def test_tracks_rom_region(self) -> None:
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo') roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
sp_s2 = next(r for r in roms if r['name'] == 'sp-s2.sp1') sp_s2 = next(r for r in roms if r["name"] == "sp-s2.sp1")
sm1 = next(r for r in roms if r['name'] == 'sm1.sm1') sm1 = next(r for r in roms if r["name"] == "sm1.sm1")
self.assertEqual(sp_s2['region'], 'mainbios') self.assertEqual(sp_s2["region"], "mainbios")
self.assertEqual(sm1['region'], 'audiocpu') self.assertEqual(sm1["region"], "audiocpu")
def test_returns_empty_for_unknown_set(self) -> None: def test_returns_empty_for_unknown_set(self) -> None:
roms = parse_rom_block(NEOGEO_FIXTURE, 'nonexistent') roms = parse_rom_block(NEOGEO_FIXTURE, "nonexistent")
self.assertEqual(roms, []) self.assertEqual(roms, [])
def test_good_rom_not_flagged_bad_dump(self) -> None: def test_good_rom_not_flagged_bad_dump(self) -> None:
roms = parse_rom_block(NODUMP_FIXTURE, 'testnd') roms = parse_rom_block(NODUMP_FIXTURE, "testnd")
good = next(r for r in roms if r['name'] == 'good.rom') good = next(r for r in roms if r["name"] == "good.rom")
self.assertFalse(good['bad_dump']) self.assertFalse(good["bad_dump"])
def test_crc32_sha1_lowercase(self) -> None: def test_crc32_sha1_lowercase(self) -> None:
fixture = """\ fixture = """\
@@ -189,9 +189,9 @@ ROM_START( upper )
ROM_LOAD( "test.rom", 0x00000, 0x4000, CRC(AABBCCDD) SHA1(AABBCCDDEEFF00112233AABBCCDDEEFF00112233) ) ROM_LOAD( "test.rom", 0x00000, 0x4000, CRC(AABBCCDD) SHA1(AABBCCDDEEFF00112233AABBCCDDEEFF00112233) )
ROM_END ROM_END
""" """
roms = parse_rom_block(fixture, 'upper') roms = parse_rom_block(fixture, "upper")
self.assertEqual(roms[0]['crc32'], 'aabbccdd') self.assertEqual(roms[0]["crc32"], "aabbccdd")
self.assertEqual(roms[0]['sha1'], 'aabbccddeeff00112233aabbccddeeff00112233') self.assertEqual(roms[0]["sha1"], "aabbccddeeff00112233aabbccddeeff00112233")
class TestParseMameSourceTree(unittest.TestCase): class TestParseMameSourceTree(unittest.TestCase):
@@ -199,26 +199,26 @@ class TestParseMameSourceTree(unittest.TestCase):
def test_walks_source_tree(self) -> None: def test_walks_source_tree(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
mame_dir = os.path.join(tmpdir, 'src', 'mame', 'snk') mame_dir = os.path.join(tmpdir, "src", "mame", "snk")
os.makedirs(mame_dir) os.makedirs(mame_dir)
filepath = os.path.join(mame_dir, 'neogeo.cpp') filepath = os.path.join(mame_dir, "neogeo.cpp")
with open(filepath, 'w') as f: with open(filepath, "w") as f:
f.write(NEOGEO_FIXTURE) f.write(NEOGEO_FIXTURE)
results = parse_mame_source_tree(tmpdir) results = parse_mame_source_tree(tmpdir)
self.assertIn('neogeo', results) self.assertIn("neogeo", results)
self.assertEqual(len(results['neogeo']['roms']), 3) self.assertEqual(len(results["neogeo"]["roms"]), 3)
self.assertEqual( self.assertEqual(
results['neogeo']['source_file'], results["neogeo"]["source_file"],
'src/mame/snk/neogeo.cpp', "src/mame/snk/neogeo.cpp",
) )
def test_ignores_non_source_files(self) -> None: def test_ignores_non_source_files(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
mame_dir = os.path.join(tmpdir, 'src', 'mame') mame_dir = os.path.join(tmpdir, "src", "mame")
os.makedirs(mame_dir) os.makedirs(mame_dir)
# Write a .txt file that should be ignored # Write a .txt file that should be ignored
with open(os.path.join(mame_dir, 'notes.txt'), 'w') as f: with open(os.path.join(mame_dir, "notes.txt"), "w") as f:
f.write(NEOGEO_FIXTURE) f.write(NEOGEO_FIXTURE)
results = parse_mame_source_tree(tmpdir) results = parse_mame_source_tree(tmpdir)
@@ -226,13 +226,13 @@ class TestParseMameSourceTree(unittest.TestCase):
def test_scans_devices_dir(self) -> None: def test_scans_devices_dir(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
dev_dir = os.path.join(tmpdir, 'src', 'devices', 'bus') dev_dir = os.path.join(tmpdir, "src", "devices", "bus")
os.makedirs(dev_dir) os.makedirs(dev_dir)
with open(os.path.join(dev_dir, 'test.cpp'), 'w') as f: with open(os.path.join(dev_dir, "test.cpp"), "w") as f:
f.write(DEVICE_FIXTURE) f.write(DEVICE_FIXTURE)
results = parse_mame_source_tree(tmpdir) results = parse_mame_source_tree(tmpdir)
self.assertIn('bbcb', results) self.assertIn("bbcb", results)
def test_empty_tree(self) -> None: def test_empty_tree(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
@@ -240,5 +240,5 @@ class TestParseMameSourceTree(unittest.TestCase):
self.assertEqual(results, {}) self.assertEqual(results, {})
if __name__ == '__main__': if __name__ == "__main__":
unittest.main() unittest.main()

20
tests/test_pack_integrity.py
View File

@@ -25,11 +25,11 @@ def _platform_has_pack(platform_name: str) -> bool:
return False return False
sys.path.insert(0, os.path.join(REPO_ROOT, "scripts")) sys.path.insert(0, os.path.join(REPO_ROOT, "scripts"))
from common import load_platform_config from common import load_platform_config
config = load_platform_config(platform_name, PLATFORMS_DIR) config = load_platform_config(platform_name, PLATFORMS_DIR)
display = config.get("platform", platform_name).replace(" ", "_") display = config.get("platform", platform_name).replace(" ", "_")
return any( return any(
f.endswith("_BIOS_Pack.zip") and display in f f.endswith("_BIOS_Pack.zip") and display in f for f in os.listdir(DIST_DIR)
for f in os.listdir(DIST_DIR)
) )
@@ -40,10 +40,18 @@ class PackIntegrityTest(unittest.TestCase):
if not _platform_has_pack(platform_name): if not _platform_has_pack(platform_name):
self.skipTest(f"no pack found for {platform_name}") self.skipTest(f"no pack found for {platform_name}")
result = subprocess.run( result = subprocess.run(
[sys.executable, "scripts/generate_pack.py", [
"--platform", platform_name, sys.executable,
"--verify-packs", "--output-dir", "dist/"], "scripts/generate_pack.py",
capture_output=True, text=True, cwd=REPO_ROOT, "--platform",
platform_name,
"--verify-packs",
"--output-dir",
"dist/",
],
capture_output=True,
text=True,
cwd=REPO_ROOT,
) )
if result.returncode != 0: if result.returncode != 0:
self.fail( self.fail(