mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-04-13 12:22:33 -05:00
chore: lint and format entire codebase
Run ruff check --fix: remove unused imports (F401), fix f-strings without placeholders (F541), remove unused variables (F841), fix duplicate dict key (F601). Run isort --profile black: normalize import ordering across all files. Run ruff format: apply consistent formatting (black-compatible) to all 58 Python files. 3 intentional E402 remain (imports after require_yaml() must execute after yaml is available).
This commit is contained in:
Abdessamad Derraz
@@ -13,22 +13,22 @@ import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from scripts.scraper.fbneo_parser import parse_fbneo_source_tree
|
||||
from scripts.scraper._hash_merge import compute_diff, merge_fbneo_profile
|
||||
from scripts.scraper.fbneo_parser import parse_fbneo_source_tree
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
REPO_URL = 'https://github.com/finalburnneo/FBNeo.git'
|
||||
REPO_URL = "https://github.com/finalburnneo/FBNeo.git"
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
CLONE_DIR = REPO_ROOT / 'tmp' / 'fbneo'
|
||||
CACHE_PATH = REPO_ROOT / 'data' / 'fbneo-hashes.json'
|
||||
EMULATORS_DIR = REPO_ROOT / 'emulators'
|
||||
CLONE_DIR = REPO_ROOT / "tmp" / "fbneo"
|
||||
CACHE_PATH = REPO_ROOT / "data" / "fbneo-hashes.json"
|
||||
EMULATORS_DIR = REPO_ROOT / "emulators"
|
||||
STALE_HOURS = 24
|
||||
|
||||
|
||||
@@ -37,8 +37,8 @@ def _is_cache_fresh() -> bool:
|
||||
if not CACHE_PATH.exists():
|
||||
return False
|
||||
try:
|
||||
data = json.loads(CACHE_PATH.read_text(encoding='utf-8'))
|
||||
fetched_at = datetime.fromisoformat(data['fetched_at'])
|
||||
data = json.loads(CACHE_PATH.read_text(encoding="utf-8"))
|
||||
fetched_at = datetime.fromisoformat(data["fetched_at"])
|
||||
return datetime.now(timezone.utc) - fetched_at < timedelta(hours=STALE_HOURS)
|
||||
except (json.JSONDecodeError, KeyError, ValueError):
|
||||
return False
|
||||
@@ -53,8 +53,14 @@ def _sparse_clone() -> None:
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'git', 'clone', '--depth', '1', '--filter=blob:none',
|
||||
'--sparse', REPO_URL, str(CLONE_DIR),
|
||||
"git",
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"--filter=blob:none",
|
||||
"--sparse",
|
||||
REPO_URL,
|
||||
str(CLONE_DIR),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
@@ -62,7 +68,7 @@ def _sparse_clone() -> None:
|
||||
)
|
||||
|
||||
subprocess.run(
|
||||
['git', 'sparse-checkout', 'set', 'src/burn/drv', 'src/burner/resource.h'],
|
||||
["git", "sparse-checkout", "set", "src/burn/drv", "src/burner/resource.h"],
|
||||
cwd=CLONE_DIR,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
@@ -76,42 +82,44 @@ def _extract_version() -> tuple[str, str]:
|
||||
Returns (version, commit_sha). Falls back to resource.h if no tag.
|
||||
"""
|
||||
result = subprocess.run(
|
||||
['git', 'describe', '--tags', '--abbrev=0'],
|
||||
["git", "describe", "--tags", "--abbrev=0"],
|
||||
cwd=CLONE_DIR,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
# Prefer real version tags over pseudo-tags like "latest"
|
||||
version = 'unknown'
|
||||
version = "unknown"
|
||||
if result.returncode == 0:
|
||||
tag = result.stdout.strip()
|
||||
if tag and tag != 'latest':
|
||||
if tag and tag != "latest":
|
||||
version = tag
|
||||
# Fallback: resource.h
|
||||
if version == 'unknown':
|
||||
if version == "unknown":
|
||||
version = _version_from_resource_h()
|
||||
# Last resort: use GitHub API for latest real release tag
|
||||
if version == 'unknown':
|
||||
if version == "unknown":
|
||||
try:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
req = urllib.request.Request(
|
||||
'https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10',
|
||||
headers={'User-Agent': 'retrobios-scraper/1.0'},
|
||||
"https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10",
|
||||
headers={"User-Agent": "retrobios-scraper/1.0"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
import json as json_mod
|
||||
|
||||
tags = json_mod.loads(resp.read())
|
||||
for t in tags:
|
||||
if t['name'] != 'latest' and t['name'].startswith('v'):
|
||||
version = t['name']
|
||||
if t["name"] != "latest" and t["name"].startswith("v"):
|
||||
version = t["name"]
|
||||
break
|
||||
except (urllib.error.URLError, OSError):
|
||||
pass
|
||||
|
||||
sha_result = subprocess.run(
|
||||
['git', 'rev-parse', 'HEAD'],
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=CLONE_DIR,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
@@ -124,17 +132,17 @@ def _extract_version() -> tuple[str, str]:
|
||||
|
||||
def _version_from_resource_h() -> str:
|
||||
"""Fallback: parse VER_FULL_VERSION_STR from resource.h."""
|
||||
resource_h = CLONE_DIR / 'src' / 'burner' / 'resource.h'
|
||||
resource_h = CLONE_DIR / "src" / "burner" / "resource.h"
|
||||
if not resource_h.exists():
|
||||
return 'unknown'
|
||||
return "unknown"
|
||||
|
||||
text = resource_h.read_text(encoding='utf-8', errors='replace')
|
||||
text = resource_h.read_text(encoding="utf-8", errors="replace")
|
||||
for line in text.splitlines():
|
||||
if 'VER_FULL_VERSION_STR' in line:
|
||||
if "VER_FULL_VERSION_STR" in line:
|
||||
parts = line.split('"')
|
||||
if len(parts) >= 2:
|
||||
return parts[1]
|
||||
return 'unknown'
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _cleanup() -> None:
|
||||
@@ -146,33 +154,33 @@ def _cleanup() -> None:
|
||||
def fetch_and_cache(force: bool = False) -> dict[str, Any]:
|
||||
"""Clone, parse, and write JSON cache. Returns the cache dict."""
|
||||
if not force and _is_cache_fresh():
|
||||
log.info('cache fresh, skipping clone (use --force to override)')
|
||||
return json.loads(CACHE_PATH.read_text(encoding='utf-8'))
|
||||
log.info("cache fresh, skipping clone (use --force to override)")
|
||||
return json.loads(CACHE_PATH.read_text(encoding="utf-8"))
|
||||
|
||||
try:
|
||||
log.info('sparse cloning %s', REPO_URL)
|
||||
log.info("sparse cloning %s", REPO_URL)
|
||||
_sparse_clone()
|
||||
|
||||
log.info('extracting version')
|
||||
log.info("extracting version")
|
||||
version, commit = _extract_version()
|
||||
|
||||
log.info('parsing source tree')
|
||||
log.info("parsing source tree")
|
||||
bios_sets = parse_fbneo_source_tree(str(CLONE_DIR))
|
||||
|
||||
cache: dict[str, Any] = {
|
||||
'source': 'finalburnneo/FBNeo',
|
||||
'version': version,
|
||||
'commit': commit,
|
||||
'fetched_at': datetime.now(timezone.utc).isoformat(),
|
||||
'bios_sets': bios_sets,
|
||||
"source": "finalburnneo/FBNeo",
|
||||
"version": version,
|
||||
"commit": commit,
|
||||
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
||||
"bios_sets": bios_sets,
|
||||
}
|
||||
|
||||
CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
CACHE_PATH.write_text(
|
||||
json.dumps(cache, indent=2, ensure_ascii=False) + '\n',
|
||||
encoding='utf-8',
|
||||
json.dumps(cache, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
log.info('wrote %d BIOS sets to %s', len(bios_sets), CACHE_PATH)
|
||||
log.info("wrote %d BIOS sets to %s", len(bios_sets), CACHE_PATH)
|
||||
|
||||
return cache
|
||||
finally:
|
||||
@@ -182,48 +190,50 @@ def fetch_and_cache(force: bool = False) -> dict[str, Any]:
|
||||
def _find_fbneo_profiles() -> list[Path]:
|
||||
"""Find emulator profiles whose upstream references finalburnneo/FBNeo."""
|
||||
profiles: list[Path] = []
|
||||
for path in sorted(EMULATORS_DIR.glob('*.yml')):
|
||||
if path.name.endswith('.old.yml'):
|
||||
for path in sorted(EMULATORS_DIR.glob("*.yml")):
|
||||
if path.name.endswith(".old.yml"):
|
||||
continue
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding='utf-8'))
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except (yaml.YAMLError, OSError):
|
||||
continue
|
||||
if not data or not isinstance(data, dict):
|
||||
continue
|
||||
upstream = data.get('upstream', '')
|
||||
if isinstance(upstream, str) and 'finalburnneo/fbneo' in upstream.lower():
|
||||
upstream = data.get("upstream", "")
|
||||
if isinstance(upstream, str) and "finalburnneo/fbneo" in upstream.lower():
|
||||
profiles.append(path)
|
||||
return profiles
|
||||
|
||||
|
||||
def _format_diff(profile_name: str, diff: dict[str, Any], show_added: bool = True) -> str:
|
||||
def _format_diff(
|
||||
profile_name: str, diff: dict[str, Any], show_added: bool = True
|
||||
) -> str:
|
||||
"""Format diff for a single profile."""
|
||||
lines: list[str] = []
|
||||
lines.append(f' {profile_name}:')
|
||||
lines.append(f" {profile_name}:")
|
||||
|
||||
added = diff.get('added', [])
|
||||
updated = diff.get('updated', [])
|
||||
oos = diff.get('out_of_scope', 0)
|
||||
added = diff.get("added", [])
|
||||
updated = diff.get("updated", [])
|
||||
oos = diff.get("out_of_scope", 0)
|
||||
|
||||
if not added and not updated:
|
||||
lines.append(' no changes')
|
||||
lines.append(" no changes")
|
||||
if oos:
|
||||
lines.append(f' . {oos} out of scope')
|
||||
return '\n'.join(lines)
|
||||
lines.append(f" . {oos} out of scope")
|
||||
return "\n".join(lines)
|
||||
|
||||
if show_added:
|
||||
for label in added:
|
||||
lines.append(f' + {label}')
|
||||
lines.append(f" + {label}")
|
||||
elif added:
|
||||
lines.append(f' + {len(added)} new ROMs available (main profile only)')
|
||||
lines.append(f" + {len(added)} new ROMs available (main profile only)")
|
||||
for label in updated:
|
||||
lines.append(f' ~ {label}')
|
||||
lines.append(f' = {diff["unchanged"]} unchanged')
|
||||
lines.append(f" ~ {label}")
|
||||
lines.append(f" = {diff['unchanged']} unchanged")
|
||||
if oos:
|
||||
lines.append(f' . {oos} out of scope')
|
||||
lines.append(f" . {oos} out of scope")
|
||||
|
||||
return '\n'.join(lines)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def run(
|
||||
@@ -234,82 +244,84 @@ def run(
|
||||
"""Main entry point for the scraper."""
|
||||
cache = fetch_and_cache(force=force)
|
||||
|
||||
version = cache.get('version', 'unknown')
|
||||
commit = cache.get('commit', '?')[:12]
|
||||
bios_sets = cache.get('bios_sets', {})
|
||||
version = cache.get("version", "unknown")
|
||||
commit = cache.get("commit", "?")[:12]
|
||||
bios_sets = cache.get("bios_sets", {})
|
||||
profiles = _find_fbneo_profiles()
|
||||
|
||||
if json_output:
|
||||
result: dict[str, Any] = {
|
||||
'source': cache.get('source'),
|
||||
'version': version,
|
||||
'commit': cache.get('commit'),
|
||||
'bios_set_count': len(bios_sets),
|
||||
'profiles': {},
|
||||
"source": cache.get("source"),
|
||||
"version": version,
|
||||
"commit": cache.get("commit"),
|
||||
"bios_set_count": len(bios_sets),
|
||||
"profiles": {},
|
||||
}
|
||||
for path in profiles:
|
||||
diff = compute_diff(str(path), str(CACHE_PATH), mode='fbneo')
|
||||
result['profiles'][path.stem] = diff
|
||||
diff = compute_diff(str(path), str(CACHE_PATH), mode="fbneo")
|
||||
result["profiles"][path.stem] = diff
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
header = (
|
||||
f'fbneo-hashes: {len(bios_sets)} BIOS sets '
|
||||
f'from finalburnneo/FBNeo @ {version} ({commit})'
|
||||
f"fbneo-hashes: {len(bios_sets)} BIOS sets "
|
||||
f"from finalburnneo/FBNeo @ {version} ({commit})"
|
||||
)
|
||||
print(header)
|
||||
print()
|
||||
|
||||
if not profiles:
|
||||
print(' no matching emulator profiles found')
|
||||
print(" no matching emulator profiles found")
|
||||
return 0
|
||||
|
||||
for path in profiles:
|
||||
is_main = path.name == 'fbneo.yml'
|
||||
diff = compute_diff(str(path), str(CACHE_PATH), mode='fbneo')
|
||||
is_main = path.name == "fbneo.yml"
|
||||
diff = compute_diff(str(path), str(CACHE_PATH), mode="fbneo")
|
||||
print(_format_diff(path.stem, diff, show_added=is_main))
|
||||
|
||||
effective_added = diff['added'] if is_main else []
|
||||
if not dry_run and (effective_added or diff['updated']):
|
||||
effective_added = diff["added"] if is_main else []
|
||||
if not dry_run and (effective_added or diff["updated"]):
|
||||
merge_fbneo_profile(str(path), str(CACHE_PATH), write=True, add_new=is_main)
|
||||
log.info('merged changes into %s', path.name)
|
||||
log.info("merged changes into %s", path.name)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Scrape FBNeo BIOS set hashes from upstream source',
|
||||
description="Scrape FBNeo BIOS set hashes from upstream source",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='show diff without writing changes',
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="show diff without writing changes",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--force',
|
||||
action='store_true',
|
||||
help='force re-clone even if cache is fresh',
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="force re-clone even if cache is fresh",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--json',
|
||||
action='store_true',
|
||||
dest='json_output',
|
||||
help='output diff as JSON',
|
||||
"--json",
|
||||
action="store_true",
|
||||
dest="json_output",
|
||||
help="output diff as JSON",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(name)s: %(message)s',
|
||||
format="%(name)s: %(message)s",
|
||||
)
|
||||
|
||||
sys.exit(run(
|
||||
dry_run=args.dry_run,
|
||||
force=args.force,
|
||||
json_output=args.json_output,
|
||||
))
|
||||
sys.exit(
|
||||
run(
|
||||
dry_run=args.dry_run,
|
||||
force=args.force,
|
||||
json_output=args.json_output,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user