kmoody/libretro
1
0
Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-04-13 12:22:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: lint and format entire codebase

Browse Source 
Run ruff check --fix: remove unused imports (F401), fix f-strings
without placeholders (F541), remove unused variables (F841), fix
duplicate dict key (F601).

Run isort --profile black: normalize import ordering across all files.

Run ruff format: apply consistent formatting (black-compatible) to
all 58 Python files.

3 intentional E402 remain (imports after require_yaml() must execute
after yaml is available).
This commit is contained in:
Abdessamad Derraz
2026-04-01 13:17:55 +02:00
parent a2d30557e4
commit 0a272dc4e9
56 changed files with 5115 additions and 2679 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
scripts/scraper/mame_hash_scraper.py
View File

@@ -21,16 +21,16 @@ from typing import Any
import yaml
from .mame_parser import parse_mame_source_tree
from ._hash_merge import compute_diff, merge_mame_profile
from .mame_parser import parse_mame_source_tree
log = logging.getLogger(__name__)
_ROOT = Path(__file__).resolve().parents[2]
_CACHE_PATH = _ROOT / 'data' / 'mame-hashes.json'
_CLONE_DIR = _ROOT / 'tmp' / 'mame'
_EMULATORS_DIR = _ROOT / 'emulators'
_REPO_URL = 'https://github.com/mamedev/mame.git'
_CACHE_PATH = _ROOT / "data" / "mame-hashes.json"
_CLONE_DIR = _ROOT / "tmp" / "mame"
_EMULATORS_DIR = _ROOT / "emulators"
_REPO_URL = "https://github.com/mamedev/mame.git"
_STALE_HOURS = 24
@@ -41,7 +41,7 @@ def _load_cache() -> dict[str, Any] | None:
if not _CACHE_PATH.exists():
return None
try:
with open(_CACHE_PATH, encoding='utf-8') as f:
with open(_CACHE_PATH, encoding="utf-8") as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
return None
@@ -50,7 +50,7 @@ def _load_cache() -> dict[str, Any] | None:
def _is_stale(cache: dict[str, Any] | None) -> bool:
if cache is None:
return True
fetched_at = cache.get('fetched_at')
fetched_at = cache.get("fetched_at")
if not fetched_at:
return True
try:
@@ -63,17 +63,19 @@ def _is_stale(cache: dict[str, Any] | None) -> bool:
def _write_cache(data: dict[str, Any]) -> None:
_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
with open(_CACHE_PATH, 'w', encoding='utf-8') as f:
with open(_CACHE_PATH, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
log.info('cache written to %s', _CACHE_PATH)
log.info("cache written to %s", _CACHE_PATH)
# ── Git operations ───────────────────────────────────────────────────
def _run_git(args: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
def _run_git(
args: list[str], cwd: Path | None = None
) -> subprocess.CompletedProcess[str]:
return subprocess.run(
['git', *args],
["git", *args],
cwd=cwd,
check=True,
capture_output=True,
@@ -86,17 +88,20 @@ def _sparse_clone() -> None:
shutil.rmtree(_CLONE_DIR)
_CLONE_DIR.parent.mkdir(parents=True, exist_ok=True)
log.info('sparse cloning mamedev/mame into %s', _CLONE_DIR)
_run_git([
'clone',
'--depth', '1',
'--filter=blob:none',
'--sparse',
_REPO_URL,
str(_CLONE_DIR),
])
log.info("sparse cloning mamedev/mame into %s", _CLONE_DIR)
_run_git(
['sparse-checkout', 'set', 'src/mame', 'src/devices'],
[
"clone",
"--depth",
"1",
"--filter=blob:none",
"--sparse",
_REPO_URL,
str(_CLONE_DIR),
]
)
_run_git(
["sparse-checkout", "set", "src/mame", "src/devices"],
cwd=_CLONE_DIR,
)
@@ -106,41 +111,41 @@ def _get_version() -> str:
# Use GitHub API to get the latest release tag.
try:
req = urllib.request.Request(
'https://api.github.com/repos/mamedev/mame/releases/latest',
headers={'User-Agent': 'retrobios-scraper/1.0',
'Accept': 'application/vnd.github.v3+json'},
"https://api.github.com/repos/mamedev/mame/releases/latest",
headers={
"User-Agent": "retrobios-scraper/1.0",
"Accept": "application/vnd.github.v3+json",
},
)
with urllib.request.urlopen(req, timeout=10) as resp:
data = json.loads(resp.read())
tag = data.get('tag_name', '')
tag = data.get("tag_name", "")
if tag:
return _parse_version_tag(tag)
except (urllib.error.URLError, json.JSONDecodeError, OSError):
pass
return 'unknown'
return "unknown"
def _parse_version_tag(tag: str) -> str:
prefix = 'mame'
prefix = "mame"
raw = tag.removeprefix(prefix) if tag.startswith(prefix) else tag
if raw.isdigit() and len(raw) >= 4:
return f'{raw[0]}.{raw[1:]}'
return f"{raw[0]}.{raw[1:]}"
return raw
def _get_commit() -> str:
try:
result = _run_git(['rev-parse', 'HEAD'], cwd=_CLONE_DIR)
result = _run_git(["rev-parse", "HEAD"], cwd=_CLONE_DIR)
return result.stdout.strip()
except subprocess.CalledProcessError:
return ''
return ""
def _cleanup() -> None:
if _CLONE_DIR.exists():
log.info('cleaning up %s', _CLONE_DIR)
log.info("cleaning up %s", _CLONE_DIR)
shutil.rmtree(_CLONE_DIR)
@@ -149,18 +154,21 @@ def _cleanup() -> None:
def _find_mame_profiles() -> list[Path]:
profiles: list[Path] = []
for path in sorted(_EMULATORS_DIR.glob('*.yml')):
if path.name.endswith('.old.yml'):
for path in sorted(_EMULATORS_DIR.glob("*.yml")):
if path.name.endswith(".old.yml"):
continue
try:
with open(path, encoding='utf-8') as f:
with open(path, encoding="utf-8") as f:
data = yaml.safe_load(f)
if not isinstance(data, dict):
continue
upstream = data.get('upstream', '')
upstream = data.get("upstream", "")
# Only match profiles tracking current MAME (not frozen snapshots
# which have upstream like "mamedev/mame/tree/mame0139")
if isinstance(upstream, str) and upstream.rstrip('/') == 'https://github.com/mamedev/mame':
if (
isinstance(upstream, str)
and upstream.rstrip("/") == "https://github.com/mamedev/mame"
):
profiles.append(path)
except (yaml.YAMLError, OSError):
continue
@@ -179,36 +187,36 @@ def _format_diff(
lines: list[str] = []
name = profile_path.stem
added = diff.get('added', [])
updated = diff.get('updated', [])
removed = diff.get('removed', [])
unchanged = diff.get('unchanged', 0)
added = diff.get("added", [])
updated = diff.get("updated", [])
removed = diff.get("removed", [])
unchanged = diff.get("unchanged", 0)
if not added and not updated and not removed:
lines.append(f' {name}:')
lines.append(' no changes')
lines.append(f" {name}:")
lines.append(" no changes")
return lines
lines.append(f' {name}:')
lines.append(f" {name}:")
if show_added:
bios_sets = hashes.get('bios_sets', {})
bios_sets = hashes.get("bios_sets", {})
for set_name in added:
rom_count = len(bios_sets.get(set_name, {}).get('roms', []))
source_file = bios_sets.get(set_name, {}).get('source_file', '')
source_line = bios_sets.get(set_name, {}).get('source_line', '')
ref = f'{source_file}:{source_line}' if source_file else ''
lines.append(f' + {set_name}.zip ({ref}, {rom_count} ROMs)')
rom_count = len(bios_sets.get(set_name, {}).get("roms", []))
source_file = bios_sets.get(set_name, {}).get("source_file", "")
source_line = bios_sets.get(set_name, {}).get("source_line", "")
ref = f"{source_file}:{source_line}" if source_file else ""
lines.append(f" + {set_name}.zip ({ref}, {rom_count} ROMs)")
elif added:
lines.append(f' + {len(added)} new sets available (main profile only)')
lines.append(f" + {len(added)} new sets available (main profile only)")
for set_name in updated:
lines.append(f' ~ {set_name}.zip (contents changed)')
lines.append(f" ~ {set_name}.zip (contents changed)")
oos = diff.get('out_of_scope', 0)
lines.append(f' = {unchanged} unchanged')
oos = diff.get("out_of_scope", 0)
lines.append(f" = {unchanged} unchanged")
if oos:
lines.append(f' . {oos} out of scope (not BIOS root sets)')
lines.append(f" . {oos} out of scope (not BIOS root sets)")
return lines
@@ -218,7 +226,7 @@ def _format_diff(
def _fetch_hashes(force: bool) -> dict[str, Any]:
cache = _load_cache()
if not force and not _is_stale(cache):
log.info('using cached data from %s', cache.get('fetched_at', ''))
log.info("using cached data from %s", cache.get("fetched_at", ""))
return cache # type: ignore[return-value]
try:
@@ -228,11 +236,11 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
commit = _get_commit()
data: dict[str, Any] = {
'source': 'mamedev/mame',
'version': version,
'commit': commit,
'fetched_at': datetime.now(timezone.utc).isoformat(),
'bios_sets': bios_sets,
"source": "mamedev/mame",
"version": version,
"commit": commit,
"fetched_at": datetime.now(timezone.utc).isoformat(),
"bios_sets": bios_sets,
}
_write_cache(data)
return data
@@ -243,34 +251,36 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
def _run(args: argparse.Namespace) -> None:
hashes = _fetch_hashes(args.force)
total_sets = len(hashes.get('bios_sets', {}))
version = hashes.get('version', 'unknown')
commit = hashes.get('commit', '')[:12]
total_sets = len(hashes.get("bios_sets", {}))
version = hashes.get("version", "unknown")
commit = hashes.get("commit", "")[:12]
if args.json:
json.dump(hashes, sys.stdout, indent=2, ensure_ascii=False)
sys.stdout.write('\n')
sys.stdout.write("\n")
return
print(f'mame-hashes: {total_sets} BIOS root sets from mamedev/mame'
f' @ {version} ({commit})')
print(
f"mame-hashes: {total_sets} BIOS root sets from mamedev/mame"
f" @ {version} ({commit})"
)
print()
profiles = _find_mame_profiles()
if not profiles:
print(' no profiles with mamedev/mame upstream found')
print(" no profiles with mamedev/mame upstream found")
return
for profile_path in profiles:
is_main = profile_path.name == 'mame.yml'
diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode='mame')
is_main = profile_path.name == "mame.yml"
diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode="mame")
lines = _format_diff(profile_path, diff, hashes, show_added=is_main)
for line in lines:
print(line)
if not args.dry_run:
updated = diff.get('updated', [])
added = diff.get('added', []) if is_main else []
updated = diff.get("updated", [])
added = diff.get("added", []) if is_main else []
if added or updated:
merge_mame_profile(
str(profile_path),
@@ -278,32 +288,32 @@ def _run(args: argparse.Namespace) -> None:
write=True,
add_new=is_main,
)
log.info('merged into %s', profile_path.name)
log.info("merged into %s", profile_path.name)
print()
if args.dry_run:
print('(dry run, no files modified)')
print("(dry run, no files modified)")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog='mame_hash_scraper',
description='Fetch MAME BIOS hashes from source and merge into profiles.',
prog="mame_hash_scraper",
description="Fetch MAME BIOS hashes from source and merge into profiles.",
)
parser.add_argument(
'--dry-run',
action='store_true',
help='show diff only, do not modify profiles',
"--dry-run",
action="store_true",
help="show diff only, do not modify profiles",
)
parser.add_argument(
'--json',
action='store_true',
help='output raw JSON to stdout',
"--json",
action="store_true",
help="output raw JSON to stdout",
)
parser.add_argument(
'--force',
action='store_true',
help='re-fetch even if cache is fresh',
"--force",
action="store_true",
help="re-fetch even if cache is fresh",
)
return parser
@@ -311,12 +321,12 @@ def build_parser() -> argparse.ArgumentParser:
def main() -> None:
logging.basicConfig(
level=logging.INFO,
format='%(levelname)s: %(message)s',
format="%(levelname)s: %(message)s",
)
parser = build_parser()
args = parser.parse_args()
_run(args)
if __name__ == '__main__':
if __name__ == "__main__":
main()