mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-04-13 12:22:33 -05:00
0a272dc4e9
Run ruff check --fix: remove unused imports (F401), fix f-strings without placeholders (F541), remove unused variables (F841), fix duplicate dict key (F601). Run isort --profile black: normalize import ordering across all files. Run ruff format: apply consistent formatting (black-compatible) to all 58 Python files. 3 intentional E402 remain (imports after require_yaml() must execute after yaml is available).
208 lines
7.4 KiB
Python
208 lines
7.4 KiB
Python
"""Scraper for RetroArch buildbot nightly targets.
|
|
|
|
Source: https://buildbot.libretro.com/nightly/
|
|
Fetches directory listings per target to determine available cores.
|
|
|
|
Buildbot structure varies by platform:
|
|
- linux: {path}/latest/ -> *_libretro.so.zip
|
|
- windows: {path}/latest/ -> *_libretro.dll.zip
|
|
- apple/osx: {path}/latest/ -> *_libretro.dylib.zip
|
|
- android: android/latest/{arch}/ -> *_libretro_android.so.zip
|
|
- switch: nintendo/switch/libnx/latest/ -> *_libretro_libnx.nro.zip
|
|
- 3ds: nintendo/3ds/latest/3dsx/ -> *_libretro.3dsx.zip
|
|
- wii/ngc: {path}/latest/ -> *_libretro_{plat}.dol.zip
|
|
- wiiu: nintendo/wiiu/latest/ -> *_libretro.rpx.zip
|
|
- psp: playstation/psp/latest/ -> *_libretro_psp.PBP.zip
|
|
- ps2: playstation/ps2/latest/ -> *_libretro_ps2.elf.zip
|
|
- vita: bundles only (VPK) - no individual cores
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import re
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
|
|
import yaml
|
|
|
|
from . import BaseTargetScraper
|
|
|
|
PLATFORM_NAME = "retroarch"
|
|
|
|
BUILDBOT_URL = "https://buildbot.libretro.com/nightly/"
|
|
|
|
# (url_path_under_nightly, target_name, architecture)
|
|
# url_path must end at the directory containing core files
|
|
TARGETS: list[tuple[str, str, str]] = [
|
|
("linux/x86_64/latest", "linux-x86_64", "x86_64"),
|
|
("linux/armhf/latest", "linux-armhf", "armhf"),
|
|
("linux/armv7-neon-hf/latest", "linux-armv7-neon-hf", "armv7"),
|
|
("windows/x86_64/latest", "windows-x86_64", "x86_64"),
|
|
("windows/x86/latest", "windows-x86", "x86"),
|
|
("android/latest/arm64-v8a", "android-arm64-v8a", "aarch64"),
|
|
("android/latest/armeabi-v7a", "android-armeabi-v7a", "armv7"),
|
|
("android/latest/x86_64", "android-x86_64", "x86_64"),
|
|
("android/latest/x86", "android-x86", "x86"),
|
|
("apple/osx/x86_64/latest", "osx-x86_64", "x86_64"),
|
|
("apple/osx/arm64/latest", "osx-arm64", "aarch64"),
|
|
("apple/ios-arm64/latest", "ios-arm64", "aarch64"),
|
|
("apple/tvos-arm64/latest", "tvos-arm64", "aarch64"),
|
|
("nintendo/switch/libnx/latest", "nintendo-switch", "aarch64"),
|
|
("nintendo/3ds/latest/3dsx", "nintendo-3ds", "arm"),
|
|
("nintendo/ngc/latest", "nintendo-gamecube", "ppc"),
|
|
("nintendo/wii/latest", "nintendo-wii", "ppc"),
|
|
("nintendo/wiiu/latest", "nintendo-wiiu", "ppc"),
|
|
("playstation/ps2/latest", "playstation-ps2", "mips"),
|
|
("playstation/psp/latest", "playstation-psp", "mips"),
|
|
# vita: only VPK bundles on buildbot -cores listed via libretro-super recipes
|
|
]
|
|
|
|
# Recipe-based targets: (recipe_path_under_RECIPE_BASE_URL, target_name, architecture)
|
|
RECIPE_TARGETS: list[tuple[str, str, str]] = [
|
|
("playstation/vita", "playstation-vita", "armv7"),
|
|
]
|
|
|
|
RECIPE_BASE_URL = (
|
|
"https://raw.githubusercontent.com/libretro/libretro-super/master/recipes/"
|
|
)
|
|
|
|
# Match any href containing _libretro followed by a platform-specific extension
|
|
# Covers: .so.zip, .dll.zip, .dylib.zip, .nro.zip, .dol.zip, .rpx.zip,
|
|
# .3dsx.zip, .PBP.zip, .elf.zip, _android.so.zip
|
|
_HREF_RE = re.compile(
|
|
r'href="([^"]*?(\w+)_libretro[^"]*?\.zip)"',
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
# Extract core name: everything before _libretro
|
|
_CORE_NAME_RE = re.compile(r"^(.+?)_libretro")
|
|
|
|
|
|
class Scraper(BaseTargetScraper):
|
|
"""Fetches core lists per target from RetroArch buildbot nightly."""
|
|
|
|
def __init__(self, url: str = BUILDBOT_URL):
|
|
super().__init__(url=url)
|
|
|
|
def _fetch_url(self, url: str) -> str | None:
|
|
try:
|
|
req = urllib.request.Request(
|
|
url, headers={"User-Agent": "retrobios-scraper/1.0"}
|
|
)
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return resp.read().decode("utf-8")
|
|
except urllib.error.URLError as e:
|
|
print(f" skip {url}: {e}", file=sys.stderr)
|
|
return None
|
|
|
|
def _fetch_cores_for_target(self, path: str) -> list[str]:
|
|
url = f"{self.url}{path}/"
|
|
html = self._fetch_url(url)
|
|
if html is None:
|
|
return []
|
|
cores: list[str] = []
|
|
seen: set[str] = set()
|
|
for match in _HREF_RE.finditer(html):
|
|
href = match.group(1)
|
|
filename = href.split("/")[-1]
|
|
m = _CORE_NAME_RE.match(filename)
|
|
if m:
|
|
core = m.group(1)
|
|
if core not in seen:
|
|
seen.add(core)
|
|
cores.append(core)
|
|
return sorted(cores)
|
|
|
|
def _parse_recipe_cores(self, text: str) -> list[str]:
|
|
cores: list[str] = []
|
|
seen: set[str] = set()
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
parts = line.split()
|
|
if not parts:
|
|
continue
|
|
core = parts[0]
|
|
if core not in seen:
|
|
seen.add(core)
|
|
cores.append(core)
|
|
return sorted(cores)
|
|
|
|
def _fetch_cores_for_recipe(self, recipe_path: str) -> list[str]:
|
|
url = f"{RECIPE_BASE_URL}{recipe_path}"
|
|
text = self._fetch_url(url)
|
|
if text is None:
|
|
return []
|
|
return self._parse_recipe_cores(text)
|
|
|
|
def fetch_targets(self) -> dict:
|
|
targets: dict[str, dict] = {}
|
|
for path, target_name, arch in TARGETS:
|
|
print(f" fetching {target_name}...", file=sys.stderr)
|
|
cores = self._fetch_cores_for_target(path)
|
|
if not cores:
|
|
print(f" warning: no cores found for {target_name}", file=sys.stderr)
|
|
continue
|
|
targets[target_name] = {
|
|
"architecture": arch,
|
|
"cores": cores,
|
|
}
|
|
print(f" {target_name}: {len(cores)} cores", file=sys.stderr)
|
|
for recipe_path, target_name, arch in RECIPE_TARGETS:
|
|
print(f" fetching {target_name} (recipe)...", file=sys.stderr)
|
|
cores = self._fetch_cores_for_recipe(recipe_path)
|
|
if not cores:
|
|
print(f" warning: no cores found for {target_name}", file=sys.stderr)
|
|
continue
|
|
targets[target_name] = {
|
|
"architecture": arch,
|
|
"cores": cores,
|
|
}
|
|
print(f" {target_name}: {len(cores)} cores", file=sys.stderr)
|
|
return {
|
|
"platform": "retroarch",
|
|
"source": self.url,
|
|
"scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"targets": targets,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Scrape RetroArch buildbot nightly targets"
|
|
)
|
|
parser.add_argument("--dry-run", action="store_true", help="Show target summary")
|
|
parser.add_argument("--output", "-o", help="Output YAML file")
|
|
args = parser.parse_args()
|
|
|
|
scraper = Scraper()
|
|
data = scraper.fetch_targets()
|
|
|
|
total_cores = sum(len(t["cores"]) for t in data["targets"].values())
|
|
print(
|
|
f"\n{len(data['targets'])} targets, {total_cores} total core entries",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
if args.dry_run:
|
|
for name, info in sorted(data["targets"].items()):
|
|
print(
|
|
f" {name:30s} {info['architecture']:10s} {len(info['cores']):>4d} cores"
|
|
)
|
|
return
|
|
|
|
if args.output:
|
|
scraper.write_output(data, args.output)
|
|
print(f"Written to {args.output}")
|
|
return
|
|
|
|
print(yaml.dump(data, default_flow_style=False, sort_keys=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|