mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-04-13 12:22:33 -05:00
chore: add MAME and RetroDECK ROM sets
This commit is contained in:
Abdessamad Derraz
@@ -112,11 +112,16 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
|
||||
merged["systems"][sys_id] = override
|
||||
config = merged
|
||||
|
||||
# Resolve shared group includes
|
||||
# Resolve shared group includes (cached to avoid re-parsing per call)
|
||||
shared_path = os.path.join(platforms_dir, "_shared.yml")
|
||||
if os.path.exists(shared_path):
|
||||
with open(shared_path) as f:
|
||||
shared = yaml.safe_load(f) or {}
|
||||
if not hasattr(load_platform_config, "_shared_cache"):
|
||||
load_platform_config._shared_cache = {}
|
||||
cache_key = os.path.realpath(shared_path)
|
||||
if cache_key not in load_platform_config._shared_cache:
|
||||
with open(shared_path) as f:
|
||||
load_platform_config._shared_cache[cache_key] = yaml.safe_load(f) or {}
|
||||
shared = load_platform_config._shared_cache[cache_key]
|
||||
shared_groups = shared.get("shared_groups", {})
|
||||
for system in config.get("systems", {}).values():
|
||||
for group_name in system.get("includes", []):
|
||||
@@ -288,8 +293,11 @@ def build_zip_contents_index(db: dict, max_entry_size: int = 512 * 1024 * 1024)
|
||||
for info in zf.infolist():
|
||||
if info.is_dir() or info.file_size > max_entry_size:
|
||||
continue
|
||||
data = zf.read(info.filename)
|
||||
index[hashlib.md5(data).hexdigest()] = sha1
|
||||
h = hashlib.md5()
|
||||
with zf.open(info.filename) as inner:
|
||||
for chunk in iter(lambda: inner.read(65536), b""):
|
||||
h.update(chunk)
|
||||
index[h.hexdigest()] = sha1
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
continue
|
||||
return index
|
||||
|
||||
@@ -49,7 +49,7 @@ def _canonical_name(filepath: Path) -> str:
|
||||
return name
|
||||
|
||||
|
||||
def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> dict:
|
||||
def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict, dict]:
|
||||
"""Scan bios directory and compute hashes, using cache when possible."""
|
||||
files = {}
|
||||
aliases = {}
|
||||
|
||||
@@ -224,6 +224,7 @@ def generate_pack(
|
||||
emulators_dir: str = "emulators",
|
||||
zip_contents: dict | None = None,
|
||||
data_registry: dict | None = None,
|
||||
emu_profiles: dict | None = None,
|
||||
) -> str | None:
|
||||
"""Generate a ZIP pack for a platform.
|
||||
|
||||
@@ -269,7 +270,7 @@ def generate_pack(
|
||||
full_dest = dest
|
||||
|
||||
dedup_key = full_dest
|
||||
already_packed = dedup_key in seen_destinations
|
||||
already_packed = dedup_key in seen_destinations or dedup_key.lower() in seen_lower
|
||||
|
||||
storage = file_entry.get("storage", "embedded")
|
||||
|
||||
@@ -364,7 +365,8 @@ def generate_pack(
|
||||
total_files += 1
|
||||
|
||||
# Core requirements: files platform's cores need but YAML doesn't declare
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
if emu_profiles is None:
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
core_files = _collect_emulator_extras(
|
||||
config, emulators_dir, db,
|
||||
seen_destinations, base_dest, emu_profiles,
|
||||
@@ -374,7 +376,16 @@ def generate_pack(
|
||||
dest = _sanitize_path(fe.get("destination", fe["name"]))
|
||||
if not dest:
|
||||
continue
|
||||
full_dest = f"{base_dest}/{dest}" if base_dest else dest
|
||||
# Core extras use flat filenames; prepend base_destination or
|
||||
# default to the platform's most common BIOS path prefix
|
||||
if base_dest:
|
||||
full_dest = f"{base_dest}/{dest}"
|
||||
elif "/" not in dest:
|
||||
# Bare filename with empty base_destination — infer bios/ prefix
|
||||
# to match platform conventions (RetroDECK: ~/retrodeck/bios/)
|
||||
full_dest = f"bios/{dest}"
|
||||
else:
|
||||
full_dest = dest
|
||||
if full_dest in seen_destinations:
|
||||
continue
|
||||
# Skip case-insensitive duplicates (Windows/macOS FS safety)
|
||||
@@ -513,6 +524,7 @@ def main():
|
||||
if updated:
|
||||
print(f"Refreshed {updated} data director{'ies' if updated > 1 else 'y'}")
|
||||
|
||||
emu_profiles = load_emulator_profiles(args.emulators_dir)
|
||||
groups = group_identical_platforms(platforms, args.platforms_dir)
|
||||
|
||||
for group_platforms, representative in groups:
|
||||
@@ -528,6 +540,7 @@ def main():
|
||||
representative, args.platforms_dir, db, args.bios_dir, args.output_dir,
|
||||
include_extras=args.include_extras, emulators_dir=args.emulators_dir,
|
||||
zip_contents=zip_contents, data_registry=data_registry,
|
||||
emu_profiles=emu_profiles,
|
||||
)
|
||||
if zip_path and len(group_platforms) > 1:
|
||||
# Rename ZIP to include all platform names
|
||||
|
||||
@@ -47,6 +47,31 @@ def compute_coverage(platform_name: str, platforms_dir: str, db: dict) -> dict:
|
||||
|
||||
SITE_URL = "https://abdess.github.io/retrobios/"
|
||||
RELEASE_URL = "../../releases/latest"
|
||||
REPO = "Abdess/retrobios"
|
||||
|
||||
|
||||
def fetch_contributors() -> list[dict]:
|
||||
"""Fetch contributors from GitHub API, exclude bots."""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
url = f"https://api.github.com/repos/{REPO}/contributors"
|
||||
headers = {"User-Agent": "retrobios-readme/1.0"}
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
if token:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
owner = REPO.split("/")[0]
|
||||
return [
|
||||
c for c in data
|
||||
if not c.get("login", "").endswith("[bot]")
|
||||
and c.get("type") == "User"
|
||||
and c.get("login") != owner
|
||||
]
|
||||
except (urllib.error.URLError, urllib.error.HTTPError):
|
||||
return []
|
||||
|
||||
|
||||
def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
@@ -111,6 +136,25 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"",
|
||||
f"Full file listings, platform coverage, emulator profiles, and gap analysis: **[{SITE_URL}]({SITE_URL})**",
|
||||
"",
|
||||
])
|
||||
|
||||
contributors = fetch_contributors()
|
||||
if contributors:
|
||||
lines.extend([
|
||||
"## Contributors",
|
||||
"",
|
||||
])
|
||||
for c in contributors:
|
||||
login = c["login"]
|
||||
avatar = c.get("avatar_url", "")
|
||||
url = c.get("html_url", f"https://github.com/{login}")
|
||||
lines.append(
|
||||
f'<a href="{url}"><img src="{avatar}" width="50" title="{login}"></a>'
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"## Contributing",
|
||||
"",
|
||||
"See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.",
|
||||
|
||||
@@ -26,7 +26,8 @@ except ImportError:
|
||||
sys.exit(1)
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import load_database, load_platform_config
|
||||
from common import load_database, load_emulator_profiles, load_platform_config
|
||||
from generate_readme import compute_coverage
|
||||
from verify import verify_platform
|
||||
|
||||
DOCS_DIR = "docs"
|
||||
@@ -65,46 +66,6 @@ def _status_icon(pct: float) -> str:
|
||||
return "partial"
|
||||
|
||||
|
||||
def compute_coverage(platform_name: str, platforms_dir: str, db: dict) -> dict:
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
result = verify_platform(config, db)
|
||||
sc = result.get("status_counts", {})
|
||||
ok = sc.get("ok", 0)
|
||||
untested = sc.get("untested", 0)
|
||||
missing = sc.get("missing", 0)
|
||||
total = result["total_files"]
|
||||
present = ok + untested
|
||||
pct = (present / total * 100) if total > 0 else 0
|
||||
return {
|
||||
"platform": config.get("platform", platform_name),
|
||||
"total": total,
|
||||
"verified": ok,
|
||||
"untested": untested,
|
||||
"missing": missing,
|
||||
"present": present,
|
||||
"percentage": pct,
|
||||
"mode": config.get("verification_mode", "existence"),
|
||||
"details": result["details"],
|
||||
"config": config,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Load emulator profiles
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_emulator_profiles(emulators_dir: str) -> dict[str, dict]:
|
||||
profiles = {}
|
||||
emu_path = Path(emulators_dir)
|
||||
if not emu_path.exists():
|
||||
return profiles
|
||||
for f in sorted(emu_path.glob("*.yml")):
|
||||
with open(f) as fh:
|
||||
profile = yaml.safe_load(fh) or {}
|
||||
profiles[f.stem] = profile
|
||||
return profiles
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Home page
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -743,7 +704,7 @@ def main():
|
||||
|
||||
# Load emulator profiles
|
||||
print("Loading emulator profiles...")
|
||||
profiles = _load_emulator_profiles(args.emulators_dir)
|
||||
profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
|
||||
unique_count = sum(1 for p in profiles.values() if p.get("type") != "alias")
|
||||
print(f" {len(profiles)} profiles ({unique_count} unique, {len(profiles) - unique_count} aliases)")
|
||||
|
||||
|
||||
@@ -116,7 +116,9 @@ def get_remote_sha(source_url: str, version: str) -> str | None:
|
||||
|
||||
|
||||
def _is_safe_tar_member(member: tarfile.TarInfo, dest: Path) -> bool:
|
||||
"""Reject path traversal and absolute paths in tar members."""
|
||||
"""Reject path traversal, absolute paths, and symlinks in tar members."""
|
||||
if member.issym() or member.islnk():
|
||||
return False
|
||||
if member.name.startswith("/") or ".." in member.name.split("/"):
|
||||
return False
|
||||
resolved = (dest / member.name).resolve()
|
||||
|
||||
@@ -2,40 +2,25 @@
|
||||
"""Scraper for RetroDECK BIOS requirements.
|
||||
|
||||
Source: https://github.com/RetroDECK/components
|
||||
Format: component_manifest.json committed at <component>/component_manifest.json
|
||||
Hash: MD5 primary, SHA256 for some entries (melonDS DSi BIOS)
|
||||
Format: component_manifest.json per component directory
|
||||
Hash: MD5 (primary), SHA256 for some entries (melonDS DSi)
|
||||
|
||||
RetroDECK verification logic:
|
||||
- MD5 or SHA256 checked against expected value per file
|
||||
- MD5 may be a list of multiple accepted hashes (xroar ROM variants) — joined
|
||||
as comma-separated string per retrobios convention
|
||||
- Files may declare paths via $bios_path, $saves_path, or $roms_path tokens
|
||||
- $saves_path entries (GameCube memory card directories) are excluded —
|
||||
these are directory placeholders, not BIOS files
|
||||
- $roms_path entries are included with a roms/ prefix in destination,
|
||||
consistent with Batocera's saves/ destination convention
|
||||
- Entries with no hash are emitted without an md5 field (existence-only),
|
||||
which is valid per the platform schema (e.g. pico-8 executables)
|
||||
RetroDECK stores BIOS requirements in component_manifest.json files,
|
||||
one per emulator component. BIOS entries can appear in three locations:
|
||||
- top-level 'bios' key
|
||||
- preset_actions.bios (duckstation, dolphin, pcsx2)
|
||||
- cores.bios (retroarch)
|
||||
|
||||
Component structure:
|
||||
RetroDECK/components (GitHub, main branch)
|
||||
├── <component>/component_manifest.json <- fetched directly via raw URL
|
||||
├── archive_later/ <- skipped
|
||||
└── archive_old/ <- skipped
|
||||
Path tokens: $bios_path, $saves_path, $roms_path map to
|
||||
~/retrodeck/bios/, ~/retrodeck/saves/, ~/retrodeck/roms/ respectively.
|
||||
$saves_path entries are directory placeholders (excluded).
|
||||
$roms_path entries (neogeo.zip etc.) get roms/ prefix in destination.
|
||||
Entries with no paths key default to bios/ (RetroDECK's default BIOS dir).
|
||||
|
||||
BIOS may appear in three locations within a manifest:
|
||||
- top-level 'bios' key (melonDS, xemu, xroar, pico-8)
|
||||
- preset_actions.bios (duckstation, dolphin, pcsx2, ppsspp)
|
||||
- cores.bios (not yet seen in practice, kept for safety)
|
||||
|
||||
ppsspp quirk: preset_actions.bios is a bare dict, not a list.
|
||||
|
||||
Adding to watch.yml (maintainer step):
|
||||
from scraper.retrodeck_scraper import Scraper as RDS
|
||||
config = RDS().generate_platform_yaml()
|
||||
with open('platforms/retrodeck.yml', 'w') as f:
|
||||
yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
||||
print(f'RetroDECK: {len(config["systems"])} systems, version={config["version"]}')
|
||||
Verification logic (api_data_processing.sh:289-405):
|
||||
- md5sum per file, compared against known_md5 (comma-separated list)
|
||||
- envsubst resolves path tokens at runtime
|
||||
- Multi-threaded on system_cpu_max_threads
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -49,530 +34,396 @@ import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
|
||||
from .base_scraper import BaseScraper, BiosRequirement
|
||||
except ImportError:
|
||||
from base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
from scraper.base_scraper import BaseScraper, BiosRequirement
|
||||
|
||||
PLATFORM_NAME = "retrodeck"
|
||||
|
||||
COMPONENTS_REPO = "RetroDECK/components"
|
||||
COMPONENTS_REPO = "RetroDECK/components"
|
||||
COMPONENTS_BRANCH = "main"
|
||||
COMPONENTS_API_URL = (
|
||||
f"https://api.github.com/repos/{COMPONENTS_REPO}"
|
||||
f"/git/trees/{COMPONENTS_BRANCH}?recursive=0"
|
||||
f"/git/trees/{COMPONENTS_BRANCH}"
|
||||
)
|
||||
RAW_BASE_URL = (
|
||||
RAW_BASE = (
|
||||
f"https://raw.githubusercontent.com/{COMPONENTS_REPO}"
|
||||
f"/{COMPONENTS_BRANCH}"
|
||||
)
|
||||
|
||||
# Top-level directories to ignore when enumerating components
|
||||
SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"}
|
||||
|
||||
# Default local path for --manifests-dir (standard flatpak install)
|
||||
DEFAULT_LOCAL_MANIFESTS = (
|
||||
"/var/lib/flatpak/app/net.retrodeck.retrodeck"
|
||||
"/current/active/files/retrodeck/components"
|
||||
)
|
||||
|
||||
# RetroDECK system ID -> retrobios slug.
|
||||
# IDs absent from this map pass through unchanged (maintainer decides on slug).
|
||||
# IDs mapped to None are skipped entirely (no retrobios equivalent).
|
||||
SYSTEM_SLUG_MAP: dict[str, str | None] = {
|
||||
# Nintendo
|
||||
"nes": "nintendo-nes",
|
||||
"snes": "nintendo-snes",
|
||||
"snesna": "nintendo-snes",
|
||||
"n64": "nintendo-64",
|
||||
"n64dd": "nintendo-64dd",
|
||||
"gc": "nintendo-gamecube",
|
||||
"wii": "wii", # no retrobios slug yet — passes through
|
||||
"wiiu": "nintendo-wii-u",
|
||||
"switch": "nintendo-switch",
|
||||
"gb": "nintendo-gb",
|
||||
"gbc": "nintendo-gbc",
|
||||
"gba": "nintendo-gba",
|
||||
"nds": "nintendo-ds",
|
||||
"3ds": "nintendo-3ds",
|
||||
"n3ds": "nintendo-3ds", # azahar uses n3ds
|
||||
"fds": "nintendo-fds",
|
||||
"sgb": "nintendo-sgb",
|
||||
"virtualboy": "nintendo-virtual-boy",
|
||||
# Sony
|
||||
"psx": "sony-playstation",
|
||||
"ps2": "sony-playstation-2",
|
||||
"ps3": "sony-playstation-3",
|
||||
"psp": "sony-psp",
|
||||
"psvita": "sony-psvita",
|
||||
# Sega
|
||||
"megadrive": "sega-mega-drive",
|
||||
"genesis": "sega-mega-drive",
|
||||
"megacd": "sega-mega-cd",
|
||||
"megacdjp": "sega-mega-cd",
|
||||
"segacd": "sega-mega-cd",
|
||||
"saturn": "sega-saturn",
|
||||
"saturnjp": "sega-saturn",
|
||||
"dreamcast": "sega-dreamcast",
|
||||
"naomi": "sega-dreamcast-arcade",
|
||||
"naomi2": "sega-dreamcast-arcade",
|
||||
"atomiswave": "sega-dreamcast-arcade",
|
||||
"sega32x": "sega32x",
|
||||
"sega32xjp": "sega32x",
|
||||
"sega32xna": "sega32x",
|
||||
"gamegear": "sega-game-gear",
|
||||
"mastersystem": "sega-master-system",
|
||||
# NEC
|
||||
"tg16": "nec-pc-engine",
|
||||
"tg-cd": "nec-pc-engine",
|
||||
"pcengine": "nec-pc-engine",
|
||||
"pcenginecd": "nec-pc-engine",
|
||||
"pcfx": "nec-pc-fx",
|
||||
# SNK
|
||||
"neogeo": "snk-neogeo",
|
||||
"neogeocd": "snk-neogeo-cd",
|
||||
"neogeocdjp": "snk-neogeo-cd",
|
||||
# Atari
|
||||
"atari2600": "atari2600", # no retrobios slug yet — passes through
|
||||
"atari800": "atari-400-800",
|
||||
"atari5200": "atari-5200",
|
||||
"atari7800": "atari-7800",
|
||||
"atarilynx": "atari-lynx",
|
||||
"atarist": "atari-st",
|
||||
"atarijaguar": "jaguar",
|
||||
# Panasonic / Philips
|
||||
"3do": "panasonic-3do",
|
||||
"cdimono1": "cdi",
|
||||
"cdtv": "amigacdtv",
|
||||
# Microsoft
|
||||
"xbox": "xbox",
|
||||
# Commodore
|
||||
"amiga": "commodore-amiga",
|
||||
"amigacd32": "amigacd32",
|
||||
"c64": "commodore-c64",
|
||||
# Tandy / Dragon
|
||||
"coco": "trs80coco",
|
||||
"dragon32": "dragon32",
|
||||
"tanodragon": "dragon32", # Tano Dragon is a Dragon 32 clone
|
||||
# Other
|
||||
"colecovision": "coleco-colecovision",
|
||||
"intellivision": "mattel-intellivision",
|
||||
"o2em": "magnavox-odyssey2",
|
||||
"msx": "microsoft-msx",
|
||||
"msx2": "microsoft-msx",
|
||||
"fmtowns": "fmtowns",
|
||||
"scummvm": "scummvm",
|
||||
"dos": "dos",
|
||||
# Explicitly skipped — no retrobios equivalent
|
||||
"mess": None,
|
||||
NON_EMULATOR_COMPONENTS = {
|
||||
"framework", "es-de", "steam-rom-manager", "flips", "portmaster",
|
||||
}
|
||||
|
||||
# Matches all saves_path typo variants seen in the wild:
|
||||
# $saves_path, $saves_paths_path, $saves_paths_paths_path, etc.
|
||||
_SAVES_PATH_RE = re.compile(r"^\$saves_\w+/")
|
||||
# RetroDECK system ID -> retrobios slug.
|
||||
# None = skip (system not relevant for BIOS packs).
|
||||
# Missing key = pass through as-is.
|
||||
SYSTEM_SLUG_MAP: dict[str, str | None] = {
|
||||
# Nintendo
|
||||
"nes": "nintendo-nes",
|
||||
"snes": "nintendo-snes",
|
||||
"snesna": "nintendo-snes",
|
||||
"n64": "nintendo-64",
|
||||
"n64dd": "nintendo-64dd",
|
||||
"gc": "nintendo-gamecube",
|
||||
"wii": "nintendo-wii",
|
||||
"wiiu": "nintendo-wii-u",
|
||||
"switch": "nintendo-switch",
|
||||
"gb": "nintendo-gb",
|
||||
"gbc": "nintendo-gbc",
|
||||
"gba": "nintendo-gba",
|
||||
"nds": "nintendo-ds",
|
||||
"3ds": "nintendo-3ds",
|
||||
"n3ds": "nintendo-3ds",
|
||||
"fds": "nintendo-fds",
|
||||
"sgb": "nintendo-sgb",
|
||||
"virtualboy": "nintendo-virtual-boy",
|
||||
# Sony
|
||||
"psx": "sony-playstation",
|
||||
"ps2": "sony-playstation-2",
|
||||
"ps3": "sony-playstation-3",
|
||||
"psp": "sony-psp",
|
||||
"psvita": "sony-psvita",
|
||||
# Sega
|
||||
"megadrive": "sega-mega-drive",
|
||||
"genesis": "sega-mega-drive",
|
||||
"megacd": "sega-mega-cd",
|
||||
"megacdjp": "sega-mega-cd",
|
||||
"segacd": "sega-mega-cd",
|
||||
"saturn": "sega-saturn",
|
||||
"saturnjp": "sega-saturn",
|
||||
"dreamcast": "sega-dreamcast",
|
||||
"naomi": "sega-dreamcast-arcade",
|
||||
"naomi2": "sega-dreamcast-arcade",
|
||||
"atomiswave": "sega-dreamcast-arcade",
|
||||
"gamegear": "sega-game-gear",
|
||||
"mastersystem": "sega-master-system",
|
||||
"sms": "sega-master-system",
|
||||
# NEC
|
||||
"pcengine": "nec-pc-engine",
|
||||
"pcenginecd": "nec-pc-engine",
|
||||
"turbografx16": "nec-pc-engine",
|
||||
"pcfx": "nec-pc-fx",
|
||||
"pc98": "nec-pc-98",
|
||||
"pc9800": "nec-pc-98",
|
||||
"pc88": "nec-pc-88",
|
||||
"pc8800": "nec-pc-88",
|
||||
# Other
|
||||
"3do": "3do",
|
||||
"amstradcpc": "amstrad-cpc",
|
||||
"arcade": "arcade",
|
||||
"mame": "arcade",
|
||||
"fbneo": "arcade",
|
||||
"atari800": "atari-400-800",
|
||||
"atari5200": "atari-5200",
|
||||
"atari7800": "atari-7800",
|
||||
"atarijaguar": "atari-jaguar",
|
||||
"atarilynx": "atari-lynx",
|
||||
"atarist": "atari-st",
|
||||
"atarixe": "atari-400-800",
|
||||
"c64": "commodore-c64",
|
||||
"amiga": "commodore-amiga",
|
||||
"cdimono1": "philips-cdi",
|
||||
"channelf": "fairchild-channel-f",
|
||||
"colecovision": "coleco-colecovision",
|
||||
"intellivision": "mattel-intellivision",
|
||||
"msx": "microsoft-msx",
|
||||
"xbox": "microsoft-xbox",
|
||||
"doom": "doom",
|
||||
"j2me": "j2me",
|
||||
"mac2": "apple-macintosh-ii",
|
||||
"macintosh": "apple-macintosh-ii",
|
||||
"apple2": "apple-ii",
|
||||
"apple2gs": "apple-iigs",
|
||||
"enterprise": "enterprise-64-128",
|
||||
"gamecom": "tiger-game-com",
|
||||
"gmaster": "hartung-game-master",
|
||||
"pokemini": "nintendo-pokemon-mini",
|
||||
"scv": "epoch-scv",
|
||||
"supervision": "watara-supervision",
|
||||
"wonderswan": "bandai-wonderswan",
|
||||
"neogeocd": "snk-neogeo-cd",
|
||||
"neogeocdjp": "snk-neogeo-cd",
|
||||
"coco": "tandy-coco",
|
||||
"trs80": "tandy-trs-80",
|
||||
"dragon": "dragon-32-64",
|
||||
"tanodragon": "dragon-32-64",
|
||||
"pico8": "pico8",
|
||||
"wolfenstein": "wolfenstein-3d",
|
||||
"zxspectrum": "sinclair-zx-spectrum",
|
||||
}
|
||||
|
||||
|
||||
def _fetch_bytes(url: str, token: str | None = None) -> bytes:
|
||||
headers = {"User-Agent": "retrobios-scraper/1.0"}
|
||||
if token:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return resp.read()
|
||||
except urllib.error.URLError as e:
|
||||
raise ConnectionError(f"Failed to fetch {url}: {e}") from e
|
||||
def _sanitize_path(p: str) -> str:
|
||||
"""Fix upstream typos in path tokens."""
|
||||
return re.sub(r"\$saves_\w+", "$saves_path", p)
|
||||
|
||||
|
||||
def _fetch_json(url: str, token: str | None = None) -> dict | list:
|
||||
return json.loads(_fetch_bytes(url, token).decode("utf-8"))
|
||||
def _resolve_path(p: str) -> str:
|
||||
"""Resolve RetroDECK path tokens to pack-relative paths."""
|
||||
p = _sanitize_path(p)
|
||||
p = p.replace("$bios_path", "bios")
|
||||
p = p.replace("$saves_path", "saves")
|
||||
p = p.replace("$roms_path", "roms")
|
||||
return p.strip("/")
|
||||
|
||||
|
||||
def _resolve_destination(raw_path: str, filename: str) -> str | None:
|
||||
"""Resolve a RetroDECK path token to a retrobios destination string.
|
||||
def _extract_bios_entries(component_val: dict) -> list[dict]:
|
||||
"""Extract BIOS entries from all three possible locations in a component.
|
||||
|
||||
Returns None if the entry should be excluded ($saves_path variants).
|
||||
$bios_path -> strip prefix; destination is bios-relative.
|
||||
$roms_path -> preserve roms/ prefix (Batocera saves/ convention).
|
||||
Bare directory paths get filename appended.
|
||||
No dedup here — dedup is done in fetch_requirements() with full
|
||||
(system, filename) key to avoid dropping valid same-filename entries
|
||||
across different systems.
|
||||
"""
|
||||
if _SAVES_PATH_RE.match(raw_path):
|
||||
return None
|
||||
entries: list[dict] = []
|
||||
|
||||
if raw_path.startswith("$bios_path/"):
|
||||
remainder = raw_path[len("$bios_path/"):].strip("/")
|
||||
if not remainder or remainder == filename:
|
||||
return filename
|
||||
# Subdirectory path — append filename if path looks like a directory
|
||||
if not remainder.endswith(tuple(".bin .rom .zip .img .bin ".split())):
|
||||
return remainder.rstrip("/") + "/" + filename
|
||||
return remainder
|
||||
def collect(bios_data: list | dict) -> None:
|
||||
if isinstance(bios_data, dict):
|
||||
bios_data = [bios_data]
|
||||
if not isinstance(bios_data, list):
|
||||
return
|
||||
for entry in bios_data:
|
||||
if isinstance(entry, dict) and entry.get("filename", "").strip():
|
||||
entries.append(entry)
|
||||
|
||||
if raw_path.startswith("$roms_path/"):
|
||||
remainder = raw_path[len("$roms_path/"):].strip("/")
|
||||
base = ("roms/" + remainder) if remainder else "roms"
|
||||
return base.rstrip("/") + "/" + filename
|
||||
if "bios" in component_val:
|
||||
collect(component_val["bios"])
|
||||
|
||||
# No recognised token — treat as bios-relative
|
||||
remainder = raw_path.strip("/")
|
||||
if not remainder:
|
||||
return filename
|
||||
return remainder.rstrip("/") + "/" + filename
|
||||
pa = component_val.get("preset_actions", {})
|
||||
if isinstance(pa, dict) and "bios" in pa:
|
||||
collect(pa["bios"])
|
||||
|
||||
cores = component_val.get("cores", {})
|
||||
if isinstance(cores, dict) and "bios" in cores:
|
||||
collect(cores["bios"])
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def _normalise_md5(raw: str | list) -> str:
|
||||
"""Return a comma-separated MD5 string.
|
||||
def _map_system(raw_system: str) -> str | None:
|
||||
"""Map RetroDECK system ID to retrobios slug.
|
||||
|
||||
xroar declares a list of accepted hashes for ROM variants;
|
||||
retrobios platform schema accepts comma-separated MD5 strings.
|
||||
Returns None for systems explicitly excluded from the map.
|
||||
Unknown systems pass through as-is.
|
||||
"""
|
||||
if isinstance(raw, list):
|
||||
return ",".join(str(h).strip().lower() for h in raw if h)
|
||||
return str(raw).strip().lower() if raw else ""
|
||||
|
||||
|
||||
def _coerce_bios_to_list(val: object) -> list:
|
||||
"""Ensure a bios value is always a list of dicts.
|
||||
|
||||
ppsspp declares preset_actions.bios as a bare dict, not a list.
|
||||
"""
|
||||
if isinstance(val, list):
|
||||
return val
|
||||
if isinstance(val, dict):
|
||||
return [val]
|
||||
return []
|
||||
|
||||
|
||||
def _parse_required(raw: object) -> bool:
|
||||
"""Coerce RetroDECK required field to bool.
|
||||
|
||||
Values seen: 'Required', 'Optional', 'At least one BIOS file required',
|
||||
'Optional, for boot logo', True, False, absent (None).
|
||||
Absent is treated as required.
|
||||
"""
|
||||
if isinstance(raw, bool):
|
||||
return raw
|
||||
if raw is None:
|
||||
return True
|
||||
return str(raw).strip().lower() not in ("optional", "false", "no", "0")
|
||||
|
||||
|
||||
def _parse_manifest(data: dict) -> list[BiosRequirement]:
|
||||
"""Parse one component_manifest.json into BiosRequirement objects."""
|
||||
requirements: list[BiosRequirement] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
|
||||
for _component_key, component_val in data.items():
|
||||
if not isinstance(component_val, dict):
|
||||
continue
|
||||
|
||||
# Component-level system fallback (may be a list for multi-system components)
|
||||
comp_system = component_val.get("system", "")
|
||||
if isinstance(comp_system, list):
|
||||
comp_system = comp_system[0] if comp_system else ""
|
||||
comp_system = str(comp_system).strip().lower()
|
||||
|
||||
# Collect bios entries from all known locations
|
||||
bios_sources: list[list] = []
|
||||
|
||||
if "bios" in component_val:
|
||||
bios_sources.append(_coerce_bios_to_list(component_val["bios"]))
|
||||
|
||||
pa = component_val.get("preset_actions", {})
|
||||
if isinstance(pa, dict) and "bios" in pa:
|
||||
bios_sources.append(_coerce_bios_to_list(pa["bios"]))
|
||||
|
||||
cores = component_val.get("cores", {})
|
||||
if isinstance(cores, dict) and "bios" in cores:
|
||||
bios_sources.append(_coerce_bios_to_list(cores["bios"]))
|
||||
|
||||
if not bios_sources:
|
||||
continue
|
||||
|
||||
for bios_list in bios_sources:
|
||||
for entry in bios_list:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
|
||||
filename = str(entry.get("filename", "")).strip()
|
||||
if not filename:
|
||||
continue
|
||||
|
||||
# System slug — entry-level preferred, component-level fallback
|
||||
entry_system = entry.get("system", comp_system)
|
||||
if isinstance(entry_system, list):
|
||||
entry_system = entry_system[0] if entry_system else comp_system
|
||||
entry_system = str(entry_system).strip().lower()
|
||||
|
||||
if entry_system in SYSTEM_SLUG_MAP:
|
||||
slug = SYSTEM_SLUG_MAP[entry_system]
|
||||
if slug is None:
|
||||
continue # explicitly skipped (e.g. mess)
|
||||
else:
|
||||
slug = entry_system # unknown — pass through
|
||||
|
||||
# Destination resolution
|
||||
paths_raw = entry.get("paths")
|
||||
if paths_raw is None:
|
||||
destination = filename
|
||||
elif isinstance(paths_raw, list):
|
||||
destination = None
|
||||
for p in paths_raw:
|
||||
resolved = _resolve_destination(str(p), filename)
|
||||
if resolved is not None:
|
||||
destination = resolved
|
||||
break
|
||||
if destination is None:
|
||||
continue # all paths were saves_path variants — skip
|
||||
else:
|
||||
destination = _resolve_destination(str(paths_raw), filename)
|
||||
if destination is None:
|
||||
continue # saves_path — skip
|
||||
|
||||
# Hash fields
|
||||
md5_val: str | None = None
|
||||
sha256_val: str | None = None
|
||||
|
||||
raw_md5 = entry.get("md5")
|
||||
if raw_md5:
|
||||
md5_val = _normalise_md5(raw_md5) or None
|
||||
|
||||
raw_sha256 = entry.get("sha256")
|
||||
if raw_sha256:
|
||||
sha256_val = str(raw_sha256).strip().lower() or None
|
||||
|
||||
required = _parse_required(entry.get("required"))
|
||||
|
||||
dedup_key = (slug, filename.lower())
|
||||
if dedup_key in seen:
|
||||
continue
|
||||
seen.add(dedup_key)
|
||||
|
||||
req = BiosRequirement(
|
||||
name=filename,
|
||||
system=slug,
|
||||
md5=md5_val,
|
||||
sha1=None,
|
||||
destination=destination,
|
||||
required=required,
|
||||
)
|
||||
req._sha256 = sha256_val # type: ignore[attr-defined]
|
||||
requirements.append(req)
|
||||
|
||||
return requirements
|
||||
if raw_system in SYSTEM_SLUG_MAP:
|
||||
return SYSTEM_SLUG_MAP[raw_system]
|
||||
return raw_system
|
||||
|
||||
|
||||
class Scraper(BaseScraper):
|
||||
"""Scraper for RetroDECK component_manifest.json files.
|
||||
"""RetroDECK BIOS scraper from component manifests."""
|
||||
|
||||
Two modes:
|
||||
remote (default): fetches manifests directly from RetroDECK/components
|
||||
via GitHub raw URLs, enumerating components via the
|
||||
GitHub API tree endpoint
|
||||
local: reads manifests from a directory on disk
|
||||
(--manifests-dir or pass manifests_dir= to __init__)
|
||||
"""
|
||||
platform_name = PLATFORM_NAME
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
manifests_dir: str | None = None,
|
||||
github_token: str | None = None,
|
||||
):
|
||||
def __init__(self, manifests_dir: str = "") -> None:
|
||||
super().__init__()
|
||||
self.manifests_dir = manifests_dir
|
||||
self.github_token = github_token or os.environ.get("GITHUB_TOKEN")
|
||||
self._release_version: str | None = None
|
||||
self._manifests: list[tuple[str, dict]] | None = None
|
||||
|
||||
# ── Remote ───────────────────────────────────────────────────────────────
|
||||
def _get_manifests(self) -> list[tuple[str, dict]]:
|
||||
"""Fetch manifests once, cache for reuse."""
|
||||
if self._manifests is None:
|
||||
self._manifests = (
|
||||
self._fetch_local_manifests()
|
||||
if self.manifests_dir
|
||||
else self._fetch_remote_manifests()
|
||||
)
|
||||
return self._manifests
|
||||
|
||||
def _list_component_dirs(self) -> list[str]:
|
||||
"""Return top-level component directory names from the GitHub API."""
|
||||
tree = _fetch_json(COMPONENTS_API_URL, self.github_token)
|
||||
return [
|
||||
def _fetch_remote_manifests(self) -> list[tuple[str, dict]]:
|
||||
"""Fetch component manifests via GitHub API."""
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
headers = {"User-Agent": "retrobios-scraper/1.0"}
|
||||
if token:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(COMPONENTS_API_URL, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
tree = json.loads(resp.read().decode())
|
||||
except (urllib.error.HTTPError, urllib.error.URLError) as e:
|
||||
raise ConnectionError(f"Failed to fetch component tree: {e}") from e
|
||||
|
||||
if tree.get("truncated"):
|
||||
print(" WARNING: GitHub tree response truncated", file=sys.stderr)
|
||||
|
||||
component_dirs = [
|
||||
item["path"]
|
||||
for item in tree.get("tree", [])
|
||||
if item["type"] == "tree" and item["path"] not in SKIP_DIRS
|
||||
]
|
||||
|
||||
def _fetch_remote_manifests(self) -> list[dict]:
|
||||
component_dirs = self._list_component_dirs()
|
||||
manifests: list[dict] = []
|
||||
for component in sorted(component_dirs):
|
||||
url = f"{RAW_BASE_URL}/{component}/component_manifest.json"
|
||||
print(f" Fetching {component}/component_manifest.json ...", file=sys.stderr)
|
||||
manifests: list[tuple[str, dict]] = []
|
||||
for comp in sorted(component_dirs):
|
||||
url = f"{RAW_BASE}/{comp}/component_manifest.json"
|
||||
print(f" {comp} ...", file=sys.stderr, end="", flush=True)
|
||||
try:
|
||||
raw = _fetch_bytes(url, self.github_token)
|
||||
manifests.append(json.loads(raw.decode("utf-8")))
|
||||
except ConnectionError:
|
||||
pass # component has no manifest — skip silently
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
manifests.append((comp, data))
|
||||
print(" ok", file=sys.stderr)
|
||||
except (urllib.error.HTTPError, urllib.error.URLError):
|
||||
print(" skip", file=sys.stderr)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" WARNING: parse error in {component}: {e}", file=sys.stderr)
|
||||
print(f" parse error: {e}", file=sys.stderr)
|
||||
return manifests
|
||||
|
||||
# ── Local ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _fetch_local_manifests(self) -> list[dict]:
|
||||
def _fetch_local_manifests(self) -> list[tuple[str, dict]]:
|
||||
"""Read manifests from local RetroDECK install."""
|
||||
root = Path(self.manifests_dir)
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(f"Manifests directory not found: {root}")
|
||||
manifests: list[dict] = []
|
||||
# Only scan top-level component directories; skip archive and hidden dirs
|
||||
for component_dir in sorted(root.iterdir()):
|
||||
if not component_dir.is_dir():
|
||||
manifests: list[tuple[str, dict]] = []
|
||||
for d in sorted(root.iterdir()):
|
||||
if not d.is_dir() or d.name in SKIP_DIRS or d.name.startswith("."):
|
||||
continue
|
||||
if component_dir.name in SKIP_DIRS or component_dir.name.startswith("."):
|
||||
continue
|
||||
manifest_path = component_dir / "component_manifest.json"
|
||||
if not manifest_path.exists():
|
||||
mf = d / "component_manifest.json"
|
||||
if not mf.exists():
|
||||
continue
|
||||
try:
|
||||
with open(manifest_path) as f:
|
||||
manifests.append(json.load(f))
|
||||
with open(mf) as f:
|
||||
manifests.append((d.name, json.load(f)))
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
print(f" WARNING: Could not parse {manifest_path}: {e}", file=sys.stderr)
|
||||
print(f" WARNING: {mf}: {e}", file=sys.stderr)
|
||||
return manifests
|
||||
|
||||
# ── BaseScraper interface ─────────────────────────────────────────────────
|
||||
|
||||
def fetch_requirements(self) -> list[BiosRequirement]:
|
||||
manifests = (
|
||||
self._fetch_local_manifests()
|
||||
if self.manifests_dir
|
||||
else self._fetch_remote_manifests()
|
||||
)
|
||||
|
||||
requirements: list[BiosRequirement] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for manifest in manifests:
|
||||
for req in _parse_manifest(manifest):
|
||||
key = (req.system, req.name.lower())
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
requirements.append(req)
|
||||
return requirements
|
||||
|
||||
def validate_format(self, raw_data: str) -> bool:
|
||||
try:
|
||||
return isinstance(json.loads(raw_data), dict)
|
||||
except json.JSONDecodeError:
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return False
|
||||
|
||||
def fetch_requirements(self) -> list[BiosRequirement]:
|
||||
manifests = self._get_manifests()
|
||||
|
||||
requirements: list[BiosRequirement] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
|
||||
for comp_name, manifest in manifests:
|
||||
for comp_key, comp_val in manifest.items():
|
||||
if not isinstance(comp_val, dict):
|
||||
continue
|
||||
|
||||
default_system = comp_val.get("system", comp_key)
|
||||
if isinstance(default_system, list):
|
||||
default_system = default_system[0] if default_system else comp_key
|
||||
|
||||
for entry in _extract_bios_entries(comp_val):
|
||||
filename = entry["filename"].strip()
|
||||
raw_system = entry.get("system", default_system)
|
||||
if isinstance(raw_system, list):
|
||||
raw_system = raw_system[0] if raw_system else default_system
|
||||
|
||||
system = _map_system(str(raw_system))
|
||||
if system is None:
|
||||
continue
|
||||
|
||||
# Resolve path
|
||||
paths_raw = entry.get("paths")
|
||||
if isinstance(paths_raw, str):
|
||||
resolved = _resolve_path(paths_raw)
|
||||
elif isinstance(paths_raw, list):
|
||||
resolved = ""
|
||||
for p in paths_raw:
|
||||
rp = _resolve_path(str(p))
|
||||
if not rp.startswith("saves"):
|
||||
resolved = rp
|
||||
break
|
||||
if not resolved:
|
||||
continue
|
||||
else:
|
||||
resolved = ""
|
||||
|
||||
# Skip saves-only entries
|
||||
if resolved.startswith("saves"):
|
||||
continue
|
||||
|
||||
# Build destination — default to bios/ if no path specified
|
||||
if resolved:
|
||||
destination = f"{resolved}/{filename}"
|
||||
else:
|
||||
destination = f"bios/{filename}"
|
||||
|
||||
# MD5 handling — sanitize upstream errors
|
||||
md5_raw = entry.get("md5", "")
|
||||
if isinstance(md5_raw, list):
|
||||
parts = [str(m).strip().lower() for m in md5_raw if m]
|
||||
elif md5_raw:
|
||||
parts = [str(md5_raw).strip().lower()]
|
||||
else:
|
||||
parts = []
|
||||
# Keep only valid 32-char hex MD5 hashes
|
||||
valid = [p for p in parts if re.fullmatch(r"[0-9a-f]{32}", p)]
|
||||
md5 = ",".join(valid)
|
||||
|
||||
required_raw = entry.get("required", "")
|
||||
required = bool(required_raw) and str(required_raw).lower() not in (
|
||||
"false", "no", "optional", "",
|
||||
)
|
||||
|
||||
key = (system, filename.lower())
|
||||
if key in seen:
|
||||
existing = next(
|
||||
(r for r in requirements if (r.system, r.name.lower()) == key),
|
||||
None,
|
||||
)
|
||||
if existing and md5 and existing.md5 and md5 != existing.md5:
|
||||
print(
|
||||
f" WARNING: {filename} ({system}): MD5 conflict "
|
||||
f"({existing.md5[:12]}... vs {md5[:12]}...)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
seen.add(key)
|
||||
|
||||
requirements.append(BiosRequirement(
|
||||
name=filename,
|
||||
system=system,
|
||||
destination=destination,
|
||||
md5=md5,
|
||||
required=required,
|
||||
))
|
||||
|
||||
return requirements
|
||||
|
||||
def generate_platform_yaml(self) -> dict:
|
||||
requirements = self.fetch_requirements()
|
||||
reqs = self.fetch_requirements()
|
||||
manifests = self._get_manifests()
|
||||
|
||||
cores = sorted({
|
||||
comp_name for comp_name, _ in manifests
|
||||
if comp_name not in SKIP_DIRS
|
||||
and comp_name not in NON_EMULATOR_COMPONENTS
|
||||
})
|
||||
|
||||
systems: dict[str, dict] = {}
|
||||
for req in requirements:
|
||||
systems.setdefault(req.system, {"files": []})
|
||||
entry: dict = {
|
||||
"name": req.name,
|
||||
for req in reqs:
|
||||
sys_entry = systems.setdefault(req.system, {"files": []})
|
||||
file_entry: dict = {
|
||||
"name": req.name,
|
||||
"destination": req.destination,
|
||||
"required": req.required,
|
||||
"required": req.required,
|
||||
}
|
||||
if req.md5:
|
||||
entry["md5"] = req.md5
|
||||
sha256 = getattr(req, "_sha256", None)
|
||||
if sha256 and not req.md5:
|
||||
entry["sha256"] = sha256
|
||||
systems[req.system]["files"].append(entry)
|
||||
|
||||
version = self._release_version or ""
|
||||
if not version:
|
||||
try:
|
||||
version = fetch_github_latest_version(COMPONENTS_REPO) or ""
|
||||
except (ConnectionError, OSError):
|
||||
pass
|
||||
file_entry["md5"] = req.md5
|
||||
sys_entry["files"].append(file_entry)
|
||||
|
||||
return {
|
||||
"platform": "RetroDECK",
|
||||
"version": version,
|
||||
"homepage": "https://retrodeck.net",
|
||||
"source": f"https://github.com/{COMPONENTS_REPO}",
|
||||
"base_destination": "bios",
|
||||
"hash_type": "md5",
|
||||
"platform": "RetroDECK",
|
||||
"version": "",
|
||||
"homepage": "https://retrodeck.net",
|
||||
"source": "https://github.com/RetroDECK/components",
|
||||
"base_destination": "",
|
||||
"hash_type": "md5",
|
||||
"verification_mode": "md5",
|
||||
"systems": systems,
|
||||
"cores": cores,
|
||||
"systems": systems,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Scrape RetroDECK component_manifest.json BIOS requirements"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--manifests-dir", metavar="DIR",
|
||||
help=(
|
||||
"Read manifests from a local directory instead of fetching from GitHub. "
|
||||
f"Live install path: {DEFAULT_LOCAL_MANIFESTS}"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token", metavar="TOKEN",
|
||||
help="GitHub personal access token (or set GITHUB_TOKEN env var)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true",
|
||||
help="Print per-system summary without generating output",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", "-o", metavar="FILE",
|
||||
help="Write generated platform YAML to FILE",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json", action="store_true",
|
||||
help="Print platform config as JSON (for debugging)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
scraper = Scraper(manifests_dir=args.manifests_dir, github_token=args.token)
|
||||
|
||||
try:
|
||||
reqs = scraper.fetch_requirements()
|
||||
except (ConnectionError, FileNotFoundError) as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.dry_run:
|
||||
by_system: dict[str, list] = {}
|
||||
for r in reqs:
|
||||
by_system.setdefault(r.system, []).append(r)
|
||||
for system, files in sorted(by_system.items()):
|
||||
req_c = sum(1 for f in files if f.required)
|
||||
opt_c = len(files) - req_c
|
||||
print(f" {system}: {req_c} required, {opt_c} optional")
|
||||
print(f"\nTotal: {len(reqs)} entries across {len(by_system)} systems")
|
||||
return
|
||||
|
||||
config = scraper.generate_platform_yaml()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(config, indent=2))
|
||||
return
|
||||
|
||||
if args.output:
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("Error: PyYAML required (pip install pyyaml)", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
def _str_representer(dumper, data):
|
||||
if any(c in data for c in "()[]{}:#"):
|
||||
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')
|
||||
return dumper.represent_scalar("tag:yaml.org,2002:str", data)
|
||||
yaml.add_representer(str, _str_representer)
|
||||
|
||||
with open(args.output, "w") as f:
|
||||
yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
||||
total = sum(len(s["files"]) for s in config["systems"].values())
|
||||
print(
|
||||
f"Written {total} entries across "
|
||||
f"{len(config['systems'])} systems to {args.output}"
|
||||
)
|
||||
return
|
||||
|
||||
systems = len(set(r.system for r in reqs))
|
||||
print(f"Scraped {len(reqs)} entries across {systems} systems. Use --dry-run, --json, or --output FILE.")
|
||||
from scraper.base_scraper import scraper_cli
|
||||
scraper_cli(Scraper, "Scrape RetroDECK BIOS requirements")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user