mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-16 14:47:49 -06:00
feat: scrape the rom folders of each setup
This commit is contained in:
1 parent
249b189ec9
commit
1aa36258b2
102 files changed
+49969
No files matched your search
@@ -0,0 +1,193 @@
|
||||
"""Scrape each setup's rom folders from its own source, at a pinned revision.
|
||||
|
||||
`platforms/romfolders/_sources/<setup>.yml` names the repositories and
|
||||
the commits to read, the format of the system list, and the facts a parser
|
||||
cannot read from a table (roms root, case rule, folders the OS never scans),
|
||||
each with its `source_ref`. A format parser reads the system list and returns
|
||||
one row per folder. The setup file it writes carries the OS's own names only:
|
||||
`scripts/romfolders.py` resolves them to repository systems and profiles.
|
||||
|
||||
Every fetch goes through `upstream.fetch_file`, cached by revision, so a
|
||||
second run reads nothing from the network and writes the same bytes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = str(Path(__file__).resolve().parents[2])
|
||||
if _SCRIPTS_DIR not in sys.path:
|
||||
sys.path.insert(0, _SCRIPTS_DIR)
|
||||
|
||||
import upstream # noqa: E402
|
||||
from safeparse import yaml_load # noqa: E402
|
||||
|
||||
CACHE_DIR = upstream.CACHE_DIR
|
||||
|
||||
# Keys a setup file carries, in the order they are written.
|
||||
SETUP_KEYS = (
|
||||
"setup", "name", "platform", "sources", "roms_root", "case_sensitive",
|
||||
"case_ref", "skip", "hidden", "media", "notes", "folders",
|
||||
)
|
||||
ROW_KEYS = (
|
||||
"folder", "name", "aliases", "extensions", "core", "emulator", "source_ref",
|
||||
"board_defaults", "arcade", "note",
|
||||
)
|
||||
|
||||
|
||||
class SourceError(RuntimeError):
|
||||
"""A pinned file is absent, or its content no longer has the shape read."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Fetcher:
|
||||
"""Reads pinned files of one setup's declared sources."""
|
||||
|
||||
sources: dict[str, dict]
|
||||
cache_dir: str = CACHE_DIR
|
||||
offline: bool = False
|
||||
|
||||
def _repo(self, key: str) -> tuple[upstream.Repo, str]:
|
||||
if key not in self.sources:
|
||||
raise SourceError(f"source {key!r} is not declared")
|
||||
spec = self.sources[key]
|
||||
repo = upstream.parse_repo(spec["repo"])
|
||||
if repo is None:
|
||||
raise SourceError(f"source {key!r}: unsupported forge {spec['repo']}")
|
||||
return repo, spec["commit"]
|
||||
|
||||
def text(self, key: str, path: str) -> str:
|
||||
"""One file at the pinned commit, or SourceError when absent."""
|
||||
repo, sha = self._repo(key)
|
||||
lines = upstream.fetch_file(repo, sha, path, self.cache_dir, self.offline)
|
||||
if lines is None:
|
||||
raise SourceError(f"{repo.slug}@{sha[:8]}:{path} does not exist")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
def optional_text(self, key: str, path: str) -> str | None:
|
||||
repo, sha = self._repo(key)
|
||||
lines = upstream.fetch_file(repo, sha, path, self.cache_dir, self.offline)
|
||||
return None if lines is None else "\n".join(lines) + "\n"
|
||||
|
||||
def tree(self, key: str, prefix: str = "") -> list[str]:
|
||||
"""Every blob path under a prefix at the pinned commit."""
|
||||
repo, sha = self._repo(key)
|
||||
if repo.family == "gitlab" and prefix.strip("/"):
|
||||
return self._gitlab_subtree(repo, sha, prefix)
|
||||
paths, truncated = upstream.list_tree(repo, sha, self.cache_dir, self.offline)
|
||||
if truncated:
|
||||
raise SourceError(f"{repo.slug}@{sha[:8]}: the forge truncated the tree")
|
||||
return sorted(p for p in paths if p.startswith(prefix))
|
||||
|
||||
def entries(self, key: str, directory: str) -> list[str]:
|
||||
"""Paths of the files and folders directly under one directory."""
|
||||
repo, sha = self._repo(key)
|
||||
if repo.family != "gitlab":
|
||||
prefix = directory.strip("/") + "/"
|
||||
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in self.tree(key, prefix)})
|
||||
return self._gitlab_listing(repo, sha, directory, recursive=False, blobs=False)
|
||||
|
||||
def _gitlab_subtree(self, repo: upstream.Repo, sha: str, prefix: str) -> list[str]:
|
||||
"""GitLab pages its tree 100 entries at a time and Recalbox's whole
|
||||
tree outruns the page cap: list only the directory asked for."""
|
||||
paths = self._gitlab_listing(repo, sha, prefix, recursive=True, blobs=True)
|
||||
return sorted(p for p in paths if p.startswith(prefix))
|
||||
|
||||
def _gitlab_listing(
|
||||
self, repo: upstream.Repo, sha: str, directory: str, *, recursive: bool, blobs: bool
|
||||
) -> list[str]:
|
||||
directory = directory.strip("/")
|
||||
paths: list[str] = []
|
||||
for page in range(1, upstream.MAX_TREE_PAGES + 1):
|
||||
url = (
|
||||
f"{repo.api_base}/projects/{upstream._project(repo)}/repository/tree"
|
||||
f"?ref={sha}&path={directory}&recursive={'true' if recursive else 'false'}"
|
||||
f"&per_page={upstream.TREE_PAGE_SIZE}&page={page}"
|
||||
)
|
||||
payload = upstream._api(url, self.cache_dir, self.offline)
|
||||
if not isinstance(payload, list):
|
||||
raise SourceError(f"{repo.slug}@{sha[:8]}: {directory} could not be listed")
|
||||
paths.extend(
|
||||
entry["path"] for entry in payload
|
||||
if isinstance(entry, dict) and entry.get("path")
|
||||
and (entry.get("type") == "blob" or not blobs)
|
||||
)
|
||||
if len(payload) < upstream.TREE_PAGE_SIZE:
|
||||
return sorted(paths)
|
||||
raise SourceError(f"{repo.slug}@{sha[:8]}: {directory} outruns the page cap")
|
||||
|
||||
|
||||
Parser = Callable[[dict, Fetcher], list[dict]]
|
||||
FORMATS: dict[str, Parser] = {}
|
||||
|
||||
|
||||
def register(name: str) -> Callable[[Parser], Parser]:
|
||||
def wrap(func: Parser) -> Parser:
|
||||
if name in FORMATS:
|
||||
raise ValueError(f"format {name!r} registered twice")
|
||||
FORMATS[name] = func
|
||||
return func
|
||||
|
||||
return wrap
|
||||
|
||||
|
||||
def load_formats() -> dict[str, Parser]:
|
||||
"""Import every parser module of this package once."""
|
||||
if not FORMATS:
|
||||
for info in pkgutil.iter_modules([str(Path(__file__).parent)]):
|
||||
if not info.name.startswith("_"):
|
||||
importlib.import_module(f"{__name__}.{info.name}")
|
||||
return FORMATS
|
||||
|
||||
|
||||
def load_sources(directory: Path) -> dict[str, dict]:
|
||||
"""Each setup's declared sources and facts, keyed by its file name."""
|
||||
out: dict[str, dict] = {}
|
||||
for path in sorted(Path(directory).glob("*.yml")):
|
||||
with path.open(encoding="utf-8") as handle:
|
||||
out[path.stem] = yaml_load(handle) or {}
|
||||
return out
|
||||
|
||||
|
||||
def _clean_row(row: dict) -> dict:
|
||||
unknown = set(row) - set(ROW_KEYS)
|
||||
if unknown:
|
||||
raise ValueError(f"folder {row.get('folder')!r}: unknown keys {sorted(unknown)}")
|
||||
out = {key: row[key] for key in ROW_KEYS if row.get(key) not in (None, [], {}, "")}
|
||||
if out.get("name") == out["folder"]:
|
||||
del out["name"]
|
||||
if "extensions" in out:
|
||||
out["extensions"] = sorted({e.lower() for e in out["extensions"]})
|
||||
if "aliases" in out:
|
||||
out["aliases"] = sorted(set(out["aliases"]) - {out["folder"]})
|
||||
if not out["aliases"]:
|
||||
del out["aliases"]
|
||||
return out
|
||||
|
||||
|
||||
def scrape_setup(setup: str, spec: dict, fetcher: Fetcher) -> dict:
|
||||
"""The setup document: declared facts, then the parsed folders."""
|
||||
formats = load_formats()
|
||||
fmt = spec.get("format")
|
||||
rows: list[dict] = []
|
||||
if fmt:
|
||||
if fmt not in formats:
|
||||
raise SourceError(f"{setup}: no parser for format {fmt!r}")
|
||||
rows = formats[fmt](spec, fetcher)
|
||||
if not rows and not spec.get("allow_empty"):
|
||||
raise SourceError(f"{setup}: format {fmt} read no folder")
|
||||
folders: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
clean = _clean_row(row)
|
||||
key = clean["folder"]
|
||||
if key in folders:
|
||||
raise SourceError(f"{setup}: folder {key!r} read twice")
|
||||
folders[key] = clean
|
||||
doc = {"setup": setup, **{k: spec[k] for k in SETUP_KEYS[1:-1] if k in spec}}
|
||||
doc["folders"] = [folders[k] for k in sorted(folders, key=lambda f: (f.lower(), f))]
|
||||
return doc
|
||||
Reference in new issue
Block a user