mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-16 06:44:43 -06:00
feat: scrape recalbox rom folders per board
GitLab refused the hundred raw reads per IP. A GitLab directory is now read from one archive request, else a shallow sparse git fetch, and the cache keeps its file list so offline runs know what is absent. Each board builds system.ini against its defconfig: a core whose package the board does not enable is dropped (pisnes was the SNES default everywhere), and a system with no core is not built. The row names the default most boards share, board_defaults the others. A files: list names whole files and is carried as files through the index and the installer. A folder no name resolves, run by a core of several machines, now holds those machines: thomson, multivision and pcv2 had no fallback, so their games got no file. Arcade-indexed cores keep naming files per set.
This commit is contained in:
1 parent
a05fc599d9
commit
d38cd58036
64 files changed
+2516
-103
No files matched your search
@@ -256,18 +256,28 @@ def _arcade_profiles(directory: Path) -> set[str]:
|
||||
|
||||
|
||||
def resolve_folder(
|
||||
row: dict, platform: str, profile: str | None, resolver: SystemResolver, profiles: Profiles
|
||||
row: dict, platform: str, profile: str | None, resolver: SystemResolver, profiles: Profiles,
|
||||
arcade: frozenset[str] | set[str] = frozenset(),
|
||||
) -> Resolution:
|
||||
"""The systems a folder holds: its names first, else its core's one machine.
|
||||
"""The systems a folder holds: its names first, else its core's machines.
|
||||
|
||||
A name nothing spells, run by a core of one machine, is that machine
|
||||
(ECWolf's wolf3d folder, a port's own folder).
|
||||
(ECWolf's wolf3d folder, a port's own folder). Run by a core of several,
|
||||
its games are one of them (theodore's thomson folder), and every one
|
||||
is kept: left empty, the folder had no fallback and its games got no file.
|
||||
An arcade core is the exception: the arcade index names each set's
|
||||
files, and its hundred machines would make any unknown file a fallback
|
||||
for all of them.
|
||||
"""
|
||||
resolution = resolver.resolve(folder_names(row), platform)
|
||||
own = [str(s) for s in profiles.profiles.get(profile or "", {}).get("systems") or []]
|
||||
if not resolution.systems and len(own) == 1:
|
||||
if resolution.systems or not own:
|
||||
return resolution
|
||||
if len(own) == 1:
|
||||
return Resolution([resolver.canonical(own[0])], f"only system of {profile}")
|
||||
return resolution
|
||||
if profile in arcade:
|
||||
return resolution
|
||||
return Resolution(sorted({resolver.canonical(s) for s in own}), f"systems of {profile}")
|
||||
|
||||
|
||||
def build_setups(
|
||||
@@ -297,13 +307,13 @@ def build_setups(
|
||||
for row in doc.get("folders", []):
|
||||
counts["folders"] += 1
|
||||
profile = cores.resolve(row.get("core"))
|
||||
resolution = resolve_folder(row, platform, profile, resolver, profiles)
|
||||
resolution = resolve_folder(row, platform, profile, resolver, profiles, arcade)
|
||||
published: dict = {
|
||||
"folder": row["folder"],
|
||||
"systems": resolution.systems,
|
||||
"basis": resolution.basis,
|
||||
}
|
||||
for key in ("name", "aliases", "extensions", "core", "emulator", "note"):
|
||||
for key in ("name", "aliases", "extensions", "files", "core", "emulator", "note"):
|
||||
if row.get(key):
|
||||
published[key] = row[key]
|
||||
if profile:
|
||||
|
||||
@@ -17,7 +17,7 @@ import importlib
|
||||
import pkgutil
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = str(Path(__file__).resolve().parents[2])
|
||||
@@ -35,7 +35,7 @@ SETUP_KEYS = (
|
||||
"case_ref", "skip", "hidden", "media", "notes", "folders",
|
||||
)
|
||||
ROW_KEYS = (
|
||||
"folder", "name", "aliases", "extensions", "core", "emulator", "source_ref",
|
||||
"folder", "name", "aliases", "extensions", "files", "core", "emulator", "source_ref",
|
||||
"board_defaults", "arcade", "note",
|
||||
)
|
||||
|
||||
@@ -51,6 +51,7 @@ class Fetcher:
|
||||
sources: dict[str, dict]
|
||||
cache_dir: str = CACHE_DIR
|
||||
offline: bool = False
|
||||
_cached_dirs: dict[str, dict[str, set[str]]] = field(default_factory=dict, init=False, repr=False)
|
||||
|
||||
def _repo(self, key: str) -> tuple[upstream.Repo, str]:
|
||||
if key not in self.sources:
|
||||
@@ -71,6 +72,9 @@ class Fetcher:
|
||||
|
||||
def optional_text(self, key: str, path: str) -> str | None:
|
||||
repo, sha = self._repo(key)
|
||||
for directory, held in self._cached_dirs.get(key, {}).items():
|
||||
if path.startswith(directory + "/") and path not in held:
|
||||
return None
|
||||
lines = upstream.fetch_file(repo, sha, path, self.cache_dir, self.offline)
|
||||
return None if lines is None else "\n".join(lines) + "\n"
|
||||
|
||||
@@ -84,9 +88,21 @@ class Fetcher:
|
||||
raise SourceError(f"{repo.slug}@{sha[:8]}: the forge truncated the tree")
|
||||
return sorted(p for p in paths if p.startswith(prefix))
|
||||
|
||||
def cache_directory(self, key: str, directory: str) -> None:
|
||||
"""Fetch a whole GitLab directory in one archive request before its
|
||||
files are read one by one. A no-op on other forges."""
|
||||
repo, sha = self._repo(key)
|
||||
held = upstream.cache_gitlab_directory(repo, sha, directory, self.cache_dir, self.offline)
|
||||
if held is not None:
|
||||
self._cached_dirs.setdefault(key, {})[directory.strip("/")] = held
|
||||
|
||||
def entries(self, key: str, directory: str) -> list[str]:
|
||||
"""Paths of the files and folders directly under one directory."""
|
||||
repo, sha = self._repo(key)
|
||||
held = self._cached_dirs.get(key, {}).get(directory.strip("/"))
|
||||
if held is not None:
|
||||
prefix = directory.strip("/") + "/"
|
||||
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in held})
|
||||
if repo.family != "gitlab":
|
||||
prefix = directory.strip("/") + "/"
|
||||
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in self.tree(key, prefix)})
|
||||
|
||||
@@ -27,6 +27,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import re
|
||||
import shlex
|
||||
from collections import Counter
|
||||
from collections.abc import Iterator
|
||||
|
||||
from safeparse import parse_untrusted_xml, yaml_load_upstream
|
||||
@@ -607,15 +608,72 @@ def _ini_sections(text: str) -> tuple[dict[str, dict[str, str]], dict[str, int]]
|
||||
return sections, lines
|
||||
|
||||
|
||||
# romfs2 systemholder.py reads core.0 to core.19.
|
||||
_RECALBOX_CORE_SLOTS = 20
|
||||
_RECALBOX_TARGET = "BR2_PACKAGE_RECALBOX_TARGET_"
|
||||
_ENABLED = re.compile(r"^(BR2_\w+)=y\s*$")
|
||||
# FolderData.cpp IsMatching: a list opening with files: holds whole file
|
||||
# names, each bounded by a space or an end of the list.
|
||||
_FILES_PREFIX = "files:"
|
||||
|
||||
|
||||
def _recalbox_names(listed: str) -> list[str]:
|
||||
"""The whole names of a files: list. A name may hold spaces
|
||||
("prince of persia (1990).zip"): a list with a word that has no dot
|
||||
is one name, not several."""
|
||||
words = listed.split()
|
||||
if words and all("." in word for word in words):
|
||||
return words
|
||||
return [listed.strip()] if listed.strip() else []
|
||||
|
||||
|
||||
def _recalbox_boards(fetcher: Fetcher, key: str, configs_dir: str) -> dict[str, tuple[str, dict[str, int]]]:
|
||||
"""Each board's tag and the packages its defconfig enables, with their
|
||||
lines. The tag is the lowercased BR2_PACKAGE_RECALBOX_TARGET_<tag>
|
||||
(romfs2 configin.py), the suffix a system.ini key or section takes."""
|
||||
fetcher.cache_directory(key, configs_dir)
|
||||
boards: dict[str, tuple[str, dict[str, int]]] = {}
|
||||
for path in fetcher.entries(key, configs_dir):
|
||||
if not path.endswith("_defconfig"):
|
||||
continue
|
||||
enabled: dict[str, int] = {}
|
||||
for number, line in enumerate(fetcher.text(key, path).splitlines(), 1):
|
||||
found = _ENABLED.match(line.strip())
|
||||
if found:
|
||||
enabled.setdefault(found.group(1), number)
|
||||
tags = [name[len(_RECALBOX_TARGET):].lower() for name in enabled if name.startswith(_RECALBOX_TARGET)]
|
||||
if tags:
|
||||
boards[tags[0]] = (path, enabled)
|
||||
if not boards:
|
||||
raise SourceError(f"{configs_dir}: no defconfig names a Recalbox target")
|
||||
return boards
|
||||
|
||||
|
||||
def _recalbox_get(
|
||||
sections: dict[str, dict[str, str]], lines: dict[str, int], section: str, key: str, board: str
|
||||
) -> tuple[str | None, int | None]:
|
||||
"""systemholder.py __get: the board's section, then the board's key,
|
||||
then the plain key."""
|
||||
for where, name in ((f"{section}.{board}", key), (section, f"{key}.{board}"), (section, key)):
|
||||
if name in sections.get(where, {}):
|
||||
return sections[where][name], lines.get(f"{where}/{name}")
|
||||
return None, None
|
||||
|
||||
|
||||
@register("recalbox")
|
||||
def recalbox(spec: dict, fetcher: Fetcher) -> list[dict]:
|
||||
"""One system.ini per system. The default core is the lowest
|
||||
`priority` (EmulatorManager), and `priority.<board>` changes it on that
|
||||
board. A `.ext!` extension is an archive the frontend hands over
|
||||
unopened."""
|
||||
"""One system.ini per system, built once per board. A core is kept on
|
||||
a board whose defconfig enables its `package`, and a system left with
|
||||
none is not built there (romfs2). The default core is the lowest
|
||||
`priority` (EmulatorManager), the last listed on a tie. The row names the
|
||||
default most boards share, and `board_defaults` every board that differs.
|
||||
A `.ext!` extension is an archive the frontend hands over unopened, and
|
||||
a `files:` list names whole files instead of extensions."""
|
||||
params = spec["params"]
|
||||
key = params.get("source", "os")
|
||||
boards = _recalbox_boards(fetcher, key, params["configs_dir"])
|
||||
rows: list[dict] = []
|
||||
fetcher.cache_directory(key, params["systems_dir"])
|
||||
for entry in fetcher.entries(key, params["systems_dir"]):
|
||||
path = f"{entry}/system.ini"
|
||||
text = fetcher.optional_text(key, path)
|
||||
@@ -626,42 +684,53 @@ def recalbox(spec: dict, fetcher: Fetcher) -> list[dict]:
|
||||
folder = _strip_prefix(system.get("roms.folder", ""), ["%ROOT%/"])
|
||||
if not folder:
|
||||
continue
|
||||
cores = []
|
||||
for section, values in sections.items():
|
||||
parts = section.split(".")
|
||||
if parts[0] != "core" or len(parts) != 2:
|
||||
continue
|
||||
cores.append({
|
||||
"section": section, "emulator": values.get("emulator"), "core": values.get("core"),
|
||||
"priority": int(values.get("priority", "255")),
|
||||
"boards": {k.split(".", 1)[1]: int(v) for k, v in values.items() if k.startswith("priority.")},
|
||||
"extensions": values.get("extensions", "").split(),
|
||||
})
|
||||
if not cores:
|
||||
picks: dict[str, dict] = {}
|
||||
extensions: set[str] = set()
|
||||
names: set[str] = set()
|
||||
for board, (config_path, enabled) in sorted(boards.items()):
|
||||
cores = []
|
||||
for slot in range(_RECALBOX_CORE_SLOTS):
|
||||
section = f"core.{slot}"
|
||||
if section not in sections and f"{section}.{board}" not in sections:
|
||||
continue
|
||||
package, _ = _recalbox_get(sections, lines, section, "package", board)
|
||||
if package not in enabled:
|
||||
continue
|
||||
priority, priority_line = _recalbox_get(sections, lines, section, "priority", board)
|
||||
core, core_line = _recalbox_get(sections, lines, section, "core", board)
|
||||
emulator, _ = _recalbox_get(sections, lines, section, "emulator", board)
|
||||
exts, _ = _recalbox_get(sections, lines, section, "extensions", board)
|
||||
if (exts or "").startswith(_FILES_PREFIX):
|
||||
names.update(_recalbox_names(exts[len(_FILES_PREFIX):]))
|
||||
else:
|
||||
extensions.update((exts or "").split())
|
||||
cores.append({
|
||||
"core": core, "emulator": emulator, "priority": int(priority or "255"),
|
||||
"source_ref": "; ".join([
|
||||
_ref(key, path, priority_line, core_line),
|
||||
_ref(key, config_path, enabled[package]),
|
||||
]),
|
||||
})
|
||||
if cores:
|
||||
picks[board] = min(reversed(cores), key=lambda c: c["priority"])
|
||||
if not picks:
|
||||
continue
|
||||
# EmulatorManager walks the cores from the last and keeps a strictly
|
||||
# lower priority: on a tie the last core listed wins.
|
||||
pick = min(reversed(cores), key=lambda c: c["priority"])
|
||||
board_defaults: dict[str, dict] = {}
|
||||
for board in sorted({b for c in cores for b in c["boards"]}):
|
||||
best = min(reversed(cores), key=lambda c: c["boards"].get(board, c["priority"]))
|
||||
if best is not pick:
|
||||
board_defaults[board] = _board(
|
||||
best["core"], best["emulator"],
|
||||
_ref(key, path, lines.get(f"{best['section']}/priority.{board}")))
|
||||
extensions = sorted({e for c in cores for e in c["extensions"]})
|
||||
shared = Counter((pick["core"], pick["emulator"]) for pick in picks.values())
|
||||
core, emulator = max(shared, key=lambda pair: shared[pair])
|
||||
first = next(pick for pick in picks.values() if (pick["core"], pick["emulator"]) == (core, emulator))
|
||||
board_defaults = {
|
||||
board: _board(pick["core"], pick["emulator"], pick["source_ref"])
|
||||
for board, pick in picks.items() if (pick["core"], pick["emulator"]) != (core, emulator)
|
||||
}
|
||||
archive = any(e.endswith("!") for e in extensions)
|
||||
rows.append({
|
||||
"folder": folder,
|
||||
"name": system.get("name"),
|
||||
"extensions": [e.rstrip("!") for e in extensions],
|
||||
"core": pick["core"],
|
||||
"emulator": pick["emulator"],
|
||||
"source_ref": "; ".join([
|
||||
_ref(key, path, lines.get("system/roms.folder")),
|
||||
_ref(key, path, lines.get(f"{pick['section']}/priority"),
|
||||
lines.get(f"{pick['section']}/core")),
|
||||
]),
|
||||
"extensions": sorted({e.rstrip("!") for e in extensions}),
|
||||
"files": sorted(names),
|
||||
"core": core,
|
||||
"emulator": emulator,
|
||||
"source_ref": "; ".join([_ref(key, path, lines.get("system/roms.folder")), first["source_ref"]]),
|
||||
"board_defaults": board_defaults,
|
||||
"arcade": sections.get("properties", {}).get("type") == "arcade" or None,
|
||||
"note": "an extension ending in ! is an archive the frontend does not open" if archive else None,
|
||||
|
||||
+87
-1
@@ -9,10 +9,13 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -246,7 +249,9 @@ def _http_failure(url: str, exc: urllib.error.HTTPError) -> UpstreamError:
|
||||
"""
|
||||
host = urllib.parse.urlsplit(url).hostname or ""
|
||||
if exc.code == 429:
|
||||
return RateLimitError(f"{url}: HTTP 429", host)
|
||||
retry_after = (exc.headers.get("Retry-After") or "").strip() if exc.headers else ""
|
||||
wait = f", retry after {retry_after} s" if retry_after.isdigit() else ""
|
||||
return RateLimitError(f"{url}: HTTP 429{wait}", host)
|
||||
if exc.code in (410, 451):
|
||||
reason = "withdrawn for legal reasons" if exc.code == 451 else "gone"
|
||||
return GoneError(f"{url}: HTTP {exc.code}, {reason}")
|
||||
@@ -954,6 +959,87 @@ def _gitlab_tree(
|
||||
return paths, True
|
||||
|
||||
|
||||
def cache_gitlab_directory(
|
||||
repo: Repo, sha: str, directory: str, cache_dir: str, offline: bool = False
|
||||
) -> set[str] | None:
|
||||
"""Every file under one GitLab directory, cached from one request.
|
||||
|
||||
GitLab counts anonymous requests per IP: reading Recalbox's hundred
|
||||
system.ini files one raw URL at a time ran into HTTP 429 before the
|
||||
end. The archive endpoint serves the directory in one request, and when
|
||||
the API itself is refused, a shallow sparse git fetch does, since GitLab
|
||||
does not count git traffic against that quota. Each file goes to the
|
||||
cache `fetch_file` reads, and the list of paths is kept so a later run,
|
||||
offline included, knows what is absent. None when the list is not
|
||||
known (offline, never fetched).
|
||||
"""
|
||||
directory = directory.strip("/")
|
||||
index = cache_path(cache_dir, repo, sha, f".archives/{directory}.json")
|
||||
if index.is_file():
|
||||
return set(json.loads(index.read_text(encoding="utf-8")))
|
||||
if offline or repo.family != "gitlab":
|
||||
return None
|
||||
try:
|
||||
files = _gitlab_archive_files(repo, sha, directory)
|
||||
except RateLimitError:
|
||||
files = _sparse_git_files(repo, sha, directory, cache_dir)
|
||||
if not files:
|
||||
raise UpstreamError(f"{repo.slug}@{sha[:8]}: nothing under {directory}")
|
||||
for path, text in files.items():
|
||||
write_cache(cache_path(cache_dir, repo, sha, path), text)
|
||||
write_cache(index, json.dumps(sorted(files)))
|
||||
return set(files)
|
||||
|
||||
|
||||
def _gitlab_archive_files(repo: Repo, sha: str, directory: str) -> dict[str, str]:
|
||||
url = (
|
||||
f"{repo.api_base}/projects/{_project(repo)}/repository/archive.tar.gz"
|
||||
f"?sha={sha}&path={urllib.parse.quote(directory)}"
|
||||
)
|
||||
body = _fetch(url)
|
||||
if body is None:
|
||||
return {}
|
||||
prefix = directory + "/"
|
||||
files: dict[str, str] = {}
|
||||
with tarfile.open(fileobj=io.BytesIO(body), mode="r:gz") as archive:
|
||||
for member in archive:
|
||||
if not member.isfile():
|
||||
continue
|
||||
# The archive wraps everything in one <project>-<sha>-<path> folder.
|
||||
inner = member.name.split("/", 1)[-1]
|
||||
handle = archive.extractfile(member) if inner.startswith(prefix) else None
|
||||
if handle is not None:
|
||||
files[inner] = handle.read().decode("utf-8", errors="replace")
|
||||
return files
|
||||
|
||||
|
||||
def _sparse_git_files(repo: Repo, sha: str, directory: str, cache_dir: str) -> dict[str, str]:
|
||||
"""One directory at one commit through git: depth 1, blobs of that
|
||||
directory only, in a scratch work tree removed afterwards."""
|
||||
url = f"https://{repo.host}/{repo.slug}.git"
|
||||
Path(cache_dir).mkdir(parents=True, exist_ok=True)
|
||||
work = Path(tempfile.mkdtemp(dir=cache_dir, prefix=".sparse-"))
|
||||
try:
|
||||
steps = (
|
||||
["init", "--quiet"],
|
||||
["remote", "add", "origin", url],
|
||||
["sparse-checkout", "set", "--no-cone", f"/{directory}/"],
|
||||
["fetch", "--quiet", "--depth", "1", "--filter=blob:none", "origin", sha],
|
||||
["checkout", "--quiet", "FETCH_HEAD"],
|
||||
)
|
||||
for argv in steps:
|
||||
result = _git(argv, work)
|
||||
if result.returncode:
|
||||
raise UpstreamError(f"{url}: git {argv[0]} failed: {result.stderr.strip()}")
|
||||
root = work / directory
|
||||
return {
|
||||
f"{directory}/{p.relative_to(root).as_posix()}": p.read_text(encoding="utf-8", errors="replace")
|
||||
for p in sorted(root.rglob("*")) if p.is_file()
|
||||
}
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
|
||||
|
||||
def list_tree(
|
||||
repo: Repo, sha: str, cache_dir: str, offline: bool = False
|
||||
) -> tuple[list[str], bool]:
|
||||
|
||||
Reference in new issue
Block a user