feat: scrape recalbox rom folders per board

GitLab refused the hundred raw reads per IP. A GitLab directory is now
read from one archive request, else a shallow sparse git fetch, and the
cache keeps its file list so offline runs know what is absent.

Each board builds system.ini against its defconfig: a core whose package
the board does not enable is dropped (pisnes was the SNES default
everywhere), and a system with no core is not built. The row names the
default most boards share, board_defaults the others. A files: list
names whole files and is carried as files through the index and the
installer.

A folder no name resolves, run by a core of several machines, now holds
those machines: thomson, multivision and pcv2 had no fallback, so their
games got no file. Arcade-indexed cores keep naming files per set.
This commit is contained in:
Abdessamad Derraz committed 2026-11-04 16:48:34 +01:00
1 parent a05fc599d9
commit d38cd58036
64 files changed
+2516 -103

No files matched your search

+17 -7
View File
@@ -256,18 +256,28 @@ def _arcade_profiles(directory: Path) -> set[str]:
def resolve_folder(
row: dict, platform: str, profile: str | None, resolver: SystemResolver, profiles: Profiles
row: dict, platform: str, profile: str | None, resolver: SystemResolver, profiles: Profiles,
arcade: frozenset[str] | set[str] = frozenset(),
) -> Resolution:
"""The systems a folder holds: its names first, else its core's one machine.
"""The systems a folder holds: its names first, else its core's machines.
A name nothing spells, run by a core of one machine, is that machine
(ECWolf's wolf3d folder, a port's own folder).
(ECWolf's wolf3d folder, a port's own folder). Run by a core of several,
its games are one of them (theodore's thomson folder), and every one
is kept: left empty, the folder had no fallback and its games got no file.
An arcade core is the exception: the arcade index names each set's
files, and its hundred machines would make any unknown file a fallback
for all of them.
"""
resolution = resolver.resolve(folder_names(row), platform)
own = [str(s) for s in profiles.profiles.get(profile or "", {}).get("systems") or []]
if not resolution.systems and len(own) == 1:
if resolution.systems or not own:
return resolution
if len(own) == 1:
return Resolution([resolver.canonical(own[0])], f"only system of {profile}")
return resolution
if profile in arcade:
return resolution
return Resolution(sorted({resolver.canonical(s) for s in own}), f"systems of {profile}")
def build_setups(
@@ -297,13 +307,13 @@ def build_setups(
for row in doc.get("folders", []):
counts["folders"] += 1
profile = cores.resolve(row.get("core"))
resolution = resolve_folder(row, platform, profile, resolver, profiles)
resolution = resolve_folder(row, platform, profile, resolver, profiles, arcade)
published: dict = {
"folder": row["folder"],
"systems": resolution.systems,
"basis": resolution.basis,
}
for key in ("name", "aliases", "extensions", "core", "emulator", "note"):
for key in ("name", "aliases", "extensions", "files", "core", "emulator", "note"):
if row.get(key):
published[key] = row[key]
if profile:
+18 -2
View File
@@ -17,7 +17,7 @@ import importlib
import pkgutil
import sys
from collections.abc import Callable
from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path
_SCRIPTS_DIR = str(Path(__file__).resolve().parents[2])
@@ -35,7 +35,7 @@ SETUP_KEYS = (
"case_ref", "skip", "hidden", "media", "notes", "folders",
)
ROW_KEYS = (
"folder", "name", "aliases", "extensions", "core", "emulator", "source_ref",
"folder", "name", "aliases", "extensions", "files", "core", "emulator", "source_ref",
"board_defaults", "arcade", "note",
)
@@ -51,6 +51,7 @@ class Fetcher:
sources: dict[str, dict]
cache_dir: str = CACHE_DIR
offline: bool = False
_cached_dirs: dict[str, dict[str, set[str]]] = field(default_factory=dict, init=False, repr=False)
def _repo(self, key: str) -> tuple[upstream.Repo, str]:
if key not in self.sources:
@@ -71,6 +72,9 @@ class Fetcher:
def optional_text(self, key: str, path: str) -> str | None:
repo, sha = self._repo(key)
for directory, held in self._cached_dirs.get(key, {}).items():
if path.startswith(directory + "/") and path not in held:
return None
lines = upstream.fetch_file(repo, sha, path, self.cache_dir, self.offline)
return None if lines is None else "\n".join(lines) + "\n"
@@ -84,9 +88,21 @@ class Fetcher:
raise SourceError(f"{repo.slug}@{sha[:8]}: the forge truncated the tree")
return sorted(p for p in paths if p.startswith(prefix))
def cache_directory(self, key: str, directory: str) -> None:
"""Fetch a whole GitLab directory in one archive request before its
files are read one by one. A no-op on other forges."""
repo, sha = self._repo(key)
held = upstream.cache_gitlab_directory(repo, sha, directory, self.cache_dir, self.offline)
if held is not None:
self._cached_dirs.setdefault(key, {})[directory.strip("/")] = held
def entries(self, key: str, directory: str) -> list[str]:
"""Paths of the files and folders directly under one directory."""
repo, sha = self._repo(key)
held = self._cached_dirs.get(key, {}).get(directory.strip("/"))
if held is not None:
prefix = directory.strip("/") + "/"
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in held})
if repo.family != "gitlab":
prefix = directory.strip("/") + "/"
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in self.tree(key, prefix)})
+104 -35
View File
@@ -27,6 +27,7 @@ from __future__ import annotations
import json
import re
import shlex
from collections import Counter
from collections.abc import Iterator
from safeparse import parse_untrusted_xml, yaml_load_upstream
@@ -607,15 +608,72 @@ def _ini_sections(text: str) -> tuple[dict[str, dict[str, str]], dict[str, int]]
return sections, lines
# romfs2 systemholder.py reads core.0 to core.19.
_RECALBOX_CORE_SLOTS = 20
_RECALBOX_TARGET = "BR2_PACKAGE_RECALBOX_TARGET_"
_ENABLED = re.compile(r"^(BR2_\w+)=y\s*$")
# FolderData.cpp IsMatching: a list opening with files: holds whole file
# names, each bounded by a space or an end of the list.
_FILES_PREFIX = "files:"
def _recalbox_names(listed: str) -> list[str]:
"""The whole names of a files: list. A name may hold spaces
("prince of persia (1990).zip"): a list with a word that has no dot
is one name, not several."""
words = listed.split()
if words and all("." in word for word in words):
return words
return [listed.strip()] if listed.strip() else []
def _recalbox_boards(fetcher: Fetcher, key: str, configs_dir: str) -> dict[str, tuple[str, dict[str, int]]]:
"""Each board's tag and the packages its defconfig enables, with their
lines. The tag is the lowercased BR2_PACKAGE_RECALBOX_TARGET_<tag>
(romfs2 configin.py), the suffix a system.ini key or section takes."""
fetcher.cache_directory(key, configs_dir)
boards: dict[str, tuple[str, dict[str, int]]] = {}
for path in fetcher.entries(key, configs_dir):
if not path.endswith("_defconfig"):
continue
enabled: dict[str, int] = {}
for number, line in enumerate(fetcher.text(key, path).splitlines(), 1):
found = _ENABLED.match(line.strip())
if found:
enabled.setdefault(found.group(1), number)
tags = [name[len(_RECALBOX_TARGET):].lower() for name in enabled if name.startswith(_RECALBOX_TARGET)]
if tags:
boards[tags[0]] = (path, enabled)
if not boards:
raise SourceError(f"{configs_dir}: no defconfig names a Recalbox target")
return boards
def _recalbox_get(
sections: dict[str, dict[str, str]], lines: dict[str, int], section: str, key: str, board: str
) -> tuple[str | None, int | None]:
"""systemholder.py __get: the board's section, then the board's key,
then the plain key."""
for where, name in ((f"{section}.{board}", key), (section, f"{key}.{board}"), (section, key)):
if name in sections.get(where, {}):
return sections[where][name], lines.get(f"{where}/{name}")
return None, None
@register("recalbox")
def recalbox(spec: dict, fetcher: Fetcher) -> list[dict]:
"""One system.ini per system. The default core is the lowest
`priority` (EmulatorManager), and `priority.<board>` changes it on that
board. A `.ext!` extension is an archive the frontend hands over
unopened."""
"""One system.ini per system, built once per board. A core is kept on
a board whose defconfig enables its `package`, and a system left with
none is not built there (romfs2). The default core is the lowest
`priority` (EmulatorManager), the last listed on a tie. The row names the
default most boards share, and `board_defaults` every board that differs.
A `.ext!` extension is an archive the frontend hands over unopened, and
a `files:` list names whole files instead of extensions."""
params = spec["params"]
key = params.get("source", "os")
boards = _recalbox_boards(fetcher, key, params["configs_dir"])
rows: list[dict] = []
fetcher.cache_directory(key, params["systems_dir"])
for entry in fetcher.entries(key, params["systems_dir"]):
path = f"{entry}/system.ini"
text = fetcher.optional_text(key, path)
@@ -626,42 +684,53 @@ def recalbox(spec: dict, fetcher: Fetcher) -> list[dict]:
folder = _strip_prefix(system.get("roms.folder", ""), ["%ROOT%/"])
if not folder:
continue
cores = []
for section, values in sections.items():
parts = section.split(".")
if parts[0] != "core" or len(parts) != 2:
continue
cores.append({
"section": section, "emulator": values.get("emulator"), "core": values.get("core"),
"priority": int(values.get("priority", "255")),
"boards": {k.split(".", 1)[1]: int(v) for k, v in values.items() if k.startswith("priority.")},
"extensions": values.get("extensions", "").split(),
})
if not cores:
picks: dict[str, dict] = {}
extensions: set[str] = set()
names: set[str] = set()
for board, (config_path, enabled) in sorted(boards.items()):
cores = []
for slot in range(_RECALBOX_CORE_SLOTS):
section = f"core.{slot}"
if section not in sections and f"{section}.{board}" not in sections:
continue
package, _ = _recalbox_get(sections, lines, section, "package", board)
if package not in enabled:
continue
priority, priority_line = _recalbox_get(sections, lines, section, "priority", board)
core, core_line = _recalbox_get(sections, lines, section, "core", board)
emulator, _ = _recalbox_get(sections, lines, section, "emulator", board)
exts, _ = _recalbox_get(sections, lines, section, "extensions", board)
if (exts or "").startswith(_FILES_PREFIX):
names.update(_recalbox_names(exts[len(_FILES_PREFIX):]))
else:
extensions.update((exts or "").split())
cores.append({
"core": core, "emulator": emulator, "priority": int(priority or "255"),
"source_ref": "; ".join([
_ref(key, path, priority_line, core_line),
_ref(key, config_path, enabled[package]),
]),
})
if cores:
picks[board] = min(reversed(cores), key=lambda c: c["priority"])
if not picks:
continue
# EmulatorManager walks the cores from the last and keeps a strictly
# lower priority: on a tie the last core listed wins.
pick = min(reversed(cores), key=lambda c: c["priority"])
board_defaults: dict[str, dict] = {}
for board in sorted({b for c in cores for b in c["boards"]}):
best = min(reversed(cores), key=lambda c: c["boards"].get(board, c["priority"]))
if best is not pick:
board_defaults[board] = _board(
best["core"], best["emulator"],
_ref(key, path, lines.get(f"{best['section']}/priority.{board}")))
extensions = sorted({e for c in cores for e in c["extensions"]})
shared = Counter((pick["core"], pick["emulator"]) for pick in picks.values())
core, emulator = max(shared, key=lambda pair: shared[pair])
first = next(pick for pick in picks.values() if (pick["core"], pick["emulator"]) == (core, emulator))
board_defaults = {
board: _board(pick["core"], pick["emulator"], pick["source_ref"])
for board, pick in picks.items() if (pick["core"], pick["emulator"]) != (core, emulator)
}
archive = any(e.endswith("!") for e in extensions)
rows.append({
"folder": folder,
"name": system.get("name"),
"extensions": [e.rstrip("!") for e in extensions],
"core": pick["core"],
"emulator": pick["emulator"],
"source_ref": "; ".join([
_ref(key, path, lines.get("system/roms.folder")),
_ref(key, path, lines.get(f"{pick['section']}/priority"),
lines.get(f"{pick['section']}/core")),
]),
"extensions": sorted({e.rstrip("!") for e in extensions}),
"files": sorted(names),
"core": core,
"emulator": emulator,
"source_ref": "; ".join([_ref(key, path, lines.get("system/roms.folder")), first["source_ref"]]),
"board_defaults": board_defaults,
"arcade": sections.get("properties", {}).get("type") == "arcade" or None,
"note": "an extension ending in ! is an archive the frontend does not open" if archive else None,
+87 -1
View File
@@ -9,10 +9,13 @@ from __future__ import annotations
import hashlib
import http.client
import io
import json
import os
import shutil
import socket
import subprocess
import tarfile
import tempfile
import time
import urllib.error
@@ -246,7 +249,9 @@ def _http_failure(url: str, exc: urllib.error.HTTPError) -> UpstreamError:
"""
host = urllib.parse.urlsplit(url).hostname or ""
if exc.code == 429:
return RateLimitError(f"{url}: HTTP 429", host)
retry_after = (exc.headers.get("Retry-After") or "").strip() if exc.headers else ""
wait = f", retry after {retry_after} s" if retry_after.isdigit() else ""
return RateLimitError(f"{url}: HTTP 429{wait}", host)
if exc.code in (410, 451):
reason = "withdrawn for legal reasons" if exc.code == 451 else "gone"
return GoneError(f"{url}: HTTP {exc.code}, {reason}")
@@ -954,6 +959,87 @@ def _gitlab_tree(
return paths, True
def cache_gitlab_directory(
repo: Repo, sha: str, directory: str, cache_dir: str, offline: bool = False
) -> set[str] | None:
"""Every file under one GitLab directory, cached from one request.
GitLab counts anonymous requests per IP: reading Recalbox's hundred
system.ini files one raw URL at a time ran into HTTP 429 before the
end. The archive endpoint serves the directory in one request, and when
the API itself is refused, a shallow sparse git fetch does, since GitLab
does not count git traffic against that quota. Each file goes to the
cache `fetch_file` reads, and the list of paths is kept so a later run,
offline included, knows what is absent. None when the list is not
known (offline, never fetched).
"""
directory = directory.strip("/")
index = cache_path(cache_dir, repo, sha, f".archives/{directory}.json")
if index.is_file():
return set(json.loads(index.read_text(encoding="utf-8")))
if offline or repo.family != "gitlab":
return None
try:
files = _gitlab_archive_files(repo, sha, directory)
except RateLimitError:
files = _sparse_git_files(repo, sha, directory, cache_dir)
if not files:
raise UpstreamError(f"{repo.slug}@{sha[:8]}: nothing under {directory}")
for path, text in files.items():
write_cache(cache_path(cache_dir, repo, sha, path), text)
write_cache(index, json.dumps(sorted(files)))
return set(files)
def _gitlab_archive_files(repo: Repo, sha: str, directory: str) -> dict[str, str]:
url = (
f"{repo.api_base}/projects/{_project(repo)}/repository/archive.tar.gz"
f"?sha={sha}&path={urllib.parse.quote(directory)}"
)
body = _fetch(url)
if body is None:
return {}
prefix = directory + "/"
files: dict[str, str] = {}
with tarfile.open(fileobj=io.BytesIO(body), mode="r:gz") as archive:
for member in archive:
if not member.isfile():
continue
# The archive wraps everything in one <project>-<sha>-<path> folder.
inner = member.name.split("/", 1)[-1]
handle = archive.extractfile(member) if inner.startswith(prefix) else None
if handle is not None:
files[inner] = handle.read().decode("utf-8", errors="replace")
return files
def _sparse_git_files(repo: Repo, sha: str, directory: str, cache_dir: str) -> dict[str, str]:
"""One directory at one commit through git: depth 1, blobs of that
directory only, in a scratch work tree removed afterwards."""
url = f"https://{repo.host}/{repo.slug}.git"
Path(cache_dir).mkdir(parents=True, exist_ok=True)
work = Path(tempfile.mkdtemp(dir=cache_dir, prefix=".sparse-"))
try:
steps = (
["init", "--quiet"],
["remote", "add", "origin", url],
["sparse-checkout", "set", "--no-cone", f"/{directory}/"],
["fetch", "--quiet", "--depth", "1", "--filter=blob:none", "origin", sha],
["checkout", "--quiet", "FETCH_HEAD"],
)
for argv in steps:
result = _git(argv, work)
if result.returncode:
raise UpstreamError(f"{url}: git {argv[0]} failed: {result.stderr.strip()}")
root = work / directory
return {
f"{directory}/{p.relative_to(root).as_posix()}": p.read_text(encoding="utf-8", errors="replace")
for p in sorted(root.rglob("*")) if p.is_file()
}
finally:
shutil.rmtree(work, ignore_errors=True)
def list_tree(
repo: Repo, sha: str, cache_dir: str, offline: bool = False
) -> tuple[list[str], bool]: