feat: scrape recalbox rom folders per board

GitLab refused the hundred raw reads per IP. A GitLab directory is now
read from one archive request, else a shallow sparse git fetch, and the
cache keeps its file list so offline runs know what is absent.

Each board builds system.ini against its defconfig: a core whose package
the board does not enable is dropped (pisnes was the SNES default
everywhere), and a system with no core is not built. The row names the
default most boards share, board_defaults the others. A files: list
names whole files and is carried as files through the index and the
installer.

A folder no name resolves, run by a core of several machines, now holds
those machines: thomson, multivision and pcv2 had no fallback, so their
games got no file. Arcade-indexed cores keep naming files per set.
This commit is contained in:
Abdessamad Derraz committed 2026-11-04 16:48:34 +01:00
1 parent a05fc599d9
commit d38cd58036
64 files changed
+2516 -103

No files matched your search

+18 -2
View File
@@ -17,7 +17,7 @@ import importlib
import pkgutil
import sys
from collections.abc import Callable
from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path
_SCRIPTS_DIR = str(Path(__file__).resolve().parents[2])
@@ -35,7 +35,7 @@ SETUP_KEYS = (
"case_ref", "skip", "hidden", "media", "notes", "folders",
)
ROW_KEYS = (
"folder", "name", "aliases", "extensions", "core", "emulator", "source_ref",
"folder", "name", "aliases", "extensions", "files", "core", "emulator", "source_ref",
"board_defaults", "arcade", "note",
)
@@ -51,6 +51,7 @@ class Fetcher:
sources: dict[str, dict]
cache_dir: str = CACHE_DIR
offline: bool = False
_cached_dirs: dict[str, dict[str, set[str]]] = field(default_factory=dict, init=False, repr=False)
def _repo(self, key: str) -> tuple[upstream.Repo, str]:
if key not in self.sources:
@@ -71,6 +72,9 @@ class Fetcher:
def optional_text(self, key: str, path: str) -> str | None:
repo, sha = self._repo(key)
for directory, held in self._cached_dirs.get(key, {}).items():
if path.startswith(directory + "/") and path not in held:
return None
lines = upstream.fetch_file(repo, sha, path, self.cache_dir, self.offline)
return None if lines is None else "\n".join(lines) + "\n"
@@ -84,9 +88,21 @@ class Fetcher:
raise SourceError(f"{repo.slug}@{sha[:8]}: the forge truncated the tree")
return sorted(p for p in paths if p.startswith(prefix))
def cache_directory(self, key: str, directory: str) -> None:
"""Fetch a whole GitLab directory in one archive request before its
files are read one by one. A no-op on other forges."""
repo, sha = self._repo(key)
held = upstream.cache_gitlab_directory(repo, sha, directory, self.cache_dir, self.offline)
if held is not None:
self._cached_dirs.setdefault(key, {})[directory.strip("/")] = held
def entries(self, key: str, directory: str) -> list[str]:
"""Paths of the files and folders directly under one directory."""
repo, sha = self._repo(key)
held = self._cached_dirs.get(key, {}).get(directory.strip("/"))
if held is not None:
prefix = directory.strip("/") + "/"
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in held})
if repo.family != "gitlab":
prefix = directory.strip("/") + "/"
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in self.tree(key, prefix)})
+104 -35
View File
@@ -27,6 +27,7 @@ from __future__ import annotations
import json
import re
import shlex
from collections import Counter
from collections.abc import Iterator
from safeparse import parse_untrusted_xml, yaml_load_upstream
@@ -607,15 +608,72 @@ def _ini_sections(text: str) -> tuple[dict[str, dict[str, str]], dict[str, int]]
return sections, lines
# romfs2 systemholder.py reads core.0 to core.19.
_RECALBOX_CORE_SLOTS = 20
_RECALBOX_TARGET = "BR2_PACKAGE_RECALBOX_TARGET_"
_ENABLED = re.compile(r"^(BR2_\w+)=y\s*$")
# FolderData.cpp IsMatching: a list opening with files: holds whole file
# names, each bounded by a space or an end of the list.
_FILES_PREFIX = "files:"
def _recalbox_names(listed: str) -> list[str]:
"""The whole names of a files: list. A name may hold spaces
("prince of persia (1990).zip"): a list with a word that has no dot
is one name, not several."""
words = listed.split()
if words and all("." in word for word in words):
return words
return [listed.strip()] if listed.strip() else []
def _recalbox_boards(fetcher: Fetcher, key: str, configs_dir: str) -> dict[str, tuple[str, dict[str, int]]]:
"""Each board's tag and the packages its defconfig enables, with their
lines. The tag is the lowercased BR2_PACKAGE_RECALBOX_TARGET_<tag>
(romfs2 configin.py), the suffix a system.ini key or section takes."""
fetcher.cache_directory(key, configs_dir)
boards: dict[str, tuple[str, dict[str, int]]] = {}
for path in fetcher.entries(key, configs_dir):
if not path.endswith("_defconfig"):
continue
enabled: dict[str, int] = {}
for number, line in enumerate(fetcher.text(key, path).splitlines(), 1):
found = _ENABLED.match(line.strip())
if found:
enabled.setdefault(found.group(1), number)
tags = [name[len(_RECALBOX_TARGET):].lower() for name in enabled if name.startswith(_RECALBOX_TARGET)]
if tags:
boards[tags[0]] = (path, enabled)
if not boards:
raise SourceError(f"{configs_dir}: no defconfig names a Recalbox target")
return boards
def _recalbox_get(
sections: dict[str, dict[str, str]], lines: dict[str, int], section: str, key: str, board: str
) -> tuple[str | None, int | None]:
"""systemholder.py __get: the board's section, then the board's key,
then the plain key."""
for where, name in ((f"{section}.{board}", key), (section, f"{key}.{board}"), (section, key)):
if name in sections.get(where, {}):
return sections[where][name], lines.get(f"{where}/{name}")
return None, None
@register("recalbox")
def recalbox(spec: dict, fetcher: Fetcher) -> list[dict]:
"""One system.ini per system. The default core is the lowest
`priority` (EmulatorManager), and `priority.<board>` changes it on that
board. A `.ext!` extension is an archive the frontend hands over
unopened."""
"""One system.ini per system, built once per board. A core is kept on
a board whose defconfig enables its `package`, and a system left with
none is not built there (romfs2). The default core is the lowest
`priority` (EmulatorManager), the last listed on a tie. The row names the
default most boards share, and `board_defaults` every board that differs.
A `.ext!` extension is an archive the frontend hands over unopened, and
a `files:` list names whole files instead of extensions."""
params = spec["params"]
key = params.get("source", "os")
boards = _recalbox_boards(fetcher, key, params["configs_dir"])
rows: list[dict] = []
fetcher.cache_directory(key, params["systems_dir"])
for entry in fetcher.entries(key, params["systems_dir"]):
path = f"{entry}/system.ini"
text = fetcher.optional_text(key, path)
@@ -626,42 +684,53 @@ def recalbox(spec: dict, fetcher: Fetcher) -> list[dict]:
folder = _strip_prefix(system.get("roms.folder", ""), ["%ROOT%/"])
if not folder:
continue
cores = []
for section, values in sections.items():
parts = section.split(".")
if parts[0] != "core" or len(parts) != 2:
continue
cores.append({
"section": section, "emulator": values.get("emulator"), "core": values.get("core"),
"priority": int(values.get("priority", "255")),
"boards": {k.split(".", 1)[1]: int(v) for k, v in values.items() if k.startswith("priority.")},
"extensions": values.get("extensions", "").split(),
})
if not cores:
picks: dict[str, dict] = {}
extensions: set[str] = set()
names: set[str] = set()
for board, (config_path, enabled) in sorted(boards.items()):
cores = []
for slot in range(_RECALBOX_CORE_SLOTS):
section = f"core.{slot}"
if section not in sections and f"{section}.{board}" not in sections:
continue
package, _ = _recalbox_get(sections, lines, section, "package", board)
if package not in enabled:
continue
priority, priority_line = _recalbox_get(sections, lines, section, "priority", board)
core, core_line = _recalbox_get(sections, lines, section, "core", board)
emulator, _ = _recalbox_get(sections, lines, section, "emulator", board)
exts, _ = _recalbox_get(sections, lines, section, "extensions", board)
if (exts or "").startswith(_FILES_PREFIX):
names.update(_recalbox_names(exts[len(_FILES_PREFIX):]))
else:
extensions.update((exts or "").split())
cores.append({
"core": core, "emulator": emulator, "priority": int(priority or "255"),
"source_ref": "; ".join([
_ref(key, path, priority_line, core_line),
_ref(key, config_path, enabled[package]),
]),
})
if cores:
picks[board] = min(reversed(cores), key=lambda c: c["priority"])
if not picks:
continue
# EmulatorManager walks the cores from the last and keeps a strictly
# lower priority: on a tie the last core listed wins.
pick = min(reversed(cores), key=lambda c: c["priority"])
board_defaults: dict[str, dict] = {}
for board in sorted({b for c in cores for b in c["boards"]}):
best = min(reversed(cores), key=lambda c: c["boards"].get(board, c["priority"]))
if best is not pick:
board_defaults[board] = _board(
best["core"], best["emulator"],
_ref(key, path, lines.get(f"{best['section']}/priority.{board}")))
extensions = sorted({e for c in cores for e in c["extensions"]})
shared = Counter((pick["core"], pick["emulator"]) for pick in picks.values())
core, emulator = max(shared, key=lambda pair: shared[pair])
first = next(pick for pick in picks.values() if (pick["core"], pick["emulator"]) == (core, emulator))
board_defaults = {
board: _board(pick["core"], pick["emulator"], pick["source_ref"])
for board, pick in picks.items() if (pick["core"], pick["emulator"]) != (core, emulator)
}
archive = any(e.endswith("!") for e in extensions)
rows.append({
"folder": folder,
"name": system.get("name"),
"extensions": [e.rstrip("!") for e in extensions],
"core": pick["core"],
"emulator": pick["emulator"],
"source_ref": "; ".join([
_ref(key, path, lines.get("system/roms.folder")),
_ref(key, path, lines.get(f"{pick['section']}/priority"),
lines.get(f"{pick['section']}/core")),
]),
"extensions": sorted({e.rstrip("!") for e in extensions}),
"files": sorted(names),
"core": core,
"emulator": emulator,
"source_ref": "; ".join([_ref(key, path, lines.get("system/roms.folder")), first["source_ref"]]),
"board_defaults": board_defaults,
"arcade": sections.get("properties", {}).get("type") == "arcade" or None,
"note": "an extension ending in ! is an archive the frontend does not open" if archive else None,