mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-18 23:57:55 -06:00
feat: scrape recalbox rom folders per board
GitLab refused the hundred raw reads per IP. A GitLab directory is now read from one archive request, else a shallow sparse git fetch, and the cache keeps its file list so offline runs know what is absent. Each board builds system.ini against its defconfig: a core whose package the board does not enable is dropped (pisnes was the SNES default everywhere), and a system with no core is not built. The row names the default most boards share, board_defaults the others. A files: list names whole files and is carried as files through the index and the installer. A folder no name resolves, run by a core of several machines, now holds those machines: thomson, multivision and pcv2 had no fallback, so their games got no file. Arcade-indexed cores keep naming files per set.
This commit is contained in:
1 parent
a05fc599d9
commit
d38cd58036
64 files changed
+2516
-103
No files matched your search
@@ -17,7 +17,7 @@ import importlib
|
||||
import pkgutil
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
_SCRIPTS_DIR = str(Path(__file__).resolve().parents[2])
|
||||
@@ -35,7 +35,7 @@ SETUP_KEYS = (
|
||||
"case_ref", "skip", "hidden", "media", "notes", "folders",
|
||||
)
|
||||
ROW_KEYS = (
|
||||
"folder", "name", "aliases", "extensions", "core", "emulator", "source_ref",
|
||||
"folder", "name", "aliases", "extensions", "files", "core", "emulator", "source_ref",
|
||||
"board_defaults", "arcade", "note",
|
||||
)
|
||||
|
||||
@@ -51,6 +51,7 @@ class Fetcher:
|
||||
sources: dict[str, dict]
|
||||
cache_dir: str = CACHE_DIR
|
||||
offline: bool = False
|
||||
_cached_dirs: dict[str, dict[str, set[str]]] = field(default_factory=dict, init=False, repr=False)
|
||||
|
||||
def _repo(self, key: str) -> tuple[upstream.Repo, str]:
|
||||
if key not in self.sources:
|
||||
@@ -71,6 +72,9 @@ class Fetcher:
|
||||
|
||||
def optional_text(self, key: str, path: str) -> str | None:
|
||||
repo, sha = self._repo(key)
|
||||
for directory, held in self._cached_dirs.get(key, {}).items():
|
||||
if path.startswith(directory + "/") and path not in held:
|
||||
return None
|
||||
lines = upstream.fetch_file(repo, sha, path, self.cache_dir, self.offline)
|
||||
return None if lines is None else "\n".join(lines) + "\n"
|
||||
|
||||
@@ -84,9 +88,21 @@ class Fetcher:
|
||||
raise SourceError(f"{repo.slug}@{sha[:8]}: the forge truncated the tree")
|
||||
return sorted(p for p in paths if p.startswith(prefix))
|
||||
|
||||
def cache_directory(self, key: str, directory: str) -> None:
|
||||
"""Fetch a whole GitLab directory in one archive request before its
|
||||
files are read one by one. A no-op on other forges."""
|
||||
repo, sha = self._repo(key)
|
||||
held = upstream.cache_gitlab_directory(repo, sha, directory, self.cache_dir, self.offline)
|
||||
if held is not None:
|
||||
self._cached_dirs.setdefault(key, {})[directory.strip("/")] = held
|
||||
|
||||
def entries(self, key: str, directory: str) -> list[str]:
|
||||
"""Paths of the files and folders directly under one directory."""
|
||||
repo, sha = self._repo(key)
|
||||
held = self._cached_dirs.get(key, {}).get(directory.strip("/"))
|
||||
if held is not None:
|
||||
prefix = directory.strip("/") + "/"
|
||||
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in held})
|
||||
if repo.family != "gitlab":
|
||||
prefix = directory.strip("/") + "/"
|
||||
return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in self.tree(key, prefix)})
|
||||
|
||||
Reference in new issue
Block a user