chore: merge the minui family platforms

This commit is contained in:
Abdessamad Derraz committed 2026-10-22 04:09:17 +02:00
commit 841017c887
34 files changed
+13514 -77

No files matched your search

+72 -2
View File
@@ -1221,6 +1221,12 @@ def resolve_local_file(
"""
sha1 = file_entry.get("sha1")
name = file_entry.get("name", "")
# A platform that hands each core a system directory of its own (MinUI's
# Bios/<TAG>/) writes that folder in front of the core's path. The tail
# the collection indexes is the core's path alone.
system_dir = file_entry.get("system_dir")
if system_dir and dest_hint.startswith(f"{system_dir}/"):
dest_hint = dest_hint[len(system_dir) + 1:]
# A name written with its directory is the destination when no path is
# given: gam4980/E.BIN resolved by its basename took another model's OS
# in verify --emulator and the emulator pack, while the platform pack,
@@ -2106,6 +2112,11 @@ def group_identical_platforms(
# One archive cannot serve a filesystem that folds case and one
# that does not: Lakka keeps bios.col beside BIOS.col.
bool(config.get("case_insensitive_fs")),
# Where each core's files go: the same cores handed other
# folders make another pack.
sorted(
tag for tag in standalone_set(config) if tag.startswith(CORE_DIR_PREFIX)
),
]
fp = hashlib.sha1(f"{fp}|{emulators}".encode()).hexdigest()
if target_cores_cache:
@@ -2252,6 +2263,55 @@ DUAL_MODE_PREFIX = "dual:"
# Marks, in the same set, a whole directory the platform's launcher hands a
# standalone emulator (launchers.handed_directories), as JSON.
HANDED_PREFIX = "handed:"
# Marks, in the same set, a libretro core and the system directory the
# platform hands it (``core_system_dirs``), as JSON.
CORE_DIR_PREFIX = "coredir:"
def core_system_dirs(
emu_name: str, profile: dict, standalone_cores: Iterable[str]
) -> list[str] | None:
"""The system directories a platform hands this libretro core, or None.
None when the platform declares no per-core directory: the core reads
the one system directory every core shares. A platform that declares
them (MinUI hands each launch tag its own Bios/<TAG>/) gives a core it
lists no directory at all, so the list is then empty.
"""
pairs = [
json.loads(tag[len(CORE_DIR_PREFIX):])
for tag in standalone_cores if tag.startswith(CORE_DIR_PREFIX)
]
if not pairs:
return None
names = {emu_name, *(str(c) for c in profile.get("cores") or [])}
return sorted({directory for core, directory in pairs if core in names})
def system_dir_view(profile: dict, directory: str) -> dict:
"""The profile as a core reads it from one handed system directory.
Each file keeps its own path, relative to the system directory, and
carries the directory: destinations put it in front, resolution takes
it off again (resolve_local_file), so the collection is still searched
by the core's own path. A core's archive subfolder moves with it.
"""
view = {
**profile,
"files": [
{**f, "system_dir": directory} if isinstance(f, dict) else f
for f in profile.get("files") or []
],
}
if profile.get("archive_prefix"):
view["archive_prefix"] = f"{directory}/{profile['archive_prefix']}"
return view
def system_dir_path(entry: dict, path: str) -> str:
"""``path`` below the system directory the entry carries, if any."""
directory = entry.get("system_dir")
return f"{directory}/{path}" if directory else path
def standalone_set(config: dict) -> set[str]:
@@ -2263,7 +2323,14 @@ def standalone_set(config: dict) -> set[str]:
"""
names = {str(c) for c in config.get("standalone_cores") or []}
names |= {DUAL_MODE_PREFIX + str(c) for c in config.get("dual_mode_cores") or []}
if names:
# The system directory each core is handed travels the same way, so
# platform_layouts places a core's files below it for every reader.
names |= {
CORE_DIR_PREFIX + json.dumps([str(core), directory])
for directory, entry in (config.get("core_system_dirs") or {}).items()
for core in entry.get("cores") or []
}
if names - {n for n in names if n.startswith(CORE_DIR_PREFIX)}:
from launchers import handed_directories # noqa: PLC0415
names |= {
@@ -2328,7 +2395,10 @@ def platform_layouts(
(what the build reads from the system directory, where it reads it).
"""
if not runs_standalone(emu_name, profile, standalone_cores):
return [(profile, False)]
directories = core_system_dirs(emu_name, profile, standalone_cores)
if directories is None:
return [(profile, False)]
return [(system_dir_view(profile, d), False) for d in directories]
standalone = profile
root = standalone_root_for(emu_name, profile, standalone_cores)
if root:
+3 -2
View File
@@ -41,6 +41,7 @@ from common import (
require_yaml,
resolve_local_file,
platform_layouts,
system_dir_path,
standalone_set,
select_emulator_profiles,
)
@@ -652,7 +653,7 @@ def slot_key(f: dict, standalone: bool = False) -> tuple:
return (
fname,
f.get("archive"),
where or fname,
system_dir_path(f, where or fname),
f.get("system"),
f.get("variant_group"),
f.get("mode", "both"),
@@ -727,7 +728,7 @@ def _cross_reference_profile(
if is_standalone:
effective_path = f.get("standalone_path") or f.get("path") or fname
else:
effective_path = f.get("path") or fname
effective_path = system_dir_path(f, f.get("path") or fname)
slot = slot_key(f, is_standalone)
seen_key = slot
if len(layouts) > 1:
+1
View File
@@ -1157,6 +1157,7 @@ def generate_pack(
standalone_copies=(
_pack_registry.get(platform_name, {}).get("install", {}) or {}
).get("standalone_copies", []),
launch_folders=len(config.get("core_system_dirs") or {}),
)
_write_generated_member(zf, "README.txt", readme_text)
+19 -3
View File
@@ -22,6 +22,7 @@ sys.path.insert(0, os.path.dirname(__file__))
from common import (
compute_composition,
count_catalog_matched,
declared_root,
github_token,
list_registered_platforms,
load_database,
@@ -167,6 +168,19 @@ def extract_targets(platforms_dir: str) -> list[tuple[str, str]]:
return sorted(rows, key=lambda row: row[0].casefold())
def own_root(config: dict) -> str:
"""The folder a pack written from an empty base_destination starts with.
RetroDECK writes bios/ in front of its files, the MinUI family hands its
cores folders below Bios/.
"""
root = declared_root(config)
if root:
return root
tops = {directory.split("/", 1)[0] for directory in config.get("core_system_dirs") or {}}
return tops.pop() if len(tops) == 1 else "bios"
def extract_notes(platforms_dir: str) -> list[str]:
"""A pack whose entries carry their own root extracts above the BIOS folder."""
folders = dict(extract_targets(platforms_dir))
@@ -176,8 +190,8 @@ def extract_notes(platforms_dir: str) -> list[str]:
display = config.get("platform", key)
if config.get("base_destination") == "" and display in folders:
notes.append(
f"The {display} pack already carries its own `bios/` folder, so it"
f" extracts into `{folders[display]}` rather than into the BIOS folder."
f"The {display} pack already carries its own `{own_root(config)}/` folder,"
f" so it extracts into `{folders[display]}` rather than into the BIOS folder."
)
return notes
@@ -470,7 +484,9 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
" a checksum, the fingerprint of a file's contents, which catches a"
" corrupt or unexpected copy. RetroArch, Lakka and RetroPie check"
" that the file is present under its name without reading its"
" contents (`path_is_valid` in RetroArch's `core_info.c`): the"
" contents (`path_is_valid` in RetroArch's `core_info.c`), and"
" MinUI, NextUI and MyMinUI hand each core its folder without"
" opening a file in it: the"
" Coverage table says which applies to you. Independently of that,"
" the collection records five fingerprints per file, and wherever"
" an emulator's code states an expected size or hash, that value is"
+31 -3
View File
@@ -2158,7 +2158,9 @@ def generate_platform_page(
f"| Contributed by | [@{username}](https://github.com/{username}):"
f" {contribution}{pr_link} |"
)
lines.extend(["", *_platform_install_block(name, display)])
detect = ((registry or {}).get(name, {}).get("install") or {}).get("detect") or []
on_card = any(rule.get("method") == "card_marker" for rule in detect)
lines.extend(["", *_platform_install_block(name, display, on_card)])
lines.extend(
[
"",
@@ -3565,9 +3567,35 @@ def _install_block(name: str, profile: dict) -> list[str]:
]
def _platform_install_block(name: str, display: str) -> list[str]:
def _platform_install_block(name: str, display: str, on_card: bool = False) -> list[str]:
"""The platform's pack, installed file by file: what the ZIP holds, with
resume, verification and only what is missing."""
resume, verification and only what is missing.
``on_card``: the files go on the frontend's SD card, installed from the
computer it is mounted on, whose root is named.
"""
if on_card:
return [
'???+ tip "Install with one command"',
"",
f" With the {display} SD card mounted, the installer downloads the files "
"of the pack that are missing or differ from it into `Bios/<TAG>/` on "
"the card, checks each one, and resumes where it stopped. `--dest` "
"names the card's root.",
"",
" Linux, macOS:",
"",
" ```sh",
f" {install_command('sh', f'--platform {name} --dest /path/to/card')}",
" ```",
"",
" Windows (PowerShell, E: being the card):",
"",
" ```powershell",
" " + install_command("powershell", f"--platform {name} --dest E:\\"),
" ```",
"",
]
return [
'???+ tip "Install with one command"',
"",
+81 -42
View File
@@ -15,7 +15,7 @@ from common import load_emulator_profiles
from common import resolution_is_hash_exact
from common import resolve_local_file
from common import resolve_platform_cores
from common import platform_layouts
from common import core_system_dirs, platform_layouts, system_dir_path
from common import standalone_set as platform_standalone_set
from common import sanitize_pack_path
from common import size_fits
@@ -331,7 +331,9 @@ def _extra_dest(u: dict) -> str:
the file's name after it ("cafeLibs/" + "snd_user.rpl").
"""
archive = u.get("archive")
raw = archive if archive else (u.get("path") or u["name"])
# An archive goes at the root of the core's system directory, which a
# platform handing each core its own folder writes in the path.
raw = (u.get("path") or archive) if archive else (u.get("path") or u["name"])
return f"{raw}{u['name']}" if raw.endswith("/") else raw
@@ -479,6 +481,8 @@ def _agnostic_scan_extras(
seed = f.get("path") or ""
if standalone:
seed = f.get("standalone_path") or seed
else:
seed = system_dir_path(f, seed or fname)
folder = seed.rsplit("/", 1)[0] if "/" in seed else ""
dest = f"{folder}/{scan_name}" if folder else scan_name
full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest
@@ -727,6 +731,7 @@ def _archive_prefix_extras(
extras_prefix: str,
by_name: dict,
taken: dict[str, dict] | None = None,
standalone_set: set[str] | None = None,
) -> list[dict]:
"""A second copy of an archive under the subdirectory a core reads.
@@ -744,45 +749,75 @@ def _archive_prefix_extras(
continue
if emu_name not in relevant:
continue
prefix = profile.get("archive_prefix", "")
if not prefix:
# A platform that hands each core its own system directory moves the
# subfolder with it, once per directory.
views = [profile]
if core_system_dirs(emu_name, profile, standalone_set or set()) is not None:
views = [
view for view, standalone
in platform_layouts(emu_name, profile, standalone_set or set())
if not standalone
]
for view in views:
extras.extend(_archive_prefix_copies(
emu_name, view, covered_names, seen_dests, extras_prefix,
by_name, taken,
))
return extras
def _archive_prefix_copies(
emu_name: str,
profile: dict,
covered_names: set,
seen_dests: set,
extras_prefix: str,
by_name: dict,
taken: dict[str, dict] | None,
) -> list[dict]:
"""The prefixed copies of one layout's archives."""
extras: list[dict] = []
prefix = profile.get("archive_prefix", "")
if not prefix:
return extras
profile_archives: dict[str, dict] = {}
for f in profile.get("files", []):
archive = f.get("archive", "")
if archive:
profile_archives.setdefault(archive, f)
for archive_name, archive_entry in sorted(profile_archives.items()):
if archive_name not in covered_names:
continue
profile_archives: dict[str, dict] = {}
for f in profile.get("files", []):
archive = f.get("archive", "")
if archive:
profile_archives.setdefault(archive, f)
for archive_name, archive_entry in sorted(profile_archives.items()):
if archive_name not in covered_names:
continue
dest = f"{prefix}/{archive_name}"
full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest
if full_dest in seen_dests and (taken is None or full_dest not in taken):
continue
if not by_name.get(archive_name):
continue
extra = {
"name": archive_name,
"destination": dest,
"required": True,
"hle_fallback": False,
"source_emulator": profile.get("emulator", emu_name),
"source_profile": emu_name,
# Keyed as the gap report keys an archive: name and path alike.
"source_name": archive_name,
"source_path": archive_name,
"category": archive_entry.get("category", "bios"),
"source_system": archive_entry.get("system"),
"source_systems": entry_system_ids(archive_entry, profile),
"region": archive_entry.get("region"),
"variant_group": archive_entry.get("variant_group"),
}
members = archive_members(archive_name, profile.get("files", []))
if members:
extra["members"] = members
extra["claim_order"] = (3, 0, emu_name)
if _take(extra, full_dest, seen_dests, taken):
extras.append(extra)
dest = f"{prefix}/{archive_name}"
full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest
if full_dest in seen_dests and (taken is None or full_dest not in taken):
continue
if not by_name.get(archive_name):
continue
extra = {
"name": archive_name,
"destination": dest,
"required": True,
"hle_fallback": False,
"source_emulator": profile.get("emulator", emu_name),
"source_profile": emu_name,
# Keyed as the gap report keys an archive: name and path alike.
"source_name": archive_name,
"source_path": system_dir_path(archive_entry, archive_name),
"category": archive_entry.get("category", "bios"),
"source_system": archive_entry.get("system"),
"source_systems": entry_system_ids(archive_entry, profile),
"region": archive_entry.get("region"),
"variant_group": archive_entry.get("variant_group"),
}
members = archive_members(archive_name, profile.get("files", []))
if members:
extra["members"] = members
if archive_entry.get("system_dir"):
extra["system_dir"] = archive_entry["system_dir"]
extra["claim_order"] = (3, 0, emu_name)
if _take(extra, full_dest, seen_dests, taken):
extras.append(extra)
return extras
@@ -1046,7 +1081,7 @@ def _collect_emulator_extras(
if is_standalone:
raw = f.get("standalone_path") or f.get("path") or fname
else:
raw = f.get("path") or fname
raw = system_dir_path(f, f.get("path") or fname)
dest = f"{raw}{fname}" if raw.endswith("/") else raw
# A bare name is a destination too: the core reads it at the root
# of the directory core files go to, which the platform may not
@@ -1087,6 +1122,8 @@ def _collect_emulator_extras(
extra[identity] = f[identity]
if f.get("contents"):
extra["contents"] = f["contents"]
if f.get("system_dir"):
extra["system_dir"] = f["system_dir"]
extra["claim_order"] = (2, 0, emu_name)
if _take(extra, full_dest, seen_dests, taken):
extras.append(extra)
@@ -1094,7 +1131,7 @@ def _collect_emulator_extras(
extras.extend(
_archive_prefix_extras(
profiles, relevant, covered_names, seen_dests, extras_prefix,
by_name, taken,
by_name, taken, standalone_set,
)
)
@@ -1211,6 +1248,8 @@ def _report_extra(u: dict, name: str, dest: str) -> dict:
}
if u.get("members"):
extra["members"] = u["members"]
if u.get("system_dir"):
extra["system_dir"] = u["system_dir"]
# The archive layout the core opens ranks same-named copies by what
# they hold. Without it the first copy indexed won.
if u.get("contents"):
+60 -7
View File
@@ -12,9 +12,46 @@ def _detect_systems(detect: list[dict]) -> list[str]:
"""The operating systems the registry's detection rules cover, in order."""
order = ("linux", "darwin", "windows")
seen = {str(rule.get("os", "")) for rule in detect}
if "any" in seen:
return list(order)
return [name for name in order if name in seen]
def _card_section(platform_name: str, display: str) -> str:
"""The one-line install for a frontend whose files live on its SD card.
The installer runs on the computer the card is mounted on, and the card
root is named: no folder on that computer is one the frontend reads.
"""
return (
" Option A: Automatic (recommended)\n"
" ---------------------------------\n"
" With the SD card mounted on a computer, name its root with --dest.\n"
" Linux and macOS: run this in a terminal:\n\n"
f" curl -fsSL {INSTALL_SH} | sh -s -- --platform {platform_name} --dest /path/to/card\n\n"
" Windows: run this in PowerShell (E: being the card):\n\n"
f" &([scriptblock]::Create((irm {INSTALL_PS1}))) --platform {platform_name} --dest E:\\\n\n"
f" The installer writes each file where {display} hands it to the core\n"
" and checks every file.\n\n"
)
def _card_manual(display: str) -> str:
"""Where the files of a per-tag Bios/ layout go, copied by hand."""
return (
" Option B: Manual\n"
" ----------------\n"
" 1. Put the SD card in your PC\n"
" 2. Extract all files from this archive directly into the root of the\n"
" card: the archive holds the Bios/ folder, with one subfolder per\n"
" launch tag (Bios/PS/, Bios/GBA/, ...)\n"
" 3. Overwrite if asked\n\n"
f" {display} hands each core the folder named after the tag in\n"
" parentheses of its Roms folder (\"Sony PlayStation (PS)\" reads\n"
" Bios/PS/). A Roms folder can be renamed as long as the tag stays.\n\n"
)
def _automatic_section(
display: str, detect: list[dict], standalone_copies: list[dict] | None,
) -> str:
@@ -234,24 +271,40 @@ def _build_readme(
system_filter: list[str] | None = None,
detect: list[dict] | None = None,
standalone_copies: list[dict] | None = None,
launch_folders: int = 0,
) -> str:
"""Build a personalized step-by-step README for each platform pack."""
"""Build a personalized step-by-step README for each platform pack.
``launch_folders`` counts the folders a platform that hands each core
its own system directory gives them, where it names no system.
"""
narrowings = narrowings or []
sep = "=" * 50
scope = (
f"{launch_folders} launch folders" if launch_folders and not num_systems
else f"{num_systems} systems"
)
header = (
f"{sep}\n"
f" RETROBIOS - {platform_display} BIOS Pack\n"
f" {total_files} files for {num_systems} systems\n"
f" {total_files} files for {scope}\n"
f"{sep}\n\n"
)
detect = list(detect or [])
bios_paths = [str(rule.get("bios_path", "")) for rule in detect]
guide = (
"INSTALLATION GUIDE\n\n"
+ _automatic_section(platform_display, detect, standalone_copies)
+ MANUAL_GUIDES.get(platform_name, _generic_manual(bios_paths))
)
if any(rule.get("method") == "card_marker" for rule in detect):
guide = (
"INSTALLATION GUIDE\n\n"
+ _card_section(platform_name, platform_display)
+ _card_manual(platform_display)
)
else:
guide = (
"INSTALLATION GUIDE\n\n"
+ _automatic_section(platform_display, detect, standalone_copies)
+ MANUAL_GUIDES.get(platform_name, _generic_manual(bios_paths))
)
if regions:
region_help = (
+348
View File
@@ -0,0 +1,348 @@
#!/usr/bin/env python3
"""Scraper for MinUI and the frontends built on it (NextUI, MyMinUI).
Source: the frontend's repository at a pinned commit
Format: one emulator pak per launch tag, ``skeleton/SYSTEM/<device>/paks/Emus/
<TAG>.pak/launch.sh`` (base install) or ``skeleton/EXTRAS/Emus/<device>/
<TAG>.pak/launch.sh`` (extras bundle), whose script names the libretro core
in ``EMU_EXE`` and starts it under minarch.
None of the three publishes a BIOS list in code: the frontend hands every
core the folder ``<SD>/Bios/<TAG>/`` as its libretro system directory
(``core.bios_dir``, returned for RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY) and
never opens or hashes a BIOS file itself. What the code does state is which
core each tag runs, and that tag decides the folder. So the platform file
carries no file entry: it maps each folder to the cores launched with it,
and the files are the ones the core profiles document, placed below that
folder.
Paks that start a standalone binary (NDS.pak runs DraStic, P8-NATIVE.pak
the PICO-8 executable) set no ``EMU_EXE`` and are not libretro launches:
they are counted and left out.
"""
from __future__ import annotations
import json
import re
import urllib.parse
from collections.abc import Callable, Mapping
from dataclasses import dataclass
try:
from .base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
fetch_bytes,
github_headers,
)
except ImportError:
from base_scraper import (
BaseScraper,
BiosRequirement,
fetch_branch_head,
fetch_bytes,
github_headers,
)
PLATFORM_NAME = "minui"
@dataclass(frozen=True)
class Variant:
"""One frontend of the family: where its code lives and what it is called."""
key: str
name: str
repo: str
branch: str
homepage: str
# The file that sets core.bios_dir.
minarch_source: str
VARIANTS: dict[str, Variant] = {
"minui": Variant(
key="minui",
name="MinUI",
repo="shauninman/MinUI",
branch="main",
homepage="https://github.com/shauninman/MinUI",
minarch_source="workspace/all/minarch/minarch.c",
),
"nextui": Variant(
key="nextui",
name="NextUI",
repo="LoveRetro/NextUI",
branch="main",
homepage="https://nextui.loveretro.games",
minarch_source="workspace/all/minarch/ma_core.c",
),
"myminui": Variant(
key="myminui",
name="MyMinUI",
repo="Turro75/MyMinUI",
branch="main",
homepage="https://github.com/Turro75/MyMinUI",
minarch_source="workspace/all/minarch/minarch.c",
),
}
# The folder minarch hands a core: SDCARD_PATH "/Bios/" and the launch tag.
BIOS_ROOT = "Bios"
_BIOS_DIR = re.compile(
r'sprintf\(\s*\(char\s*\*\)\s*core\.bios_dir\s*,\s*SDCARD_PATH\s*"/Bios/%s"\s*,\s*core\.tag\s*\)'
)
_PAK = re.compile(
r"^skeleton/(?:SYSTEM/(?P<base>[^/]+)/paks/Emus|EXTRAS/Emus/(?P<extra>[^/]+))"
r"/(?P<tag>[^/]+)\.pak/launch\.sh$"
)
_EMU_EXE = re.compile(r"^[ \t]*EMU_EXE=([A-Za-z0-9_.-]+)[ \t]*$", re.MULTILINE)
# Both ways a pak starts its core: minarch on the core's .so directly, or
# MyMinUI's launch_rom.sh, which runs minarch (or RetroArch with
# system_directory set to the same folder) on ${EMU_EXE}_libretro.so.
_STARTS_CORE = re.compile(
r'^[ \t]*(?:nice\s+-?\d+\s+)?minarch\.elf\s+"\$CORES_PATH/\$\{EMU_EXE\}_libretro\.so"'
r"|^[ \t]*launch_rom\.sh\s",
re.MULTILINE,
)
# Each device builds its cores from workspace/<device>/cores/makefile through
# the template in workspace/all/cores/makefile: `<X>_REPO ?=
# https://github.com/libretro/<X>` and the library `<X>_libretro.so` unless
# `<X>_CORE` names another.
_RECIPE = re.compile(r"^workspace/(?P<device>[^/]+)/cores/makefile$")
_CORES_LIST = re.compile(r"^CORES\s*\+?=(.*)$", re.MULTILINE)
_CORE_VAR = re.compile(r"^(\S+?)_(REPO|CORE)\s*\??=\s*(\S+)", re.MULTILINE)
# A library built from a project other than the libretro core of that name:
# MyMinUI builds dosbox-pure into dosbox_libretro.so. The core is the project
# the bytes come from.
BUILT_AS = {
"https://github.com/libretro/dosbox-pure": "dosbox_pure",
}
@dataclass(frozen=True)
class Pak:
"""One emulator pak as its launch script states it."""
tag: str
device: str
core: str
path: str
line: int
def read_pak(path: str, text: str) -> Pak | None:
"""The libretro launch a pak's script makes, or None for a standalone one.
A script that names a core but starts it in a way not read here raises:
the shape of the paks changed and the scrape must say so.
"""
match = _PAK.match(path)
if not match:
return None
exe = _EMU_EXE.search(text)
if not exe:
return None
if not _STARTS_CORE.search(text):
raise ValueError(f"{path}: EMU_EXE={exe.group(1)} but no minarch launch")
line = text.count("\n", 0, exe.start()) + 1
return Pak(
tag=match.group("tag"),
device=match.group("base") or match.group("extra"),
core=exe.group(1),
path=path,
line=line,
)
def normalize_repository(url: str) -> str:
"""A repository URL without the .git suffix or a trailing slash."""
url = url.strip().rstrip("/")
return url[: -len(".git")] if url.endswith(".git") else url
def read_recipes(text: str) -> dict[str, str]:
"""Library file name to source repository, for one device's core makefile."""
text = "\n".join(line.split("#", 1)[0] for line in text.splitlines())
names = [name for match in _CORES_LIST.finditer(text) for name in match.group(1).split()]
values: dict[tuple[str, str], str] = {
(match.group(1), match.group(2)): match.group(3) for match in _CORE_VAR.finditer(text)
}
return {
values.get((name, "CORE"), f"{name}_libretro.so"): normalize_repository(
values.get((name, "REPO"), f"https://github.com/libretro/{name}")
)
for name in names
}
def bios_dir_ref(source: str, text: str) -> str:
"""``file:line`` of the statement that sets the per-tag system directory."""
match = _BIOS_DIR.search(text)
if not match:
raise ValueError(f"{source}: core.bios_dir is no longer SDCARD_PATH/Bios/<tag>")
return f"{source}:{text.count(chr(10), 0, match.start()) + 1}"
def core_identity(library: str, repositories: Mapping[str, set[str]]) -> str:
"""The core a pak's library is, by the project it is built from."""
stem = library.removesuffix("_libretro.so")
built = sorted({BUILT_AS[r] for r in repositories.get(library, set()) if r in BUILT_AS})
if len(built) > 1:
raise ValueError(f"{library} is built from projects of {', '.join(built)}")
return built[0] if built else stem
def core_system_dirs(
paks: list[Pak], repositories: Mapping[str, set[str]] | None = None
) -> dict[str, dict]:
"""Each Bios/<TAG> folder with the cores its paks run and where they ship."""
repositories = repositories or {}
by_dir: dict[str, dict] = {}
for pak in sorted(paks, key=lambda p: (p.tag, p.device, p.path)):
entry = by_dir.setdefault(
f"{BIOS_ROOT}/{pak.tag}",
{"cores": [], "devices": [], "source_ref": f"{pak.path}:{pak.line}"},
)
core = core_identity(f"{pak.core}_libretro.so", repositories)
if core not in entry["cores"]:
entry["cores"].append(core)
if pak.device not in entry["devices"]:
entry["devices"].append(pak.device)
for entry in by_dir.values():
entry["cores"].sort()
entry["devices"].sort()
return dict(sorted(by_dir.items()))
def platform_config(
variant: Variant, revision: str, date: str, files: Mapping[str, str]
) -> dict:
"""The platform file for one frontend, from its sources at ``revision``.
``files`` maps repository paths to their text: every pak launch script,
every device's core makefile and the minarch source that sets the
system directory.
"""
if variant.minarch_source not in files:
raise ValueError(f"{variant.repo}@{revision}: no {variant.minarch_source}")
reference = bios_dir_ref(variant.minarch_source, files[variant.minarch_source])
paks = [pak for path, text in sorted(files.items()) if (pak := read_pak(path, text))]
if not paks:
raise ValueError(f"{variant.repo}@{revision}: no pak starts a libretro core")
repositories: dict[str, set[str]] = {}
for path, text in files.items():
if _RECIPE.match(path):
for library, repository in read_recipes(text).items():
repositories.setdefault(library, set()).add(repository)
dirs = core_system_dirs(paks, repositories)
built_from = {
core_identity(f"{pak.core}_libretro.so", repositories): sorted(
repositories.get(f"{pak.core}_libretro.so", set())
)
for pak in paks
}
return {
"platform": variant.name,
"version": date,
"homepage": variant.homepage,
"source": f"https://github.com/{variant.repo}/tree/{revision}",
# The card root: every core reads below Bios/<TAG>/ on it.
"base_destination": "",
"hash_type": "sha1",
# minarch hands the folder and never reads or hashes a file in it.
"verification_mode": "existence",
"system_directory_ref": reference,
"cores": sorted({core for entry in dirs.values() for core in entry["cores"]}),
"core_system_dirs": dirs,
# Where each core's library is built from, read from the device
# makefiles. A pak may ship a library no makefile builds.
"core_repositories": {core: repos for core, repos in sorted(built_from.items()) if repos},
"systems": {},
}
class Scraper(BaseScraper):
"""Pak scraper for one frontend of the MinUI family."""
lists_files = False
variant: Variant = VARIANTS["minui"]
def __init__(
self,
url: str | None = None,
fetch: Callable[[str, dict[str, str] | None], bytes] | None = None,
) -> None:
self._fetch = fetch or (lambda u, h=None: fetch_bytes(u, headers=h))
self.revision = ""
if url is None:
self.revision = fetch_branch_head(self.variant.repo, self.variant.branch)
url = f"https://github.com/{self.variant.repo}/tree/{self.revision}"
super().__init__(url=url)
self._config: dict | None = None
def _api(self, path: str) -> dict:
url = f"https://api.github.com/repos/{self.variant.repo}/{path}"
try:
return json.loads(self._fetch(url, github_headers()))
except (ConnectionError, json.JSONDecodeError) as exc:
raise RuntimeError(f"cannot read {url}: {exc}") from exc
def _raw(self, path: str) -> str:
url = (
f"https://raw.githubusercontent.com/{self.variant.repo}/{self.revision}/"
f"{urllib.parse.quote(path)}"
)
return self._fetch(url, None).decode("utf-8", errors="replace")
def config(self) -> dict:
"""The platform file, read once."""
if self._config is not None:
return self._config
if not self.revision:
self.revision = self.url.rstrip("/").rsplit("/", 1)[-1]
tree = self._api(f"git/trees/{self.revision}?recursive=1")
if tree.get("truncated"):
raise RuntimeError(f"{self.variant.repo}@{self.revision}: tree listing truncated")
paths = [
item["path"] for item in tree.get("tree", [])
if item.get("type") == "blob"
and (
_PAK.match(item["path"])
or _RECIPE.match(item["path"])
or item["path"] == self.variant.minarch_source
)
]
commit = self._api(f"commits/{self.revision}")
date = commit.get("commit", {}).get("committer", {}).get("date", "")[:10]
if not date:
raise RuntimeError(f"{self.variant.repo}@{self.revision}: commit has no date")
files = {path: self._raw(path) for path in paths}
self._config = platform_config(self.variant, self.revision, date, files)
return self._config
def fetch_requirements(self) -> list[BiosRequirement]:
"""None: the frontend names no BIOS file in code, its cores do."""
self.config()
return []
def validate_format(self, raw_data: str) -> bool:
return bool(self.config().get("core_system_dirs"))
def generate_platform_yaml(self) -> dict:
return self.config()
def main() -> None:
try:
from .base_scraper import scraper_cli
except ImportError:
from base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape the MinUI pak list")
if __name__ == "__main__":
main()
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Scraper for MyMinUI's pak list: the MinUI scraper on MyMinUI's repository."""
from __future__ import annotations
try:
from . import minui_scraper
except ImportError:
import minui_scraper
PLATFORM_NAME = "myminui"
class Scraper(minui_scraper.Scraper):
"""Pak scraper for MyMinUI."""
variant = minui_scraper.VARIANTS["myminui"]
def main() -> None:
try:
from .base_scraper import scraper_cli
except ImportError:
from base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape the MyMinUI pak list")
if __name__ == "__main__":
main()
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env python3
"""Scraper for NextUI's pak list: the MinUI scraper on NextUI's repository."""
from __future__ import annotations
try:
from . import minui_scraper
except ImportError:
import minui_scraper
PLATFORM_NAME = "nextui"
class Scraper(minui_scraper.Scraper):
"""Pak scraper for NextUI."""
variant = minui_scraper.VARIANTS["nextui"]
def main() -> None:
try:
from .base_scraper import scraper_cli
except ImportError:
from base_scraper import scraper_cli
scraper_cli(Scraper, "Scrape the NextUI pak list")
if __name__ == "__main__":
main()
+8 -3
View File
@@ -35,6 +35,7 @@ from common import (
resolve_platform_cores,
platform_layouts,
standalone_set as platform_standalone_set,
system_dir_path,
variant_base_name,
)
from packresolve import preferred_entries
@@ -227,7 +228,7 @@ def profile_claims(
dest = (
(entry.get("standalone_path") or entry.get("path"))
if is_standalone
else entry.get("path")
else system_dir_path(entry, entry.get("path") or entry.get("name") or "")
) or entry.get("name") or ""
if not dest:
continue
@@ -299,8 +300,12 @@ def _archive_claims(
members = entry.get("members")
if not members:
continue
system = next(f.get("system") for f in files if f.get("archive") == archive)
for dest in (archive, f"{prefix}/{archive}") if prefix else (archive,):
member = next(f for f in files if f.get("archive") == archive)
system = member.get("system")
if member.get("system_dir"):
entry = {**entry, "system_dir": member["system_dir"]}
root = system_dir_path(entry, archive)
for dest in (root, f"{prefix}/{archive}") if prefix else (root,):
if placement is None:
full = f"{base_dest}/{dest}" if base_dest else dest
else:
+8 -2
View File
@@ -76,6 +76,7 @@ from common import (
entry_system_ids,
sanitize_pack_path,
select_emulator_profiles,
system_dir_path,
)
yaml = require_yaml()
@@ -576,7 +577,7 @@ def find_undeclared_files(
if is_standalone:
dest = f.get("standalone_path") or f.get("path") or fname
else:
dest = f.get("path") or fname
dest = system_dir_path(f, f.get("path") or fname)
raw_regions = f.get("region") or []
region_key = tuple(
str(value) for value in (
@@ -615,6 +616,7 @@ def find_undeclared_files(
if archive:
archive_key = (
archive,
f.get("system_dir"),
f.get("system"),
f.get("variant_group"),
region_key,
@@ -634,7 +636,7 @@ def find_undeclared_files(
"variant_group": f.get("variant_group"),
"name": archive,
"archive": archive,
"path": archive,
"path": system_dir_path(f, archive),
"required": False,
"hle_fallback": False,
"category": f.get("category", "bios"),
@@ -647,6 +649,8 @@ def find_undeclared_files(
"archive_required_count": 0,
"members": archive_members(archive, profile.get("files", [])),
}
if f.get("system_dir"):
archive_entries[archive_key]["system_dir"] = f["system_dir"]
entry = archive_entries[archive_key]
entry["archive_file_count"] += 1
if f.get("required", False):
@@ -738,6 +742,8 @@ def find_undeclared_files(
}
if f.get("contents"):
entry["contents"] = f["contents"]
if f.get("system_dir"):
entry["system_dir"] = f["system_dir"]
members = with_archive_members(f, profile.get("files", [])).get("members")
if members:
entry["members"] = members