mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-06 01:53:56 -06:00
chore: merge core data trees
This commit is contained in:
Abdessamad Derraz committed
commit
83b50c9acf
35 files changed
+243686
-5703
No files matched your search
@@ -73,7 +73,7 @@ data_directories:
|
||||
# buildbot source: https://buildbot.libretro.com/assets/system/blueMSX.zip
|
||||
# ref: libretro.c:1196-1197: system_dir/Machines/ + system_dir/Databases/
|
||||
- ref: bluemsx
|
||||
installed_on: [batocera, recalbox, retrobat, retrodeck]
|
||||
installed_on: [batocera, knulli, recalbox, retrobat, retrodeck, retropie]
|
||||
source_ref: "libretro.c:1196-1197, 1447-1449, Src/Board/Machine.c:637-656"
|
||||
|
||||
files:
|
||||
|
||||
@@ -51,7 +51,8 @@ pack_structure:
|
||||
|
||||
data_directories:
|
||||
- ref: dolphin-sys
|
||||
installed_on: [batocera, lakka, retrobat, retrodeck]
|
||||
installed_on: [batocera, knulli, lakka, retrobat, retrodeck]
|
||||
unread_on: [recalbox, romm]
|
||||
mode: libretro
|
||||
destination: "Sys"
|
||||
source_ref: "source Source/Core/Common/CommonPaths.h:128-141"
|
||||
|
||||
@@ -49,9 +49,11 @@ archive_prefix: fbneo
|
||||
|
||||
data_directories:
|
||||
- ref: fbneo-cheats
|
||||
unread_on: [romm]
|
||||
destination: fbneo/cheats
|
||||
source_ref: "source src/burner/conc.cpp:167"
|
||||
- ref: fbneo-samples
|
||||
unread_on: [romm]
|
||||
destination: fbneo
|
||||
source_ref: "source src/burn/snd/samples.cpp:21-35"
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ notes: |
|
||||
|
||||
data_directories:
|
||||
- ref: freeintv-overlays
|
||||
unread_on: [batocera, knulli, recalbox, romm]
|
||||
destination: freeintv_overlays
|
||||
required: false
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ pack_structure:
|
||||
|
||||
data_directories:
|
||||
- ref: dolphin-sys
|
||||
unread_on: [romm]
|
||||
mode: libretro
|
||||
destination: "Sys"
|
||||
source_ref: "Source/Core/Common/CommonPaths.h:130-131"
|
||||
|
||||
@@ -40,6 +40,7 @@ notes: |
|
||||
data_directories:
|
||||
- ref: lrps2
|
||||
installed_on: [batocera, lakka, recalbox, retrobat, retrodeck]
|
||||
unread_on: [romm]
|
||||
destination: ""
|
||||
source_ref: "source pcsx2/GameDatabase.cpp:49, source pcsx2/GameDatabase.cpp:979"
|
||||
|
||||
|
||||
+2
-1
@@ -23,7 +23,8 @@ resources_directory: "pcsx2/resources/"
|
||||
data_directories:
|
||||
- ref: pcsx2-resources
|
||||
destination: "pcsx2/resources"
|
||||
source_ref: "pcee2-libretro/Libretro.cpp:543-571, pcsx2/Pcsx2Config.cpp:2417-2429"
|
||||
bundled: true
|
||||
source_ref: "pcee2-libretro/Libretro.cpp:543-571, pcsx2/Pcsx2Config.cpp:2417-2429, cmake/EmbedResources.cmake:19-31, 46-62"
|
||||
|
||||
notes: |
|
||||
libretro port of current PCSX2, merged up to upstream ab0b401f (v2.9.138,
|
||||
|
||||
@@ -21,7 +21,7 @@ pack_structure:
|
||||
|
||||
data_directories:
|
||||
- ref: ppsspp-assets
|
||||
installed_on: [batocera, emudeck, lakka, recalbox, retrobat, retrodeck, retropie]
|
||||
installed_on: [amberelec, batocera, emudeck, lakka, recalbox, retrobat, retrodeck, retropie, romm]
|
||||
mode: libretro
|
||||
destination: ""
|
||||
source_ref: "libretro/libretro.cpp:1228-1237, libretro/libretro.cpp:1283"
|
||||
|
||||
@@ -42,7 +42,8 @@ notes: |
|
||||
data_directories:
|
||||
- ref: stonesoup
|
||||
installed_on: [recalbox]
|
||||
source_ref: "crawl-ref/source/files.cc:375-458"
|
||||
load_from: content_dir
|
||||
source_ref: "crawl-ref/libretro/libretro.cc:178-197, crawl-ref/source/files.cc:375-458"
|
||||
|
||||
files: []
|
||||
# no BIOS/firmware, game data loaded via data_directories
|
||||
+15155
-2307
File diff suppressed because it is too large.
Load diff
+13966
-2
File diff suppressed because it is too large.
Load diff
+24886
-2
File diff suppressed because it is too large.
Load diff
+15603
-2307
File diff suppressed because it is too large.
Load diff
+14670
-2
File diff suppressed because it is too large.
Load diff
+15436
-2
File diff suppressed because it is too large.
Load diff
+27928
-2
File diff suppressed because it is too large.
Load diff
+14859
-3
File diff suppressed because it is too large.
Load diff
+13966
-2
File diff suppressed because it is too large.
Load diff
+15884
-2
File diff suppressed because it is too large.
Load diff
+14338
-2
File diff suppressed because it is too large.
Load diff
+15758
-2
File diff suppressed because it is too large.
Load diff
+15138
-972
File diff suppressed because it is too large.
Load diff
+25575
-2
File diff suppressed because it is too large.
Load diff
@@ -9,6 +9,14 @@
|
||||
# - for_platforms: restricts which platforms use this entry
|
||||
# if absent, entry is available to all platforms
|
||||
#
|
||||
# A platform pack carries the trees its own systems declare and those of the
|
||||
# libretro cores it runs (a profile's data_directories). A platform whose
|
||||
# image installs the tree where its core reads it is left out of
|
||||
# for_platforms: the profile records it (installed_on), and the comment above
|
||||
# the entry cites the install step at the revision the platform YAML pins.
|
||||
# A platform that never opens the tree is recorded with unread_on.
|
||||
# tests/test_core_data_trees.py fails on a reader with neither.
|
||||
#
|
||||
# Each entry cites the original emulator source code that requires the directory.
|
||||
|
||||
data_directories:
|
||||
@@ -19,37 +27,73 @@ data_directories:
|
||||
# =========================================================================
|
||||
|
||||
# ref: DolphinLibretro/Boot.cpp:72-73 (system/dolphin-emu/Sys/)
|
||||
# Installed (emulators/dolphin.yml installed_on, the other readers follow):
|
||||
# batocera batocera.linux@batocera-43.1 package/batocera/emulators/retroarch/libretro/libretro-dolphin/libretro-dolphin.mk:43-44
|
||||
# knulli batocera.linux@487014d9 libretro-dolphin.mk:36-37, knulli-linux@20260511 board/scripts/post-build-script.sh:262
|
||||
# lakka Lakka-LibreELEC@v6.1 packages/lakka/libretro_cores/dolphin/package.mk:35-36
|
||||
# retrobat retrobat-bios@158add8d dolphin-emu/Sys/, retrobat@8.3.1 build.ini get_bios
|
||||
# retrodeck components@fcd0d729 retroarch/component_prepare.sh:61
|
||||
# Never opened: recalbox (libretroRetroarch.py:452-453 hands the core bios/gamecube,
|
||||
# package/libretro-dolphin/0003-recalbox-bios-paths.patch drops dolphin-emu/Sys),
|
||||
# romm (EmulatorJS 4.2.3 ships no Dolphin core and RomM hands a core one file).
|
||||
dolphin-sys:
|
||||
source_url: "https://buildbot.libretro.com/assets/system/Dolphin.zip"
|
||||
source_type: zip
|
||||
for_platforms: [retroarch, lakka, retropie]
|
||||
for_platforms: [emudeck, retroarch, retropie, rocknix]
|
||||
local_cache: data/dolphin-sys
|
||||
strip_components: 2
|
||||
exclude: [Themes]
|
||||
description: "Dolphin system data (GameSettings, DSP, fonts, shaders)"
|
||||
|
||||
# ref: ppsspp/ext/native/ui/ui_screen.cpp (system/PPSSPP/)
|
||||
# Installed (emulators/ppsspp.yml installed_on):
|
||||
# amberelec AmberELEC@a21975bf packages/amberelec/sources/autostart.sh:152-154, core patched
|
||||
# to read /roms/gamedata/ppsspp (packages/games/libretro/ppsspp/patches/001-fix_savedir.patch)
|
||||
# batocera libretro-ppsspp.mk:109-110, core patched to /usr/share/ppsspp/PPSSPP (000-batocera-path.patch:10)
|
||||
# emudeck EmuDeck@dddb3022 functions/EmuScripts/emuDeckRetroArch.sh:27, 343
|
||||
# lakka packages/lakka/libretro_cores/ppsspp/package.mk:72-73
|
||||
# recalbox recalbox@10.1.1 package/libretro-ppsspp/libretro-ppsspp.mk:86, ppsspp-0000-fix-recalbox-path.patch
|
||||
# retrobat retrobat-bios@158add8d PPSSPP/
|
||||
# retrodeck retroarch/component_prepare.sh:64
|
||||
# retropie RetroPie-Setup@46128cd2 scriptmodules/libretrocores/lr-ppsspp.sh:55-56
|
||||
# romm EmulatorJS@v4.2.3 emulator.js:922-923, GameManager.js:346-368
|
||||
# knulli stays: the A133 and A527 boards overlay a prebuilt core that reads
|
||||
# bios/PPSSPP (board/allwinner/a133/fsoverlay/usr/lib/libretro/ppsspp_libretro.so).
|
||||
ppsspp-assets:
|
||||
source_url: "https://buildbot.libretro.com/assets/system/PPSSPP.zip"
|
||||
source_type: zip
|
||||
for_platforms: [retroarch, lakka, retropie]
|
||||
for_platforms: [knulli, retroarch, rocknix]
|
||||
local_cache: data/ppsspp-assets
|
||||
strip_components: 1
|
||||
description: "PPSSPP fonts, backgrounds, shaders, lang files"
|
||||
|
||||
# ref: bluemsx-libretro/system/ (system/Databases/ + system/Machines/)
|
||||
# Installed (emulators/bluemsx.yml installed_on):
|
||||
# batocera libretro-bluemsx/libretro-bluemsx.mk:38-39
|
||||
# knulli batocera.linux@487014d9 libretro-bluemsx.mk:37-38, post-build-script.sh:262
|
||||
# recalbox package/libretro-bluemsx/libretro-bluemsx.mk:27
|
||||
# retrobat retrobat-bios@158add8d Databases/, Machines/
|
||||
# retrodeck retroarch/component_prepare.sh:122, 124
|
||||
# retropie scriptmodules/libretrocores/lr-bluemsx.sh:33-34, 59
|
||||
bluemsx:
|
||||
source_url: "https://buildbot.libretro.com/assets/system/blueMSX.zip"
|
||||
source_type: zip
|
||||
for_platforms: [retroarch, lakka, retropie]
|
||||
for_platforms: [amberelec, emudeck, emuelec, lakka, nextui, retroarch, rocknix]
|
||||
local_cache: data/bluemsx
|
||||
description: "blueMSX Databases + Machines configs"
|
||||
|
||||
# ref: pcsx2/libretro/main.cpp (system/pcsx2/)
|
||||
# Installed (emulators/lrps2.yml installed_on):
|
||||
# batocera libretro-ps2/libretro-ps2.mk:37-39
|
||||
# lakka packages/lakka/libretro_cores/lrps2/package.mk:33-34
|
||||
# recalbox package/libretro-ps2/libretro-ps2.mk:35-37 (bios/ps2/resources, where its patch reads)
|
||||
# retrobat retrobat-bios@158add8d pcsx2/resources/GameIndex.yaml
|
||||
# retrodeck retroarch/component_prepare.sh:95
|
||||
# Never opened: romm (EmulatorJS 4.2.3 ships no PS2 core).
|
||||
lrps2:
|
||||
source_url: "https://buildbot.libretro.com/assets/system/LRPS2.zip"
|
||||
source_type: zip
|
||||
for_platforms: [retroarch, lakka, retropie]
|
||||
for_platforms: [emudeck, knulli, retroarch, retropie, rocknix]
|
||||
local_cache: data/lrps2
|
||||
description: "LRPS2 GameIndex.yaml + resources"
|
||||
|
||||
@@ -132,10 +176,16 @@ data_directories:
|
||||
description: "XRick Rick Dangerous game data"
|
||||
|
||||
# ref: dinothawr/libretro.cpp (system/dinothawr/)
|
||||
# Installed (emulators/dinothawr.yml installed_on):
|
||||
# lakka packages/lakka/libretro_cores/dinothawr/package.mk:17-18
|
||||
# recalbox package/libretro-dinothawr/libretro-dinothawr.mk:32-33 (as content, roms/ports)
|
||||
# retrobat retrobat-bios@158add8d dinothawr/
|
||||
# retrodeck retroarch/component_prepare.sh:59
|
||||
# retropie scriptmodules/libretrocores/lr-dinothawr.sh:50 (as content, roms/ports)
|
||||
dinothawr:
|
||||
source_url: "https://buildbot.libretro.com/assets/system/Dinothawr.zip"
|
||||
source_type: zip
|
||||
for_platforms: [retroarch, lakka, retropie]
|
||||
for_platforms: [emudeck, emuelec, retroarch, rocknix]
|
||||
local_cache: data/dinothawr
|
||||
description: "Dinothawr game data"
|
||||
|
||||
@@ -217,6 +267,8 @@ data_directories:
|
||||
# freeintv overlays: Assets/Overlays.zip in repo (not a directory)
|
||||
# Already archived in bios/Mattel/Intellivision/freeintv_overlays/
|
||||
# No buildbot source, sourced manually from core repo
|
||||
# Never opened: batocera, knulli and recalbox build FreeIntv 85bf25a3, which
|
||||
# predates the overlays (libretro-freeintv.mk), and romm hands a core one file.
|
||||
freeintv-overlays:
|
||||
source_url: "https://github.com/libretro/FreeIntv/raw/{version}/Assets/Overlays.zip"
|
||||
source_type: zip
|
||||
@@ -237,6 +289,7 @@ data_directories:
|
||||
description: "DCSS game data (tiles, fonts, databases, lua, level descriptions)"
|
||||
|
||||
# ref: FBNeo src/burner/libretro/libretro.cpp (system/fbneo/cheats/)
|
||||
# No platform installs it. Never opened: romm (one file per core).
|
||||
fbneo-cheats:
|
||||
source_url: "https://github.com/finalburnneo/FBNeo-cheats/archive/refs/heads/{version}.zip"
|
||||
source_type: zip
|
||||
@@ -246,17 +299,26 @@ data_directories:
|
||||
description: "FinalBurn Neo per-game cheat definitions"
|
||||
|
||||
# ref: FBNeo src/burn/snd/samples.cpp (system/fbneo/samples/)
|
||||
# No platform installs it (batocera, knulli, recalbox, retrobat, retrodeck and
|
||||
# retropie create an empty fbneo/samples). Never opened: romm (RomM hands
|
||||
# EmulatorJS one file, views/Player/EmulatorJS/Player.vue:368-370).
|
||||
fbneo-samples:
|
||||
source_url: "https://github.com/Abdess/retrobios/releases/download/large-files/fbneo-samples.zip"
|
||||
source_type: zip
|
||||
for_platforms: [retroarch, lakka, retropie]
|
||||
for_platforms: [amberelec, batocera, emudeck, emuelec, knulli, lakka, myminui, nextui, recalbox, retroarch, retrobat, retrodeck, retropie, rocknix]
|
||||
local_cache: data/fbneo-samples
|
||||
description: "FinalBurn Neo per-game audio samples (49 arcade games)"
|
||||
|
||||
# ref: dirksimple (system/)
|
||||
# Installed (emulators/dirksimple.yml installed_on):
|
||||
# recalbox package/libretro-dirksimple/libretro-dirksimple.mk:21 (bios/daphne/DirkSimple, where its patch reads)
|
||||
# retrodeck retroarch/component_prepare.sh:60
|
||||
# retropie scriptmodules/libretrocores/lr-dirksimple.sh:49
|
||||
# lakka stays: packages/lakka/libretro_cores/dirksimple/package.mk:17-18 copies
|
||||
# data/games to DirkSimple/games, one level above DirkSimple/data/games.
|
||||
dirksimple:
|
||||
source_url: "https://buildbot.libretro.com/assets/system/DirkSimple.zip"
|
||||
source_type: zip
|
||||
for_platforms: [retroarch, lakka, retropie]
|
||||
for_platforms: [emudeck, lakka, retroarch, rocknix]
|
||||
local_cache: data/dirksimple
|
||||
description: "DirkSimple Dragon's Lair engine data"
|
||||
@@ -390,6 +390,21 @@
|
||||
"items": {"type": "string"},
|
||||
"description": "Platforms (registry keys) whose launcher points the emulator at other files, so this entry is never opened there."
|
||||
},
|
||||
"load_from": {
|
||||
"description": "Where the emulator reads the tree when not below the system directory, keyed by build when the builds differ (content_dir: beside the content). A platform pack carries only a tree read from the system directory.",
|
||||
"oneOf": [
|
||||
{"type": "string"},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"minProperties": 1,
|
||||
"properties": {
|
||||
"libretro": {"type": "string"},
|
||||
"standalone": {"type": "string"}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"source_ref": {
|
||||
"type": "string"
|
||||
}
|
||||
|
||||
@@ -333,10 +333,10 @@ def write_small_list(path: Path) -> None:
|
||||
def _child(spec: dict) -> int:
|
||||
"""install.main() as the sample runs it, in its own process. Only the
|
||||
transport and the machine are stood in for: no connection leaves the
|
||||
loopback, detection finds the card folder the spec names, a file of a
|
||||
landed run reports its bytes written without fetching them, and a held
|
||||
run finds every file in place. The counts, sizes and words are the
|
||||
installer's own."""
|
||||
loopback, detection finds the card folder the spec names, a file or a
|
||||
data tree of a landed run reports its bytes written without fetching
|
||||
them, and a held run finds every file and tree in place. The counts,
|
||||
sizes and words are the installer's own."""
|
||||
import concurrent.futures # noqa: PLC0415
|
||||
import socket # noqa: PLC0415
|
||||
from unittest import mock # noqa: PLC0415
|
||||
@@ -362,6 +362,13 @@ def _child(spec: dict) -> int:
|
||||
def held(files: list[dict], _bios: Path, **_kwargs: object) -> tuple:
|
||||
return [], list(files), []
|
||||
|
||||
def tree_landed(entry: dict, _bios: Path, state: object = None) -> dict:
|
||||
return {"present": 0, "installed": len(entry["files"]), "changed": 0,
|
||||
"absent": 0, "error": "", "not_kept": []}
|
||||
|
||||
def tree_held(entry: dict, bios: Path, remember: bool = False) -> tuple:
|
||||
return bios / entry.get("dest", ""), {}, len(entry["files"])
|
||||
|
||||
patches = [
|
||||
mock.patch.object(socket, "create_connection", loopback_only),
|
||||
# One worker and the submission order: the progress lines of a log.
|
||||
@@ -376,8 +383,10 @@ def _child(spec: dict) -> int:
|
||||
]
|
||||
if spec.get("landed"):
|
||||
patches.append(mock.patch.object(_install, "_download_one", landed))
|
||||
patches.append(mock.patch.object(_install, "install_data_directory", tree_landed))
|
||||
if spec.get("held"):
|
||||
patches.append(mock.patch.object(_install, "check_local", held))
|
||||
patches.append(mock.patch.object(_install, "data_directory_state", tree_held))
|
||||
with contextlib.ExitStack() as stack:
|
||||
for patch in patches:
|
||||
stack.enter_context(patch)
|
||||
|
||||
+204
-56
@@ -69,9 +69,14 @@ from common import (
|
||||
MANUFACTURER_PREFIXES,
|
||||
require_yaml,
|
||||
ProfileSelectionError,
|
||||
core_system_dirs,
|
||||
declared_root,
|
||||
entry_serves_systems,
|
||||
entry_system_ids,
|
||||
platform_layouts,
|
||||
sanitize_pack_path,
|
||||
select_emulator_profiles,
|
||||
standalone_set,
|
||||
system_core_reach,
|
||||
upstream_profile_index,
|
||||
yaml_load,
|
||||
@@ -100,6 +105,7 @@ from validation import (
|
||||
_build_validation_index,
|
||||
_parse_validation,
|
||||
deliverable,
|
||||
read_from_system_dir,
|
||||
with_archive_members,
|
||||
extra_alternatives,
|
||||
)
|
||||
@@ -376,6 +382,97 @@ def platform_list_version(config: dict, platform: str) -> str:
|
||||
PACK_DOCUMENTS = ("README.txt", "manifest.json")
|
||||
|
||||
|
||||
def core_data_trees(
|
||||
config: dict,
|
||||
profiles: dict[str, dict],
|
||||
platform_name: str,
|
||||
data_registry: dict | None,
|
||||
target_cores: set[str] | None = None,
|
||||
) -> list[dict]:
|
||||
"""The data directories the platform's cores read, as tree references.
|
||||
|
||||
A profile's ``data_directories`` reach the pack of every platform that
|
||||
runs the core as a libretro core, below the system directory it hands
|
||||
the core (and the build's own folder, ``pack_structure``), as the
|
||||
emulator pack lays them out. Left out: a tree the platform's package
|
||||
installs, for this core or another reading the same tree
|
||||
(``installed_data_refs``), one the build bundles or the platform never
|
||||
opens (``deliverable``), and one whose registry entry names the
|
||||
platforms it serves (``for_platforms``) without this one. A standalone build reads its data below its own folder, which a
|
||||
tree's destination does not describe, so its trees stay out, as does a
|
||||
tree read beside the content (``load_from``).
|
||||
|
||||
Each reference carries the profile's systems (``source_systems``, as a
|
||||
core extra does), which --system and --split narrow on.
|
||||
"""
|
||||
if not data_registry:
|
||||
return []
|
||||
installed = installed_data_refs(platform_name)
|
||||
handed = standalone_set(config)
|
||||
# Relative to base_destination, which the members are written under. A
|
||||
# platform with none writes its BIOS folder into every destination, and
|
||||
# the trees go there as the core extras do.
|
||||
root = declared_root(config)
|
||||
trees: list[dict] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for core in sorted(_platform_cores(config, profiles, target_cores)):
|
||||
profile = profiles[core]
|
||||
if all(standalone for _view, standalone in platform_layouts(core, profile, handed)):
|
||||
continue
|
||||
declared = [t for t in profile.get("data_directories") or [] if isinstance(t, dict)]
|
||||
directories = core_system_dirs(core, profile, handed)
|
||||
structure = str((profile.get("pack_structure") or {}).get("libretro") or "")
|
||||
for tree in deliverable(declared, False, platform_name):
|
||||
if not read_from_system_dir(tree):
|
||||
continue
|
||||
ref = str(tree.get("ref") or "")
|
||||
entry = data_registry.get(ref)
|
||||
if entry is None or ref in installed:
|
||||
continue
|
||||
allowed = entry.get("for_platforms")
|
||||
if allowed and platform_name not in allowed:
|
||||
continue
|
||||
for directory in [""] if directories is None else directories:
|
||||
destination = prefixed_destination("/".join(
|
||||
part.strip("/")
|
||||
for part in (directory, structure, str(tree.get("destination") or ""))
|
||||
if part.strip("/")
|
||||
), root) or root
|
||||
if (ref, destination) in seen:
|
||||
continue
|
||||
seen.add((ref, destination))
|
||||
trees.append({
|
||||
"ref": ref,
|
||||
"destination": destination,
|
||||
"required": tree.get("required"),
|
||||
"core": core,
|
||||
"source_systems": entry_system_ids(tree, profile),
|
||||
})
|
||||
return trees
|
||||
|
||||
|
||||
def _in_data_cache(path: str, data_registry: dict | None) -> bool:
|
||||
"""Whether a resolved file lies in one of the data directory caches."""
|
||||
real = os.path.realpath(path)
|
||||
return any(
|
||||
real.startswith(os.path.realpath(entry["local_cache"]) + os.sep)
|
||||
for entry in (data_registry or {}).values()
|
||||
if entry.get("local_cache")
|
||||
)
|
||||
|
||||
|
||||
def _tree_references(
|
||||
pack_systems: dict, core_trees: Iterable[dict],
|
||||
) -> Iterator[tuple[tuple[str, ...], dict]]:
|
||||
"""(systems, reference) for each tree a pack carries: those its systems
|
||||
declare, then those its cores read."""
|
||||
for sys_id, system in sorted(pack_systems.items()):
|
||||
for dd in system.get("data_directories", []):
|
||||
yield (sys_id,), dd
|
||||
for tree in core_trees:
|
||||
yield tuple(tree["source_systems"]), tree
|
||||
|
||||
|
||||
def _data_directory_members(
|
||||
pack_systems: dict,
|
||||
data_registry: dict | None,
|
||||
@@ -387,12 +484,20 @@ def _data_directory_members(
|
||||
seen_parents: set,
|
||||
refs_out: dict | None = None,
|
||||
required_only: bool = False,
|
||||
core_trees: Iterable[dict] = (),
|
||||
reclaim: dict[str, str] | None = None,
|
||||
) -> Iterator[tuple[str, str]]:
|
||||
"""Yield (source, destination) for the cached data directories a
|
||||
platform's systems declare.
|
||||
platform's systems declare, then those its cores read (``core_trees``,
|
||||
from core_data_trees).
|
||||
|
||||
``reclaim`` maps a destination already taken to the cache file it was
|
||||
resolved to: the manifest found the platform's entry there and cannot
|
||||
list it as a file, so the tree yields it once more and the installer
|
||||
gets it from the tree's archive.
|
||||
|
||||
``refs_out``, when given, maps each yielded destination to
|
||||
(registry key, system id, directory destination), for the manifest.
|
||||
(registry key, system ids, directory destination), for the manifest.
|
||||
|
||||
These are not BIOS and carry no hash of their own: they are whole
|
||||
trees an emulator reads, refreshed from upstream, so what ships is
|
||||
@@ -402,54 +507,61 @@ def _data_directory_members(
|
||||
core runs without, as it leaves out such a file.
|
||||
"""
|
||||
# Data directories from _data_dirs.yml
|
||||
for sys_id, system in sorted(pack_systems.items()):
|
||||
for dd in system.get("data_directories", []):
|
||||
ref_key = dd.get("ref", "")
|
||||
if not ref_key or not data_registry or ref_key not in data_registry:
|
||||
continue
|
||||
if required_only and dd.get("required") is False:
|
||||
continue
|
||||
if ref_key in installed_data_refs(platform_name):
|
||||
continue
|
||||
entry = data_registry[ref_key]
|
||||
allowed = entry.get("for_platforms")
|
||||
if allowed and platform_name not in allowed:
|
||||
continue
|
||||
local_path = entry.get("local_cache", "")
|
||||
if not local_path or not os.path.isdir(local_path):
|
||||
print(
|
||||
f" WARNING: data directory '{ref_key}' not cached at {local_path} (run refresh_data_dirs.py)"
|
||||
)
|
||||
_MISSING_DATA_DIRS.add(ref_key)
|
||||
continue
|
||||
dd_dest = dd.get("destination", "")
|
||||
if base_dest and dd_dest:
|
||||
dd_prefix = f"{base_dest}/{dd_dest}"
|
||||
elif base_dest:
|
||||
dd_prefix = base_dest
|
||||
else:
|
||||
dd_prefix = dd_dest
|
||||
# Held while the caller writes what is yielded: a refresh swapping
|
||||
# the tree waits until the walk is done.
|
||||
with cache_lock(local_path, shared=True):
|
||||
for root, _dirs, filenames in os.walk(local_path):
|
||||
for fname in filenames:
|
||||
src = os.path.join(root, fname)
|
||||
rel = os.path.relpath(src, local_path)
|
||||
full = f"{dd_prefix}/{rel}"
|
||||
if full in seen_destinations or (
|
||||
full.lower() in seen_lower and case_insensitive
|
||||
):
|
||||
continue
|
||||
if _has_path_conflict(full, seen_destinations, seen_parents):
|
||||
continue
|
||||
seen_destinations.add(full)
|
||||
_register_path(full, seen_destinations, seen_parents)
|
||||
if case_insensitive:
|
||||
seen_lower.add(full.lower())
|
||||
for systems, dd in _tree_references(pack_systems, core_trees):
|
||||
ref_key = dd.get("ref", "")
|
||||
if not ref_key or not data_registry or ref_key not in data_registry:
|
||||
continue
|
||||
if required_only and dd.get("required") is False:
|
||||
continue
|
||||
if ref_key in installed_data_refs(platform_name):
|
||||
continue
|
||||
entry = data_registry[ref_key]
|
||||
allowed = entry.get("for_platforms")
|
||||
if allowed and platform_name not in allowed:
|
||||
continue
|
||||
local_path = entry.get("local_cache", "")
|
||||
if not local_path or not os.path.isdir(local_path):
|
||||
print(
|
||||
f" WARNING: data directory '{ref_key}' not cached at {local_path} (run refresh_data_dirs.py)"
|
||||
)
|
||||
_MISSING_DATA_DIRS.add(ref_key)
|
||||
continue
|
||||
dd_dest = dd.get("destination", "")
|
||||
if base_dest and dd_dest:
|
||||
dd_prefix = f"{base_dest}/{dd_dest}"
|
||||
elif base_dest:
|
||||
dd_prefix = base_dest
|
||||
else:
|
||||
dd_prefix = dd_dest
|
||||
# Held while the caller writes what is yielded: a refresh swapping
|
||||
# the tree waits until the walk is done.
|
||||
with cache_lock(local_path, shared=True):
|
||||
for root, _dirs, filenames in os.walk(local_path):
|
||||
for fname in filenames:
|
||||
src = os.path.join(root, fname)
|
||||
rel = os.path.relpath(src, local_path)
|
||||
full = f"{dd_prefix}/{rel}" if dd_prefix else rel
|
||||
if reclaim and full in reclaim and (
|
||||
os.path.normpath(reclaim[full]) == os.path.normpath(src)
|
||||
):
|
||||
del reclaim[full]
|
||||
if refs_out is not None:
|
||||
refs_out[full] = (ref_key, sys_id, dd_prefix)
|
||||
refs_out[full] = (ref_key, systems, dd_prefix)
|
||||
yield src, full
|
||||
continue
|
||||
if full in seen_destinations or (
|
||||
full.lower() in seen_lower and case_insensitive
|
||||
):
|
||||
continue
|
||||
if _has_path_conflict(full, seen_destinations, seen_parents):
|
||||
continue
|
||||
seen_destinations.add(full)
|
||||
_register_path(full, seen_destinations, seen_parents)
|
||||
if case_insensitive:
|
||||
seen_lower.add(full.lower())
|
||||
if refs_out is not None:
|
||||
refs_out[full] = (ref_key, systems, dd_prefix)
|
||||
yield src, full
|
||||
|
||||
|
||||
def _pack_data_directories(
|
||||
@@ -464,13 +576,14 @@ def _pack_data_directories(
|
||||
seen_lower: set,
|
||||
seen_parents: set,
|
||||
required_only: bool = False,
|
||||
core_trees: Iterable[dict] = (),
|
||||
) -> int:
|
||||
"""Add the data directories to the pack. Returns how many files went in."""
|
||||
added = 0
|
||||
for src, full in _data_directory_members(
|
||||
pack_systems, data_registry, platform_name, base_dest,
|
||||
case_insensitive, seen_destinations, seen_lower, seen_parents,
|
||||
required_only=required_only,
|
||||
required_only=required_only, core_trees=core_trees,
|
||||
):
|
||||
_add_pack_member(zf, src, _flat(full, base_dest, flatten))
|
||||
added += 1
|
||||
@@ -1169,10 +1282,23 @@ def generate_pack(
|
||||
save_count += 1
|
||||
total_files += 1
|
||||
|
||||
# The trees the cores read follow them as their extras do: by the
|
||||
# systems asked for, and those of no system in the part of their own.
|
||||
core_trees = (
|
||||
[] if source == "platform"
|
||||
else core_data_trees(
|
||||
config, emu_profiles, platform_name, data_registry, target_cores
|
||||
)
|
||||
)
|
||||
if extras_only:
|
||||
owned = list(platform_scope_systems(config, emu_profiles, target_cores))
|
||||
core_trees = [t for t in core_trees if not _extras_for_systems([t], owned)]
|
||||
elif system_filter:
|
||||
core_trees = _extras_for_systems(core_trees, system_filter)
|
||||
total_files += _pack_data_directories(
|
||||
zf, pack_systems, data_registry, platform_name, base_dest,
|
||||
flatten, case_insensitive, seen_destinations, seen_lower,
|
||||
seen_parents, required_only=required_only,
|
||||
seen_parents, required_only=required_only, core_trees=core_trees,
|
||||
)
|
||||
|
||||
# README.txt for users: personalized step-by-step per platform
|
||||
@@ -4334,6 +4460,9 @@ def generate_manifest(
|
||||
# Sizes of the files the pack carries and the installer cannot fetch: the
|
||||
# collection does not index them, a data directory cache answered.
|
||||
pack_only_sizes: list[int] = []
|
||||
# Platform entries resolved into a data directory cache: destination ->
|
||||
# (cache file, size). Settled once the trees are walked.
|
||||
cache_held: dict[str, tuple[str, int]] = {}
|
||||
rename_notes: set[str] = set()
|
||||
slot_overrides = slots.pack_overrides(
|
||||
config, emu_profiles, db, zip_contents, data_registry,
|
||||
@@ -4564,7 +4693,10 @@ def generate_manifest(
|
||||
if entry is None:
|
||||
record_omission(full_dest, file_entry, sys_id, "not_found", None)
|
||||
if file_size and full_dest not in seen_destinations:
|
||||
pack_only_sizes.append(file_size)
|
||||
if _in_data_cache(local_path, data_registry):
|
||||
cache_held[full_dest] = (local_path, file_size)
|
||||
else:
|
||||
pack_only_sizes.append(file_size)
|
||||
seen_destinations.add(dedup_key)
|
||||
_register_path(dedup_key, seen_destinations, seen_parents)
|
||||
if case_insensitive:
|
||||
@@ -4643,12 +4775,28 @@ def generate_manifest(
|
||||
# Phase 3: data directories. They are not files of the collection: the
|
||||
# installer fetches each from its source archive and keeps the members
|
||||
# whose hash the list below records, the same tree the pack carries.
|
||||
data_refs: dict[str, tuple[str, str, str]] = {}
|
||||
data_refs: dict[str, tuple[str, tuple[str, ...], str]] = {}
|
||||
reclaim = {full: path for full, (path, _size) in cache_held.items()}
|
||||
data_members = list(_data_directory_members(
|
||||
pack_systems, data_registry, platform_name, base_dest,
|
||||
case_insensitive, seen_destinations, seen_lower, seen_parents,
|
||||
data_refs, required_only=required_only,
|
||||
core_trees=(
|
||||
[] if source == "platform"
|
||||
else core_data_trees(
|
||||
config, emu_profiles, platform_name, data_registry, target_cores
|
||||
)
|
||||
),
|
||||
reclaim=reclaim,
|
||||
))
|
||||
# A platform entry found in a tree the pack carries is that tree's member:
|
||||
# the installer rebuilds it from the tree's archive. One found in a tree
|
||||
# the pack does not carry stays omitted, held by the pack alone.
|
||||
for full_dest, (_path, size) in cache_held.items():
|
||||
if full_dest in reclaim:
|
||||
pack_only_sizes.append(size)
|
||||
else:
|
||||
omitted_by_destination.pop(full_dest, None)
|
||||
data_sizes = pack_only_sizes + [os.path.getsize(src) for src, _dest in data_members]
|
||||
data_directories = _manifest_data_directories(
|
||||
data_members, data_refs, data_registry or {}, base_dest
|
||||
@@ -4738,7 +4886,7 @@ def generate_manifest(
|
||||
|
||||
def _manifest_data_directories(
|
||||
members: list[tuple[str, str]],
|
||||
refs: dict[str, tuple[str, str, str]],
|
||||
refs: dict[str, tuple[str, tuple[str, ...], str]],
|
||||
registry: dict,
|
||||
base_dest: str,
|
||||
) -> list[dict]:
|
||||
@@ -4752,7 +4900,7 @@ def _manifest_data_directories(
|
||||
"""
|
||||
grouped: dict[tuple[str, str], dict] = {}
|
||||
for src, full in members:
|
||||
ref, system, prefix = refs[full]
|
||||
ref, systems, prefix = refs[full]
|
||||
entry = registry.get(ref, {})
|
||||
key = (ref, prefix)
|
||||
group = grouped.get(key)
|
||||
@@ -4775,8 +4923,7 @@ def _manifest_data_directories(
|
||||
group["source_path"] = str(entry.get("source_path", "")).format(version=version)
|
||||
if entry.get("exclude"):
|
||||
group["exclude"] = [str(e) for e in entry["exclude"]]
|
||||
if system not in group["systems"]:
|
||||
group["systems"].append(system)
|
||||
group["systems"].extend(s for s in systems if s not in group["systems"])
|
||||
rel = full[len(prefix):].lstrip("/") if prefix else full
|
||||
hashes = compute_hashes(src, frozenset({"sha1"}))
|
||||
group["files"][rel] = {"sha1": hashes["sha1"], "size": os.path.getsize(src)}
|
||||
@@ -5222,6 +5369,7 @@ from packpaths import ( # noqa: E402,F401
|
||||
from packextras import ( # noqa: E402,F401
|
||||
ExtraResolver,
|
||||
extra_full_destination,
|
||||
prefixed_destination,
|
||||
save_entry_fields,
|
||||
save_manifest_dest,
|
||||
_detect_extras_prefix,
|
||||
|
||||
+10
-3
@@ -433,19 +433,26 @@ def parse_pack_paths(output: str) -> dict[str, str]:
|
||||
_PACK_NOTES = re.compile(r"(^README\.txt$|^manifest\.json$|(^|/)(INSTRUCTIONS|RENAMED)_[^/]*\.txt$)")
|
||||
|
||||
|
||||
def _tree_members(platform: str, platforms_dir: str) -> dict[str, int]:
|
||||
def _tree_members(
|
||||
platform: str, platforms_dir: str, emulators_dir: str = "emulators",
|
||||
) -> dict[str, int]:
|
||||
"""Archive path -> size of every file the platform's data trees can add."""
|
||||
import generate_pack # noqa: PLC0415
|
||||
from common import load_platform_config # noqa: PLC0415
|
||||
from common import load_emulator_profiles, load_platform_config # noqa: PLC0415
|
||||
from packpaths import _flat # noqa: PLC0415
|
||||
|
||||
config = load_platform_config(platform, platforms_dir)
|
||||
base_dest = config.get("base_destination", "")
|
||||
registry = load_data_dir_registry(platforms_dir)
|
||||
core_trees = generate_pack.core_data_trees(
|
||||
config, load_emulator_profiles(emulators_dir), platform, registry,
|
||||
)
|
||||
members: dict[str, int] = {}
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
for src, full in generate_pack._data_directory_members(
|
||||
config.get("systems", {}), load_data_dir_registry(platforms_dir),
|
||||
config.get("systems", {}), registry,
|
||||
platform, base_dest, False, set(), set(), set(),
|
||||
core_trees=core_trees,
|
||||
):
|
||||
members[_flat(full, base_dest, True)] = os.path.getsize(src)
|
||||
return members
|
||||
|
||||
@@ -5544,7 +5544,10 @@ class DocumentedExamplesHoldTogether(unittest.TestCase):
|
||||
page = (ROOT / "wiki" / "installer.md").read_text(encoding="utf-8")
|
||||
downloading = re.search(r"Downloading (\d+) files", page).group(1)
|
||||
done = re.search(r"Done\. (\d+) files installed", page).group(1)
|
||||
self.assertEqual(done, downloading)
|
||||
# The data trees come after the download, counted in the index.
|
||||
index = page[page.index("Fetching file index for batocera"):]
|
||||
trees = re.search(r"^ .* and (\d+) data files", index, re.M)
|
||||
self.assertEqual(int(done), int(downloading) + (int(trees.group(1)) if trees else 0))
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
"""A data directory a core reads reaches every platform that runs the core.
|
||||
|
||||
The registry entries of the buildbot trees named RetroArch, Lakka and
|
||||
RetroPie only, and a platform pack carried only the trees its own systems
|
||||
declared. AmberELEC and EmuELEC run the blueMSX core, whose package copies
|
||||
the .so alone: their packs filled the machine configs by name from the
|
||||
cache, and the install lists left 192 of them out. A platform that runs a
|
||||
reader now gets the tree, or the profile says its package installs it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
import common # noqa: E402
|
||||
import generate_pack as builder # noqa: E402
|
||||
from validation import deliverable, read_from_system_dir # noqa: E402
|
||||
|
||||
PROFILES = {
|
||||
"demo": {
|
||||
"emulator": "Demo",
|
||||
"type": "libretro",
|
||||
"systems": ["demo-system"],
|
||||
"cores": ["demo"],
|
||||
"pack_structure": {"libretro": "Demo"},
|
||||
"data_directories": [
|
||||
{"ref": "shared", "destination": "Sys"},
|
||||
{"ref": "restricted"},
|
||||
{"ref": "installed", "installed_on": ["plat"]},
|
||||
{"ref": "beside", "load_from": "content_dir"},
|
||||
],
|
||||
},
|
||||
"solo": {
|
||||
"emulator": "Solo",
|
||||
"type": "standalone",
|
||||
"systems": ["demo-system"],
|
||||
"cores": ["solo"],
|
||||
"data_directories": [{"ref": "shared"}],
|
||||
},
|
||||
}
|
||||
|
||||
REGISTRY = {
|
||||
"shared": {"local_cache": "data/shared"},
|
||||
"restricted": {"local_cache": "data/restricted", "for_platforms": ["other"]},
|
||||
"installed": {"local_cache": "data/installed"},
|
||||
"beside": {"local_cache": "data/beside"},
|
||||
}
|
||||
|
||||
CONFIG = {
|
||||
"platform": "Plat",
|
||||
"cores": ["demo", "solo"],
|
||||
"standalone_cores": ["solo"],
|
||||
"systems": {"demo-system": {"files": []}},
|
||||
}
|
||||
|
||||
|
||||
class CoreDataTreesTest(unittest.TestCase):
|
||||
def trees(self, platform: str = "plat", registry: dict | None = None) -> list[dict]:
|
||||
return builder.core_data_trees(
|
||||
CONFIG, PROFILES, platform, REGISTRY if registry is None else registry
|
||||
)
|
||||
|
||||
def test_tree_below_the_build_folder(self) -> None:
|
||||
shared = [t for t in self.trees() if t["ref"] == "shared"]
|
||||
self.assertEqual([(t["core"], t["destination"]) for t in shared], [("demo", "Demo/Sys")])
|
||||
self.assertEqual(shared[0]["source_systems"], ["demo-system"])
|
||||
|
||||
def test_for_platforms_names_who_gets_it(self) -> None:
|
||||
self.assertNotIn("restricted", {t["ref"] for t in self.trees()})
|
||||
self.assertIn("restricted", {t["ref"] for t in self.trees("other")})
|
||||
|
||||
def test_installed_tree_stays_out(self) -> None:
|
||||
self.assertNotIn("installed", {t["ref"] for t in self.trees()})
|
||||
self.assertIn("installed", {t["ref"] for t in self.trees("other")})
|
||||
|
||||
def test_tree_read_beside_the_content_stays_out(self) -> None:
|
||||
self.assertNotIn("beside", {t["ref"] for t in self.trees("other")})
|
||||
|
||||
def test_standalone_build_brings_no_tree(self) -> None:
|
||||
self.assertNotIn("solo", {t["core"] for t in self.trees("other")})
|
||||
|
||||
def test_tree_below_the_declared_bios_folder(self) -> None:
|
||||
config = {
|
||||
**CONFIG,
|
||||
"base_destination": "",
|
||||
"systems": {"demo-system": {"files": [{"name": "a.bin", "destination": "bios/a.bin"}]}},
|
||||
}
|
||||
trees = builder.core_data_trees(config, PROFILES, "plat", REGISTRY)
|
||||
self.assertEqual(
|
||||
[t["destination"] for t in trees if t["ref"] == "shared"], ["bios/Demo/Sys"]
|
||||
)
|
||||
|
||||
def test_no_registry_no_tree(self) -> None:
|
||||
self.assertEqual(builder.core_data_trees(CONFIG, PROFILES, "plat", None), [])
|
||||
|
||||
|
||||
class EveryReaderHasTheTreeOrAReason(unittest.TestCase):
|
||||
"""Reads the repository: platforms/_data_dirs.yml against the profiles."""
|
||||
|
||||
def test_for_platforms_leaves_no_reader_without_a_reason(self) -> None:
|
||||
platforms_dir = str(REPO_ROOT / "platforms")
|
||||
registry = common.load_data_dir_registry(platforms_dir)
|
||||
profiles = common.load_emulator_profiles(str(REPO_ROOT / "emulators"))
|
||||
unexplained = []
|
||||
for platform in common.list_registered_platforms(platforms_dir, include_archived=True):
|
||||
config = common.load_platform_config(platform, platforms_dir)
|
||||
handed = common.standalone_set(config)
|
||||
for core in sorted(common.resolve_platform_cores(config, profiles)):
|
||||
profile = profiles[core]
|
||||
if all(standalone for _view, standalone
|
||||
in common.platform_layouts(core, profile, handed)):
|
||||
continue
|
||||
trees = [t for t in profile.get("data_directories") or [] if isinstance(t, dict)]
|
||||
installed = common.installed_data_refs(platform)
|
||||
for tree in deliverable(trees, False, platform):
|
||||
allowed = registry.get(tree.get("ref"), {}).get("for_platforms")
|
||||
if tree.get("ref") in installed or not read_from_system_dir(tree):
|
||||
continue
|
||||
if allowed and platform not in allowed:
|
||||
unexplained.append(f"{tree['ref']} on {platform} ({core})")
|
||||
self.assertEqual(
|
||||
unexplained, [],
|
||||
"these platforms run a core that reads the tree, are not in its "
|
||||
"for_platforms, and no profile says their package installs it "
|
||||
"(installed_on) or that they never open it (unread_on)",
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.path.isdir(REPO_ROOT / "data" / "bluemsx"), "data/bluemsx is not cached"
|
||||
)
|
||||
class AmberelecCarriesTheMachines(unittest.TestCase):
|
||||
"""AmberELEC's blueMSX package installs the core alone, so the pack and
|
||||
the install list bring the Machines tree its BIOS list names."""
|
||||
|
||||
def test_machine_configs_come_with_the_tree(self) -> None:
|
||||
platforms_dir = str(REPO_ROOT / "platforms")
|
||||
cwd = os.getcwd()
|
||||
os.chdir(REPO_ROOT)
|
||||
try:
|
||||
config = common.load_platform_config("amberelec", platforms_dir)
|
||||
registry = common.load_data_dir_registry(platforms_dir)
|
||||
profiles = common.load_emulator_profiles(str(REPO_ROOT / "emulators"))
|
||||
trees = builder.core_data_trees(config, profiles, "amberelec", registry)
|
||||
refs: dict = {}
|
||||
members = dict(
|
||||
(full, src) for src, full in builder._data_directory_members(
|
||||
config.get("systems", {}), registry, "amberelec",
|
||||
config.get("base_destination", ""), False, set(), set(), set(),
|
||||
refs, core_trees=trees,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
# Below bios/, where the platform writes every destination.
|
||||
self.assertIn("bios/Machines/COL - ColecoVision/config.ini", members)
|
||||
self.assertEqual(
|
||||
refs["bios/Machines/COL - ColecoVision/config.ini"][0], "bluemsx"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -845,9 +845,14 @@ class FirstInstallWalkthrough(unittest.TestCase):
|
||||
self.assertNotRegex(self.text, r"\d+ (in place|missing|files installed)")
|
||||
self.assertIn(f" {install.EVERYTHING_IN_PLACE}", self.text)
|
||||
step3 = _generated(self.run, "run-index")
|
||||
count, size = re.search(r"\n (\d+) files \(([\d.]+ [KMG]B)\)", step3).groups()
|
||||
count = re.search(r"\n (\d+) files \([\d.]+ [KMG]B\)", step3).group(1)
|
||||
self.assertIn(f"0 in place, {count} missing", step3)
|
||||
self.assertIn(f"Downloading {count} files ({size})", _generated(self.run, "run-download"))
|
||||
# The data trees are fetched after the files, from their archives:
|
||||
# the download line counts the files alone.
|
||||
trees = re.search(r" and (\d+) data files", step3)
|
||||
downloaded = int(count) - (int(trees.group(1)) if trees else 0)
|
||||
self.assertRegex(_generated(self.run, "run-download"),
|
||||
rf"Downloading {downloaded} files \([\d.]+ [KMG]B\)")
|
||||
step4 = _generated(self.run, "run-end")
|
||||
self.assertIn(f"Done. {count} files installed", step4)
|
||||
self.assertIn(f"{count} in place, 0 missing", _generated(self.run, "run-check"))
|
||||
|
||||
@@ -20,6 +20,10 @@
|
||||
"emulators/data-dolphin-sys-2.md": "emulators/dolphin.md",
|
||||
"emulators/data-dolphin-sys-3.md": "emulators/dolphin.md",
|
||||
"emulators/data-dolphin-sys.md": "emulators/dolphin.md",
|
||||
"emulators/data-fbneo-cheats-2.md": "emulators/fbneo.md",
|
||||
"emulators/data-fbneo-cheats-3.md": "emulators/fbneo.md",
|
||||
"emulators/data-fbneo-cheats-4.md": "emulators/fbneo.md",
|
||||
"emulators/data-fbneo-cheats.md": "emulators/fbneo.md",
|
||||
"emulators/mame-files-1.md": "emulators/mame.md",
|
||||
"emulators/mame-files-10.md": "emulators/mame.md",
|
||||
"emulators/mame-files-2.md": "emulators/mame.md",
|
||||
|
||||
+14
-14
@@ -403,13 +403,13 @@ A RetroArch dry run into an empty folder, with the list of 2026-11-05, printed t
|
||||
```
|
||||
Fetching file index for retroarch...
|
||||
106 files read from RetroArch's save directory go to a folder per core under /home/me/.config/retroarch/saves
|
||||
11002 files (6.7 GB)
|
||||
3272 BIOS files (4.1 GB), 5089 game data files (2.5 GB) and 2641 data files (40.2 MB).
|
||||
14942 files (6.8 GB)
|
||||
3272 BIOS files (4.1 GB), 5089 game data files (2.5 GB) and 6581 data files (135.4 MB).
|
||||
Game data: files a game or a game engine reads, of no use to the other cores. Data files: the files an emulator reads from a data folder of its own, such as Dolphin's Sys and PPSSPP's assets.
|
||||
Not installable: 37 files are not in the collection, 2 of them required.
|
||||
|
||||
Checking existing files...
|
||||
0 in place, 11002 missing, 0 wrong copies
|
||||
0 in place, 14942 missing, 0 wrong copies
|
||||
```
|
||||
<!-- /generated: dry-run -->
|
||||
|
||||
@@ -468,13 +468,13 @@ few minutes on a memory card, because every file is hashed:
|
||||
```
|
||||
Fetching file index for batocera...
|
||||
23 files read from Batocera's save directory go to a folder per system under /userdata/saves
|
||||
14724 files (9.2 GB)
|
||||
3180 BIOS files (4.4 GB) and 11544 game data files (4.8 GB).
|
||||
Game data: files a game or a game engine reads, of no use to the other cores.
|
||||
18196 files (9.3 GB)
|
||||
3180 BIOS files (4.4 GB), 11544 game data files (4.8 GB) and 3472 data files (81.8 MB).
|
||||
Game data: files a game or a game engine reads, of no use to the other cores. Data files: the files an emulator reads from a data folder of its own, such as Dolphin's Sys and PPSSPP's assets.
|
||||
Not installable: 426 files are not in the collection, 80 of them required.
|
||||
|
||||
Checking existing files...
|
||||
0 in place, 14724 missing, 0 wrong copies
|
||||
0 in place, 18196 missing, 0 wrong copies
|
||||
```
|
||||
<!-- /generated: run-index -->
|
||||
|
||||
@@ -504,8 +504,8 @@ command again picks up where it stopped.
|
||||
<!-- generated: run-end -->
|
||||
|
||||
```
|
||||
Done. 14724 files installed.
|
||||
Installed: 3180 BIOS files and 11544 game data files.
|
||||
Done. 18196 files installed.
|
||||
Installed: 3180 BIOS files, 11544 game data files and 3472 data files.
|
||||
BIOS folder: /userdata/bios
|
||||
426 files were not installed: the collection does not hold them, or its copy fails their check.
|
||||
Needed to start an emulator (1 file):
|
||||
@@ -559,7 +559,7 @@ The same command with `--check` reads the folder and writes nothing:
|
||||
|
||||
```
|
||||
Checking existing files...
|
||||
14724 in place, 0 missing, 0 wrong copies
|
||||
18196 in place, 0 missing, 0 wrong copies
|
||||
|
||||
Everything is in place.
|
||||
```
|
||||
@@ -753,8 +753,8 @@ by `scripts/generate_installer_doc.py`):
|
||||
<!-- generated: line-menu -->
|
||||
|
||||
```
|
||||
RetroArch: 11002 files, 6.7 GB.
|
||||
3272 BIOS files (4.1 GB), 5089 game data files (2.5 GB) and 2641 data files (40.2 MB).
|
||||
RetroArch: 14942 files, 6.8 GB.
|
||||
3272 BIOS files (4.1 GB), 5089 game data files (2.5 GB) and 6581 data files (135.4 MB).
|
||||
Game data: files a game or a game engine reads, of no use to the other cores. Data files: the files an emulator reads from a data folder of its own, such as Dolphin's Sys and PPSSPP's assets.
|
||||
BIOS folder: /home/me/.config/retroarch/system
|
||||
Not available here, and needed to start its emulator: sfiii2a.zip (Capcom Cps3). Your own copy goes in the BIOS folder.
|
||||
@@ -768,8 +768,8 @@ RetroArch: 11002 files, 6.7 GB.
|
||||
|
||||
Counted again for your choices. Nothing is written before Enter.
|
||||
|
||||
RetroArch: 5913 files, 4.2 GB.
|
||||
3272 BIOS files (4.1 GB) and 2641 data files (40.2 MB). 5089 game data files left out.
|
||||
RetroArch: 9853 files, 4.3 GB.
|
||||
3272 BIOS files (4.1 GB) and 6581 data files (135.4 MB). 5089 game data files left out.
|
||||
Game data: files a game or a game engine reads, of no use to the other cores. Data files: the files an emulator reads from a data folder of its own, such as Dolphin's Sys and PPSSPP's assets.
|
||||
BIOS folder: /home/me/.config/retroarch/system
|
||||
Not available here, and needed to start its emulator: sfiii2a.zip (Capcom Cps3). Your own copy goes in the BIOS folder.
|
||||
|
||||
+1
-1
@@ -548,7 +548,7 @@ reports the profile as uncheckable, and the CI job that runs
|
||||
| `mode` | no | default mode: `standalone`, `libretro`, or `both` |
|
||||
| `verification` | no | how the core verifies BIOS: `existence`, `md5`, `sha1`, `crc32` |
|
||||
| `files` | yes, unless `type: alias` | list of file entries |
|
||||
| `data_directories` | no | whole directory trees the core needs, referencing `_data_dirs.yml` keys. An entry may carry `mode:` when only one build reads the tree, and `required: false` when the core runs without it (`--required-only` then leaves it out) |
|
||||
| `data_directories` | no | whole directory trees the core needs, referencing `_data_dirs.yml` keys. An entry may carry `mode:` when only one build reads the tree, `required: false` when the core runs without it (`--required-only` then leaves it out), and `load_from: content_dir` when the core reads it beside the content rather than in the system directory. Every platform that runs the core as a libretro core gets the tree in its pack, unless the entry names it in `installed_on` or `unread_on`, or the `_data_dirs.yml` entry has a `for_platforms` list without it |
|
||||
| `notes` | no | free-form technical notes |
|
||||
| `note` | no | single-paragraph variant of `notes` |
|
||||
| `exclusion_note` | no | why the profile has no files despite .info declaring firmware |
|
||||
|
||||
@@ -52,7 +52,7 @@ Every module in `tests/`, by the area it guards.
|
||||
| Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` |
|
||||
| File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_archive_recipes`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_np2_fonts`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` |
|
||||
| Verification, cross-reference and profile checks | synthetic + profiles | `test_bios_layout`, `test_cache_independent_counts`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_names`, `test_profile_paths`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` |
|
||||
| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
|
||||
| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_core_data_trees`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
|
||||
| Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
|
||||
| Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
|
||||
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_games`, `test_install_offline`, `test_install_selection`, `test_install_tui` |
|
||||
|
||||
Reference in new issue
Block a user