feat: pack the files a directory entry names

This commit is contained in:
Abdessamad Derraz committed 2026-10-04 17:58:10 +02:00
1 parent 9059234810
commit 2efc37a738
12 files changed
+465 -4

No files matched your search

+3
View File
@@ -45,6 +45,7 @@ files:
# ================================================================
- name: "Animations/"
type: directory
category: game_data
path: "jazz2/Content/Animations/"
standalone_path: "Content/Animations/"
@@ -54,6 +55,7 @@ files:
note: "Native graphics, sounds and the UI fonts, opened through OpenContentFile."
- name: "Metadata/"
type: directory
category: game_data
path: "jazz2/Content/Metadata/"
standalone_path: "Content/Metadata/"
@@ -63,6 +65,7 @@ files:
note: "Actor .res descriptors (Common, Enemy, Player, UI, Boss, ...)."
- name: "Translations/"
type: directory
category: game_data
path: "jazz2/Content/Translations/"
standalone_path: "Content/Translations/"
+6
View File
@@ -26,6 +26,7 @@ notes: |
files:
- name: "Standard"
type: directory
description: "RPG Maker XP Run-Time Package"
category: game_data
path: "mkxp-z/RTP/Standard/"
@@ -33,6 +34,7 @@ files:
source_ref: "src/core.cpp:1081-1124, src/core.cpp:1126-1197"
- name: "RPGVX"
type: directory
description: "RPG Maker VX Run-Time Package"
category: game_data
path: "mkxp-z/RTP/RPGVX/"
@@ -40,6 +42,7 @@ files:
source_ref: "src/core.cpp:1081-1124, src/core.cpp:1126-1197"
- name: "RPGVXAce"
type: directory
description: "RPG Maker VX Ace Run-Time Package"
category: game_data
path: "mkxp-z/RTP/RPGVXAce/"
@@ -47,6 +50,7 @@ files:
source_ref: "src/core.cpp:1081-1124, src/core.cpp:1126-1197"
- name: "Fonts"
type: directory
description: "Fallback font directory (.otf/.ttf)"
category: game_data
path: "mkxp-z/Fonts/"
@@ -54,6 +58,7 @@ files:
source_ref: "src/core.cpp:1199-1211"
- name: "Preload"
type: directory
description: "Ruby scripts loaded before game scripts"
category: game_data
path: "mkxp-z/Scripts/Preload/"
@@ -61,6 +66,7 @@ files:
source_ref: "src/core.cpp:891-934"
- name: "Postload"
type: directory
description: "Ruby scripts loaded after game scripts (RGSS3 only)"
category: game_data
path: "mkxp-z/Scripts/Postload/"
+5
View File
@@ -67,6 +67,7 @@ files:
# --- ADPCM audio samples (optional, game-specific) ---
- name: "nestopia/samples/moepro/"
type: directory
system: nintendo-nes
description: "Moero Pro Yakyuu audio samples (16 .wav files, 00.wav-15.wav)"
required: false
@@ -74,6 +75,7 @@ files:
source_ref: "libretro/libretro.cpp:183-391 (load_wav), libretro/libretro.cpp:451 (LOAD_SAMPLE_MOERO_PRO_YAKYUU callback), source/core/NstSoundPlayer.hpp:49 (16 samples)"
- name: "nestopia/samples/moepro88/"
type: directory
system: nintendo-nes
description: "Moero Pro Yakyuu '88 audio samples (20 .wav files, 00.wav-19.wav)"
required: false
@@ -81,6 +83,7 @@ files:
source_ref: "libretro/libretro.cpp:183-391 (load_wav), libretro/libretro.cpp:453 (LOAD_SAMPLE_MOERO_PRO_YAKYUU_88 callback), source/core/NstSoundPlayer.hpp:50 (20 samples)"
- name: "nestopia/samples/mptennis/"
type: directory
system: nintendo-nes
description: "Moero Pro Tennis audio samples (19 .wav files, 00.wav-18.wav)"
required: false
@@ -88,6 +91,7 @@ files:
source_ref: "libretro/libretro.cpp:183-391 (load_wav), libretro/libretro.cpp:455 (LOAD_SAMPLE_MOERO_PRO_TENNIS callback), source/core/NstSoundPlayer.hpp:51 (19 samples)"
- name: "nestopia/samples/terao/"
type: directory
system: nintendo-nes
description: "Terao no Dosukoi Oozumou audio samples (6 .wav files, 00.wav-05.wav)"
required: false
@@ -95,6 +99,7 @@ files:
source_ref: "libretro/libretro.cpp:183-391 (load_wav), libretro/libretro.cpp:457 (LOAD_SAMPLE_TERAO_NO_DOSUKOI_OOZUMOU callback), source/core/NstSoundPlayer.hpp:52 (6 samples)"
- name: "nestopia/samples/ftaerobi/"
type: directory
system: nintendo-nes
description: "Aerobics Studio audio samples (8 .wav files, 00.wav-07.wav)"
required: false
+1
View File
@@ -50,6 +50,7 @@ files:
- name: "data/"
path: "nxengine/data/"
type: directory
category: game_data
system: cave-story
description: "Full game asset directory tree (399 files: sprites, NPC sheets, stage maps, scripts, backgrounds)"
+1
View File
@@ -81,6 +81,7 @@ files:
source_ref: "libretro.c:1162-1167"
- name: "voice/"
type: directory
required: false
category: game_data
note: "The Voice speech synthesis WAV samples. 9 banks (E4, E8-EF) × 128 samples."
+6
View File
@@ -90,18 +90,21 @@ files:
# ================================================================
- name: "graphics/"
type: directory
category: game_data
required: true
source_ref: "src/main/asset_pack.cpp:274, 326-327, src/gfx.cpp:121-229, src/load_gfx.cpp:120-123"
note: "graphics/ui/ must exist for a directory to be scanned as a pack; GFX_t::load rejects the pack when a required UI image is missing."
- name: "sound/"
type: directory
category: game_data
required: false
source_ref: "src/sound.cpp:1319"
note: "SFX directory, files named by sounds.ini. Game runs without sounds."
- name: "music/"
type: directory
category: game_data
required: false
source_ref: "src/sound.cpp:1318"
@@ -132,6 +135,7 @@ files:
note: "Read when intro.lvlx is absent."
- name: "introset/"
type: directory
category: game_data
required: false
source_ref: "src/game_main.cpp:194-222, 247"
@@ -150,12 +154,14 @@ files:
note: "Read when outro.lvlx is absent."
- name: "worlds/"
type: directory
category: game_data
required: false
source_ref: "src/main/menu_main.cpp:361-370"
note: "Episodes shipped with the pack, listed next to the user's worlds directory."
- name: "battle/"
type: directory
category: game_data
required: false
source_ref: "src/main/menu_main.cpp:701-706"
+36 -1
View File
@@ -425,7 +425,11 @@
"type": "string"
},
"type": {
"type": "string"
"description": "Set on an entry that names a directory the code reads whole. The entry then stands for every file under path.",
"type": "string",
"enum": [
"directory"
]
},
"variant_group": {
"type": "string",
@@ -599,9 +603,40 @@
"unsourceable": {
"type": "string"
}
},
"allOf": [
{
"if": {
"properties": {
"name": {
"pattern": "/$"
}
}
},
"then": {
"required": [
"type"
]
}
},
{
"if": {
"required": [
"type"
]
},
"then": {
"properties": {
"path": {
"type": "string",
"pattern": "/$"
}
}
}
}
]
}
},
"source_branch": {
"type": "string",
"description": "Branch the port was read from, when it is not the default one."
+95
View File
@@ -1116,6 +1116,101 @@ def group_identical_platforms(
return result
_directory_members_cache: dict[int, tuple[dict, dict[str, dict[str, str]]]] = {}
def directory_members(db: dict, directory: str) -> dict[str, str]:
"""Files the collection holds under a directory an emulator reads whole.
Returns {destination: sha1}, each destination starting with the directory
as the profile writes it. The collection and the destination meet on a
tail, so the members are the indexed tails that begin with it.
A generic directory name can sit in two unrelated trees (``voice/``,
``data/``). Members that do not share one parent, or a destination two
contents claim, would merge them into one emulator's directory: nothing
is returned and the entry stays a reported gap.
"""
tail = directory.strip("/")
if not tail:
return {}
by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
cached = _directory_members_cache.get(id(by_path_suffix))
if cached is None or cached[0] is not by_path_suffix:
cached = (by_path_suffix, {})
_directory_members_cache[id(by_path_suffix)] = cached
if tail in cached[1]:
return cached[1][tail]
prefix = f"{tail}/"
infix = f"/{prefix}"
held: dict[str, list[str]] = {}
parent: dict[str, str] = {}
for key, sha1s in by_path_suffix.items():
if key.startswith(prefix):
held[key] = sha1s
continue
at = key.find(infix)
if at > 0:
member = key[at + 1:]
if len(key[:at]) > len(parent.get(member, "")):
parent[member] = key[:at]
held = {
dest: sha1s for dest, sha1s in held.items() if "/.variants/" not in dest
}
one_tree = len({parent.get(dest, "") for dest in held}) <= 1
one_content = all(len(sha1s) == 1 for sha1s in held.values())
members = (
{dest: held[dest][0] for dest in sorted(held)}
if one_tree and one_content
else {}
)
cached[1][tail] = members
return members
def expand_directory_entries(
files: list[dict], db: dict, standalone: bool = False
) -> list[dict]:
"""Replace each ``type: directory`` entry by the files held under it.
A profile declares the directory its code walks; what a pack can carry
is the files. Each member keeps the entry's fields and is pinned to the
content the collection holds at that destination. A directory nothing
answers is kept as declared, one gap, and a member the profile also
names as a file is left to that entry.
"""
def destination(entry: dict) -> str:
chosen = entry.get("standalone_path") if standalone else None
return chosen or entry.get("path") or entry.get("name", "")
if not any(entry.get("type") == "directory" for entry in files):
return files
named = {
destination(entry) for entry in files if entry.get("type") != "directory"
}
expanded: list[dict] = []
for entry in files:
if entry.get("type") != "directory":
expanded.append(entry)
continue
members = directory_members(db, destination(entry))
if not members:
expanded.append(entry)
continue
inherited = {
key: value
for key, value in entry.items()
if key not in ("type", "standalone_path")
}
expanded.extend(
{**inherited, "name": dest, "path": dest, "sha1": sha1}
for dest, sha1 in members.items()
if dest not in named
)
return expanded
def runs_standalone(
emu_name: str, profile: dict, standalone_cores: set[str]
) -> bool:
+6 -1
View File
@@ -36,6 +36,7 @@ from common import (
build_zip_contents_index,
check_inside_zip,
compute_hashes,
expand_directory_entries,
expand_platform_declared_names,
fetch_large_file,
filter_systems_by_target,
@@ -1288,7 +1289,11 @@ def generate_emulator_pack(
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for emu_name, profile in sorted(selected):
pack_structure = profile.get("pack_structure")
files = filter_files_by_mode(profile.get("files", []), standalone)
files = expand_directory_entries(
filter_files_by_mode(profile.get("files", []), standalone),
db,
standalone,
)
# A tree only one build reads follows that build, like a file.
for dd in filter_files_by_mode(
profile.get("data_directories", []), standalone
+9 -2
View File
@@ -38,6 +38,7 @@ from common import (
build_zip_contents_index,
check_inside_zip,
compute_hashes,
expand_directory_entries,
expand_platform_declared_names,
filter_systems_by_target,
group_identical_platforms,
@@ -458,7 +459,9 @@ def find_undeclared_files(
is_standalone = runs_standalone(emu_name, profile, standalone_set)
for f in profile.get("files", []):
for f in expand_directory_entries(
profile.get("files", []), db, is_standalone
):
fname = f.get("name", "")
effective_path = (
f.get("standalone_path") if is_standalone else f.get("path")
@@ -1363,7 +1366,11 @@ def verify_emulator(
)
for emu_name, profile in selected:
files = filter_files_by_mode(profile.get("files", []), standalone)
files = expand_directory_entries(
filter_files_by_mode(profile.get("files", []), standalone),
db,
standalone,
)
if region_drops:
files = [
fe
+296
View File
@@ -0,0 +1,296 @@
"""A profile entry that names a directory stands for the files under it.
Some cores read a whole tree: EasyRPG looks for an RTP under `rtp/2000/`,
O2EM for speech samples under `voice/`. The profile declares the directory,
the collection holds the files, and for a long time nothing joined the two:
the entry was resolved as one file called `rtp/2000/`, found nowhere, and
reported missing while 3 584 collected files stayed out of every pack.
"""
from __future__ import annotations
import hashlib
import json
import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
import yaml
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import common # noqa: E402
import generate_db # noqa: E402
import generate_pack as builder # noqa: E402
import verify # noqa: E402
PROFILE = """\
emulator: Demo
type: libretro
display_name: Demo
systems: [demo-system]
cores: [demo]
files:
- name: "RTP 2000"
path: "rtp/2000/"
type: directory
category: game_data
required: false
- name: "Fonts"
path: "demo/Fonts/"
type: directory
category: game_data
required: false
"""
PLATFORM = {
"platform": "Demo",
"verification_mode": "existence",
"base_destination": "",
"cores": ["demo"],
"systems": {
"demo-system": {
"files": [{"name": "boot.bin", "destination": "boot.bin"}],
},
},
}
class DirectoryFixture(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.bios = self.root / "bios"
self.emulators = self.root / "emulators"
self.platforms = self.root / "platforms"
for directory in (self.bios, self.emulators, self.platforms):
directory.mkdir()
self.files: dict[str, dict] = {}
self.sha1: dict[str, str] = {}
self._add("SystemA/boot.bin", b"boot")
self._add("Engine/demo/rtp/2000/Backdrop/a.png", b"backdrop")
self._add("Engine/demo/rtp/2000/Music/b.mid", b"music")
self._add("Engine/demo/rtp/2000/.variants/a.png.1234", b"older backdrop")
(self.platforms / "demo.yml").write_text(yaml.dump(PLATFORM))
(self.platforms / "_registry.yml").write_text(
yaml.dump({"platforms": {"demo": {"status": "active"}}})
)
common._emulator_profiles_cache.clear()
def tearDown(self):
common._emulator_profiles_cache.clear()
self._tmp.cleanup()
def _add(self, relative: str, payload: bytes) -> None:
path = self.bios / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(payload)
sha1 = hashlib.sha1(payload).hexdigest()
self.sha1[relative] = sha1
self.files[sha1] = {
"path": f"bios/{relative}",
"name": path.name,
"size": len(payload),
"sha1": sha1,
"md5": hashlib.md5(payload).hexdigest(),
"sha256": hashlib.sha256(payload).hexdigest(),
"crc32": f"{len(payload):08x}",
}
def _db(self) -> dict:
"""The index generate_db builds, over files stored where the test put them."""
indexes = generate_db.build_indexes(self.files, {})
stored = {
sha1: {**record, "path": str(self.root / record["path"])}
for sha1, record in self.files.items()
}
return {"files": stored, "indexes": indexes}
def _profiles(self, body: str = PROFILE) -> dict:
(self.emulators / "demo.yml").write_text(body)
common._emulator_profiles_cache.clear()
return common.load_emulator_profiles(str(self.emulators))
class DirectoryMembers(DirectoryFixture):
def test_every_file_under_the_tail_is_a_member(self):
self.assertEqual(
common.directory_members(self._db(), "rtp/2000/"),
{
"rtp/2000/Backdrop/a.png": self.sha1[
"Engine/demo/rtp/2000/Backdrop/a.png"
],
"rtp/2000/Music/b.mid": self.sha1["Engine/demo/rtp/2000/Music/b.mid"],
},
)
def test_a_directory_nobody_collected_has_no_member(self):
self.assertEqual(common.directory_members(self._db(), "demo/Fonts/"), {})
def test_two_trees_answering_one_tail_prove_nothing(self):
"""`voice/` under two unrelated parents is two directories, and merging
them would hand one emulator the other's files."""
self._add("Console/voice/E480.WAV", b"speech")
self._add("Other/app/voice/hello.wav", b"another program")
self.assertEqual(common.directory_members(self._db(), "voice/"), {})
def test_one_tree_under_a_generic_name_is_enough(self):
self._add("Console/voice/E480.WAV", b"speech")
self.assertEqual(
common.directory_members(self._db(), "voice/"),
{"voice/E480.WAV": self.sha1["Console/voice/E480.WAV"]},
)
class CrossReferenceReadsADirectory(DirectoryFixture):
def _report(self, body: str = PROFILE) -> list[dict]:
profiles = self._profiles(body)
return verify.find_undeclared_files(
PLATFORM, str(self.emulators), self._db(), profiles
)
def test_each_held_file_is_reported_as_held(self):
held = {
entry["path"]: entry["sha1"]
for entry in self._report()
if entry["in_repo"]
}
self.assertEqual(
held,
{
"rtp/2000/Backdrop/a.png": self.sha1[
"Engine/demo/rtp/2000/Backdrop/a.png"
],
"rtp/2000/Music/b.mid": self.sha1["Engine/demo/rtp/2000/Music/b.mid"],
},
)
def test_a_directory_with_nothing_collected_stays_one_gap(self):
gaps = [entry for entry in self._report() if not entry["in_repo"]]
self.assertEqual([(g["name"], g["path"]) for g in gaps], [("Fonts", "demo/Fonts/")])
def test_a_file_the_profile_also_names_is_reported_once(self):
body = PROFILE + (
' - name: "b.mid"\n'
' path: "rtp/2000/Music/b.mid"\n'
" category: game_data\n"
" required: true\n"
)
paths = [entry["path"] for entry in self._report(body)]
self.assertEqual(paths.count("rtp/2000/Music/b.mid"), 1)
class PackCarriesADirectory(DirectoryFixture):
EXPECTED = {"rtp/2000/Backdrop/a.png", "rtp/2000/Music/b.mid"}
def test_a_platform_pack_ships_the_tree(self):
profiles = self._profiles()
out = self.root / "dist"
out.mkdir()
zip_path = builder.generate_pack(
"demo", str(self.platforms), self._db(), str(self.bios), str(out),
include_extras=True, emulators_dir=str(self.emulators),
emu_profiles=profiles, offline=True,
)
with zipfile.ZipFile(zip_path) as archive:
names = set(archive.namelist())
self.assertEqual(
archive.read("rtp/2000/Backdrop/a.png"), b"backdrop"
)
self.assertTrue(self.EXPECTED <= names, sorted(names))
self.assertFalse([n for n in names if ".variants" in n])
def test_the_install_manifest_lists_the_tree(self):
profiles = self._profiles()
manifest = builder.generate_manifest(
"demo", str(self.platforms), self._db(), str(self.bios),
str(self.platforms / "_registry.yml"),
emulators_dir=str(self.emulators), emu_profiles=profiles,
offline=True,
)
dests = {entry["dest"] for entry in manifest["files"]}
self.assertTrue(self.EXPECTED <= dests, sorted(dests))
json.dumps(manifest)
def test_an_emulator_pack_ships_the_tree(self):
self._profiles()
out = self.root / "emu"
out.mkdir()
zip_path = builder.generate_emulator_pack(
["demo"], str(self.emulators), self._db(), str(self.bios), str(out),
offline=True,
)
with zipfile.ZipFile(zip_path) as archive:
names = set(archive.namelist())
self.assertTrue(self.EXPECTED <= names, sorted(names))
def test_verify_counts_what_the_emulator_pack_ships(self):
self._profiles()
result = verify.verify_emulator(["demo"], str(self.emulators), self._db())
held = {
detail["name"]
for detail in result["details"]
if detail["status"] == verify.Status.OK
}
self.assertEqual(held, self.EXPECTED)
missing = [
detail["name"]
for detail in result["details"]
if detail["status"] == verify.Status.MISSING
]
self.assertEqual(missing, ["Fonts"])
class ProfileContract(unittest.TestCase):
"""One way to say an entry is a directory, so one place reads it."""
def _errors(self, entry: dict) -> list[str]:
from jsonschema import Draft202012Validator
schema = json.loads(
(REPO_ROOT / "schemas" / "emulator.schema.json").read_text()
)
validator = Draft202012Validator(schema["properties"]["files"]["items"])
return [error.message for error in validator.iter_errors(entry)]
def test_a_name_that_is_a_directory_must_say_so(self):
self.assertTrue(self._errors({"name": "voice/"}))
self.assertEqual(self._errors({"name": "voice/", "type": "directory"}), [])
def test_the_marker_takes_no_other_value(self):
self.assertTrue(self._errors({"name": "RTP", "path": "rtp/", "type": "folder"}))
def test_a_directory_is_written_with_its_trailing_slash(self):
self.assertTrue(
self._errors({"name": "RTP", "path": "rtp/2000", "type": "directory"})
)
self.assertEqual(
self._errors({"name": "RTP", "path": "rtp/2000/", "type": "directory"}),
[],
)
def test_every_profile_follows_it(self):
offenders = []
for path in sorted((REPO_ROOT / "emulators").glob("*.yml")):
document = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
for entry in document.get("files") or []:
if self._errors_for_directory_rules(entry):
offenders.append(f"{path.name}: {entry.get('name')}")
self.assertEqual(offenders, [])
@staticmethod
def _errors_for_directory_rules(entry: dict) -> bool:
name = str(entry.get("name", ""))
path = entry.get("path")
if entry.get("type") == "directory":
return isinstance(path, str) and not path.endswith("/")
return name.endswith("/") or "type" in entry
if __name__ == "__main__":
unittest.main()
+1
View File
@@ -372,6 +372,7 @@ which CI validates every profile against.
| `load_from` | directory the core reads the file from when it is not the system directory |
| `source_ref` | source file and line number (e.g. `boot.cpp:42`), read at the profile's `source_commit` when set. A dict splits `standalone` from `libretro` when they diverge |
| `path` | destination path relative to system directory |
| `type` | `directory` when the code reads a whole tree (an RTP, a folder of speech samples). `path` is then the directory, written with its trailing slash, and the entry stands for every file the collection holds under it. A name ending in `/` requires it |
| `description` | what this file is |
| `note` | additional context |
| `contents` | structure of files inside a BIOS ZIP (`name`, `description`, `size`, `crc32`) |