diff --git a/emulators/jazz2-native.yml b/emulators/jazz2-native.yml index 7b4be7d3..ff09da49 100644 --- a/emulators/jazz2-native.yml +++ b/emulators/jazz2-native.yml @@ -45,6 +45,7 @@ files: # ================================================================ - name: "Animations/" + type: directory category: game_data path: "jazz2/Content/Animations/" standalone_path: "Content/Animations/" @@ -54,6 +55,7 @@ files: note: "Native graphics, sounds and the UI fonts, opened through OpenContentFile." - name: "Metadata/" + type: directory category: game_data path: "jazz2/Content/Metadata/" standalone_path: "Content/Metadata/" @@ -63,6 +65,7 @@ files: note: "Actor .res descriptors (Common, Enemy, Player, UI, Boss, ...)." - name: "Translations/" + type: directory category: game_data path: "jazz2/Content/Translations/" standalone_path: "Content/Translations/" diff --git a/emulators/mkxp_z.yml b/emulators/mkxp_z.yml index 3cea7053..57451b8c 100644 --- a/emulators/mkxp_z.yml +++ b/emulators/mkxp_z.yml @@ -26,6 +26,7 @@ notes: | files: - name: "Standard" + type: directory description: "RPG Maker XP Run-Time Package" category: game_data path: "mkxp-z/RTP/Standard/" @@ -33,6 +34,7 @@ files: source_ref: "src/core.cpp:1081-1124, src/core.cpp:1126-1197" - name: "RPGVX" + type: directory description: "RPG Maker VX Run-Time Package" category: game_data path: "mkxp-z/RTP/RPGVX/" @@ -40,6 +42,7 @@ files: source_ref: "src/core.cpp:1081-1124, src/core.cpp:1126-1197" - name: "RPGVXAce" + type: directory description: "RPG Maker VX Ace Run-Time Package" category: game_data path: "mkxp-z/RTP/RPGVXAce/" @@ -47,6 +50,7 @@ files: source_ref: "src/core.cpp:1081-1124, src/core.cpp:1126-1197" - name: "Fonts" + type: directory description: "Fallback font directory (.otf/.ttf)" category: game_data path: "mkxp-z/Fonts/" @@ -54,6 +58,7 @@ files: source_ref: "src/core.cpp:1199-1211" - name: "Preload" + type: directory description: "Ruby scripts loaded before game scripts" category: game_data path: "mkxp-z/Scripts/Preload/" @@ -61,6 +66,7 @@ files: source_ref: "src/core.cpp:891-934" - name: "Postload" + type: directory description: "Ruby scripts loaded after game scripts (RGSS3 only)" category: game_data path: "mkxp-z/Scripts/Postload/" diff --git a/emulators/nestopia.yml b/emulators/nestopia.yml index 269d7b42..afd53bb8 100644 --- a/emulators/nestopia.yml +++ b/emulators/nestopia.yml @@ -67,6 +67,7 @@ files: # --- ADPCM audio samples (optional, game-specific) --- - name: "nestopia/samples/moepro/" + type: directory system: nintendo-nes description: "Moero Pro Yakyuu audio samples (16 .wav files, 00.wav-15.wav)" required: false @@ -74,6 +75,7 @@ files: source_ref: "libretro/libretro.cpp:183-391 (load_wav), libretro/libretro.cpp:451 (LOAD_SAMPLE_MOERO_PRO_YAKYUU callback), source/core/NstSoundPlayer.hpp:49 (16 samples)" - name: "nestopia/samples/moepro88/" + type: directory system: nintendo-nes description: "Moero Pro Yakyuu '88 audio samples (20 .wav files, 00.wav-19.wav)" required: false @@ -81,6 +83,7 @@ files: source_ref: "libretro/libretro.cpp:183-391 (load_wav), libretro/libretro.cpp:453 (LOAD_SAMPLE_MOERO_PRO_YAKYUU_88 callback), source/core/NstSoundPlayer.hpp:50 (20 samples)" - name: "nestopia/samples/mptennis/" + type: directory system: nintendo-nes description: "Moero Pro Tennis audio samples (19 .wav files, 00.wav-18.wav)" required: false @@ -88,6 +91,7 @@ files: source_ref: "libretro/libretro.cpp:183-391 (load_wav), libretro/libretro.cpp:455 (LOAD_SAMPLE_MOERO_PRO_TENNIS callback), source/core/NstSoundPlayer.hpp:51 (19 samples)" - name: "nestopia/samples/terao/" + type: directory system: nintendo-nes description: "Terao no Dosukoi Oozumou audio samples (6 .wav files, 00.wav-05.wav)" required: false @@ -95,6 +99,7 @@ files: source_ref: "libretro/libretro.cpp:183-391 (load_wav), libretro/libretro.cpp:457 (LOAD_SAMPLE_TERAO_NO_DOSUKOI_OOZUMOU callback), source/core/NstSoundPlayer.hpp:52 (6 samples)" - name: "nestopia/samples/ftaerobi/" + type: directory system: nintendo-nes description: "Aerobics Studio audio samples (8 .wav files, 00.wav-07.wav)" required: false diff --git a/emulators/nxengine.yml b/emulators/nxengine.yml index 96cf6d87..c3913136 100644 --- a/emulators/nxengine.yml +++ b/emulators/nxengine.yml @@ -50,6 +50,7 @@ files: - name: "data/" path: "nxengine/data/" + type: directory category: game_data system: cave-story description: "Full game asset directory tree (399 files: sprites, NPC sheets, stage maps, scripts, backgrounds)" diff --git a/emulators/o2em.yml b/emulators/o2em.yml index 1d3de452..b158c74d 100644 --- a/emulators/o2em.yml +++ b/emulators/o2em.yml @@ -81,6 +81,7 @@ files: source_ref: "libretro.c:1162-1167" - name: "voice/" + type: directory required: false category: game_data note: "The Voice speech synthesis WAV samples. 9 banks (E4, E8-EF) × 128 samples." diff --git a/emulators/thextech.yml b/emulators/thextech.yml index 1f714c95..5b86f5de 100644 --- a/emulators/thextech.yml +++ b/emulators/thextech.yml @@ -90,18 +90,21 @@ files: # ================================================================ - name: "graphics/" + type: directory category: game_data required: true source_ref: "src/main/asset_pack.cpp:274, 326-327, src/gfx.cpp:121-229, src/load_gfx.cpp:120-123" note: "graphics/ui/ must exist for a directory to be scanned as a pack; GFX_t::load rejects the pack when a required UI image is missing." - name: "sound/" + type: directory category: game_data required: false source_ref: "src/sound.cpp:1319" note: "SFX directory, files named by sounds.ini. Game runs without sounds." - name: "music/" + type: directory category: game_data required: false source_ref: "src/sound.cpp:1318" @@ -132,6 +135,7 @@ files: note: "Read when intro.lvlx is absent." - name: "introset/" + type: directory category: game_data required: false source_ref: "src/game_main.cpp:194-222, 247" @@ -150,12 +154,14 @@ files: note: "Read when outro.lvlx is absent." - name: "worlds/" + type: directory category: game_data required: false source_ref: "src/main/menu_main.cpp:361-370" note: "Episodes shipped with the pack, listed next to the user's worlds directory." - name: "battle/" + type: directory category: game_data required: false source_ref: "src/main/menu_main.cpp:701-706" diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 1fc45500..8e8a9ccd 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -425,7 +425,11 @@ "type": "string" }, "type": { - "type": "string" + "description": "Set on an entry that names a directory the code reads whole. The entry then stands for every file under path.", + "type": "string", + "enum": [ + "directory" + ] }, "variant_group": { "type": "string", @@ -599,7 +603,38 @@ "unsourceable": { "type": "string" } - } + }, + "allOf": [ + { + "if": { + "properties": { + "name": { + "pattern": "/$" + } + } + }, + "then": { + "required": [ + "type" + ] + } + }, + { + "if": { + "required": [ + "type" + ] + }, + "then": { + "properties": { + "path": { + "type": "string", + "pattern": "/$" + } + } + } + } + ] } }, "source_branch": { diff --git a/scripts/common.py b/scripts/common.py index 0c84a827..f290d0f4 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1116,6 +1116,101 @@ def group_identical_platforms( return result +_directory_members_cache: dict[int, tuple[dict, dict[str, dict[str, str]]]] = {} + + +def directory_members(db: dict, directory: str) -> dict[str, str]: + """Files the collection holds under a directory an emulator reads whole. + + Returns {destination: sha1}, each destination starting with the directory + as the profile writes it. The collection and the destination meet on a + tail, so the members are the indexed tails that begin with it. + + A generic directory name can sit in two unrelated trees (``voice/``, + ``data/``). Members that do not share one parent, or a destination two + contents claim, would merge them into one emulator's directory: nothing + is returned and the entry stays a reported gap. + """ + tail = directory.strip("/") + if not tail: + return {} + by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {}) + cached = _directory_members_cache.get(id(by_path_suffix)) + if cached is None or cached[0] is not by_path_suffix: + cached = (by_path_suffix, {}) + _directory_members_cache[id(by_path_suffix)] = cached + if tail in cached[1]: + return cached[1][tail] + + prefix = f"{tail}/" + infix = f"/{prefix}" + held: dict[str, list[str]] = {} + parent: dict[str, str] = {} + for key, sha1s in by_path_suffix.items(): + if key.startswith(prefix): + held[key] = sha1s + continue + at = key.find(infix) + if at > 0: + member = key[at + 1:] + if len(key[:at]) > len(parent.get(member, "")): + parent[member] = key[:at] + held = { + dest: sha1s for dest, sha1s in held.items() if "/.variants/" not in dest + } + one_tree = len({parent.get(dest, "") for dest in held}) <= 1 + one_content = all(len(sha1s) == 1 for sha1s in held.values()) + members = ( + {dest: held[dest][0] for dest in sorted(held)} + if one_tree and one_content + else {} + ) + cached[1][tail] = members + return members + + +def expand_directory_entries( + files: list[dict], db: dict, standalone: bool = False +) -> list[dict]: + """Replace each ``type: directory`` entry by the files held under it. + + A profile declares the directory its code walks; what a pack can carry + is the files. Each member keeps the entry's fields and is pinned to the + content the collection holds at that destination. A directory nothing + answers is kept as declared, one gap, and a member the profile also + names as a file is left to that entry. + """ + def destination(entry: dict) -> str: + chosen = entry.get("standalone_path") if standalone else None + return chosen or entry.get("path") or entry.get("name", "") + + if not any(entry.get("type") == "directory" for entry in files): + return files + named = { + destination(entry) for entry in files if entry.get("type") != "directory" + } + expanded: list[dict] = [] + for entry in files: + if entry.get("type") != "directory": + expanded.append(entry) + continue + members = directory_members(db, destination(entry)) + if not members: + expanded.append(entry) + continue + inherited = { + key: value + for key, value in entry.items() + if key not in ("type", "standalone_path") + } + expanded.extend( + {**inherited, "name": dest, "path": dest, "sha1": sha1} + for dest, sha1 in members.items() + if dest not in named + ) + return expanded + + def runs_standalone( emu_name: str, profile: dict, standalone_cores: set[str] ) -> bool: diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 05ce3417..146db1ee 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -36,6 +36,7 @@ from common import ( build_zip_contents_index, check_inside_zip, compute_hashes, + expand_directory_entries, expand_platform_declared_names, fetch_large_file, filter_systems_by_target, @@ -1288,7 +1289,11 @@ def generate_emulator_pack( with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: for emu_name, profile in sorted(selected): pack_structure = profile.get("pack_structure") - files = filter_files_by_mode(profile.get("files", []), standalone) + files = expand_directory_entries( + filter_files_by_mode(profile.get("files", []), standalone), + db, + standalone, + ) # A tree only one build reads follows that build, like a file. for dd in filter_files_by_mode( profile.get("data_directories", []), standalone diff --git a/scripts/verify.py b/scripts/verify.py index 04e487d9..3baa89e5 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -38,6 +38,7 @@ from common import ( build_zip_contents_index, check_inside_zip, compute_hashes, + expand_directory_entries, expand_platform_declared_names, filter_systems_by_target, group_identical_platforms, @@ -458,7 +459,9 @@ def find_undeclared_files( is_standalone = runs_standalone(emu_name, profile, standalone_set) - for f in profile.get("files", []): + for f in expand_directory_entries( + profile.get("files", []), db, is_standalone + ): fname = f.get("name", "") effective_path = ( f.get("standalone_path") if is_standalone else f.get("path") @@ -1363,7 +1366,11 @@ def verify_emulator( ) for emu_name, profile in selected: - files = filter_files_by_mode(profile.get("files", []), standalone) + files = expand_directory_entries( + filter_files_by_mode(profile.get("files", []), standalone), + db, + standalone, + ) if region_drops: files = [ fe diff --git a/tests/test_directory_entries.py b/tests/test_directory_entries.py new file mode 100644 index 00000000..9f5ae507 --- /dev/null +++ b/tests/test_directory_entries.py @@ -0,0 +1,296 @@ +"""A profile entry that names a directory stands for the files under it. + +Some cores read a whole tree: EasyRPG looks for an RTP under `rtp/2000/`, +O2EM for speech samples under `voice/`. The profile declares the directory, +the collection holds the files, and for a long time nothing joined the two: +the entry was resolved as one file called `rtp/2000/`, found nowhere, and +reported missing while 3 584 collected files stayed out of every pack. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import common # noqa: E402 +import generate_db # noqa: E402 +import generate_pack as builder # noqa: E402 +import verify # noqa: E402 + +PROFILE = """\ +emulator: Demo +type: libretro +display_name: Demo +systems: [demo-system] +cores: [demo] +files: + - name: "RTP 2000" + path: "rtp/2000/" + type: directory + category: game_data + required: false + - name: "Fonts" + path: "demo/Fonts/" + type: directory + category: game_data + required: false +""" + +PLATFORM = { + "platform": "Demo", + "verification_mode": "existence", + "base_destination": "", + "cores": ["demo"], + "systems": { + "demo-system": { + "files": [{"name": "boot.bin", "destination": "boot.bin"}], + }, + }, +} + + +class DirectoryFixture(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.bios = self.root / "bios" + self.emulators = self.root / "emulators" + self.platforms = self.root / "platforms" + for directory in (self.bios, self.emulators, self.platforms): + directory.mkdir() + self.files: dict[str, dict] = {} + self.sha1: dict[str, str] = {} + + self._add("SystemA/boot.bin", b"boot") + self._add("Engine/demo/rtp/2000/Backdrop/a.png", b"backdrop") + self._add("Engine/demo/rtp/2000/Music/b.mid", b"music") + self._add("Engine/demo/rtp/2000/.variants/a.png.1234", b"older backdrop") + + (self.platforms / "demo.yml").write_text(yaml.dump(PLATFORM)) + (self.platforms / "_registry.yml").write_text( + yaml.dump({"platforms": {"demo": {"status": "active"}}}) + ) + common._emulator_profiles_cache.clear() + + def tearDown(self): + common._emulator_profiles_cache.clear() + self._tmp.cleanup() + + def _add(self, relative: str, payload: bytes) -> None: + path = self.bios / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + sha1 = hashlib.sha1(payload).hexdigest() + self.sha1[relative] = sha1 + self.files[sha1] = { + "path": f"bios/{relative}", + "name": path.name, + "size": len(payload), + "sha1": sha1, + "md5": hashlib.md5(payload).hexdigest(), + "sha256": hashlib.sha256(payload).hexdigest(), + "crc32": f"{len(payload):08x}", + } + + def _db(self) -> dict: + """The index generate_db builds, over files stored where the test put them.""" + indexes = generate_db.build_indexes(self.files, {}) + stored = { + sha1: {**record, "path": str(self.root / record["path"])} + for sha1, record in self.files.items() + } + return {"files": stored, "indexes": indexes} + + def _profiles(self, body: str = PROFILE) -> dict: + (self.emulators / "demo.yml").write_text(body) + common._emulator_profiles_cache.clear() + return common.load_emulator_profiles(str(self.emulators)) + + +class DirectoryMembers(DirectoryFixture): + def test_every_file_under_the_tail_is_a_member(self): + self.assertEqual( + common.directory_members(self._db(), "rtp/2000/"), + { + "rtp/2000/Backdrop/a.png": self.sha1[ + "Engine/demo/rtp/2000/Backdrop/a.png" + ], + "rtp/2000/Music/b.mid": self.sha1["Engine/demo/rtp/2000/Music/b.mid"], + }, + ) + + def test_a_directory_nobody_collected_has_no_member(self): + self.assertEqual(common.directory_members(self._db(), "demo/Fonts/"), {}) + + def test_two_trees_answering_one_tail_prove_nothing(self): + """`voice/` under two unrelated parents is two directories, and merging + them would hand one emulator the other's files.""" + self._add("Console/voice/E480.WAV", b"speech") + self._add("Other/app/voice/hello.wav", b"another program") + self.assertEqual(common.directory_members(self._db(), "voice/"), {}) + + def test_one_tree_under_a_generic_name_is_enough(self): + self._add("Console/voice/E480.WAV", b"speech") + self.assertEqual( + common.directory_members(self._db(), "voice/"), + {"voice/E480.WAV": self.sha1["Console/voice/E480.WAV"]}, + ) + + +class CrossReferenceReadsADirectory(DirectoryFixture): + def _report(self, body: str = PROFILE) -> list[dict]: + profiles = self._profiles(body) + return verify.find_undeclared_files( + PLATFORM, str(self.emulators), self._db(), profiles + ) + + def test_each_held_file_is_reported_as_held(self): + held = { + entry["path"]: entry["sha1"] + for entry in self._report() + if entry["in_repo"] + } + self.assertEqual( + held, + { + "rtp/2000/Backdrop/a.png": self.sha1[ + "Engine/demo/rtp/2000/Backdrop/a.png" + ], + "rtp/2000/Music/b.mid": self.sha1["Engine/demo/rtp/2000/Music/b.mid"], + }, + ) + + def test_a_directory_with_nothing_collected_stays_one_gap(self): + gaps = [entry for entry in self._report() if not entry["in_repo"]] + self.assertEqual([(g["name"], g["path"]) for g in gaps], [("Fonts", "demo/Fonts/")]) + + def test_a_file_the_profile_also_names_is_reported_once(self): + body = PROFILE + ( + ' - name: "b.mid"\n' + ' path: "rtp/2000/Music/b.mid"\n' + " category: game_data\n" + " required: true\n" + ) + paths = [entry["path"] for entry in self._report(body)] + self.assertEqual(paths.count("rtp/2000/Music/b.mid"), 1) + + +class PackCarriesADirectory(DirectoryFixture): + EXPECTED = {"rtp/2000/Backdrop/a.png", "rtp/2000/Music/b.mid"} + + def test_a_platform_pack_ships_the_tree(self): + profiles = self._profiles() + out = self.root / "dist" + out.mkdir() + zip_path = builder.generate_pack( + "demo", str(self.platforms), self._db(), str(self.bios), str(out), + include_extras=True, emulators_dir=str(self.emulators), + emu_profiles=profiles, offline=True, + ) + with zipfile.ZipFile(zip_path) as archive: + names = set(archive.namelist()) + self.assertEqual( + archive.read("rtp/2000/Backdrop/a.png"), b"backdrop" + ) + self.assertTrue(self.EXPECTED <= names, sorted(names)) + self.assertFalse([n for n in names if ".variants" in n]) + + def test_the_install_manifest_lists_the_tree(self): + profiles = self._profiles() + manifest = builder.generate_manifest( + "demo", str(self.platforms), self._db(), str(self.bios), + str(self.platforms / "_registry.yml"), + emulators_dir=str(self.emulators), emu_profiles=profiles, + offline=True, + ) + dests = {entry["dest"] for entry in manifest["files"]} + self.assertTrue(self.EXPECTED <= dests, sorted(dests)) + json.dumps(manifest) + + def test_an_emulator_pack_ships_the_tree(self): + self._profiles() + out = self.root / "emu" + out.mkdir() + zip_path = builder.generate_emulator_pack( + ["demo"], str(self.emulators), self._db(), str(self.bios), str(out), + offline=True, + ) + with zipfile.ZipFile(zip_path) as archive: + names = set(archive.namelist()) + self.assertTrue(self.EXPECTED <= names, sorted(names)) + + def test_verify_counts_what_the_emulator_pack_ships(self): + self._profiles() + result = verify.verify_emulator(["demo"], str(self.emulators), self._db()) + held = { + detail["name"] + for detail in result["details"] + if detail["status"] == verify.Status.OK + } + self.assertEqual(held, self.EXPECTED) + missing = [ + detail["name"] + for detail in result["details"] + if detail["status"] == verify.Status.MISSING + ] + self.assertEqual(missing, ["Fonts"]) + + +class ProfileContract(unittest.TestCase): + """One way to say an entry is a directory, so one place reads it.""" + + def _errors(self, entry: dict) -> list[str]: + from jsonschema import Draft202012Validator + + schema = json.loads( + (REPO_ROOT / "schemas" / "emulator.schema.json").read_text() + ) + validator = Draft202012Validator(schema["properties"]["files"]["items"]) + return [error.message for error in validator.iter_errors(entry)] + + def test_a_name_that_is_a_directory_must_say_so(self): + self.assertTrue(self._errors({"name": "voice/"})) + self.assertEqual(self._errors({"name": "voice/", "type": "directory"}), []) + + def test_the_marker_takes_no_other_value(self): + self.assertTrue(self._errors({"name": "RTP", "path": "rtp/", "type": "folder"})) + + def test_a_directory_is_written_with_its_trailing_slash(self): + self.assertTrue( + self._errors({"name": "RTP", "path": "rtp/2000", "type": "directory"}) + ) + self.assertEqual( + self._errors({"name": "RTP", "path": "rtp/2000/", "type": "directory"}), + [], + ) + + def test_every_profile_follows_it(self): + offenders = [] + for path in sorted((REPO_ROOT / "emulators").glob("*.yml")): + document = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + for entry in document.get("files") or []: + if self._errors_for_directory_rules(entry): + offenders.append(f"{path.name}: {entry.get('name')}") + self.assertEqual(offenders, []) + + @staticmethod + def _errors_for_directory_rules(entry: dict) -> bool: + name = str(entry.get("name", "")) + path = entry.get("path") + if entry.get("type") == "directory": + return isinstance(path, str) and not path.endswith("/") + return name.endswith("/") or "type" in entry + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index a5a278a3..b5fe28f4 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -372,6 +372,7 @@ which CI validates every profile against. | `load_from` | directory the core reads the file from when it is not the system directory | | `source_ref` | source file and line number (e.g. `boot.cpp:42`), read at the profile's `source_commit` when set. A dict splits `standalone` from `libretro` when they diverge | | `path` | destination path relative to system directory | +| `type` | `directory` when the code reads a whole tree (an RTP, a folder of speech samples). `path` is then the directory, written with its trailing slash, and the entry stands for every file the collection holds under it. A name ending in `/` requires it | | `description` | what this file is | | `note` | additional context | | `contents` | structure of files inside a BIOS ZIP (`name`, `description`, `size`, `crc32`) |