diff --git a/emulators/pcee2.yml b/emulators/pcee2.yml index d45fb9549..36335723f 100644 --- a/emulators/pcee2.yml +++ b/emulators/pcee2.yml @@ -23,7 +23,8 @@ resources_directory: "pcsx2/resources/" data_directories: - ref: pcsx2-resources destination: "pcsx2/resources" - source_ref: "pcee2-libretro/Libretro.cpp:543-571, pcsx2/Pcsx2Config.cpp:2417-2429" + bundled: true + source_ref: "pcee2-libretro/Libretro.cpp:543-571, pcsx2/Pcsx2Config.cpp:2417-2429, cmake/EmbedResources.cmake:19-31, 46-62" notes: | libretro port of current PCSX2, merged up to upstream ab0b401f (v2.9.138, diff --git a/emulators/stonesoup.yml b/emulators/stonesoup.yml index 9d03cf32c..a668dbe4e 100644 --- a/emulators/stonesoup.yml +++ b/emulators/stonesoup.yml @@ -42,7 +42,8 @@ notes: | data_directories: - ref: stonesoup installed_on: [recalbox] - source_ref: "crawl-ref/source/files.cc:375-458" + load_from: content_dir + source_ref: "crawl-ref/libretro/libretro.cc:178-197, crawl-ref/source/files.cc:375-458" files: [] # no BIOS/firmware, game data loaded via data_directories diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 9ca4a8418..dab1a1d7a 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -390,6 +390,21 @@ "items": {"type": "string"}, "description": "Platforms (registry keys) whose launcher points the emulator at other files, so this entry is never opened there." }, + "load_from": { + "description": "Where the emulator reads the tree when not below the system directory, keyed by build when the builds differ (content_dir: beside the content). A platform pack carries only a tree read from the system directory.", + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "libretro": {"type": "string"}, + "standalone": {"type": "string"} + } + } + ] + }, "source_ref": { "type": "string" } diff --git a/scripts/generate_installer_doc.py b/scripts/generate_installer_doc.py index f19dc77c0..ee2973261 100644 --- a/scripts/generate_installer_doc.py +++ b/scripts/generate_installer_doc.py @@ -333,10 +333,10 @@ def write_small_list(path: Path) -> None: def _child(spec: dict) -> int: """install.main() as the sample runs it, in its own process. Only the transport and the machine are stood in for: no connection leaves the - loopback, detection finds the card folder the spec names, a file of a - landed run reports its bytes written without fetching them, and a held - run finds every file in place. The counts, sizes and words are the - installer's own.""" + loopback, detection finds the card folder the spec names, a file or a + data tree of a landed run reports its bytes written without fetching + them, and a held run finds every file and tree in place. The counts, + sizes and words are the installer's own.""" import concurrent.futures # noqa: PLC0415 import socket # noqa: PLC0415 from unittest import mock # noqa: PLC0415 @@ -362,6 +362,13 @@ def _child(spec: dict) -> int: def held(files: list[dict], _bios: Path, **_kwargs: object) -> tuple: return [], list(files), [] + def tree_landed(entry: dict, _bios: Path, state: object = None) -> dict: + return {"present": 0, "installed": len(entry["files"]), "changed": 0, + "absent": 0, "error": "", "not_kept": []} + + def tree_held(entry: dict, bios: Path, remember: bool = False) -> tuple: + return bios / entry.get("dest", ""), {}, len(entry["files"]) + patches = [ mock.patch.object(socket, "create_connection", loopback_only), # One worker and the submission order: the progress lines of a log. @@ -376,8 +383,10 @@ def _child(spec: dict) -> int: ] if spec.get("landed"): patches.append(mock.patch.object(_install, "_download_one", landed)) + patches.append(mock.patch.object(_install, "install_data_directory", tree_landed)) if spec.get("held"): patches.append(mock.patch.object(_install, "check_local", held)) + patches.append(mock.patch.object(_install, "data_directory_state", tree_held)) with contextlib.ExitStack() as stack: for patch in patches: stack.enter_context(patch) diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index b41606dba..4d7fc9c7d 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -69,9 +69,14 @@ from common import ( MANUFACTURER_PREFIXES, require_yaml, ProfileSelectionError, + core_system_dirs, + declared_root, entry_serves_systems, + entry_system_ids, + platform_layouts, sanitize_pack_path, select_emulator_profiles, + standalone_set, system_core_reach, upstream_profile_index, yaml_load, @@ -100,6 +105,7 @@ from validation import ( _build_validation_index, _parse_validation, deliverable, + read_from_system_dir, with_archive_members, extra_alternatives, ) @@ -376,6 +382,97 @@ def platform_list_version(config: dict, platform: str) -> str: PACK_DOCUMENTS = ("README.txt", "manifest.json") +def core_data_trees( + config: dict, + profiles: dict[str, dict], + platform_name: str, + data_registry: dict | None, + target_cores: set[str] | None = None, +) -> list[dict]: + """The data directories the platform's cores read, as tree references. + + A profile's ``data_directories`` reach the pack of every platform that + runs the core as a libretro core, below the system directory it hands + the core (and the build's own folder, ``pack_structure``), as the + emulator pack lays them out. Left out: a tree the platform's package + installs, for this core or another reading the same tree + (``installed_data_refs``), one the build bundles or the platform never + opens (``deliverable``), and one whose registry entry names the + platforms it serves (``for_platforms``) without this one. A standalone build reads its data below its own folder, which a + tree's destination does not describe, so its trees stay out, as does a + tree read beside the content (``load_from``). + + Each reference carries the profile's systems (``source_systems``, as a + core extra does), which --system and --split narrow on. + """ + if not data_registry: + return [] + installed = installed_data_refs(platform_name) + handed = standalone_set(config) + # Relative to base_destination, which the members are written under. A + # platform with none writes its BIOS folder into every destination, and + # the trees go there as the core extras do. + root = declared_root(config) + trees: list[dict] = [] + seen: set[tuple[str, str]] = set() + for core in sorted(_platform_cores(config, profiles, target_cores)): + profile = profiles[core] + if all(standalone for _view, standalone in platform_layouts(core, profile, handed)): + continue + declared = [t for t in profile.get("data_directories") or [] if isinstance(t, dict)] + directories = core_system_dirs(core, profile, handed) + structure = str((profile.get("pack_structure") or {}).get("libretro") or "") + for tree in deliverable(declared, False, platform_name): + if not read_from_system_dir(tree): + continue + ref = str(tree.get("ref") or "") + entry = data_registry.get(ref) + if entry is None or ref in installed: + continue + allowed = entry.get("for_platforms") + if allowed and platform_name not in allowed: + continue + for directory in [""] if directories is None else directories: + destination = prefixed_destination("/".join( + part.strip("/") + for part in (directory, structure, str(tree.get("destination") or "")) + if part.strip("/") + ), root) or root + if (ref, destination) in seen: + continue + seen.add((ref, destination)) + trees.append({ + "ref": ref, + "destination": destination, + "required": tree.get("required"), + "core": core, + "source_systems": entry_system_ids(tree, profile), + }) + return trees + + +def _in_data_cache(path: str, data_registry: dict | None) -> bool: + """Whether a resolved file lies in one of the data directory caches.""" + real = os.path.realpath(path) + return any( + real.startswith(os.path.realpath(entry["local_cache"]) + os.sep) + for entry in (data_registry or {}).values() + if entry.get("local_cache") + ) + + +def _tree_references( + pack_systems: dict, core_trees: Iterable[dict], +) -> Iterator[tuple[tuple[str, ...], dict]]: + """(systems, reference) for each tree a pack carries: those its systems + declare, then those its cores read.""" + for sys_id, system in sorted(pack_systems.items()): + for dd in system.get("data_directories", []): + yield (sys_id,), dd + for tree in core_trees: + yield tuple(tree["source_systems"]), tree + + def _data_directory_members( pack_systems: dict, data_registry: dict | None, @@ -387,12 +484,20 @@ def _data_directory_members( seen_parents: set, refs_out: dict | None = None, required_only: bool = False, + core_trees: Iterable[dict] = (), + reclaim: dict[str, str] | None = None, ) -> Iterator[tuple[str, str]]: """Yield (source, destination) for the cached data directories a - platform's systems declare. + platform's systems declare, then those its cores read (``core_trees``, + from core_data_trees). + + ``reclaim`` maps a destination already taken to the cache file it was + resolved to: the manifest found the platform's entry there and cannot + list it as a file, so the tree yields it once more and the installer + gets it from the tree's archive. ``refs_out``, when given, maps each yielded destination to - (registry key, system id, directory destination), for the manifest. + (registry key, system ids, directory destination), for the manifest. These are not BIOS and carry no hash of their own: they are whole trees an emulator reads, refreshed from upstream, so what ships is @@ -402,54 +507,61 @@ def _data_directory_members( core runs without, as it leaves out such a file. """ # Data directories from _data_dirs.yml - for sys_id, system in sorted(pack_systems.items()): - for dd in system.get("data_directories", []): - ref_key = dd.get("ref", "") - if not ref_key or not data_registry or ref_key not in data_registry: - continue - if required_only and dd.get("required") is False: - continue - if ref_key in installed_data_refs(platform_name): - continue - entry = data_registry[ref_key] - allowed = entry.get("for_platforms") - if allowed and platform_name not in allowed: - continue - local_path = entry.get("local_cache", "") - if not local_path or not os.path.isdir(local_path): - print( - f" WARNING: data directory '{ref_key}' not cached at {local_path} (run refresh_data_dirs.py)" - ) - _MISSING_DATA_DIRS.add(ref_key) - continue - dd_dest = dd.get("destination", "") - if base_dest and dd_dest: - dd_prefix = f"{base_dest}/{dd_dest}" - elif base_dest: - dd_prefix = base_dest - else: - dd_prefix = dd_dest - # Held while the caller writes what is yielded: a refresh swapping - # the tree waits until the walk is done. - with cache_lock(local_path, shared=True): - for root, _dirs, filenames in os.walk(local_path): - for fname in filenames: - src = os.path.join(root, fname) - rel = os.path.relpath(src, local_path) - full = f"{dd_prefix}/{rel}" - if full in seen_destinations or ( - full.lower() in seen_lower and case_insensitive - ): - continue - if _has_path_conflict(full, seen_destinations, seen_parents): - continue - seen_destinations.add(full) - _register_path(full, seen_destinations, seen_parents) - if case_insensitive: - seen_lower.add(full.lower()) + for systems, dd in _tree_references(pack_systems, core_trees): + ref_key = dd.get("ref", "") + if not ref_key or not data_registry or ref_key not in data_registry: + continue + if required_only and dd.get("required") is False: + continue + if ref_key in installed_data_refs(platform_name): + continue + entry = data_registry[ref_key] + allowed = entry.get("for_platforms") + if allowed and platform_name not in allowed: + continue + local_path = entry.get("local_cache", "") + if not local_path or not os.path.isdir(local_path): + print( + f" WARNING: data directory '{ref_key}' not cached at {local_path} (run refresh_data_dirs.py)" + ) + _MISSING_DATA_DIRS.add(ref_key) + continue + dd_dest = dd.get("destination", "") + if base_dest and dd_dest: + dd_prefix = f"{base_dest}/{dd_dest}" + elif base_dest: + dd_prefix = base_dest + else: + dd_prefix = dd_dest + # Held while the caller writes what is yielded: a refresh swapping + # the tree waits until the walk is done. + with cache_lock(local_path, shared=True): + for root, _dirs, filenames in os.walk(local_path): + for fname in filenames: + src = os.path.join(root, fname) + rel = os.path.relpath(src, local_path) + full = f"{dd_prefix}/{rel}" if dd_prefix else rel + if reclaim and full in reclaim and ( + os.path.normpath(reclaim[full]) == os.path.normpath(src) + ): + del reclaim[full] if refs_out is not None: - refs_out[full] = (ref_key, sys_id, dd_prefix) + refs_out[full] = (ref_key, systems, dd_prefix) yield src, full + continue + if full in seen_destinations or ( + full.lower() in seen_lower and case_insensitive + ): + continue + if _has_path_conflict(full, seen_destinations, seen_parents): + continue + seen_destinations.add(full) + _register_path(full, seen_destinations, seen_parents) + if case_insensitive: + seen_lower.add(full.lower()) + if refs_out is not None: + refs_out[full] = (ref_key, systems, dd_prefix) + yield src, full def _pack_data_directories( @@ -464,13 +576,14 @@ def _pack_data_directories( seen_lower: set, seen_parents: set, required_only: bool = False, + core_trees: Iterable[dict] = (), ) -> int: """Add the data directories to the pack. Returns how many files went in.""" added = 0 for src, full in _data_directory_members( pack_systems, data_registry, platform_name, base_dest, case_insensitive, seen_destinations, seen_lower, seen_parents, - required_only=required_only, + required_only=required_only, core_trees=core_trees, ): _add_pack_member(zf, src, _flat(full, base_dest, flatten)) added += 1 @@ -1169,10 +1282,23 @@ def generate_pack( save_count += 1 total_files += 1 + # The trees the cores read follow them as their extras do: by the + # systems asked for, and those of no system in the part of their own. + core_trees = ( + [] if source == "platform" + else core_data_trees( + config, emu_profiles, platform_name, data_registry, target_cores + ) + ) + if extras_only: + owned = list(platform_scope_systems(config, emu_profiles, target_cores)) + core_trees = [t for t in core_trees if not _extras_for_systems([t], owned)] + elif system_filter: + core_trees = _extras_for_systems(core_trees, system_filter) total_files += _pack_data_directories( zf, pack_systems, data_registry, platform_name, base_dest, flatten, case_insensitive, seen_destinations, seen_lower, - seen_parents, required_only=required_only, + seen_parents, required_only=required_only, core_trees=core_trees, ) # README.txt for users: personalized step-by-step per platform @@ -4334,6 +4460,9 @@ def generate_manifest( # Sizes of the files the pack carries and the installer cannot fetch: the # collection does not index them, a data directory cache answered. pack_only_sizes: list[int] = [] + # Platform entries resolved into a data directory cache: destination -> + # (cache file, size). Settled once the trees are walked. + cache_held: dict[str, tuple[str, int]] = {} rename_notes: set[str] = set() slot_overrides = slots.pack_overrides( config, emu_profiles, db, zip_contents, data_registry, @@ -4564,7 +4693,10 @@ def generate_manifest( if entry is None: record_omission(full_dest, file_entry, sys_id, "not_found", None) if file_size and full_dest not in seen_destinations: - pack_only_sizes.append(file_size) + if _in_data_cache(local_path, data_registry): + cache_held[full_dest] = (local_path, file_size) + else: + pack_only_sizes.append(file_size) seen_destinations.add(dedup_key) _register_path(dedup_key, seen_destinations, seen_parents) if case_insensitive: @@ -4643,12 +4775,28 @@ def generate_manifest( # Phase 3: data directories. They are not files of the collection: the # installer fetches each from its source archive and keeps the members # whose hash the list below records, the same tree the pack carries. - data_refs: dict[str, tuple[str, str, str]] = {} + data_refs: dict[str, tuple[str, tuple[str, ...], str]] = {} + reclaim = {full: path for full, (path, _size) in cache_held.items()} data_members = list(_data_directory_members( pack_systems, data_registry, platform_name, base_dest, case_insensitive, seen_destinations, seen_lower, seen_parents, data_refs, required_only=required_only, + core_trees=( + [] if source == "platform" + else core_data_trees( + config, emu_profiles, platform_name, data_registry, target_cores + ) + ), + reclaim=reclaim, )) + # A platform entry found in a tree the pack carries is that tree's member: + # the installer rebuilds it from the tree's archive. One found in a tree + # the pack does not carry stays omitted, held by the pack alone. + for full_dest, (_path, size) in cache_held.items(): + if full_dest in reclaim: + pack_only_sizes.append(size) + else: + omitted_by_destination.pop(full_dest, None) data_sizes = pack_only_sizes + [os.path.getsize(src) for src, _dest in data_members] data_directories = _manifest_data_directories( data_members, data_refs, data_registry or {}, base_dest @@ -4738,7 +4886,7 @@ def generate_manifest( def _manifest_data_directories( members: list[tuple[str, str]], - refs: dict[str, tuple[str, str, str]], + refs: dict[str, tuple[str, tuple[str, ...], str]], registry: dict, base_dest: str, ) -> list[dict]: @@ -4752,7 +4900,7 @@ def _manifest_data_directories( """ grouped: dict[tuple[str, str], dict] = {} for src, full in members: - ref, system, prefix = refs[full] + ref, systems, prefix = refs[full] entry = registry.get(ref, {}) key = (ref, prefix) group = grouped.get(key) @@ -4775,8 +4923,7 @@ def _manifest_data_directories( group["source_path"] = str(entry.get("source_path", "")).format(version=version) if entry.get("exclude"): group["exclude"] = [str(e) for e in entry["exclude"]] - if system not in group["systems"]: - group["systems"].append(system) + group["systems"].extend(s for s in systems if s not in group["systems"]) rel = full[len(prefix):].lstrip("/") if prefix else full hashes = compute_hashes(src, frozenset({"sha1"})) group["files"][rel] = {"sha1": hashes["sha1"], "size": os.path.getsize(src)} @@ -5222,6 +5369,7 @@ from packpaths import ( # noqa: E402,F401 from packextras import ( # noqa: E402,F401 ExtraResolver, extra_full_destination, + prefixed_destination, save_entry_fields, save_manifest_dest, _detect_extras_prefix, diff --git a/scripts/pipeline.py b/scripts/pipeline.py index e4300b034..90f08e7dc 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -433,19 +433,26 @@ def parse_pack_paths(output: str) -> dict[str, str]: _PACK_NOTES = re.compile(r"(^README\.txt$|^manifest\.json$|(^|/)(INSTRUCTIONS|RENAMED)_[^/]*\.txt$)") -def _tree_members(platform: str, platforms_dir: str) -> dict[str, int]: +def _tree_members( + platform: str, platforms_dir: str, emulators_dir: str = "emulators", +) -> dict[str, int]: """Archive path -> size of every file the platform's data trees can add.""" import generate_pack # noqa: PLC0415 - from common import load_platform_config # noqa: PLC0415 + from common import load_emulator_profiles, load_platform_config # noqa: PLC0415 from packpaths import _flat # noqa: PLC0415 config = load_platform_config(platform, platforms_dir) base_dest = config.get("base_destination", "") + registry = load_data_dir_registry(platforms_dir) + core_trees = generate_pack.core_data_trees( + config, load_emulator_profiles(emulators_dir), platform, registry, + ) members: dict[str, int] = {} with contextlib.redirect_stdout(io.StringIO()): for src, full in generate_pack._data_directory_members( - config.get("systems", {}), load_data_dir_registry(platforms_dir), + config.get("systems", {}), registry, platform, base_dest, False, set(), set(), set(), + core_trees=core_trees, ): members[_flat(full, base_dest, True)] = os.path.getsize(src) return members diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 94317c432..ed9cfbf35 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5544,7 +5544,10 @@ class DocumentedExamplesHoldTogether(unittest.TestCase): page = (ROOT / "wiki" / "installer.md").read_text(encoding="utf-8") downloading = re.search(r"Downloading (\d+) files", page).group(1) done = re.search(r"Done\. (\d+) files installed", page).group(1) - self.assertEqual(done, downloading) + # The data trees come after the download, counted in the index. + index = page[page.index("Fetching file index for batocera"):] + trees = re.search(r"^ .* and (\d+) data files", index, re.M) + self.assertEqual(int(done), int(downloading) + (int(trees.group(1)) if trees else 0)) diff --git a/tests/test_core_data_trees.py b/tests/test_core_data_trees.py new file mode 100644 index 000000000..6cc8c8ca0 --- /dev/null +++ b/tests/test_core_data_trees.py @@ -0,0 +1,169 @@ +"""A data directory a core reads reaches every platform that runs the core. + +The registry entries of the buildbot trees named RetroArch, Lakka and +RetroPie only, and a platform pack carried only the trees its own systems +declared. AmberELEC and EmuELEC run the blueMSX core, whose package copies +the .so alone: their packs filled the machine configs by name from the +cache, and the install lists left 192 of them out. A platform that runs a +reader now gets the tree, or the profile says its package installs it. +""" + +from __future__ import annotations + +import os +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import common # noqa: E402 +import generate_pack as builder # noqa: E402 +from validation import deliverable, read_from_system_dir # noqa: E402 + +PROFILES = { + "demo": { + "emulator": "Demo", + "type": "libretro", + "systems": ["demo-system"], + "cores": ["demo"], + "pack_structure": {"libretro": "Demo"}, + "data_directories": [ + {"ref": "shared", "destination": "Sys"}, + {"ref": "restricted"}, + {"ref": "installed", "installed_on": ["plat"]}, + {"ref": "beside", "load_from": "content_dir"}, + ], + }, + "solo": { + "emulator": "Solo", + "type": "standalone", + "systems": ["demo-system"], + "cores": ["solo"], + "data_directories": [{"ref": "shared"}], + }, +} + +REGISTRY = { + "shared": {"local_cache": "data/shared"}, + "restricted": {"local_cache": "data/restricted", "for_platforms": ["other"]}, + "installed": {"local_cache": "data/installed"}, + "beside": {"local_cache": "data/beside"}, +} + +CONFIG = { + "platform": "Plat", + "cores": ["demo", "solo"], + "standalone_cores": ["solo"], + "systems": {"demo-system": {"files": []}}, +} + + +class CoreDataTreesTest(unittest.TestCase): + def trees(self, platform: str = "plat", registry: dict | None = None) -> list[dict]: + return builder.core_data_trees( + CONFIG, PROFILES, platform, REGISTRY if registry is None else registry + ) + + def test_tree_below_the_build_folder(self) -> None: + shared = [t for t in self.trees() if t["ref"] == "shared"] + self.assertEqual([(t["core"], t["destination"]) for t in shared], [("demo", "Demo/Sys")]) + self.assertEqual(shared[0]["source_systems"], ["demo-system"]) + + def test_for_platforms_names_who_gets_it(self) -> None: + self.assertNotIn("restricted", {t["ref"] for t in self.trees()}) + self.assertIn("restricted", {t["ref"] for t in self.trees("other")}) + + def test_installed_tree_stays_out(self) -> None: + self.assertNotIn("installed", {t["ref"] for t in self.trees()}) + self.assertIn("installed", {t["ref"] for t in self.trees("other")}) + + def test_tree_read_beside_the_content_stays_out(self) -> None: + self.assertNotIn("beside", {t["ref"] for t in self.trees("other")}) + + def test_standalone_build_brings_no_tree(self) -> None: + self.assertNotIn("solo", {t["core"] for t in self.trees("other")}) + + def test_tree_below_the_declared_bios_folder(self) -> None: + config = { + **CONFIG, + "base_destination": "", + "systems": {"demo-system": {"files": [{"name": "a.bin", "destination": "bios/a.bin"}]}}, + } + trees = builder.core_data_trees(config, PROFILES, "plat", REGISTRY) + self.assertEqual( + [t["destination"] for t in trees if t["ref"] == "shared"], ["bios/Demo/Sys"] + ) + + def test_no_registry_no_tree(self) -> None: + self.assertEqual(builder.core_data_trees(CONFIG, PROFILES, "plat", None), []) + + +class EveryReaderHasTheTreeOrAReason(unittest.TestCase): + """Reads the repository: platforms/_data_dirs.yml against the profiles.""" + + def test_for_platforms_leaves_no_reader_without_a_reason(self) -> None: + platforms_dir = str(REPO_ROOT / "platforms") + registry = common.load_data_dir_registry(platforms_dir) + profiles = common.load_emulator_profiles(str(REPO_ROOT / "emulators")) + unexplained = [] + for platform in common.list_registered_platforms(platforms_dir, include_archived=True): + config = common.load_platform_config(platform, platforms_dir) + handed = common.standalone_set(config) + for core in sorted(common.resolve_platform_cores(config, profiles)): + profile = profiles[core] + if all(standalone for _view, standalone + in common.platform_layouts(core, profile, handed)): + continue + trees = [t for t in profile.get("data_directories") or [] if isinstance(t, dict)] + installed = common.installed_data_refs(platform) + for tree in deliverable(trees, False, platform): + allowed = registry.get(tree.get("ref"), {}).get("for_platforms") + if tree.get("ref") in installed or not read_from_system_dir(tree): + continue + if allowed and platform not in allowed: + unexplained.append(f"{tree['ref']} on {platform} ({core})") + self.assertEqual( + unexplained, [], + "these platforms run a core that reads the tree, are not in its " + "for_platforms, and no profile says their package installs it " + "(installed_on) or that they never open it (unread_on)", + ) + + +@unittest.skipUnless( + os.path.isdir(REPO_ROOT / "data" / "bluemsx"), "data/bluemsx is not cached" +) +class AmberelecCarriesTheMachines(unittest.TestCase): + """AmberELEC's blueMSX package installs the core alone, so the pack and + the install list bring the Machines tree its BIOS list names.""" + + def test_machine_configs_come_with_the_tree(self) -> None: + platforms_dir = str(REPO_ROOT / "platforms") + cwd = os.getcwd() + os.chdir(REPO_ROOT) + try: + config = common.load_platform_config("amberelec", platforms_dir) + registry = common.load_data_dir_registry(platforms_dir) + profiles = common.load_emulator_profiles(str(REPO_ROOT / "emulators")) + trees = builder.core_data_trees(config, profiles, "amberelec", registry) + refs: dict = {} + members = dict( + (full, src) for src, full in builder._data_directory_members( + config.get("systems", {}), registry, "amberelec", + config.get("base_destination", ""), False, set(), set(), set(), + refs, core_trees=trees, + ) + ) + finally: + os.chdir(cwd) + # Below bios/, where the platform writes every destination. + self.assertIn("bios/Machines/COL - ColecoVision/config.ini", members) + self.assertEqual( + refs["bios/Machines/COL - ColecoVision/config.ini"][0], "bluemsx" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 8b42cafaa..0935b2bcb 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -845,9 +845,14 @@ class FirstInstallWalkthrough(unittest.TestCase): self.assertNotRegex(self.text, r"\d+ (in place|missing|files installed)") self.assertIn(f" {install.EVERYTHING_IN_PLACE}", self.text) step3 = _generated(self.run, "run-index") - count, size = re.search(r"\n (\d+) files \(([\d.]+ [KMG]B)\)", step3).groups() + count = re.search(r"\n (\d+) files \([\d.]+ [KMG]B\)", step3).group(1) self.assertIn(f"0 in place, {count} missing", step3) - self.assertIn(f"Downloading {count} files ({size})", _generated(self.run, "run-download")) + # The data trees are fetched after the files, from their archives: + # the download line counts the files alone. + trees = re.search(r" and (\d+) data files", step3) + downloaded = int(count) - (int(trees.group(1)) if trees else 0) + self.assertRegex(_generated(self.run, "run-download"), + rf"Downloading {downloaded} files \([\d.]+ [KMG]B\)") step4 = _generated(self.run, "run-end") self.assertIn(f"Done. {count} files installed", step4) self.assertIn(f"{count} in place, 0 missing", _generated(self.run, "run-check")) diff --git a/wiki/profiling.md b/wiki/profiling.md index bd63cb157..feb7414ce 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -548,7 +548,7 @@ reports the profile as uncheckable, and the CI job that runs | `mode` | no | default mode: `standalone`, `libretro`, or `both` | | `verification` | no | how the core verifies BIOS: `existence`, `md5`, `sha1`, `crc32` | | `files` | yes, unless `type: alias` | list of file entries | -| `data_directories` | no | whole directory trees the core needs, referencing `_data_dirs.yml` keys. An entry may carry `mode:` when only one build reads the tree, and `required: false` when the core runs without it (`--required-only` then leaves it out) | +| `data_directories` | no | whole directory trees the core needs, referencing `_data_dirs.yml` keys. An entry may carry `mode:` when only one build reads the tree, `required: false` when the core runs without it (`--required-only` then leaves it out), and `load_from: content_dir` when the core reads it beside the content rather than in the system directory. Every platform that runs the core as a libretro core gets the tree in its pack, unless the entry names it in `installed_on` or `unread_on`, or the `_data_dirs.yml` entry has a `for_platforms` list without it | | `notes` | no | free-form technical notes | | `note` | no | single-paragraph variant of `notes` | | `exclusion_note` | no | why the profile has no files despite .info declaring firmware | diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index ed925b0e1..e20528b72 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -52,7 +52,7 @@ Every module in `tests/`, by the area it guards. | Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` | | File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_archive_recipes`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_np2_fonts`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` | | Verification, cross-reference and profile checks | synthetic + profiles | `test_bios_layout`, `test_cache_independent_counts`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_names`, `test_profile_paths`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | -| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | +| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_core_data_trees`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_games`, `test_install_offline`, `test_install_selection`, `test_install_tui` |