diff --git a/install.ps1 b/install.ps1 index 7a8b8a040..32699e0d1 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "9dcac77739d3cf609879117fb16bcf798c4b6432b8647cf49dbef52d6650adeb" + $defaultInstallSha256 = "68d93a9905c324bc378ead0c0a7cbbecaf47a3c113265cb9f7c40ad40338b9a3" # The pin above refuses any changed byte: this cap only stops a body that # cannot be the installer (a sign-in portal's page) before it is hashed. $maximumInstallerBytes = 8MB diff --git a/install.py b/install.py index 937bf30a9..0f8a6d6c7 100644 --- a/install.py +++ b/install.py @@ -4237,7 +4237,7 @@ def _validate_manifest(data: object, plat: str) -> dict: ): raise ValueError("manifest total_omitted does not match omitted_files") - for field in ("core_names", "system_cores", "system_names"): + for field in ("core_names", "downloaded_cores", "system_cores", "system_names"): mapping = data.get(field) if mapping is None: continue @@ -4828,18 +4828,28 @@ def _filter_by_target( return result -def designated_cores(manifest: dict, names: list[str]) -> set[str]: +def designated_cores( + manifest: dict, names: list[str], target: str | None = None +) -> set[str]: """The cores a list of names designates, the way the pack builder reads it. A target list, or a core found installed, names a core the buildbot's way (`mednafen_psx`), and a file names the profile that reads it - (`beetle_psx`). `core_names` gives each profile its other names. + (`beetle_psx`). `core_names` gives each profile its other names. A core + only the platform's RetroArch downloader brings runs on the boards + `downloaded_cores` gives it: Batocera's bcm2711 list names its own + amiberry, and the buildbot's amiberry core is no download there. """ wanted = set(names) chosen = set(wanted) for profile, aliases in (manifest.get("core_names") or {}).items(): if wanted.intersection(aliases): chosen.add(profile) + if target is not None: + chosen -= { + profile for profile, boards in (manifest.get("downloaded_cores") or {}).items() + if target not in boards + } return chosen @@ -4849,6 +4859,7 @@ def narrow_to_cores( files: list[dict] | None = None, omitted: list[dict] | None = None, declared_by_info: Collection[str] = (), + target: str | None = None, ) -> tuple[list[dict], list[dict]]: """Keep what the named cores read, as a pack built for them carries. @@ -4859,9 +4870,10 @@ def narrow_to_cores( way to the first named core claiming its destination. A manifest without the two maps keeps the platform's list whole and matches cores by name. A destination in `declared_by_info`, what the .info of an installed core - no profile describes declares, stays whatever is named. + no profile describes declares, stays whatever is named. `target` is the + board the names are the list of. """ - cores = designated_cores(manifest, names) + cores = designated_cores(manifest, names, target) reach = manifest.get("system_cores") or {} info_dests = {dest.casefold() for dest in declared_by_info} @@ -16175,7 +16187,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: else: before = len(files) files, omitted_files = narrow_to_cores( - manifest, target_info["cores"], files, omitted_files + manifest, target_info["cores"], files, omitted_files, + target=target, ) print(f" Filtered {before} to {len(files)} files for target {target}") diff --git a/install.sh b/install.sh index 2723cd262..88f7d3b79 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="9dcac77739d3cf609879117fb16bcf798c4b6432b8647cf49dbef52d6650adeb" +DEFAULT_INSTALL_SHA256="68d93a9905c324bc378ead0c0a7cbbecaf47a3c113265cb9f7c40ad40338b9a3" # The pin above refuses any changed byte: this cap only stops a body that # cannot be the installer (a sign-in portal's page) before it is hashed. MAX_INSTALLER_BYTES=8388608 diff --git a/install/launchers/index.json b/install/launchers/index.json index a7757974e..eec6fb563 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1150418, - "sha256": "9dcac77739d3cf609879117fb16bcf798c4b6432b8647cf49dbef52d6650adeb", + "size": 1151002, + "sha256": "68d93a9905c324bc378ead0c0a7cbbecaf47a3c113265cb9f7c40ad40338b9a3", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1204043, - "sha256": "24a025696054f5a9371eab36b0f43332ea1cd3c35a706619a6ddc2559c4aa220" + "size": 1204627, + "sha256": "45d6a1c8ccdf2309783c391be51e274939890b25c5b78ee10b435155a934c271" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1150418, - "sha256": "9dcac77739d3cf609879117fb16bcf798c4b6432b8647cf49dbef52d6650adeb", + "size": 1151002, + "sha256": "68d93a9905c324bc378ead0c0a7cbbecaf47a3c113265cb9f7c40ad40338b9a3", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1150418, - "sha256": "9dcac77739d3cf609879117fb16bcf798c4b6432b8647cf49dbef52d6650adeb", + "size": 1151002, + "sha256": "68d93a9905c324bc378ead0c0a7cbbecaf47a3c113265cb9f7c40ad40338b9a3", "executable": false }, { diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index ea313b6b8..51eaf7f50 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -130,6 +130,18 @@ "uniqueItems": true } }, + "downloaded_cores": { + "description": "Each core only the platform's RetroArch core downloader brings, with the boards and board aliases whose downloader offers it. install.py --target keeps the files of such a core on those boards only.", + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + } + }, "system_cores": { "description": "Each platform system with the cores whose presence keeps it. A system absent here is kept whatever cores are available.", "type": "object", diff --git a/scripts/common.py b/scripts/common.py index 4d8ab742a..f11f86c44 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -6,7 +6,7 @@ and file resolution - eliminates DRY violations across scripts. from __future__ import annotations -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Collection, Iterable, Mapping from collections import Counter import argparse @@ -3230,9 +3230,30 @@ def resolve_platform_cores( Alias profiles are always excluded (they point to another profile). If target_cores is provided, result is intersected with it. """ + result, downloaded_only = _platform_profiles(config, profiles) + if target_cores is not None: + result = result & designated_profiles(profiles, target_cores) + result -= { + profile for profile, names in downloaded_only.items() + if not downloadable_on(target_cores, names) + } + return result + + +def downloaded_only_profiles( + config: dict, profiles: dict[str, dict] +) -> dict[str, frozenset[str]]: + """The profiles a platform reaches only through its RetroArch core + downloader, each with the installable names that reach it.""" + return _platform_profiles(config, profiles)[1] + + +def _platform_profiles( + config: dict, profiles: dict[str, dict] +) -> tuple[set[str], dict[str, frozenset[str]]]: cores_config = config.get("cores") platform_key = str(config.get("_key") or "") - downloaded_only: dict[str, str] = {} + downloaded_only: dict[str, set[str]] = {} # The builds the platform compiles itself, by the names they answer to. # Such a name means that build there, and another revision of the same # project is not what the platform runs: ROCKNIX and RetroPie build only @@ -3323,9 +3344,11 @@ def resolve_platform_cores( # A core reached only through the RetroArch downloader runs on the # boards whose downloader offers it: Batocera's x86_64 RetroArch # installs the amiberry core, its armv7 and aarch64 boards cannot. + # Every installable name counts: Batocera's x86_64 downloader offers + # nside_sfc_balanced, not the higan_sfc_balanced it also lists. for c in config.get("installable_cores") or []: for profile in libretro_builds(c) - dual - result: - downloaded_only.setdefault(profile, str(c)) + downloaded_only.setdefault(profile, set()).add(str(c)) result |= dual | set(downloaded_only) # Support "all_libretro" as a list element: combines all libretro # profiles with explicitly listed standalone cores (e.g. RetroDECK @@ -3343,13 +3366,7 @@ def resolve_platform_cores( and not p.get("built_by") } - if target_cores is not None: - result = result & designated_profiles(profiles, target_cores) - result -= { - profile for profile, c in downloaded_only.items() - if not downloadable_on(target_cores, c) - } - return result + return result, {p: frozenset(names) for p, names in downloaded_only.items()} @functools.lru_cache(maxsize=1) @@ -3362,14 +3379,18 @@ def _retroarch_target_sets() -> tuple[frozenset[str], ...]: ) -def downloadable_on(target_cores: set[str], name: str) -> bool: - """Whether a board's core set holds a RetroArch download list offering `name`. +def downloadable_on(target_cores: set[str], names: Collection[str]) -> bool: + """Whether a board's core set holds a RetroArch download list offering + one of `names`. A board with a downloader carries a whole RetroArch targets entry in its set (load_target_config, `downloads`), a board without one only what it builds. """ - return any(name in found and found <= target_cores for found in _retroarch_target_sets()) + return any( + found <= target_cores and not found.isdisjoint(names) + for found in _retroarch_target_sets() + ) def unresolved_platform_cores(config: dict, profiles: dict[str, dict]) -> set[str]: diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index b62ce5a06..fd3a39032 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -86,6 +86,9 @@ from common import ( ProfileSelectionError, core_system_dirs, declared_root, + downloadable_on, + downloaded_only_profiles, + list_available_targets, entry_serves_systems, entry_system_ids, platform_layouts, @@ -5291,6 +5294,9 @@ def generate_manifest( omitted_by_destination.values(), key=lambda entry: entry["dest"] ), "core_names": core_names, + **({"downloaded_cores": downloaded} if (downloaded := _downloaded_cores( + config, emu_profiles, core_names, platforms_dir, + )) else {}), "system_cores": system_cores, # The names the platform itself gives its systems (Batocera's psx), # which --system accepts beside the ids. @@ -5445,6 +5451,39 @@ def _manifest_data_directories( return trees +def _downloaded_cores( + config: dict, + profiles: dict[str, dict], + core_names: dict[str, list[str]], + platforms_dir: str, +) -> dict[str, list[str]]: + """The cores only the platform's RetroArch downloader brings, each with + the boards, aliases included, whose downloader offers it. + + A target list names the board's standalone amiberry and the buildbot's + amiberry core alike, so the installer cannot tell from the name whether + the board reaches the core. `resolve_platform_cores` decides it, and + `install.py --target` reads its answer here. + """ + reached = { + profile: names + for profile, names in downloaded_only_profiles(config, profiles).items() + if profile in core_names + } + if not reached: + return {} + platform = str(config.get("_key") or "") + boards: dict[str, list[str]] = {profile: [] for profile in reached} + for target in list_available_targets(platform, platforms_dir): + cores = load_target_config(platform, target["name"], platforms_dir) + if cores is None: + continue + for profile, names in reached.items(): + if downloadable_on(cores, names): + boards[profile] += [target["name"], *target.get("aliases", [])] + return {profile: sorted(names) for profile, names in sorted(boards.items())} + + def _manifest_core_index( config: dict, profiles: dict[str, dict], diff --git a/tests/test_install_cores.py b/tests/test_install_cores.py index a4e3c88ab..6ff2c2285 100644 --- a/tests/test_install_cores.py +++ b/tests/test_install_cores.py @@ -113,6 +113,38 @@ class NarrowToCores(unittest.TestCase): ) +class ACoreOnlyTheDownloaderBrings(unittest.TestCase): + """Batocera builds amiberry standalone on every board, and only its + x86_64 RetroArch downloads the amiberry core: both answer to amiberry + in a board's core list. The builder reads the downloader's reach + (resolve_platform_cores) and the list carries it in downloaded_cores.""" + + MANIFEST = { + "core_names": {"amiberry": [], "amiberry-7": ["amiberry"]}, + "downloaded_cores": {"amiberry": ["x86_64", "pc"]}, + "system_cores": {"commodore-amiga": ["amiberry", "amiberry-7"]}, + "files": [ + {"dest": "kick31.rom", "cores": ["amiberry-7"], "systems": ["commodore-amiga"]}, + {"dest": "roms/a4091.rom", "cores": ["amiberry"], "systems": ["commodore-amiga"]}, + ], + "omitted_files": [], + } + + def test_a_board_without_the_download_keeps_its_own_build(self): + files, _ = install.narrow_to_cores(self.MANIFEST, ["amiberry"], target="bcm2711") + self.assertEqual([f["dest"] for f in files], ["kick31.rom"]) + + def test_a_board_or_its_alias_with_the_download_keeps_both(self): + for target in ("x86_64", "pc"): + with self.subTest(target=target): + files, _ = install.narrow_to_cores(self.MANIFEST, ["amiberry"], target=target) + self.assertEqual([f["dest"] for f in files], ["kick31.rom", "roms/a4091.rom"]) + + def test_an_installed_core_is_no_board(self): + files, _ = install.narrow_to_cores(self.MANIFEST, ["amiberry"]) + self.assertEqual([f["dest"] for f in files], ["kick31.rom", "roms/a4091.rom"]) + + class ACoreTheListDoesNotKnow(unittest.TestCase): """A core newer than the list (parallel_dc, issue 87) loses nothing its .info declares.""" diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 852699169..76f488d43 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -1015,7 +1015,7 @@ class InstallMatchesATargetManifest(unittest.TestCase): )) full = self._manifest(platform) built = self._manifest(platform, set(cores), target) - files, omitted = install.narrow_to_cores(full, cores) + files, omitted = install.narrow_to_cores(full, cores, target=target) # A case-insensitive platform reads one file under both # spellings (BlastEm's 32X_M_BIOS.bin, PicoDrive's .BIN). from common import load_platform_config diff --git a/tests/test_target_scrapers.py b/tests/test_target_scrapers.py index 65a90da7d..0a1bc8848 100644 --- a/tests/test_target_scrapers.py +++ b/tests/test_target_scrapers.py @@ -335,6 +335,42 @@ class CoreDownloaderReach(unittest.TestCase): self.assertIn(DUAL_MODE_PREFIX + "amiberry", standalone_set(config)) +class DownloadedCoresReachTheirBoards(unittest.TestCase): + """A core only the downloader brings runs where the board's downloader + offers one of its names, and the install list says on which boards.""" + + @classmethod + def setUpClass(cls): + from common import load_emulator_profiles, load_platform_config # noqa: PLC0415 + + cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + cls.config = load_platform_config("batocera", str(REPO_ROOT / "platforms")) + + def test_every_installable_name_counts(self): + """Batocera lists higan_sfc_balanced and nside_sfc_balanced, and the + buildbot offers the second: reading the first alone dropped the + core from x86_64.""" + from common import load_target_config, resolve_platform_cores # noqa: PLC0415 + + root = str(REPO_ROOT / "platforms") + x86 = load_target_config("batocera", "x86_64", root) + self.assertIn("higan_sfc_balanced", resolve_platform_cores(self.config, self.profiles, x86)) + rpi4 = load_target_config("batocera", "bcm2711", root) + resolved = resolve_platform_cores(self.config, self.profiles, rpi4) + self.assertNotIn("amiberry", resolved) + self.assertIn("amiberry-7.1.1-332", resolved) + + def test_the_list_names_the_boards_of_each_downloaded_core(self): + import generate_pack # noqa: PLC0415 + + core_names = {"amiberry": [], "higan_sfc_balanced": ["nside_sfc_balanced"]} + boards = generate_pack._downloaded_cores( + self.config, self.profiles, core_names, str(REPO_ROOT / "platforms")) + self.assertEqual(set(boards), {"amiberry", "higan_sfc_balanced"}) + self.assertIn("x86_64", boards["amiberry"]) + self.assertNotIn("bcm2711", boards["amiberry"]) + + class KnulliPerBoard(unittest.TestCase): """KNULLI builds per board on Batocera's model from its own tree: configs/knulli-*.board, package/system/knulli-system/Config.in and