From c638c12ca8d089fd0456b89cbfbb2d26893d2cb0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:08:16 +0200 Subject: [PATCH 01/11] fix: replay the builder's held and one-file groups install.region_drops now skips one-file groups (:file:) and lets no omitted match displace an installable file, as region.resolve_region_drops does on the builder side. The property test draws both cases and compares against the builder. --- install.ps1 | 2 +- install.py | 43 ++++++++++++++++----- install.sh | 2 +- tests/test_install_selection.py | 68 +++++++++++++++++++++++++++++---- 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/install.ps1 b/install.ps1 index 2c16dbe0..fa2ce047 100644 --- a/install.ps1 +++ b/install.ps1 @@ -12,7 +12,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "376edf794195d74f21e598854b5f3e90fb1955513835264bc9326ad4d2acf9c9" + $defaultInstallSha256 = "973d64627af05b239555ab5a83a8902747b7a742bc1176f1b994cf7ac74774f9" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 32e6a681..4e18bb17 100644 --- a/install.py +++ b/install.py @@ -142,6 +142,8 @@ _REGION_PARENT = { member: parent for parent, members in REGION_TREE.items() for member in members } REGIONS = frozenset({WORLD_REGION} | set(REGION_TREE) | set(_REGION_PARENT)) +# Marks a region group named after one file that no system names. +FILE_GROUP = ":file:" MAX_SELECTION_TAGS = 1000 SELECTION_FIELDS = ("systems", "regions", "region_groups", "region_system_groups") # The files cores load for systems the platform does not list (game engines, @@ -2220,7 +2222,10 @@ def _region_rank(file_regions: set[str], requested: list[str]) -> int: def region_drops( - entries: list[dict], requested: list[str], systems: list[str] | None = None + entries: list[dict], + requested: list[str], + systems: list[str] | None = None, + held: set[str] | None = None, ) -> set[str]: """Destinations a region priority list withdraws. @@ -2228,10 +2233,16 @@ def region_drops( groups the manifest records. Per group, the best-ranked regional file stays. A world file beats unmatched regional ones the code does not search before it (`priority`, lower first). When nothing matches and no world - file exists, every regional file stays, so a group is never emptied. A file with no region never competes and always stays. + file exists, every regional file stays, so a group is never emptied. A + file with no region never competes and always stays. A core extra competes whatever systems are chosen, while a platform's own declaration competes only when its system is kept, as in the pack. + + ``held`` names the destinations the manifest can install. A match it + lacks never displaces a held file: the American BIOS stays under + --region japan when the Japanese one is missing. A group named after a + single file (FILE_GROUP) competes with nothing and keeps nothing. """ if not requested: return set() @@ -2253,7 +2264,9 @@ def region_drops( groups.setdefault(group, []).append((set(regions), entry["dest"], priority)) keep: set[str] = set() drop: set[str] = set() - for members in groups.values(): + for group_id, members in groups.items(): + if FILE_GROUP in group_id: + continue world = {dest for regions, dest, _p in members if WORLD_REGION in regions} ranks = {dest: priority for _regions, dest, priority in members} regional = [ @@ -2265,12 +2278,19 @@ def region_drops( if not regional: continue matched = [(rank, dest) for rank, dest in regional if rank < len(requested)] - if matched: - best = min(rank for rank, _dest in matched) - keep |= {dest for rank, dest in matched if rank == best} - drop |= {dest for _rank, dest in regional} - elif world: - world_ranks = [ranks[dest] for dest in world] + winners, rivals = matched, world + if held is not None: + winners = [(rank, dest) for rank, dest in matched if dest in held] + rivals = world & held + if winners: + best = min(rank for rank, _dest in winners) + keep |= { + dest for rank, dest in matched + if rank == best or (rank < best and dest not in (held or ())) + } + drop |= {dest for _rank, dest in regional if dest not in keep} + elif rivals: + world_ranks = [ranks[dest] for dest in rivals] best_world = min(world_ranks) if None not in world_ranks else None for _rank, dest in regional: ours = ranks[dest] @@ -2310,7 +2330,10 @@ def narrow( files = _filter_by_target(files, cores) omitted = _filter_by_target(omitted, cores) if regions: - dropped = region_drops(files + omitted, regions, systems) + # The builder's held set: what the collection serves. An omission + # answers the request but takes no installable file's place. + held = {f["dest"] for f in files} + dropped = region_drops(files + omitted, regions, systems, held) files = [f for f in files if f["dest"] not in dropped] omitted = [o for o in omitted if o["dest"] not in dropped] if systems: diff --git a/install.sh b/install.sh index a7948a1c..fa86e185 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="376edf794195d74f21e598854b5f3e90fb1955513835264bc9326ad4d2acf9c9" +DEFAULT_INSTALL_SHA256="973d64627af05b239555ab5a83a8902747b7a742bc1176f1b994cf7ac74774f9" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 66c8d9dc..6c2eb183 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -43,6 +43,7 @@ class OneRegionVocabulary(unittest.TestCase): self.assertEqual(install.REGION_ALIASES, region.ALIASES) self.assertEqual(install.WORLD_REGION, region.WORLD) self.assertEqual(install.REGIONS, region.REGIONS) + self.assertEqual(install.FILE_GROUP, region.FILE_GROUP) class TheRegionRuleIsTheBuilders(unittest.TestCase): @@ -72,26 +73,68 @@ class TheRegionRuleIsTheBuilders(unittest.TestCase): index[dest] = {"regions": regions, "has_untagged": not regions, "emulators": [], "priorities": {priority}} - groups.setdefault(f"g{g}", []).append((dest, dest)) - if f"g{g}" not in entries[dest]["region_groups"]: - entries[dest]["region_groups"].append(f"g{g}") + # A lone core file no system names gets a group of its own. + group = f"g{g}" if rng.random() < 0.85 else f"e{region.FILE_GROUP}{dest}" + groups.setdefault(group, []).append((dest, dest)) + if group not in entries[dest]["region_groups"]: + entries[dest]["region_groups"].append(group) for entry in entries.values(): if not entry["regions"]: del entry["regions"], entry["region_groups"] entry.pop("priority", None) requested = rng.sample(["north-america", "europe", "japan", "uk", "canada"], rng.randint(1, 3)) - return groups, index, list(entries.values()), requested + held = {dest for dest in entries if rng.random() < 0.7} + return groups, index, list(entries.values()), requested, held def test_two_thousand_random_platforms(self): rng = random.Random(20261010) for trial in range(2000): - groups, index, entries, requested = self._case(rng) + groups, index, entries, requested, held = self._case(rng) with self.subTest(trial=trial): self.assertEqual( install.region_drops(entries, requested), region.resolve_region_drops(groups, index, requested), ) + self.assertEqual( + install.region_drops(entries, requested, held=held), + region.resolve_region_drops(groups, index, requested, held), + ) + + def test_narrow_passes_the_installable_files_as_held(self): + """The builder passes what the collection holds (generate_pack + _select_variants). An omission is the installer's unheld file.""" + rng = random.Random(20261021) + for trial in range(500): + groups, index, entries, requested, held = self._case(rng) + files = [dict(e, size=1, cores=None) for e in entries if e["dest"] in held] + omitted = [dict(e, cores=None) for e in entries if e["dest"] not in held] + kept, still_omitted = install.narrow(files, omitted, [], [], requested) + drops = region.resolve_region_drops(groups, index, requested, held) + with self.subTest(trial=trial): + self.assertEqual( + {f["dest"] for f in kept}, + {f["dest"] for f in files} - drops, + ) + self.assertEqual( + {o["dest"] for o in still_omitted}, + {o["dest"] for o in omitted} - drops, + ) + + def test_a_lone_core_file_leaves_its_systems_drop(self): + """Dolphin's GC/EUR/IPL.bin is a core extra no system names, plus the + GameCube's own declaration. The builder skips the one-file group, so + --region us drops it.""" + entries = [ + {"dest": "GC/USA/IPL.bin", "regions": ["north-america"], + "region_system_groups": ["nintendo-gamecube"]}, + {"dest": "GC/EUR/IPL.bin", "regions": ["europe"], + "region_groups": [f"dolphin{region.FILE_GROUP}GC/EUR/IPL.bin"], + "region_system_groups": ["nintendo-gamecube"]}, + ] + self.assertEqual( + install.region_drops(entries, ["north-america"]), {"GC/EUR/IPL.bin"} + ) class Narrowing(unittest.TestCase): @@ -138,12 +181,21 @@ class Narrowing(unittest.TestCase): self.assertIn("scph5501.bin", files) self.assertEqual(omitted, []) - def test_an_unavailable_match_still_wins_as_the_builder_counts_it(self): + def test_an_unavailable_match_never_displaces_a_held_file(self): + """The builder keeps the held BIOS when the requested one is missing + (region.resolve_region_drops, held). The missing one stays reported.""" files, omitted = self._dests([], [], ["europe"]) - self.assertNotIn("scph5500.bin", files) - self.assertNotIn("scph5501.bin", files) + self.assertIn("scph5500.bin", files) + self.assertIn("scph5501.bin", files) self.assertEqual(omitted, ["scph5502.bin"]) + def test_a_missing_japanese_bios_keeps_the_american_one(self): + files = [dict(self.FILES[1])] + omitted = [dict(self.OMITTED[0], dest="scph5500.bin", regions=["japan"])] + kept, still = install.narrow(files, omitted, [], [], ["japan"]) + self.assertEqual([f["dest"] for f in kept], ["scph5501.bin"]) + self.assertEqual([o["dest"] for o in still], ["scph5500.bin"]) + def test_no_match_keeps_every_regional_file(self): files, _ = self._dests([], [], ["brazil"]) self.assertIn("scph5500.bin", files) From 9b1d378d2fe8051f9138b908239071d09baf6a33 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:11:54 +0200 Subject: [PATCH 02/11] fix: never truncate a copy another run renames On Windows a verified partial copy is closed before its rename. A second run that claimed it in that moment truncated it and the first rename failed with a sharing violation, reported as an unwritable folder. A complete partial is now left whole there, the rename waits for a sharing violation to clear, and a sharing violation no longer reads as a refused folder. --- install.ps1 | 2 +- install.py | 90 +++++++++++++++++++++++++++++++++++++++-- install.sh | 2 +- tests/test_install.py | 94 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 5 deletions(-) diff --git a/install.ps1 b/install.ps1 index fa2ce047..2165680f 100644 --- a/install.ps1 +++ b/install.ps1 @@ -12,7 +12,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "973d64627af05b239555ab5a83a8902747b7a742bc1176f1b994cf7ac74774f9" + $defaultInstallSha256 = "007fffece97716aa9e1ed0606a8453f2da8a1262711aa68ca910673956b2b341" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 4e18bb17..406d7f8d 100644 --- a/install.py +++ b/install.py @@ -2576,6 +2576,7 @@ def _download_one( resumable = expected_size >= RESUME_FROM_SIZE reason = "download failed" refused = False + full_partial = False for attempt in range(1, MAX_RETRIES + 1): tmp_path: Path | None = None counted = 0 @@ -2594,7 +2595,17 @@ def _download_one( tmp_path = None else: offset = os.fstat(partial.fileno()).st_size - if offset >= expected_size: + if offset >= expected_size and _NO_RENAME_WHILE_OPEN: + # The run that verified this copy releases its lock + # to rename it: truncating it there destroyed that + # copy and failed the rename. It is left alone and + # this run writes a private file. + partial.close() + partial = None + tmp_path = None + offset = 0 + full_partial = True + elif offset >= expected_size: partial.truncate(0) offset = 0 request = urllib.request.Request(url) @@ -2665,11 +2676,15 @@ def _download_one( # written with sudo has to stay readable by the emulator's user. os.chmod(tmp_path, (0o777 if f.get("executable") else 0o666) & ~_UMASK) _clear_read_only(dest) - if partial is not None and os.name == "nt": + if partial is not None and _NO_RENAME_WHILE_OPEN: # Windows renames no open file. Elsewhere the lock is held # until the copy has its final name. partial.close() - os.replace(tmp_path, dest) + _replace_released(tmp_path, dest, expected) + else: + os.replace(tmp_path, dest) + if full_partial: + _drop_full_partial(dest, expected_size) return f["dest"], True, "" except _Interrupted: @@ -2739,6 +2754,71 @@ def _download_one( return f["dest"], False, reason +# Windows renames no file another handle holds open, and Python opens files +# without FILE_SHARE_DELETE: a locked partial copy is closed before its rename. +_NO_RENAME_WHILE_OPEN = os.name == "nt" +# Seconds a closed partial copy waits for a sharing violation to clear: a +# second run closing it again, or a scanner reading the new file. +RELEASE_WAIT = 30.0 +_SHARING_VIOLATIONS = (32, 33) + + +def _replace_released(source: Path, dest: Path, expected: dict[str, str]) -> None: + """Rename a verified copy whose lock was just released. + + Another run may open the copy in that moment, which refuses the rename + with a sharing violation until it lets go. When the copy is gone, that + run placed the same file and removed it: the destination is checked + against the manifest's hashes before it counts as installed. + """ + deadline = time.monotonic() + RELEASE_WAIT + while True: + try: + os.replace(source, dest) + return + except FileNotFoundError: + if expected and dest.is_file() and _digest_file(dest, tuple(expected)) == expected: + return + raise + except PermissionError as exc: + if (getattr(exc, "winerror", None) not in _SHARING_VIOLATIONS + or time.monotonic() >= deadline): + raise + time.sleep(0.2) + + +def _drop_full_partial(dest: Path, expected_size: int) -> None: + """Remove a complete partial copy this run left alone. + + A copy at full size that nothing holds is what a run stopped between its + check and its rename leaves behind. The destination now holds the + verified file, so the copy is only clutter. One another run holds is + left to it. + """ + path = dest.parent / f".{dest.name}.part" + if not path.is_file(): + return + try: + handle = _claim_partial(path) + except OSError: + return + if handle is None: + return + with handle: + complete = os.fstat(handle.fileno()).st_size >= expected_size + if complete: + try: + path.unlink() + except OSError: + pass # another run opened it meanwhile and finishes with it + else: + try: + if path.stat().st_size == 0: + path.unlink() + except OSError: + pass # created by the claim and taken since by another run + + def _partial_size(path: Path) -> int: """Bytes already held by a partial copy. Anything but a plain file is dropped.""" try: @@ -2823,6 +2903,10 @@ NOT_WRITABLE = "folder not writable" def _write_refused(exc: OSError) -> bool: + # A sharing violation is a file another process holds open, which a + # later attempt can find released: no refusal of the folder. + if getattr(exc, "winerror", None) in _SHARING_VIOLATIONS: + return False return isinstance(exc, PermissionError) or exc.errno == errno.EROFS diff --git a/install.sh b/install.sh index fa86e185..00463b67 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="973d64627af05b239555ab5a83a8902747b7a742bc1176f1b994cf7ac74774f9" +DEFAULT_INSTALL_SHA256="007fffece97716aa9e1ed0606a8453f2da8a1262711aa68ca910673956b2b341" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 3ae1f221..a70ef8da 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3586,5 +3586,99 @@ class DestAloneFollowsWhatIsFoundHere(unittest.TestCase): self.assertIn("Which platform's files go to /media/card?", out) +class _FakeResponse: + """What urlopen returns, serving one payload from its start.""" + + def __init__(self, payload: bytes) -> None: + self.status = 200 + self.headers = {"Content-Length": str(len(payload))} + self._body = io.BytesIO(payload) + + def __enter__(self) -> _FakeResponse: + return self + + def __exit__(self, *_exc: object) -> None: + return None + + def read(self, size: int = -1) -> bytes: + return self._body.read(size) + + +def _sharing_violation() -> PermissionError: + exc = PermissionError(13, "The process cannot access the file") + exc.winerror = 32 + return exc + + +class ACompletePartialIsHandedOffOnWindows(unittest.TestCase): + """Windows renames no open file, so the run that verified a large copy + released its lock before the rename. A second run claimed the copy in + that moment, found it complete and truncated it, and the first run's + rename failed with a sharing violation reported as "folder not + writable".""" + + PAYLOAD = bytes(range(256)) * 200 + ENTRY = { + "dest": "big.bin", "repo_path": "bios/big.bin", "size": len(PAYLOAD), + "sha256": hashlib.sha256(PAYLOAD).hexdigest(), + } + + def _run(self, tmp: Path, seen: list[int]): + part = tmp / ".big.bin.part" + + def urlopen(request, timeout=None): + seen.append(part.stat().st_size if part.exists() else -1) + return _FakeResponse(self.PAYLOAD) + + with unittest.mock.patch.object(install, "RESUME_FROM_SIZE", 1024), \ + unittest.mock.patch.object(install, "_NO_RENAME_WHILE_OPEN", True), \ + unittest.mock.patch.object(install.urllib.request, "urlopen", urlopen): + return install._download_one(dict(self.ENTRY), tmp) + + def test_the_other_runs_copy_is_left_whole(self): + tmp = Path(_tempdir(self)) + (tmp / ".big.bin.part").write_bytes(self.PAYLOAD) + seen: list[int] = [] + self.assertEqual(self._run(tmp, seen), ("big.bin", True, "")) + # The download ran while the complete copy was still whole. + self.assertEqual(seen, [len(self.PAYLOAD)]) + self.assertEqual((tmp / "big.bin").read_bytes(), self.PAYLOAD) + # Nothing held it afterwards: it was a leftover, now removed. + self.assertFalse((tmp / ".big.bin.part").exists()) + + def test_the_rename_waits_for_the_other_run_to_let_go(self): + tmp = Path(_tempdir(self)) + source = tmp / "copy" + source.write_bytes(b"x") + real_replace = os.replace + calls = [] + + def replace(src, dst): + calls.append(src) + if len(calls) < 3: + raise _sharing_violation() + return real_replace(src, dst) + + with unittest.mock.patch.object(install.os, "replace", replace), \ + unittest.mock.patch.object(install.time, "sleep"): + install._replace_released(source, tmp / "dest", {}) + self.assertEqual(len(calls), 3) + self.assertEqual((tmp / "dest").read_bytes(), b"x") + + def test_a_copy_the_other_run_placed_counts_once_verified(self): + tmp = Path(_tempdir(self)) + dest = tmp / "dest" + dest.write_bytes(self.PAYLOAD) + expected = {"sha256": self.ENTRY["sha256"]} + install._replace_released(tmp / "gone", dest, expected) + dest.write_bytes(b"other bytes") + with self.assertRaises(FileNotFoundError): + install._replace_released(tmp / "gone", dest, expected) + + def test_a_sharing_violation_is_no_refused_folder(self): + self.assertFalse(install._write_refused(_sharing_violation())) + self.assertTrue(install._write_refused(PermissionError(13, "denied"))) + + if __name__ == "__main__": unittest.main() From 1392f00c8dcc178525fe015bcdbacda2019f3eb7 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:13:04 +0200 Subject: [PATCH 03/11] fix: replace a link at a standalone copy's name A link already at the destination was skipped without a word and counted as "dir not found" while the folder existed, so the emulator kept the file the link pointed at. The link now gives way to the file, written beside it and renamed over the name, and a name leading out of the folder is a named failure. --- install.ps1 | 2 +- install.py | 41 ++++++++++++++++++++++++++++++++++------- install.sh | 2 +- tests/test_install.py | 18 +++++++++++++----- wiki/installer.md | 11 ++++++----- 5 files changed, 55 insertions(+), 19 deletions(-) diff --git a/install.ps1 b/install.ps1 index 2165680f..5ba1d7ae 100644 --- a/install.ps1 +++ b/install.ps1 @@ -12,7 +12,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "007fffece97716aa9e1ed0606a8453f2da8a1262711aa68ca910673956b2b341" + $defaultInstallSha256 = "df46e48289c89c5c542095b01c16208ef54696df472685c99327e5022fcd7ab8" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 406d7f8d..cba224d1 100644 --- a/install.py +++ b/install.py @@ -3538,7 +3538,9 @@ def do_standalone_copies( Returns (copied, skipped, failed): skipped is a target directory that does not exist, failed a copy the system refused (permission, disk - full, read-only mount), which used to be counted as "dir not found". + full, read-only mount) or a name leading out of the folder, which used + to be counted as "dir not found". A link already at a file's name is + replaced by the file, never written through. Each failure is also appended to failures, by absolute path, for the summary to name. """ @@ -3603,14 +3605,23 @@ def do_standalone_copies( resolved_dir = target_dir.resolve() for src in sources: dest = target_dir / src.name - # A symlink already sitting at the destination would redirect - # the write outside the directory the user opted into, and a - # crafted source name would climb out of it. - if dest.is_symlink() or not _within(dest, resolved_dir): - skipped += 1 + linked = dest.is_symlink() + # A crafted source name would climb out of the directory. + if not linked and not _within(dest, resolved_dir): + print(f" ERROR: {dest}: {OUTSIDE_TARGET}") + failed += 1 + if failures is not None: + failures.append((str(dest), OUTSIDE_TARGET)) continue try: - shutil.copy2(src, dest) + if linked: + # Writing through a link already at the name would + # leave the folder the user opted into. The link + # itself gives way to the file. + _replace_with_copy(src, dest) + print(f" {dest}: link replaced by the file") + else: + shutil.copy2(src, dest) copied += 1 except OSError as exc: print(f" ERROR: {dest}: {exc.strerror or exc}") @@ -3623,6 +3634,22 @@ def do_standalone_copies( return copied, skipped, failed +OUTSIDE_TARGET = "name leads outside the emulator folder" + + +def _replace_with_copy(src: Path, dest: Path) -> None: + """Put a copy of src at dest's name without following what is there.""" + fd, scratch = tempfile.mkstemp(dir=dest.parent, prefix=f".{dest.name}.", suffix=".part") + os.close(fd) + try: + shutil.copy2(src, scratch) + os.replace(scratch, dest) + except OSError: + if os.path.lexists(scratch): + os.unlink(scratch) + raise + + def format_size(n: int) -> str: """Human-readable file size.""" if n < 1024: diff --git a/install.sh b/install.sh index 00463b67..de28778f 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="007fffece97716aa9e1ed0606a8453f2da8a1262711aa68ca910673956b2b341" +DEFAULT_INSTALL_SHA256="df46e48289c89c5c542095b01c16208ef54696df472685c99327e5022fcd7ab8" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index a70ef8da..c1753935 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1608,6 +1608,9 @@ class TestStandaloneCopyTargetsAreUntrusted(unittest.TestCase): self.assertIsInstance(install._validate_manifest(manifest, "retroarch"), dict) def test_symlinked_destination_is_not_written_through(self): + """A link at the name was skipped in silence and counted as "dir + not found" while the folder existed: the emulator kept whatever the + link pointed at. The link now gives way to the file.""" bios = Path(_tempdir(self)) (bios / "boot.bin").write_text("rom") target = Path(_tempdir(self)) @@ -1615,12 +1618,17 @@ class TestStandaloneCopyTargetsAreUntrusted(unittest.TestCase): outside.write_text("untouched") (target / "boot.bin").symlink_to(outside) - copied, skipped, _ = install.do_standalone_copies( - self._manifest(str(target)), bios, "linux" - ) - self.assertEqual(copied, 0) - self.assertEqual(skipped, 1) + out = io.StringIO() + with contextlib.redirect_stdout(out): + copied, skipped, failed = install.do_standalone_copies( + self._manifest(str(target)), bios, "linux" + ) + self.assertEqual((copied, skipped, failed), (1, 0, 0)) self.assertEqual(outside.read_text(), "untouched") + self.assertFalse((target / "boot.bin").is_symlink()) + self.assertEqual((target / "boot.bin").read_text(), "rom") + self.assertIn("link replaced", out.getvalue()) + self.assertEqual(sorted(p.name for p in target.iterdir()), ["boot.bin"]) @unittest.skipUnless(shutil.which("pwsh"), "pwsh not available") diff --git a/wiki/installer.md b/wiki/installer.md index d736822d..862571ed 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -404,9 +404,10 @@ run rather than a file to copy. Targets are per OS, WSL falling back to the Linux ones. A directory that does not exist is skipped rather than created, so nothing is copied for an emulator -that is not installed. A destination that is already a symbolic link is skipped -too: the copy leaves the tree the user opted into, and a link there would -redirect the write somewhere else again. +that is not installed. A destination that is already a symbolic link is +replaced by the file rather than written through: a write through the link +would land outside the folder the user opted into. The summary counts each +copy as copied, skipped (folder absent) or failed, and names every failure. ## Android @@ -478,8 +479,8 @@ user's own layout, and EmuDeck links `bios/shadps4/sys_modules` into shadPS4's data directory where the emulator actually reads it. The boundary is lexical, not a resolved-path comparison, which is what keeps that case working. Standalone copies are stricter, since they write outside the tree the user -selected: a symlink already sitting at the destination is skipped rather than -followed. +selected: a symlink already sitting at the destination is replaced by the +file rather than followed. ## Changing the installer From 8a19d647a89504bddf1809840444f51021dc1e5e Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:15:24 +0200 Subject: [PATCH 04/11] fix: pass -v and every short flag to install.py A [Parameter()] attribute made install.ps1 an advanced script: its common parameters took -v (-Verbose), -vb, -d and -ea before install.py saw them, and the scriptblock form printed PowerShell's own VERBOSE lines. The wrapper now forwards $args as typed. --- install.ps1 | 9 ++++----- tests/test_install.py | 23 +++++++++++++++++++++++ wiki/installer.md | 4 +++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/install.ps1 b/install.ps1 index 5ba1d7ae..8bb01c51 100644 --- a/install.ps1 +++ b/install.ps1 @@ -3,12 +3,11 @@ # Everything runs in a script block of its own: piped into iex, the script # would otherwise leave its variables and $ErrorActionPreference = "Stop" # in the user's session, where every later error would stop their commands. +# Neither the script nor the block declares a parameter: one [Parameter()] +# attribute makes it an advanced one, whose common parameters took -v +# (-Verbose), -vb, -d (-Debug) and -ea before install.py could see them. & { - [CmdletBinding()] - param( - [Parameter(ValueFromRemainingArguments = $true)] - [string[]]$InstallerArguments - ) + $InstallerArguments = @($args) $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" diff --git a/tests/test_install.py b/tests/test_install.py index c1753935..49b949c7 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -2276,6 +2276,29 @@ class ThePowerShellWrapperPassesTheStatusOn(unittest.TestCase): self.assertNotIn("installer failed", proc.stdout + proc.stderr) +class ThePowerShellWrapperPassesEveryArgument(unittest.TestCase): + """A [Parameter()] attribute made install.ps1 an advanced script, whose + common parameters took -v (as -Verbose), -vb, -d and -ea: the documented + `-v` never reached install.py.""" + + @unittest.skipUnless(shutil.which("pwsh"), "needs PowerShell") + def test_short_flags_reach_the_installer(self): + folder = Path(_tempdir(self)) + shutil.copy(REPO_ROOT / "install.ps1", folder / "install.ps1") + (folder / "install.py").write_text( + "import json, sys\nprint(json.dumps(sys.argv[1:]))\n" + ) + for args in (["-v", "--check"], ["-vb"], ["-Verbose"], ["-d", "-ea", "x"], + ["--platform", "a b"], []): + with self.subTest(args=args): + proc = subprocess.run( + ["pwsh", "-NoProfile", "-File", str(folder / "install.ps1"), *args], + capture_output=True, text=True, timeout=120, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(json.loads(proc.stdout), args) + + @unittest.skipIf(os.name == "nt" or os.geteuid() == 0, "needs a folder this user cannot write") class AnUnwritableFolderIsCodeTwo(unittest.TestCase): """A BIOS folder that could not be created ended with 1, while the same diff --git a/wiki/installer.md b/wiki/installer.md index 862571ed..c8349519 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -510,7 +510,9 @@ fields, the installer refuses to narrow rather than install everything. `install.ps1` passes the installer's code on unchanged: run as a file it exits with it, run through `irm \| iex` it sets `$LASTEXITCODE` (an `exit` there -would close the PowerShell window). Its body runs in a script block of its own, +would close the PowerShell window). It declares no parameter of its own, so +every argument reaches `install.py` as typed, short forms such as `-v` +included. Its body runs in a script block of its own, so `irm \| iex` leaves neither its variables nor its `$ErrorActionPreference` in the session. From 253a41ebc8d3bf673f59d23f638a15f04274c2ae Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:22:08 +0200 Subject: [PATCH 05/11] feat: list the names --emulator accepts An unknown --emulator name pointed at --list-cores and counted 441 names while --list-cores prints 189: the buildbot names --emulator accepts (mednafen_psx for beetle_psx) were listed nowhere. --list-emulators prints each accepted name beside the emulator it stands for, and the hint names it. --- install.ps1 | 2 +- install.py | 52 +++++++++++++++++++++++++++++++++++-------- install.sh | 2 +- tests/test_install.py | 36 ++++++++++++++++++++++++++++++ wiki/installer.md | 3 ++- 5 files changed, 83 insertions(+), 12 deletions(-) diff --git a/install.ps1 b/install.ps1 index 8bb01c51..3f26cd07 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "df46e48289c89c5c542095b01c16208ef54696df472685c99327e5022fcd7ab8" + $defaultInstallSha256 = "2409cff28ad4eed1546161aa5abd6808fcd6a010bda7943eb1076f4ab0559c9f" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index cba224d1..e3dae02f 100644 --- a/install.py +++ b/install.py @@ -2408,7 +2408,8 @@ def resolve_choices(values: list[str], known: list[str], label: str) -> list[str if unknown: if not known: raise ValueError(f"unknown {label}: {', '.join(unknown)}. This list records none.") - listing = "--list-systems" if label == "system" else "--list-cores" + listing = {"system": "--list-systems", "emulator": "--list-emulators"}.get( + label, "--list-cores") raise ValueError( f"unknown {label}: {', '.join(unknown)}. {did_you_mean(unknown, known, listing)}" ) @@ -3923,6 +3924,33 @@ def _print_choices( print(f" {name:<{width}} {_sized(held)}") +def emulator_names(manifest: dict, files: list[dict], omitted: list[dict]) -> dict[str, str]: + """Every name --emulator accepts, with the profile each one stands for. + + A profile answers to its own name and to the names its core is built + under (core_names: beetle_psx and mednafen_psx). A core named by a file + and by no profile stands for itself. + """ + names = {core: core for core in available_choices(files, omitted)["cores"]} + for profile, aliases in (manifest.get("core_names") or {}).items(): + names[profile] = profile + for alias in aliases: + names.setdefault(alias, profile) + return names + + +def _print_emulators(plat: str, manifest: dict, files: list[dict], omitted: list[dict]) -> None: + """What --emulator accepts for a platform, one name per line.""" + names = emulator_names(manifest, files, omitted) + if not names: + print(f" {plat} records no emulators") + return + width = min(max(len(name) for name in names), 40) + for name in sorted(names): + profile = names[name] + print(f" {name:<{width}} {'' if profile == name else 'same as ' + profile}".rstrip()) + + def _older_list_advice() -> str: """How to reach a file list that records what an option needs. @@ -3952,11 +3980,7 @@ def _narrow_to_emulators( "which systems, which --emulator needs.", file=sys.stderr) print(_older_list_advice(), file=sys.stderr) sys.exit(1) - known = sorted({ - name - for profile, aliases in (manifest.get("core_names") or {}).items() - for name in [profile, *aliases] - } | set(available_choices(files, omitted)["cores"])) + known = sorted(emulator_names(manifest, files, omitted)) try: chosen = resolve_choices(names, known, "emulator") except ValueError as exc: @@ -4093,7 +4117,7 @@ def main() -> None: action="append", metavar="NAME", help="install what these emulators read on the platform, as a pack " - "built for them carries (names: --list-cores)", + "built for them carries (names: --list-emulators)", ) parser.add_argument( "--installed-cores", @@ -4138,6 +4162,11 @@ def main() -> None: action="store_true", help="list the cores whose extra files a platform installs and exit", ) + parser.add_argument( + "--list-emulators", + action="store_true", + help="list every name --emulator accepts for a platform and exit", + ) parser.add_argument( "--no-input", action="store_true", @@ -4183,7 +4212,7 @@ def main() -> None: parser.error(f"{option} needs at least one name") if args.quiet and ( args.check or args.dry_run or args.list_platforms or args.list_targets - or args.list_systems or args.list_cores + or args.list_systems or args.list_cores or args.list_emulators ): parser.error("--quiet applies to an install, not to --check, --dry-run or a listing") if args.every_board and args.target: @@ -4271,7 +4300,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: """One run of the installer, once the options are read.""" term = Terminal(real_stdout, quiet=args.quiet) interactive = sys.stdin.isatty() and not args.no_input and not args.quiet - listing = args.list_targets or args.list_systems or args.list_cores + listing = (args.list_targets or args.list_systems or args.list_cores + or args.list_emulators) print(f"{term.bold('RetroBIOS')} {term.dim(installer_id())}\n") os_type = detect_os() @@ -4500,6 +4530,10 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: ) print(f" Narrowed {before} -> {len(files)} files to the installed cores") + if args.list_emulators: + _print_emulators(plat_name, manifest, files, omitted_files) + continue + if args.emulator: files, omitted_files = _narrow_to_emulators( args.emulator, manifest, plat_name, files, omitted_files diff --git a/install.sh b/install.sh index de28778f..d7a4c4d5 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="df46e48289c89c5c542095b01c16208ef54696df472685c99327e5022fcd7ab8" +DEFAULT_INSTALL_SHA256="2409cff28ad4eed1546161aa5abd6808fcd6a010bda7943eb1076f4ab0559c9f" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 49b949c7..06b714f3 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3711,5 +3711,41 @@ class ACompletePartialIsHandedOffOnWindows(unittest.TestCase): self.assertTrue(install._write_refused(PermissionError(13, "denied"))) +class TheEmulatorNamesHaveTheirOwnListing(unittest.TestCase, _LoopbackRun): + """--emulator nope said "--list-cores lists all 441" and --list-cores + printed 189: the buildbot names --emulator accepts were listed nowhere.""" + + def setUp(self): + self.env = self.serve(self) + + def test_the_hint_counts_what_the_listing_prints(self): + proc = _install_run(self.env, "--platform", "retroarch", "--no-input", + "--emulator", "nope") + self.assertEqual(proc.returncode, 1, proc.stdout) + found = re.search(r"--list-emulators lists all (\d+)\.", proc.stderr) + self.assertIsNotNone(found, proc.stderr) + listing = _install_run(self.env, "--platform", "retroarch", "--no-input", + "--list-emulators") + self.assertEqual(listing.returncode, 0, listing.stderr) + names = [re.split(r"\s{2,}", line.strip())[0] + for line in listing.stdout.splitlines() if line.startswith(" ")] + manifest = json.loads((REPO_ROOT / "install" / "retroarch.json").read_text()) + accepted = install.emulator_names( + manifest, manifest["files"], manifest.get("omitted_files", [])) + self.assertEqual(int(found.group(1)), len(accepted)) + self.assertTrue(set(accepted) <= set(names), set(accepted) - set(names)) + self.assertIn("mednafen_psx", accepted) + self.assertRegex(listing.stdout, r"mednafen_psx\s+same as beetle_psx") + + def test_every_listed_name_is_accepted(self): + manifest = {"core_names": {"beetle_psx": ["mednafen_psx"]}} + files = [{"dest": "a", "cores": ["pcsx_rearmed"]}] + names = install.emulator_names(manifest, files, []) + self.assertEqual(names, {"beetle_psx": "beetle_psx", "mednafen_psx": "beetle_psx", + "pcsx_rearmed": "pcsx_rearmed"}) + self.assertEqual(install.resolve_choices(["MEDNAFEN_PSX"], sorted(names), "emulator"), + ["mednafen_psx"]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index c8349519..e8b6b3c8 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -230,10 +230,11 @@ what the options name, everything by default. | `--cores-dir PATH` | The folder RetroArch loads its cores from, for `--installed-cores` when it is not found. Implies `--installed-cores` | | `--system NAMES` | Install only these systems. Comma-separated or repeated. `--list-systems` prints the names, `other` included | | `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names | -| `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-cores` prints the names | +| `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included | | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | | `--list-systems` | Print a platform's systems with the files and size each holds | | `--list-cores` | Print the cores whose extra files a platform installs | +| `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | | `--no-input` | Never ask a question, even at a terminal | | `--check` | Report and exit without writing | | `--dry-run` | Show what would be downloaded, by system and with sizes, without writing anything | From 4f20b5daaf8946dcdbbef6293e04d4c7ea25e6aa Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:25:19 +0200 Subject: [PATCH 06/11] fix: name the network when every transfer fails A run whose downloads were all cut ended on "Location that was being written to" and "once that is resolved", pointing at the disk. A failure caused by the connection (lookup, reset, timeout, TLS, a cut transfer, HTTP 5xx or 429) is now marked as such, and the summary says to check the connection and HTTPS_PROXY. --- install.ps1 | 2 +- install.py | 42 +++++++++++++++++++++++++++++++++- install.sh | 2 +- tests/test_install.py | 53 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/install.ps1 b/install.ps1 index 3f26cd07..6bc44b5f 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "2409cff28ad4eed1546161aa5abd6808fcd6a010bda7943eb1076f4ab0559c9f" + $defaultInstallSha256 = "caec72e5ebd146509e3b1fabc7d8d4c5c921567592d6d1581df6e5cc50f3e96f" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index e3dae02f..2c87e2f0 100644 --- a/install.py +++ b/install.py @@ -26,6 +26,7 @@ import platform import re import shlex import shutil +import socket import stat import subprocess import sys @@ -2711,6 +2712,8 @@ def _download_one( reason = f"HTTP {exc.code}" if exc.code in (403, 429): reason += " (GitHub rate limit or access refused)" + if exc.code >= 500 or exc.code == 429: + reason = NetworkReason(reason) if verbose: print(f" Attempt {attempt} failed: {exc}", file=sys.stderr) if tmp_path is not None and partial is None and tmp_path.exists(): @@ -2731,6 +2734,8 @@ def _download_one( # place is not a folder this user cannot write. reason = READ_ONLY if _read_only_file(dest) else NOT_WRITABLE refused = True + elif _transfer_failure(exc): + reason = NetworkReason(getattr(exc, "reason", "") or exc) else: reason = str(getattr(exc, "reason", "") or exc) if verbose: @@ -3071,6 +3076,34 @@ def _plural(count: int, word: str) -> str: return f"{count} {word}" if count == 1 else f"{count} {word}s" +class NetworkReason(str): + """A failure the connection caused: cut, timed out, refused by a server + or a proxy. The summary names the network instead of the folder.""" + + +def _transfer_failure(exc: BaseException) -> bool: + """Whether a download error came from the connection, not the disk.""" + # urlopen raises URLError for the name lookup, the connection and TLS. + # A cut read surfaces as a timeout (socket.timeout is no TimeoutError + # before 3.10), a reset, an SSL error or a size that does not add up. + ssl_error = getattr(sys.modules.get("ssl"), "SSLError", ()) + if isinstance(exc, (urllib.error.URLError, http.client.HTTPException, ValueError, + ConnectionError, TimeoutError, socket.timeout)): + return True + if ssl_error and isinstance(exc, ssl_error): + return True + return isinstance(exc, OSError) and exc.errno in ( + errno.ECONNRESET, errno.ECONNREFUSED, errno.ECONNABORTED, errno.ETIMEDOUT, + errno.EHOSTUNREACH, errno.ENETUNREACH, errno.ENETDOWN, errno.EPIPE, + ) + + +NETWORK_FAILURE_ADVICE = ( + " The transfers were cut or refused on the way: check the connection, and " + "HTTPS_PROXY if this network needs a proxy.\n" + f" More help: {TROUBLESHOOTING_PAGE}") + + def _report_outcome( paths: list[Path], downloaded: int, @@ -3132,11 +3165,18 @@ def _report_outcome( if any(reason == READ_ONLY for _dest, reason in failures or []): advice.append(" Files marked read-only are in the way: clear that attribute " "(or delete them), then run again.") + if any(isinstance(reason, NetworkReason) for _dest, reason in failures or []): + advice.append(NETWORK_FAILURE_ADVICE) paths = paths + [folder for folder in elsewhere if folder not in paths] if errors and not downloaded: print(f"\nNothing was installed. {_plural(errors, 'file')} failed.") - print(f"Location that was being written to: {_locations(paths)}") + if failures and all(isinstance(r, NetworkReason) for _d, r in failures): + # The folder was never the problem: naming it as the place being + # written to sent people looking at their disk. + print(f"Destination: {_locations(paths)}") + else: + print(f"Location that was being written to: {_locations(paths)}") for line in advice: print(line) print("Run the same command again once that is resolved. " diff --git a/install.sh b/install.sh index d7a4c4d5..72d44f87 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="2409cff28ad4eed1546161aa5abd6808fcd6a010bda7943eb1076f4ab0559c9f" +DEFAULT_INSTALL_SHA256="caec72e5ebd146509e3b1fabc7d8d4c5c921567592d6d1581df6e5cc50f3e96f" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 06b714f3..b6282f3a 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -19,6 +19,7 @@ import threading import unittest from types import SimpleNamespace import unittest.mock +import urllib.error from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent @@ -3747,5 +3748,57 @@ class TheEmulatorNamesHaveTheirOwnListing(unittest.TestCase, _LoopbackRun): ["mednafen_psx"]) +class ANetworkFailureNamesTheNetwork(unittest.TestCase): + """Every transfer cut by the connection ended on "Location that was being + written to" and "once that is resolved", naming the folder and nothing + about the connection or the proxy.""" + + def _fail(self, error: BaseException) -> str: + def urlopen(*_args, **_kwargs): + raise error + + entry = {"dest": "a.bin", "repo_path": "bios/a.bin", "size": 10, + "sha256": "0" * 64} + with unittest.mock.patch.object(install.urllib.request, "urlopen", urlopen), \ + unittest.mock.patch.object(install, "_retry_delay", return_value=0): + _dest, ok, reason = install._download_one(entry, Path(_tempdir(self))) + self.assertFalse(ok) + return reason + + def test_cuts_and_refusals_are_network_failures(self): + import http.client + import socket + + for error in ( + urllib.error.URLError(socket.gaierror(-2, "Name or service not known")), + http.client.IncompleteRead(b"abc", 10), + ConnectionResetError(104, "Connection reset by peer"), + socket.timeout("timed out"), + urllib.error.HTTPError("u", 503, "Service Unavailable", None, None), + ): + with self.subTest(error=error): + self.assertIsInstance(self._fail(error), install.NetworkReason) + + def test_a_full_disk_and_a_missing_file_are_not(self): + import errno + + for error in (OSError(errno.ENOSPC, "No space left on device"), + urllib.error.HTTPError("u", 404, "Not Found", None, None)): + with self.subTest(error=error): + self.assertNotIsInstance(self._fail(error), install.NetworkReason) + + def test_the_summary_points_at_the_connection(self): + failures = [("a.bin", install.NetworkReason("timed out")), + ("b.bin", install.NetworkReason("Connection reset by peer"))] + out = io.StringIO() + with contextlib.redirect_stdout(out), self.assertRaises(SystemExit) as raised: + install._report_outcome([Path("/bios")], 0, 0, 2, 0, False, failures=failures) + self.assertEqual(raised.exception.code, 2) + text = out.getvalue() + self.assertIn("HTTPS_PROXY", text) + self.assertIn("Destination: /bios", text) + self.assertNotIn("being written to", text) + + if __name__ == "__main__": unittest.main() From cdb2cfa6f0c6468b3fb0c9aa8971e081afbb5c1a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:28:35 +0200 Subject: [PATCH 07/11] feat: find RetroBat through the folder it records RetroBat.exe writes its folder to HKCU\SOFTWARE\RetroBat LatestKnownInstallPath on every start (retrobat-executable 702da66a, Program.cs:57 and 99, RegistryTools.cs:15-50). Only C:\RetroBat and the profile folder were looked at, so an install on D: was missed and a forced --platform retrobat wrote into C:\RetroBat\bios. The value is read through winreg, or reg.exe under WSL, before the fixed folders. --- install.ps1 | 2 +- install.py | 59 +++++++++++++++++++++++++++++++++++++++++ install.sh | 2 +- platforms/_registry.yml | 8 ++++++ tests/test_install.py | 47 ++++++++++++++++++++++++++++++++ wiki/installer.md | 1 + 6 files changed, 117 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index 6bc44b5f..b5d6fbf8 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "caec72e5ebd146509e3b1fabc7d8d4c5c921567592d6d1581df6e5cc50f3e96f" + $defaultInstallSha256 = "f05f862fcebc16e99d692a20a09163665828ea0383643fcc96a41b11d77fcff7" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 2c87e2f0..17a06481 100644 --- a/install.py +++ b/install.py @@ -1092,6 +1092,64 @@ def _detect_by_executable(os_type: str, found: list[tuple[str, Path]]) -> None: found.append((platform_name, newest.parent / subdir)) +# The registry's registry rules: (os, platform, key, value, BIOS folder under +# the folder the value names). RetroBat.exe writes its own folder there on +# every start (retrobat-executable 702da66a, RetroBat/RetroBat/Program.cs:57 +# and 99, RegistryTools.cs:15-50), wherever it was installed. The path rules +# only knew C:\RetroBat and the profile folder. +_REGISTRY_RULES: tuple[tuple[str, str, str, str, str], ...] = ( + ("windows", "retrobat", "HKCU\\SOFTWARE\\RetroBat", "LatestKnownInstallPath", "bios"), +) + + +def _read_registry(key: str, value: str) -> str | None: + """A string value of the Windows registry, read natively or, under WSL, + through reg.exe. None when the key, the value or the means is missing.""" + if os.name == "nt": + import winreg # noqa: PLC0415 + + hive, _, subkey = key.partition("\\") + roots = {"HKCU": winreg.HKEY_CURRENT_USER, "HKLM": winreg.HKEY_LOCAL_MACHINE} + try: + with winreg.OpenKey(roots[hive], subkey) as handle: + data, _kind = winreg.QueryValueEx(handle, value) + except (OSError, KeyError): + return None + return data.strip() if isinstance(data, str) and data.strip() else None + if not _WSL["active"]: + return None + try: + result = subprocess.run( + ["reg.exe", "query", key, "/v", value], + capture_output=True, text=True, errors="replace", timeout=10, + stdin=subprocess.DEVNULL, cwd="/mnt/c" if Path("/mnt/c").is_dir() else None, + ) + except (OSError, subprocess.SubprocessError): + return None + match = re.search( + rf"^\s*{re.escape(value)}\s+REG_(?:EXPAND_)?SZ\s+(.+?)\s*$", result.stdout, re.M + ) + return match.group(1) if match else None + + +def _detect_by_registry(os_type: str, found: list[tuple[str, Path]]) -> None: + """Add a platform whose own program names its folder in the registry. + + A folder since moved or removed is passed over, so the path rules still + get their turn. + """ + systems = ("windows",) if os_type in ("windows", "wsl") else () + for rule_os, platform_name, key, value, subdir in _REGISTRY_RULES: + if rule_os not in systems or any(name == platform_name for name, _ in found): + continue + folder = _read_registry(key, value) + if not folder: + continue + base = _host_path(folder.rstrip("\\/") or folder) + if base.is_dir(): + found.append((platform_name, base / subdir)) + + def _detect_by_path(os_type: str, found: list[tuple[str, Path]]) -> None: """Add what a registry path rule finds, for a platform not found yet.""" systems = ("linux", "windows") if os_type == "wsl" else (os_type,) @@ -1239,6 +1297,7 @@ def detect_platforms(os_type: str) -> list[tuple[str, Path]]: found.append(("bizhawk", bizhawk)) _detect_by_executable(os_type, found) + _detect_by_registry(os_type, found) _detect_by_path(os_type, found) return found diff --git a/install.sh b/install.sh index 72d44f87..54028c51 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="caec72e5ebd146509e3b1fabc7d8d4c5c921567592d6d1581df6e5cc50f3e96f" +DEFAULT_INSTALL_SHA256="f05f862fcebc16e99d692a20a09163665828ea0383643fcc96a41b11d77fcff7" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/platforms/_registry.yml b/platforms/_registry.yml index d8ef431d..495ccdc5 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -351,6 +351,14 @@ platforms: target_source: null install: detect: + # RetroBat.exe writes its own folder there on every start, wherever it + # was installed (retrobat-executable 702da66a, + # RetroBat/RetroBat/Program.cs:57 and 99, RegistryTools.cs:15-50). + - os: windows + method: registry + key: 'HKCU\SOFTWARE\RetroBat' + value: LatestKnownInstallPath + bios_subdir: bios # installer.iss: DefaultDirName=C:\RetroBat, the first place to look. - os: windows method: path_exists diff --git a/tests/test_install.py b/tests/test_install.py index b6282f3a..46daaf4b 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3800,5 +3800,52 @@ class ANetworkFailureNamesTheNetwork(unittest.TestCase): self.assertNotIn("being written to", text) +class RetroBatIsFoundWhereItSaysItIs(unittest.TestCase): + """RetroBat on D:\\RetroBat was not detected, and --platform retrobat + wrote 7.6 GB into C:\\RetroBat\\bios, a folder it does not read. + RetroBat.exe writes its folder to the registry on every start.""" + + def test_the_table_is_the_registry(self): + import yaml # noqa: PLC0415 + + registry = yaml.safe_load((REPO_ROOT / "platforms" / "_registry.yml").read_text()) + rules = { + (rule["os"], name, rule["key"], rule["value"], rule["bios_subdir"]) + for name, entry in registry["platforms"].items() + for rule in (entry.get("install") or {}).get("detect", []) + if rule["method"] == "registry" + } + self.assertEqual(set(install._REGISTRY_RULES), rules) + + def test_the_registry_folder_wins_over_the_default(self): + folder = Path(_tempdir(self)) + with unittest.mock.patch.object(install, "_read_registry", + return_value=str(folder) + "\\"): + found: list = [] + install._detect_by_registry("windows", found) + self.assertEqual(found, [("retrobat", folder / "bios")]) + + def test_a_folder_since_removed_is_passed_over(self): + gone = Path(_tempdir(self)) / "moved" + with unittest.mock.patch.object(install, "_read_registry", return_value=str(gone)): + found: list = [] + install._detect_by_registry("windows", found) + self.assertEqual(found, []) + + def test_wsl_reads_it_through_reg_exe(self): + output = ("\nHKEY_CURRENT_USER\\SOFTWARE\\RetroBat\n" + " LatestKnownInstallPath REG_SZ D:\\Games\\RetroBat\\\n\n") + with unittest.mock.patch.dict(install._WSL, {"active": True, "root": "/mnt"}), \ + unittest.mock.patch.object(install.os, "name", "posix"), \ + unittest.mock.patch.object( + install.subprocess, "run", + return_value=SimpleNamespace(stdout=output, returncode=0)) as run: + value = install._read_registry("HKCU\\SOFTWARE\\RetroBat", "LatestKnownInstallPath") + self.assertEqual(value, "D:\\Games\\RetroBat\\") + self.assertEqual(install._host_path(value.rstrip("\\")), + Path("/mnt/d/Games/RetroBat")) + self.assertEqual(run.call_args[0][0][:2], ["reg.exe", "query"]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index e8b6b3c8..63ffad36 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -318,6 +318,7 @@ WSL does not pass them. A forced `--platform retrobat` falls back to | Evidence | Platform | BIOS directory | |----------|----------|----------------| +| `HKCU\SOFTWARE\RetroBat` `LatestKnownInstallPath`, which `RetroBat.exe` writes on every start (read through `reg.exe` under WSL) | RetroBat | that folder + `\bios`, before the fixed folders above | | `~/.config/EmuDeck/settings.sh` | EmuDeck | `emulationPath` + `/bios` | | `%APPDATA%\EmuDeck\settings.ps1` | EmuDeck | `$emulationPath` + `\bios` | | `~/.var/app/net.retrodeck.retrodeck/config/retrodeck/retrodeck.json` | RetroDECK | `paths.rd_home_path`, falling back to the pre-migration `retrodeck.cfg` and its `rdhome` | From 15cb52d5513e3fa81c2e01a11ea2a70231241177 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:31:35 +0200 Subject: [PATCH 08/11] fix: find EmuDeck Windows where its scripts look functions/all.ps1:4-14 (emudeck-we 037460ef) reads the APPDATA settings.ps1, else the one under the profile, and takes C:\Emulation when neither names a folder. Only the first was read and a forced platform fell back to %USERPROFILE%\Emulation\bios. A network install names its share through $VolName (JSONtoPS1.ps1:166-172), and a UTF-16 settings file is decoded. --- install.ps1 | 2 +- install.py | 43 +++++++++++++++++++++++-------- install.sh | 2 +- platforms/_registry.yml | 7 +++++ tests/test_install.py | 57 ++++++++++++++++++++++++++++++++++++++++- wiki/installer.md | 4 +-- 6 files changed, 100 insertions(+), 15 deletions(-) diff --git a/install.ps1 b/install.ps1 index b5d6fbf8..d0de691a 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "f05f862fcebc16e99d692a20a09163665828ea0383643fcc96a41b11d77fcff7" + $defaultInstallSha256 = "67bba818ec05142e431155c27919e087d0d5aa1d5e148019ebae630f0dd26bdf" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 17a06481..6c8fbe6e 100644 --- a/install.py +++ b/install.py @@ -206,6 +206,9 @@ def _os_default_dests(os_type: str) -> dict[str, Path]: "retroarch": _host_path("C:/RetroArch-Win64/system"), # RetroBat's installer defaults to C:\RetroBat (installer.iss). "retrobat": _host_path("C:/RetroBat/bios"), + # EmuDeck's Windows edition: C:\Emulation when its settings name + # no folder (emudeck-we functions/all.ps1:12-14). + "emudeck": _host_path("C:/Emulation/bios"), } if os_type == "wsl": # RetroArch may be the Linux one inside the distribution, while RetroBat @@ -749,7 +752,11 @@ def _parse_ps1_var(path: Path, key: str) -> str | None: return None normalized = key.lstrip("$") try: - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + raw = path.read_bytes() + # Windows PowerShell's Out-File writes UTF-16, Set-Content -Encoding + # UTF8 a byte order mark: EmuDeck's settings.ps1 passes through both. + encoding = "utf-16" if raw[:2] in (b"\xff\xfe", b"\xfe\xff") else "utf-8-sig" + for line in raw.decode(encoding, errors="replace").splitlines(): line = line.strip() check = line.lstrip("$") if check.startswith(f"{normalized}="): @@ -1231,13 +1238,9 @@ def detect_platforms(os_type: str) -> list[tuple[str, Path]]: break if os_type in ("windows", "wsl"): - # EmuDeck Windows - home = Path.home() - emudeck_ps1 = Path(os.environ.get("APPDATA", "")) / "EmuDeck" / "settings.ps1" - if emudeck_ps1.exists(): - emu_path = _parse_ps1_var(emudeck_ps1, "$emulationPath") - if emu_path: - found.append(("emudeck", Path(emu_path) / "bios")) + emudeck = _emudeck_windows_bios() + if emudeck is not None: + found.append(("emudeck", emudeck)) # RetroArch Windows: open_default_config_file reads retroarch.cfg # beside the exe, then under %APPDATA%\RetroArch, and a value left at @@ -1302,6 +1305,27 @@ def detect_platforms(os_type: str) -> list[tuple[str, Path]]: return found +def _emudeck_windows_bios() -> Path | None: + """The BIOS folder of EmuDeck's Windows edition, as its scripts find it. + + functions/all.ps1:4-14 (emudeck-we 037460ef) loads + %APPDATA%\\EmuDeck\\settings.ps1, else %USERPROFILE%\\EmuDeck\\settings.ps1, + and takes C:\\Emulation when neither sets $emulationPath. A network + install sets it through a drive mapped to $VolName in that session + (JSONtoPS1.ps1:166-172), which is the share itself here. + """ + candidates = [Path(os.environ[var]) / "EmuDeck" / "settings.ps1" + for var in ("APPDATA", "USERPROFILE") if os.environ.get(var)] + settings = next((path for path in candidates if path.is_file()), None) + if settings is None: + return None + emu_path = _parse_ps1_var(settings, "$emulationPath") + if emu_path and ("$" in emu_path or "+" in emu_path): + share = _parse_ps1_var(settings, "$VolName") + emu_path = f"{share.rstrip(chr(92))}\\Emulation" if share else None + return _host_path(emu_path or "C:\\Emulation") / "bios" + + # Identity Name in pkg/msvc-uwp/RetroArch-msvcUWP/Package.appxmanifest. RETROARCH_UWP_ID = "1e4cf179-f3c2-404f-b9f3-cb2070a5aad8" @@ -3828,8 +3852,7 @@ def _default_dest(os_type: str, platform_name: str) -> Path | None: if own is not None: return own if os_type == "windows": - # EmuDeck's Windows edition keeps the same folder under the profile. - return DEFAULT_DESTS["emudeck"] if platform_name == "emudeck" else None + return None if platform_name == "retrobat": return None return DEFAULT_DESTS.get(platform_name) diff --git a/install.sh b/install.sh index 54028c51..41a56140 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="f05f862fcebc16e99d692a20a09163665828ea0383643fcc96a41b11d77fcff7" +DEFAULT_INSTALL_SHA256="67bba818ec05142e431155c27919e087d0d5aa1d5e148019ebae630f0dd26bdf" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 495ccdc5..433339a8 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -392,11 +392,18 @@ platforms: - os: linux method: path_exists path: $HOME/Emulation/bios + # functions/all.ps1:4-14 (emudeck-we 037460ef): the APPDATA settings, + # else the profile ones, and C:\Emulation when neither names a folder. - os: windows method: config_file config: '%APPDATA%\EmuDeck\settings.ps1' parse_key: $emulationPath bios_subdir: bios + - os: windows + method: config_file + config: '%USERPROFILE%\EmuDeck\settings.ps1' + parse_key: $emulationPath + bios_subdir: bios standalone_copies: *standalone_copies lakka: config: lakka.yml diff --git a/tests/test_install.py b/tests/test_install.py index 46daaf4b..31dd80d5 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3180,8 +3180,9 @@ class NoFallbackOutsideWhatAPlatformReads(unittest.TestCase): self.assertIsNone(install._default_dest("windows", "batocera")) self.assertIsNone(install._default_dest("windows", "retropie")) self.assertEqual(install._default_dest("linux", "batocera"), Path("/userdata/bios")) + # emudeck-we functions/all.ps1:12-14, not the profile folder. self.assertEqual( - install._default_dest("windows", "emudeck"), install.DEFAULT_DESTS["emudeck"] + install._default_dest("windows", "emudeck"), install._host_path("C:/Emulation/bios") ) def test_a_forced_platform_without_a_folder_is_refused(self): @@ -3847,5 +3848,59 @@ class RetroBatIsFoundWhereItSaysItIs(unittest.TestCase): self.assertEqual(run.call_args[0][0][:2], ["reg.exe", "query"]) +class EmuDeckWindowsIsFoundAsItsScriptsFindIt(unittest.TestCase): + """EmuDeck's Windows scripts read %APPDATA%\\EmuDeck\\settings.ps1, else + %USERPROFILE%\\EmuDeck\\settings.ps1, and take C:\\Emulation when neither + names a folder (functions/all.ps1:4-14). Only the first was read, and a + forced --platform emudeck wrote into %USERPROFILE%\\Emulation\\bios.""" + + def _env(self) -> tuple[Path, Path]: + root = Path(_tempdir(self)) + appdata, profile = root / "appdata", root / "profile" + patcher = unittest.mock.patch.dict( + os.environ, {"APPDATA": str(appdata), "USERPROFILE": str(profile)}) + patcher.start() + self.addCleanup(patcher.stop) + return appdata, profile + + @staticmethod + def _settings(folder: Path, text: str, encoding: str = "utf-8-sig") -> None: + (folder / "EmuDeck").mkdir(parents=True, exist_ok=True) + (folder / "EmuDeck" / "settings.ps1").write_bytes(text.encode(encoding)) + + def test_the_profile_settings_are_read_when_appdata_has_none(self): + _appdata, profile = self._env() + self._settings(profile, '\n$emulationPath="/games/Emulation"\n') + self.assertEqual(install._emudeck_windows_bios(), Path("/games/Emulation/bios")) + + def test_appdata_comes_first(self): + appdata, profile = self._env() + self._settings(profile, '$emulationPath="/b/Emulation"\n') + self._settings(appdata, '$emulationPath="/a/Emulation"\n', "utf-16") + self.assertEqual(install._emudeck_windows_bios(), Path("/a/Emulation/bios")) + + def test_settings_naming_no_folder_mean_c_emulation(self): + appdata, _profile = self._env() + self._settings(appdata, '$device="Windows"\n') + self.assertEqual(install._emudeck_windows_bios(), + install._host_path("C:\\Emulation") / "bios") + + def test_a_network_install_is_its_share(self): + appdata, _profile = self._env() + self._settings(appdata, '$networkInstallation="true"\n' + '$VolName="/nas/games"\n' + '$emulationPath=$testdrive.ToString() + ":\\Emulation"\n') + self.assertEqual(install._emudeck_windows_bios(), + Path("/nas/games\\Emulation") / "bios") + + def test_no_settings_detects_nothing(self): + self._env() + self.assertIsNone(install._emudeck_windows_bios()) + + def test_a_forced_platform_falls_back_to_c_emulation(self): + self.assertEqual(install._default_dest("windows", "emudeck"), + install._host_path("C:/Emulation/bios")) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index 63ffad36..a649467a 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -320,7 +320,7 @@ WSL does not pass them. A forced `--platform retrobat` falls back to |----------|----------|----------------| | `HKCU\SOFTWARE\RetroBat` `LatestKnownInstallPath`, which `RetroBat.exe` writes on every start (read through `reg.exe` under WSL) | RetroBat | that folder + `\bios`, before the fixed folders above | | `~/.config/EmuDeck/settings.sh` | EmuDeck | `emulationPath` + `/bios` | -| `%APPDATA%\EmuDeck\settings.ps1` | EmuDeck | `$emulationPath` + `\bios` | +| `%APPDATA%\EmuDeck\settings.ps1`, else `%USERPROFILE%\EmuDeck\settings.ps1` | EmuDeck | `$emulationPath` + `\bios`, `C:\Emulation\bios` when it names none (a network install: the `$VolName` share + `\Emulation\bios`) | | `~/.var/app/net.retrodeck.retrodeck/config/retrodeck/retrodeck.json` | RetroDECK | `paths.rd_home_path`, falling back to the pre-migration `retrodeck.cfg` and its `rdhome` | | `~/.var/app/org.libretro.RetroArch/config/retroarch/retroarch.cfg` | RetroArch (Flatpak) | `system_directory` | | `~/snap/retroarch/current/.config/retroarch/retroarch.cfg` | RetroArch (Snap) | `system_directory` | @@ -363,7 +363,7 @@ When a platform named with `--platform` is not found on the machine, the destination falls back to the directory its detection looks for: `/userdata/bios` for Batocera, `/recalbox/share/bios` for Recalbox, `/storage/system` for Lakka, `~/retrodeck` for RetroDECK, `~/Emulation/bios` -for EmuDeck, `/storage/roms/bios` for ROCKNIX, `/media/fat/games` for MiSTer, +for EmuDeck (`C:\Emulation\bios` on Windows), `/storage/roms/bios` for ROCKNIX, `/media/fat/games` for MiSTer, `~/RetroPie/BIOS` for RetroPie, `/romm/library/bios` for RomM, `C:\RetroBat\bios` for RetroBat, and RetroArch's own default system directory for RetroArch. BizHawk has no such folder: it reads `Firmware` From edda080c59f362bb260fdff852302ffaf538caf4 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:35:45 +0200 Subject: [PATCH 09/11] fix: read muOS RetroArch from its own config muOS runs RetroArch on a config copied from its default, which reads /run/muos/storage/bios, bound to MUOS/bios on the second card when that card holds one, and loads cores from /opt/muos/share/core (MustardOS/internal f41f4a89, script/control/retroarch.sh:8-20, script/device/bind.sh:116-150, retroarch.default.cfg:2806,3151). The installer wrote to /mnt/mmc/MUOS/bios and --installed-cores found no core. The config now decides both. --- install.ps1 | 2 +- install.py | 30 +++++++++++++++++++++++-- install.sh | 2 +- platforms/_registry.yml | 15 ++++++++++++- tests/test_install.py | 50 +++++++++++++++++++++++++++++++++++++++++ wiki/installer.md | 3 ++- 6 files changed, 96 insertions(+), 6 deletions(-) diff --git a/install.ps1 b/install.ps1 index d0de691a..19f51c43 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "67bba818ec05142e431155c27919e087d0d5aa1d5e148019ebae630f0dd26bdf" + $defaultInstallSha256 = "e85a34efb84620b8c65858d209c9741a715f8ff6a61397f9ee161c0931a9a049" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 6c8fbe6e..fdd3f467 100644 --- a/install.py +++ b/install.py @@ -779,7 +779,7 @@ _EMBEDDED_RULES: tuple[tuple[str, str, tuple[str, ...], str], ...] = ( ("batocera", "file_exists", ("/etc/batocera-version",), "/userdata/bios"), ("recalbox", "file_exists", ("/recalbox/recalbox.version",), "/recalbox/share/bios"), ("recalbox", "file_exists", ("/usr/bin/recalbox-settings",), "/recalbox/share/bios"), - ("retroarch", "file_exists", ("/opt/muos",), "/mnt/mmc/MUOS/bios"), + ("retroarch", "file_exists", ("/opt/muos",), "/run/muos/storage/bios"), ("retroarch", "file_exists", ("/mnt/mmc/MUOS",), "/mnt/mmc/MUOS/bios"), ("retroarch", "file_exists", ("/home/ark", "/opt/system"), "/roms/bios"), ("retroarch", "file_exists", ("/mnt/vendor/bin/dmenu.bin",), "/mnt/mmc/bios"), @@ -803,6 +803,30 @@ def _detect_embedded() -> list[tuple[str, Path]]: return [] +# muOS runs RetroArch on the config it copies from its default on first launch +# (internal f41f4a89, script/control/retroarch.sh:8-20). The default names +# /run/muos/storage/bios, a bind mount of MUOS/bios on the second card when +# that card holds one, else on the first (script/device/bind.sh:4,116-150), +# and loads cores from /opt/muos/share/core (retroarch.default.cfg:2806,3151). +MUOS_ROOT = Path("/opt/muos") +MUOS_RETROARCH_CONFIGS = ( + MUOS_ROOT / "share" / "info" / "config" / "retroarch.cfg", + MUOS_ROOT / "share" / "emulator" / "retroarch" / "retroarch.default.cfg", +) + + +def _muos_system_dir() -> Path | None: + """The system directory of muOS's RetroArch, read from its config, which + also records the config for finding the installed cores.""" + if not MUOS_ROOT.is_dir(): + return None + for cfg in MUOS_RETROARCH_CONFIGS: + system_dir = _parse_retroarch_system_dir(cfg) + if system_dir is not None: + return system_dir + return None + + def _android_storage_root() -> Path: """Shared storage root, the parent RetroArch builds its defaults from.""" value = os.environ.get("EXTERNAL_STORAGE", "").strip() @@ -1175,7 +1199,9 @@ def detect_platforms(os_type: str) -> list[tuple[str, Path]]: found.extend(_detect_android()) if os_type in ("linux", "wsl"): - found.extend(_detect_embedded()) + embedded = _detect_embedded() + muos = _muos_system_dir() if embedded and embedded[0][0] == "retroarch" else None + found.extend([("retroarch", muos)] if muos else embedded) # EmuDeck (Linux/SteamOS) home = Path.home() diff --git a/install.sh b/install.sh index 41a56140..a9fb019f 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="67bba818ec05142e431155c27919e087d0d5aa1d5e148019ebae630f0dd26bdf" +DEFAULT_INSTALL_SHA256="e85a34efb84620b8c65858d209c9741a715f8ff6a61397f9ee161c0931a9a049" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 433339a8..819438d7 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -269,10 +269,23 @@ platforms: parse_key: system_directory # Handheld firmwares that run RetroArch as their frontend: muOS, # ArkOS, and the stock Linux of Anbernic devices (dmenu). + # muOS copies its default config on first launch and RetroArch reads + # /run/muos/storage/bios, bound to MUOS/bios on the second card when + # it holds one, else the first (MustardOS/internal f41f4a89, + # script/control/retroarch.sh:8-20, script/device/bind.sh:4,116-150, + # share/emulator/retroarch/retroarch.default.cfg:2806,3151). + - os: linux + method: config_file + config: /opt/muos/share/info/config/retroarch.cfg + parse_key: system_directory + - os: linux + method: config_file + config: /opt/muos/share/emulator/retroarch/retroarch.default.cfg + parse_key: system_directory - os: linux method: file_exists file: /opt/muos - bios_path: /mnt/mmc/MUOS/bios + bios_path: /run/muos/storage/bios - os: linux method: file_exists file: /mnt/mmc/MUOS diff --git a/tests/test_install.py b/tests/test_install.py index 31dd80d5..44db4d6f 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3902,5 +3902,55 @@ class EmuDeckWindowsIsFoundAsItsScriptsFindIt(unittest.TestCase): install._host_path("C:/Emulation/bios")) +class MuOsIsReadFromItsRetroArchConfig(unittest.TestCase): + """muOS wrote to /mnt/mmc/MUOS/bios while RetroArch reads + /run/muos/storage/bios, bound to the second card when it holds MUOS/bios, + and --installed-cores found no core in /opt/muos/share/core.""" + + def _muos(self, *, user_config: bool) -> tuple[Path, Path]: + root = Path(_tempdir(self)) + muos = root / "opt" / "muos" + system_dir, cores = root / "run" / "bios", root / "core" + cores.mkdir(parents=True) + configs = (muos / "share/info/config/retroarch.cfg", + muos / "share/emulator/retroarch/retroarch.default.cfg") + default = configs[1] + default.parent.mkdir(parents=True) + default.write_text(f'libretro_directory = "{cores}"\n' + f'system_directory = "{root / "default-bios"}"\n') + if user_config: + configs[0].parent.mkdir(parents=True) + configs[0].write_text(f'libretro_directory = "{cores}"\n' + f'system_directory = "{system_dir}"\n') + for patcher in ( + unittest.mock.patch.object(install, "MUOS_ROOT", muos), + unittest.mock.patch.object(install, "MUOS_RETROARCH_CONFIGS", configs), + unittest.mock.patch.object( + install, "_detect_embedded", + lambda: [("retroarch", Path("/run/muos/storage/bios"))]), + unittest.mock.patch.dict(os.environ, {"HOME": str(root / "home")}), + unittest.mock.patch.dict(install._RETROARCH_CONFIGS, clear=True), + ): + patcher.start() + self.addCleanup(patcher.stop) + return (system_dir if user_config else root / "default-bios"), cores + + def test_the_users_config_decides(self): + system_dir, cores = self._muos(user_config=True) + found = install.detect_platforms("linux") + self.assertIn(("retroarch", system_dir), found) + self.assertNotIn(("retroarch", Path("/run/muos/storage/bios")), found) + self.assertIn(cores, install.core_dirs("retroarch", system_dir)) + + def test_before_the_first_launch_the_default_does(self): + system_dir, cores = self._muos(user_config=False) + self.assertIn(("retroarch", system_dir), install.detect_platforms("linux")) + self.assertIn(cores, install.core_dirs("retroarch", system_dir)) + + def test_without_a_config_the_bound_folder_is_used(self): + rule = next(r for r in install._EMBEDDED_RULES if r[2] == ("/opt/muos",)) + self.assertEqual(rule[3], "/run/muos/storage/bios") + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index a649467a..1937c0cb 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -283,7 +283,8 @@ wins, since a machine is only one of them. | `/etc/os-release` `ID=lakka` | Lakka | `/storage/system` | | `/etc/batocera-version` | Batocera | `/userdata/bios` | | `/recalbox/recalbox.version` or `/usr/bin/recalbox-settings` | Recalbox | `/recalbox/share/bios` | -| `/opt/muos` or `/mnt/mmc/MUOS/` | RetroArch | `/mnt/mmc/MUOS/bios` | +| `/opt/muos` | RetroArch (muOS) | `system_directory` of `/opt/muos/share/info/config/retroarch.cfg`, else of the default it is copied from: `/run/muos/storage/bios`, which muOS binds to `MUOS/bios` on the second card when that card holds one, else the first. Cores are read from that config's `libretro_directory` (`/opt/muos/share/core`) | +| `/mnt/mmc/MUOS/` | RetroArch | `/mnt/mmc/MUOS/bios` | | `/home/ark` and `/opt/system` | RetroArch | `/roms/bios` | | `/mnt/vendor/bin/dmenu.bin` | RetroArch | `/mnt/mmc/bios` | From 89c533e9bad96614c07512e8babc593bb81fe23d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:44:27 +0200 Subject: [PATCH 10/11] fix: detect Batocera and Knulli by build files No Batocera build writes /etc/batocera-version: batocera-system.mk:21-22 (batocera.linux 38845725, unchanged since batocera-30) writes /usr/share/batocera/batocera.version. Knulli (knulli-linux 57afa662) adds OS_NAME=knulli to os-release, writes /usr/share/knulli/knulli.version and removes /usr/share/batocera, and no Knulli source writes /etc/knulli-release. Neither was detected. A Knulli board is named and not narrowed by Batocera's per-board lists, and an unlisted board is named. --- install.ps1 | 2 +- install.py | 47 ++++++++++++++++++++++++++++++++++++----- install.sh | 2 +- platforms/_registry.yml | 16 ++++++++++---- tests/test_install.py | 41 ++++++++++++++++++++++++++++++++++- wiki/installer.md | 9 +++++--- 6 files changed, 102 insertions(+), 15 deletions(-) diff --git a/install.ps1 b/install.ps1 index 19f51c43..eb5abf7d 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "e85a34efb84620b8c65858d209c9741a715f8ff6a61397f9ee161c0931a9a049" + $defaultInstallSha256 = "5ed37d4dabc5a6b28a9093f38a25506b20ec2f49c3587692efd695b017d1caf4" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index fdd3f467..73bfd482 100644 --- a/install.py +++ b/install.py @@ -717,6 +717,28 @@ _BOARD_SOURCES: dict[str, tuple[str, str]] = { } +# A derivative installed under a platform's name, with its own board file. +# Knulli moves /usr/share/batocera to /usr/share/knulli (knulli-linux +# 57afa662, board/scripts/post-build-script.sh:264-278) and writes its board +# to knulli.arch (package/system/knulli-system/knulli-system.mk:98-99). Its +# packages are its own, so Batocera's per-board core lists do not apply. +_DERIVATIVE_BOARDS: dict[str, tuple[str, str]] = { + "batocera": ("Knulli", "/usr/share/knulli/knulli.arch"), +} + + +def derivative_board(platform: str) -> tuple[str, str] | None: + """(derivative, board) when a derivative of the platform names its board.""" + name, where = _DERIVATIVE_BOARDS.get(platform, ("", "")) + if not where: + return None + try: + board = Path(where).read_text(encoding="utf-8", errors="replace").strip().lower() + except OSError: + return None + return (name, board) if board else None + + def detect_board(platform: str) -> str | None: """The board this system was built for, or None where it names none.""" kind, where = _BOARD_SOURCES.get(platform, ("", "")) @@ -767,16 +789,22 @@ def _parse_ps1_var(path: Path, key: str) -> str | None: return None -# Embedded systems, first match wins: Knulli also carries the Batocera -# markers. The registry publishes the same rules as each manifest's -# `detect`, and a test holds the two equal. MiSTer is a path rule below. +# Embedded systems, first match wins. The registry publishes the same rules +# as each manifest's `detect`, and a test holds the two equal. MiSTer is a +# path rule below. _EMBEDDED_RULES: tuple[tuple[str, str, tuple[str, ...], str], ...] = ( # ROCKNIX writes OS_NAME and no ID since 2025 (scripts/image). Older # images wrote ID=rocknix. ("rocknix", "os_release", ("OS_NAME=rocknix", "ID=rocknix"), "/storage/roms/bios"), - ("batocera", "file_exists", ("/etc/knulli-release",), "/userdata/bios"), + # Knulli (knulli-linux 57afa662): OS_NAME in os-release + # (board/scripts/post-build-script.sh:233-236) and knulli.version + # (package/system/knulli-system/knulli-system.mk:98-102). + ("batocera", "os_release", ("OS_NAME=knulli",), "/userdata/bios"), + ("batocera", "file_exists", ("/usr/share/knulli/knulli.version",), "/userdata/bios"), ("lakka", "os_release", ("ID=lakka",), "/storage/system"), - ("batocera", "file_exists", ("/etc/batocera-version",), "/userdata/bios"), + # batocera.linux 38845725, batocera-system.mk:21-22 (the same lines in + # batocera-30): batocera.arch and batocera.version. + ("batocera", "file_exists", ("/usr/share/batocera/batocera.version",), "/userdata/bios"), ("recalbox", "file_exists", ("/recalbox/recalbox.version",), "/recalbox/share/bios"), ("recalbox", "file_exists", ("/usr/bin/recalbox-settings",), "/recalbox/share/bios"), ("retroarch", "file_exists", ("/opt/muos",), "/run/muos/storage/bios"), @@ -4633,6 +4661,15 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: if board in known: target = board print(f" Board {board} detected (--every-board installs all boards)") + else: + print(f" Board {board} has no {PLATFORM_NAMES.get(plat_name, plat_name)} " + "core list: installing the files of every board") + else: + derived = derivative_board(plat_name) + if derived: + print(f" {derived[0]} board {derived[1]} detected. {derived[0]} " + "builds its own set of cores and publishes no core list " + "per board: installing the files of every board") if target: targets = target_lists.get(plat_name) if targets is None: diff --git a/install.sh b/install.sh index a9fb019f..094090f8 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="e85a34efb84620b8c65858d209c9741a715f8ff6a61397f9ee161c0931a9a049" +DEFAULT_INSTALL_SHA256="5ed37d4dabc5a6b28a9093f38a25506b20ec2f49c3587692efd695b017d1caf4" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 819438d7..beb31913 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -313,14 +313,22 @@ platforms: target_source: https://github.com/batocera-linux/batocera.linux install: detect: - # Knulli is a Batocera derivative and carries its markers too. + # Knulli, a Batocera derivative (knulli-cfw/knulli-linux 57afa662): + # OS_NAME in os-release (board/scripts/post-build-script.sh:233-236), + # and knulli.version (package/system/knulli-system/knulli-system.mk:98-102). - os: linux - method: file_exists - file: /etc/knulli-release + method: os_release + match: [OS_NAME=knulli] bios_path: /userdata/bios - os: linux method: file_exists - file: /etc/batocera-version + file: /usr/share/knulli/knulli.version + bios_path: /userdata/bios + # batocera.linux 38845725, package/batocera/core/batocera-system/ + # batocera-system.mk:21-22 writes batocera.arch and batocera.version. + - os: linux + method: file_exists + file: /usr/share/batocera/batocera.version bios_path: /userdata/bios standalone_copies: *standalone_copies recalbox: diff --git a/tests/test_install.py b/tests/test_install.py index 44db4d6f..fa44f502 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1028,7 +1028,8 @@ class DetectionFollowsTheRegistry(unittest.TestCase): with unittest.mock.patch.object(install, "_parse_os_release", return_value={}), \ unittest.mock.patch.object( Path, "exists", - lambda p: str(p) in ("/etc/knulli-release", "/etc/batocera-version"), + lambda p: str(p) in ("/usr/share/knulli/knulli.version", + "/usr/share/batocera/batocera.version"), ): self.assertEqual(install._detect_embedded(), [("batocera", Path("/userdata/bios"))]) # What ROCKNIX writes since 2025: OS_NAME and HW_DEVICE, no ID. @@ -3952,5 +3953,43 @@ class MuOsIsReadFromItsRetroArchConfig(unittest.TestCase): self.assertEqual(rule[3], "/run/muos/storage/bios") +class BatoceraAndKnulliByWhatTheirBuildsWrite(unittest.TestCase): + """No Batocera build writes /etc/batocera-version (batocera-system.mk + writes /usr/share/batocera/batocera.version), and Knulli writes no + /etc/knulli-release and removes /usr/share/batocera: neither was + detected, and a Knulli board got every board's files without a word.""" + + def _detect(self, release: dict, files: tuple[str, ...]): + with unittest.mock.patch.object(install, "_parse_os_release", return_value=release), \ + unittest.mock.patch.object(Path, "exists", lambda p: str(p) in files): + return install._detect_embedded() + + def test_batocera_by_its_version_file(self): + self.assertEqual(self._detect({}, ("/usr/share/batocera/batocera.version",)), + [("batocera", Path("/userdata/bios"))]) + self.assertEqual(self._detect({}, ("/etc/batocera-version",)), []) + + def test_knulli_by_its_os_release_or_version_file(self): + self.assertEqual(self._detect({"OS_NAME": "knulli"}, ()), + [("batocera", Path("/userdata/bios"))]) + self.assertEqual(self._detect({}, ("/usr/share/knulli/knulli.version",)), + [("batocera", Path("/userdata/bios"))]) + + def test_a_knulli_board_is_named_and_not_narrowed(self): + board = Path(_tempdir(self)) / "knulli.arch" + board.write_text("a133") + with unittest.mock.patch.dict(install._DERIVATIVE_BOARDS, + {"batocera": ("Knulli", str(board))}): + self.assertEqual(install.derivative_board("batocera"), ("Knulli", "a133")) + self.assertIsNone(install.derivative_board("recalbox")) + + def test_the_run_says_why_every_board_is_installed(self): + source = (REPO_ROOT / "install.py").read_text(encoding="utf-8") + block = source[source.index("board = detect_board(plat_name)"):] + block = block[:block.index("if target:")] + self.assertIn("has no", block) + self.assertIn("derivative_board(plat_name)", block) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index 1937c0cb..190acfcc 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -279,9 +279,9 @@ wins, since a machine is only one of them. | Evidence | Platform | BIOS directory | |----------|----------|----------------| | `/etc/os-release` `OS_NAME=ROCKNIX` (or `ID=rocknix` on images before 2025) | ROCKNIX | `/storage/roms/bios` | -| `/etc/knulli-release` | Batocera | `/userdata/bios` | +| `/etc/os-release` `OS_NAME=knulli` or `/usr/share/knulli/knulli.version` | Batocera (Knulli) | `/userdata/bios` | | `/etc/os-release` `ID=lakka` | Lakka | `/storage/system` | -| `/etc/batocera-version` | Batocera | `/userdata/bios` | +| `/usr/share/batocera/batocera.version` | Batocera | `/userdata/bios` | | `/recalbox/recalbox.version` or `/usr/bin/recalbox-settings` | Recalbox | `/recalbox/share/bios` | | `/opt/muos` | RetroArch (muOS) | `system_directory` of `/opt/muos/share/info/config/retroarch.cfg`, else of the default it is copied from: `/run/muos/storage/bios`, which muOS binds to `MUOS/bios` on the second card when that card holds one, else the first. Cores are read from that config's `libretro_directory` (`/opt/muos/share/core`) | | `/mnt/mmc/MUOS/` | RetroArch | `/mnt/mmc/MUOS/bios` | @@ -290,7 +290,10 @@ wins, since a machine is only one of them. Batocera, Recalbox, ROCKNIX and Lakka also name the board they were built for, and that board narrows the install unless `--target` or `--every-board` says -otherwise (see the options above). +otherwise (see the options above). A board with no core list of its own is +named and gets the files of every board. Knulli names its board in +`/usr/share/knulli/knulli.arch` but builds its own set of cores, so Batocera's +per-board lists are not applied to it and the run says so. Some platforms keep their BIOS in a fixed directory, and that directory is the evidence. They are looked for after the configurations below, for a platform From 1c4686ab930c6b56b02062e1c85565f754886dba Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:53:23 +0200 Subject: [PATCH 11/11] feat: place RomM firmware by its own template RomM 5.4 reads each platform's firmware where filesystem.structure.firmware in config.yml says, bios/{platform} by default, and suggests {platform}/bios for a {platform}/roms library (rommapp/romm 5.4.0, backend/config/config_manager.py:58,101-109,205-225,564-568,1105-1124). The installer only wrote bios//. It now reads the template from config.yml beside the library, takes --romm-firmware when that file is elsewhere, refuses a platform-first library with no template, and detects RomM by /romm/config/config.yml too. --- install.ps1 | 2 +- install.py | 141 ++++++++++++++++++++++++++++++++++++++++ install.sh | 2 +- platforms/_registry.yml | 8 +++ tests/test_install.py | 95 +++++++++++++++++++++++++++ wiki/installer.md | 19 ++++++ 6 files changed, 265 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index eb5abf7d..5eec478f 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "5ed37d4dabc5a6b28a9093f38a25506b20ec2f49c3587692efd695b017d1caf4" + $defaultInstallSha256 = "ddd92362d6b82d9d1797e1325d45898b4ed265c504b861d1b645d44c33d05fdb" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 73bfd482..33978b44 100644 --- a/install.py +++ b/install.py @@ -1100,6 +1100,9 @@ _PATH_RULES: tuple[tuple[str, str, str, str | None], ...] = ( ("linux", "emudeck", "$HOME/Emulation/bios", None), ("linux", "retrodeck", "$HOME/.var/app/net.retrodeck.retrodeck", "$HOME/retrodeck"), ("linux", "romm", "/romm/library/bios", None), + # A library laid out as {platform}/bios has no library/bios: RomM's + # config folder is the evidence (config_manager.py:264-265). + ("linux", "romm", "/romm/config/config.yml", "/romm/library/bios"), ("linux", "retropie", "$HOME/RetroPie/BIOS", None), ("windows", "bizhawk", "%USERPROFILE%\\BizHawk\\Firmware", None), ("linux", "misterfpga", "/media/fat/MiSTer", "/media/fat/games"), @@ -4127,6 +4130,111 @@ def _print_emulators(plat: str, manifest: dict, files: list[dict], omitted: list print(f" {name:<{width}} {'' if profile == name else 'same as ' + profile}".rstrip()) +# RomM 5.4 places each platform's firmware by `filesystem.structure.firmware` +# in config.yml, "bios/{platform}" by default, relative to the library +# (rommapp/romm 5.4.0 68f71259, backend/config/config_manager.py:58,101-109, +# 205-225,564-568). The library and config.yml both sit under the RomM base +# folder (config/__init__.py:46-48, config_manager.py:264-265). A library +# laid out as {platform}/roms makes RomM stop until config.yml declares its +# templates, and RomM suggests firmware: "{platform}/bios" there +# (config_manager.py:1105-1124). + + +def parse_romm_firmware(template: str) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The literal folders before and after {platform}, as RomM parses them.""" + sections = [section for section in template.strip().split("/") if section] + if not sections: + raise ValueError("the RomM firmware template is empty") + + def macro(section: str) -> str | None: + if section.startswith("{") and section.endswith("}"): + return section[1:-1].strip() + return None + + if any(section in (".", "..") for section in sections): + raise ValueError(f"the RomM firmware template {template!r} climbs out of the library") + platform_at = [i for i, section in enumerate(sections) if macro(section) == "platform"] + if len(platform_at) != 1: + raise ValueError(f"the RomM firmware template {template!r} needs one '{{platform}}'") + before, after = sections[:platform_at[0]], sections[platform_at[0] + 1:] + if any(macro(section) is not None for section in before + after): + raise ValueError(f"the RomM firmware template {template!r} takes literal folder " + "names around '{platform}'") + return tuple(before), tuple(after) + + +def _romm_configured_firmware(config: Path) -> str | None: + """filesystem.structure.firmware from RomM's config.yml, read without a + YAML library: only that nested scalar is needed.""" + try: + lines = config.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return None + path: list[tuple[int, str]] = [] + for line in lines: + stripped = line.split(" #", 1)[0].rstrip() + if not stripped.strip() or stripped.lstrip().startswith(("#", "-")): + continue + indent = len(stripped) - len(stripped.lstrip()) + key, sep, value = stripped.strip().partition(":") + if not sep: + continue + while path and path[-1][0] >= indent: + path.pop() + keys = [name for _indent, name in path] + [key.strip().strip("'\"")] + if keys == ["filesystem", "structure", "firmware"] and value.strip(): + return value.strip().strip("'\"") + path.append((indent, key.strip().strip("'\""))) + return None + + +def romm_firmware_layout( + bios_path: Path, template: str | None +) -> tuple[tuple[str, ...], tuple[str, ...]] | None: + """Where RomM reads each platform's firmware, relative to its library. + + bios_path is the folder the default template names, so its parent is the + library. The template comes from --romm-firmware, else from config.yml + beside the library. None keeps the pack's own bios/ layout. + """ + library = bios_path.parent + if template is None: + template = _romm_configured_firmware(library.parent / "config" / "config.yml") + if template is not None: + return parse_romm_firmware(template) + platform_first = not (library / "roms").is_dir() and any( + path.is_dir() for path in library.glob("*/roms") + ) + if platform_first: + raise ValueError( + f"{library} keeps its roms inside each platform folder, and RomM then " + "reads firmware where filesystem.structure.firmware in its config.yml " + "says, which was not found here. RomM suggests \"{platform}/bios\" for " + "this layout: run again with --romm-firmware \"{platform}/bios\", or " + "with the value config.yml holds." + ) + return None + + +def romm_relayout( + files: list[dict], bios_path: Path, + before: tuple[str, ...], after: tuple[str, ...], +) -> tuple[list[dict], Path]: + """The manifest's bios// entries under RomM's template, + with the library as the root they are relative to.""" + if (before, after) == (("bios",), ()): + return files, bios_path + moved = [] + for entry in files: + slug, _, rest = str(entry["dest"]).partition("/") + if not rest: + # Firmware of no platform: RomM lists none, the default place stays. + moved.append({**entry, "dest": f"bios/{slug}"}) + continue + moved.append({**entry, "dest": "/".join((*before, slug, *after, rest))}) + return moved, bios_path.parent + + def _older_list_advice() -> str: """How to reach a file list that records what an option needs. @@ -4338,6 +4446,13 @@ def main() -> None: action="store_true", help="list the cores whose extra files a platform installs and exit", ) + parser.add_argument( + "--romm-firmware", + metavar="TEMPLATE", + help="RomM: where each platform's firmware goes in the library, as " + "filesystem.structure.firmware in its config.yml (default: read " + "from config.yml, else bios/{platform})", + ) parser.add_argument( "--list-emulators", action="store_true", @@ -4595,6 +4710,18 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: "--platform to install it alone.", file=sys.stderr) sys.exit(1) + if args.romm_firmware is not None: + # Applied or refused, never ignored. + if not any(name == "romm" for name, _path in platforms): + print("Error: --romm-firmware applies to RomM, which this run does not " + "install for.", file=sys.stderr) + sys.exit(1) + try: + parse_romm_firmware(args.romm_firmware) + except ValueError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + # Every check --installed-cores makes before anything is written: a # refusal for the second platform must not follow an install of the first. cores_found: dict[tuple[str, Path], list[str]] = {} @@ -4785,6 +4912,20 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: total_omitted += len(omitted_files) omitted_seen.extend(omitted_files) + if plat_name == "romm": + try: + layout = romm_firmware_layout(bios_path, args.romm_firmware) + except ValueError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + if layout is not None: + files, library = romm_relayout(files, bios_path, *layout) + if library != bios_path: + installed_paths[installed_paths.index(bios_path)] = library + print(f" Firmware laid out as {'/'.join(layout[0] + ('{platform}',) + layout[1])} " + f"under {library}") + bios_path = library + print("\nChecking existing files...") to_download, up_to_date, mismatched = check_local( files, bios_path, jobs=args.jobs, term=term diff --git a/install.sh b/install.sh index 094090f8..7d214e39 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="5ed37d4dabc5a6b28a9093f38a25506b20ec2f49c3587692efd695b017d1caf4" +DEFAULT_INSTALL_SHA256="ddd92362d6b82d9d1797e1325d45898b4ed265c504b861d1b645d44c33d05fdb" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/platforms/_registry.yml b/platforms/_registry.yml index beb31913..22c77a2d 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -511,6 +511,14 @@ platforms: - os: linux method: path_exists path: /romm/library/bios + # RomM 5.4 places firmware by filesystem.structure.firmware in this + # file, relative to /romm/library (rommapp/romm 5.4.0, + # backend/config/config_manager.py:58,101-109,264-265,564-568): a + # {platform}/bios library has no library/bios to find. + - os: linux + method: path_exists + path: /romm/config/config.yml + bios_path: /romm/library/bios cores: - atari800 - clk diff --git a/tests/test_install.py b/tests/test_install.py index fa44f502..dfd31f8f 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3991,5 +3991,100 @@ class BatoceraAndKnulliByWhatTheirBuildsWrite(unittest.TestCase): self.assertIn("derivative_board(plat_name)", block) +class RomMFirmwareFollowsItsTemplate(unittest.TestCase): + """RomM 5.4 reads firmware where filesystem.structure.firmware says, and + a {platform}/roms library is told to declare "{platform}/bios". The + installer only wrote bios//, where such a library lists none.""" + + CONFIG = ( + "# RomM\n" + "exclude:\n" + " platforms: []\n" + "filesystem:\n" + " structure:\n" + " default: \"{platform}/roms/{game}\"\n" + " firmware: \"{platform}/bios\" # as RomM suggests\n" + "system:\n" + " firmware: elsewhere\n" + ) + + def _romm(self, config: str | None) -> Path: + base = Path(_tempdir(self)) + library = base / "library" + (library / "psx" / "roms").mkdir(parents=True) + if config is not None: + (base / "config").mkdir() + (base / "config" / "config.yml").write_text(config) + return library / "bios" + + def test_templates_parse_as_romm_parses_them(self): + self.assertEqual(install.parse_romm_firmware("bios/{platform}"), (("bios",), ())) + self.assertEqual(install.parse_romm_firmware("{platform}/bios"), ((), ("bios",))) + self.assertEqual(install.parse_romm_firmware("/a/{platform}/b/c/"), + (("a",), ("b", "c"))) + for bad in ("", "bios", "{platform}/{game}", "{region}/{platform}", + "../{platform}", "{platform}/{platform}"): + with self.subTest(template=bad), self.assertRaises(ValueError): + install.parse_romm_firmware(bad) + + def test_the_template_is_read_from_config_yml(self): + bios = self._romm(self.CONFIG) + layout = install.romm_firmware_layout(bios, None) + self.assertEqual(layout, ((), ("bios",))) + files = [{"dest": "psx/scph5501.bin"}, {"dest": "3do/panafz1.bin"}] + moved, root = install.romm_relayout(files, bios, *layout) + self.assertEqual(root, bios.parent) + self.assertEqual([f["dest"] for f in moved], + ["psx/bios/scph5501.bin", "3do/bios/panafz1.bin"]) + + def test_the_default_keeps_the_pack_layout(self): + bios = self._romm("filesystem:\n structure:\n firmware: bios/{platform}\n") + files = [{"dest": "psx/scph5501.bin"}] + self.assertEqual(install.romm_relayout(files, bios, *install.romm_firmware_layout( + bios, None)), (files, bios)) + + def test_a_platform_first_library_without_a_template_is_refused(self): + bios = self._romm(None) + with self.assertRaises(ValueError) as raised: + install.romm_firmware_layout(bios, None) + self.assertIn('--romm-firmware "{platform}/bios"', str(raised.exception)) + self.assertEqual(install.romm_firmware_layout(bios, "{platform}/bios"), + ((), ("bios",))) + + def test_the_option_is_refused_where_romm_is_not_installed(self): + err = io.StringIO() + with unittest.mock.patch.object(install, "detect_os", return_value="linux"), \ + unittest.mock.patch("sys.argv", [ + "install.py", "--platform", "batocera", "--dest", _tempdir(self), + "--romm-firmware", "{platform}/bios", "--no-input"]), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(err), self.assertRaises(SystemExit) as caught: + install.main() + self.assertEqual(caught.exception.code, 1) + self.assertIn("--romm-firmware applies to RomM", err.getvalue()) + + +class RomMFirmwareInstallsUnderItsTemplate(unittest.TestCase, _LoopbackRun): + """The whole run, over loopback: config.yml beside the library decides.""" + + def setUp(self): + self.env = self.serve(self) + + def test_lynx_lands_in_its_platform_folder(self): + base = self.home / "romm" + (base / "library" / "lynx" / "roms").mkdir(parents=True) + (base / "config").mkdir() + (base / "config" / "config.yml").write_text( + "filesystem:\n structure:\n default: \"{platform}/roms/{game}\"\n" + " firmware: \"{platform}/bios\"\n") + proc = _install_run(self.env, "--platform", "romm", "--dest", + str(base / "library" / "bios"), "--system", "atari-lynx", + "--no-input") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertTrue((base / "library" / "lynx" / "bios" / "lynxboot.img").is_file()) + self.assertFalse((base / "library" / "bios").exists()) + self.assertIn("{platform}/bios", proc.stdout) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index 190acfcc..88a42e22 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -235,6 +235,7 @@ what the options name, everything by default. | `--list-systems` | Print a platform's systems with the files and size each holds | | `--list-cores` | Print the cores whose extra files a platform installs | | `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | +| `--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) | | `--no-input` | Never ask a question, even at a terminal | | `--check` | Report and exit without writing | | `--dry-run` | Show what would be downloaded, by system and with sizes, without writing anything | @@ -306,6 +307,7 @@ not already found that way. | `~/Emulation/bios` | EmuDeck | the same | | `~/.var/app/net.retrodeck.retrodeck` | RetroDECK | `~/retrodeck` | | `/romm/library/bios` | RomM | the same | +| `/romm/config/config.yml` | RomM | `/romm/library/bios`, laid out by its firmware template | | `~/RetroPie/BIOS` | RetroPie | the same | | `%USERPROFILE%\BizHawk\Firmware` | BizHawk | the same | | `EmuHawk.exe` or `EmuHawkMono.sh` in a `BizHawk*` folder of the home folder or one level down (`~/Games/BizHawk-2.11.1`), the newest | BizHawk | `Firmware` beside it | @@ -415,6 +417,23 @@ replaced by the file rather than written through: a write through the link would land outside the folder the user opted into. The summary counts each copy as copied, skipped (folder absent) or failed, and names every failure. +## RomM firmware layout + +RomM 5.4 places each platform's firmware where `filesystem.structure.firmware` +in its `config.yml` says, relative to the library: `bios/{platform}` by +default, the layout of the pack. A library laid out as `{platform}/roms` makes +RomM stop until `config.yml` declares its templates, and RomM suggests +`firmware: "{platform}/bios"` for it. + +The installer reads that value from `config.yml` in the `config` folder beside +the library, `/romm/config/config.yml` inside the container, and places each +file accordingly: with `{platform}/bios`, `psx/scph5501.bin` goes to +`library/psx/bios/scph5501.bin`. `--dest` still names the library's `bios` +folder, whose parent is the library. Run from the host of a container, where +`config.yml` sits elsewhere, `--romm-firmware` gives the same value. A +library laid out as `{platform}/roms` with no template found is refused +rather than filled in the wrong place, and the message gives the command. + ## Android Android reports itself as Linux, so the runtime is named from its own