From 9ea1630caad027e1bf731a46d30f3ff70e9adac8 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:52:37 +0200 Subject: [PATCH] feat: install case twins where case is kept The RetroArch list is folded for Windows and macOS. Its manifest now carries the names a case-keeping directory also needs, and the installer adds them after probing the BIOS directory. --- install.ps1 | 2 +- install.py | 111 +++- install.sh | 2 +- install/bizhawk.json | 14 +- install/retroarch.json | 93 +++ schemas/install-manifest.schema.json | 920 ++++++++++++++------------- scripts/generate_pack.py | 32 +- tests/test_audit_regressions.py | 6 +- tests/test_install.py | 52 ++ wiki/troubleshooting.md | 9 + 10 files changed, 748 insertions(+), 493 deletions(-) diff --git a/install.ps1 b/install.ps1 index ac2fdecf..acc331ca 100644 --- a/install.ps1 +++ b/install.ps1 @@ -8,7 +8,7 @@ param( $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -$defaultInstallSha256 = "b6a90d5cecf61b6bcb2a48ef440a57c56d0792ca0986c2c5dd9ba952dda3a221" +$defaultInstallSha256 = "8faaf944ff4e0cc23e9049299145077567163837eaabdd65c530554980080ed7" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index df7c6a76..d66b3783 100644 --- a/install.py +++ b/install.py @@ -1099,6 +1099,28 @@ def core_dirs(plat: str, bios_path: Path) -> list[Path]: return list(dict.fromkeys(dirs)) +def keeps_case(directory: Path) -> bool: + """Whether names differing only by case are distinct files there. + + Probed on the nearest existing directory, since the filesystem decides, + not the operating system: an ext4 drive mounted on macOS keeps case and + a casefolded ext4 directory folds it. Unprobeable: the system's default. + """ + probe_dir = directory + while not probe_dir.is_dir() and probe_dir != probe_dir.parent: + probe_dir = probe_dir.parent + try: + handle, name = tempfile.mkstemp(prefix=".retrobios-case-", dir=str(probe_dir)) + except OSError: + return sys.platform not in ("win32", "darwin") + os.close(handle) + try: + twin = os.path.join(os.path.dirname(name), os.path.basename(name).upper()) + return not os.path.exists(twin) + finally: + os.unlink(name) + + def installed_cores(dirs: list[Path]) -> list[str]: """The cores present in these directories, by the name the buildbot uses. @@ -1336,6 +1358,46 @@ def _check_slot(entry: dict, where: str, dest: str) -> int: return size +def _check_entry( + entry: object, where: str, seen_destinations: set[str] +) -> tuple[int, int]: + """Check one file entry; return its size and the largest copy it may fetch.""" + if not isinstance(entry, dict): + raise ValueError(f"{where} must be an object") + dest = str(_safe_relative_path(entry.get("dest"), f"{where}.dest")) + if dest in seen_destinations: + raise ValueError(f"duplicate manifest destination: {dest}") + seen_destinations.add(dest) + + size = _check_download(entry, where, dest) + largest = size + alternatives = entry.get("alternatives", []) + if not isinstance(alternatives, list) or len(alternatives) > MAX_SELECTION_TAGS: + raise ValueError(f"invalid alternatives for {dest}") + for alt_index, alternative in enumerate(alternatives): + if ( + not isinstance(alternative, dict) + or not alternative.get("cores") + or not set(alternative) <= set(_COPY_FIELDS) + ): + raise ValueError(f"invalid alternative for {dest}") + # Narrowing can swap in any copy: the cap holds for the largest. + largest = max(largest, _check_download( + alternative, f"{where}.alternatives[{alt_index}]", dest + )) + largest = max(largest, _check_slot(entry, where, dest)) + absent = entry.get("absent_claims", []) + if not isinstance(absent, list) or len(absent) > MAX_SELECTION_TAGS: + raise ValueError(f"invalid absent claims for {dest}") + for claim in absent: + if not isinstance(claim, dict): + raise ValueError(f"invalid absent claim for {dest}") + _check_omission(claim, dest) + for field in (*SELECTION_FIELDS, "declared_in"): + _check_tags(entry, field, dest) + return size, largest + + def _validate_manifest(data: object, plat: str) -> dict: """Validate the untrusted install-manifest boundary using stdlib only.""" if not isinstance(data, dict): @@ -1352,41 +1414,21 @@ def _validate_manifest(data: object, plat: str) -> dict: total_size = 0 largest_total = 0 for index, entry in enumerate(files): - if not isinstance(entry, dict): - raise ValueError(f"files[{index}] must be an object") - dest = str(_safe_relative_path(entry.get("dest"), f"files[{index}].dest")) - if dest in seen_destinations: - raise ValueError(f"duplicate manifest destination: {dest}") - seen_destinations.add(dest) - - size = _check_download(entry, f"files[{index}]", dest) + size, largest = _check_entry(entry, f"files[{index}]", seen_destinations) total_size += size - alternatives = entry.get("alternatives", []) - if not isinstance(alternatives, list) or len(alternatives) > MAX_SELECTION_TAGS: - raise ValueError(f"invalid alternatives for {dest}") - for alt_index, alternative in enumerate(alternatives): - where = f"files[{index}].alternatives[{alt_index}]" - if ( - not isinstance(alternative, dict) - or not alternative.get("cores") - or not set(alternative) <= set(_COPY_FIELDS) - ): - raise ValueError(f"invalid alternative for {dest}") - # Narrowing can swap in any copy: the cap holds for the largest. - size = max(size, _check_download(alternative, where, dest)) - size = max(size, _check_slot(entry, f"files[{index}]", dest)) - absent = entry.get("absent_claims", []) - if not isinstance(absent, list) or len(absent) > MAX_SELECTION_TAGS: - raise ValueError(f"invalid absent claims for {dest}") - for claim in absent: - if not isinstance(claim, dict): - raise ValueError(f"invalid absent claim for {dest}") - _check_omission(claim, dest) - largest_total += size + largest_total += largest + if largest_total > MAX_TOTAL_DOWNLOAD_SIZE: + raise ValueError("manifest total size exceeds safety limit") + variants = data.get("case_variants", []) + if not isinstance(variants, list) or len(variants) > MAX_MANIFEST_FILES: + raise ValueError("invalid case_variants list") + for index, entry in enumerate(variants): + _size, largest = _check_entry( + entry, f"case_variants[{index}]", seen_destinations + ) + largest_total += largest if largest_total > MAX_TOTAL_DOWNLOAD_SIZE: raise ValueError("manifest total size exceeds safety limit") - for field in (*SELECTION_FIELDS, "declared_in"): - _check_tags(entry, field, dest) declared_total_files = data.get("total_files") if declared_total_files is not None and declared_total_files != len(files): @@ -3154,6 +3196,11 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: manifest = fetch_manifest(plat_name) files = manifest.get("files", []) omitted_files = manifest.get("omitted_files", []) + variants = manifest.get("case_variants", []) + if variants and keeps_case(bios_path): + # The list is folded for Windows and macOS; here bios.col and + # BIOS.col are two files that two cores open. + files = files + variants if args.list_targets: targets = fetch_targets(plat_name) diff --git a/install.sh b/install.sh index c5c50a03..79362611 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="b6a90d5cecf61b6bcb2a48ef440a57c56d0792ca0986c2c5dd9ba952dda3a221" +DEFAULT_INSTALL_SHA256="8faaf944ff4e0cc23e9049299145077567163837eaabdd65c530554980080ed7" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/bizhawk.json b/install/bizhawk.json index b6456dd3..51dcb329 100644 --- a/install/bizhawk.json +++ b/install/bizhawk.json @@ -5,7 +5,7 @@ "platform": "bizhawk", "display_name": "BizHawk", "version": "1.0", - "generated": "2026-10-18T00:39:54Z", + "generated": "2026-10-18T11:24:19Z", "base_destination": "Firmware", "detect": [ { @@ -12165,5 +12165,17 @@ "storage": "release", "release_asset": "dsi_nand.bin.8b79a73c" } + ], + "case_variants": [ + { + "dest": "dsi_firmware.bin", + "sha1": "d2a5af338f09c5cbdd5d7628db5b9c075c69b616", + "sha256": "1d7c771de535e6e07182fc9820ac32e76c6f0b5bcefd256a861fdfab144761ec", + "size": 131072, + "repo_path": "bios/Nintendo/Nintendo DS/dsi_firmware.bin", + "cores": [ + "melonds" + ] + } ] } \ No newline at end of file diff --git a/install/retroarch.json b/install/retroarch.json index b12d72ae..c6fbf48b 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -71870,5 +71870,98 @@ "sony-playstation-2" ] } + ], + "case_variants": [ + { + "dest": "bios.col", + "sha1": "45bedc4cbdeac66c7df59e9e599195c778d86a92", + "sha256": "990bf1956f10207d8781b619eb74f89b00d921c8d45c95c334c16c8cceca09ad", + "size": 8192, + "repo_path": "bios/Coleco/ColecoVision/BIOS.col", + "cores": [ + "mesence" + ], + "systems": [ + "coleco-colecovision" + ] + }, + { + "dest": "np2kai/2608_bd.wav", + "sha1": "0a56c142ef40cec50f3ee56a6e42d0029c9e2818", + "sha256": "802fb0ce40878e98335fc56d11eff6382982e9f09ebc83055b0a469df13f7900", + "size": 19192, + "repo_path": "bios/NEC/PC-98/2608_bd.wav", + "cores": [ + "np2kai" + ], + "systems": [ + "nec-pc-98" + ] + }, + { + "dest": "np2kai/2608_sd.wav", + "sha1": "3c79663ef74c0b0439d13351326eb1c52a657008", + "sha256": "e5877cd1e4821cbbc9a450491a189032c82906c05f7fc9f2e48ee0d06b3ad18c", + "size": 15558, + "repo_path": "bios/NEC/PC-98/2608_sd.wav", + "cores": [ + "np2kai" + ], + "systems": [ + "nec-pc-98" + ] + }, + { + "dest": "np2kai/2608_top.wav", + "sha1": "aa4a8f766a86b830687d5083fd3b9db0652f46fc", + "sha256": "d7bcf2da99a09589096e9bcee98da43303c588cca4e89b1a263adace52038cd3", + "size": 57016, + "repo_path": "bios/NEC/PC-98/2608_top.wav", + "cores": [ + "np2kai" + ], + "systems": [ + "nec-pc-98" + ] + }, + { + "dest": "np2kai/2608_hh.wav", + "sha1": "12f676cef249b82480b6f19c454e234b435ca7b6", + "sha256": "216c81ff9a84af0dfbf21dd8df6827c25fca9fdeefe9bc8426f7a00d6ff2264e", + "size": 36722, + "repo_path": "bios/NEC/PC-98/2608_hh.wav", + "cores": [ + "np2kai" + ], + "systems": [ + "nec-pc-98" + ] + }, + { + "dest": "np2kai/2608_tom.wav", + "sha1": "9513fb4a3f41e75a972a273a5104cbd834c1e2c5", + "sha256": "8e2c9ccfa8caff03dbd0d945d7c1d4de15292817409fe6fabda07f008f97dbc6", + "size": 23092, + "repo_path": "bios/NEC/PC-98/2608_tom.wav", + "cores": [ + "np2kai" + ], + "systems": [ + "nec-pc-98" + ] + }, + { + "dest": "np2kai/2608_rim.wav", + "sha1": "c65592330c9dd84011151daed52f9aec926b7e56", + "sha256": "6f7d1996001c74452a9ca68b41a2767925b5b947c699096a8ad15bbc658af5a3", + "size": 5288, + "repo_path": "bios/NEC/PC-98/2608_rim.wav", + "cores": [ + "np2kai" + ], + "systems": [ + "nec-pc-98" + ] + } ] } \ No newline at end of file diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index b5dbf3fb..fe93d111 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -294,474 +294,486 @@ "files": { "type": "array", "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "dest", - "sha1", - "sha256", - "size", - "repo_path", - "cores" - ], - "properties": { - "dest": { - "type": "string", - "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" - }, - "sha1": { - "type": "string", - "pattern": "^[a-f0-9]{40}$" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "size": { - "type": "integer", - "minimum": 0 - }, - "repo_path": { - "type": "string" - }, - "cores": { - "oneOf": [ - { - "type": "null" + "$ref": "#/$defs/file" + } + }, + "case_variants": { + "description": "Files the folded list serves under another case; installed only where the target directory keeps case.", + "type": "array", + "items": { + "$ref": "#/$defs/file" + } + } + }, + "$defs": { + "file": { + "type": "object", + "additionalProperties": false, + "required": [ + "dest", + "sha1", + "sha256", + "size", + "repo_path", + "cores" + ], + "properties": { + "dest": { + "type": "string", + "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" + }, + "sha1": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "repo_path": { + "type": "string" + }, + "cores": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" }, - { + "uniqueItems": true + } + ] + }, + "storage": { + "const": "release" + }, + "release_asset": { + "type": "string", + "pattern": "^[^/\\\\]+$" + }, + "systems": { + "description": "Every platform system that declares the file, for the installer's --system.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "regions": { + "description": "The regions the emulator code selects the file for. Absent when it never competes.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "argentina", + "asia", + "asia-ntsc", + "asia-pal", + "australia", + "brazil", + "canada", + "china", + "denmark", + "europe", + "finland", + "france", + "germany", + "greece", + "hong-kong", + "india", + "italy", + "japan", + "latin-america", + "mexico", + "netherlands", + "new-zealand", + "north-america", + "norway", + "oceania", + "poland", + "portugal", + "russia", + "singapore", + "south-korea", + "spain", + "sweden", + "taiwan", + "uk", + "world" + ] + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "region_groups": { + "description": "Groups the file competes in as a core extra, whatever systems are chosen.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "region_system_groups": { + "description": "Groups the file competes in as the platform's own declaration: present only when that system is kept.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "priority": { + "description": "The rank the code searches the file at, lower first: a world file withdraws an unmatched regional one only when it ranks no worse.", + "type": "integer" + }, + "alternatives": { + "description": "Copies other cores read at this destination when they disagree on its content, in the order the pack builder tries them; a narrowing that drops the entry's cores takes the first whose cores remain.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "sha1", + "sha256", + "size", + "repo_path", + "cores" + ], + "properties": { + "sha1": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "repo_path": { + "type": "string" + }, + "storage": { + "const": "release" + }, + "release_asset": { + "type": "string", + "pattern": "^[^/\\\\]+$" + }, + "cores": { "type": "array", "items": { "type": "string" }, + "minItems": 1, "uniqueItems": true - } - ] - }, - "storage": { - "const": "release" - }, - "release_asset": { - "type": "string", - "pattern": "^[^/\\\\]+$" - }, - "systems": { - "description": "Every platform system that declares the file, for the installer's --system.", - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 256 - }, - "minItems": 1, - "maxItems": 1000, - "uniqueItems": true - }, - "regions": { - "description": "The regions the emulator code selects the file for. Absent when it never competes.", - "type": "array", - "items": { - "type": "string", - "enum": [ - "argentina", - "asia", - "asia-ntsc", - "asia-pal", - "australia", - "brazil", - "canada", - "china", - "denmark", - "europe", - "finland", - "france", - "germany", - "greece", - "hong-kong", - "india", - "italy", - "japan", - "latin-america", - "mexico", - "netherlands", - "new-zealand", - "north-america", - "norway", - "oceania", - "poland", - "portugal", - "russia", - "singapore", - "south-korea", - "spain", - "sweden", - "taiwan", - "uk", - "world" - ] - }, - "minItems": 1, - "maxItems": 1000, - "uniqueItems": true - }, - "region_groups": { - "description": "Groups the file competes in as a core extra, whatever systems are chosen.", - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 256 - }, - "minItems": 1, - "maxItems": 1000, - "uniqueItems": true - }, - "region_system_groups": { - "description": "Groups the file competes in as the platform's own declaration: present only when that system is kept.", - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 256 - }, - "minItems": 1, - "maxItems": 1000, - "uniqueItems": true - }, - "priority": { - "description": "The rank the code searches the file at, lower first: a world file withdraws an unmatched regional one only when it ranks no worse.", - "type": "integer" - }, - "alternatives": { - "description": "Copies other cores read at this destination when they disagree on its content, in the order the pack builder tries them; a narrowing that drops the entry's cores takes the first whose cores remain.", - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "sha1", - "sha256", - "size", - "repo_path", - "cores" - ], - "properties": { - "sha1": { - "type": "string", - "pattern": "^[a-f0-9]{40}$" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "size": { - "type": "integer", - "minimum": 0 - }, - "repo_path": { - "type": "string" - }, - "storage": { - "const": "release" - }, - "release_asset": { - "type": "string", - "pattern": "^[^/\\\\]+$" - }, - "cores": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - }, - "executable": { - "type": "boolean" - } - } - } - }, - "slot_choices": { - "description": "On a platform file a core's own source contradicts while the frontend only checks the path: the copy each core's presence puts there, in the order the pack builder settles the slot. A narrowing serves the first whose cores remain; one naming only cores, or none remaining, leaves `declared_choices`.", - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "cores" - ], - "properties": { - "sha1": { - "type": "string", - "pattern": "^[a-f0-9]{40}$" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "size": { - "type": "integer", - "minimum": 0 - }, - "repo_path": { - "type": "string" - }, - "storage": { - "const": "release" - }, - "release_asset": { - "type": "string", - "pattern": "^[^/\\\\]+$" - }, - "cores": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - }, - "executable": { - "type": "boolean" - } - } - } - }, - "declared_choices": { - "description": "The platform's own answer at the destination by the declaring systems that remain, in the order the pack builder takes the declarations; a narrowing serves the first whose systems remain.", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "sha1", - "sha256", - "size", - "repo_path", - "systems" - ], - "properties": { - "sha1": { - "type": "string", - "pattern": "^[a-f0-9]{40}$" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "size": { - "type": "integer", - "minimum": 0 - }, - "repo_path": { - "type": "string" - }, - "storage": { - "const": "release" - }, - "release_asset": { - "type": "string", - "pattern": "^[^/\\\\]+$" - }, - "executable": { - "type": "boolean" - }, - "systems": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1, - "uniqueItems": true - } - } - } - }, - "executable": { - "description": "The repository keeps the file executable; the installer gives it the execute bits the umask allows.", - "type": "boolean" - }, - "declared_in": { - "description": "The platform systems declaring the file, when fewer than `systems`, which also names the systems whose cores read it there.", - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "uniqueItems": true - }, - "absent_claims": { - "description": "Cores that want this destination and have nothing held, after the cores served here: a narrowing that keeps none of those takes the first named one as a file left out, as a pack built for it lists it.", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "name", - "system", - "required", - "reason", - "cores" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "system": { - "type": "string" - }, - "required": { - "type": "boolean" - }, - "reason": { - "enum": [ - "hash_mismatch", - "not_found", - "external", - "user_provided", - "no_platform_slug" - ] - }, - "cores": { - "oneOf": [ - { - "type": "null" - }, - { - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true - } - ] - }, - "regions": { - "description": "The regions the emulator code selects the file for. Absent when it never competes.", - "type": "array", - "items": { - "type": "string", - "enum": [ - "argentina", - "asia", - "asia-ntsc", - "asia-pal", - "australia", - "brazil", - "canada", - "china", - "denmark", - "europe", - "finland", - "france", - "germany", - "greece", - "hong-kong", - "india", - "italy", - "japan", - "latin-america", - "mexico", - "netherlands", - "new-zealand", - "north-america", - "norway", - "oceania", - "poland", - "portugal", - "russia", - "singapore", - "south-korea", - "spain", - "sweden", - "taiwan", - "uk", - "world" - ] - }, - "minItems": 1, - "maxItems": 1000, - "uniqueItems": true - }, - "region_groups": { - "description": "Groups the file competes in as a core extra, whatever systems are chosen.", - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 256 - }, - "minItems": 1, - "maxItems": 1000, - "uniqueItems": true - }, - "region_system_groups": { - "description": "Groups the file competes in as the platform's own declaration: present only when that system is kept.", - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 256 - }, - "minItems": 1, - "maxItems": 1000, - "uniqueItems": true - }, - "systems": { - "description": "Every platform system that declares the file, for the installer's --system.", - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "maxLength": 256 - }, - "minItems": 1, - "maxItems": 1000, - "uniqueItems": true - }, - "priority": { - "description": "The rank the code searches the file at, lower first: a world file withdraws an unmatched regional one only when it ranks no worse.", - "type": "integer" - }, - "declared_in": { - "description": "The platform systems declaring the file, when fewer than `systems`, which also names the systems whose cores read it there.", - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "uniqueItems": true - } + }, + "executable": { + "type": "boolean" } } } }, - "oneOf": [ - { + "slot_choices": { + "description": "On a platform file a core's own source contradicts while the frontend only checks the path: the copy each core's presence puts there, in the order the pack builder settles the slot. A narrowing serves the first whose cores remain; one naming only cores, or none remaining, leaves `declared_choices`.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "cores" + ], + "properties": { + "sha1": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "repo_path": { + "type": "string" + }, + "storage": { + "const": "release" + }, + "release_asset": { + "type": "string", + "pattern": "^[^/\\\\]+$" + }, + "cores": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + }, + "executable": { + "type": "boolean" + } + } + } + }, + "declared_choices": { + "description": "The platform's own answer at the destination by the declaring systems that remain, in the order the pack builder takes the declarations; a narrowing serves the first whose systems remain.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "sha1", + "sha256", + "size", + "repo_path", + "systems" + ], + "properties": { + "sha1": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "repo_path": { + "type": "string" + }, + "storage": { + "const": "release" + }, + "release_asset": { + "type": "string", + "pattern": "^[^/\\\\]+$" + }, + "executable": { + "type": "boolean" + }, + "systems": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1, + "uniqueItems": true + } + } + } + }, + "executable": { + "description": "The repository keeps the file executable; the installer gives it the execute bits the umask allows.", + "type": "boolean" + }, + "declared_in": { + "description": "The platform systems declaring the file, when fewer than `systems`, which also names the systems whose cores read it there.", + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "absent_claims": { + "description": "Cores that want this destination and have nothing held, after the cores served here: a narrowing that keeps none of those takes the first named one as a file left out, as a pack built for it lists it.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "system", + "required", + "reason", + "cores" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "system": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "reason": { + "enum": [ + "hash_mismatch", + "not_found", + "external", + "user_provided", + "no_platform_slug" + ] + }, + "cores": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + ] + }, + "regions": { + "description": "The regions the emulator code selects the file for. Absent when it never competes.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "argentina", + "asia", + "asia-ntsc", + "asia-pal", + "australia", + "brazil", + "canada", + "china", + "denmark", + "europe", + "finland", + "france", + "germany", + "greece", + "hong-kong", + "india", + "italy", + "japan", + "latin-america", + "mexico", + "netherlands", + "new-zealand", + "north-america", + "norway", + "oceania", + "poland", + "portugal", + "russia", + "singapore", + "south-korea", + "spain", + "sweden", + "taiwan", + "uk", + "world" + ] + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "region_groups": { + "description": "Groups the file competes in as a core extra, whatever systems are chosen.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "region_system_groups": { + "description": "Groups the file competes in as the platform's own declaration: present only when that system is kept.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "systems": { + "description": "Every platform system that declares the file, for the installer's --system.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "priority": { + "description": "The rank the code searches the file at, lower first: a world file withdraws an unmatched regional one only when it ranks no worse.", + "type": "integer" + }, + "declared_in": { + "description": "The platform systems declaring the file, when fewer than `systems`, which also names the systems whose cores read it there.", + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + } + } + } + } + }, + "oneOf": [ + { + "required": [ + "release_asset" + ] + }, + { + "properties": { + "repo_path": { + "type": "string", + "pattern": "^bios/" + } + }, + "not": { "required": [ "release_asset" ] - }, - { - "properties": { - "repo_path": { - "type": "string", - "pattern": "^bios/" - } - }, - "not": { - "required": [ - "release_asset" - ] - } } - ] - } + } + ] } } } diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 5eaf49e3..dfe198e4 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -3764,6 +3764,10 @@ def _record_unplaceable(unplaceable: list[dict], record_omission: Callable[..., ) +# Systems whose default filesystems fold case (NTFS, APFS, HFS+). +CASE_FOLDING_OS = frozenset({"windows", "darwin"}) + + def generate_manifest( platform_name: str, platforms_dir: str, @@ -3780,13 +3784,17 @@ def generate_manifest( offline: bool | None = None, required_only: bool = False, data_registry: dict | None = None, + fold_case: bool | None = None, ) -> dict: """Generate a JSON manifest for a platform (same resolution as generate_pack). Returns a dict ready for JSON serialization with file inventory, - install hints, and download metadata. + install hints, and download metadata. ``fold_case`` overrides the + platform's ``case_insensitive_fs``, for the case variants below. """ config = load_platform_config(platform_name, platforms_dir) + if fold_case is not None: + config = {**config, "case_insensitive_fs": fold_case} if zip_contents is None: zip_contents = {} if emu_profiles is None: @@ -4165,6 +4173,28 @@ def generate_manifest( "system_cores": system_cores, "files": manifest_files, } + if case_insensitive and fold_case is None and any( + d.get("os") not in CASE_FOLDING_OS for d in detect + ): + # RetroArch also runs where case is kept: there Mesen-CE opens + # bios.col and np2kai np2kai/2608_bd.wav, which the folded list + # serves only as BIOS.col and 2608_BD.WAV. The installer adds these + # where the directory keeps case; the pack, extracted on Windows + # or macOS too, stays folded. + kept = generate_manifest( + platform_name, platforms_dir, db, bios_dir, registry_path, + emulators_dir, zip_contents, emu_profiles, target_cores, source, + regions, target_name, offline, required_only, data_registry, + fold_case=False, + ) + folded = {entry["dest"] for entry in manifest_files} + folded_lower = {dest.lower() for dest in folded} + variants = [ + entry for entry in kept["files"] + if entry["dest"] not in folded and entry["dest"].lower() in folded_lower + ] + if variants: + result["case_variants"] = variants return result diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 8252bc27..78a10cde 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -1871,7 +1871,7 @@ class EveryManifestEntryIsFetchable(unittest.TestCase): data = json.loads(path.read_text()) orphans = [ entry["dest"] - for entry in data.get("files", []) + for entry in data.get("files", []) + data.get("case_variants", []) if not entry.get("repo_path") and not entry.get("release_asset") ] self.assertEqual( @@ -1927,7 +1927,7 @@ class EveryManifestEntryIsFetchable(unittest.TestCase): data = json.loads(path.read_text(encoding="utf-8")) stale = [ (entry["dest"], entry["repo_path"], files.get(entry["sha1"], {}).get("path")) - for entry in data.get("files", []) + for entry in data.get("files", []) + data.get("case_variants", []) if entry.get("repo_path") and files.get(entry.get("sha1", ""), {}).get("path") != entry["repo_path"] ] @@ -1954,7 +1954,7 @@ class EveryManifestEntryIsFetchable(unittest.TestCase): with self.subTest(manifest=path.name): data = json.loads(path.read_text(encoding="utf-8")) wrong = [] - for entry in data.get("files", []): + for entry in data.get("files", []) + data.get("case_variants", []): repo_path = entry.get("repo_path") if not repo_path: continue diff --git a/tests/test_install.py b/tests/test_install.py index bfd39060..eed1fff3 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -2171,5 +2171,57 @@ class SudoAdviceNamesTheFolder(unittest.TestCase): self.assertIn("platform_of={path: plat for plat, path in platforms}", source) + +class CaseVariants(unittest.TestCase): + """The RetroArch list is folded for Windows and macOS: a Linux user got + BIOS.col and np2kai/2608_BD.WAV only, while Mesen-CE opens bios.col and + np2kai np2kai/2608_bd.wav. The manifest carries those as case_variants, + installed where the directory keeps case.""" + + def _manifest(self, variants): + entry = { + "dest": "BIOS.col", "sha1": "a" * 40, "sha256": "b" * 64, + "size": 8192, "repo_path": "bios/Coleco/BIOS.col", "cores": None, + } + return { + "manifest_version": 2, "platform": "retroarch", "files": [entry], + "total_files": 1, "total_size": 8192, "case_variants": variants, + } + + def test_a_kept_case_directory_is_detected(self): + directory = Path(_tempdir(self)) / "not" / "yet" + self.assertTrue(install.keeps_case(directory)) + self.assertEqual(list(Path(directory.parents[1]).iterdir()), []) + + def test_variants_cross_the_trust_boundary(self): + variant = { + "dest": "bios.col", "sha1": "a" * 40, "sha256": "b" * 64, + "size": 8192, "repo_path": "bios/Coleco/BIOS.col", "cores": ["mesence"], + } + install._validate_manifest(self._manifest([variant]), "retroarch") + for bad in ({**variant, "dest": "../x"}, {**variant, "dest": "BIOS.col"}): + with self.subTest(dest=bad["dest"]), self.assertRaises(ValueError): + install._validate_manifest(self._manifest([bad]), "retroarch") + + def test_main_adds_them_only_where_case_is_kept(self): + source = (REPO_ROOT / "install.py").read_text(encoding="utf-8") + compact = " ".join(source.split()) + self.assertIn( + 'variants = manifest.get("case_variants", []) ' + "if variants and keeps_case(bios_path):", compact, + ) + + def test_the_published_retroarch_list_carries_them(self): + manifest = json.loads((REPO_ROOT / "install" / "retroarch.json").read_text()) + variants = {e["dest"] for e in manifest.get("case_variants", [])} + self.assertIn("bios.col", variants) + lakka = json.loads((REPO_ROOT / "install" / "lakka.json").read_text()) + folded = {e["dest"] for e in manifest["files"]} + self.assertEqual( + {e["dest"] for e in lakka["files"]}, folded | variants, + "a case-keeping RetroArch install equals the Lakka list", + ) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md index bb7aacd0..4303a16d 100644 --- a/wiki/troubleshooting.md +++ b/wiki/troubleshooting.md @@ -86,6 +86,15 @@ Batocera's verification uses `casefold()` for case-insensitive matching, but the actual emulator may still require exact case. When in doubt, use the exact filename from the emulator profile. +The RetroArch pack is built for every system RetroArch runs on, so it holds one +name per file: `BIOS.col`, never `bios.col` beside it, since extracting both on +Windows or macOS would overwrite one with the other. On Linux, Mesen-CE opens +`bios.col` and NP2kai opens the lowercase `np2kai/2608_*.wav`, which the pack +only holds as `BIOS.col` and `2608_*.WAV`. The installer checks whether the +BIOS directory keeps case and, if it does, also fetches these names. If you +extract the ZIP on Linux by hand, take the Lakka pack instead: it is the +RetroArch pack with both spellings. + ## Hash mismatch / UNTESTED `verify.py` reports `UNTESTED` for a file.