diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index effa0246..36998516 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -168,14 +168,16 @@ jobs: ${PACKS} ### Install - Download, extract to your emulator's BIOS directory. + Download the pack matching your frontend, extract to the BIOS directory. - | Platform | Path | - |----------|------| - | RetroArch / Lakka | system/ | - | Batocera | /userdata/bios/ | - | Recalbox | /recalbox/share/bios/ | - | RetroBat | bios/ | + | Platform | Pack | Path | + |----------|------|------| + | RetroArch / Lakka | RetroArch_Lakka_BIOS_Pack.zip | system/ | + | Batocera | Batocera_BIOS_Pack.zip | /userdata/bios/ | + | Recalbox | Recalbox_BIOS_Pack.zip | /recalbox/share/bios/ | + | RetroBat | RetroBat_BIOS_Pack.zip | bios/ | + | RetroDECK | RetroDECK_BIOS_Pack.zip | ~/retrodeck/bios/ | + | EmuDeck | EmuDeck_BIOS_Pack.zip | Emulation/bios/ | ### Changes ${CHANGES} diff --git a/database.json b/database.json index cd8479c6..77b8497e 100644 --- a/database.json +++ b/database.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-03-25T05:56:02Z", + "generated_at": "2026-03-25T11:16:01Z", "total_files": 6733, "total_size": 5288644732, "files": { @@ -1163,16 +1163,6 @@ "crc32": "b28f7112", "adler32": "591b377c" }, - "ac4b78d53c7a97da2451ca35498395d8dd1e3024": { - "path": "bios/Arcade/Arcade/Firmware.19.0.0.zip", - "name": "Firmware.19.0.0.zip", - "size": 338076508, - "sha1": "ac4b78d53c7a97da2451ca35498395d8dd1e3024", - "md5": "72d6c73306c7f0b76723f989e7e1bdd1", - "sha256": "2f3791655e6c1b56f07a309b69ce8ea35d8412695599bbb6d4b0e29d1b044b66", - "crc32": "77228c84", - "adler32": "471a3291" - }, "5426d52e17e0ff9195fabbb42f704342e556d08e": { "path": "bios/Arcade/Arcade/acpsx.zip", "name": "acpsx.zip", @@ -1843,16 +1833,6 @@ "crc32": "065d69d0", "adler32": "3e9b6373" }, - "add40c002084e8e25768671877b2aa603aaf5cb1": { - "path": "bios/Arcade/Arcade/maclc3.zip", - "name": "maclc3.zip", - "size": 189428461, - "sha1": "add40c002084e8e25768671877b2aa603aaf5cb1", - "md5": "aff722788800df5b22d5a07cf8e558ee", - "sha256": "e663e456e88f475b3cacc06e75f50605e700789aa327b6648c627a560762a5d6", - "crc32": "81f21918", - "adler32": "dbd42440" - }, "4e0202f8430cb4842184df7b5418e32620156c7b": { "path": "bios/Arcade/Arcade/macsbios.zip", "name": "macsbios.zip", @@ -33253,16 +33233,6 @@ "crc32": "df558b58", "adler32": "4f2d77b8" }, - "b48f44194fe918aaaec5298861479512b581d661": { - "path": "bios/Nintendo/Nintendo DS/dsi_nand.bin", - "name": "dsi_nand.bin", - "size": 251658304, - "sha1": "b48f44194fe918aaaec5298861479512b581d661", - "md5": "dfafb1908da8f527df7a372e649b50be", - "sha256": "f57d9bf00529bec35d58404faff029a193fd2ccda0a83403ec4e6cc32626721b", - "crc32": "416bf51a", - "adler32": "3b3e7d56" - }, "3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3": { "path": "bios/Nintendo/Nintendo DS/dsi_sd_card.bin", "name": "dsi_sd_card.bin", @@ -65473,16 +65443,6 @@ "crc32": "2c3bcd32", "adler32": "26412551" }, - "093f8698b54b78dcb701de2043f82639de51d63b": { - "path": "bios/Sony/PlayStation 3/PS3UPDAT.PUP", - "name": "PS3UPDAT.PUP", - "size": 206126236, - "sha1": "093f8698b54b78dcb701de2043f82639de51d63b", - "md5": "05fe32f5dc8c78acbcd84d36ee7fdc5b", - "sha256": "69070a95780f59fc9e0d82bcf53eb9b28fd4ed4a7d54d0a40045f80422fd98d6", - "crc32": "24bdb2db", - "adler32": "1ec0b1c3" - }, "6bf1ae9fb01915966b715836253592cbf588b406": { "path": "bios/Sony/PlayStation Portable/Roboto-Condensed.ttf", "name": "Roboto-Condensed.ttf", @@ -66473,26 +66433,6 @@ "crc32": "c0c3a1fe", "adler32": "ea4fd486" }, - "ed3a4cb264fff283209f10ae58c96c6090fed187": { - "path": "bios/Sony/PlayStation Vita/PSP2UPDAT.PUP", - "name": "PSP2UPDAT.PUP", - "size": 56778752, - "sha1": "ed3a4cb264fff283209f10ae58c96c6090fed187", - "md5": "59dcf059d3328fb67be7e51f8aa33418", - "sha256": "c3c03fc7363dd573d90e5157629bf11551f434b283cc898d9ffc71dd716b791c", - "crc32": "082ecf86", - "adler32": "620a2ff1" - }, - "cc72dfcc964577cc29112ef368c28f55277c237c": { - "path": "bios/Sony/PlayStation Vita/PSVUPDAT.PUP", - "name": "PSVUPDAT.PUP", - "size": 133834240, - "sha1": "cc72dfcc964577cc29112ef368c28f55277c237c", - "md5": "f2c7b12fe85496ec88a0391b514d6e3b", - "sha256": "6ef6dc8da6db026f28647713e473486d770087a605c52a8d751bfca7478386cf", - "crc32": "39075d41", - "adler32": "75d71010" - }, "b184f1c1febf66c8168fcae0b8aa37a5754f79db": { "path": "bios/Synertek/SYM-1/SYM.ROM", "name": "SYM.ROM", @@ -67332,6 +67272,66 @@ "sha256": "2a86458696e83eb924fc6c6fda3ca5d320ca90885bd6c2f32d121757ade389bb", "crc32": "74b76447", "adler32": "701e6531" + }, + "ac4b78d53c7a97da2451ca35498395d8dd1e3024": { + "path": "bios/Arcade/Arcade/Firmware.19.0.0.zip", + "name": "Firmware.19.0.0.zip", + "size": 338076508, + "sha1": "ac4b78d53c7a97da2451ca35498395d8dd1e3024", + "md5": "72d6c73306c7f0b76723f989e7e1bdd1", + "sha256": "2f3791655e6c1b56f07a309b69ce8ea35d8412695599bbb6d4b0e29d1b044b66", + "crc32": "77228c84", + "adler32": "471a3291" + }, + "add40c002084e8e25768671877b2aa603aaf5cb1": { + "path": "bios/Arcade/Arcade/maclc3.zip", + "name": "maclc3.zip", + "size": 189428461, + "sha1": "add40c002084e8e25768671877b2aa603aaf5cb1", + "md5": "aff722788800df5b22d5a07cf8e558ee", + "sha256": "e663e456e88f475b3cacc06e75f50605e700789aa327b6648c627a560762a5d6", + "crc32": "81f21918", + "adler32": "dbd42440" + }, + "b48f44194fe918aaaec5298861479512b581d661": { + "path": "bios/Nintendo/Nintendo DS/dsi_nand.bin", + "name": "dsi_nand.bin", + "size": 251658304, + "sha1": "b48f44194fe918aaaec5298861479512b581d661", + "md5": "dfafb1908da8f527df7a372e649b50be", + "sha256": "f57d9bf00529bec35d58404faff029a193fd2ccda0a83403ec4e6cc32626721b", + "crc32": "416bf51a", + "adler32": "3b3e7d56" + }, + "093f8698b54b78dcb701de2043f82639de51d63b": { + "path": "bios/Sony/PlayStation 3/PS3UPDAT.PUP", + "name": "PS3UPDAT.PUP", + "size": 206126236, + "sha1": "093f8698b54b78dcb701de2043f82639de51d63b", + "md5": "05fe32f5dc8c78acbcd84d36ee7fdc5b", + "sha256": "69070a95780f59fc9e0d82bcf53eb9b28fd4ed4a7d54d0a40045f80422fd98d6", + "crc32": "24bdb2db", + "adler32": "1ec0b1c3" + }, + "ed3a4cb264fff283209f10ae58c96c6090fed187": { + "path": "bios/Sony/PlayStation Vita/PSP2UPDAT.PUP", + "name": "PSP2UPDAT.PUP", + "size": 56778752, + "sha1": "ed3a4cb264fff283209f10ae58c96c6090fed187", + "md5": "59dcf059d3328fb67be7e51f8aa33418", + "sha256": "c3c03fc7363dd573d90e5157629bf11551f434b283cc898d9ffc71dd716b791c", + "crc32": "082ecf86", + "adler32": "620a2ff1" + }, + "cc72dfcc964577cc29112ef368c28f55277c237c": { + "path": "bios/Sony/PlayStation Vita/PSVUPDAT.PUP", + "name": "PSVUPDAT.PUP", + "size": 133834240, + "sha1": "cc72dfcc964577cc29112ef368c28f55277c237c", + "md5": "f2c7b12fe85496ec88a0391b514d6e3b", + "sha256": "6ef6dc8da6db026f28647713e473486d770087a605c52a8d751bfca7478386cf", + "crc32": "39075d41", + "adler32": "75d71010" } }, "indexes": { @@ -67452,7 +67452,6 @@ "fcb298d97792b9e9bdd3296cc6be10b6": "eb2a867578a05bbf8741e9fe7204301062df0cb8", "ddb8aacffffffa608ddbb4a6d6dda5ec": "0b6519209766ed883e3fca4c61bf866804c89004", "6c6c0c726cbf15e81785eb7592fdb51c": "de463b0577dfd1027bf7de523ff67a0fff861cdb", - "72d6c73306c7f0b76723f989e7e1bdd1": "ac4b78d53c7a97da2451ca35498395d8dd1e3024", "fcb631bf18a56f2d5b077fa846bab4a6": "5426d52e17e0ff9195fabbb42f704342e556d08e", "3f348c88af99a40fbd11fa435f28c69d": "e18c5e9ca21654dfd724aa54e625b386e6ffb2c5", "c266fc58905af1e246dffadc84301042": "beaf97c4a0e0792b8db65648f9dabb6a54ae0549", @@ -67520,7 +67519,6 @@ "7b51d463324b6bf26e86c4afb7316a3a": "8cf0aa7f9dca4d77485e605fb0e2173a734633bf", "7a14456d6e8afaf540f2368fead25f26": "78c8e1c3c033b65758b7e53a9346b44d037fea7f", "d048a9ff941041de45c26474a0da40aa": "65a2f2cee74c316d5f40b68deda66787609df353", - "aff722788800df5b22d5a07cf8e558ee": "add40c002084e8e25768671877b2aa603aaf5cb1", "34530e248d96e7171af19155af315378": "4e0202f8430cb4842184df7b5418e32620156c7b", "b48fb4fb35dc348f4904a318dbf9a712": "697551fcf9557ae33e31096b118a0c6769700a2e", "6bb005f55ba39bc5f6b330b663da1a58": "c9ee16e26e03496195a7bff151efbdd89da01204", @@ -70661,7 +70659,6 @@ "bfd8292fbf0a251647a23c5cb310a97a": "f1ad917e0affaeb8d2114c7ecd02b9f938c3cbd9", "94bc5094607c5e6598d50472c52f27f2": "1cf9e67c2c703bb9961bbcdd39cd2c7e319a803b", "8daa89fd280b3e5ec79fbab73ad6684e": "d2a5af338f09c5cbdd5d7628db5b9c075c69b616", - "dfafb1908da8f527df7a372e649b50be": "b48f44194fe918aaaec5298861479512b581d661", "b6d81b360a5672d80c27430f39153e2c": "3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3", "74f23348012d7b3e1cc216c47192ffeb": "3773f52559d5ac4fc6d8aefe35bce58730ae8181", "e45033d9b0fa6b0de071292bba7c9d13": "cfe072921ee3fb93f688743f8beef89043c3e9ad", @@ -73883,7 +73880,6 @@ "5dea62f70439682a6cee16ba3823d11e": "6eddec30056cde7c664a0cf508dcad29353a12bb", "de6da198a7359d1200c3eeb6df9c7eda": "40ecf6138c99a0aba775ef93240b295025a45500", "44552702b05697a14ccbe2ca22ee7139": "47d2ec4b342649e4c391043ab915d4435f9d180d", - "05fe32f5dc8c78acbcd84d36ee7fdc5b": "093f8698b54b78dcb701de2043f82639de51d63b", "55caa30ec34ef081ded15615db54eafe": "6bf1ae9fb01915966b715836253592cbf588b406", "a062688b08c70a42ff2a0acff6c46d93": "08325554623568bb9babadc10213bfc0b1151766", "ad0542e2956a8dddf52357f28a8a7d9c": "9d6c9874c1d6a0c57a1345f211154fe1e494b55a", @@ -73983,8 +73979,6 @@ "331d6806a56d7370515d94a66616eca6": "78632d0fe9dd77bf9a2264f192fae6f0af03a71c", "c96bb62586bf81dd6237c417f8cf3bb8": "18985a2079c7570c13cf39e0d001eef87538cd15", "8b5f60b56c3da8365b973dba570c53a5": "3ae832c9800fcaa007eccfc48f24242967c111f8", - "59dcf059d3328fb67be7e51f8aa33418": "ed3a4cb264fff283209f10ae58c96c6090fed187", - "f2c7b12fe85496ec88a0391b514d6e3b": "cc72dfcc964577cc29112ef368c28f55277c237c", "e59fdf56762c480ba4dfe1b3ec5fb86d": "b184f1c1febf66c8168fcae0b8aa37a5754f79db", "1d33d70f35b33873fc75941d95ad1ffa": "567c5b5054552a2771eafa7966844a146f0dde96", "b81dc552536796d234c08587bac7be43": "f2fa8d8e940f1d91a1b1624013df5dca0bb1ee44", @@ -74068,7 +74062,13 @@ "6f68e4baf89c8ee4623c19617319184b": "cee76080884af97c20059da0eb1ca956a835f3d0", "2010e5b85f9e1d60685ccb3d84a17115": "c7cc306fb921754ba00794153292d533cf0765ef", "39e5bc84ce9aac3a2d297d8aeb2a0d05": "22bcfeb5b6c6481569b90db96aa3f4b5f06c8848", - "a471e64e9f69afbe59c10cc94ed1b184": "ecfc092fe6371dbf38e238a8ba5f90785b5db52d" + "a471e64e9f69afbe59c10cc94ed1b184": "ecfc092fe6371dbf38e238a8ba5f90785b5db52d", + "72d6c73306c7f0b76723f989e7e1bdd1": "ac4b78d53c7a97da2451ca35498395d8dd1e3024", + "aff722788800df5b22d5a07cf8e558ee": "add40c002084e8e25768671877b2aa603aaf5cb1", + "dfafb1908da8f527df7a372e649b50be": "b48f44194fe918aaaec5298861479512b581d661", + "05fe32f5dc8c78acbcd84d36ee7fdc5b": "093f8698b54b78dcb701de2043f82639de51d63b", + "59dcf059d3328fb67be7e51f8aa33418": "ed3a4cb264fff283209f10ae58c96c6090fed187", + "f2c7b12fe85496ec88a0391b514d6e3b": "cc72dfcc964577cc29112ef368c28f55277c237c" }, "by_name": { "3do_arcade_saot.bin": [ @@ -74428,9 +74428,6 @@ "de463b0577dfd1027bf7de523ff67a0fff861cdb", "12516c82f52a8abb252fba754f95b7952c295e6f" ], - "Firmware.19.0.0.zip": [ - "ac4b78d53c7a97da2451ca35498395d8dd1e3024" - ], "acpsx.zip": [ "5426d52e17e0ff9195fabbb42f704342e556d08e" ], @@ -83271,9 +83268,6 @@ "dsi_firmware.bin": [ "d2a5af338f09c5cbdd5d7628db5b9c075c69b616" ], - "dsi_nand.bin": [ - "b48f44194fe918aaaec5298861479512b581d661" - ], "dsi_sd_card.bin": [ "3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3" ], @@ -92088,9 +92082,6 @@ "rom1.bin": [ "47d2ec4b342649e4c391043ab915d4435f9d180d" ], - "PS3UPDAT.PUP": [ - "093f8698b54b78dcb701de2043f82639de51d63b" - ], "Roboto-Condensed.ttf": [ "6bf1ae9fb01915966b715836253592cbf588b406" ], @@ -92386,9 +92377,6 @@ "3ae832c9800fcaa007eccfc48f24242967c111f8", "ed3a4cb264fff283209f10ae58c96c6090fed187" ], - "PSVUPDAT.PUP": [ - "cc72dfcc964577cc29112ef368c28f55277c237c" - ], "coco.zip": [ "567c5b5054552a2771eafa7966844a146f0dde96", "e31fbbb831f32886e57410183bc4e85d36c2dc7a" @@ -92623,6 +92611,18 @@ "data.zip": [ "ecfc092fe6371dbf38e238a8ba5f90785b5db52d" ], + "Firmware.19.0.0.zip": [ + "ac4b78d53c7a97da2451ca35498395d8dd1e3024" + ], + "dsi_nand.bin": [ + "b48f44194fe918aaaec5298861479512b581d661" + ], + "PS3UPDAT.PUP": [ + "093f8698b54b78dcb701de2043f82639de51d63b" + ], + "PSVUPDAT.PUP": [ + "cc72dfcc964577cc29112ef368c28f55277c237c" + ], "disk2-16boot.rom": [ "d4181c9f046aafc3fb326b381baac809d9e38d16" ], @@ -95473,7 +95473,6 @@ "ad37f2de": "eb2a867578a05bbf8741e9fe7204301062df0cb8", "b0e03aa6": "0b6519209766ed883e3fca4c61bf866804c89004", "b28f7112": "de463b0577dfd1027bf7de523ff67a0fff861cdb", - "77228c84": "ac4b78d53c7a97da2451ca35498395d8dd1e3024", "9c9601ca": "5426d52e17e0ff9195fabbb42f704342e556d08e", "2c87c283": "e18c5e9ca21654dfd724aa54e625b386e6ffb2c5", "da9beacc": "beaf97c4a0e0792b8db65648f9dabb6a54ae0549", @@ -95541,7 +95540,6 @@ "ce2a2c77": "8cf0aa7f9dca4d77485e605fb0e2173a734633bf", "7325a518": "78c8e1c3c033b65758b7e53a9346b44d037fea7f", "065d69d0": "65a2f2cee74c316d5f40b68deda66787609df353", - "81f21918": "add40c002084e8e25768671877b2aa603aaf5cb1", "946c6bb8": "4e0202f8430cb4842184df7b5418e32620156c7b", "bfce92a3": "697551fcf9557ae33e31096b118a0c6769700a2e", "7c57bff1": "c9ee16e26e03496195a7bff151efbdd89da01204", @@ -98682,7 +98680,6 @@ "7389b815": "f1ad917e0affaeb8d2114c7ecd02b9f938c3cbd9", "62efc03d": "1cf9e67c2c703bb9961bbcdd39cd2c7e319a803b", "df558b58": "d2a5af338f09c5cbdd5d7628db5b9c075c69b616", - "416bf51a": "b48f44194fe918aaaec5298861479512b581d661", "a738ea1c": "3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3", "6f6d289b": "3773f52559d5ac4fc6d8aefe35bce58730ae8181", "945f9dc9": "cfe072921ee3fb93f688743f8beef89043c3e9ad", @@ -101904,7 +101901,6 @@ "0b805686": "6eddec30056cde7c664a0cf508dcad29353a12bb", "a3e573a0": "40ecf6138c99a0aba775ef93240b295025a45500", "2c3bcd32": "47d2ec4b342649e4c391043ab915d4435f9d180d", - "24bdb2db": "093f8698b54b78dcb701de2043f82639de51d63b", "72d2fd97": "6bf1ae9fb01915966b715836253592cbf588b406", "c48bf84b": "08325554623568bb9babadc10213bfc0b1151766", "2b629346": "9d6c9874c1d6a0c57a1345f211154fe1e494b55a", @@ -102004,8 +102000,6 @@ "616f5e40": "78632d0fe9dd77bf9a2264f192fae6f0af03a71c", "92ddefae": "18985a2079c7570c13cf39e0d001eef87538cd15", "c0c3a1fe": "3ae832c9800fcaa007eccfc48f24242967c111f8", - "082ecf86": "ed3a4cb264fff283209f10ae58c96c6090fed187", - "39075d41": "cc72dfcc964577cc29112ef368c28f55277c237c", "44295096": "b184f1c1febf66c8168fcae0b8aa37a5754f79db", "31c53421": "567c5b5054552a2771eafa7966844a146f0dde96", "d13aefe2": "f2fa8d8e940f1d91a1b1624013df5dca0bb1ee44", @@ -102089,7 +102083,13 @@ "8419990c": "cee76080884af97c20059da0eb1ca956a835f3d0", "3cacb086": "c7cc306fb921754ba00794153292d533cf0765ef", "0a4e2e07": "22bcfeb5b6c6481569b90db96aa3f4b5f06c8848", - "74b76447": "ecfc092fe6371dbf38e238a8ba5f90785b5db52d" + "74b76447": "ecfc092fe6371dbf38e238a8ba5f90785b5db52d", + "77228c84": "ac4b78d53c7a97da2451ca35498395d8dd1e3024", + "81f21918": "add40c002084e8e25768671877b2aa603aaf5cb1", + "416bf51a": "b48f44194fe918aaaec5298861479512b581d661", + "24bdb2db": "093f8698b54b78dcb701de2043f82639de51d63b", + "082ecf86": "ed3a4cb264fff283209f10ae58c96c6090fed187", + "39075d41": "cc72dfcc964577cc29112ef368c28f55277c237c" }, "by_path_suffix": { ".variants/aa310.zip": [ diff --git a/scripts/common.py b/scripts/common.py index e9dfe5b6..656958cb 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -416,16 +416,23 @@ def group_identical_platforms( """Group platforms that produce identical packs (same files + base_destination). Returns [(group_of_platform_names, representative), ...]. + The representative is the root platform (one that does not inherit). """ fingerprints: dict[str, list[str]] = {} representatives: dict[str, str] = {} + inherits: dict[str, bool] = {} for platform in platforms: try: + raw_path = os.path.join(platforms_dir, f"{platform}.yml") + with open(raw_path) as f: + raw = yaml.safe_load(f) or {} + inherits[platform] = "inherits" in raw config = load_platform_config(platform, platforms_dir) except FileNotFoundError: fingerprints.setdefault(platform, []).append(platform) representatives.setdefault(platform, platform) + inherits[platform] = False continue base_dest = config.get("base_destination", "") @@ -440,9 +447,16 @@ def group_identical_platforms( fp = hashlib.sha1("|".join(sorted(entries)).encode()).hexdigest() fingerprints.setdefault(fp, []).append(platform) - representatives.setdefault(fp, platform) + # Prefer the root platform (no inherits) as representative + if fp not in representatives or (not inherits[platform] and inherits.get(representatives[fp], False)): + representatives[fp] = platform - return [(group, representatives[fp]) for fp, group in fingerprints.items()] + result = [] + for fp, group in fingerprints.items(): + rep = representatives[fp] + ordered = [rep] + [p for p in group if p != rep] + result.append((ordered, rep)) + return result def resolve_platform_cores( @@ -537,8 +551,10 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: "min_size": None, "max_size": None, "crc32": set(), "md5": set(), "sha1": set(), "sha256": set(), "adler32": set(), "crypto_only": set(), + "emulators": set(), } sources[fname] = {} + index[fname]["emulators"].add(emu_name) index[fname]["checks"].update(checks) # Track non-reproducible crypto checks index[fname]["crypto_only"].update( @@ -584,6 +600,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: for v in index.values(): v["checks"] = sorted(v["checks"]) v["crypto_only"] = sorted(v["crypto_only"]) + v["emulators"] = sorted(v["emulators"]) # Keep hash sets as frozensets for O(1) lookup in check_file_validation return index diff --git a/scripts/generate_db.py b/scripts/generate_db.py index b9df8c9c..5098b887 100644 --- a/scripts/generate_db.py +++ b/scripts/generate_db.py @@ -216,6 +216,46 @@ def save_cache(cache_path: str, cache: dict): json.dump(cache, f) +def _load_gitignored_bios_paths() -> set[str]: + """Read .gitignore and return bios/ paths that are listed (large files).""" + gitignore = Path(".gitignore") + if not gitignore.exists(): + return set() + paths = set() + for line in gitignore.read_text().splitlines(): + line = line.strip() + if line.startswith("bios/") and not line.startswith("#"): + paths.add(line) + return paths + + +def _preserve_large_file_entries(files: dict, db_path: str) -> int: + """Preserve database entries for large files not on disk. + + Large files (>50 MB) are stored as GitHub release assets and listed + in .gitignore. When generate_db runs locally without them, their + entries would be lost. This reads the existing database and re-adds + entries whose paths match .gitignore bios/ entries. + """ + gitignored = _load_gitignored_bios_paths() + if not gitignored: + return 0 + + try: + with open(db_path) as f: + existing_db = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return 0 + + count = 0 + for sha1, entry in existing_db.get("files", {}).items(): + path = entry.get("path", "") + if path in gitignored and sha1 not in files: + files[sha1] = entry + count += 1 + return count + + def main(): parser = argparse.ArgumentParser(description="Generate multi-indexed BIOS database") parser.add_argument("--force", action="store_true", help="Force rehash all files") @@ -236,6 +276,11 @@ def main(): if not files: print("Warning: No BIOS files found", file=sys.stderr) + # Preserve entries for large files stored as release assets (.gitignore) + preserved = _preserve_large_file_entries(files, args.output) + if preserved: + print(f" Preserved {preserved} large file entries from existing database") + platform_aliases = _collect_all_aliases(files) for sha1, name_list in platform_aliases.items(): for alias_entry in name_list: diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index f32803f3..3523146d 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -94,6 +94,47 @@ def fetch_large_file(name: str, dest_dir: str = ".cache/large", return cached +def _find_candidate_satisfying_both( + file_entry: dict, + db: dict, + local_path: str, + validation_index: dict, + bios_dir: str, +) -> str | None: + """Search for a repo file that satisfies both platform MD5 and emulator validation. + + When the current file passes platform verification but fails emulator checks, + search all candidates with the same name for one that passes both. + Returns a better path, or None if no upgrade found. + """ + fname = file_entry.get("name", "") + if not fname: + return None + entry = validation_index.get(fname) + if not entry: + return None + + md5_expected = file_entry.get("md5", "") + md5_set = {m.strip().lower() for m in md5_expected.split(",") if m.strip()} if md5_expected else set() + + by_name = db.get("indexes", {}).get("by_name", {}) + files_db = db.get("files", {}) + + for sha1 in by_name.get(fname, []): + candidate = files_db.get(sha1, {}) + path = candidate.get("path", "") + if not path or not os.path.exists(path) or os.path.realpath(path) == os.path.realpath(local_path): + continue + # Must still satisfy platform MD5 + if md5_set and candidate.get("md5", "").lower() not in md5_set: + continue + # Check emulator validation + reason = check_file_validation(path, fname, validation_index, bios_dir) + if reason is None: + return path + return None + + def _sanitize_path(raw: str) -> str: """Strip path traversal components from a relative path.""" raw = raw.replace("\\", "/") @@ -118,10 +159,11 @@ def resolve_file(file_entry: dict, db: dict, bios_dir: str, path, status = resolve_local_file(file_entry, db, zip_contents, dest_hint=dest_hint) - if path: + if path and status != "hash_mismatch": return path, status - # Last resort: large files from GitHub release assets + # Large files from GitHub release assets — tried when local file is + # missing OR has a hash mismatch (wrong variant on disk) name = file_entry.get("name", "") sha1 = file_entry.get("sha1") md5_raw = file_entry.get("md5", "") @@ -131,6 +173,10 @@ def resolve_file(file_entry: dict, db: dict, bios_dir: str, if cached: return cached, "release_asset" + # Fall back to hash_mismatch local file if release asset unavailable + if path: + return path, status + return None, "not_found" @@ -362,20 +408,28 @@ def generate_pack( else: file_status.setdefault(dedup_key, "ok") - # Emulator-level validation (matches verify.py behavior) - # In existence mode: validation is informational (warning, not downgrade) - # In md5 mode: validation downgrades OK to UNTESTED + # Emulator-level validation: informational only for platform packs. + # Platform verification (existence/md5) is the authority for pack status. + # Emulator checks are supplementary — logged but don't downgrade. + # When a discrepancy is found, try to find a file satisfying both. if (file_status.get(dedup_key) == "ok" and local_path and validation_index): fname = file_entry.get("name", "") - reason = check_file_validation(local_path, fname, validation_index) + reason = check_file_validation(local_path, fname, validation_index, + bios_dir) if reason: - if verification_mode == "existence": - # Existence mode: file present = OK, validation is extra info - file_reasons.setdefault(dedup_key, reason) + better = _find_candidate_satisfying_both( + file_entry, db, local_path, validation_index, bios_dir, + ) + if better: + local_path = better else: - file_status[dedup_key] = "untested" - file_reasons[dedup_key] = reason + ventry = validation_index.get(fname, {}) + emus = ", ".join(ventry.get("emulators", [])) + file_reasons.setdefault( + dedup_key, + f"{platform_display} says OK but {emus} says {reason}", + ) if already_packed: continue @@ -475,7 +529,7 @@ def generate_pack( for key, reason in sorted(file_reasons.items()): status = file_status.get(key, "") - label = "UNTESTED" + label = "UNTESTED" if status == "untested" else "DISCREPANCY" print(f" {label}: {key} — {reason}") for name in missing_files: print(f" MISSING: {name}") @@ -915,10 +969,11 @@ def main(): groups = group_identical_platforms(platforms, args.platforms_dir) for group_platforms, representative in groups: - if len(group_platforms) > 1: - names = [load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms] - combined_name = " + ".join(names) - print(f"\nGenerating shared pack for {combined_name}...") + variants = [p for p in group_platforms if p != representative] + if variants: + all_names = [load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms] + label = " / ".join(all_names) + print(f"\nGenerating pack for {label}...") else: print(f"\nGenerating pack for {representative}...") @@ -929,10 +984,10 @@ def main(): zip_contents=zip_contents, data_registry=data_registry, emu_profiles=emu_profiles, ) - if zip_path and len(group_platforms) > 1: - names = [load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms] - combined_filename = "_".join(n.replace(" ", "") for n in names) + "_BIOS_Pack.zip" - new_path = os.path.join(os.path.dirname(zip_path), combined_filename) + if zip_path and variants: + all_names = [load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms] + combined = "_".join(n.replace(" ", "") for n in all_names) + "_BIOS_Pack.zip" + new_path = os.path.join(os.path.dirname(zip_path), combined) if new_path != zip_path: os.rename(zip_path, new_path) print(f" Renamed -> {os.path.basename(new_path)}")