mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-04-13 04:12:33 -05:00
Compare commits
16 Commits
e5859eb761
...
cbb86c7746
| Author | SHA1 | Date | |
|---|---|---|---|
|
cbb86c7746 | ||
|
|
9bbd39369d | ||
|
|
a1333137a0 | ||
|
|
1efe95228f | ||
|
|
074e3371f2 | ||
|
|
85cc23398a | ||
|
|
47a68c1a11 | ||
|
|
5f579d1851 | ||
|
|
2c1c2a7bfe | ||
|
|
423a1b201e | ||
|
|
9c6b3dfe96 | ||
|
|
b070fa41de | ||
|
|
0a272dc4e9 | ||
|
|
a2d30557e4 | ||
|
|
0e6db8abdf | ||
|
|
6eca4c416a |
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "batocera",
|
||||
"display_name": "Batocera",
|
||||
"version": "1.0",
|
||||
"generated": "2026-03-31T21:00:28Z",
|
||||
"generated": "2026-04-01T14:41:41Z",
|
||||
"base_destination": "bios",
|
||||
"detect": [
|
||||
{
|
||||
@@ -14,8 +15,8 @@
|
||||
}
|
||||
],
|
||||
"standalone_copies": [],
|
||||
"total_files": 1523,
|
||||
"total_size": 3888059911,
|
||||
"total_files": 1540,
|
||||
"total_size": 3891615271,
|
||||
"files": [
|
||||
{
|
||||
"dest": "panafz1.bin",
|
||||
@@ -2588,6 +2589,15 @@
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick31.rom",
|
||||
"sha1": "3b7f1493b27e212830f989f26ca76c02049f09ca",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/kick31.rom",
|
||||
"cores": [
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick.rom",
|
||||
"sha1": "891e9a547772fe0c6c19b610baf8bc4ea7fcb785",
|
||||
@@ -4109,6 +4119,15 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "Amiga/Kickstart-v1.3-rev34.5-1987-Commodore-A500-A1000-A2000-CDTV.rom",
|
||||
"sha1": "891e9a547772fe0c6c19b610baf8bc4ea7fcb785",
|
||||
"size": 262144,
|
||||
"repo_path": "bios/Commodore/Amiga/Kickstart-v1.3-rev34.5-1987-Commodore-A500-A1000-A2000-CDTV.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AmstradCPC/os464.rom",
|
||||
"sha1": "475c8080065a7aa9984daca0415a3d70a5305be2",
|
||||
@@ -4163,6 +4182,15 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AmstradCPC/amsdos.rom",
|
||||
"sha1": "39102c8e9cb55fcc0b9b62098780ed4a3cb6a4bb",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Amstrad/CPC/amsdos.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleII/apple2o.rom",
|
||||
"sha1": "78008be557f643e956a106121bcc182c0fb9ea6d",
|
||||
@@ -4199,6 +4227,33 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleII/apple2-character.rom",
|
||||
"sha1": "f9d312f128c9557d9d6ac03bfad6c3ddf83e5659",
|
||||
"size": 2048,
|
||||
"repo_path": "bios/Apple/Apple II/apple2-character.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleII/apple2eu-character.rom",
|
||||
"sha1": "7060de104046736529c1e8a687a0dd7b84f8c51b",
|
||||
"size": 4096,
|
||||
"repo_path": "bios/Apple/Apple II/apple2eu-character.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleII/apple2e-character.rom",
|
||||
"sha1": "b2b5d87f52693817fc747df087a4aa1ddcdb1f10",
|
||||
"size": 4096,
|
||||
"repo_path": "bios/Apple/Apple II/apple2e-character.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleII/scsi.rom",
|
||||
"sha1": "3d7166f05daad1b022fa04c2569e788580158095",
|
||||
@@ -4226,6 +4281,24 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleIIgs/apple2gs.chr",
|
||||
"sha1": "34e2443e2ef960a36c047a09ed5a93f471797f89",
|
||||
"size": 4096,
|
||||
"repo_path": "bios/Apple/Apple II/apple2gs.chr",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleIIgs/341s0632-2",
|
||||
"sha1": "141d18c36a617ab9dce668445440d34354be0672",
|
||||
"size": 4096,
|
||||
"repo_path": "bios/Apple/Apple II/341s0632-2",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AtariST/tos100.img",
|
||||
"sha1": "9a6e4c88533a9eaa4d55cdc040e47443e0226eb2",
|
||||
@@ -4262,6 +4335,15 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "DiskII/state-machine-16.rom",
|
||||
"sha1": "bc39fbd5b9a8d2287ac5d0a42e639fc4d3c2f9d4",
|
||||
"size": 256,
|
||||
"repo_path": "bios/Apple/Apple II/DiskII/state-machine-16.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "DiskII/boot-13.rom",
|
||||
"sha1": "afd060e6f35faf3bb0146fa889fc787adf56330a",
|
||||
@@ -4289,6 +4371,15 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "Enterprise/exos20.bin",
|
||||
"sha1": "6033a0535136c40c47137e4d1cd9273c06d5fdff",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Enterprise/64-128/exos20.bin",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "Enterprise/exos23.bin",
|
||||
"sha1": "c6241e1c248193108ce38b9a8e9dd33972cf47ba",
|
||||
@@ -4415,6 +4506,15 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "MSX/msx-japanese.rom",
|
||||
"sha1": "302afb5d8be26c758309ca3df611ae69cced2821",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Microsoft/MSX/CLK/msx-japanese.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "MSX/msx-american.rom",
|
||||
"sha1": "3656bb3bbc17d280d2016fe4f6ff3cded3082a41",
|
||||
@@ -5164,6 +5264,15 @@
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1001_v20.bin",
|
||||
"sha1": "649895efd79d14790eabb362e94eb0622093dfb9",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph1001_v20.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph3500.bin",
|
||||
"sha1": "e38466a4ba8005fba7e9e3c7b9efeba7205bee3f",
|
||||
@@ -6084,6 +6193,15 @@
|
||||
"fMSX"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "MSX2PEXT.ROM",
|
||||
"sha1": "fe0254cbfc11405b79e7c86c7769bd6322b04995",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Microsoft/MSX/MSX2PEXT.ROM",
|
||||
"cores": [
|
||||
"fMSX"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "DISK.ROM",
|
||||
"sha1": "032cb1c1c75b9a191fa1230978971698d9d2a17f",
|
||||
@@ -6651,6 +6769,15 @@
|
||||
"Genesis Plus GX"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ROM",
|
||||
"sha1": "e4fc7560b69d062cb2da5b1ffbe11cd1ca03cc37",
|
||||
"size": 131072,
|
||||
"repo_path": "bios/Apple/Apple IIGS/ROM",
|
||||
"cores": [
|
||||
"GSplus"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "c600.rom",
|
||||
"sha1": "d4181c9f046aafc3fb326b381baac809d9e38d16",
|
||||
@@ -8406,6 +8533,15 @@
|
||||
"PicoDrive"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "SegaCDBIOS9303.bin",
|
||||
"sha1": "5adb6c3af218c60868e6b723ec47e36bbdf5e6f0",
|
||||
"size": 131072,
|
||||
"repo_path": "bios/Sega/Mega CD/SegaCDBIOS9303.bin",
|
||||
"cores": [
|
||||
"PicoDrive"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "us_scd1_9210.bin",
|
||||
"sha1": "f4f315adcef9b8feb0364c21ab7f0eaf5457f3ed",
|
||||
@@ -8703,6 +8839,15 @@
|
||||
"QUASI88"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "dc_bios.bin",
|
||||
"sha1": "8951d1bb219ab2ff8583033d2119c899cc81f18c",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Sega/Dreamcast/dc_bios.bin",
|
||||
"cores": [
|
||||
"Redream"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "same_cdi/bios/cdimono2.zip",
|
||||
"sha1": "9492247203b71c12d88fad0a5437376941c7870a",
|
||||
@@ -8856,6 +9001,15 @@
|
||||
"shadps4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "BIOS.col",
|
||||
"sha1": "45bedc4cbdeac66c7df59e9e599195c778d86a92",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Coleco/ColecoVision/BIOS.col",
|
||||
"cores": [
|
||||
"SMS Plus GX"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "squirreljme-0.3.0-fast.jar",
|
||||
"sha1": "7c4cd0a5451eedeac9b328f48408dbc312198ccf",
|
||||
@@ -9072,6 +9226,15 @@
|
||||
"Stella 2023"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "MYTOWNS.ROM",
|
||||
"sha1": "e245f8086df57ce6e48853f0e13525f738e5c4d8",
|
||||
"size": 32,
|
||||
"repo_path": "bios/Fujitsu/FM Towns/MYTOWNS.ROM",
|
||||
"cores": [
|
||||
"tsugaru"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "FMT_ALL.ROM",
|
||||
"sha1": "262aae14f334bc21499f7e2bfe8b7ec1079b1e04",
|
||||
@@ -10152,6 +10315,15 @@
|
||||
"Vircon32"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/IPLROM.X1T",
|
||||
"sha1": "44620f57a25f0bcac2b57ca2b0f1ebad3bf305d3",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Sharp/X1/.variants/IPLROM.X1T.44620f57",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/FNT0808.X1",
|
||||
"sha1": "1c1a0d8c9f4c446ccd7470516b215ddca5052fb2",
|
||||
@@ -10269,6 +10441,24 @@
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "bas13.rom",
|
||||
"sha1": "28b92bebe35fa4f026a084416d6ea3b1552b63d3",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/bas13.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "bas12.rom",
|
||||
"sha1": "0f14dc46c647510eb0b7bd3f53e33da07907d04f",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/bas12.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "bas11.rom",
|
||||
"sha1": "cecb7c24ff1e0ab5836e4a7a8eb1b8e01f1fded3",
|
||||
@@ -10278,6 +10468,24 @@
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "bas10.rom",
|
||||
"sha1": "1f08455cd48ce6a06132aea15c4778f264e19539",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/bas10.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "extbas11.rom",
|
||||
"sha1": "ad927fb4f30746d820cb8b860ebb585e7f095dea",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/extbas11.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "extbas10.rom",
|
||||
"sha1": "7275f1e3f165ff6a4657e4e5e24cb8b817239f54",
|
||||
@@ -10287,6 +10495,42 @@
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "coco3.rom",
|
||||
"sha1": "e0d82953fb6fd03768604933df1ce8bc51fc427d",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Tandy/CoCo/coco3.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "coco3p.rom",
|
||||
"sha1": "631e383068b1f52a8f419f4114b69501b21cf379",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Tandy/CoCo/coco3p.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "mc10.rom",
|
||||
"sha1": "4afff2b4c120334481aab7b02c3552bf76f1bc43",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/mc10.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "alice.rom",
|
||||
"sha1": "c2166b91e6396a311f486832012aa43e0d2b19f8",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/alice.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "deluxe.rom",
|
||||
"sha1": "d89196292b9ebd787647cf91bbb83c63da2b4390",
|
||||
@@ -10359,6 +10603,15 @@
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "disk11.rom",
|
||||
"sha1": "10bdc5aa2d7d7f205f67b47b19003a4bd89defd1",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/disk11.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "hdbdw3bck.rom",
|
||||
"sha1": "8fd64f1c246489e0bf2b3743ae76332ff324716a",
|
||||
@@ -10989,69 +11242,6 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "Amiga/Kickstart-v1.3-rev34.5-1987-Commodore-A500-A1000-A2000-CDTV.rom",
|
||||
"sha1": "891e9a547772fe0c6c19b610baf8bc4ea7fcb785",
|
||||
"size": 262144,
|
||||
"repo_path": "bios/Commodore/Amiga/Kickstart-v1.3-rev34.5-1987-Commodore-A500-A1000-A2000-CDTV.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AmstradCPC/amsdos.rom",
|
||||
"sha1": "39102c8e9cb55fcc0b9b62098780ed4a3cb6a4bb",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Amstrad/CPC/amsdos.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleII/apple2-character.rom",
|
||||
"sha1": "f9d312f128c9557d9d6ac03bfad6c3ddf83e5659",
|
||||
"size": 2048,
|
||||
"repo_path": "bios/Apple/Apple II/apple2-character.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleII/apple2eu-character.rom",
|
||||
"sha1": "7060de104046736529c1e8a687a0dd7b84f8c51b",
|
||||
"size": 4096,
|
||||
"repo_path": "bios/Apple/Apple II/apple2eu-character.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleII/apple2e-character.rom",
|
||||
"sha1": "b2b5d87f52693817fc747df087a4aa1ddcdb1f10",
|
||||
"size": 4096,
|
||||
"repo_path": "bios/Apple/Apple II/apple2e-character.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleIIgs/apple2gs.chr",
|
||||
"sha1": "34e2443e2ef960a36c047a09ed5a93f471797f89",
|
||||
"size": 4096,
|
||||
"repo_path": "bios/Apple/Apple II/apple2gs.chr",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "AppleIIgs/341s0632-2",
|
||||
"sha1": "141d18c36a617ab9dce668445440d34354be0672",
|
||||
"size": 4096,
|
||||
"repo_path": "bios/Apple/Apple II/341s0632-2",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ColecoVision/coleco.rom",
|
||||
"sha1": "45bedc4cbdeac66c7df59e9e599195c778d86a92",
|
||||
@@ -11061,33 +11251,6 @@
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "DiskII/state-machine-16.rom",
|
||||
"sha1": "bc39fbd5b9a8d2287ac5d0a42e639fc4d3c2f9d4",
|
||||
"size": 256,
|
||||
"repo_path": "bios/Apple/Apple II/DiskII/state-machine-16.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "Enterprise/exos20.bin",
|
||||
"sha1": "6033a0535136c40c47137e4d1cd9273c06d5fdff",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Enterprise/64-128/exos20.bin",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "MSX/msx-japanese.rom",
|
||||
"sha1": "302afb5d8be26c758309ca3df611ae69cced2821",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Microsoft/MSX/CLK/msx-japanese.rom",
|
||||
"cores": [
|
||||
"Clock Signal (CLK)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "GBA/gba_bios.bin",
|
||||
"sha1": "300c20df6731a33952ded8c436f7f186d25d3492",
|
||||
@@ -11412,15 +11575,6 @@
|
||||
"ScummVM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/IPLROM.X1T",
|
||||
"sha1": "44620f57a25f0bcac2b57ca2b0f1ebad3bf305d3",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Sharp/X1/.variants/IPLROM.X1T.44620f57",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fbneo/bubsys.zip",
|
||||
"sha1": "1c0ffcd308b0c8c6dbb74ad8b811a0767200d366",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "bizhawk",
|
||||
"display_name": "BizHawk",
|
||||
"version": "1.0",
|
||||
"generated": "2026-03-31T21:00:33Z",
|
||||
"generated": "2026-04-01T14:00:10Z",
|
||||
"base_destination": "Firmware",
|
||||
"detect": [
|
||||
{
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "emudeck",
|
||||
"display_name": "EmuDeck",
|
||||
"version": "1.0",
|
||||
"generated": "2026-03-31T21:00:40Z",
|
||||
"generated": "2026-04-01T14:41:53Z",
|
||||
"base_destination": "bios",
|
||||
"detect": [
|
||||
{
|
||||
@@ -50,8 +51,8 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"total_files": 509,
|
||||
"total_size": 3267803462,
|
||||
"total_files": 526,
|
||||
"total_size": 3276716358,
|
||||
"files": [
|
||||
{
|
||||
"dest": "colecovision.rom",
|
||||
@@ -932,6 +933,24 @@
|
||||
"Dolphin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1000.bin",
|
||||
"sha1": "343883a7b555646da8cee54aadd2795b6e7dd070",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph1000.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1001.bin",
|
||||
"sha1": "10155d8d6e6e832d6ea66db9bc098321fb5e8ebf",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph1001.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1002a.bin",
|
||||
"sha1": "20b98f3d80f11cbf5a7bfd0779b0e63760ecc62c",
|
||||
@@ -941,6 +960,132 @@
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1002b.bin",
|
||||
"sha1": "76cf6b1b2a7c571a6ad07f2bac0db6cd8f71e2cc",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph1002b.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1002c.bin",
|
||||
"sha1": "b6a11579caef3875504fcf3831b8e3922746df2c",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph1002c.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "dtlh1100.bin",
|
||||
"sha1": "73107d468fc7cb1d2c5b18b269715dd889ecef06",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/dtlh1100.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph3000.bin",
|
||||
"sha1": "b06f4a861f74270be819aa2a07db8d0563a7cc4e",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph3000.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1001_v20.bin",
|
||||
"sha1": "649895efd79d14790eabb362e94eb0622093dfb9",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph1001_v20.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph3500.bin",
|
||||
"sha1": "e38466a4ba8005fba7e9e3c7b9efeba7205bee3f",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph3500.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1001_v21.bin",
|
||||
"sha1": "ca7af30b50d9756cbd764640126c454cff658479",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph1001_v21.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph5000.bin",
|
||||
"sha1": "ffa7f9a7fb19d773a0c3985a541c8e5623d2c30d",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph5000.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph7000.bin",
|
||||
"sha1": "77b10118d21ac7ffa9b35f9c4fd814da240eb3e9",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph7000.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph7000w.bin",
|
||||
"sha1": "1b0dbdb23da9dc0776aac58d0755dc80fea20975",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph7000w.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph7001.bin",
|
||||
"sha1": "14df4f6c1e367ce097c11deae21566b4fe5647a9",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph7001.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph7002.bin",
|
||||
"sha1": "8d5de56a79954f29e9006929ba3fed9b6a418c1d",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph7002.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph100.bin",
|
||||
"sha1": "339a48f4fcf63e10b5b867b8c93cfd40945faf6c",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph100.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph101_v44.bin",
|
||||
"sha1": "7771d6e90980408f753891648685def6dd42ef6d",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph101_v44.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph101_v45.bin",
|
||||
"sha1": "dcffe16bd90a723499ad46c641424981338d8378",
|
||||
@@ -968,6 +1113,15 @@
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph1000r.bin",
|
||||
"sha1": "7082bd57141fa0007b3adcd031f7ba23a20108a0",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph1000r.bin",
|
||||
"cores": [
|
||||
"DuckStation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ps2_scph18000.bin",
|
||||
"sha1": "d7d6be084f51354bc951d8fa2d8d912aa70abc5e",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "lakka",
|
||||
"display_name": "Lakka",
|
||||
"version": "1.0",
|
||||
"generated": "2026-03-31T21:00:57Z",
|
||||
"generated": "2026-04-01T14:42:08Z",
|
||||
"base_destination": "system",
|
||||
"detect": [
|
||||
{
|
||||
@@ -14,8 +15,8 @@
|
||||
}
|
||||
],
|
||||
"standalone_copies": [],
|
||||
"total_files": 1609,
|
||||
"total_size": 5248935496,
|
||||
"total_files": 1620,
|
||||
"total_size": 5255358024,
|
||||
"files": [
|
||||
{
|
||||
"dest": "3do_arcade_saot.bin",
|
||||
@@ -3158,6 +3159,15 @@
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick31.rom",
|
||||
"sha1": "3b7f1493b27e212830f989f26ca76c02049f09ca",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/kick31.rom",
|
||||
"cores": [
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick.rom",
|
||||
"sha1": "891e9a547772fe0c6c19b610baf8bc4ea7fcb785",
|
||||
@@ -3968,6 +3978,15 @@
|
||||
"DirectXBox"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "GC/USA/IPL.bin",
|
||||
"sha1": "ef9194ab4804aa0aa8540d846caf291b28331165",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Nintendo/GameCube/GC/USA/IPL.bin",
|
||||
"cores": [
|
||||
"Dolphin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "Wii/shared2/sys/SYSCONF",
|
||||
"sha1": "3256c026284a24fb99d2ec1558d95db3b5dcc2e9",
|
||||
@@ -5195,6 +5214,15 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-204.rom",
|
||||
"sha1": "c5839f5cb98a7a8947065c3ed2f14f5f42e334a1",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-204.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-205.rom",
|
||||
"sha1": "02843c4253bbd29aba535b0aa3bd9a85034ecde4",
|
||||
@@ -5204,6 +5232,24 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-120.rom",
|
||||
"sha1": "11f9e62cf299f72184835b7b2a70a16333fc0d88",
|
||||
"size": 262144,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-120.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a1200.rom",
|
||||
"sha1": "e21545723fe8374e91342617604f1b3d703094f1",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-a1200.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a3000.rom",
|
||||
"sha1": "f8e210d72b4c4853e0c9b85d223ba20e3d1b36ee",
|
||||
@@ -5213,6 +5259,33 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a4000.rom",
|
||||
"sha1": "5fe04842d04a489720f0f4bb0e46948199406f49",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-a4000.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-cd32.rom",
|
||||
"sha1": "3525be8887f79b5929e017b42380a79edfee542d",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-cd32.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-ext-310-cd32.rom",
|
||||
"sha1": "5bef3d628ce59cc02a66e6e4ae0da48f60e78f7f",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-ext-310-cd32.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "cd32fmv.rom",
|
||||
"sha1": "03ca81c7a7b259cf64bc9582863eca0f6529f435",
|
||||
@@ -5222,6 +5295,15 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-ext-130-cdtv.rom",
|
||||
"sha1": "7ba40ffa17e500ed9fed041f3424bd81d9c907be",
|
||||
"size": 262144,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-ext-130-cdtv.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fuse/48.rom",
|
||||
"sha1": "5ea7c2b824672e914525d1d5c419d71b84a426a2",
|
||||
@@ -9424,6 +9506,24 @@
|
||||
"QUASI88"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "dc_bios.bin",
|
||||
"sha1": "8951d1bb219ab2ff8583033d2119c899cc81f18c",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Sega/Dreamcast/dc_bios.bin",
|
||||
"cores": [
|
||||
"Redream"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "dc_flash.bin",
|
||||
"sha1": "94d44d7f9529ec1642ba3771ed3c5f756d5bc872",
|
||||
"size": 131072,
|
||||
"repo_path": "bios/Sega/Dreamcast/dc_flash.bin",
|
||||
"cores": [
|
||||
"Redream"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph_v11j.bin",
|
||||
"sha1": "b06f4a861f74270be819aa2a07db8d0563a7cc4e",
|
||||
@@ -9487,6 +9587,15 @@
|
||||
"Rustation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph102.bin",
|
||||
"sha1": "beb0ac693c0dc26daf5665b3314db81480fa5c7c",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph102.bin",
|
||||
"cores": [
|
||||
"Rustation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "rvvm/fw_payload.bin",
|
||||
"sha1": "c603ebeea2816d5c52985170aa7ac4b9dd5f7a8d",
|
||||
@@ -10711,6 +10820,15 @@
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/FNT0808.X1",
|
||||
"sha1": "1c1a0d8c9f4c446ccd7470516b215ddca5052fb2",
|
||||
"size": 2048,
|
||||
"repo_path": "bios/Sharp/X1/FNT0808.X1",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/FNT0816.X1",
|
||||
"sha1": "4f06d20c997a79ee6af954b69498147789bf1847",
|
||||
@@ -11467,15 +11585,6 @@
|
||||
"Citra"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "GC/USA/IPL.bin",
|
||||
"sha1": "ef9194ab4804aa0aa8540d846caf291b28331165",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Nintendo/GameCube/GC/USA/IPL.bin",
|
||||
"cores": [
|
||||
"Dolphin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "GC/EUR/IPL.bin",
|
||||
"sha1": "80b8744ff5e43585392f55546bd03a673d11ef5f",
|
||||
@@ -12043,15 +12152,6 @@
|
||||
"UME 2015"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/FNT0808.X1",
|
||||
"sha1": "1c1a0d8c9f4c446ccd7470516b215ddca5052fb2",
|
||||
"size": 2048,
|
||||
"repo_path": "bios/Sharp/X1/FNT0808.X1",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fbneo/bubsys.zip",
|
||||
"sha1": "1c0ffcd308b0c8c6dbb74ad8b811a0767200d366",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "recalbox",
|
||||
"display_name": "Recalbox",
|
||||
"version": "1.0",
|
||||
"generated": "2026-03-31T21:01:26Z",
|
||||
"generated": "2026-04-01T14:42:39Z",
|
||||
"base_destination": "bios",
|
||||
"detect": [
|
||||
{
|
||||
@@ -14,8 +15,8 @@
|
||||
}
|
||||
],
|
||||
"standalone_copies": [],
|
||||
"total_files": 1093,
|
||||
"total_size": 3499462394,
|
||||
"total_files": 1097,
|
||||
"total_size": 3500142330,
|
||||
"files": [
|
||||
{
|
||||
"dest": "3do/panafz1.bin",
|
||||
@@ -1992,6 +1993,15 @@
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick31.rom",
|
||||
"sha1": "3b7f1493b27e212830f989f26ca76c02049f09ca",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/kick31.rom",
|
||||
"cores": [
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick.rom",
|
||||
"sha1": "891e9a547772fe0c6c19b610baf8bc4ea7fcb785",
|
||||
@@ -4219,6 +4229,15 @@
|
||||
"Genesis Plus GX"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ROM",
|
||||
"sha1": "e4fc7560b69d062cb2da5b1ffbe11cd1ca03cc37",
|
||||
"size": 131072,
|
||||
"repo_path": "bios/Apple/Apple IIGS/ROM",
|
||||
"cores": [
|
||||
"GSplus"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "c600.rom",
|
||||
"sha1": "d4181c9f046aafc3fb326b381baac809d9e38d16",
|
||||
@@ -6937,6 +6956,24 @@
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "alpha-basic.rom",
|
||||
"sha1": "1983b4fb398e3dd9668d424c666c5a0b3f1e2b69",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Dragon/Dragon/alpha-basic.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "alice.rom",
|
||||
"sha1": "c2166b91e6396a311f486832012aa43e0d2b19f8",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/alice.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "deluxe.rom",
|
||||
"sha1": "d89196292b9ebd787647cf91bbb83c63da2b4390",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "retroarch",
|
||||
"display_name": "RetroArch",
|
||||
"version": "1.0",
|
||||
"generated": "2026-03-31T21:00:57Z",
|
||||
"generated": "2026-04-01T14:42:08Z",
|
||||
"base_destination": "system",
|
||||
"detect": [
|
||||
{
|
||||
@@ -32,8 +33,8 @@
|
||||
}
|
||||
],
|
||||
"standalone_copies": [],
|
||||
"total_files": 1609,
|
||||
"total_size": 5248935496,
|
||||
"total_files": 1620,
|
||||
"total_size": 5255358024,
|
||||
"files": [
|
||||
{
|
||||
"dest": "3do_arcade_saot.bin",
|
||||
@@ -3176,6 +3177,15 @@
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick31.rom",
|
||||
"sha1": "3b7f1493b27e212830f989f26ca76c02049f09ca",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/kick31.rom",
|
||||
"cores": [
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick.rom",
|
||||
"sha1": "891e9a547772fe0c6c19b610baf8bc4ea7fcb785",
|
||||
@@ -3986,6 +3996,15 @@
|
||||
"DirectXBox"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "GC/USA/IPL.bin",
|
||||
"sha1": "ef9194ab4804aa0aa8540d846caf291b28331165",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Nintendo/GameCube/GC/USA/IPL.bin",
|
||||
"cores": [
|
||||
"Dolphin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "Wii/shared2/sys/SYSCONF",
|
||||
"sha1": "3256c026284a24fb99d2ec1558d95db3b5dcc2e9",
|
||||
@@ -5213,6 +5232,15 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-204.rom",
|
||||
"sha1": "c5839f5cb98a7a8947065c3ed2f14f5f42e334a1",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-204.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-205.rom",
|
||||
"sha1": "02843c4253bbd29aba535b0aa3bd9a85034ecde4",
|
||||
@@ -5222,6 +5250,24 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-120.rom",
|
||||
"sha1": "11f9e62cf299f72184835b7b2a70a16333fc0d88",
|
||||
"size": 262144,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-120.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a1200.rom",
|
||||
"sha1": "e21545723fe8374e91342617604f1b3d703094f1",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-a1200.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a3000.rom",
|
||||
"sha1": "f8e210d72b4c4853e0c9b85d223ba20e3d1b36ee",
|
||||
@@ -5231,6 +5277,33 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a4000.rom",
|
||||
"sha1": "5fe04842d04a489720f0f4bb0e46948199406f49",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-a4000.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-cd32.rom",
|
||||
"sha1": "3525be8887f79b5929e017b42380a79edfee542d",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-cd32.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-ext-310-cd32.rom",
|
||||
"sha1": "5bef3d628ce59cc02a66e6e4ae0da48f60e78f7f",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-ext-310-cd32.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "cd32fmv.rom",
|
||||
"sha1": "03ca81c7a7b259cf64bc9582863eca0f6529f435",
|
||||
@@ -5240,6 +5313,15 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-ext-130-cdtv.rom",
|
||||
"sha1": "7ba40ffa17e500ed9fed041f3424bd81d9c907be",
|
||||
"size": 262144,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-ext-130-cdtv.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fuse/48.rom",
|
||||
"sha1": "5ea7c2b824672e914525d1d5c419d71b84a426a2",
|
||||
@@ -9442,6 +9524,24 @@
|
||||
"QUASI88"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "dc_bios.bin",
|
||||
"sha1": "8951d1bb219ab2ff8583033d2119c899cc81f18c",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Sega/Dreamcast/dc_bios.bin",
|
||||
"cores": [
|
||||
"Redream"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "dc_flash.bin",
|
||||
"sha1": "94d44d7f9529ec1642ba3771ed3c5f756d5bc872",
|
||||
"size": 131072,
|
||||
"repo_path": "bios/Sega/Dreamcast/dc_flash.bin",
|
||||
"cores": [
|
||||
"Redream"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph_v11j.bin",
|
||||
"sha1": "b06f4a861f74270be819aa2a07db8d0563a7cc4e",
|
||||
@@ -9505,6 +9605,15 @@
|
||||
"Rustation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph102.bin",
|
||||
"sha1": "beb0ac693c0dc26daf5665b3314db81480fa5c7c",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph102.bin",
|
||||
"cores": [
|
||||
"Rustation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "rvvm/fw_payload.bin",
|
||||
"sha1": "c603ebeea2816d5c52985170aa7ac4b9dd5f7a8d",
|
||||
@@ -10729,6 +10838,15 @@
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/FNT0808.X1",
|
||||
"sha1": "1c1a0d8c9f4c446ccd7470516b215ddca5052fb2",
|
||||
"size": 2048,
|
||||
"repo_path": "bios/Sharp/X1/FNT0808.X1",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/FNT0816.X1",
|
||||
"sha1": "4f06d20c997a79ee6af954b69498147789bf1847",
|
||||
@@ -11485,15 +11603,6 @@
|
||||
"Citra"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "GC/USA/IPL.bin",
|
||||
"sha1": "ef9194ab4804aa0aa8540d846caf291b28331165",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Nintendo/GameCube/GC/USA/IPL.bin",
|
||||
"cores": [
|
||||
"Dolphin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "GC/EUR/IPL.bin",
|
||||
"sha1": "80b8744ff5e43585392f55546bd03a673d11ef5f",
|
||||
@@ -12061,15 +12170,6 @@
|
||||
"UME 2015"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/FNT0808.X1",
|
||||
"sha1": "1c1a0d8c9f4c446ccd7470516b215ddca5052fb2",
|
||||
"size": 2048,
|
||||
"repo_path": "bios/Sharp/X1/FNT0808.X1",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fbneo/bubsys.zip",
|
||||
"sha1": "1c0ffcd308b0c8c6dbb74ad8b811a0767200d366",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "retrobat",
|
||||
"display_name": "RetroBat",
|
||||
"version": "1.0",
|
||||
"generated": "2026-03-31T21:01:36Z",
|
||||
"generated": "2026-04-01T14:42:50Z",
|
||||
"base_destination": "bios",
|
||||
"detect": [
|
||||
{
|
||||
@@ -13,8 +14,8 @@
|
||||
}
|
||||
],
|
||||
"standalone_copies": [],
|
||||
"total_files": 1160,
|
||||
"total_size": 4297510031,
|
||||
"total_files": 1162,
|
||||
"total_size": 4297772175,
|
||||
"files": [
|
||||
{
|
||||
"dest": "panafz1.bin",
|
||||
@@ -4453,6 +4454,15 @@
|
||||
"Genesis Plus GX"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ROM",
|
||||
"sha1": "e4fc7560b69d062cb2da5b1ffbe11cd1ca03cc37",
|
||||
"size": 131072,
|
||||
"repo_path": "bios/Apple/Apple IIGS/ROM",
|
||||
"cores": [
|
||||
"GSplus"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "c600.rom",
|
||||
"sha1": "d4181c9f046aafc3fb326b381baac809d9e38d16",
|
||||
@@ -6352,6 +6362,15 @@
|
||||
"PicoDrive"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "SegaCDBIOS9303.bin",
|
||||
"sha1": "5adb6c3af218c60868e6b723ec47e36bbdf5e6f0",
|
||||
"size": 131072,
|
||||
"repo_path": "bios/Sega/Mega CD/SegaCDBIOS9303.bin",
|
||||
"cores": [
|
||||
"PicoDrive"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "us_scd1_9210.bin",
|
||||
"sha1": "f4f315adcef9b8feb0364c21ab7f0eaf5457f3ed",
|
||||
@@ -7105,6 +7124,15 @@
|
||||
"storage": "release",
|
||||
"release_asset": "PSP2UPDAT.PUP"
|
||||
},
|
||||
{
|
||||
"dest": "xmil/IPLROM.X1T",
|
||||
"sha1": "44620f57a25f0bcac2b57ca2b0f1ebad3bf305d3",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Sharp/X1/.variants/IPLROM.X1T.44620f57",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/FNT0808.X1",
|
||||
"sha1": "1c1a0d8c9f4c446ccd7470516b215ddca5052fb2",
|
||||
@@ -8194,15 +8222,6 @@
|
||||
"ScummVM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "xmil/IPLROM.X1T",
|
||||
"sha1": "44620f57a25f0bcac2b57ca2b0f1ebad3bf305d3",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Sharp/X1/.variants/IPLROM.X1T.44620f57",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fbneo/bubsys.zip",
|
||||
"sha1": "1c0ffcd308b0c8c6dbb74ad8b811a0767200d366",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "retrodeck",
|
||||
"display_name": "RetroDECK",
|
||||
"version": "1.0",
|
||||
"generated": "2026-04-01T09:05:30Z",
|
||||
"generated": "2026-04-01T14:43:08Z",
|
||||
"base_destination": "",
|
||||
"detect": [
|
||||
{
|
||||
@@ -14,8 +15,8 @@
|
||||
}
|
||||
],
|
||||
"standalone_copies": [],
|
||||
"total_files": 3127,
|
||||
"total_size": 5865074692,
|
||||
"total_files": 3144,
|
||||
"total_size": 5871582893,
|
||||
"files": [
|
||||
{
|
||||
"dest": "bios/panafz1.bin",
|
||||
@@ -13775,6 +13776,15 @@
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick31.rom",
|
||||
"sha1": "3b7f1493b27e212830f989f26ca76c02049f09ca",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/kick31.rom",
|
||||
"cores": [
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "kick.rom",
|
||||
"sha1": "891e9a547772fe0c6c19b610baf8bc4ea7fcb785",
|
||||
@@ -13910,6 +13920,24 @@
|
||||
"Amiberry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "BB01R4_OS.ROM",
|
||||
"sha1": "decde89fbae90adb591ad2fc553d35f49030c129",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Atari/400-800/BB01R4_OS.ROM",
|
||||
"cores": [
|
||||
"Atari800"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "XEGAME.ROM",
|
||||
"sha1": "a107db7f16a1129cf9d933c9cf4f013b068c9e82",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Atari/400-800/XEGAME.ROM",
|
||||
"cores": [
|
||||
"Atari800"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "sysdata/keys.txt",
|
||||
"sha1": "9edc52be45201ec99f6e4ceb5dc6abfc633c4eae",
|
||||
@@ -14207,6 +14235,15 @@
|
||||
"bsnes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "st018.data.rom",
|
||||
"sha1": "b19c0f8f207d62fdabf4bf71442826063bccc626",
|
||||
"size": 32768,
|
||||
"repo_path": "bios/Nintendo/SNES/st018.data.rom",
|
||||
"cores": [
|
||||
"bsnes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "sgb.boot.rom",
|
||||
"sha1": "aa2f50a77dfb4823da96ba99309085a3c6278515",
|
||||
@@ -16164,6 +16201,15 @@
|
||||
"ep128emu-core"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ep128emu/roms/zx48.rom",
|
||||
"sha1": "5ea7c2b824672e914525d1d5c419d71b84a426a2",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Enterprise/64-128/zx48.rom",
|
||||
"cores": [
|
||||
"ep128emu-core"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "hiscore.dat",
|
||||
"sha1": "7381472bf046126257e51a0124e4553282f020e5",
|
||||
@@ -16343,6 +16389,15 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-204.rom",
|
||||
"sha1": "c5839f5cb98a7a8947065c3ed2f14f5f42e334a1",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-204.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-205.rom",
|
||||
"sha1": "02843c4253bbd29aba535b0aa3bd9a85034ecde4",
|
||||
@@ -16352,6 +16407,24 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-120.rom",
|
||||
"sha1": "11f9e62cf299f72184835b7b2a70a16333fc0d88",
|
||||
"size": 262144,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-120.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a1200.rom",
|
||||
"sha1": "e21545723fe8374e91342617604f1b3d703094f1",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-a1200.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a3000.rom",
|
||||
"sha1": "f8e210d72b4c4853e0c9b85d223ba20e3d1b36ee",
|
||||
@@ -16361,6 +16434,33 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-a4000.rom",
|
||||
"sha1": "5fe04842d04a489720f0f4bb0e46948199406f49",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-a4000.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-os-310-cd32.rom",
|
||||
"sha1": "3525be8887f79b5929e017b42380a79edfee542d",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-os-310-cd32.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-ext-310-cd32.rom",
|
||||
"sha1": "5bef3d628ce59cc02a66e6e4ae0da48f60e78f7f",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-ext-310-cd32.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "cd32fmv.rom",
|
||||
"sha1": "03ca81c7a7b259cf64bc9582863eca0f6529f435",
|
||||
@@ -16370,6 +16470,15 @@
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "amiga-ext-130-cdtv.rom",
|
||||
"sha1": "7ba40ffa17e500ed9fed041f3424bd81d9c907be",
|
||||
"size": 262144,
|
||||
"repo_path": "bios/Commodore/Amiga/amiga-ext-130-cdtv.rom",
|
||||
"cores": [
|
||||
"FS-UAE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fuse/48.rom",
|
||||
"sha1": "5ea7c2b824672e914525d1d5c419d71b84a426a2",
|
||||
@@ -16775,6 +16884,15 @@
|
||||
"Genesis Plus GX"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ROM",
|
||||
"sha1": "e4fc7560b69d062cb2da5b1ffbe11cd1ca03cc37",
|
||||
"size": 131072,
|
||||
"repo_path": "bios/Apple/Apple IIGS/ROM",
|
||||
"cores": [
|
||||
"GSplus"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "c600.rom",
|
||||
"sha1": "d4181c9f046aafc3fb326b381baac809d9e38d16",
|
||||
@@ -17569,6 +17687,15 @@
|
||||
"MAME 2009"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "bctvidbs.zip",
|
||||
"sha1": "5024b3bfd04ccd2061eb60d2eca254e8faf44f5c",
|
||||
"size": 3753,
|
||||
"repo_path": "bios/Arcade/Arcade/bctvidbs.zip",
|
||||
"cores": [
|
||||
"MAME 2009"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "cd32.zip",
|
||||
"sha1": "2b43d67e90767a43b435b3a9f504346cff0f64ca",
|
||||
@@ -17985,6 +18112,60 @@
|
||||
"Mupen64Plus-Next"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_bd.wav",
|
||||
"sha1": "0a56c142ef40cec50f3ee56a6e42d0029c9e2818",
|
||||
"size": 19192,
|
||||
"repo_path": "bios/NEC/PC-98/2608_bd.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_sd.wav",
|
||||
"sha1": "3c79663ef74c0b0439d13351326eb1c52a657008",
|
||||
"size": 15558,
|
||||
"repo_path": "bios/NEC/PC-98/2608_sd.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_top.wav",
|
||||
"sha1": "aa4a8f766a86b830687d5083fd3b9db0652f46fc",
|
||||
"size": 57016,
|
||||
"repo_path": "bios/NEC/PC-98/2608_top.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_hh.wav",
|
||||
"sha1": "12f676cef249b82480b6f19c454e234b435ca7b6",
|
||||
"size": 36722,
|
||||
"repo_path": "bios/NEC/PC-98/2608_hh.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_tom.wav",
|
||||
"sha1": "9513fb4a3f41e75a972a273a5104cbd834c1e2c5",
|
||||
"size": 23092,
|
||||
"repo_path": "bios/NEC/PC-98/2608_tom.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_rim.wav",
|
||||
"sha1": "c65592330c9dd84011151daed52f9aec926b7e56",
|
||||
"size": 5288,
|
||||
"repo_path": "bios/NEC/PC-98/2608_rim.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/scsi.rom",
|
||||
"sha1": "3d7166f05daad1b022fa04c2569e788580158095",
|
||||
@@ -19683,6 +19864,15 @@
|
||||
"QUASI88"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "quasi88/n88jisho.rom",
|
||||
"sha1": "deef0cc2a9734ba891a6d6c022aa70ffc66f783e",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/NEC/PC-98/n88jisho.rom",
|
||||
"cores": [
|
||||
"QUASI88"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "quasi88/font.rom",
|
||||
"sha1": "78ba9960f135372825ab7244b5e4e73a810002ff",
|
||||
@@ -19710,6 +19900,15 @@
|
||||
"QUASI88"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "dc_bios.bin",
|
||||
"sha1": "8951d1bb219ab2ff8583033d2119c899cc81f18c",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Sega/Dreamcast/dc_bios.bin",
|
||||
"cores": [
|
||||
"Redream"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "PS3UPDAT.PUP",
|
||||
"sha1": "093f8698b54b78dcb701de2043f82639de51d63b",
|
||||
@@ -19784,6 +19983,15 @@
|
||||
"Rustation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "scph102.bin",
|
||||
"sha1": "beb0ac693c0dc26daf5665b3314db81480fa5c7c",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/scph102.bin",
|
||||
"cores": [
|
||||
"Rustation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "rvvm/fw_payload.bin",
|
||||
"sha1": "c603ebeea2816d5c52985170aa7ac4b9dd5f7a8d",
|
||||
@@ -21417,6 +21625,15 @@
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "alpha-basic.rom",
|
||||
"sha1": "1983b4fb398e3dd9668d424c666c5a0b3f1e2b69",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Dragon/Dragon/alpha-basic.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "mc10.rom",
|
||||
"sha1": "4afff2b4c120334481aab7b02c3552bf76f1bc43",
|
||||
@@ -21489,6 +21706,15 @@
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "cp400ext.rom",
|
||||
"sha1": "a348a165009a6de1ae6fc18ed77137b38b6ed46d",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Tandy/CoCo/cp400ext.rom",
|
||||
"cores": [
|
||||
"XRoar"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "hdbdw3bck.rom",
|
||||
"sha1": "8fd64f1c246489e0bf2b3743ae76332ff324716a",
|
||||
@@ -22380,15 +22606,6 @@
|
||||
"ep128emu-core"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ep128emu/roms/zx48.rom",
|
||||
"sha1": "5ea7c2b824672e914525d1d5c419d71b84a426a2",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Enterprise/64-128/zx48.rom",
|
||||
"cores": [
|
||||
"ep128emu-core"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fbalpha2012/hiscore.dat",
|
||||
"sha1": "7381472bf046126257e51a0124e4553282f020e5",
|
||||
@@ -22560,60 +22777,6 @@
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_bd.wav",
|
||||
"sha1": "0a56c142ef40cec50f3ee56a6e42d0029c9e2818",
|
||||
"size": 19192,
|
||||
"repo_path": "bios/NEC/PC-98/2608_bd.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_sd.wav",
|
||||
"sha1": "3c79663ef74c0b0439d13351326eb1c52a657008",
|
||||
"size": 15558,
|
||||
"repo_path": "bios/NEC/PC-98/2608_sd.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_top.wav",
|
||||
"sha1": "aa4a8f766a86b830687d5083fd3b9db0652f46fc",
|
||||
"size": 57016,
|
||||
"repo_path": "bios/NEC/PC-98/2608_top.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_hh.wav",
|
||||
"sha1": "12f676cef249b82480b6f19c454e234b435ca7b6",
|
||||
"size": 36722,
|
||||
"repo_path": "bios/NEC/PC-98/2608_hh.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_tom.wav",
|
||||
"sha1": "9513fb4a3f41e75a972a273a5104cbd834c1e2c5",
|
||||
"size": 23092,
|
||||
"repo_path": "bios/NEC/PC-98/2608_tom.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "np2/2608_rim.wav",
|
||||
"sha1": "c65592330c9dd84011151daed52f9aec926b7e56",
|
||||
"size": 5288,
|
||||
"repo_path": "bios/NEC/PC-98/2608_rim.wav",
|
||||
"cores": [
|
||||
"nekop2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "neocd/neocd_f.rom",
|
||||
"sha1": "a5f4a7a627b3083c979f6ebe1fabc5d2df6d083b",
|
||||
@@ -22920,15 +23083,6 @@
|
||||
"QUASI88"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "quasi88/n88jisho.rom",
|
||||
"sha1": "deef0cc2a9734ba891a6d6c022aa70ffc66f783e",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/NEC/PC-98/n88jisho.rom",
|
||||
"cores": [
|
||||
"QUASI88"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "same_cdi/bios/cdimono1.zip",
|
||||
"sha1": "5d0b1b55b0d0958a5c9069c3219d4da5a87a6b93",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
{
|
||||
"manifest_version": 1,
|
||||
"source": "full",
|
||||
"platform": "romm",
|
||||
"display_name": "RomM",
|
||||
"version": "1.0",
|
||||
"generated": "2026-04-01T09:05:31Z",
|
||||
"generated": "2026-04-01T14:43:09Z",
|
||||
"base_destination": "bios",
|
||||
"detect": [
|
||||
{
|
||||
@@ -13,8 +14,8 @@
|
||||
}
|
||||
],
|
||||
"standalone_copies": [],
|
||||
"total_files": 530,
|
||||
"total_size": 1073883091,
|
||||
"total_files": 535,
|
||||
"total_size": 1076531155,
|
||||
"files": [
|
||||
{
|
||||
"dest": "3do/3do_arcade_saot.bin",
|
||||
@@ -2640,6 +2641,15 @@
|
||||
"Atari800"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "psx/psxonpsp660.bin",
|
||||
"sha1": "96880d1ca92a016ff054be5159bb06fe03cb4e14",
|
||||
"size": 524288,
|
||||
"repo_path": "bios/Sony/PlayStation/psxonpsp660.bin",
|
||||
"cores": [
|
||||
"Beetle PSX (Mednafen PSX)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "psx/ps1_rom.bin",
|
||||
"sha1": "c40146361eb8cf670b19fdc9759190257803cab7",
|
||||
@@ -2667,6 +2677,15 @@
|
||||
"DirectXBox"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ngc/GC/USA/IPL.bin",
|
||||
"sha1": "a1837968288253ed541f2b11440b68f5a9b33875",
|
||||
"size": 2097152,
|
||||
"repo_path": "bios/Nintendo/GameCube/GC/JAP/IPL.bin",
|
||||
"cores": [
|
||||
"Dolphin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "ngc/GC/dsp_rom.bin",
|
||||
"sha1": "f4f683a49d7eb4155566f793f2c1c27e90159992",
|
||||
@@ -2768,6 +2787,15 @@
|
||||
"ep128emu-core"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "acpc/ep128emu/roms/zx48.rom",
|
||||
"sha1": "5ea7c2b824672e914525d1d5c419d71b84a426a2",
|
||||
"size": 16384,
|
||||
"repo_path": "bios/Enterprise/64-128/zx48.rom",
|
||||
"cores": [
|
||||
"ep128emu-core"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "fds/gamegenie.nes",
|
||||
"sha1": "f430a0d752a9fa0c7032db8131f9090d18f71779",
|
||||
@@ -3508,6 +3536,15 @@
|
||||
"px68k"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "gamegear/BIOS.col",
|
||||
"sha1": "45bedc4cbdeac66c7df59e9e599195c778d86a92",
|
||||
"size": 8192,
|
||||
"repo_path": "bios/Coleco/ColecoVision/BIOS.col",
|
||||
"cores": [
|
||||
"SMS Plus GX"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "j2me/squirreljme-0.3.0-fast.jar",
|
||||
"sha1": "7c4cd0a5451eedeac9b328f48408dbc312198ccf",
|
||||
@@ -3616,6 +3653,15 @@
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "x1/xmil/FNT0808.X1",
|
||||
"sha1": "1c1a0d8c9f4c446ccd7470516b215ddca5052fb2",
|
||||
"size": 2048,
|
||||
"repo_path": "bios/Sharp/X1/FNT0808.X1",
|
||||
"cores": [
|
||||
"X Millennium"
|
||||
]
|
||||
},
|
||||
{
|
||||
"dest": "x1/xmil/FNT0816.X1",
|
||||
"sha1": "4f06d20c997a79ee6af954b69498147789bf1847",
|
||||
|
||||
@@ -21,12 +21,17 @@ import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from common import list_registered_platforms, load_database, load_platform_config, require_yaml
|
||||
from common import (
|
||||
list_registered_platforms,
|
||||
load_database,
|
||||
load_platform_config,
|
||||
require_yaml,
|
||||
)
|
||||
|
||||
yaml = require_yaml()
|
||||
|
||||
@@ -83,14 +88,16 @@ def find_missing(config: dict, db: dict) -> list[dict]:
|
||||
found = any(m in by_md5 for m in md5_list)
|
||||
|
||||
if not found:
|
||||
missing.append({
|
||||
"name": name,
|
||||
"system": sys_id,
|
||||
"sha1": sha1,
|
||||
"md5": md5,
|
||||
"size": file_entry.get("size"),
|
||||
"destination": file_entry.get("destination", name),
|
||||
})
|
||||
missing.append(
|
||||
{
|
||||
"name": name,
|
||||
"system": sys_id,
|
||||
"sha1": sha1,
|
||||
"md5": md5,
|
||||
"size": file_entry.get("size"),
|
||||
"destination": file_entry.get("destination", name),
|
||||
}
|
||||
)
|
||||
|
||||
return missing
|
||||
|
||||
@@ -139,14 +146,16 @@ def step2_scan_branches(entry: dict) -> bytes | None:
|
||||
try:
|
||||
subprocess.run(
|
||||
["git", "rev-parse", "--verify", ref],
|
||||
capture_output=True, check=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
continue
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "ls-tree", "-r", "--name-only", ref],
|
||||
capture_output=True, text=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
for filepath in result.stdout.strip().split("\n"):
|
||||
@@ -154,7 +163,8 @@ def step2_scan_branches(entry: dict) -> bytes | None:
|
||||
try:
|
||||
blob = subprocess.run(
|
||||
["git", "show", f"{ref}:{filepath}"],
|
||||
capture_output=True, check=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
if verify_content(blob.stdout, entry):
|
||||
return blob.stdout
|
||||
@@ -172,7 +182,9 @@ def step3_search_public_repos(entry: dict) -> bytes | None:
|
||||
for url_template in PUBLIC_REPOS:
|
||||
url = url_template.format(name=name)
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-fetch/1.0"})
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "retrobios-fetch/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = _read_limited(resp)
|
||||
if data is None:
|
||||
@@ -185,7 +197,9 @@ def step3_search_public_repos(entry: dict) -> bytes | None:
|
||||
if "/" in destination:
|
||||
url = url_template.format(name=destination)
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-fetch/1.0"})
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "retrobios-fetch/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = _read_limited(resp)
|
||||
if data is None:
|
||||
@@ -206,7 +220,9 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
|
||||
for path in [name, f"system/{name}", f"bios/{name}"]:
|
||||
url = f"https://archive.org/download/{collection_id}/{path}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-fetch/1.0"})
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "retrobios-fetch/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = _read_limited(resp)
|
||||
if data is None:
|
||||
@@ -221,12 +237,13 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
|
||||
return None
|
||||
|
||||
search_url = (
|
||||
f"https://archive.org/advancedsearch.php?"
|
||||
f"q=sha1:{sha1}&output=json&rows=1"
|
||||
f"https://archive.org/advancedsearch.php?q=sha1:{sha1}&output=json&rows=1"
|
||||
)
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(search_url, headers={"User-Agent": "retrobios-fetch/1.0"})
|
||||
req = urllib.request.Request(
|
||||
search_url, headers={"User-Agent": "retrobios-fetch/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
result = json.loads(resp.read())
|
||||
docs = result.get("response", {}).get("docs", [])
|
||||
@@ -235,7 +252,9 @@ def step4_search_archive_org(entry: dict) -> bytes | None:
|
||||
if identifier:
|
||||
dl_url = f"https://archive.org/download/{identifier}/{name}"
|
||||
try:
|
||||
req2 = urllib.request.Request(dl_url, headers={"User-Agent": "retrobios-fetch/1.0"})
|
||||
req2 = urllib.request.Request(
|
||||
dl_url, headers={"User-Agent": "retrobios-fetch/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req2, timeout=30) as resp2:
|
||||
data = _read_limited(resp2)
|
||||
if data is not None and verify_content(data, entry):
|
||||
@@ -297,7 +316,7 @@ def fetch_missing(
|
||||
continue
|
||||
|
||||
if dry_run:
|
||||
print(f" [DRY RUN] Would search branches, repos, archive.org")
|
||||
print(" [DRY RUN] Would search branches, repos, archive.org")
|
||||
still_missing.append(entry)
|
||||
stats["not_found"] += 1
|
||||
continue
|
||||
@@ -323,7 +342,7 @@ def fetch_missing(
|
||||
stats["found"] += 1
|
||||
continue
|
||||
|
||||
print(f" [5] Not found - needs community contribution")
|
||||
print(" [5] Not found - needs community contribution")
|
||||
still_missing.append(entry)
|
||||
stats["not_found"] += 1
|
||||
|
||||
@@ -345,16 +364,20 @@ def generate_issue_body(missing: list[dict], platform: str) -> str:
|
||||
for entry in missing:
|
||||
sha1 = entry.get("sha1") or "N/A"
|
||||
md5 = entry.get("md5") or "N/A"
|
||||
lines.append(f"| `{entry['name']}` | {entry['system']} | `{sha1[:12]}...` | `{md5[:12]}...` |")
|
||||
lines.append(
|
||||
f"| `{entry['name']}` | {entry['system']} | `{sha1[:12]}...` | `{md5[:12]}...` |"
|
||||
)
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"### How to Contribute",
|
||||
"",
|
||||
"1. Fork this repository",
|
||||
"2. Add the BIOS file to `bios/Manufacturer/Console/`",
|
||||
"3. Create a Pull Request - checksums are verified automatically",
|
||||
])
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"### How to Contribute",
|
||||
"",
|
||||
"1. Fork this repository",
|
||||
"2. Add the BIOS file to `bios/Manufacturer/Console/`",
|
||||
"3. Create a Pull Request - checksums are verified automatically",
|
||||
]
|
||||
)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -363,11 +386,15 @@ def main():
|
||||
parser = argparse.ArgumentParser(description="Auto-fetch missing BIOS files")
|
||||
parser.add_argument("--platform", "-p", help="Platform to check")
|
||||
parser.add_argument("--all", action="store_true", help="Check all platforms")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Don't download, just report")
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true", help="Don't download, just report"
|
||||
)
|
||||
parser.add_argument("--db", default=DEFAULT_DB)
|
||||
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
|
||||
parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR)
|
||||
parser.add_argument("--create-issues", action="store_true", help="Output GitHub Issue bodies")
|
||||
parser.add_argument(
|
||||
"--create-issues", action="store_true", help="Output GitHub Issue bodies"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.db):
|
||||
@@ -378,7 +405,8 @@ def main():
|
||||
|
||||
if args.all:
|
||||
platforms = list_registered_platforms(
|
||||
args.platforms_dir, include_archived=True,
|
||||
args.platforms_dir,
|
||||
include_archived=True,
|
||||
)
|
||||
elif args.platform:
|
||||
platforms = [args.platform]
|
||||
@@ -389,19 +417,19 @@ def main():
|
||||
all_still_missing = {}
|
||||
|
||||
for platform in sorted(platforms):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Platform: {platform}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
try:
|
||||
config = load_platform_config(platform, args.platforms_dir)
|
||||
except FileNotFoundError:
|
||||
print(f" Config not found, skipping")
|
||||
print(" Config not found, skipping")
|
||||
continue
|
||||
|
||||
missing = find_missing(config, db)
|
||||
if not missing:
|
||||
print(f" All BIOS files present!")
|
||||
print(" All BIOS files present!")
|
||||
continue
|
||||
|
||||
print(f" {len(missing)} missing files")
|
||||
@@ -414,9 +442,9 @@ def main():
|
||||
print(f"\n Results: {stats['found']} found, {stats['not_found']} not found")
|
||||
|
||||
if args.create_issues and all_still_missing:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print("GitHub Issue Bodies")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
for platform, missing in all_still_missing.items():
|
||||
print(f"\n--- Issue for {platform} ---\n")
|
||||
print(generate_issue_body(missing, platform))
|
||||
|
||||
@@ -9,6 +9,7 @@ Usage:
|
||||
python scripts/check_buildbot_system.py --update
|
||||
python scripts/check_buildbot_system.py --json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
@@ -36,10 +37,14 @@ def fetch_index() -> set[str]:
|
||||
"""Fetch .index from buildbot, return set of ZIP filenames."""
|
||||
req = urllib.request.Request(INDEX_URL, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
|
||||
return {line.strip() for line in resp.read().decode().splitlines() if line.strip()}
|
||||
return {
|
||||
line.strip() for line in resp.read().decode().splitlines() if line.strip()
|
||||
}
|
||||
|
||||
|
||||
def load_tracked_entries(registry_path: str = DEFAULT_REGISTRY) -> dict[str, tuple[str, str]]:
|
||||
def load_tracked_entries(
|
||||
registry_path: str = DEFAULT_REGISTRY,
|
||||
) -> dict[str, tuple[str, str]]:
|
||||
"""Load buildbot entries from _data_dirs.yml.
|
||||
|
||||
Returns {decoded_zip_name: (key, source_url)}.
|
||||
@@ -64,8 +69,9 @@ def load_tracked_entries(registry_path: str = DEFAULT_REGISTRY) -> dict[str, tup
|
||||
def get_remote_etag(url: str) -> str | None:
|
||||
"""HEAD request to get ETag."""
|
||||
try:
|
||||
req = urllib.request.Request(url, method="HEAD",
|
||||
headers={"User-Agent": USER_AGENT})
|
||||
req = urllib.request.Request(
|
||||
url, method="HEAD", headers={"User-Agent": USER_AGENT}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
|
||||
return resp.headers.get("ETag") or resp.headers.get("Last-Modified") or ""
|
||||
except (urllib.error.URLError, OSError):
|
||||
@@ -114,8 +120,15 @@ def check(registry_path: str = DEFAULT_REGISTRY) -> dict:
|
||||
status = "OK"
|
||||
else:
|
||||
status = "UPDATED"
|
||||
results.append({"zip": z, "status": status, "key": key,
|
||||
"stored_etag": stored, "remote_etag": remote or ""})
|
||||
results.append(
|
||||
{
|
||||
"zip": z,
|
||||
"status": status,
|
||||
"key": key,
|
||||
"stored_etag": stored,
|
||||
"remote_etag": remote or "",
|
||||
}
|
||||
)
|
||||
|
||||
return {"entries": results}
|
||||
|
||||
@@ -144,8 +157,13 @@ def update_changed(report: dict) -> None:
|
||||
if e["status"] == "UPDATED" and e.get("key"):
|
||||
log.info("refreshing %s ...", e["key"])
|
||||
subprocess.run(
|
||||
[sys.executable, "scripts/refresh_data_dirs.py",
|
||||
"--force", "--key", e["key"]],
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/refresh_data_dirs.py",
|
||||
"--force",
|
||||
"--key",
|
||||
e["key"],
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
|
||||
@@ -155,10 +173,15 @@ def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Check buildbot system directory for changes",
|
||||
)
|
||||
parser.add_argument("--update", action="store_true",
|
||||
help="Auto-refresh changed entries")
|
||||
parser.add_argument("--json", action="store_true", dest="json_output",
|
||||
help="Machine-readable JSON output")
|
||||
parser.add_argument(
|
||||
"--update", action="store_true", help="Auto-refresh changed entries"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
dest="json_output",
|
||||
help="Machine-readable JSON output",
|
||||
)
|
||||
parser.add_argument("--registry", default=DEFAULT_REGISTRY)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -26,9 +26,11 @@ def require_yaml():
|
||||
"""Import and return yaml, exiting if PyYAML is not installed."""
|
||||
try:
|
||||
import yaml as _yaml
|
||||
|
||||
return _yaml
|
||||
except ImportError:
|
||||
import sys
|
||||
|
||||
print("Error: PyYAML required (pip install pyyaml)", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
@@ -154,12 +156,17 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
|
||||
if "inherits" in config:
|
||||
parent = load_platform_config(config["inherits"], platforms_dir)
|
||||
merged = {**parent}
|
||||
merged.update({k: v for k, v in config.items() if k not in ("inherits", "overrides")})
|
||||
merged.update(
|
||||
{k: v for k, v in config.items() if k not in ("inherits", "overrides")}
|
||||
)
|
||||
if "overrides" in config and "systems" in config["overrides"]:
|
||||
merged.setdefault("systems", {})
|
||||
for sys_id, override in config["overrides"]["systems"].items():
|
||||
if sys_id in merged["systems"]:
|
||||
merged["systems"][sys_id] = {**merged["systems"][sys_id], **override}
|
||||
merged["systems"][sys_id] = {
|
||||
**merged["systems"][sys_id],
|
||||
**override,
|
||||
}
|
||||
else:
|
||||
merged["systems"][sys_id] = override
|
||||
config = merged
|
||||
@@ -346,12 +353,14 @@ def list_available_targets(
|
||||
result = []
|
||||
for tname, tdata in sorted(data.get("targets", {}).items()):
|
||||
aliases = overrides.get(tname, {}).get("aliases", [])
|
||||
result.append({
|
||||
"name": tname,
|
||||
"architecture": tdata.get("architecture", ""),
|
||||
"core_count": len(tdata.get("cores", [])),
|
||||
"aliases": aliases,
|
||||
})
|
||||
result.append(
|
||||
{
|
||||
"name": tname,
|
||||
"architecture": tdata.get("architecture", ""),
|
||||
"core_count": len(tdata.get("cores", [])),
|
||||
"aliases": aliases,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@@ -398,7 +407,9 @@ def resolve_local_file(
|
||||
if hint_base and hint_base not in names_to_try:
|
||||
names_to_try.append(hint_base)
|
||||
|
||||
md5_list = [m.strip().lower() for m in md5_raw.split(",") if m.strip()] if md5_raw else []
|
||||
md5_list = (
|
||||
[m.strip().lower() for m in md5_raw.split(",") if m.strip()] if md5_raw else []
|
||||
)
|
||||
files_db = db.get("files", {})
|
||||
by_md5 = db.get("indexes", {}).get("by_md5", {})
|
||||
by_name = db.get("indexes", {}).get("by_name", {})
|
||||
@@ -480,7 +491,9 @@ def resolve_local_file(
|
||||
|
||||
if candidates:
|
||||
if zipped_file:
|
||||
candidates = [(p, m) for p, m in candidates if ".zip" in os.path.basename(p)]
|
||||
candidates = [
|
||||
(p, m) for p, m in candidates if ".zip" in os.path.basename(p)
|
||||
]
|
||||
if md5_set:
|
||||
for path, db_md5 in candidates:
|
||||
if ".zip" in os.path.basename(path):
|
||||
@@ -530,7 +543,11 @@ def resolve_local_file(
|
||||
if canonical and canonical != name:
|
||||
canonical_entry = {"name": canonical}
|
||||
result = resolve_local_file(
|
||||
canonical_entry, db, zip_contents, dest_hint, _depth=_depth + 1,
|
||||
canonical_entry,
|
||||
db,
|
||||
zip_contents,
|
||||
dest_hint,
|
||||
_depth=_depth + 1,
|
||||
data_dir_registry=data_dir_registry,
|
||||
)
|
||||
if result[0]:
|
||||
@@ -643,9 +660,7 @@ def build_zip_contents_index(db: dict, max_entry_size: int = 512 * 1024 * 1024)
|
||||
if path.endswith(".zip") and os.path.exists(path):
|
||||
zip_entries.append((path, sha1))
|
||||
|
||||
fingerprint = frozenset(
|
||||
(path, os.path.getmtime(path)) for path, _ in zip_entries
|
||||
)
|
||||
fingerprint = frozenset((path, os.path.getmtime(path)) for path, _ in zip_entries)
|
||||
if _zip_contents_cache is not None and _zip_contents_cache[0] == fingerprint:
|
||||
return _zip_contents_cache[1]
|
||||
|
||||
@@ -672,7 +687,8 @@ _emulator_profiles_cache: dict[tuple[str, bool], dict[str, dict]] = {}
|
||||
|
||||
|
||||
def load_emulator_profiles(
|
||||
emulators_dir: str, skip_aliases: bool = True,
|
||||
emulators_dir: str,
|
||||
skip_aliases: bool = True,
|
||||
) -> dict[str, dict]:
|
||||
"""Load all emulator YAML profiles from a directory (cached)."""
|
||||
cache_key = (os.path.realpath(emulators_dir), skip_aliases)
|
||||
@@ -701,7 +717,8 @@ def load_emulator_profiles(
|
||||
|
||||
|
||||
def group_identical_platforms(
|
||||
platforms: list[str], platforms_dir: str,
|
||||
platforms: list[str],
|
||||
platforms_dir: str,
|
||||
target_cores_cache: dict[str, set[str] | None] | None = None,
|
||||
) -> list[tuple[list[str], str]]:
|
||||
"""Group platforms that produce identical packs (same files + base_destination).
|
||||
@@ -744,7 +761,9 @@ def group_identical_platforms(
|
||||
fp = hashlib.sha1(f"{fp}|{tc_str}".encode()).hexdigest()
|
||||
fingerprints.setdefault(fp, []).append(platform)
|
||||
# Prefer the root platform (no inherits) as representative
|
||||
if fp not in representatives or (not inherits[platform] and inherits.get(representatives[fp], False)):
|
||||
if fp not in representatives or (
|
||||
not inherits[platform] and inherits.get(representatives[fp], False)
|
||||
):
|
||||
representatives[fp] = platform
|
||||
|
||||
result = []
|
||||
@@ -756,7 +775,8 @@ def group_identical_platforms(
|
||||
|
||||
|
||||
def resolve_platform_cores(
|
||||
config: dict, profiles: dict[str, dict],
|
||||
config: dict,
|
||||
profiles: dict[str, dict],
|
||||
target_cores: set[str] | None = None,
|
||||
) -> set[str]:
|
||||
"""Resolve which emulator profiles are relevant for a platform.
|
||||
@@ -773,9 +793,9 @@ def resolve_platform_cores(
|
||||
|
||||
if cores_config == "all_libretro":
|
||||
result = {
|
||||
name for name, p in profiles.items()
|
||||
if "libretro" in p.get("type", "")
|
||||
and p.get("type") != "alias"
|
||||
name
|
||||
for name, p in profiles.items()
|
||||
if "libretro" in p.get("type", "") and p.get("type") != "alias"
|
||||
}
|
||||
elif isinstance(cores_config, list):
|
||||
core_set = {str(c) for c in cores_config}
|
||||
@@ -786,25 +806,22 @@ def resolve_platform_cores(
|
||||
core_to_profile[name] = name
|
||||
for core_name in p.get("cores", []):
|
||||
core_to_profile[str(core_name)] = name
|
||||
result = {
|
||||
core_to_profile[c]
|
||||
for c in core_set
|
||||
if c in core_to_profile
|
||||
}
|
||||
result = {core_to_profile[c] for c in core_set if c in core_to_profile}
|
||||
# Support "all_libretro" as a list element: combines all libretro
|
||||
# profiles with explicitly listed standalone cores (e.g. RetroDECK
|
||||
# ships RetroArch + standalone emulators)
|
||||
if "all_libretro" in core_set or "retroarch" in core_set:
|
||||
result |= {
|
||||
name for name, p in profiles.items()
|
||||
if "libretro" in p.get("type", "")
|
||||
and p.get("type") != "alias"
|
||||
name
|
||||
for name, p in profiles.items()
|
||||
if "libretro" in p.get("type", "") and p.get("type") != "alias"
|
||||
}
|
||||
else:
|
||||
# Fallback: system ID intersection with normalization
|
||||
norm_plat_systems = {_norm_system_id(s) for s in config.get("systems", {})}
|
||||
result = {
|
||||
name for name, p in profiles.items()
|
||||
name
|
||||
for name, p in profiles.items()
|
||||
if {_norm_system_id(s) for s in p.get("systems", [])} & norm_plat_systems
|
||||
and p.get("type") != "alias"
|
||||
}
|
||||
@@ -826,11 +843,34 @@ def resolve_platform_cores(
|
||||
|
||||
|
||||
MANUFACTURER_PREFIXES = (
|
||||
"acorn-", "apple-", "microsoft-", "nintendo-", "sony-", "sega-",
|
||||
"snk-", "panasonic-", "nec-", "epoch-", "mattel-", "fairchild-",
|
||||
"hartung-", "tiger-", "magnavox-", "philips-", "bandai-", "casio-",
|
||||
"coleco-", "commodore-", "sharp-", "sinclair-", "atari-", "sammy-",
|
||||
"gce-", "interton-", "texas-instruments-", "videoton-",
|
||||
"acorn-",
|
||||
"apple-",
|
||||
"microsoft-",
|
||||
"nintendo-",
|
||||
"sony-",
|
||||
"sega-",
|
||||
"snk-",
|
||||
"panasonic-",
|
||||
"nec-",
|
||||
"epoch-",
|
||||
"mattel-",
|
||||
"fairchild-",
|
||||
"hartung-",
|
||||
"tiger-",
|
||||
"magnavox-",
|
||||
"philips-",
|
||||
"bandai-",
|
||||
"casio-",
|
||||
"coleco-",
|
||||
"commodore-",
|
||||
"sharp-",
|
||||
"sinclair-",
|
||||
"atari-",
|
||||
"sammy-",
|
||||
"gce-",
|
||||
"interton-",
|
||||
"texas-instruments-",
|
||||
"videoton-",
|
||||
)
|
||||
|
||||
|
||||
@@ -877,7 +917,7 @@ def _norm_system_id(sid: str) -> str:
|
||||
s = SYSTEM_ALIASES.get(s, s)
|
||||
for prefix in MANUFACTURER_PREFIXES:
|
||||
if s.startswith(prefix):
|
||||
s = s[len(prefix):]
|
||||
s = s[len(prefix) :]
|
||||
break
|
||||
return s.replace("-", "")
|
||||
|
||||
@@ -984,9 +1024,9 @@ def expand_platform_declared_names(config: dict, db: dict) -> set[str]:
|
||||
import re
|
||||
|
||||
_TIMESTAMP_PATTERNS = [
|
||||
re.compile(r'"generated_at":\s*"[^"]*"'), # database.json
|
||||
re.compile(r'\*Auto-generated on [^*]*\*'), # README.md
|
||||
re.compile(r'\*Generated on [^*]*\*'), # docs site pages
|
||||
re.compile(r'"generated_at":\s*"[^"]*"'), # database.json
|
||||
re.compile(r"\*Auto-generated on [^*]*\*"), # README.md
|
||||
re.compile(r"\*Generated on [^*]*\*"), # docs site pages
|
||||
]
|
||||
|
||||
|
||||
@@ -1023,8 +1063,12 @@ LARGE_FILES_REPO = "Abdess/retrobios"
|
||||
LARGE_FILES_CACHE = ".cache/large"
|
||||
|
||||
|
||||
def fetch_large_file(name: str, dest_dir: str = LARGE_FILES_CACHE,
|
||||
expected_sha1: str = "", expected_md5: str = "") -> str | None:
|
||||
def fetch_large_file(
|
||||
name: str,
|
||||
dest_dir: str = LARGE_FILES_CACHE,
|
||||
expected_sha1: str = "",
|
||||
expected_md5: str = "",
|
||||
) -> str | None:
|
||||
"""Download a large file from the 'large-files' GitHub release if not cached."""
|
||||
cached = os.path.join(dest_dir, name)
|
||||
if os.path.exists(cached):
|
||||
@@ -1033,7 +1077,9 @@ def fetch_large_file(name: str, dest_dir: str = LARGE_FILES_CACHE,
|
||||
if expected_sha1 and hashes["sha1"].lower() != expected_sha1.lower():
|
||||
os.unlink(cached)
|
||||
elif expected_md5:
|
||||
md5_list = [m.strip().lower() for m in expected_md5.split(",") if m.strip()]
|
||||
md5_list = [
|
||||
m.strip().lower() for m in expected_md5.split(",") if m.strip()
|
||||
]
|
||||
if hashes["md5"].lower() not in md5_list:
|
||||
os.unlink(cached)
|
||||
else:
|
||||
@@ -1122,8 +1168,9 @@ def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None:
|
||||
file_count = len(systems[sys_id].get("files", []))
|
||||
mfr = systems[sys_id].get("manufacturer", "")
|
||||
mfr_display = f" [{mfr.split('|')[0]}]" if mfr else ""
|
||||
print(f" {sys_id:35s} ({file_count} file{'s' if file_count != 1 else ''}){mfr_display}")
|
||||
|
||||
print(
|
||||
f" {sys_id:35s} ({file_count} file{'s' if file_count != 1 else ''}){mfr_display}"
|
||||
)
|
||||
|
||||
|
||||
def build_target_cores_cache(
|
||||
|
||||
@@ -19,7 +19,13 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import list_registered_platforms, load_database, load_emulator_profiles, load_platform_config, require_yaml
|
||||
from common import (
|
||||
list_registered_platforms,
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
require_yaml,
|
||||
)
|
||||
|
||||
yaml = require_yaml()
|
||||
|
||||
@@ -28,11 +34,15 @@ DEFAULT_PLATFORMS_DIR = "platforms"
|
||||
DEFAULT_DB = "database.json"
|
||||
|
||||
|
||||
def load_platform_files(platforms_dir: str) -> tuple[dict[str, set[str]], dict[str, set[str]]]:
|
||||
def load_platform_files(
|
||||
platforms_dir: str,
|
||||
) -> tuple[dict[str, set[str]], dict[str, set[str]]]:
|
||||
"""Load all platform configs and collect declared filenames + data_directories per system."""
|
||||
declared = {}
|
||||
platform_data_dirs = {}
|
||||
for platform_name in list_registered_platforms(platforms_dir, include_archived=True):
|
||||
for platform_name in list_registered_platforms(
|
||||
platforms_dir, include_archived=True
|
||||
):
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
for sys_id, system in config.get("systems", {}).items():
|
||||
for fe in system.get("files", []):
|
||||
@@ -46,8 +56,9 @@ def load_platform_files(platforms_dir: str) -> tuple[dict[str, set[str]], dict[s
|
||||
return declared, platform_data_dirs
|
||||
|
||||
|
||||
def _build_supplemental_index(data_root: str = "data",
|
||||
bios_root: str = "bios") -> set[str]:
|
||||
def _build_supplemental_index(
|
||||
data_root: str = "data", bios_root: str = "bios"
|
||||
) -> set[str]:
|
||||
"""Build a set of filenames and directory names in data/ and inside bios/ ZIPs."""
|
||||
names: set[str] = set()
|
||||
root_path = Path(data_root)
|
||||
@@ -76,12 +87,15 @@ def _build_supplemental_index(data_root: str = "data",
|
||||
names.add(dpath.name + "/")
|
||||
names.add(dpath.name.lower() + "/")
|
||||
import zipfile
|
||||
|
||||
for zpath in bios_path.rglob("*.zip"):
|
||||
try:
|
||||
with zipfile.ZipFile(zpath) as zf:
|
||||
for member in zf.namelist():
|
||||
if not member.endswith("/"):
|
||||
basename = member.rsplit("/", 1)[-1] if "/" in member else member
|
||||
basename = (
|
||||
member.rsplit("/", 1)[-1] if "/" in member else member
|
||||
)
|
||||
names.add(basename)
|
||||
names.add(basename.lower())
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
@@ -89,8 +103,12 @@ def _build_supplemental_index(data_root: str = "data",
|
||||
return names
|
||||
|
||||
|
||||
def _find_in_repo(fname: str, by_name: dict[str, list], by_name_lower: dict[str, str],
|
||||
data_names: set[str] | None = None) -> bool:
|
||||
def _find_in_repo(
|
||||
fname: str,
|
||||
by_name: dict[str, list],
|
||||
by_name_lower: dict[str, str],
|
||||
data_names: set[str] | None = None,
|
||||
) -> bool:
|
||||
if fname in by_name:
|
||||
return True
|
||||
# For directory entries or paths, extract the meaningful basename
|
||||
@@ -170,7 +188,9 @@ def cross_reference(
|
||||
if not in_repo:
|
||||
path_field = f.get("path", "")
|
||||
if path_field and path_field != fname:
|
||||
in_repo = _find_in_repo(path_field, by_name, by_name_lower, data_names)
|
||||
in_repo = _find_in_repo(
|
||||
path_field, by_name, by_name_lower, data_names
|
||||
)
|
||||
# Try MD5 hash match (handles files that exist under different names)
|
||||
if not in_repo:
|
||||
md5_raw = f.get("md5", "")
|
||||
@@ -231,9 +251,11 @@ def print_report(report: dict) -> None:
|
||||
status = f"{data['gap_in_repo']} in repo, {data['gap_missing']} missing"
|
||||
|
||||
print(f"\n{data['emulator']} ({', '.join(data['systems'])})")
|
||||
print(f" {data['total_files']} files in profile, "
|
||||
f"{data['platform_covered']} declared by platforms, "
|
||||
f"{gaps} undeclared")
|
||||
print(
|
||||
f" {data['total_files']} files in profile, "
|
||||
f"{data['platform_covered']} declared by platforms, "
|
||||
f"{gaps} undeclared"
|
||||
)
|
||||
|
||||
if gaps > 0:
|
||||
print(f" Gaps: {status}")
|
||||
@@ -259,7 +281,9 @@ def main():
|
||||
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
|
||||
parser.add_argument("--db", default=DEFAULT_DB)
|
||||
parser.add_argument("--emulator", "-e", help="Analyze single emulator")
|
||||
parser.add_argument("--platform", "-p", help="Platform name (required for --target)")
|
||||
parser.add_argument(
|
||||
"--platform", "-p", help="Platform name (required for --target)"
|
||||
)
|
||||
parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)")
|
||||
parser.add_argument("--json", action="store_true", help="JSON output")
|
||||
args = parser.parse_args()
|
||||
@@ -272,7 +296,10 @@ def main():
|
||||
if not args.platform:
|
||||
parser.error("--target requires --platform")
|
||||
from common import load_target_config, resolve_platform_cores
|
||||
target_cores = load_target_config(args.platform, args.target, args.platforms_dir)
|
||||
|
||||
target_cores = load_target_config(
|
||||
args.platform, args.target, args.platforms_dir
|
||||
)
|
||||
config = load_platform_config(args.platform, args.platforms_dir)
|
||||
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
|
||||
profiles = {k: v for k, v in profiles.items() if k in relevant}
|
||||
|
||||
@@ -14,6 +14,7 @@ Source refs:
|
||||
Azahar src/core/hw/rsa/rsa.cpp
|
||||
Azahar src/core/file_sys/otp.cpp
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
@@ -22,9 +23,9 @@ import subprocess
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Key file parsing (keys.txt / aes_keys.txt format)
|
||||
|
||||
|
||||
def parse_keys_file(path: str | Path) -> dict[str, dict[str, bytes]]:
|
||||
"""Parse a 3DS keys file with :AES, :RSA, :ECC sections.
|
||||
|
||||
@@ -67,6 +68,7 @@ def find_keys_file(bios_dir: str | Path) -> Path | None:
|
||||
|
||||
# Pure Python RSA-2048 PKCS1v15 SHA256 verification (zero dependencies)
|
||||
|
||||
|
||||
def _rsa_verify_pkcs1v15_sha256(
|
||||
message: bytes,
|
||||
signature: bytes,
|
||||
@@ -98,14 +100,29 @@ def _rsa_verify_pkcs1v15_sha256(
|
||||
# PKCS#1 v1.5 signature encoding: 0x00 0x01 [0xFF padding] 0x00 [DigestInfo]
|
||||
# DigestInfo for SHA-256:
|
||||
# SEQUENCE { SEQUENCE { OID sha256, NULL }, OCTET STRING hash }
|
||||
digest_info_prefix = bytes([
|
||||
0x30, 0x31, # SEQUENCE (49 bytes)
|
||||
0x30, 0x0D, # SEQUENCE (13 bytes)
|
||||
0x06, 0x09, # OID (9 bytes)
|
||||
0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, # sha256
|
||||
0x05, 0x00, # NULL
|
||||
0x04, 0x20, # OCTET STRING (32 bytes)
|
||||
])
|
||||
digest_info_prefix = bytes(
|
||||
[
|
||||
0x30,
|
||||
0x31, # SEQUENCE (49 bytes)
|
||||
0x30,
|
||||
0x0D, # SEQUENCE (13 bytes)
|
||||
0x06,
|
||||
0x09, # OID (9 bytes)
|
||||
0x60,
|
||||
0x86,
|
||||
0x48,
|
||||
0x01,
|
||||
0x65,
|
||||
0x03,
|
||||
0x04,
|
||||
0x02,
|
||||
0x01, # sha256
|
||||
0x05,
|
||||
0x00, # NULL
|
||||
0x04,
|
||||
0x20, # OCTET STRING (32 bytes)
|
||||
]
|
||||
)
|
||||
|
||||
sha256_hash = hashlib.sha256(message).digest()
|
||||
expected_digest_info = digest_info_prefix + sha256_hash
|
||||
@@ -122,11 +139,13 @@ def _rsa_verify_pkcs1v15_sha256(
|
||||
|
||||
# AES-128-CBC decryption (with fallback)
|
||||
|
||||
|
||||
def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
|
||||
"""Decrypt AES-128-CBC without padding."""
|
||||
# Try cryptography library first
|
||||
try:
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
|
||||
cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
|
||||
decryptor = cipher.decryptor()
|
||||
return decryptor.update(data) + decryptor.finalize()
|
||||
@@ -136,6 +155,7 @@ def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
|
||||
# Try pycryptodome
|
||||
try:
|
||||
from Crypto.Cipher import AES # type: ignore[import-untyped]
|
||||
|
||||
cipher = AES.new(key, AES.MODE_CBC, iv)
|
||||
return cipher.decrypt(data)
|
||||
except ImportError:
|
||||
@@ -145,8 +165,15 @@ def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"openssl", "enc", "-aes-128-cbc", "-d",
|
||||
"-K", key.hex(), "-iv", iv.hex(), "-nopad",
|
||||
"openssl",
|
||||
"enc",
|
||||
"-aes-128-cbc",
|
||||
"-d",
|
||||
"-K",
|
||||
key.hex(),
|
||||
"-iv",
|
||||
iv.hex(),
|
||||
"-nopad",
|
||||
],
|
||||
input=data,
|
||||
capture_output=True,
|
||||
@@ -162,6 +189,7 @@ def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
|
||||
|
||||
# File verification functions
|
||||
|
||||
|
||||
def verify_secure_info_a(
|
||||
filepath: str | Path,
|
||||
keys: dict[str, dict[str, bytes]],
|
||||
@@ -204,7 +232,10 @@ def verify_secure_info_a(
|
||||
continue
|
||||
modified_body = bytes([test_region]) + body[1:]
|
||||
if _rsa_verify_pkcs1v15_sha256(modified_body, signature, modulus, exponent):
|
||||
return False, f"signature invalid (region changed from {test_region} to {region_byte})"
|
||||
return (
|
||||
False,
|
||||
f"signature invalid (region changed from {test_region} to {region_byte})",
|
||||
)
|
||||
|
||||
return False, "signature invalid"
|
||||
|
||||
@@ -307,7 +338,7 @@ def verify_otp(
|
||||
|
||||
Returns (valid, reason_string).
|
||||
"""
|
||||
from sect233r1 import ecdsa_verify_sha256, _ec_mul, _Gx, _Gy, _N
|
||||
from sect233r1 import _N, _ec_mul, _Gx, _Gy, ecdsa_verify_sha256
|
||||
|
||||
data = bytearray(Path(filepath).read_bytes())
|
||||
|
||||
@@ -322,7 +353,10 @@ def verify_otp(
|
||||
magic = struct.unpack_from("<I", data, 0)[0]
|
||||
if magic != 0xDEADB00F:
|
||||
if not otp_key or not otp_iv:
|
||||
return False, "encrypted OTP but missing AES keys (otpKey/otpIV) in keys file"
|
||||
return (
|
||||
False,
|
||||
"encrypted OTP but missing AES keys (otpKey/otpIV) in keys file",
|
||||
)
|
||||
try:
|
||||
data = bytearray(_aes_128_cbc_decrypt(bytes(data), otp_key, otp_iv))
|
||||
except RuntimeError as e:
|
||||
@@ -343,7 +377,10 @@ def verify_otp(
|
||||
ecc_keys = keys.get("ECC", {})
|
||||
root_public_xy = ecc_keys.get("rootPublicXY")
|
||||
if not root_public_xy or len(root_public_xy) != 60:
|
||||
return True, "decrypted, magic valid, SHA-256 valid (ECC skipped: no rootPublicXY)"
|
||||
return (
|
||||
True,
|
||||
"decrypted, magic valid, SHA-256 valid (ECC skipped: no rootPublicXY)",
|
||||
)
|
||||
|
||||
# Extract CTCert fields from OTP body
|
||||
device_id = struct.unpack_from("<I", data, 0x04)[0]
|
||||
@@ -368,9 +405,7 @@ def verify_otp(
|
||||
pub_point = _ec_mul(priv_key_int, (_Gx, _Gy))
|
||||
if pub_point is None:
|
||||
return False, "ECC cert: derived public key is point at infinity"
|
||||
pub_key_xy = (
|
||||
pub_point[0].to_bytes(30, "big") + pub_point[1].to_bytes(30, "big")
|
||||
)
|
||||
pub_key_xy = pub_point[0].to_bytes(30, "big") + pub_point[1].to_bytes(30, "big")
|
||||
|
||||
# Build certificate body (what was signed)
|
||||
# Issuer: "Nintendo CA - G3_NintendoCTR2prod" or "...dev"
|
||||
@@ -379,12 +414,12 @@ def verify_otp(
|
||||
issuer_str = b"Nintendo CA - G3_NintendoCTR2prod"
|
||||
else:
|
||||
issuer_str = b"Nintendo CA - G3_NintendoCTR2dev"
|
||||
issuer[:len(issuer_str)] = issuer_str
|
||||
issuer[: len(issuer_str)] = issuer_str
|
||||
|
||||
# Name: "CT{device_id:08X}-{system_type:02X}"
|
||||
name = bytearray(0x40)
|
||||
name_str = f"CT{device_id:08X}-{system_type:02X}".encode()
|
||||
name[:len(name_str)] = name_str
|
||||
name[: len(name_str)] = name_str
|
||||
|
||||
# Key type = 2 (ECC), big-endian u32
|
||||
key_type = struct.pack(">I", 2)
|
||||
|
||||
@@ -17,6 +17,7 @@ Two types of deduplication:
|
||||
|
||||
After dedup, run generate_db.py --force to rebuild database indexes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
@@ -110,13 +111,10 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
|
||||
unique_names = sorted(by_name.keys())
|
||||
if len(unique_names) > 1:
|
||||
# Check if these are all in MAME/Arcade dirs AND all ZIPs
|
||||
all_mame_zip = (
|
||||
all(
|
||||
any(_is_mame_dir(p) for p in name_paths)
|
||||
for name_paths in by_name.values()
|
||||
)
|
||||
and all(n.endswith(".zip") for n in unique_names)
|
||||
)
|
||||
all_mame_zip = all(
|
||||
any(_is_mame_dir(p) for p in name_paths)
|
||||
for name_paths in by_name.values()
|
||||
) and all(n.endswith(".zip") for n in unique_names)
|
||||
if all_mame_zip:
|
||||
# MAME device clones: different ZIP names, same ROM content
|
||||
# Keep one canonical, remove clones, record in clone map
|
||||
@@ -202,7 +200,9 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
|
||||
|
||||
prefix = "Would remove" if dry_run else "Removed"
|
||||
print(f"\n{prefix}: {total_removed} files")
|
||||
print(f"Space {'to save' if dry_run else 'saved'}: {total_saved / 1024 / 1024:.1f} MB")
|
||||
print(
|
||||
f"Space {'to save' if dry_run else 'saved'}: {total_saved / 1024 / 1024:.1f} MB"
|
||||
)
|
||||
if not dry_run and empty_cleaned:
|
||||
print(f"Cleaned {empty_cleaned} empty directories")
|
||||
|
||||
@@ -211,21 +211,27 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
|
||||
clone_path = "_mame_clones.json"
|
||||
if dry_run:
|
||||
print(f"\nWould write MAME clone map: {clone_path}")
|
||||
print(f" {len(mame_clones)} canonical ZIPs with "
|
||||
f"{sum(len(v['clones']) for v in mame_clones.values())} clones")
|
||||
print(
|
||||
f" {len(mame_clones)} canonical ZIPs with "
|
||||
f"{sum(len(v['clones']) for v in mame_clones.values())} clones"
|
||||
)
|
||||
else:
|
||||
with open(clone_path, "w") as f:
|
||||
json.dump(mame_clones, f, indent=2, sort_keys=True)
|
||||
print(f"\nWrote MAME clone map: {clone_path}")
|
||||
print(f" {len(mame_clones)} canonical ZIPs with "
|
||||
f"{sum(len(v['clones']) for v in mame_clones.values())} clones")
|
||||
print(
|
||||
f" {len(mame_clones)} canonical ZIPs with "
|
||||
f"{sum(len(v['clones']) for v in mame_clones.values())} clones"
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Deduplicate bios/ directory")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview without deleting")
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true", help="Preview without deleting"
|
||||
)
|
||||
parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -22,10 +22,10 @@ Usage:
|
||||
]
|
||||
build_deterministic_zip("neogeo.zip", recipe, atom_store)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import struct
|
||||
import zipfile
|
||||
import zlib
|
||||
from io import BytesIO
|
||||
@@ -63,7 +63,9 @@ def build_deterministic_zip(
|
||||
# Sort by filename for deterministic order
|
||||
sorted_recipe = sorted(recipe, key=lambda r: r["name"])
|
||||
|
||||
with zipfile.ZipFile(str(output_path), "w", compression, compresslevel=_COMPRESS_LEVEL) as zf:
|
||||
with zipfile.ZipFile(
|
||||
str(output_path), "w", compression, compresslevel=_COMPRESS_LEVEL
|
||||
) as zf:
|
||||
for entry in sorted_recipe:
|
||||
name = entry["name"]
|
||||
expected_crc = entry.get("crc32", "").lower()
|
||||
@@ -127,12 +129,14 @@ def extract_atoms_with_names(zip_path: str | Path) -> list[dict]:
|
||||
continue
|
||||
data = zf.read(info.filename)
|
||||
crc = format(zlib.crc32(data) & 0xFFFFFFFF, "08x")
|
||||
result.append({
|
||||
"name": info.filename,
|
||||
"crc32": crc,
|
||||
"size": len(data),
|
||||
"data": data,
|
||||
})
|
||||
result.append(
|
||||
{
|
||||
"name": info.filename,
|
||||
"crc32": crc,
|
||||
"size": len(data),
|
||||
"data": data,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@@ -154,7 +158,9 @@ def verify_zip_determinism(zip_path: str | Path) -> tuple[bool, str, str]:
|
||||
# Rebuild to memory
|
||||
buf = BytesIO()
|
||||
sorted_recipe = sorted(recipe, key=lambda r: r["name"])
|
||||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED, compresslevel=_COMPRESS_LEVEL) as zf:
|
||||
with zipfile.ZipFile(
|
||||
buf, "w", zipfile.ZIP_DEFLATED, compresslevel=_COMPRESS_LEVEL
|
||||
) as zf:
|
||||
for entry in sorted_recipe:
|
||||
info = zipfile.ZipInfo(filename=entry["name"], date_time=_FIXED_DATE_TIME)
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
|
||||
@@ -78,13 +78,17 @@ def _format_terminal(report: dict) -> str:
|
||||
lines.append(f" + {m['name']} [{cores}]")
|
||||
for h in div.get("hash_mismatch", []):
|
||||
ht = h["hash_type"]
|
||||
lines.append(f" ~ {h['name']} {ht}: {h[f'truth_{ht}']} != {h[f'scraped_{ht}']}")
|
||||
lines.append(
|
||||
f" ~ {h['name']} {ht}: {h[f'truth_{ht}']} != {h[f'scraped_{ht}']}"
|
||||
)
|
||||
for p in div.get("extra_phantom", []):
|
||||
lines.append(f" - {p['name']} (phantom)")
|
||||
for u in div.get("extra_unprofiled", []):
|
||||
lines.append(f" ? {u['name']} (unprofiled)")
|
||||
for r in div.get("required_mismatch", []):
|
||||
lines.append(f" ! {r['name']} required: {r['truth_required']} != {r['scraped_required']}")
|
||||
lines.append(
|
||||
f" ! {r['name']} required: {r['truth_required']} != {r['scraped_required']}"
|
||||
)
|
||||
|
||||
uncovered = report.get("uncovered_systems", [])
|
||||
if uncovered:
|
||||
@@ -125,13 +129,17 @@ def _format_markdown(report: dict) -> str:
|
||||
lines.append(f"- **Add** `{m['name']}`{refs}")
|
||||
for h in div.get("hash_mismatch", []):
|
||||
ht = h["hash_type"]
|
||||
lines.append(f"- **Fix hash** `{h['name']}` {ht}: `{h[f'truth_{ht}']}` != `{h[f'scraped_{ht}']}`")
|
||||
lines.append(
|
||||
f"- **Fix hash** `{h['name']}` {ht}: `{h[f'truth_{ht}']}` != `{h[f'scraped_{ht}']}`"
|
||||
)
|
||||
for p in div.get("extra_phantom", []):
|
||||
lines.append(f"- **Remove** `{p['name']}` (phantom)")
|
||||
for u in div.get("extra_unprofiled", []):
|
||||
lines.append(f"- **Check** `{u['name']}` (unprofiled cores)")
|
||||
for r in div.get("required_mismatch", []):
|
||||
lines.append(f"- **Fix required** `{r['name']}`: truth={r['truth_required']}, scraped={r['scraped_required']}")
|
||||
lines.append(
|
||||
f"- **Fix required** `{r['name']}`: truth={r['truth_required']}, scraped={r['scraped_required']}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
uncovered = report.get("uncovered_systems", [])
|
||||
@@ -148,17 +156,25 @@ def _format_markdown(report: dict) -> str:
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Compare scraped vs truth YAMLs")
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--all", action="store_true", help="diff all registered platforms")
|
||||
group.add_argument(
|
||||
"--all", action="store_true", help="diff all registered platforms"
|
||||
)
|
||||
group.add_argument("--platform", help="diff a single platform")
|
||||
parser.add_argument("--json", action="store_true", dest="json_output", help="JSON output")
|
||||
parser.add_argument("--format", choices=["terminal", "markdown"], default="terminal")
|
||||
parser.add_argument(
|
||||
"--json", action="store_true", dest="json_output", help="JSON output"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--format", choices=["terminal", "markdown"], default="terminal"
|
||||
)
|
||||
parser.add_argument("--truth-dir", default="dist/truth")
|
||||
parser.add_argument("--platforms-dir", default="platforms")
|
||||
parser.add_argument("--include-archived", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.all:
|
||||
platforms = list_registered_platforms(args.platforms_dir, include_archived=args.include_archived)
|
||||
platforms = list_registered_platforms(
|
||||
args.platforms_dir, include_archived=args.include_archived
|
||||
)
|
||||
else:
|
||||
platforms = [args.platform]
|
||||
|
||||
@@ -169,7 +185,10 @@ def main() -> None:
|
||||
truth = _load_truth(args.truth_dir, platform)
|
||||
if truth is None:
|
||||
if not args.json_output:
|
||||
print(f"skip {platform}: no truth YAML in {args.truth_dir}/", file=sys.stderr)
|
||||
print(
|
||||
f"skip {platform}: no truth YAML in {args.truth_dir}/",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
|
||||
@@ -16,8 +16,8 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
@@ -31,10 +31,13 @@ REPO = "Abdess/retrobios"
|
||||
def get_latest_release() -> dict:
|
||||
"""Fetch latest release info from GitHub API."""
|
||||
url = f"{GITHUB_API}/repos/{REPO}/releases/latest"
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "retrobios-downloader/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
})
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "retrobios-downloader/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
@@ -71,7 +74,9 @@ def find_asset(release: dict, platform: str) -> dict | None:
|
||||
|
||||
def download_file(url: str, dest: str, expected_size: int = 0):
|
||||
"""Download a file with progress indication."""
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-downloader/1.0"})
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "retrobios-downloader/1.0"}
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=300) as resp:
|
||||
total = int(resp.headers.get("Content-Length", expected_size))
|
||||
@@ -88,7 +93,11 @@ def download_file(url: str, dest: str, expected_size: int = 0):
|
||||
if total > 0:
|
||||
pct = downloaded * 100 // total
|
||||
bar = "=" * (pct // 2) + " " * (50 - pct // 2)
|
||||
print(f"\r [{bar}] {pct}% ({downloaded:,}/{total:,})", end="", flush=True)
|
||||
print(
|
||||
f"\r [{bar}] {pct}% ({downloaded:,}/{total:,})",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print()
|
||||
|
||||
@@ -114,11 +123,14 @@ def verify_files(platform: str, dest_dir: str, release: dict):
|
||||
return
|
||||
|
||||
import tempfile
|
||||
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".json", delete=False)
|
||||
tmp.close()
|
||||
|
||||
try:
|
||||
download_file(db_asset["browser_download_url"], tmp.name, db_asset.get("size", 0))
|
||||
download_file(
|
||||
db_asset["browser_download_url"], tmp.name, db_asset.get("size", 0)
|
||||
)
|
||||
with open(tmp.name) as f:
|
||||
db = json.load(f)
|
||||
finally:
|
||||
@@ -142,7 +154,9 @@ def verify_files(platform: str, dest_dir: str, release: dict):
|
||||
break
|
||||
else:
|
||||
mismatched += 1
|
||||
print(f" MISMATCH: {name} (expected {sha1[:12]}..., got {local_sha1[:12]}...)")
|
||||
print(
|
||||
f" MISMATCH: {name} (expected {sha1[:12]}..., got {local_sha1[:12]}...)"
|
||||
)
|
||||
found = True
|
||||
break
|
||||
|
||||
@@ -166,7 +180,7 @@ def show_info(platform: str, release: dict):
|
||||
|
||||
print(f" Platform: {platform}")
|
||||
print(f" File: {asset['name']}")
|
||||
print(f" Size: {asset['size']:,} bytes ({asset['size'] / (1024*1024):.1f} MB)")
|
||||
print(f" Size: {asset['size']:,} bytes ({asset['size'] / (1024 * 1024):.1f} MB)")
|
||||
print(f" Downloads: {asset.get('download_count', 'N/A')}")
|
||||
print(f" Updated: {asset.get('updated_at', 'N/A')}")
|
||||
|
||||
@@ -200,7 +214,12 @@ Examples:
|
||||
print(f" - {p}")
|
||||
else:
|
||||
print("No platform packs found in latest release")
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, OSError, json.JSONDecodeError) as e:
|
||||
except (
|
||||
urllib.error.URLError,
|
||||
urllib.error.HTTPError,
|
||||
OSError,
|
||||
json.JSONDecodeError,
|
||||
) as e:
|
||||
print(f"Error: {e}")
|
||||
return
|
||||
|
||||
@@ -233,6 +252,7 @@ Examples:
|
||||
sys.exit(1)
|
||||
|
||||
import tempfile
|
||||
|
||||
fd, zip_path = tempfile.mkstemp(suffix=".zip")
|
||||
os.close(fd)
|
||||
|
||||
|
||||
@@ -9,11 +9,9 @@ from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import yaml
|
||||
|
||||
from common import list_registered_platforms, load_platform_config
|
||||
from exporter import discover_exporters
|
||||
|
||||
|
||||
OUTPUT_FILENAMES: dict[str, str] = {
|
||||
"retroarch": "System.dat",
|
||||
"lakka": "System.dat",
|
||||
@@ -94,23 +92,31 @@ def main() -> None:
|
||||
group.add_argument("--all", action="store_true", help="export all platforms")
|
||||
group.add_argument("--platform", help="export a single platform")
|
||||
parser.add_argument(
|
||||
"--output-dir", default="dist/upstream", help="output directory",
|
||||
"--output-dir",
|
||||
default="dist/upstream",
|
||||
help="output directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--truth-dir", default="dist/truth", help="truth YAML directory",
|
||||
"--truth-dir",
|
||||
default="dist/truth",
|
||||
help="truth YAML directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platforms-dir", default="platforms", help="platform configs directory",
|
||||
"--platforms-dir",
|
||||
default="platforms",
|
||||
help="platform configs directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-archived", action="store_true",
|
||||
"--include-archived",
|
||||
action="store_true",
|
||||
help="include archived platforms",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.all:
|
||||
platforms = list_registered_platforms(
|
||||
args.platforms_dir, include_archived=args.include_archived,
|
||||
args.platforms_dir,
|
||||
include_archived=args.include_archived,
|
||||
)
|
||||
else:
|
||||
platforms = [args.platform]
|
||||
|
||||
@@ -38,7 +38,8 @@ class BaseExporter(ABC):
|
||||
|
||||
@staticmethod
|
||||
def _display_name(
|
||||
sys_id: str, scraped_sys: dict | None = None,
|
||||
sys_id: str,
|
||||
scraped_sys: dict | None = None,
|
||||
) -> str:
|
||||
"""Get display name for a system from scraped data or slug."""
|
||||
if scraped_sys:
|
||||
@@ -47,9 +48,28 @@ class BaseExporter(ABC):
|
||||
return name
|
||||
# Fallback: convert slug to display name with acronym handling
|
||||
_UPPER = {
|
||||
"3do", "cdi", "cpc", "cps1", "cps2", "cps3", "dos", "gba",
|
||||
"gbc", "hle", "msx", "nes", "nds", "ngp", "psp", "psx",
|
||||
"sms", "snes", "stv", "tvc", "vb", "zx",
|
||||
"3do",
|
||||
"cdi",
|
||||
"cpc",
|
||||
"cps1",
|
||||
"cps2",
|
||||
"cps3",
|
||||
"dos",
|
||||
"gba",
|
||||
"gbc",
|
||||
"hle",
|
||||
"msx",
|
||||
"nes",
|
||||
"nds",
|
||||
"ngp",
|
||||
"psp",
|
||||
"psx",
|
||||
"sms",
|
||||
"snes",
|
||||
"stv",
|
||||
"tvc",
|
||||
"vb",
|
||||
"zx",
|
||||
}
|
||||
parts = sys_id.replace("-", " ").split()
|
||||
result = []
|
||||
|
||||
@@ -11,8 +11,6 @@ from pathlib import Path
|
||||
from .base_exporter import BaseExporter
|
||||
|
||||
|
||||
|
||||
|
||||
class Exporter(BaseExporter):
|
||||
"""Export truth data to Batocera batocera-systems format."""
|
||||
|
||||
@@ -44,7 +42,9 @@ class Exporter(BaseExporter):
|
||||
continue
|
||||
|
||||
native_id = native_map.get(sys_id, sys_id)
|
||||
scraped_sys = scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
|
||||
scraped_sys = (
|
||||
scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
|
||||
)
|
||||
display_name = self._display_name(sys_id, scraped_sys)
|
||||
|
||||
# Build md5 lookup from scraped data for this system
|
||||
@@ -74,9 +74,7 @@ class Exporter(BaseExporter):
|
||||
# Original format requires md5 for every entry — skip without
|
||||
if not md5:
|
||||
continue
|
||||
bios_parts.append(
|
||||
f'{{ "md5": "{md5}", "file": "bios/{dest}" }}'
|
||||
)
|
||||
bios_parts.append(f'{{ "md5": "{md5}", "file": "bios/{dest}" }}')
|
||||
|
||||
bios_str = ", ".join(bios_parts)
|
||||
line = (
|
||||
|
||||
@@ -156,7 +156,9 @@ class Exporter(BaseExporter):
|
||||
continue
|
||||
md5 = fe.get("md5", "")
|
||||
if isinstance(md5, list):
|
||||
md5s.extend(m for m in md5 if m and re.fullmatch(r"[a-f0-9]{32}", m))
|
||||
md5s.extend(
|
||||
m for m in md5 if m and re.fullmatch(r"[a-f0-9]{32}", m)
|
||||
)
|
||||
elif md5 and re.fullmatch(r"[a-f0-9]{32}", md5):
|
||||
md5s.append(md5)
|
||||
if md5s:
|
||||
@@ -195,7 +197,8 @@ class Exporter(BaseExporter):
|
||||
# Only flag if the system has usable data for the function type
|
||||
if cfg["pattern"] == "md5":
|
||||
has_md5 = any(
|
||||
fe.get("md5") and isinstance(fe.get("md5"), str)
|
||||
fe.get("md5")
|
||||
and isinstance(fe.get("md5"), str)
|
||||
and re.fullmatch(r"[a-f0-9]{32}", fe["md5"])
|
||||
for fe in sys_data["files"]
|
||||
)
|
||||
|
||||
@@ -15,8 +15,6 @@ from pathlib import Path
|
||||
from .base_exporter import BaseExporter
|
||||
|
||||
|
||||
|
||||
|
||||
class Exporter(BaseExporter):
|
||||
"""Export truth data to Recalbox es_bios.xml format."""
|
||||
|
||||
@@ -51,7 +49,9 @@ class Exporter(BaseExporter):
|
||||
continue
|
||||
|
||||
native_id = native_map.get(sys_id, sys_id)
|
||||
scraped_sys = scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
|
||||
scraped_sys = (
|
||||
scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
|
||||
)
|
||||
display_name = self._display_name(sys_id, scraped_sys)
|
||||
|
||||
lines.append(f' <system fullname="{display_name}" platform="{native_id}">')
|
||||
@@ -85,7 +85,9 @@ class Exporter(BaseExporter):
|
||||
|
||||
# Build cores string from _cores
|
||||
cores_list = fe.get("_cores", [])
|
||||
core_str = ",".join(f"libretro/{c}" for c in cores_list) if cores_list else ""
|
||||
core_str = (
|
||||
",".join(f"libretro/{c}" for c in cores_list) if cores_list else ""
|
||||
)
|
||||
|
||||
attrs = [f'path="{path}"']
|
||||
if md5:
|
||||
@@ -97,7 +99,7 @@ class Exporter(BaseExporter):
|
||||
if core_str:
|
||||
attrs.append(f'core="{core_str}"')
|
||||
|
||||
lines.append(f' <bios {" ".join(attrs)} />')
|
||||
lines.append(f" <bios {' '.join(attrs)} />")
|
||||
|
||||
lines.append(" </system>")
|
||||
|
||||
@@ -125,6 +127,9 @@ class Exporter(BaseExporter):
|
||||
if name.startswith("_") or self._is_pattern(name):
|
||||
continue
|
||||
dest = self._dest(fe)
|
||||
if name.lower() not in exported_paths and dest.lower() not in exported_paths:
|
||||
if (
|
||||
name.lower() not in exported_paths
|
||||
and dest.lower() not in exported_paths
|
||||
):
|
||||
issues.append(f"missing: {name}")
|
||||
return issues
|
||||
|
||||
@@ -15,8 +15,6 @@ from pathlib import Path
|
||||
from .base_exporter import BaseExporter
|
||||
|
||||
|
||||
|
||||
|
||||
class Exporter(BaseExporter):
|
||||
"""Export truth data to RetroBat batocera-systems.json format."""
|
||||
|
||||
@@ -47,7 +45,9 @@ class Exporter(BaseExporter):
|
||||
continue
|
||||
|
||||
native_id = native_map.get(sys_id, sys_id)
|
||||
scraped_sys = scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
|
||||
scraped_sys = (
|
||||
scraped_data.get("systems", {}).get(sys_id) if scraped_data else None
|
||||
)
|
||||
display_name = self._display_name(sys_id, scraped_sys)
|
||||
bios_files: list[OrderedDict] = []
|
||||
|
||||
@@ -70,7 +70,9 @@ class Exporter(BaseExporter):
|
||||
|
||||
if bios_files:
|
||||
if native_id in output:
|
||||
existing_files = {e.get("file") for e in output[native_id]["biosFiles"]}
|
||||
existing_files = {
|
||||
e.get("file") for e in output[native_id]["biosFiles"]
|
||||
}
|
||||
for entry in bios_files:
|
||||
if entry.get("file") not in existing_files:
|
||||
output[native_id]["biosFiles"].append(entry)
|
||||
|
||||
@@ -170,7 +170,9 @@ class Exporter(BaseExporter):
|
||||
if native_id in manifest:
|
||||
# Merge into existing component (multiple truth systems
|
||||
# may map to the same native ID)
|
||||
existing_names = {e["filename"] for e in manifest[native_id]["bios"]}
|
||||
existing_names = {
|
||||
e["filename"] for e in manifest[native_id]["bios"]
|
||||
}
|
||||
for entry in bios_entries:
|
||||
if entry["filename"] not in existing_names:
|
||||
manifest[native_id]["bios"].append(entry)
|
||||
|
||||
@@ -58,16 +58,18 @@ class Exporter(BaseExporter):
|
||||
]
|
||||
if version:
|
||||
lines.append(f"\tversion {version}")
|
||||
lines.extend([
|
||||
'\tauthor "libretro"',
|
||||
'\thomepage "https://github.com/libretro/libretro-database/blob/master/dat/System.dat"',
|
||||
'\turl "https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"',
|
||||
")",
|
||||
"",
|
||||
"game (",
|
||||
'\tname "System"',
|
||||
'\tcomment "System"',
|
||||
])
|
||||
lines.extend(
|
||||
[
|
||||
'\tauthor "libretro"',
|
||||
'\thomepage "https://github.com/libretro/libretro-database/blob/master/dat/System.dat"',
|
||||
'\turl "https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"',
|
||||
")",
|
||||
"",
|
||||
"game (",
|
||||
'\tname "System"',
|
||||
'\tcomment "System"',
|
||||
]
|
||||
)
|
||||
|
||||
systems = truth_data.get("systems", {})
|
||||
for sys_id in sorted(systems):
|
||||
|
||||
@@ -44,7 +44,11 @@ def _canonical_name(filepath: Path) -> str:
|
||||
if "/.variants/" in str(filepath) or "\\.variants\\" in str(filepath):
|
||||
# naomi2.zip.da79eca4 -> naomi2.zip
|
||||
parts = name.rsplit(".", 1)
|
||||
if len(parts) == 2 and len(parts[1]) == 8 and all(c in "0123456789abcdef" for c in parts[1]):
|
||||
if (
|
||||
len(parts) == 2
|
||||
and len(parts[1]) == 8
|
||||
and all(c in "0123456789abcdef" for c in parts[1])
|
||||
):
|
||||
return parts[0]
|
||||
return name
|
||||
|
||||
@@ -83,7 +87,9 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
|
||||
if existing_is_variant and not is_variant:
|
||||
if sha1 not in aliases:
|
||||
aliases[sha1] = []
|
||||
aliases[sha1].append({"name": files[sha1]["name"], "path": files[sha1]["path"]})
|
||||
aliases[sha1].append(
|
||||
{"name": files[sha1]["name"], "path": files[sha1]["path"]}
|
||||
)
|
||||
files[sha1] = {
|
||||
"path": rel_path,
|
||||
"name": _canonical_name(filepath),
|
||||
@@ -93,7 +99,9 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
|
||||
else:
|
||||
if sha1 not in aliases:
|
||||
aliases[sha1] = []
|
||||
aliases[sha1].append({"name": _canonical_name(filepath), "path": rel_path})
|
||||
aliases[sha1].append(
|
||||
{"name": _canonical_name(filepath), "path": rel_path}
|
||||
)
|
||||
else:
|
||||
entry = {
|
||||
"path": rel_path,
|
||||
@@ -114,7 +122,9 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
|
||||
# Non-variant file should be primary over .variants/ file
|
||||
if sha1 not in aliases:
|
||||
aliases[sha1] = []
|
||||
aliases[sha1].append({"name": files[sha1]["name"], "path": files[sha1]["path"]})
|
||||
aliases[sha1].append(
|
||||
{"name": files[sha1]["name"], "path": files[sha1]["path"]}
|
||||
)
|
||||
files[sha1] = {
|
||||
"path": rel_path,
|
||||
"name": _canonical_name(filepath),
|
||||
@@ -124,7 +134,9 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
|
||||
else:
|
||||
if sha1 not in aliases:
|
||||
aliases[sha1] = []
|
||||
aliases[sha1].append({"name": _canonical_name(filepath), "path": rel_path})
|
||||
aliases[sha1].append(
|
||||
{"name": _canonical_name(filepath), "path": rel_path}
|
||||
)
|
||||
else:
|
||||
entry = {
|
||||
"path": rel_path,
|
||||
@@ -275,8 +287,12 @@ def _preserve_large_file_entries(files: dict, db_path: str) -> int:
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate multi-indexed BIOS database")
|
||||
parser.add_argument("--force", action="store_true", help="Force rehash all files")
|
||||
parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR, help="BIOS directory path")
|
||||
parser.add_argument("--output", "-o", default=DEFAULT_OUTPUT, help="Output JSON file")
|
||||
parser.add_argument(
|
||||
"--bios-dir", default=DEFAULT_BIOS_DIR, help="BIOS directory path"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", "-o", default=DEFAULT_OUTPUT, help="Output JSON file"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
bios_dir = Path(args.bios_dir)
|
||||
@@ -354,7 +370,10 @@ def _collect_all_aliases(files: dict) -> dict:
|
||||
if platforms_dir.is_dir():
|
||||
try:
|
||||
import yaml
|
||||
for platform_name in list_registered_platforms(str(platforms_dir), include_archived=True):
|
||||
|
||||
for platform_name in list_registered_platforms(
|
||||
str(platforms_dir), include_archived=True
|
||||
):
|
||||
config_file = platforms_dir / f"{platform_name}.yml"
|
||||
try:
|
||||
with open(config_file) as f:
|
||||
@@ -383,6 +402,7 @@ def _collect_all_aliases(files: dict) -> dict:
|
||||
try:
|
||||
sys.path.insert(0, "scripts")
|
||||
from scraper.coreinfo_scraper import Scraper as CoreInfoScraper
|
||||
|
||||
ci_reqs = CoreInfoScraper().fetch_requirements()
|
||||
for r in ci_reqs:
|
||||
basename = r.name
|
||||
@@ -400,6 +420,7 @@ def _collect_all_aliases(files: dict) -> dict:
|
||||
if emulators_dir.is_dir():
|
||||
try:
|
||||
import yaml
|
||||
|
||||
for emu_file in emulators_dir.glob("*.yml"):
|
||||
if emu_file.name.endswith(".old.yml"):
|
||||
continue
|
||||
@@ -454,10 +475,17 @@ def _collect_all_aliases(files: dict) -> dict:
|
||||
# ZX Spectrum
|
||||
["48.rom", "zx48.rom"],
|
||||
# SquirrelJME - all JARs are the same
|
||||
["squirreljme.sqc", "squirreljme.jar", "squirreljme-fast.jar",
|
||||
"squirreljme-slow.jar", "squirreljme-slow-test.jar",
|
||||
"squirreljme-0.3.0.jar", "squirreljme-0.3.0-fast.jar",
|
||||
"squirreljme-0.3.0-slow.jar", "squirreljme-0.3.0-slow-test.jar"],
|
||||
[
|
||||
"squirreljme.sqc",
|
||||
"squirreljme.jar",
|
||||
"squirreljme-fast.jar",
|
||||
"squirreljme-slow.jar",
|
||||
"squirreljme-slow-test.jar",
|
||||
"squirreljme-0.3.0.jar",
|
||||
"squirreljme-0.3.0-fast.jar",
|
||||
"squirreljme-0.3.0-slow.jar",
|
||||
"squirreljme-0.3.0-slow-test.jar",
|
||||
],
|
||||
# Arcade - FBNeo spectrum
|
||||
["spectrum.zip", "fbneo/spectrum.zip", "spec48k.zip"],
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,15 +18,29 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import list_registered_platforms, load_database, load_platform_config, write_if_changed
|
||||
from common import (
|
||||
list_registered_platforms,
|
||||
load_database,
|
||||
load_platform_config,
|
||||
write_if_changed,
|
||||
)
|
||||
from verify import verify_platform
|
||||
|
||||
def compute_coverage(platform_name: str, platforms_dir: str, db: dict,
|
||||
data_registry: dict | None = None,
|
||||
supplemental_names: set[str] | None = None) -> dict:
|
||||
|
||||
def compute_coverage(
|
||||
platform_name: str,
|
||||
platforms_dir: str,
|
||||
db: dict,
|
||||
data_registry: dict | None = None,
|
||||
supplemental_names: set[str] | None = None,
|
||||
) -> dict:
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
result = verify_platform(config, db, data_dir_registry=data_registry,
|
||||
supplemental_names=supplemental_names)
|
||||
result = verify_platform(
|
||||
config,
|
||||
db,
|
||||
data_dir_registry=data_registry,
|
||||
supplemental_names=supplemental_names,
|
||||
)
|
||||
sc = result.get("status_counts", {})
|
||||
ok = sc.get("ok", 0)
|
||||
untested = sc.get("untested", 0)
|
||||
@@ -55,8 +69,9 @@ REPO = "Abdess/retrobios"
|
||||
|
||||
def fetch_contributors() -> list[dict]:
|
||||
"""Fetch contributors from GitHub API, exclude bots."""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
url = f"https://api.github.com/repos/{REPO}/contributors"
|
||||
headers = {"User-Agent": "retrobios-readme/1.0"}
|
||||
token = os.environ.get("GITHUB_TOKEN", "")
|
||||
@@ -68,7 +83,8 @@ def fetch_contributors() -> list[dict]:
|
||||
data = json.loads(resp.read().decode())
|
||||
owner = REPO.split("/")[0]
|
||||
return [
|
||||
c for c in data
|
||||
c
|
||||
for c in data
|
||||
if not c.get("login", "").endswith("[bot]")
|
||||
and c.get("type") == "User"
|
||||
and c.get("login") != owner
|
||||
@@ -87,21 +103,28 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
|
||||
from common import load_data_dir_registry
|
||||
from cross_reference import _build_supplemental_index
|
||||
|
||||
data_registry = load_data_dir_registry(platforms_dir)
|
||||
suppl_names = _build_supplemental_index()
|
||||
|
||||
coverages = {}
|
||||
for name in platform_names:
|
||||
try:
|
||||
coverages[name] = compute_coverage(name, platforms_dir, db,
|
||||
data_registry, suppl_names)
|
||||
coverages[name] = compute_coverage(
|
||||
name, platforms_dir, db, data_registry, suppl_names
|
||||
)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
emulator_count = sum(
|
||||
1 for f in Path("emulators").glob("*.yml")
|
||||
if not f.name.endswith(".old.yml")
|
||||
) if Path("emulators").exists() else 0
|
||||
emulator_count = (
|
||||
sum(
|
||||
1
|
||||
for f in Path("emulators").glob("*.yml")
|
||||
if not f.name.endswith(".old.yml")
|
||||
)
|
||||
if Path("emulators").exists()
|
||||
else 0
|
||||
)
|
||||
|
||||
# Count systems from emulator profiles
|
||||
system_ids: set[str] = set()
|
||||
@@ -109,6 +132,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
if emu_dir.exists():
|
||||
try:
|
||||
import yaml
|
||||
|
||||
for f in emu_dir.glob("*.yml"):
|
||||
if f.name.endswith(".old.yml"):
|
||||
continue
|
||||
@@ -122,8 +146,12 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
"# RetroBIOS",
|
||||
"",
|
||||
f"Complete BIOS and firmware packs for "
|
||||
f"{', '.join(c['platform'] for c in sorted(coverages.values(), key=lambda x: x['platform'])[:-1])}"
|
||||
f", and {sorted(coverages.values(), key=lambda x: x['platform'])[-1]['platform']}.",
|
||||
f"{', '.join(c['platform'] for c in sorted(coverages.values(), key=lambda x: x[
|
||||
'platform'
|
||||
])[:-1])}"
|
||||
f", and {sorted(coverages.values(), key=lambda x: x[
|
||||
'platform'
|
||||
])[-1]['platform']}.",
|
||||
"",
|
||||
f"**{total_files:,}** verified files across **{len(system_ids)}** systems,"
|
||||
f" ready to extract into your emulator's BIOS directory.",
|
||||
@@ -170,48 +198,78 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
display = cov["platform"]
|
||||
path = extract_paths.get(display, "")
|
||||
lines.append(
|
||||
f"| {display} | {cov['total']} | {path} | "
|
||||
f"[Download]({RELEASE_URL}) |"
|
||||
f"| {display} | {cov['total']} | {path} | [Download]({RELEASE_URL}) |"
|
||||
)
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"## What's included",
|
||||
"",
|
||||
"BIOS, firmware, and system files for consoles from Atari to PlayStation 3.",
|
||||
f"Each file is checked against the emulator's source code to match what the"
|
||||
f" code actually loads at runtime.",
|
||||
"",
|
||||
f"- **{len(coverages)} platforms** supported with platform-specific verification",
|
||||
f"- **{emulator_count} emulators** profiled from source (RetroArch cores + standalone)",
|
||||
f"- **{len(system_ids)} systems** covered (NES, SNES, PlayStation, Saturn, Dreamcast, ...)",
|
||||
f"- **{total_files:,} files** verified with MD5, SHA1, CRC32 checksums",
|
||||
f"- **{size_mb:.0f} MB** total collection size",
|
||||
"",
|
||||
"## Supported systems",
|
||||
"",
|
||||
])
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## What's included",
|
||||
"",
|
||||
"BIOS, firmware, and system files for consoles from Atari to PlayStation 3.",
|
||||
"Each file is checked against the emulator's source code to match what the"
|
||||
" code actually loads at runtime.",
|
||||
"",
|
||||
f"- **{len(coverages)} platforms** supported with platform-specific verification",
|
||||
f"- **{emulator_count} emulators** profiled from source (RetroArch cores + standalone)",
|
||||
f"- **{len(system_ids)} systems** covered (NES, SNES, PlayStation, Saturn, Dreamcast, ...)",
|
||||
f"- **{total_files:,} files** verified with MD5, SHA1, CRC32 checksums",
|
||||
f"- **{size_mb:.0f} MB** total collection size",
|
||||
"",
|
||||
"## Supported systems",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
# Show well-known systems for SEO, link to full list
|
||||
well_known = [
|
||||
"NES", "SNES", "Nintendo 64", "GameCube", "Wii", "Game Boy", "Game Boy Advance",
|
||||
"Nintendo DS", "Nintendo 3DS", "Switch",
|
||||
"PlayStation", "PlayStation 2", "PlayStation 3", "PSP", "PS Vita",
|
||||
"Mega Drive", "Saturn", "Dreamcast", "Game Gear", "Master System",
|
||||
"Neo Geo", "Atari 2600", "Atari 7800", "Atari Lynx", "Atari ST",
|
||||
"MSX", "PC Engine", "TurboGrafx-16", "ColecoVision", "Intellivision",
|
||||
"Commodore 64", "Amiga", "ZX Spectrum", "Arcade (MAME)",
|
||||
"NES",
|
||||
"SNES",
|
||||
"Nintendo 64",
|
||||
"GameCube",
|
||||
"Wii",
|
||||
"Game Boy",
|
||||
"Game Boy Advance",
|
||||
"Nintendo DS",
|
||||
"Nintendo 3DS",
|
||||
"Switch",
|
||||
"PlayStation",
|
||||
"PlayStation 2",
|
||||
"PlayStation 3",
|
||||
"PSP",
|
||||
"PS Vita",
|
||||
"Mega Drive",
|
||||
"Saturn",
|
||||
"Dreamcast",
|
||||
"Game Gear",
|
||||
"Master System",
|
||||
"Neo Geo",
|
||||
"Atari 2600",
|
||||
"Atari 7800",
|
||||
"Atari Lynx",
|
||||
"Atari ST",
|
||||
"MSX",
|
||||
"PC Engine",
|
||||
"TurboGrafx-16",
|
||||
"ColecoVision",
|
||||
"Intellivision",
|
||||
"Commodore 64",
|
||||
"Amiga",
|
||||
"ZX Spectrum",
|
||||
"Arcade (MAME)",
|
||||
]
|
||||
lines.extend([
|
||||
", ".join(well_known) + f", and {len(system_ids) - len(well_known)}+ more.",
|
||||
"",
|
||||
f"Full list with per-file details: **[{SITE_URL}]({SITE_URL})**",
|
||||
"",
|
||||
"## Coverage",
|
||||
"",
|
||||
"| Platform | Coverage | Verified | Untested | Missing |",
|
||||
"|----------|----------|----------|----------|---------|",
|
||||
])
|
||||
lines.extend(
|
||||
[
|
||||
", ".join(well_known) + f", and {len(system_ids) - len(well_known)}+ more.",
|
||||
"",
|
||||
f"Full list with per-file details: **[{SITE_URL}]({SITE_URL})**",
|
||||
"",
|
||||
"## Coverage",
|
||||
"",
|
||||
"| Platform | Coverage | Verified | Untested | Missing |",
|
||||
"|----------|----------|----------|----------|---------|",
|
||||
]
|
||||
)
|
||||
|
||||
for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]):
|
||||
pct = f"{cov['percentage']:.1f}%"
|
||||
@@ -220,62 +278,66 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
f"{cov['verified']} | {cov['untested']} | {cov['missing']} |"
|
||||
)
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"## Build your own pack",
|
||||
"",
|
||||
"Clone the repo and generate packs for any platform, emulator, or system:",
|
||||
"",
|
||||
"```bash",
|
||||
"# Full platform pack",
|
||||
"python scripts/generate_pack.py --platform retroarch --output-dir dist/",
|
||||
"python scripts/generate_pack.py --platform batocera --output-dir dist/",
|
||||
"",
|
||||
"# Single emulator or system",
|
||||
"python scripts/generate_pack.py --emulator dolphin",
|
||||
"python scripts/generate_pack.py --system sony-playstation-2",
|
||||
"",
|
||||
"# List available emulators and systems",
|
||||
"python scripts/generate_pack.py --list-emulators",
|
||||
"python scripts/generate_pack.py --list-systems",
|
||||
"",
|
||||
"# Verify your BIOS collection",
|
||||
"python scripts/verify.py --all",
|
||||
"python scripts/verify.py --platform batocera",
|
||||
"python scripts/verify.py --emulator flycast",
|
||||
"python scripts/verify.py --platform retroarch --verbose # emulator ground truth",
|
||||
"```",
|
||||
"",
|
||||
f"Only dependency: Python 3 + `pyyaml`.",
|
||||
"",
|
||||
"## Documentation site",
|
||||
"",
|
||||
f"The [documentation site]({SITE_URL}) provides:",
|
||||
"",
|
||||
f"- **Per-platform pages** with file-by-file verification status and hashes",
|
||||
f"- **Per-emulator profiles** with source code references for every file",
|
||||
f"- **Per-system pages** showing which emulators and platforms cover each console",
|
||||
f"- **Gap analysis** identifying missing files and undeclared core requirements",
|
||||
f"- **Cross-reference** mapping files across {len(coverages)} platforms and {emulator_count} emulators",
|
||||
"",
|
||||
"## How it works",
|
||||
"",
|
||||
"Documentation and metadata can drift from what emulators actually load.",
|
||||
"To keep packs accurate, each file is checked against the emulator's source code.",
|
||||
"",
|
||||
"1. **Read emulator source code** - trace every file the code loads, its expected hash and size",
|
||||
"2. **Cross-reference with platforms** - match against what each platform declares",
|
||||
"3. **Build packs** - include baseline files plus what each platform's cores need",
|
||||
"4. **Verify** - run platform-native checks and emulator-level validation",
|
||||
"",
|
||||
])
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Build your own pack",
|
||||
"",
|
||||
"Clone the repo and generate packs for any platform, emulator, or system:",
|
||||
"",
|
||||
"```bash",
|
||||
"# Full platform pack",
|
||||
"python scripts/generate_pack.py --platform retroarch --output-dir dist/",
|
||||
"python scripts/generate_pack.py --platform batocera --output-dir dist/",
|
||||
"",
|
||||
"# Single emulator or system",
|
||||
"python scripts/generate_pack.py --emulator dolphin",
|
||||
"python scripts/generate_pack.py --system sony-playstation-2",
|
||||
"",
|
||||
"# List available emulators and systems",
|
||||
"python scripts/generate_pack.py --list-emulators",
|
||||
"python scripts/generate_pack.py --list-systems",
|
||||
"",
|
||||
"# Verify your BIOS collection",
|
||||
"python scripts/verify.py --all",
|
||||
"python scripts/verify.py --platform batocera",
|
||||
"python scripts/verify.py --emulator flycast",
|
||||
"python scripts/verify.py --platform retroarch --verbose # emulator ground truth",
|
||||
"```",
|
||||
"",
|
||||
"Only dependency: Python 3 + `pyyaml`.",
|
||||
"",
|
||||
"## Documentation site",
|
||||
"",
|
||||
f"The [documentation site]({SITE_URL}) provides:",
|
||||
"",
|
||||
"- **Per-platform pages** with file-by-file verification status and hashes",
|
||||
"- **Per-emulator profiles** with source code references for every file",
|
||||
"- **Per-system pages** showing which emulators and platforms cover each console",
|
||||
"- **Gap analysis** identifying missing files and undeclared core requirements",
|
||||
f"- **Cross-reference** mapping files across {len(coverages)} platforms and {emulator_count} emulators",
|
||||
"",
|
||||
"## How it works",
|
||||
"",
|
||||
"Documentation and metadata can drift from what emulators actually load.",
|
||||
"To keep packs accurate, each file is checked against the emulator's source code.",
|
||||
"",
|
||||
"1. **Read emulator source code** - trace every file the code loads, its expected hash and size",
|
||||
"2. **Cross-reference with platforms** - match against what each platform declares",
|
||||
"3. **Build packs** - include baseline files plus what each platform's cores need",
|
||||
"4. **Verify** - run platform-native checks and emulator-level validation",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
contributors = fetch_contributors()
|
||||
if contributors:
|
||||
lines.extend([
|
||||
"## Contributors",
|
||||
"",
|
||||
])
|
||||
lines.extend(
|
||||
[
|
||||
"## Contributors",
|
||||
"",
|
||||
]
|
||||
)
|
||||
for c in contributors:
|
||||
login = c["login"]
|
||||
avatar = c.get("avatar_url", "")
|
||||
@@ -285,18 +347,20 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"## Contributing",
|
||||
"",
|
||||
"See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.",
|
||||
"",
|
||||
"## License",
|
||||
"",
|
||||
"This repository provides BIOS files for personal backup and archival purposes.",
|
||||
"",
|
||||
f"*Auto-generated on {ts}*",
|
||||
])
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Contributing",
|
||||
"",
|
||||
"See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.",
|
||||
"",
|
||||
"## License",
|
||||
"",
|
||||
"This repository provides BIOS files for personal backup and archival purposes.",
|
||||
"",
|
||||
f"*Auto-generated on {ts}*",
|
||||
]
|
||||
)
|
||||
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
@@ -332,7 +396,11 @@ def main():
|
||||
print(f"{status} ./README.md")
|
||||
|
||||
contributing = generate_contributing()
|
||||
status = "Generated" if write_if_changed("CONTRIBUTING.md", contributing) else "Unchanged"
|
||||
status = (
|
||||
"Generated"
|
||||
if write_if_changed("CONTRIBUTING.md", contributing)
|
||||
else "Unchanged"
|
||||
)
|
||||
print(f"{status} ./CONTRIBUTING.md")
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -39,20 +39,28 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
group.add_argument("--all", action="store_true", help="all registered platforms")
|
||||
group.add_argument("--platform", help="single platform name")
|
||||
parser.add_argument(
|
||||
"--output-dir", default=DEFAULT_OUTPUT_DIR, help="output directory",
|
||||
"--output-dir",
|
||||
default=DEFAULT_OUTPUT_DIR,
|
||||
help="output directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target", "-t", default=None, help="hardware target filter",
|
||||
"--target",
|
||||
"-t",
|
||||
default=None,
|
||||
help="hardware target filter",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-archived", action="store_true",
|
||||
"--include-archived",
|
||||
action="store_true",
|
||||
help="include archived platforms with --all",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platforms-dir", default=DEFAULT_PLATFORMS_DIR,
|
||||
"--platforms-dir",
|
||||
default=DEFAULT_PLATFORMS_DIR,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--emulators-dir", default=DEFAULT_EMULATORS_DIR,
|
||||
"--emulators-dir",
|
||||
default=DEFAULT_EMULATORS_DIR,
|
||||
)
|
||||
parser.add_argument("--db", default=DEFAULT_DB_FILE, help="database.json path")
|
||||
return parser.parse_args(argv)
|
||||
@@ -77,7 +85,8 @@ def main(argv: list[str] | None = None) -> None:
|
||||
# Determine platforms
|
||||
if args.all:
|
||||
platforms = list_registered_platforms(
|
||||
args.platforms_dir, include_archived=args.include_archived,
|
||||
args.platforms_dir,
|
||||
include_archived=args.include_archived,
|
||||
)
|
||||
else:
|
||||
platforms = [args.platform]
|
||||
@@ -90,7 +99,9 @@ def main(argv: list[str] | None = None) -> None:
|
||||
if args.target:
|
||||
try:
|
||||
target_cores = load_target_config(
|
||||
name, args.target, args.platforms_dir,
|
||||
name,
|
||||
args.target,
|
||||
args.platforms_dir,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
print(f" {name}: no target config, skipped")
|
||||
@@ -105,15 +116,22 @@ def main(argv: list[str] | None = None) -> None:
|
||||
registry_entry = registry.get(name, {})
|
||||
|
||||
result = generate_platform_truth(
|
||||
name, config, registry_entry, profiles,
|
||||
db=db, target_cores=target_cores,
|
||||
name,
|
||||
config,
|
||||
registry_entry,
|
||||
profiles,
|
||||
db=db,
|
||||
target_cores=target_cores,
|
||||
)
|
||||
|
||||
out_path = os.path.join(args.output_dir, f"{name}.yml")
|
||||
with open(out_path, "w") as f:
|
||||
yaml.dump(
|
||||
result, f,
|
||||
default_flow_style=False, sort_keys=False, allow_unicode=True,
|
||||
result,
|
||||
f,
|
||||
default_flow_style=False,
|
||||
sort_keys=False,
|
||||
allow_unicode=True,
|
||||
)
|
||||
|
||||
n_systems = len(result.get("systems", {}))
|
||||
|
||||
@@ -78,11 +78,9 @@ BIOS_FILE_MAP = {
|
||||
"sanyotry.bin": ("3DO Company", "3DO"),
|
||||
"3do_arcade_saot.bin": ("3DO Company", "3DO"),
|
||||
"3dobios.zip": ("3DO Company", "3DO"),
|
||||
|
||||
"cpc464.rom": ("Amstrad", "CPC"),
|
||||
"cpc664.rom": ("Amstrad", "CPC"),
|
||||
"cpc6128.rom": ("Amstrad", "CPC"),
|
||||
|
||||
"neogeo.zip": ("SNK", "Neo Geo"),
|
||||
"pgm.zip": ("Arcade", "Arcade"),
|
||||
"skns.zip": ("Arcade", "Arcade"),
|
||||
@@ -94,7 +92,6 @@ BIOS_FILE_MAP = {
|
||||
"nmk004.zip": ("Arcade", "Arcade"),
|
||||
"ym2608.zip": ("Arcade", "Arcade"),
|
||||
"qsound.zip": ("Arcade", "Arcade"),
|
||||
|
||||
"ATARIBAS.ROM": ("Atari", "400-800"),
|
||||
"ATARIOSA.ROM": ("Atari", "400-800"),
|
||||
"ATARIOSB.ROM": ("Atari", "400-800"),
|
||||
@@ -106,10 +103,8 @@ BIOS_FILE_MAP = {
|
||||
"7800 BIOS (E).rom": ("Atari", "7800"),
|
||||
"lynxboot.img": ("Atari", "Lynx"),
|
||||
"tos.img": ("Atari", "ST"),
|
||||
|
||||
"colecovision.rom": ("Coleco", "ColecoVision"),
|
||||
"coleco.rom": ("Coleco", "ColecoVision"),
|
||||
|
||||
"kick33180.A500": ("Commodore", "Amiga"),
|
||||
"kick34005.A500": ("Commodore", "Amiga"),
|
||||
"kick34005.CDTV": ("Commodore", "Amiga"),
|
||||
@@ -122,33 +117,26 @@ BIOS_FILE_MAP = {
|
||||
"kick40063.A600": ("Commodore", "Amiga"),
|
||||
"kick40068.A1200": ("Commodore", "Amiga"),
|
||||
"kick40068.A4000": ("Commodore", "Amiga"),
|
||||
|
||||
"sl31253.bin": ("Fairchild", "Channel F"),
|
||||
"sl31254.bin": ("Fairchild", "Channel F"),
|
||||
"sl90025.bin": ("Fairchild", "Channel F"),
|
||||
|
||||
"prboom.wad": ("Id Software", "Doom"),
|
||||
"ecwolf.pk3": ("Id Software", "Wolfenstein 3D"),
|
||||
|
||||
"MacII.ROM": ("Apple", "Macintosh II"),
|
||||
"MacIIx.ROM": ("Apple", "Macintosh II"),
|
||||
"vMac.ROM": ("Apple", "Macintosh II"),
|
||||
|
||||
"o2rom.bin": ("Magnavox", "Odyssey2"),
|
||||
"g7400.bin": ("Philips", "Videopac+"),
|
||||
"jopac.bin": ("Philips", "Videopac+"),
|
||||
|
||||
"exec.bin": ("Mattel", "Intellivision"),
|
||||
"grom.bin": ("Mattel", "Intellivision"),
|
||||
"ECS.bin": ("Mattel", "Intellivision"),
|
||||
"IVOICE.BIN": ("Mattel", "Intellivision"),
|
||||
|
||||
"MSX.ROM": ("Microsoft", "MSX"),
|
||||
"MSX2.ROM": ("Microsoft", "MSX"),
|
||||
"MSX2EXT.ROM": ("Microsoft", "MSX"),
|
||||
"MSX2P.ROM": ("Microsoft", "MSX"),
|
||||
"MSX2PEXT.ROM": ("Microsoft", "MSX"),
|
||||
|
||||
"syscard1.pce": ("NEC", "PC Engine"),
|
||||
"syscard2.pce": ("NEC", "PC Engine"),
|
||||
"syscard2u.pce": ("NEC", "PC Engine"),
|
||||
@@ -156,7 +144,6 @@ BIOS_FILE_MAP = {
|
||||
"syscard3u.pce": ("NEC", "PC Engine"),
|
||||
"gexpress.pce": ("NEC", "PC Engine"),
|
||||
"pcfx.rom": ("NEC", "PC-FX"),
|
||||
|
||||
"disksys.rom": ("Nintendo", "Famicom Disk System"),
|
||||
"gba_bios.bin": ("Nintendo", "Game Boy Advance"),
|
||||
"gb_bios.bin": ("Nintendo", "Game Boy"),
|
||||
@@ -179,7 +166,6 @@ BIOS_FILE_MAP = {
|
||||
"dsifirmware.bin": ("Nintendo", "Nintendo DS"),
|
||||
"bios.min": ("Nintendo", "Pokemon Mini"),
|
||||
"64DD_IPL.bin": ("Nintendo", "Nintendo 64DD"),
|
||||
|
||||
"dc_boot.bin": ("Sega", "Dreamcast"),
|
||||
"dc_flash.bin": ("Sega", "Dreamcast"),
|
||||
"bios.gg": ("Sega", "Game Gear"),
|
||||
@@ -196,7 +182,6 @@ BIOS_FILE_MAP = {
|
||||
"saturn_bios.bin": ("Sega", "Saturn"),
|
||||
"sega_101.bin": ("Sega", "Saturn"),
|
||||
"stvbios.zip": ("Sega", "Saturn"),
|
||||
|
||||
"scph1001.bin": ("Sony", "PlayStation"),
|
||||
"SCPH1001.BIN": ("Sony", "PlayStation"),
|
||||
"scph5500.bin": ("Sony", "PlayStation"),
|
||||
@@ -207,7 +192,6 @@ BIOS_FILE_MAP = {
|
||||
"ps1_rom.bin": ("Sony", "PlayStation"),
|
||||
"psxonpsp660.bin": ("Sony", "PlayStation"),
|
||||
"PSXONPSP660.BIN": ("Sony", "PlayStation Portable"),
|
||||
|
||||
"scummvm.zip": ("ScummVM", "ScummVM"),
|
||||
"MT32_CONTROL.ROM": ("ScummVM", "ScummVM"),
|
||||
"MT32_PCM.ROM": ("ScummVM", "ScummVM"),
|
||||
@@ -254,8 +238,11 @@ SKIP_LARGE_ROM_DIRS = {"roms/"}
|
||||
BRANCHES = ["RetroArch", "RetroPie", "Recalbox", "batocera", "Other"]
|
||||
|
||||
SKIP_FILES = {
|
||||
"README.md", ".gitignore", "desktop.ini",
|
||||
"telemetry_id", "citra_log.txt",
|
||||
"README.md",
|
||||
".gitignore",
|
||||
"desktop.ini",
|
||||
"telemetry_id",
|
||||
"citra_log.txt",
|
||||
}
|
||||
SKIP_EXTENSIONS = {".txt", ".log", ".pem", ".nvm", ".ctg", ".exe", ".bat", ".sh"}
|
||||
|
||||
@@ -279,17 +266,33 @@ def classify_file(filepath: str) -> tuple:
|
||||
return None
|
||||
|
||||
clean = filepath
|
||||
for prefix in ("bios/", "BIOS/", "roms/fba/", "roms/fbneo/", "roms/mame/",
|
||||
"roms/mame-libretro/", "roms/neogeo/", "roms/naomi/",
|
||||
"roms/atomiswave/", "roms/macintosh/"):
|
||||
for prefix in (
|
||||
"bios/",
|
||||
"BIOS/",
|
||||
"roms/fba/",
|
||||
"roms/fbneo/",
|
||||
"roms/mame/",
|
||||
"roms/mame-libretro/",
|
||||
"roms/neogeo/",
|
||||
"roms/naomi/",
|
||||
"roms/atomiswave/",
|
||||
"roms/macintosh/",
|
||||
):
|
||||
if clean.startswith(prefix):
|
||||
clean = clean[len(prefix):]
|
||||
clean = clean[len(prefix) :]
|
||||
break
|
||||
|
||||
if filepath.startswith("roms/") and not any(
|
||||
filepath.startswith(p) for p in (
|
||||
"roms/fba/", "roms/fbneo/", "roms/mame/", "roms/mame-libretro/",
|
||||
"roms/neogeo/", "roms/naomi/", "roms/atomiswave/", "roms/macintosh/"
|
||||
filepath.startswith(p)
|
||||
for p in (
|
||||
"roms/fba/",
|
||||
"roms/fbneo/",
|
||||
"roms/mame/",
|
||||
"roms/mame-libretro/",
|
||||
"roms/neogeo/",
|
||||
"roms/naomi/",
|
||||
"roms/atomiswave/",
|
||||
"roms/macintosh/",
|
||||
)
|
||||
):
|
||||
return None
|
||||
@@ -341,12 +344,12 @@ def get_subpath(filepath: str, manufacturer: str, console: str) -> str:
|
||||
clean = filepath
|
||||
for prefix in ("bios/", "BIOS/"):
|
||||
if clean.startswith(prefix):
|
||||
clean = clean[len(prefix):]
|
||||
clean = clean[len(prefix) :]
|
||||
break
|
||||
|
||||
for prefix in PATH_PREFIX_MAP:
|
||||
if clean.startswith(prefix):
|
||||
remaining = clean[len(prefix):]
|
||||
remaining = clean[len(prefix) :]
|
||||
if "/" in remaining:
|
||||
return remaining
|
||||
return remaining
|
||||
@@ -363,16 +366,14 @@ def extract_from_branches(target: Path, dry_run: bool, existing_hashes: set) ->
|
||||
|
||||
try:
|
||||
subprocess.run(
|
||||
["git", "rev-parse", "--verify", ref],
|
||||
capture_output=True, check=True
|
||||
["git", "rev-parse", "--verify", ref], capture_output=True, check=True
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
print(f" Branch {branch} not found, skipping")
|
||||
continue
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "ls-tree", "-r", "--name-only", ref],
|
||||
capture_output=True, text=True
|
||||
["git", "ls-tree", "-r", "--name-only", ref], capture_output=True, text=True
|
||||
)
|
||||
files = result.stdout.strip().split("\n")
|
||||
print(f"\n Branch '{branch}': {len(files)} files")
|
||||
@@ -391,7 +392,8 @@ def extract_from_branches(target: Path, dry_run: bool, existing_hashes: set) ->
|
||||
try:
|
||||
blob = subprocess.run(
|
||||
["git", "show", f"{ref}:{filepath}"],
|
||||
capture_output=True, check=True
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
content = blob.stdout
|
||||
except subprocess.CalledProcessError:
|
||||
@@ -493,14 +495,20 @@ def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Migrate BIOS files to Manufacturer/Console structure"
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="Show what would be done without moving files")
|
||||
parser.add_argument("--source", default=".",
|
||||
help="Source directory (repo root)")
|
||||
parser.add_argument("--target", default="bios",
|
||||
help="Target directory for organized BIOS files")
|
||||
parser.add_argument("--include-branches", action="store_true",
|
||||
help="Also extract BIOS files from all remote branches")
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would be done without moving files",
|
||||
)
|
||||
parser.add_argument("--source", default=".", help="Source directory (repo root)")
|
||||
parser.add_argument(
|
||||
"--target", default="bios", help="Target directory for organized BIOS files"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-branches",
|
||||
action="store_true",
|
||||
help="Also extract BIOS files from all remote branches",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
source = Path(args.source)
|
||||
@@ -517,7 +525,9 @@ def main():
|
||||
print()
|
||||
|
||||
print("=== Phase 1: Local files (libretro branch) ===")
|
||||
moved, skipped, errors, existing_hashes = migrate_local(source, target, args.dry_run)
|
||||
moved, skipped, errors, existing_hashes = migrate_local(
|
||||
source, target, args.dry_run
|
||||
)
|
||||
action = "Would copy" if args.dry_run else "Copied"
|
||||
print(f"\n{action} {moved} files, skipped {skipped}")
|
||||
|
||||
@@ -529,8 +539,15 @@ def main():
|
||||
|
||||
if source.is_dir():
|
||||
known = set(SYSTEM_MAP.keys()) | {
|
||||
"bios", "scripts", "platforms", "schemas", ".github", ".cache",
|
||||
".git", "README.md", ".gitignore",
|
||||
"bios",
|
||||
"scripts",
|
||||
"platforms",
|
||||
"schemas",
|
||||
".github",
|
||||
".cache",
|
||||
".git",
|
||||
"README.md",
|
||||
".gitignore",
|
||||
}
|
||||
for d in sorted(source.iterdir()):
|
||||
if d.name not in known and not d.name.startswith("."):
|
||||
|
||||
@@ -19,10 +19,10 @@ Usage:
|
||||
python scripts/pipeline.py --skip-docs # skip steps 8-9
|
||||
python scripts/pipeline.py --offline # skip step 2
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -54,6 +54,7 @@ def parse_verify_counts(output: str) -> dict[str, tuple[int, int]]:
|
||||
Returns {group_label: (ok, total)}.
|
||||
"""
|
||||
import re
|
||||
|
||||
counts = {}
|
||||
for line in output.splitlines():
|
||||
m = re.match(r"^(.+?):\s+(\d+)/(\d+)\s+(OK|present)", line)
|
||||
@@ -71,6 +72,7 @@ def parse_pack_counts(output: str) -> dict[str, tuple[int, int]]:
|
||||
Returns {pack_label: (ok, total)}.
|
||||
"""
|
||||
import re
|
||||
|
||||
counts = {}
|
||||
current_label = ""
|
||||
for line in output.splitlines():
|
||||
@@ -84,7 +86,7 @@ def parse_pack_counts(output: str) -> dict[str, tuple[int, int]]:
|
||||
base_m = re.search(r"\((\d+) baseline", line)
|
||||
ok_m = re.search(r"(\d+)/(\d+) files OK", line)
|
||||
if base_m and ok_m:
|
||||
baseline = int(base_m.group(1))
|
||||
int(base_m.group(1))
|
||||
ok, total = int(ok_m.group(1)), int(ok_m.group(2))
|
||||
counts[current_label] = (ok, total)
|
||||
elif ok_m:
|
||||
@@ -118,12 +120,18 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
|
||||
print(f" {v_label}: MISMATCH total verify {v_total} != pack {p_total}")
|
||||
all_ok = False
|
||||
elif p_ok < v_ok:
|
||||
print(f" {v_label}: MISMATCH pack {p_ok} OK < verify {v_ok} OK (/{v_total})")
|
||||
print(
|
||||
f" {v_label}: MISMATCH pack {p_ok} OK < verify {v_ok} OK (/{v_total})"
|
||||
)
|
||||
all_ok = False
|
||||
elif p_ok == v_ok:
|
||||
print(f" {v_label}: verify {v_ok}/{v_total} == pack {p_ok}/{p_total} OK")
|
||||
print(
|
||||
f" {v_label}: verify {v_ok}/{v_total} == pack {p_ok}/{p_total} OK"
|
||||
)
|
||||
else:
|
||||
print(f" {v_label}: verify {v_ok}/{v_total}, pack {p_ok}/{p_total} OK (pack resolves more)")
|
||||
print(
|
||||
f" {v_label}: verify {v_ok}/{v_total}, pack {p_ok}/{p_total} OK (pack resolves more)"
|
||||
)
|
||||
else:
|
||||
print(f" {v_label}: {v_ok}/{v_total} (no separate pack)")
|
||||
|
||||
@@ -134,26 +142,47 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run the full retrobios pipeline")
|
||||
parser.add_argument("--include-archived", action="store_true",
|
||||
help="Include archived platforms")
|
||||
parser.add_argument("--skip-packs", action="store_true",
|
||||
help="Only regenerate DB and verify, skip pack generation")
|
||||
parser.add_argument("--skip-docs", action="store_true",
|
||||
help="Skip README and site generation")
|
||||
parser.add_argument("--offline", action="store_true",
|
||||
help="Skip data directory refresh")
|
||||
parser.add_argument("--output-dir", default="dist",
|
||||
help="Pack output directory (default: dist/)")
|
||||
parser.add_argument(
|
||||
"--include-archived", action="store_true", help="Include archived platforms"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-packs",
|
||||
action="store_true",
|
||||
help="Only regenerate DB and verify, skip pack generation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-docs", action="store_true", help="Skip README and site generation"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--offline", action="store_true", help="Skip data directory refresh"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir", default="dist", help="Pack output directory (default: dist/)"
|
||||
)
|
||||
# --include-extras is now a no-op: core requirements are always included
|
||||
parser.add_argument("--include-extras", action="store_true",
|
||||
help="(no-op) Core requirements are always included")
|
||||
parser.add_argument(
|
||||
"--include-extras",
|
||||
action="store_true",
|
||||
help="(no-op) Core requirements are always included",
|
||||
)
|
||||
parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)")
|
||||
parser.add_argument("--check-buildbot", action="store_true",
|
||||
help="Check buildbot system directory for changes")
|
||||
parser.add_argument("--with-truth", action="store_true",
|
||||
help="Generate truth YAMLs and diff against scraped")
|
||||
parser.add_argument("--with-export", action="store_true",
|
||||
help="Export native formats (implies --with-truth)")
|
||||
parser.add_argument("--source", choices=["platform", "truth", "full"], default="full")
|
||||
parser.add_argument("--all-variants", action="store_true")
|
||||
parser.add_argument(
|
||||
"--check-buildbot",
|
||||
action="store_true",
|
||||
help="Check buildbot system directory for changes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--with-truth",
|
||||
action="store_true",
|
||||
help="Generate truth YAMLs and diff against scraped",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--with-export",
|
||||
action="store_true",
|
||||
help="Export native formats (implies --with-truth)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
results = {}
|
||||
@@ -162,8 +191,15 @@ def main():
|
||||
|
||||
# Step 1: Generate database
|
||||
ok, out = run(
|
||||
[sys.executable, "scripts/generate_db.py", "--force",
|
||||
"--bios-dir", "bios", "--output", "database.json"],
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/generate_db.py",
|
||||
"--force",
|
||||
"--bios-dir",
|
||||
"bios",
|
||||
"--output",
|
||||
"database.json",
|
||||
],
|
||||
"1/8 generate database",
|
||||
)
|
||||
results["generate_db"] = ok
|
||||
@@ -216,8 +252,13 @@ def main():
|
||||
|
||||
# Step 2c: Generate truth YAMLs
|
||||
if args.with_truth or args.with_export:
|
||||
truth_cmd = [sys.executable, "scripts/generate_truth.py", "--all",
|
||||
"--output-dir", str(Path(args.output_dir) / "truth")]
|
||||
truth_cmd = [
|
||||
sys.executable,
|
||||
"scripts/generate_truth.py",
|
||||
"--all",
|
||||
"--output-dir",
|
||||
str(Path(args.output_dir) / "truth"),
|
||||
]
|
||||
if args.include_archived:
|
||||
truth_cmd.append("--include-archived")
|
||||
if args.target:
|
||||
@@ -242,9 +283,15 @@ def main():
|
||||
|
||||
# Step 2e: Export native formats
|
||||
if args.with_export:
|
||||
export_cmd = [sys.executable, "scripts/export_native.py", "--all",
|
||||
"--output-dir", str(Path(args.output_dir) / "upstream"),
|
||||
"--truth-dir", str(Path(args.output_dir) / "truth")]
|
||||
export_cmd = [
|
||||
sys.executable,
|
||||
"scripts/export_native.py",
|
||||
"--all",
|
||||
"--output-dir",
|
||||
str(Path(args.output_dir) / "upstream"),
|
||||
"--truth-dir",
|
||||
str(Path(args.output_dir) / "truth"),
|
||||
]
|
||||
if args.include_archived:
|
||||
export_cmd.append("--include-archived")
|
||||
ok, _ = run(export_cmd, "2e export native")
|
||||
@@ -267,8 +314,11 @@ def main():
|
||||
pack_output = ""
|
||||
if not args.skip_packs:
|
||||
pack_cmd = [
|
||||
sys.executable, "scripts/generate_pack.py", "--all",
|
||||
"--output-dir", args.output_dir,
|
||||
sys.executable,
|
||||
"scripts/generate_pack.py",
|
||||
"--all",
|
||||
"--output-dir",
|
||||
args.output_dir,
|
||||
]
|
||||
if args.include_archived:
|
||||
pack_cmd.append("--include-archived")
|
||||
@@ -278,6 +328,10 @@ def main():
|
||||
pack_cmd.append("--include-extras")
|
||||
if args.target:
|
||||
pack_cmd.extend(["--target", args.target])
|
||||
if args.source != "full":
|
||||
pack_cmd.extend(["--source", args.source])
|
||||
if args.all_variants:
|
||||
pack_cmd.append("--all-variants")
|
||||
ok, pack_output = run(pack_cmd, "4/8 generate packs")
|
||||
results["generate_packs"] = ok
|
||||
all_ok = all_ok and ok
|
||||
@@ -288,8 +342,12 @@ def main():
|
||||
# Step 4b: Generate install manifests
|
||||
if not args.skip_packs:
|
||||
manifest_cmd = [
|
||||
sys.executable, "scripts/generate_pack.py", "--all",
|
||||
"--manifest", "--output-dir", "install",
|
||||
sys.executable,
|
||||
"scripts/generate_pack.py",
|
||||
"--all",
|
||||
"--manifest",
|
||||
"--output-dir",
|
||||
"install",
|
||||
]
|
||||
if args.include_archived:
|
||||
manifest_cmd.append("--include-archived")
|
||||
@@ -307,8 +365,11 @@ def main():
|
||||
# Step 4c: Generate target manifests
|
||||
if not args.skip_packs:
|
||||
target_cmd = [
|
||||
sys.executable, "scripts/generate_pack.py",
|
||||
"--manifest-targets", "--output-dir", "install/targets",
|
||||
sys.executable,
|
||||
"scripts/generate_pack.py",
|
||||
"--manifest-targets",
|
||||
"--output-dir",
|
||||
"install/targets",
|
||||
]
|
||||
ok, _ = run(target_cmd, "4c/8 generate target manifests")
|
||||
results["generate_target_manifests"] = ok
|
||||
@@ -329,8 +390,12 @@ def main():
|
||||
# Step 6: Pack integrity (extract + hash verification)
|
||||
if not args.skip_packs:
|
||||
integrity_cmd = [
|
||||
sys.executable, "scripts/generate_pack.py", "--all",
|
||||
"--verify-packs", "--output-dir", args.output_dir,
|
||||
sys.executable,
|
||||
"scripts/generate_pack.py",
|
||||
"--all",
|
||||
"--verify-packs",
|
||||
"--output-dir",
|
||||
args.output_dir,
|
||||
]
|
||||
if args.include_archived:
|
||||
integrity_cmd.append("--include-archived")
|
||||
@@ -344,8 +409,14 @@ def main():
|
||||
# Step 7: Generate README
|
||||
if not args.skip_docs:
|
||||
ok, _ = run(
|
||||
[sys.executable, "scripts/generate_readme.py",
|
||||
"--db", "database.json", "--platforms-dir", "platforms"],
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/generate_readme.py",
|
||||
"--db",
|
||||
"database.json",
|
||||
"--platforms-dir",
|
||||
"platforms",
|
||||
],
|
||||
"7/8 generate readme",
|
||||
)
|
||||
results["generate_readme"] = ok
|
||||
|
||||
@@ -57,7 +57,9 @@ def _load_versions(versions_path: str = VERSIONS_FILE) -> dict[str, dict]:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _save_versions(versions: dict[str, dict], versions_path: str = VERSIONS_FILE) -> None:
|
||||
def _save_versions(
|
||||
versions: dict[str, dict], versions_path: str = VERSIONS_FILE
|
||||
) -> None:
|
||||
path = Path(versions_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
@@ -66,10 +68,13 @@ def _save_versions(versions: dict[str, dict], versions_path: str = VERSIONS_FILE
|
||||
|
||||
|
||||
def _api_request(url: str) -> dict:
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
})
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
||||
if token and "github" in url:
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
@@ -111,7 +116,9 @@ def get_remote_sha(source_url: str, version: str) -> str | None:
|
||||
data = _api_request(url)
|
||||
return data["commit"]["id"]
|
||||
except (urllib.error.URLError, KeyError, OSError) as exc:
|
||||
log.warning("failed to fetch remote SHA for %s/%s@%s: %s", owner, repo, version, exc)
|
||||
log.warning(
|
||||
"failed to fetch remote SHA for %s/%s@%s: %s", owner, repo, version, exc
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -167,7 +174,7 @@ def _download_and_extract(
|
||||
if not member.name.startswith(prefix) and member.name != source_path:
|
||||
continue
|
||||
|
||||
rel = member.name[len(prefix):]
|
||||
rel = member.name[len(prefix) :]
|
||||
if not rel:
|
||||
continue
|
||||
|
||||
@@ -285,8 +292,9 @@ def _download_and_extract_zip(
|
||||
def _get_remote_etag(source_url: str) -> str | None:
|
||||
"""HEAD request to get ETag or Last-Modified for freshness check."""
|
||||
try:
|
||||
req = urllib.request.Request(source_url, method="HEAD",
|
||||
headers={"User-Agent": USER_AGENT})
|
||||
req = urllib.request.Request(
|
||||
source_url, method="HEAD", headers={"User-Agent": USER_AGENT}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
|
||||
return resp.headers.get("ETag") or resp.headers.get("Last-Modified") or ""
|
||||
except (urllib.error.URLError, OSError):
|
||||
@@ -333,17 +341,31 @@ def refresh_entry(
|
||||
return False
|
||||
|
||||
if dry_run:
|
||||
log.info("[%s] would refresh (type: %s, cached: %s)", key, source_type, cached_tag or "none")
|
||||
log.info(
|
||||
"[%s] would refresh (type: %s, cached: %s)",
|
||||
key,
|
||||
source_type,
|
||||
cached_tag or "none",
|
||||
)
|
||||
return True
|
||||
|
||||
try:
|
||||
if source_type == "zip":
|
||||
strip = entry.get("strip_components", 0)
|
||||
file_count = _download_and_extract_zip(source_url, local_cache, exclude, strip)
|
||||
file_count = _download_and_extract_zip(
|
||||
source_url, local_cache, exclude, strip
|
||||
)
|
||||
else:
|
||||
source_path = entry["source_path"].format(version=version)
|
||||
file_count = _download_and_extract(source_url, source_path, local_cache, exclude)
|
||||
except (urllib.error.URLError, OSError, tarfile.TarError, zipfile.BadZipFile) as exc:
|
||||
file_count = _download_and_extract(
|
||||
source_url, source_path, local_cache, exclude
|
||||
)
|
||||
except (
|
||||
urllib.error.URLError,
|
||||
OSError,
|
||||
tarfile.TarError,
|
||||
zipfile.BadZipFile,
|
||||
) as exc:
|
||||
log.warning("[%s] download failed: %s", key, exc)
|
||||
return False
|
||||
|
||||
@@ -380,18 +402,30 @@ def refresh_all(
|
||||
if platform and allowed and platform not in allowed:
|
||||
continue
|
||||
results[key] = refresh_entry(
|
||||
key, entry, force=force, dry_run=dry_run, versions_path=versions_path,
|
||||
key,
|
||||
entry,
|
||||
force=force,
|
||||
dry_run=dry_run,
|
||||
versions_path=versions_path,
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Refresh cached data directories from upstream")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Refresh cached data directories from upstream"
|
||||
)
|
||||
parser.add_argument("--key", help="Refresh only this entry")
|
||||
parser.add_argument("--force", action="store_true", help="Re-download even if up to date")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview without downloading")
|
||||
parser.add_argument(
|
||||
"--force", action="store_true", help="Re-download even if up to date"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true", help="Preview without downloading"
|
||||
)
|
||||
parser.add_argument("--platform", help="Only refresh entries for this platform")
|
||||
parser.add_argument("--registry", default=DEFAULT_REGISTRY, help="Path to _data_dirs.yml")
|
||||
parser.add_argument(
|
||||
"--registry", default=DEFAULT_REGISTRY, help="Path to _data_dirs.yml"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
@@ -405,9 +439,13 @@ def main() -> None:
|
||||
if args.key not in registry:
|
||||
log.error("unknown key: %s (available: %s)", args.key, ", ".join(registry))
|
||||
raise SystemExit(1)
|
||||
refresh_entry(args.key, registry[args.key], force=args.force, dry_run=args.dry_run)
|
||||
refresh_entry(
|
||||
args.key, registry[args.key], force=args.force, dry_run=args.dry_run
|
||||
)
|
||||
else:
|
||||
refresh_all(registry, force=args.force, dry_run=args.dry_run, platform=args.platform)
|
||||
refresh_all(
|
||||
registry, force=args.force, dry_run=args.dry_run, platform=args.platform
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -34,40 +34,40 @@ def merge_mame_profile(
|
||||
profile = _load_yaml(profile_path)
|
||||
hashes = _load_json(hashes_path)
|
||||
|
||||
profile['core_version'] = hashes.get('version', profile.get('core_version'))
|
||||
profile["core_version"] = hashes.get("version", profile.get("core_version"))
|
||||
|
||||
files = profile.get('files', [])
|
||||
bios_zip, non_bios = _split_files(files, lambda f: f.get('category') == 'bios_zip')
|
||||
files = profile.get("files", [])
|
||||
bios_zip, non_bios = _split_files(files, lambda f: f.get("category") == "bios_zip")
|
||||
|
||||
existing_by_name: dict[str, dict] = {}
|
||||
for entry in bios_zip:
|
||||
key = _zip_name_to_set(entry['name'])
|
||||
key = _zip_name_to_set(entry["name"])
|
||||
existing_by_name[key] = entry
|
||||
|
||||
updated_bios: list[dict] = []
|
||||
matched_names: set[str] = set()
|
||||
|
||||
for set_name, set_data in hashes.get('bios_sets', {}).items():
|
||||
contents = _build_contents(set_data.get('roms', []))
|
||||
for set_name, set_data in hashes.get("bios_sets", {}).items():
|
||||
contents = _build_contents(set_data.get("roms", []))
|
||||
source_ref = _build_source_ref(set_data)
|
||||
|
||||
if set_name in existing_by_name:
|
||||
# Update existing entry: preserve manual fields, update contents
|
||||
entry = existing_by_name[set_name].copy()
|
||||
entry['contents'] = contents
|
||||
entry["contents"] = contents
|
||||
if source_ref:
|
||||
entry['source_ref'] = source_ref
|
||||
entry["source_ref"] = source_ref
|
||||
updated_bios.append(entry)
|
||||
matched_names.add(set_name)
|
||||
elif add_new:
|
||||
# New BIOS set — only added to the main profile
|
||||
entry = {
|
||||
'name': f'{set_name}.zip',
|
||||
'required': True,
|
||||
'category': 'bios_zip',
|
||||
'system': None,
|
||||
'source_ref': source_ref,
|
||||
'contents': contents,
|
||||
"name": f"{set_name}.zip",
|
||||
"required": True,
|
||||
"category": "bios_zip",
|
||||
"system": None,
|
||||
"source_ref": source_ref,
|
||||
"contents": contents,
|
||||
}
|
||||
updated_bios.append(entry)
|
||||
|
||||
@@ -77,7 +77,7 @@ def merge_mame_profile(
|
||||
if set_name not in matched_names:
|
||||
updated_bios.append(entry)
|
||||
|
||||
profile['files'] = non_bios + updated_bios
|
||||
profile["files"] = non_bios + updated_bios
|
||||
|
||||
if write:
|
||||
_backup_and_write(profile_path, profile)
|
||||
@@ -102,49 +102,49 @@ def merge_fbneo_profile(
|
||||
profile = _load_yaml(profile_path)
|
||||
hashes = _load_json(hashes_path)
|
||||
|
||||
profile['core_version'] = hashes.get('version', profile.get('core_version'))
|
||||
profile["core_version"] = hashes.get("version", profile.get("core_version"))
|
||||
|
||||
files = profile.get('files', [])
|
||||
archive_files, non_archive = _split_files(files, lambda f: 'archive' in f)
|
||||
files = profile.get("files", [])
|
||||
archive_files, non_archive = _split_files(files, lambda f: "archive" in f)
|
||||
|
||||
existing_by_key: dict[tuple[str, str], dict] = {}
|
||||
for entry in archive_files:
|
||||
key = (entry['archive'], entry['name'])
|
||||
key = (entry["archive"], entry["name"])
|
||||
existing_by_key[key] = entry
|
||||
|
||||
merged: list[dict] = []
|
||||
matched_keys: set[tuple[str, str]] = set()
|
||||
|
||||
for set_name, set_data in hashes.get('bios_sets', {}).items():
|
||||
archive_name = f'{set_name}.zip'
|
||||
for set_name, set_data in hashes.get("bios_sets", {}).items():
|
||||
archive_name = f"{set_name}.zip"
|
||||
source_ref = _build_source_ref(set_data)
|
||||
|
||||
for rom in set_data.get('roms', []):
|
||||
rom_name = rom['name']
|
||||
for rom in set_data.get("roms", []):
|
||||
rom_name = rom["name"]
|
||||
key = (archive_name, rom_name)
|
||||
|
||||
if key in existing_by_key:
|
||||
entry = existing_by_key[key].copy()
|
||||
entry['size'] = rom['size']
|
||||
entry['crc32'] = rom['crc32']
|
||||
if rom.get('sha1'):
|
||||
entry['sha1'] = rom['sha1']
|
||||
entry["size"] = rom["size"]
|
||||
entry["crc32"] = rom["crc32"]
|
||||
if rom.get("sha1"):
|
||||
entry["sha1"] = rom["sha1"]
|
||||
if source_ref:
|
||||
entry['source_ref'] = source_ref
|
||||
entry["source_ref"] = source_ref
|
||||
merged.append(entry)
|
||||
matched_keys.add(key)
|
||||
elif add_new:
|
||||
entry = {
|
||||
'name': rom_name,
|
||||
'archive': archive_name,
|
||||
'required': True,
|
||||
'size': rom['size'],
|
||||
'crc32': rom['crc32'],
|
||||
"name": rom_name,
|
||||
"archive": archive_name,
|
||||
"required": True,
|
||||
"size": rom["size"],
|
||||
"crc32": rom["crc32"],
|
||||
}
|
||||
if rom.get('sha1'):
|
||||
entry['sha1'] = rom['sha1']
|
||||
if rom.get("sha1"):
|
||||
entry["sha1"] = rom["sha1"]
|
||||
if source_ref:
|
||||
entry['source_ref'] = source_ref
|
||||
entry["source_ref"] = source_ref
|
||||
merged.append(entry)
|
||||
|
||||
# Entries not matched stay untouched
|
||||
@@ -152,7 +152,7 @@ def merge_fbneo_profile(
|
||||
if key not in matched_keys:
|
||||
merged.append(entry)
|
||||
|
||||
profile['files'] = non_archive + merged
|
||||
profile["files"] = non_archive + merged
|
||||
|
||||
if write:
|
||||
_backup_and_write_fbneo(profile_path, profile, hashes)
|
||||
@@ -163,7 +163,7 @@ def merge_fbneo_profile(
|
||||
def compute_diff(
|
||||
profile_path: str,
|
||||
hashes_path: str,
|
||||
mode: str = 'mame',
|
||||
mode: str = "mame",
|
||||
) -> dict[str, Any]:
|
||||
"""Compute diff between profile and hashes without writing.
|
||||
|
||||
@@ -172,7 +172,7 @@ def compute_diff(
|
||||
profile = _load_yaml(profile_path)
|
||||
hashes = _load_json(hashes_path)
|
||||
|
||||
if mode == 'mame':
|
||||
if mode == "mame":
|
||||
return _diff_mame(profile, hashes)
|
||||
return _diff_fbneo(profile, hashes)
|
||||
|
||||
@@ -181,26 +181,26 @@ def _diff_mame(
|
||||
profile: dict[str, Any],
|
||||
hashes: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
files = profile.get('files', [])
|
||||
bios_zip, _ = _split_files(files, lambda f: f.get('category') == 'bios_zip')
|
||||
files = profile.get("files", [])
|
||||
bios_zip, _ = _split_files(files, lambda f: f.get("category") == "bios_zip")
|
||||
|
||||
existing_by_name: dict[str, dict] = {}
|
||||
for entry in bios_zip:
|
||||
existing_by_name[_zip_name_to_set(entry['name'])] = entry
|
||||
existing_by_name[_zip_name_to_set(entry["name"])] = entry
|
||||
|
||||
added: list[str] = []
|
||||
updated: list[str] = []
|
||||
unchanged = 0
|
||||
|
||||
bios_sets = hashes.get('bios_sets', {})
|
||||
bios_sets = hashes.get("bios_sets", {})
|
||||
for set_name, set_data in bios_sets.items():
|
||||
if set_name not in existing_by_name:
|
||||
added.append(set_name)
|
||||
continue
|
||||
|
||||
old_entry = existing_by_name[set_name]
|
||||
new_contents = _build_contents(set_data.get('roms', []))
|
||||
old_contents = old_entry.get('contents', [])
|
||||
new_contents = _build_contents(set_data.get("roms", []))
|
||||
old_contents = old_entry.get("contents", [])
|
||||
|
||||
if _contents_differ(old_contents, new_contents):
|
||||
updated.append(set_name)
|
||||
@@ -213,11 +213,11 @@ def _diff_mame(
|
||||
)
|
||||
|
||||
return {
|
||||
'added': added,
|
||||
'updated': updated,
|
||||
'removed': [],
|
||||
'unchanged': unchanged,
|
||||
'out_of_scope': out_of_scope,
|
||||
"added": added,
|
||||
"updated": updated,
|
||||
"removed": [],
|
||||
"unchanged": unchanged,
|
||||
"out_of_scope": out_of_scope,
|
||||
}
|
||||
|
||||
|
||||
@@ -225,24 +225,24 @@ def _diff_fbneo(
|
||||
profile: dict[str, Any],
|
||||
hashes: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
files = profile.get('files', [])
|
||||
archive_files, _ = _split_files(files, lambda f: 'archive' in f)
|
||||
files = profile.get("files", [])
|
||||
archive_files, _ = _split_files(files, lambda f: "archive" in f)
|
||||
|
||||
existing_by_key: dict[tuple[str, str], dict] = {}
|
||||
for entry in archive_files:
|
||||
existing_by_key[(entry['archive'], entry['name'])] = entry
|
||||
existing_by_key[(entry["archive"], entry["name"])] = entry
|
||||
|
||||
added: list[str] = []
|
||||
updated: list[str] = []
|
||||
unchanged = 0
|
||||
|
||||
seen_keys: set[tuple[str, str]] = set()
|
||||
bios_sets = hashes.get('bios_sets', {})
|
||||
bios_sets = hashes.get("bios_sets", {})
|
||||
|
||||
for set_name, set_data in bios_sets.items():
|
||||
archive_name = f'{set_name}.zip'
|
||||
for rom in set_data.get('roms', []):
|
||||
key = (archive_name, rom['name'])
|
||||
archive_name = f"{set_name}.zip"
|
||||
for rom in set_data.get("roms", []):
|
||||
key = (archive_name, rom["name"])
|
||||
seen_keys.add(key)
|
||||
label = f"{archive_name}:{rom['name']}"
|
||||
|
||||
@@ -251,7 +251,9 @@ def _diff_fbneo(
|
||||
continue
|
||||
|
||||
old = existing_by_key[key]
|
||||
if old.get('crc32') != rom.get('crc32') or old.get('size') != rom.get('size'):
|
||||
if old.get("crc32") != rom.get("crc32") or old.get("size") != rom.get(
|
||||
"size"
|
||||
):
|
||||
updated.append(label)
|
||||
else:
|
||||
unchanged += 1
|
||||
@@ -259,11 +261,11 @@ def _diff_fbneo(
|
||||
out_of_scope = sum(1 for k in existing_by_key if k not in seen_keys)
|
||||
|
||||
return {
|
||||
'added': added,
|
||||
'updated': updated,
|
||||
'removed': [],
|
||||
'unchanged': unchanged,
|
||||
'out_of_scope': out_of_scope,
|
||||
"added": added,
|
||||
"updated": updated,
|
||||
"removed": [],
|
||||
"unchanged": unchanged,
|
||||
"out_of_scope": out_of_scope,
|
||||
}
|
||||
|
||||
|
||||
@@ -271,12 +273,12 @@ def _diff_fbneo(
|
||||
|
||||
|
||||
def _load_yaml(path: str) -> dict[str, Any]:
|
||||
with open(path, encoding='utf-8') as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
def _load_json(path: str) -> dict[str, Any]:
|
||||
with open(path, encoding='utf-8') as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@@ -295,7 +297,7 @@ def _split_files(
|
||||
|
||||
|
||||
def _zip_name_to_set(name: str) -> str:
|
||||
if name.endswith('.zip'):
|
||||
if name.endswith(".zip"):
|
||||
return name[:-4]
|
||||
return name
|
||||
|
||||
@@ -304,42 +306,42 @@ def _build_contents(roms: list[dict]) -> list[dict]:
|
||||
contents: list[dict] = []
|
||||
for rom in roms:
|
||||
entry: dict[str, Any] = {
|
||||
'name': rom['name'],
|
||||
'size': rom['size'],
|
||||
'crc32': rom['crc32'],
|
||||
"name": rom["name"],
|
||||
"size": rom["size"],
|
||||
"crc32": rom["crc32"],
|
||||
}
|
||||
if rom.get('sha1'):
|
||||
entry['sha1'] = rom['sha1']
|
||||
desc = rom.get('bios_description') or rom.get('bios_label') or ''
|
||||
if rom.get("sha1"):
|
||||
entry["sha1"] = rom["sha1"]
|
||||
desc = rom.get("bios_description") or rom.get("bios_label") or ""
|
||||
if desc:
|
||||
entry['description'] = desc
|
||||
if rom.get('bad_dump'):
|
||||
entry['bad_dump'] = True
|
||||
entry["description"] = desc
|
||||
if rom.get("bad_dump"):
|
||||
entry["bad_dump"] = True
|
||||
contents.append(entry)
|
||||
return contents
|
||||
|
||||
|
||||
def _build_source_ref(set_data: dict) -> str:
|
||||
source_file = set_data.get('source_file', '')
|
||||
source_line = set_data.get('source_line')
|
||||
source_file = set_data.get("source_file", "")
|
||||
source_line = set_data.get("source_line")
|
||||
if source_file and source_line is not None:
|
||||
return f'{source_file}:{source_line}'
|
||||
return f"{source_file}:{source_line}"
|
||||
return source_file
|
||||
|
||||
|
||||
def _contents_differ(old: list[dict], new: list[dict]) -> bool:
|
||||
if len(old) != len(new):
|
||||
return True
|
||||
old_by_name = {c['name']: c for c in old}
|
||||
old_by_name = {c["name"]: c for c in old}
|
||||
for entry in new:
|
||||
prev = old_by_name.get(entry['name'])
|
||||
prev = old_by_name.get(entry["name"])
|
||||
if prev is None:
|
||||
return True
|
||||
if prev.get('crc32') != entry.get('crc32'):
|
||||
if prev.get("crc32") != entry.get("crc32"):
|
||||
return True
|
||||
if prev.get('size') != entry.get('size'):
|
||||
if prev.get("size") != entry.get("size"):
|
||||
return True
|
||||
if prev.get('sha1') != entry.get('sha1'):
|
||||
if prev.get("sha1") != entry.get("sha1"):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -352,15 +354,15 @@ def _backup_and_write(path: str, data: dict) -> None:
|
||||
(core_version, contents, source_ref), and appends new entries.
|
||||
"""
|
||||
p = Path(path)
|
||||
backup = p.with_suffix('.old.yml')
|
||||
backup = p.with_suffix(".old.yml")
|
||||
shutil.copy2(p, backup)
|
||||
|
||||
original = p.read_text(encoding='utf-8')
|
||||
patched = _patch_core_version(original, data.get('core_version', ''))
|
||||
patched = _patch_bios_entries(patched, data.get('files', []))
|
||||
patched = _append_new_entries(patched, data.get('files', []), original)
|
||||
original = p.read_text(encoding="utf-8")
|
||||
patched = _patch_core_version(original, data.get("core_version", ""))
|
||||
patched = _patch_bios_entries(patched, data.get("files", []))
|
||||
patched = _append_new_entries(patched, data.get("files", []), original)
|
||||
|
||||
p.write_text(patched, encoding='utf-8')
|
||||
p.write_text(patched, encoding="utf-8")
|
||||
|
||||
|
||||
def _patch_core_version(text: str, version: str) -> str:
|
||||
@@ -368,8 +370,9 @@ def _patch_core_version(text: str, version: str) -> str:
|
||||
if not version:
|
||||
return text
|
||||
import re
|
||||
|
||||
return re.sub(
|
||||
r'^(core_version:\s*).*$',
|
||||
r"^(core_version:\s*).*$",
|
||||
rf'\g<1>"{version}"',
|
||||
text,
|
||||
count=1,
|
||||
@@ -390,18 +393,18 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
|
||||
# Build a lookup of what to patch
|
||||
patches: dict[str, dict] = {}
|
||||
for fe in files:
|
||||
if fe.get('category') != 'bios_zip':
|
||||
if fe.get("category") != "bios_zip":
|
||||
continue
|
||||
patches[fe['name']] = fe
|
||||
patches[fe["name"]] = fe
|
||||
|
||||
if not patches:
|
||||
return text
|
||||
|
||||
lines = text.split('\n')
|
||||
lines = text.split("\n")
|
||||
# Find all entry start positions (line indices)
|
||||
entry_starts: list[tuple[int, str]] = []
|
||||
for i, line in enumerate(lines):
|
||||
m = re.match(r'^ - name:\s*(.+?)\s*$', line)
|
||||
m = re.match(r"^ - name:\s*(.+?)\s*$", line)
|
||||
if m:
|
||||
entry_starts.append((i, m.group(1).strip('"').strip("'")))
|
||||
|
||||
@@ -412,8 +415,8 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
|
||||
continue
|
||||
|
||||
fe = patches[entry_name]
|
||||
contents = fe.get('contents', [])
|
||||
source_ref = fe.get('source_ref', '')
|
||||
contents = fe.get("contents", [])
|
||||
source_ref = fe.get("source_ref", "")
|
||||
|
||||
# Find the last "owned" line of this entry
|
||||
# Owned = indented with 4+ spaces (field lines of this entry)
|
||||
@@ -422,11 +425,11 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
|
||||
stripped = lines[j].strip()
|
||||
if not stripped:
|
||||
break # blank line = end of entry
|
||||
if stripped.startswith('#'):
|
||||
if stripped.startswith("#"):
|
||||
break # comment = belongs to next entry
|
||||
if re.match(r'^ - ', lines[j]):
|
||||
if re.match(r"^ - ", lines[j]):
|
||||
break # next list item
|
||||
if re.match(r'^ ', lines[j]) or re.match(r'^ \w', lines[j]):
|
||||
if re.match(r"^ ", lines[j]) or re.match(r"^ \w", lines[j]):
|
||||
last_owned = j
|
||||
else:
|
||||
break
|
||||
@@ -435,7 +438,7 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
|
||||
if source_ref:
|
||||
found_sr = False
|
||||
for j in range(start_line + 1, last_owned + 1):
|
||||
if re.match(r'^ source_ref:', lines[j]):
|
||||
if re.match(r"^ source_ref:", lines[j]):
|
||||
lines[j] = f' source_ref: "{source_ref}"'
|
||||
found_sr = True
|
||||
break
|
||||
@@ -447,10 +450,10 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
|
||||
contents_start = None
|
||||
contents_end = None
|
||||
for j in range(start_line + 1, last_owned + 1):
|
||||
if re.match(r'^ contents:', lines[j]):
|
||||
if re.match(r"^ contents:", lines[j]):
|
||||
contents_start = j
|
||||
elif contents_start is not None:
|
||||
if re.match(r'^ ', lines[j]):
|
||||
if re.match(r"^ ", lines[j]):
|
||||
contents_end = j
|
||||
else:
|
||||
break
|
||||
@@ -458,29 +461,29 @@ def _patch_bios_entries(text: str, files: list[dict]) -> str:
|
||||
contents_end = contents_start
|
||||
|
||||
if contents_start is not None:
|
||||
del lines[contents_start:contents_end + 1]
|
||||
last_owned -= (contents_end - contents_start + 1)
|
||||
del lines[contents_start : contents_end + 1]
|
||||
last_owned -= contents_end - contents_start + 1
|
||||
|
||||
# Insert new contents after last owned line
|
||||
if contents:
|
||||
new_lines = _format_contents(contents).split('\n')
|
||||
new_lines = _format_contents(contents).split("\n")
|
||||
for k, cl in enumerate(new_lines):
|
||||
lines.insert(last_owned + 1 + k, cl)
|
||||
|
||||
return '\n'.join(lines)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _append_new_entries(text: str, files: list[dict], original: str) -> str:
|
||||
"""Append new bios_zip entries (system=None) that aren't in the original."""
|
||||
# Parse original to get existing entry names (more reliable than text search)
|
||||
existing_data = yaml.safe_load(original) or {}
|
||||
existing_names = {f['name'] for f in existing_data.get('files', [])}
|
||||
existing_names = {f["name"] for f in existing_data.get("files", [])}
|
||||
|
||||
new_entries = []
|
||||
for fe in files:
|
||||
if fe.get('category') != 'bios_zip' or fe.get('system') is not None:
|
||||
if fe.get("category") != "bios_zip" or fe.get("system") is not None:
|
||||
continue
|
||||
if fe['name'] in existing_names:
|
||||
if fe["name"] in existing_names:
|
||||
continue
|
||||
new_entries.append(fe)
|
||||
|
||||
@@ -489,36 +492,36 @@ def _append_new_entries(text: str, files: list[dict], original: str) -> str:
|
||||
|
||||
lines = []
|
||||
for fe in new_entries:
|
||||
lines.append(f'\n - name: {fe["name"]}')
|
||||
lines.append(f' required: {str(fe["required"]).lower()}')
|
||||
lines.append(f' category: bios_zip')
|
||||
if fe.get('source_ref'):
|
||||
lines.append(f"\n - name: {fe['name']}")
|
||||
lines.append(f" required: {str(fe['required']).lower()}")
|
||||
lines.append(" category: bios_zip")
|
||||
if fe.get("source_ref"):
|
||||
lines.append(f' source_ref: "{fe["source_ref"]}"')
|
||||
if fe.get('contents'):
|
||||
lines.append(_format_contents(fe['contents']))
|
||||
if fe.get("contents"):
|
||||
lines.append(_format_contents(fe["contents"]))
|
||||
|
||||
if lines:
|
||||
text = text.rstrip('\n') + '\n' + '\n'.join(lines) + '\n'
|
||||
text = text.rstrip("\n") + "\n" + "\n".join(lines) + "\n"
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _format_contents(contents: list[dict]) -> str:
|
||||
"""Format a contents list as YAML text."""
|
||||
lines = [' contents:']
|
||||
lines = [" contents:"]
|
||||
for rom in contents:
|
||||
lines.append(f' - name: {rom["name"]}')
|
||||
if rom.get('description'):
|
||||
lines.append(f' description: {rom["description"]}')
|
||||
if rom.get('size'):
|
||||
lines.append(f' size: {rom["size"]}')
|
||||
if rom.get('crc32'):
|
||||
lines.append(f" - name: {rom['name']}")
|
||||
if rom.get("description"):
|
||||
lines.append(f" description: {rom['description']}")
|
||||
if rom.get("size"):
|
||||
lines.append(f" size: {rom['size']}")
|
||||
if rom.get("crc32"):
|
||||
lines.append(f' crc32: "{rom["crc32"]}"')
|
||||
if rom.get('sha1'):
|
||||
if rom.get("sha1"):
|
||||
lines.append(f' sha1: "{rom["sha1"]}"')
|
||||
if rom.get('bad_dump'):
|
||||
lines.append(f' bad_dump: true')
|
||||
return '\n'.join(lines)
|
||||
if rom.get("bad_dump"):
|
||||
lines.append(" bad_dump: true")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None:
|
||||
@@ -529,37 +532,38 @@ def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None:
|
||||
Existing entries are left untouched (CRC32 changes are rare).
|
||||
"""
|
||||
p = Path(path)
|
||||
backup = p.with_suffix('.old.yml')
|
||||
backup = p.with_suffix(".old.yml")
|
||||
shutil.copy2(p, backup)
|
||||
|
||||
original = p.read_text(encoding='utf-8')
|
||||
patched = _patch_core_version(original, data.get('core_version', ''))
|
||||
original = p.read_text(encoding="utf-8")
|
||||
patched = _patch_core_version(original, data.get("core_version", ""))
|
||||
|
||||
# Identify new ROM entries by comparing parsed data keys, not text search
|
||||
existing_data = yaml.safe_load(original) or {}
|
||||
existing_keys = {
|
||||
(f['archive'], f['name'])
|
||||
for f in existing_data.get('files', [])
|
||||
if f.get('archive')
|
||||
(f["archive"], f["name"])
|
||||
for f in existing_data.get("files", [])
|
||||
if f.get("archive")
|
||||
}
|
||||
new_roms = [
|
||||
f for f in data.get('files', [])
|
||||
if f.get('archive') and (f['archive'], f['name']) not in existing_keys
|
||||
f
|
||||
for f in data.get("files", [])
|
||||
if f.get("archive") and (f["archive"], f["name"]) not in existing_keys
|
||||
]
|
||||
|
||||
if new_roms:
|
||||
lines = []
|
||||
for fe in new_roms:
|
||||
lines.append(f' - name: "{fe["name"]}"')
|
||||
lines.append(f' archive: {fe["archive"]}')
|
||||
lines.append(f' required: {str(fe.get("required", True)).lower()}')
|
||||
if fe.get('size'):
|
||||
lines.append(f' size: {fe["size"]}')
|
||||
if fe.get('crc32'):
|
||||
lines.append(f" archive: {fe['archive']}")
|
||||
lines.append(f" required: {str(fe.get('required', True)).lower()}")
|
||||
if fe.get("size"):
|
||||
lines.append(f" size: {fe['size']}")
|
||||
if fe.get("crc32"):
|
||||
lines.append(f' crc32: "{fe["crc32"]}"')
|
||||
if fe.get('source_ref'):
|
||||
if fe.get("source_ref"):
|
||||
lines.append(f' source_ref: "{fe["source_ref"]}"')
|
||||
lines.append('')
|
||||
patched = patched.rstrip('\n') + '\n\n' + '\n'.join(lines)
|
||||
lines.append("")
|
||||
patched = patched.rstrip("\n") + "\n\n" + "\n".join(lines)
|
||||
|
||||
p.write_text(patched, encoding='utf-8')
|
||||
p.write_text(patched, encoding="utf-8")
|
||||
|
||||
@@ -4,8 +4,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -14,6 +14,7 @@ from pathlib import Path
|
||||
@dataclass
|
||||
class BiosRequirement:
|
||||
"""A single BIOS file requirement from a platform source."""
|
||||
|
||||
name: str
|
||||
system: str
|
||||
sha1: str | None = None
|
||||
@@ -29,9 +30,12 @@ class BiosRequirement:
|
||||
@dataclass
|
||||
class ChangeSet:
|
||||
"""Differences between scraped requirements and current config."""
|
||||
|
||||
added: list[BiosRequirement] = field(default_factory=list)
|
||||
removed: list[BiosRequirement] = field(default_factory=list)
|
||||
modified: list[tuple[BiosRequirement, BiosRequirement]] = field(default_factory=list)
|
||||
modified: list[tuple[BiosRequirement, BiosRequirement]] = field(
|
||||
default_factory=list
|
||||
)
|
||||
|
||||
@property
|
||||
def has_changes(self) -> bool:
|
||||
@@ -80,7 +84,9 @@ class BaseScraper(ABC):
|
||||
if not self.url:
|
||||
raise ValueError("No source URL configured")
|
||||
try:
|
||||
req = urllib.request.Request(self.url, headers={"User-Agent": "retrobios-scraper/1.0"})
|
||||
req = urllib.request.Request(
|
||||
self.url, headers={"User-Agent": "retrobios-scraper/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
self._raw_data = _read_limited(resp).decode("utf-8")
|
||||
return self._raw_data
|
||||
@@ -113,35 +119,49 @@ class BaseScraper(ABC):
|
||||
changes.added.append(req)
|
||||
else:
|
||||
existing_file = existing[key]
|
||||
if req.sha1 and existing_file.get("sha1") and req.sha1 != existing_file["sha1"]:
|
||||
changes.modified.append((
|
||||
BiosRequirement(
|
||||
name=existing_file["name"],
|
||||
system=key[0],
|
||||
sha1=existing_file.get("sha1"),
|
||||
md5=existing_file.get("md5"),
|
||||
),
|
||||
req,
|
||||
))
|
||||
elif req.md5 and existing_file.get("md5") and req.md5 != existing_file["md5"]:
|
||||
changes.modified.append((
|
||||
BiosRequirement(
|
||||
name=existing_file["name"],
|
||||
system=key[0],
|
||||
md5=existing_file.get("md5"),
|
||||
),
|
||||
req,
|
||||
))
|
||||
if (
|
||||
req.sha1
|
||||
and existing_file.get("sha1")
|
||||
and req.sha1 != existing_file["sha1"]
|
||||
):
|
||||
changes.modified.append(
|
||||
(
|
||||
BiosRequirement(
|
||||
name=existing_file["name"],
|
||||
system=key[0],
|
||||
sha1=existing_file.get("sha1"),
|
||||
md5=existing_file.get("md5"),
|
||||
),
|
||||
req,
|
||||
)
|
||||
)
|
||||
elif (
|
||||
req.md5
|
||||
and existing_file.get("md5")
|
||||
and req.md5 != existing_file["md5"]
|
||||
):
|
||||
changes.modified.append(
|
||||
(
|
||||
BiosRequirement(
|
||||
name=existing_file["name"],
|
||||
system=key[0],
|
||||
md5=existing_file.get("md5"),
|
||||
),
|
||||
req,
|
||||
)
|
||||
)
|
||||
|
||||
for key in existing:
|
||||
if key not in scraped_map:
|
||||
f = existing[key]
|
||||
changes.removed.append(BiosRequirement(
|
||||
name=f["name"],
|
||||
system=key[0],
|
||||
sha1=f.get("sha1"),
|
||||
md5=f.get("md5"),
|
||||
))
|
||||
changes.removed.append(
|
||||
BiosRequirement(
|
||||
name=f["name"],
|
||||
system=key[0],
|
||||
sha1=f.get("sha1"),
|
||||
md5=f.get("md5"),
|
||||
)
|
||||
)
|
||||
|
||||
return changes
|
||||
|
||||
@@ -163,10 +183,13 @@ def fetch_github_latest_version(repo: str) -> str | None:
|
||||
"""Fetch the latest release version tag from a GitHub repo."""
|
||||
url = f"https://api.github.com/repos/{repo}/releases/latest"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
})
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read())
|
||||
return data.get("tag_name", "")
|
||||
@@ -174,7 +197,9 @@ def fetch_github_latest_version(repo: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def scraper_cli(scraper_class: type, description: str = "Scrape BIOS requirements") -> None:
|
||||
def scraper_cli(
|
||||
scraper_class: type, description: str = "Scrape BIOS requirements"
|
||||
) -> None:
|
||||
"""Shared CLI entry point for all scrapers. Eliminates main() boilerplate."""
|
||||
import argparse
|
||||
|
||||
@@ -203,13 +228,23 @@ def scraper_cli(scraper_class: type, description: str = "Scrape BIOS requirement
|
||||
return
|
||||
|
||||
if args.json:
|
||||
data = [{"name": r.name, "system": r.system, "sha1": r.sha1, "md5": r.md5,
|
||||
"size": r.size, "required": r.required} for r in reqs]
|
||||
data = [
|
||||
{
|
||||
"name": r.name,
|
||||
"system": r.system,
|
||||
"sha1": r.sha1,
|
||||
"md5": r.md5,
|
||||
"size": r.size,
|
||||
"required": r.required,
|
||||
}
|
||||
for r in reqs
|
||||
]
|
||||
print(json.dumps(data, indent=2))
|
||||
return
|
||||
|
||||
if args.output:
|
||||
import yaml
|
||||
|
||||
# Use scraper's generate_platform_yaml() if available (includes
|
||||
# platform metadata, cores list, standalone_cores, etc.)
|
||||
if hasattr(scraper, "generate_platform_yaml"):
|
||||
@@ -224,7 +259,11 @@ def scraper_cli(scraper_class: type, description: str = "Scrape BIOS requirement
|
||||
if req.native_id:
|
||||
sys_entry["native_id"] = req.native_id
|
||||
config["systems"][sys_id] = sys_entry
|
||||
entry = {"name": req.name, "destination": req.destination or req.name, "required": req.required}
|
||||
entry = {
|
||||
"name": req.name,
|
||||
"destination": req.destination or req.name,
|
||||
"required": req.required,
|
||||
}
|
||||
if req.sha1:
|
||||
entry["sha1"] = req.sha1
|
||||
if req.md5:
|
||||
@@ -265,10 +304,13 @@ def fetch_github_latest_tag(repo: str, prefix: str = "") -> str | None:
|
||||
"""Fetch the most recent matching tag from a GitHub repo."""
|
||||
url = f"https://api.github.com/repos/{repo}/tags?per_page=50"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
})
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
tags = json.loads(resp.read())
|
||||
for tag in tags:
|
||||
|
||||
@@ -12,8 +12,8 @@ import ast
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
@@ -102,7 +102,6 @@ SYSTEM_SLUG_MAP = {
|
||||
"dragon64": "dragon64",
|
||||
"mc10": "mc10",
|
||||
"msx2+": "microsoft-msx",
|
||||
"msxturbor": "microsoft-msx",
|
||||
"spectravideo": "spectravideo",
|
||||
"tvc": "videoton-tvc",
|
||||
"enterprise": "enterprise-64-128",
|
||||
@@ -116,7 +115,7 @@ SYSTEM_SLUG_MAP = {
|
||||
}
|
||||
|
||||
|
||||
_MD5_RE = re.compile(r'^[a-fA-F0-9]+$')
|
||||
_MD5_RE = re.compile(r"^[a-fA-F0-9]+$")
|
||||
|
||||
|
||||
def _load_md5_index() -> dict[str, str]:
|
||||
@@ -183,11 +182,11 @@ class Scraper(BaseScraper):
|
||||
|
||||
def _extract_systems_dict(self, raw: str) -> dict:
|
||||
"""Extract and parse the 'systems' dict from the Python source via ast.literal_eval."""
|
||||
match = re.search(r'^systems\s*=\s*\{', raw, re.MULTILINE)
|
||||
match = re.search(r"^systems\s*=\s*\{", raw, re.MULTILINE)
|
||||
if not match:
|
||||
raise ValueError("Could not find 'systems = {' in batocera-systems")
|
||||
|
||||
start = match.start() + raw[match.start():].index("{")
|
||||
start = match.start() + raw[match.start() :].index("{")
|
||||
depth = 0
|
||||
i = start
|
||||
in_str = False
|
||||
@@ -195,7 +194,7 @@ class Scraper(BaseScraper):
|
||||
while i < len(raw):
|
||||
ch = raw[i]
|
||||
if in_str:
|
||||
if ch == '\\':
|
||||
if ch == "\\":
|
||||
i += 2
|
||||
continue
|
||||
if ch == str_ch:
|
||||
@@ -214,7 +213,7 @@ class Scraper(BaseScraper):
|
||||
i += 1
|
||||
i += 1
|
||||
|
||||
dict_str = raw[start:i + 1]
|
||||
dict_str = raw[start : i + 1]
|
||||
|
||||
lines = []
|
||||
for line in dict_str.split("\n"):
|
||||
@@ -224,7 +223,7 @@ class Scraper(BaseScraper):
|
||||
j = 0
|
||||
while j < len(line):
|
||||
ch = line[j]
|
||||
if ch == '\\' and j + 1 < len(line):
|
||||
if ch == "\\" and j + 1 < len(line):
|
||||
clean.append(ch)
|
||||
clean.append(line[j + 1])
|
||||
j += 2
|
||||
@@ -246,8 +245,8 @@ class Scraper(BaseScraper):
|
||||
clean_dict_str = "\n".join(lines)
|
||||
|
||||
# OrderedDict({...}) -> just the inner dict literal
|
||||
clean_dict_str = re.sub(r'OrderedDict\(\s*\{', '{', clean_dict_str)
|
||||
clean_dict_str = re.sub(r'\}\s*\)', '}', clean_dict_str)
|
||||
clean_dict_str = re.sub(r"OrderedDict\(\s*\{", "{", clean_dict_str)
|
||||
clean_dict_str = re.sub(r"\}\s*\)", "}", clean_dict_str)
|
||||
|
||||
try:
|
||||
return ast.literal_eval(clean_dict_str)
|
||||
@@ -279,22 +278,24 @@ class Scraper(BaseScraper):
|
||||
|
||||
name = file_path.split("/")[-1] if "/" in file_path else file_path
|
||||
|
||||
requirements.append(BiosRequirement(
|
||||
name=name,
|
||||
system=system_slug,
|
||||
md5=md5 or None,
|
||||
destination=file_path,
|
||||
required=True,
|
||||
zipped_file=zipped_file or None,
|
||||
native_id=sys_key,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=name,
|
||||
system=system_slug,
|
||||
md5=md5 or None,
|
||||
destination=file_path,
|
||||
required=True,
|
||||
zipped_file=zipped_file or None,
|
||||
native_id=sys_key,
|
||||
)
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
def validate_format(self, raw_data: str) -> bool:
|
||||
"""Validate batocera-systems format."""
|
||||
has_systems = "systems" in raw_data and "biosFiles" in raw_data
|
||||
has_dict = re.search(r'^systems\s*=\s*\{', raw_data, re.MULTILINE) is not None
|
||||
has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None
|
||||
has_md5 = '"md5"' in raw_data
|
||||
has_file = '"file"' in raw_data
|
||||
return has_systems and has_dict and has_md5 and has_file
|
||||
@@ -336,7 +337,9 @@ class Scraper(BaseScraper):
|
||||
|
||||
systems[req.system]["files"].append(entry)
|
||||
|
||||
tag = fetch_github_latest_tag("batocera-linux/batocera.linux", prefix="batocera-")
|
||||
tag = fetch_github_latest_tag(
|
||||
"batocera-linux/batocera.linux", prefix="batocera-"
|
||||
)
|
||||
batocera_version = ""
|
||||
if tag:
|
||||
num = tag.removeprefix("batocera-")
|
||||
@@ -344,7 +347,9 @@ class Scraper(BaseScraper):
|
||||
batocera_version = num
|
||||
if not batocera_version:
|
||||
# Preserve existing version when fetch fails (offline mode)
|
||||
existing = Path(__file__).resolve().parents[2] / "platforms" / "batocera.yml"
|
||||
existing = (
|
||||
Path(__file__).resolve().parents[2] / "platforms" / "batocera.yml"
|
||||
)
|
||||
if existing.exists():
|
||||
with open(existing) as f:
|
||||
old = yaml.safe_load(f) or {}
|
||||
@@ -369,6 +374,7 @@ class Scraper(BaseScraper):
|
||||
|
||||
def main():
|
||||
from scripts.scraper.base_scraper import scraper_cli
|
||||
|
||||
scraper_cli(Scraper, "Scrape batocera BIOS requirements")
|
||||
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ the Ideal non-bad option is selected as canonical.
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
try:
|
||||
from .base_scraper import (
|
||||
@@ -108,12 +107,33 @@ SYSTEM_ID_MAP: dict[str, str] = {
|
||||
|
||||
# Cores that overlap with BizHawk's system coverage
|
||||
BIZHAWK_CORES = [
|
||||
"gambatte", "mgba", "sameboy", "melonds", "snes9x", "bsnes",
|
||||
"beetle_psx", "beetle_saturn", "beetle_pce", "beetle_pcfx",
|
||||
"beetle_wswan", "beetle_vb", "beetle_ngp", "opera", "stella",
|
||||
"picodrive", "ppsspp", "handy", "quicknes", "genesis_plus_gx",
|
||||
"ares", "mupen64plus_next", "puae", "prboom", "virtualjaguar",
|
||||
"vice_x64", "mame",
|
||||
"gambatte",
|
||||
"mgba",
|
||||
"sameboy",
|
||||
"melonds",
|
||||
"snes9x",
|
||||
"bsnes",
|
||||
"beetle_psx",
|
||||
"beetle_saturn",
|
||||
"beetle_pce",
|
||||
"beetle_pcfx",
|
||||
"beetle_wswan",
|
||||
"beetle_vb",
|
||||
"beetle_ngp",
|
||||
"opera",
|
||||
"stella",
|
||||
"picodrive",
|
||||
"ppsspp",
|
||||
"handy",
|
||||
"quicknes",
|
||||
"genesis_plus_gx",
|
||||
"ares",
|
||||
"mupen64plus_next",
|
||||
"puae",
|
||||
"prboom",
|
||||
"virtualjaguar",
|
||||
"vice_x64",
|
||||
"mame",
|
||||
]
|
||||
|
||||
|
||||
@@ -137,9 +157,7 @@ def _safe_arithmetic(expr: str) -> int:
|
||||
def _strip_comments(source: str) -> str:
|
||||
"""Remove block comments and #if false blocks."""
|
||||
source = re.sub(r"/\*.*?\*/", "", source, flags=re.DOTALL)
|
||||
source = re.sub(
|
||||
r"#if\s+false\b.*?#endif", "", source, flags=re.DOTALL
|
||||
)
|
||||
source = re.sub(r"#if\s+false\b.*?#endif", "", source, flags=re.DOTALL)
|
||||
return source
|
||||
|
||||
|
||||
@@ -158,14 +176,14 @@ def parse_firmware_database(
|
||||
var_to_hash: dict[str, str] = {}
|
||||
|
||||
file_re = re.compile(
|
||||
r'(?:var\s+(\w+)\s*=\s*)?'
|
||||
r'File\(\s*'
|
||||
r"(?:var\s+(\w+)\s*=\s*)?"
|
||||
r"File\(\s*"
|
||||
r'(?:"([A-Fa-f0-9]+)"|SHA1Checksum\.Dummy)\s*,\s*'
|
||||
r'([^,]+?)\s*,\s*'
|
||||
r"([^,]+?)\s*,\s*"
|
||||
r'"([^"]+)"\s*,\s*'
|
||||
r'"([^"]*)"'
|
||||
r'(?:\s*,\s*isBad:\s*(true|false))?'
|
||||
r'\s*\)'
|
||||
r"(?:\s*,\s*isBad:\s*(true|false))?"
|
||||
r"\s*\)"
|
||||
)
|
||||
|
||||
for m in file_re.finditer(source):
|
||||
@@ -194,15 +212,15 @@ def parse_firmware_database(
|
||||
|
||||
# FirmwareAndOption one-liner
|
||||
fao_re = re.compile(
|
||||
r'FirmwareAndOption\(\s*'
|
||||
r"FirmwareAndOption\(\s*"
|
||||
r'(?:"([A-Fa-f0-9]+)"|SHA1Checksum\.Dummy)\s*,\s*'
|
||||
r'([^,]+?)\s*,\s*'
|
||||
r"([^,]+?)\s*,\s*"
|
||||
r'"([^"]+)"\s*,\s*'
|
||||
r'"([^"]+)"\s*,\s*'
|
||||
r'"([^"]+)"\s*,\s*'
|
||||
r'"([^"]*)"'
|
||||
r'(?:\s*,\s*FirmwareOptionStatus\.(\w+))?'
|
||||
r'\s*\)'
|
||||
r"(?:\s*,\s*FirmwareOptionStatus\.(\w+))?"
|
||||
r"\s*\)"
|
||||
)
|
||||
|
||||
# Firmware(system, id, desc)
|
||||
@@ -213,10 +231,10 @@ def parse_firmware_database(
|
||||
# Option(system, id, in varref|File(...), status?)
|
||||
option_re = re.compile(
|
||||
r'Option\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*'
|
||||
r'(?:in\s+(\w+)'
|
||||
r"(?:in\s+(\w+)"
|
||||
r'|File\(\s*"([A-Fa-f0-9]+)"\s*,\s*([^,]+?)\s*,\s*"([^"]+)"\s*,\s*"([^"]*)"\s*\))'
|
||||
r'(?:\s*,\s*FirmwareOptionStatus\.(\w+))?'
|
||||
r'\s*\)'
|
||||
r"(?:\s*,\s*FirmwareOptionStatus\.(\w+))?"
|
||||
r"\s*\)"
|
||||
)
|
||||
|
||||
# Collect firmware slots
|
||||
@@ -269,15 +287,17 @@ def parse_firmware_database(
|
||||
desc = m.group(6)
|
||||
status = m.group(7) or "Acceptable"
|
||||
|
||||
records.append({
|
||||
"system": system,
|
||||
"firmware_id": fw_id,
|
||||
"sha1": sha1,
|
||||
"name": name,
|
||||
"size": _safe_arithmetic(size_expr),
|
||||
"description": desc,
|
||||
"status": status,
|
||||
})
|
||||
records.append(
|
||||
{
|
||||
"system": system,
|
||||
"firmware_id": fw_id,
|
||||
"sha1": sha1,
|
||||
"name": name,
|
||||
"size": _safe_arithmetic(size_expr),
|
||||
"description": desc,
|
||||
"status": status,
|
||||
}
|
||||
)
|
||||
|
||||
# Build records from Firmware+Option pairs, picking best option
|
||||
for (system, fw_id), options in slot_options.items():
|
||||
@@ -291,15 +311,17 @@ def parse_firmware_database(
|
||||
viable.sort(key=lambda x: STATUS_RANK.get(x[1], 2), reverse=True)
|
||||
best_file, best_status = viable[0]
|
||||
|
||||
records.append({
|
||||
"system": system,
|
||||
"firmware_id": fw_id,
|
||||
"sha1": best_file["sha1"],
|
||||
"name": best_file["name"],
|
||||
"size": best_file["size"],
|
||||
"description": best_file.get("description", desc),
|
||||
"status": best_status,
|
||||
})
|
||||
records.append(
|
||||
{
|
||||
"system": system,
|
||||
"firmware_id": fw_id,
|
||||
"sha1": best_file["sha1"],
|
||||
"name": best_file["name"],
|
||||
"size": best_file["size"],
|
||||
"description": best_file.get("description", desc),
|
||||
"status": best_status,
|
||||
}
|
||||
)
|
||||
|
||||
return records, files_by_hash
|
||||
|
||||
|
||||
@@ -13,19 +13,24 @@ Complements libretro_scraper (System.dat) with:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
|
||||
except ImportError:
|
||||
# Allow running directly: python scripts/scraper/coreinfo_scraper.py
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
from scraper.base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
|
||||
from scraper.base_scraper import (
|
||||
BaseScraper,
|
||||
BiosRequirement,
|
||||
fetch_github_latest_version,
|
||||
)
|
||||
|
||||
PLATFORM_NAME = "libretro_coreinfo"
|
||||
|
||||
@@ -168,11 +173,13 @@ def _extract_firmware(info: dict) -> list[dict]:
|
||||
if _is_native_lib(path):
|
||||
continue
|
||||
|
||||
firmware.append({
|
||||
"path": path,
|
||||
"desc": desc,
|
||||
"optional": opt.lower() == "true",
|
||||
})
|
||||
firmware.append(
|
||||
{
|
||||
"path": path,
|
||||
"desc": desc,
|
||||
"optional": opt.lower() == "true",
|
||||
}
|
||||
)
|
||||
|
||||
return firmware
|
||||
|
||||
@@ -182,7 +189,7 @@ def _extract_md5_from_notes(info: dict) -> dict[str, str]:
|
||||
notes = info.get("notes", "")
|
||||
md5_map = {}
|
||||
|
||||
for match in re.finditer(r'\(!\)\s+(.+?)\s+\(md5\):\s+([a-f0-9]{32})', notes):
|
||||
for match in re.finditer(r"\(!\)\s+(.+?)\s+\(md5\):\s+([a-f0-9]{32})", notes):
|
||||
filename = match.group(1).strip()
|
||||
md5 = match.group(2)
|
||||
md5_map[filename] = md5
|
||||
@@ -202,15 +209,19 @@ class Scraper(BaseScraper):
|
||||
# Use the tree API to get all files at once
|
||||
url = f"{GITHUB_API}/git/trees/master?recursive=1"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
})
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read())
|
||||
|
||||
return [
|
||||
item["path"] for item in data.get("tree", [])
|
||||
item["path"]
|
||||
for item in data.get("tree", [])
|
||||
if item["path"].endswith("_libretro.info")
|
||||
]
|
||||
except (urllib.error.URLError, json.JSONDecodeError) as e:
|
||||
@@ -220,7 +231,9 @@ class Scraper(BaseScraper):
|
||||
"""Fetch and parse a single .info file."""
|
||||
url = f"{RAW_BASE}/{filename}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-scraper/1.0"})
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "retrobios-scraper/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
content = resp.read().decode("utf-8")
|
||||
return _parse_info_file(content)
|
||||
@@ -253,17 +266,25 @@ class Scraper(BaseScraper):
|
||||
|
||||
basename = path.split("/")[-1] if "/" in path else path
|
||||
# Full path when basename is generic to avoid SGB1.sfc/program.rom vs SGB2.sfc/program.rom collisions
|
||||
GENERIC_NAMES = {"program.rom", "data.rom", "boot.rom", "bios.bin", "firmware.bin"}
|
||||
GENERIC_NAMES = {
|
||||
"program.rom",
|
||||
"data.rom",
|
||||
"boot.rom",
|
||||
"bios.bin",
|
||||
"firmware.bin",
|
||||
}
|
||||
name = path if basename.lower() in GENERIC_NAMES else basename
|
||||
md5 = md5_map.get(basename)
|
||||
|
||||
requirements.append(BiosRequirement(
|
||||
name=name,
|
||||
system=system,
|
||||
md5=md5,
|
||||
destination=path,
|
||||
required=not fw["optional"],
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=name,
|
||||
system=system,
|
||||
md5=md5,
|
||||
destination=path,
|
||||
required=not fw["optional"],
|
||||
)
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
@@ -281,7 +302,9 @@ def main():
|
||||
"""CLI entry point."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Scrape libretro-core-info firmware requirements")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Scrape libretro-core-info firmware requirements"
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--compare-db", help="Compare against database.json")
|
||||
args = parser.parse_args()
|
||||
@@ -296,6 +319,7 @@ def main():
|
||||
|
||||
if args.compare_db:
|
||||
import json as _json
|
||||
|
||||
with open(args.compare_db) as f:
|
||||
db = _json.load(f)
|
||||
|
||||
@@ -320,6 +344,7 @@ def main():
|
||||
return
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
by_system = defaultdict(list)
|
||||
for r in reqs:
|
||||
by_system[r.system].append(r)
|
||||
|
||||
@@ -10,13 +10,13 @@ Parses files like libretro's System.dat which uses the format:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatRom:
|
||||
"""A ROM entry from a DAT file."""
|
||||
|
||||
name: str
|
||||
size: int
|
||||
crc32: str
|
||||
@@ -28,6 +28,7 @@ class DatRom:
|
||||
@dataclass
|
||||
class DatMetadata:
|
||||
"""Metadata from a DAT file header."""
|
||||
|
||||
name: str = ""
|
||||
version: str = ""
|
||||
description: str = ""
|
||||
@@ -53,7 +54,10 @@ def parse_dat(content: str) -> list[DatRom]:
|
||||
|
||||
if stripped.startswith("comment "):
|
||||
value = stripped[8:].strip().strip('"')
|
||||
if value in ("System", "System, firmware, and BIOS files used by libretro cores."):
|
||||
if value in (
|
||||
"System",
|
||||
"System, firmware, and BIOS files used by libretro cores.",
|
||||
):
|
||||
continue
|
||||
current_system = value
|
||||
|
||||
@@ -78,9 +82,16 @@ def parse_dat_metadata(content: str) -> DatMetadata:
|
||||
if in_header and stripped == ")":
|
||||
break
|
||||
if in_header:
|
||||
for field in ("name", "version", "description", "author", "homepage", "url"):
|
||||
for field in (
|
||||
"name",
|
||||
"version",
|
||||
"description",
|
||||
"author",
|
||||
"homepage",
|
||||
"url",
|
||||
):
|
||||
if stripped.startswith(f"{field} "):
|
||||
value = stripped[len(field) + 1:].strip().strip('"')
|
||||
value = stripped[len(field) + 1 :].strip().strip('"')
|
||||
setattr(meta, field, value)
|
||||
|
||||
return meta
|
||||
@@ -94,7 +105,7 @@ def _parse_rom_line(line: str, system: str) -> DatRom | None:
|
||||
if start == -1 or end == -1 or end <= start:
|
||||
return None
|
||||
|
||||
content = line[start + 1:end].strip()
|
||||
content = line[start + 1 : end].strip()
|
||||
|
||||
fields = {}
|
||||
i = 0
|
||||
|
||||
@@ -14,9 +14,8 @@ from __future__ import annotations
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
|
||||
@@ -31,8 +30,7 @@ CHECKBIOS_URL = (
|
||||
)
|
||||
|
||||
CSV_BASE_URL = (
|
||||
"https://raw.githubusercontent.com/EmuDeck/emudeck.github.io/"
|
||||
"main/docs/tables"
|
||||
"https://raw.githubusercontent.com/EmuDeck/emudeck.github.io/main/docs/tables"
|
||||
)
|
||||
|
||||
CSV_SHEETS = [
|
||||
@@ -117,10 +115,22 @@ KNOWN_BIOS_FILES = {
|
||||
{"name": "scph5502.bin", "destination": "scph5502.bin", "region": "EU"},
|
||||
],
|
||||
"sony-playstation-2": [
|
||||
{"name": "SCPH-70004_BIOS_V12_EUR_200.BIN", "destination": "SCPH-70004_BIOS_V12_EUR_200.BIN"},
|
||||
{"name": "SCPH-70004_BIOS_V12_EUR_200.EROM", "destination": "SCPH-70004_BIOS_V12_EUR_200.EROM"},
|
||||
{"name": "SCPH-70004_BIOS_V12_EUR_200.ROM1", "destination": "SCPH-70004_BIOS_V12_EUR_200.ROM1"},
|
||||
{"name": "SCPH-70004_BIOS_V12_EUR_200.ROM2", "destination": "SCPH-70004_BIOS_V12_EUR_200.ROM2"},
|
||||
{
|
||||
"name": "SCPH-70004_BIOS_V12_EUR_200.BIN",
|
||||
"destination": "SCPH-70004_BIOS_V12_EUR_200.BIN",
|
||||
},
|
||||
{
|
||||
"name": "SCPH-70004_BIOS_V12_EUR_200.EROM",
|
||||
"destination": "SCPH-70004_BIOS_V12_EUR_200.EROM",
|
||||
},
|
||||
{
|
||||
"name": "SCPH-70004_BIOS_V12_EUR_200.ROM1",
|
||||
"destination": "SCPH-70004_BIOS_V12_EUR_200.ROM1",
|
||||
},
|
||||
{
|
||||
"name": "SCPH-70004_BIOS_V12_EUR_200.ROM2",
|
||||
"destination": "SCPH-70004_BIOS_V12_EUR_200.ROM2",
|
||||
},
|
||||
],
|
||||
"sega-mega-cd": [
|
||||
{"name": "bios_CD_E.bin", "destination": "bios_CD_E.bin", "region": "EU"},
|
||||
@@ -157,17 +167,17 @@ KNOWN_BIOS_FILES = {
|
||||
}
|
||||
|
||||
_RE_ARRAY = re.compile(
|
||||
r'(?:local\s+)?(\w+)=\(\s*((?:[0-9a-fA-F]+\s*)+)\)',
|
||||
r"(?:local\s+)?(\w+)=\(\s*((?:[0-9a-fA-F]+\s*)+)\)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_RE_FUNC = re.compile(
|
||||
r'function\s+(check\w+Bios)\s*\(\)',
|
||||
r"function\s+(check\w+Bios)\s*\(\)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_RE_LOCAL_HASHES = re.compile(
|
||||
r'local\s+hashes=\(\s*((?:[0-9a-fA-F]+\s*)+)\)',
|
||||
r"local\s+hashes=\(\s*((?:[0-9a-fA-F]+\s*)+)\)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
@@ -184,7 +194,9 @@ def _fetch_url(url: str) -> str:
|
||||
class Scraper(BaseScraper):
|
||||
"""Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets."""
|
||||
|
||||
def __init__(self, checkbios_url: str = CHECKBIOS_URL, csv_base_url: str = CSV_BASE_URL):
|
||||
def __init__(
|
||||
self, checkbios_url: str = CHECKBIOS_URL, csv_base_url: str = CSV_BASE_URL
|
||||
):
|
||||
super().__init__(url=checkbios_url)
|
||||
self.checkbios_url = checkbios_url
|
||||
self.csv_base_url = csv_base_url
|
||||
@@ -241,12 +253,12 @@ class Scraper(BaseScraper):
|
||||
@staticmethod
|
||||
def _clean_markdown(text: str) -> str:
|
||||
"""Strip markdown/HTML artifacts from CSV fields."""
|
||||
text = re.sub(r'\*\*', '', text) # bold
|
||||
text = re.sub(r':material-[^:]+:\{[^}]*\}', '', text) # mkdocs material icons
|
||||
text = re.sub(r':material-[^:]+:', '', text)
|
||||
text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text) # [text](url) -> text
|
||||
text = re.sub(r'<br\s*/?>', ' ', text) # <br/>
|
||||
text = re.sub(r'<[^>]+>', '', text) # remaining HTML
|
||||
text = re.sub(r"\*\*", "", text) # bold
|
||||
text = re.sub(r":material-[^:]+:\{[^}]*\}", "", text) # mkdocs material icons
|
||||
text = re.sub(r":material-[^:]+:", "", text)
|
||||
text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text) # [text](url) -> text
|
||||
text = re.sub(r"<br\s*/?>", " ", text) # <br/>
|
||||
text = re.sub(r"<[^>]+>", "", text) # remaining HTML
|
||||
return text.strip()
|
||||
|
||||
def _parse_csv_bios(self, csv_text: str) -> list[dict]:
|
||||
@@ -274,28 +286,32 @@ class Scraper(BaseScraper):
|
||||
system_col = self._clean_markdown((row[key] or ""))
|
||||
break
|
||||
slug = None
|
||||
for part in re.split(r'[`\s/]+', folder_col):
|
||||
part = part.strip().strip('`').lower()
|
||||
for part in re.split(r"[`\s/]+", folder_col):
|
||||
part = part.strip().strip("`").lower()
|
||||
if part and part in SYSTEM_SLUG_MAP:
|
||||
slug = SYSTEM_SLUG_MAP[part]
|
||||
break
|
||||
if not slug:
|
||||
clean = re.sub(r'[^a-z0-9\-]', '', folder_col.strip().strip('`').lower())
|
||||
clean = re.sub(
|
||||
r"[^a-z0-9\-]", "", folder_col.strip().strip("`").lower()
|
||||
)
|
||||
slug = clean if clean else "unknown"
|
||||
entries.append({
|
||||
"system": slug,
|
||||
"system_name": system_col,
|
||||
"bios_raw": bios_col,
|
||||
})
|
||||
entries.append(
|
||||
{
|
||||
"system": slug,
|
||||
"system_name": system_col,
|
||||
"bios_raw": bios_col,
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
def _extract_filenames_from_bios_field(self, bios_raw: str) -> list[dict]:
|
||||
"""Extract individual BIOS filenames from a CSV BIOS field."""
|
||||
results = []
|
||||
bios_raw = re.sub(r'<br\s*/?>', ' ', bios_raw)
|
||||
bios_raw = bios_raw.replace('`', '')
|
||||
bios_raw = re.sub(r"<br\s*/?>", " ", bios_raw)
|
||||
bios_raw = bios_raw.replace("`", "")
|
||||
patterns = re.findall(
|
||||
r'[\w\-./]+\.(?:bin|rom|zip|BIN|ROM|ZIP|EROM|ROM1|ROM2|n64|txt|keys)',
|
||||
r"[\w\-./]+\.(?:bin|rom|zip|BIN|ROM|ZIP|EROM|ROM1|ROM2|n64|txt|keys)",
|
||||
bios_raw,
|
||||
)
|
||||
for p in patterns:
|
||||
@@ -324,21 +340,25 @@ class Scraper(BaseScraper):
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
requirements.append(BiosRequirement(
|
||||
name=f["name"],
|
||||
system=system,
|
||||
destination=f.get("destination", f["name"]),
|
||||
required=True,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=f["name"],
|
||||
system=system,
|
||||
destination=f.get("destination", f["name"]),
|
||||
required=True,
|
||||
)
|
||||
)
|
||||
|
||||
for md5 in system_hashes:
|
||||
requirements.append(BiosRequirement(
|
||||
name=f"{system}:{md5}",
|
||||
system=system,
|
||||
md5=md5,
|
||||
destination="",
|
||||
required=True,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=f"{system}:{md5}",
|
||||
system=system,
|
||||
md5=md5,
|
||||
destination="",
|
||||
required=True,
|
||||
)
|
||||
)
|
||||
|
||||
for sheet in CSV_SHEETS:
|
||||
csv_text = self._fetch_csv(sheet)
|
||||
@@ -353,19 +373,21 @@ class Scraper(BaseScraper):
|
||||
seen.add(key)
|
||||
if system in KNOWN_BIOS_FILES:
|
||||
continue
|
||||
requirements.append(BiosRequirement(
|
||||
name=f["name"],
|
||||
system=system,
|
||||
destination=f.get("destination", f["name"]),
|
||||
required=True,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=f["name"],
|
||||
system=system,
|
||||
destination=f.get("destination", f["name"]),
|
||||
required=True,
|
||||
)
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
def validate_format(self, raw_data: str) -> bool:
|
||||
has_ps = "PSBios=" in raw_data or "PSBios =" in raw_data
|
||||
has_func = "checkPS1BIOS" in raw_data or "checkPS2BIOS" in raw_data
|
||||
has_md5 = re.search(r'[0-9a-f]{32}', raw_data) is not None
|
||||
has_md5 = re.search(r"[0-9a-f]{32}", raw_data) is not None
|
||||
return has_ps and has_func and has_md5
|
||||
|
||||
def generate_platform_yaml(self) -> dict:
|
||||
@@ -419,14 +441,17 @@ class Scraper(BaseScraper):
|
||||
"contents/functions/EmuScripts"
|
||||
)
|
||||
name_overrides = {
|
||||
"pcsx2qt": "pcsx2", "rpcs3legacy": "rpcs3",
|
||||
"cemuproton": "cemu", "rmg": "mupen64plus_next",
|
||||
"pcsx2qt": "pcsx2",
|
||||
"rpcs3legacy": "rpcs3",
|
||||
"cemuproton": "cemu",
|
||||
"rmg": "mupen64plus_next",
|
||||
}
|
||||
skip = {"retroarch_maincfg", "retroarch"}
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
api_url, headers={"User-Agent": "retrobios-scraper/1.0"},
|
||||
api_url,
|
||||
headers={"User-Agent": "retrobios-scraper/1.0"},
|
||||
)
|
||||
data = json.loads(urllib.request.urlopen(req, timeout=30).read())
|
||||
except (urllib.error.URLError, OSError):
|
||||
@@ -454,6 +479,7 @@ class Scraper(BaseScraper):
|
||||
|
||||
def main():
|
||||
from scripts.scraper.base_scraper import scraper_cli
|
||||
|
||||
scraper_cli(Scraper, "Scrape emudeck BIOS requirements")
|
||||
|
||||
|
||||
|
||||
@@ -13,22 +13,22 @@ import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from scripts.scraper.fbneo_parser import parse_fbneo_source_tree
|
||||
from scripts.scraper._hash_merge import compute_diff, merge_fbneo_profile
|
||||
from scripts.scraper.fbneo_parser import parse_fbneo_source_tree
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
REPO_URL = 'https://github.com/finalburnneo/FBNeo.git'
|
||||
REPO_URL = "https://github.com/finalburnneo/FBNeo.git"
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
CLONE_DIR = REPO_ROOT / 'tmp' / 'fbneo'
|
||||
CACHE_PATH = REPO_ROOT / 'data' / 'fbneo-hashes.json'
|
||||
EMULATORS_DIR = REPO_ROOT / 'emulators'
|
||||
CLONE_DIR = REPO_ROOT / "tmp" / "fbneo"
|
||||
CACHE_PATH = REPO_ROOT / "data" / "fbneo-hashes.json"
|
||||
EMULATORS_DIR = REPO_ROOT / "emulators"
|
||||
STALE_HOURS = 24
|
||||
|
||||
|
||||
@@ -37,8 +37,8 @@ def _is_cache_fresh() -> bool:
|
||||
if not CACHE_PATH.exists():
|
||||
return False
|
||||
try:
|
||||
data = json.loads(CACHE_PATH.read_text(encoding='utf-8'))
|
||||
fetched_at = datetime.fromisoformat(data['fetched_at'])
|
||||
data = json.loads(CACHE_PATH.read_text(encoding="utf-8"))
|
||||
fetched_at = datetime.fromisoformat(data["fetched_at"])
|
||||
return datetime.now(timezone.utc) - fetched_at < timedelta(hours=STALE_HOURS)
|
||||
except (json.JSONDecodeError, KeyError, ValueError):
|
||||
return False
|
||||
@@ -53,8 +53,14 @@ def _sparse_clone() -> None:
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
'git', 'clone', '--depth', '1', '--filter=blob:none',
|
||||
'--sparse', REPO_URL, str(CLONE_DIR),
|
||||
"git",
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"--filter=blob:none",
|
||||
"--sparse",
|
||||
REPO_URL,
|
||||
str(CLONE_DIR),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
@@ -62,7 +68,7 @@ def _sparse_clone() -> None:
|
||||
)
|
||||
|
||||
subprocess.run(
|
||||
['git', 'sparse-checkout', 'set', 'src/burn/drv', 'src/burner/resource.h'],
|
||||
["git", "sparse-checkout", "set", "src/burn/drv", "src/burner/resource.h"],
|
||||
cwd=CLONE_DIR,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
@@ -76,42 +82,44 @@ def _extract_version() -> tuple[str, str]:
|
||||
Returns (version, commit_sha). Falls back to resource.h if no tag.
|
||||
"""
|
||||
result = subprocess.run(
|
||||
['git', 'describe', '--tags', '--abbrev=0'],
|
||||
["git", "describe", "--tags", "--abbrev=0"],
|
||||
cwd=CLONE_DIR,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
# Prefer real version tags over pseudo-tags like "latest"
|
||||
version = 'unknown'
|
||||
version = "unknown"
|
||||
if result.returncode == 0:
|
||||
tag = result.stdout.strip()
|
||||
if tag and tag != 'latest':
|
||||
if tag and tag != "latest":
|
||||
version = tag
|
||||
# Fallback: resource.h
|
||||
if version == 'unknown':
|
||||
if version == "unknown":
|
||||
version = _version_from_resource_h()
|
||||
# Last resort: use GitHub API for latest real release tag
|
||||
if version == 'unknown':
|
||||
if version == "unknown":
|
||||
try:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
req = urllib.request.Request(
|
||||
'https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10',
|
||||
headers={'User-Agent': 'retrobios-scraper/1.0'},
|
||||
"https://api.github.com/repos/finalburnneo/FBNeo/tags?per_page=10",
|
||||
headers={"User-Agent": "retrobios-scraper/1.0"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
import json as json_mod
|
||||
|
||||
tags = json_mod.loads(resp.read())
|
||||
for t in tags:
|
||||
if t['name'] != 'latest' and t['name'].startswith('v'):
|
||||
version = t['name']
|
||||
if t["name"] != "latest" and t["name"].startswith("v"):
|
||||
version = t["name"]
|
||||
break
|
||||
except (urllib.error.URLError, OSError):
|
||||
pass
|
||||
|
||||
sha_result = subprocess.run(
|
||||
['git', 'rev-parse', 'HEAD'],
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=CLONE_DIR,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
@@ -124,17 +132,17 @@ def _extract_version() -> tuple[str, str]:
|
||||
|
||||
def _version_from_resource_h() -> str:
|
||||
"""Fallback: parse VER_FULL_VERSION_STR from resource.h."""
|
||||
resource_h = CLONE_DIR / 'src' / 'burner' / 'resource.h'
|
||||
resource_h = CLONE_DIR / "src" / "burner" / "resource.h"
|
||||
if not resource_h.exists():
|
||||
return 'unknown'
|
||||
return "unknown"
|
||||
|
||||
text = resource_h.read_text(encoding='utf-8', errors='replace')
|
||||
text = resource_h.read_text(encoding="utf-8", errors="replace")
|
||||
for line in text.splitlines():
|
||||
if 'VER_FULL_VERSION_STR' in line:
|
||||
if "VER_FULL_VERSION_STR" in line:
|
||||
parts = line.split('"')
|
||||
if len(parts) >= 2:
|
||||
return parts[1]
|
||||
return 'unknown'
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _cleanup() -> None:
|
||||
@@ -146,33 +154,33 @@ def _cleanup() -> None:
|
||||
def fetch_and_cache(force: bool = False) -> dict[str, Any]:
|
||||
"""Clone, parse, and write JSON cache. Returns the cache dict."""
|
||||
if not force and _is_cache_fresh():
|
||||
log.info('cache fresh, skipping clone (use --force to override)')
|
||||
return json.loads(CACHE_PATH.read_text(encoding='utf-8'))
|
||||
log.info("cache fresh, skipping clone (use --force to override)")
|
||||
return json.loads(CACHE_PATH.read_text(encoding="utf-8"))
|
||||
|
||||
try:
|
||||
log.info('sparse cloning %s', REPO_URL)
|
||||
log.info("sparse cloning %s", REPO_URL)
|
||||
_sparse_clone()
|
||||
|
||||
log.info('extracting version')
|
||||
log.info("extracting version")
|
||||
version, commit = _extract_version()
|
||||
|
||||
log.info('parsing source tree')
|
||||
log.info("parsing source tree")
|
||||
bios_sets = parse_fbneo_source_tree(str(CLONE_DIR))
|
||||
|
||||
cache: dict[str, Any] = {
|
||||
'source': 'finalburnneo/FBNeo',
|
||||
'version': version,
|
||||
'commit': commit,
|
||||
'fetched_at': datetime.now(timezone.utc).isoformat(),
|
||||
'bios_sets': bios_sets,
|
||||
"source": "finalburnneo/FBNeo",
|
||||
"version": version,
|
||||
"commit": commit,
|
||||
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
||||
"bios_sets": bios_sets,
|
||||
}
|
||||
|
||||
CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
CACHE_PATH.write_text(
|
||||
json.dumps(cache, indent=2, ensure_ascii=False) + '\n',
|
||||
encoding='utf-8',
|
||||
json.dumps(cache, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
log.info('wrote %d BIOS sets to %s', len(bios_sets), CACHE_PATH)
|
||||
log.info("wrote %d BIOS sets to %s", len(bios_sets), CACHE_PATH)
|
||||
|
||||
return cache
|
||||
finally:
|
||||
@@ -182,48 +190,50 @@ def fetch_and_cache(force: bool = False) -> dict[str, Any]:
|
||||
def _find_fbneo_profiles() -> list[Path]:
|
||||
"""Find emulator profiles whose upstream references finalburnneo/FBNeo."""
|
||||
profiles: list[Path] = []
|
||||
for path in sorted(EMULATORS_DIR.glob('*.yml')):
|
||||
if path.name.endswith('.old.yml'):
|
||||
for path in sorted(EMULATORS_DIR.glob("*.yml")):
|
||||
if path.name.endswith(".old.yml"):
|
||||
continue
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding='utf-8'))
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except (yaml.YAMLError, OSError):
|
||||
continue
|
||||
if not data or not isinstance(data, dict):
|
||||
continue
|
||||
upstream = data.get('upstream', '')
|
||||
if isinstance(upstream, str) and 'finalburnneo/fbneo' in upstream.lower():
|
||||
upstream = data.get("upstream", "")
|
||||
if isinstance(upstream, str) and "finalburnneo/fbneo" in upstream.lower():
|
||||
profiles.append(path)
|
||||
return profiles
|
||||
|
||||
|
||||
def _format_diff(profile_name: str, diff: dict[str, Any], show_added: bool = True) -> str:
|
||||
def _format_diff(
|
||||
profile_name: str, diff: dict[str, Any], show_added: bool = True
|
||||
) -> str:
|
||||
"""Format diff for a single profile."""
|
||||
lines: list[str] = []
|
||||
lines.append(f' {profile_name}:')
|
||||
lines.append(f" {profile_name}:")
|
||||
|
||||
added = diff.get('added', [])
|
||||
updated = diff.get('updated', [])
|
||||
oos = diff.get('out_of_scope', 0)
|
||||
added = diff.get("added", [])
|
||||
updated = diff.get("updated", [])
|
||||
oos = diff.get("out_of_scope", 0)
|
||||
|
||||
if not added and not updated:
|
||||
lines.append(' no changes')
|
||||
lines.append(" no changes")
|
||||
if oos:
|
||||
lines.append(f' . {oos} out of scope')
|
||||
return '\n'.join(lines)
|
||||
lines.append(f" . {oos} out of scope")
|
||||
return "\n".join(lines)
|
||||
|
||||
if show_added:
|
||||
for label in added:
|
||||
lines.append(f' + {label}')
|
||||
lines.append(f" + {label}")
|
||||
elif added:
|
||||
lines.append(f' + {len(added)} new ROMs available (main profile only)')
|
||||
lines.append(f" + {len(added)} new ROMs available (main profile only)")
|
||||
for label in updated:
|
||||
lines.append(f' ~ {label}')
|
||||
lines.append(f' = {diff["unchanged"]} unchanged')
|
||||
lines.append(f" ~ {label}")
|
||||
lines.append(f" = {diff['unchanged']} unchanged")
|
||||
if oos:
|
||||
lines.append(f' . {oos} out of scope')
|
||||
lines.append(f" . {oos} out of scope")
|
||||
|
||||
return '\n'.join(lines)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def run(
|
||||
@@ -234,82 +244,84 @@ def run(
|
||||
"""Main entry point for the scraper."""
|
||||
cache = fetch_and_cache(force=force)
|
||||
|
||||
version = cache.get('version', 'unknown')
|
||||
commit = cache.get('commit', '?')[:12]
|
||||
bios_sets = cache.get('bios_sets', {})
|
||||
version = cache.get("version", "unknown")
|
||||
commit = cache.get("commit", "?")[:12]
|
||||
bios_sets = cache.get("bios_sets", {})
|
||||
profiles = _find_fbneo_profiles()
|
||||
|
||||
if json_output:
|
||||
result: dict[str, Any] = {
|
||||
'source': cache.get('source'),
|
||||
'version': version,
|
||||
'commit': cache.get('commit'),
|
||||
'bios_set_count': len(bios_sets),
|
||||
'profiles': {},
|
||||
"source": cache.get("source"),
|
||||
"version": version,
|
||||
"commit": cache.get("commit"),
|
||||
"bios_set_count": len(bios_sets),
|
||||
"profiles": {},
|
||||
}
|
||||
for path in profiles:
|
||||
diff = compute_diff(str(path), str(CACHE_PATH), mode='fbneo')
|
||||
result['profiles'][path.stem] = diff
|
||||
diff = compute_diff(str(path), str(CACHE_PATH), mode="fbneo")
|
||||
result["profiles"][path.stem] = diff
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
header = (
|
||||
f'fbneo-hashes: {len(bios_sets)} BIOS sets '
|
||||
f'from finalburnneo/FBNeo @ {version} ({commit})'
|
||||
f"fbneo-hashes: {len(bios_sets)} BIOS sets "
|
||||
f"from finalburnneo/FBNeo @ {version} ({commit})"
|
||||
)
|
||||
print(header)
|
||||
print()
|
||||
|
||||
if not profiles:
|
||||
print(' no matching emulator profiles found')
|
||||
print(" no matching emulator profiles found")
|
||||
return 0
|
||||
|
||||
for path in profiles:
|
||||
is_main = path.name == 'fbneo.yml'
|
||||
diff = compute_diff(str(path), str(CACHE_PATH), mode='fbneo')
|
||||
is_main = path.name == "fbneo.yml"
|
||||
diff = compute_diff(str(path), str(CACHE_PATH), mode="fbneo")
|
||||
print(_format_diff(path.stem, diff, show_added=is_main))
|
||||
|
||||
effective_added = diff['added'] if is_main else []
|
||||
if not dry_run and (effective_added or diff['updated']):
|
||||
effective_added = diff["added"] if is_main else []
|
||||
if not dry_run and (effective_added or diff["updated"]):
|
||||
merge_fbneo_profile(str(path), str(CACHE_PATH), write=True, add_new=is_main)
|
||||
log.info('merged changes into %s', path.name)
|
||||
log.info("merged changes into %s", path.name)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Scrape FBNeo BIOS set hashes from upstream source',
|
||||
description="Scrape FBNeo BIOS set hashes from upstream source",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='show diff without writing changes',
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="show diff without writing changes",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--force',
|
||||
action='store_true',
|
||||
help='force re-clone even if cache is fresh',
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="force re-clone even if cache is fresh",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--json',
|
||||
action='store_true',
|
||||
dest='json_output',
|
||||
help='output diff as JSON',
|
||||
"--json",
|
||||
action="store_true",
|
||||
dest="json_output",
|
||||
help="output diff as JSON",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(name)s: %(message)s',
|
||||
format="%(name)s: %(message)s",
|
||||
)
|
||||
|
||||
sys.exit(run(
|
||||
dry_run=args.dry_run,
|
||||
force=args.force,
|
||||
json_output=args.json_output,
|
||||
))
|
||||
sys.exit(
|
||||
run(
|
||||
dry_run=args.dry_run,
|
||||
force=args.force,
|
||||
json_output=args.json_output,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -11,18 +11,17 @@ import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_ROM_ENTRY_RE = re.compile(
|
||||
r'\{\s*"([^"]+)"\s*,\s*(0x[\da-fA-F]+)\s*,\s*(0x[\da-fA-F]+)\s*,\s*([^}]+)\}',
|
||||
)
|
||||
|
||||
_BURN_DRIVER_RE = re.compile(
|
||||
r'struct\s+BurnDriver\s+BurnDrv(\w+)\s*=\s*\{(.*?)\};',
|
||||
r"struct\s+BurnDriver\s+BurnDrv(\w+)\s*=\s*\{(.*?)\};",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
_ROM_DESC_RE = re.compile(
|
||||
r'static\s+struct\s+BurnRomInfo\s+(\w+)RomDesc\s*\[\s*\]\s*=\s*\{(.*?)\};',
|
||||
r"static\s+struct\s+BurnRomInfo\s+(\w+)RomDesc\s*\[\s*\]\s*=\s*\{(.*?)\};",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
@@ -37,7 +36,7 @@ def find_bios_sets(source: str, filename: str) -> dict[str, dict]:
|
||||
|
||||
for match in _BURN_DRIVER_RE.finditer(source):
|
||||
body = match.group(2)
|
||||
if 'BDF_BOARDROM' not in body:
|
||||
if "BDF_BOARDROM" not in body:
|
||||
continue
|
||||
|
||||
# Set name is the first quoted string in the struct body
|
||||
@@ -46,11 +45,11 @@ def find_bios_sets(source: str, filename: str) -> dict[str, dict]:
|
||||
continue
|
||||
|
||||
set_name = name_match.group(1)
|
||||
line_num = source[:match.start()].count('\n') + 1
|
||||
line_num = source[: match.start()].count("\n") + 1
|
||||
|
||||
results[set_name] = {
|
||||
'source_file': filename,
|
||||
'source_line': line_num,
|
||||
"source_file": filename,
|
||||
"source_line": line_num,
|
||||
}
|
||||
|
||||
return results
|
||||
@@ -63,9 +62,9 @@ def parse_rom_info(source: str, set_name: str) -> list[dict]:
|
||||
Sentinel entries (empty name) are skipped.
|
||||
"""
|
||||
pattern = re.compile(
|
||||
r'static\s+struct\s+BurnRomInfo\s+'
|
||||
r"static\s+struct\s+BurnRomInfo\s+"
|
||||
+ re.escape(set_name)
|
||||
+ r'RomDesc\s*\[\s*\]\s*=\s*\{(.*?)\};',
|
||||
+ r"RomDesc\s*\[\s*\]\s*=\s*\{(.*?)\};",
|
||||
re.DOTALL,
|
||||
)
|
||||
match = pattern.search(source)
|
||||
@@ -80,13 +79,15 @@ def parse_rom_info(source: str, set_name: str) -> list[dict]:
|
||||
if not name:
|
||||
continue
|
||||
size = int(entry.group(2), 16)
|
||||
crc32 = format(int(entry.group(3), 16), '08x')
|
||||
crc32 = format(int(entry.group(3), 16), "08x")
|
||||
|
||||
roms.append({
|
||||
'name': name,
|
||||
'size': size,
|
||||
'crc32': crc32,
|
||||
})
|
||||
roms.append(
|
||||
{
|
||||
"name": name,
|
||||
"size": size,
|
||||
"crc32": crc32,
|
||||
}
|
||||
)
|
||||
|
||||
return roms
|
||||
|
||||
@@ -100,7 +101,7 @@ def parse_fbneo_source_tree(base_path: str) -> dict[str, dict]:
|
||||
Returns a dict mapping set name to:
|
||||
{source_file, source_line, roms: [{name, size, crc32}, ...]}
|
||||
"""
|
||||
drv_path = Path(base_path) / 'src' / 'burn' / 'drv'
|
||||
drv_path = Path(base_path) / "src" / "burn" / "drv"
|
||||
if not drv_path.is_dir():
|
||||
return {}
|
||||
|
||||
@@ -108,20 +109,20 @@ def parse_fbneo_source_tree(base_path: str) -> dict[str, dict]:
|
||||
|
||||
for root, _dirs, files in os.walk(drv_path):
|
||||
for fname in files:
|
||||
if not fname.endswith('.cpp'):
|
||||
if not fname.endswith(".cpp"):
|
||||
continue
|
||||
|
||||
filepath = Path(root) / fname
|
||||
source = filepath.read_text(encoding='utf-8', errors='replace')
|
||||
source = filepath.read_text(encoding="utf-8", errors="replace")
|
||||
rel_path = str(filepath.relative_to(base_path))
|
||||
|
||||
bios_sets = find_bios_sets(source, rel_path)
|
||||
for set_name, meta in bios_sets.items():
|
||||
roms = parse_rom_info(source, set_name)
|
||||
results[set_name] = {
|
||||
'source_file': meta['source_file'],
|
||||
'source_line': meta['source_line'],
|
||||
'roms': roms,
|
||||
"source_file": meta["source_file"],
|
||||
"source_line": meta["source_line"],
|
||||
"roms": roms,
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
@@ -8,9 +8,8 @@ Hash: SHA1 primary
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
|
||||
from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format
|
||||
@@ -18,18 +17,17 @@ from .dat_parser import parse_dat, parse_dat_metadata, validate_dat_format
|
||||
PLATFORM_NAME = "libretro"
|
||||
|
||||
SOURCE_URL = (
|
||||
"https://raw.githubusercontent.com/libretro/libretro-database/"
|
||||
"master/dat/System.dat"
|
||||
"https://raw.githubusercontent.com/libretro/libretro-database/master/dat/System.dat"
|
||||
)
|
||||
|
||||
# Libretro cores that expect BIOS files in a subdirectory of system/.
|
||||
# System.dat lists filenames flat; the scraper prepends the prefix.
|
||||
# ref: each core's libretro.c or equivalent -see platforms/README.md
|
||||
CORE_SUBDIR_MAP = {
|
||||
"nec-pc-98": "np2kai", # libretro-np2kai/sdl/libretro.c
|
||||
"sharp-x68000": "keropi", # px68k/libretro/libretro.c
|
||||
"sega-dreamcast": "dc", # flycast/shell/libretro/libretro.cpp
|
||||
"sega-dreamcast-arcade": "dc", # flycast -same subfolder
|
||||
"nec-pc-98": "np2kai", # libretro-np2kai/sdl/libretro.c
|
||||
"sharp-x68000": "keropi", # px68k/libretro/libretro.c
|
||||
"sega-dreamcast": "dc", # flycast/shell/libretro/libretro.cpp
|
||||
"sega-dreamcast-arcade": "dc", # flycast -same subfolder
|
||||
}
|
||||
|
||||
SYSTEM_SLUG_MAP = {
|
||||
@@ -100,7 +98,6 @@ class Scraper(BaseScraper):
|
||||
def __init__(self, url: str = SOURCE_URL):
|
||||
super().__init__(url=url)
|
||||
|
||||
|
||||
def fetch_requirements(self) -> list[BiosRequirement]:
|
||||
"""Parse System.dat and return BIOS requirements."""
|
||||
raw = self._fetch_raw()
|
||||
@@ -113,7 +110,9 @@ class Scraper(BaseScraper):
|
||||
|
||||
for rom in roms:
|
||||
native_system = rom.system
|
||||
system_slug = SYSTEM_SLUG_MAP.get(native_system, native_system.lower().replace(" ", "-"))
|
||||
system_slug = SYSTEM_SLUG_MAP.get(
|
||||
native_system, native_system.lower().replace(" ", "-")
|
||||
)
|
||||
|
||||
destination = rom.name
|
||||
name = rom.name.split("/")[-1] if "/" in rom.name else rom.name
|
||||
@@ -122,17 +121,19 @@ class Scraper(BaseScraper):
|
||||
if subdir and not destination.startswith(subdir + "/"):
|
||||
destination = f"{subdir}/{destination}"
|
||||
|
||||
requirements.append(BiosRequirement(
|
||||
name=name,
|
||||
system=system_slug,
|
||||
sha1=rom.sha1 or None,
|
||||
md5=rom.md5 or None,
|
||||
crc32=rom.crc32 or None,
|
||||
size=rom.size or None,
|
||||
destination=destination,
|
||||
required=True,
|
||||
native_id=native_system,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=name,
|
||||
system=system_slug,
|
||||
sha1=rom.sha1 or None,
|
||||
md5=rom.md5 or None,
|
||||
crc32=rom.crc32 or None,
|
||||
size=rom.size or None,
|
||||
destination=destination,
|
||||
required=True,
|
||||
native_id=native_system,
|
||||
)
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
@@ -158,17 +159,22 @@ class Scraper(BaseScraper):
|
||||
"""Fetch per-core metadata from libretro-core-info .info files."""
|
||||
metadata = {}
|
||||
try:
|
||||
url = f"https://api.github.com/repos/libretro/libretro-core-info/git/trees/master?recursive=1"
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
})
|
||||
url = "https://api.github.com/repos/libretro/libretro-core-info/git/trees/master?recursive=1"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
import json
|
||||
|
||||
tree = json.loads(resp.read())
|
||||
|
||||
info_files = [
|
||||
item["path"] for item in tree.get("tree", [])
|
||||
item["path"]
|
||||
for item in tree.get("tree", [])
|
||||
if item["path"].endswith("_libretro.info")
|
||||
]
|
||||
|
||||
@@ -176,7 +182,9 @@ class Scraper(BaseScraper):
|
||||
core_name = filename.replace("_libretro.info", "")
|
||||
try:
|
||||
info_url = f"https://raw.githubusercontent.com/libretro/libretro-core-info/master/{filename}"
|
||||
req = urllib.request.Request(info_url, headers={"User-Agent": "retrobios-scraper/1.0"})
|
||||
req = urllib.request.Request(
|
||||
info_url, headers={"User-Agent": "retrobios-scraper/1.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
content = resp.read().decode("utf-8")
|
||||
|
||||
@@ -194,10 +202,11 @@ class Scraper(BaseScraper):
|
||||
system_name = info.get("systemname", "")
|
||||
manufacturer = info.get("manufacturer", "")
|
||||
display_name = info.get("display_name", "")
|
||||
categories = info.get("categories", "")
|
||||
info.get("categories", "")
|
||||
|
||||
# Map core to our system slug via firmware paths
|
||||
from .coreinfo_scraper import CORE_SYSTEM_MAP
|
||||
|
||||
system_slug = CORE_SYSTEM_MAP.get(core_name)
|
||||
if not system_slug:
|
||||
continue
|
||||
@@ -267,7 +276,11 @@ class Scraper(BaseScraper):
|
||||
# ref: Vircon32/libretro.c -virtual console, single BIOS
|
||||
"vircon32": {
|
||||
"files": [
|
||||
{"name": "Vircon32Bios.v32", "destination": "Vircon32Bios.v32", "required": True},
|
||||
{
|
||||
"name": "Vircon32Bios.v32",
|
||||
"destination": "Vircon32Bios.v32",
|
||||
"required": True,
|
||||
},
|
||||
],
|
||||
"core": "vircon32",
|
||||
"manufacturer": "Vircon",
|
||||
@@ -276,7 +289,11 @@ class Scraper(BaseScraper):
|
||||
# ref: xrick/src/sysvid.c, xrick/src/data.c -game data archive
|
||||
"xrick": {
|
||||
"files": [
|
||||
{"name": "data.zip", "destination": "xrick/data.zip", "required": True},
|
||||
{
|
||||
"name": "data.zip",
|
||||
"destination": "xrick/data.zip",
|
||||
"required": True,
|
||||
},
|
||||
],
|
||||
"core": "xrick",
|
||||
"manufacturer": "Other",
|
||||
@@ -318,27 +335,51 @@ class Scraper(BaseScraper):
|
||||
|
||||
# segasp.zip for Sega System SP (Flycast)
|
||||
if "sega-dreamcast-arcade" in systems:
|
||||
existing = {f["name"] for f in systems["sega-dreamcast-arcade"].get("files", [])}
|
||||
existing = {
|
||||
f["name"] for f in systems["sega-dreamcast-arcade"].get("files", [])
|
||||
}
|
||||
if "segasp.zip" not in existing:
|
||||
systems["sega-dreamcast-arcade"]["files"].append({
|
||||
"name": "segasp.zip",
|
||||
"destination": "dc/segasp.zip",
|
||||
"required": True,
|
||||
})
|
||||
systems["sega-dreamcast-arcade"]["files"].append(
|
||||
{
|
||||
"name": "segasp.zip",
|
||||
"destination": "dc/segasp.zip",
|
||||
"required": True,
|
||||
}
|
||||
)
|
||||
|
||||
# Extra files missing from System.dat for specific systems.
|
||||
# Each traced to the core's source code.
|
||||
EXTRA_SYSTEM_FILES = {
|
||||
# melonDS DS DSi mode -ref: JesseTG/melonds-ds/src/libretro.cpp
|
||||
"nintendo-ds": [
|
||||
{"name": "dsi_bios7.bin", "destination": "dsi_bios7.bin", "required": True},
|
||||
{"name": "dsi_bios9.bin", "destination": "dsi_bios9.bin", "required": True},
|
||||
{"name": "dsi_firmware.bin", "destination": "dsi_firmware.bin", "required": True},
|
||||
{"name": "dsi_nand.bin", "destination": "dsi_nand.bin", "required": True},
|
||||
{
|
||||
"name": "dsi_bios7.bin",
|
||||
"destination": "dsi_bios7.bin",
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"name": "dsi_bios9.bin",
|
||||
"destination": "dsi_bios9.bin",
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"name": "dsi_firmware.bin",
|
||||
"destination": "dsi_firmware.bin",
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"name": "dsi_nand.bin",
|
||||
"destination": "dsi_nand.bin",
|
||||
"required": True,
|
||||
},
|
||||
],
|
||||
# bsnes SGB naming -ref: bsnes/target-libretro/libretro.cpp
|
||||
"nintendo-sgb": [
|
||||
{"name": "sgb.boot.rom", "destination": "sgb.boot.rom", "required": False},
|
||||
{
|
||||
"name": "sgb.boot.rom",
|
||||
"destination": "sgb.boot.rom",
|
||||
"required": False,
|
||||
},
|
||||
],
|
||||
# JollyCV -ref: jollycv/libretro.c
|
||||
"coleco-colecovision": [
|
||||
@@ -348,12 +389,20 @@ class Scraper(BaseScraper):
|
||||
],
|
||||
# Kronos ST-V -ref: libretro-kronos/libretro/libretro.c
|
||||
"sega-saturn": [
|
||||
{"name": "stvbios.zip", "destination": "kronos/stvbios.zip", "required": True},
|
||||
{
|
||||
"name": "stvbios.zip",
|
||||
"destination": "kronos/stvbios.zip",
|
||||
"required": True,
|
||||
},
|
||||
],
|
||||
# PCSX ReARMed / Beetle PSX alt BIOS -ref: pcsx_rearmed/libpcsxcore/misc.c
|
||||
# docs say PSXONPSP660.bin (uppercase) but core accepts any case
|
||||
"sony-playstation": [
|
||||
{"name": "psxonpsp660.bin", "destination": "psxonpsp660.bin", "required": False},
|
||||
{
|
||||
"name": "psxonpsp660.bin",
|
||||
"destination": "psxonpsp660.bin",
|
||||
"required": False,
|
||||
},
|
||||
],
|
||||
# Dolphin GC -ref: DolphinLibretro/Boot.cpp:72-73,
|
||||
# BootManager.cpp:200-217, CommonPaths.h:139 GC_IPL="IPL.bin"
|
||||
@@ -361,15 +410,43 @@ class Scraper(BaseScraper):
|
||||
# System.dat gc-ntsc-*.bin names are NOT what Dolphin loads.
|
||||
# We add the correct Dolphin paths for BIOS + essential firmware.
|
||||
"nintendo-gamecube": [
|
||||
{"name": "gc-ntsc-12.bin", "destination": "dolphin-emu/Sys/GC/USA/IPL.bin", "required": False},
|
||||
{"name": "gc-pal-12.bin", "destination": "dolphin-emu/Sys/GC/EUR/IPL.bin", "required": False},
|
||||
{"name": "gc-ntsc-12.bin", "destination": "dolphin-emu/Sys/GC/JAP/IPL.bin", "required": False},
|
||||
{
|
||||
"name": "gc-ntsc-12.bin",
|
||||
"destination": "dolphin-emu/Sys/GC/USA/IPL.bin",
|
||||
"required": False,
|
||||
},
|
||||
{
|
||||
"name": "gc-pal-12.bin",
|
||||
"destination": "dolphin-emu/Sys/GC/EUR/IPL.bin",
|
||||
"required": False,
|
||||
},
|
||||
{
|
||||
"name": "gc-ntsc-12.bin",
|
||||
"destination": "dolphin-emu/Sys/GC/JAP/IPL.bin",
|
||||
"required": False,
|
||||
},
|
||||
# DSP firmware -ref: Source/Core/Core/HW/DSPLLE/DSPHost.cpp
|
||||
{"name": "dsp_coef.bin", "destination": "dolphin-emu/Sys/GC/dsp_coef.bin", "required": True},
|
||||
{"name": "dsp_rom.bin", "destination": "dolphin-emu/Sys/GC/dsp_rom.bin", "required": True},
|
||||
{
|
||||
"name": "dsp_coef.bin",
|
||||
"destination": "dolphin-emu/Sys/GC/dsp_coef.bin",
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"name": "dsp_rom.bin",
|
||||
"destination": "dolphin-emu/Sys/GC/dsp_rom.bin",
|
||||
"required": True,
|
||||
},
|
||||
# Fonts -ref: Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp
|
||||
{"name": "font_western.bin", "destination": "dolphin-emu/Sys/GC/font_western.bin", "required": False},
|
||||
{"name": "font_japanese.bin", "destination": "dolphin-emu/Sys/GC/font_japanese.bin", "required": False},
|
||||
{
|
||||
"name": "font_western.bin",
|
||||
"destination": "dolphin-emu/Sys/GC/font_western.bin",
|
||||
"required": False,
|
||||
},
|
||||
{
|
||||
"name": "font_japanese.bin",
|
||||
"destination": "dolphin-emu/Sys/GC/font_japanese.bin",
|
||||
"required": False,
|
||||
},
|
||||
],
|
||||
# minivmac casing -ref: minivmac/src/MYOSGLUE.c
|
||||
# doc says MacII.rom, repo has MacII.ROM -both work on case-insensitive FS
|
||||
@@ -455,6 +532,7 @@ class Scraper(BaseScraper):
|
||||
|
||||
def main():
|
||||
from scripts.scraper.base_scraper import scraper_cli
|
||||
|
||||
scraper_cli(Scraper, "Scrape libretro BIOS requirements")
|
||||
|
||||
|
||||
|
||||
@@ -21,16 +21,16 @@ from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from .mame_parser import parse_mame_source_tree
|
||||
from ._hash_merge import compute_diff, merge_mame_profile
|
||||
from .mame_parser import parse_mame_source_tree
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
_CACHE_PATH = _ROOT / 'data' / 'mame-hashes.json'
|
||||
_CLONE_DIR = _ROOT / 'tmp' / 'mame'
|
||||
_EMULATORS_DIR = _ROOT / 'emulators'
|
||||
_REPO_URL = 'https://github.com/mamedev/mame.git'
|
||||
_CACHE_PATH = _ROOT / "data" / "mame-hashes.json"
|
||||
_CLONE_DIR = _ROOT / "tmp" / "mame"
|
||||
_EMULATORS_DIR = _ROOT / "emulators"
|
||||
_REPO_URL = "https://github.com/mamedev/mame.git"
|
||||
_STALE_HOURS = 24
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ def _load_cache() -> dict[str, Any] | None:
|
||||
if not _CACHE_PATH.exists():
|
||||
return None
|
||||
try:
|
||||
with open(_CACHE_PATH, encoding='utf-8') as f:
|
||||
with open(_CACHE_PATH, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
@@ -50,7 +50,7 @@ def _load_cache() -> dict[str, Any] | None:
|
||||
def _is_stale(cache: dict[str, Any] | None) -> bool:
|
||||
if cache is None:
|
||||
return True
|
||||
fetched_at = cache.get('fetched_at')
|
||||
fetched_at = cache.get("fetched_at")
|
||||
if not fetched_at:
|
||||
return True
|
||||
try:
|
||||
@@ -63,17 +63,19 @@ def _is_stale(cache: dict[str, Any] | None) -> bool:
|
||||
|
||||
def _write_cache(data: dict[str, Any]) -> None:
|
||||
_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(_CACHE_PATH, 'w', encoding='utf-8') as f:
|
||||
with open(_CACHE_PATH, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
log.info('cache written to %s', _CACHE_PATH)
|
||||
log.info("cache written to %s", _CACHE_PATH)
|
||||
|
||||
|
||||
# ── Git operations ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _run_git(args: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
|
||||
def _run_git(
|
||||
args: list[str], cwd: Path | None = None
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
['git', *args],
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
@@ -86,17 +88,20 @@ def _sparse_clone() -> None:
|
||||
shutil.rmtree(_CLONE_DIR)
|
||||
_CLONE_DIR.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
log.info('sparse cloning mamedev/mame into %s', _CLONE_DIR)
|
||||
_run_git([
|
||||
'clone',
|
||||
'--depth', '1',
|
||||
'--filter=blob:none',
|
||||
'--sparse',
|
||||
_REPO_URL,
|
||||
str(_CLONE_DIR),
|
||||
])
|
||||
log.info("sparse cloning mamedev/mame into %s", _CLONE_DIR)
|
||||
_run_git(
|
||||
['sparse-checkout', 'set', 'src/mame', 'src/devices'],
|
||||
[
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"--filter=blob:none",
|
||||
"--sparse",
|
||||
_REPO_URL,
|
||||
str(_CLONE_DIR),
|
||||
]
|
||||
)
|
||||
_run_git(
|
||||
["sparse-checkout", "set", "src/mame", "src/devices"],
|
||||
cwd=_CLONE_DIR,
|
||||
)
|
||||
|
||||
@@ -106,41 +111,41 @@ def _get_version() -> str:
|
||||
# Use GitHub API to get the latest release tag.
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
'https://api.github.com/repos/mamedev/mame/releases/latest',
|
||||
headers={'User-Agent': 'retrobios-scraper/1.0',
|
||||
'Accept': 'application/vnd.github.v3+json'},
|
||||
"https://api.github.com/repos/mamedev/mame/releases/latest",
|
||||
headers={
|
||||
"User-Agent": "retrobios-scraper/1.0",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json.loads(resp.read())
|
||||
tag = data.get('tag_name', '')
|
||||
tag = data.get("tag_name", "")
|
||||
if tag:
|
||||
return _parse_version_tag(tag)
|
||||
except (urllib.error.URLError, json.JSONDecodeError, OSError):
|
||||
pass
|
||||
return 'unknown'
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _parse_version_tag(tag: str) -> str:
|
||||
prefix = 'mame'
|
||||
prefix = "mame"
|
||||
raw = tag.removeprefix(prefix) if tag.startswith(prefix) else tag
|
||||
if raw.isdigit() and len(raw) >= 4:
|
||||
return f'{raw[0]}.{raw[1:]}'
|
||||
return f"{raw[0]}.{raw[1:]}"
|
||||
return raw
|
||||
|
||||
|
||||
|
||||
|
||||
def _get_commit() -> str:
|
||||
try:
|
||||
result = _run_git(['rev-parse', 'HEAD'], cwd=_CLONE_DIR)
|
||||
result = _run_git(["rev-parse", "HEAD"], cwd=_CLONE_DIR)
|
||||
return result.stdout.strip()
|
||||
except subprocess.CalledProcessError:
|
||||
return ''
|
||||
return ""
|
||||
|
||||
|
||||
def _cleanup() -> None:
|
||||
if _CLONE_DIR.exists():
|
||||
log.info('cleaning up %s', _CLONE_DIR)
|
||||
log.info("cleaning up %s", _CLONE_DIR)
|
||||
shutil.rmtree(_CLONE_DIR)
|
||||
|
||||
|
||||
@@ -149,18 +154,21 @@ def _cleanup() -> None:
|
||||
|
||||
def _find_mame_profiles() -> list[Path]:
|
||||
profiles: list[Path] = []
|
||||
for path in sorted(_EMULATORS_DIR.glob('*.yml')):
|
||||
if path.name.endswith('.old.yml'):
|
||||
for path in sorted(_EMULATORS_DIR.glob("*.yml")):
|
||||
if path.name.endswith(".old.yml"):
|
||||
continue
|
||||
try:
|
||||
with open(path, encoding='utf-8') as f:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
upstream = data.get('upstream', '')
|
||||
upstream = data.get("upstream", "")
|
||||
# Only match profiles tracking current MAME (not frozen snapshots
|
||||
# which have upstream like "mamedev/mame/tree/mame0139")
|
||||
if isinstance(upstream, str) and upstream.rstrip('/') == 'https://github.com/mamedev/mame':
|
||||
if (
|
||||
isinstance(upstream, str)
|
||||
and upstream.rstrip("/") == "https://github.com/mamedev/mame"
|
||||
):
|
||||
profiles.append(path)
|
||||
except (yaml.YAMLError, OSError):
|
||||
continue
|
||||
@@ -179,36 +187,36 @@ def _format_diff(
|
||||
lines: list[str] = []
|
||||
name = profile_path.stem
|
||||
|
||||
added = diff.get('added', [])
|
||||
updated = diff.get('updated', [])
|
||||
removed = diff.get('removed', [])
|
||||
unchanged = diff.get('unchanged', 0)
|
||||
added = diff.get("added", [])
|
||||
updated = diff.get("updated", [])
|
||||
removed = diff.get("removed", [])
|
||||
unchanged = diff.get("unchanged", 0)
|
||||
|
||||
if not added and not updated and not removed:
|
||||
lines.append(f' {name}:')
|
||||
lines.append(' no changes')
|
||||
lines.append(f" {name}:")
|
||||
lines.append(" no changes")
|
||||
return lines
|
||||
|
||||
lines.append(f' {name}:')
|
||||
lines.append(f" {name}:")
|
||||
|
||||
if show_added:
|
||||
bios_sets = hashes.get('bios_sets', {})
|
||||
bios_sets = hashes.get("bios_sets", {})
|
||||
for set_name in added:
|
||||
rom_count = len(bios_sets.get(set_name, {}).get('roms', []))
|
||||
source_file = bios_sets.get(set_name, {}).get('source_file', '')
|
||||
source_line = bios_sets.get(set_name, {}).get('source_line', '')
|
||||
ref = f'{source_file}:{source_line}' if source_file else ''
|
||||
lines.append(f' + {set_name}.zip ({ref}, {rom_count} ROMs)')
|
||||
rom_count = len(bios_sets.get(set_name, {}).get("roms", []))
|
||||
source_file = bios_sets.get(set_name, {}).get("source_file", "")
|
||||
source_line = bios_sets.get(set_name, {}).get("source_line", "")
|
||||
ref = f"{source_file}:{source_line}" if source_file else ""
|
||||
lines.append(f" + {set_name}.zip ({ref}, {rom_count} ROMs)")
|
||||
elif added:
|
||||
lines.append(f' + {len(added)} new sets available (main profile only)')
|
||||
lines.append(f" + {len(added)} new sets available (main profile only)")
|
||||
|
||||
for set_name in updated:
|
||||
lines.append(f' ~ {set_name}.zip (contents changed)')
|
||||
lines.append(f" ~ {set_name}.zip (contents changed)")
|
||||
|
||||
oos = diff.get('out_of_scope', 0)
|
||||
lines.append(f' = {unchanged} unchanged')
|
||||
oos = diff.get("out_of_scope", 0)
|
||||
lines.append(f" = {unchanged} unchanged")
|
||||
if oos:
|
||||
lines.append(f' . {oos} out of scope (not BIOS root sets)')
|
||||
lines.append(f" . {oos} out of scope (not BIOS root sets)")
|
||||
return lines
|
||||
|
||||
|
||||
@@ -218,7 +226,7 @@ def _format_diff(
|
||||
def _fetch_hashes(force: bool) -> dict[str, Any]:
|
||||
cache = _load_cache()
|
||||
if not force and not _is_stale(cache):
|
||||
log.info('using cached data from %s', cache.get('fetched_at', ''))
|
||||
log.info("using cached data from %s", cache.get("fetched_at", ""))
|
||||
return cache # type: ignore[return-value]
|
||||
|
||||
try:
|
||||
@@ -228,11 +236,11 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
|
||||
commit = _get_commit()
|
||||
|
||||
data: dict[str, Any] = {
|
||||
'source': 'mamedev/mame',
|
||||
'version': version,
|
||||
'commit': commit,
|
||||
'fetched_at': datetime.now(timezone.utc).isoformat(),
|
||||
'bios_sets': bios_sets,
|
||||
"source": "mamedev/mame",
|
||||
"version": version,
|
||||
"commit": commit,
|
||||
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
||||
"bios_sets": bios_sets,
|
||||
}
|
||||
_write_cache(data)
|
||||
return data
|
||||
@@ -243,34 +251,36 @@ def _fetch_hashes(force: bool) -> dict[str, Any]:
|
||||
def _run(args: argparse.Namespace) -> None:
|
||||
hashes = _fetch_hashes(args.force)
|
||||
|
||||
total_sets = len(hashes.get('bios_sets', {}))
|
||||
version = hashes.get('version', 'unknown')
|
||||
commit = hashes.get('commit', '')[:12]
|
||||
total_sets = len(hashes.get("bios_sets", {}))
|
||||
version = hashes.get("version", "unknown")
|
||||
commit = hashes.get("commit", "")[:12]
|
||||
|
||||
if args.json:
|
||||
json.dump(hashes, sys.stdout, indent=2, ensure_ascii=False)
|
||||
sys.stdout.write('\n')
|
||||
sys.stdout.write("\n")
|
||||
return
|
||||
|
||||
print(f'mame-hashes: {total_sets} BIOS root sets from mamedev/mame'
|
||||
f' @ {version} ({commit})')
|
||||
print(
|
||||
f"mame-hashes: {total_sets} BIOS root sets from mamedev/mame"
|
||||
f" @ {version} ({commit})"
|
||||
)
|
||||
print()
|
||||
|
||||
profiles = _find_mame_profiles()
|
||||
if not profiles:
|
||||
print(' no profiles with mamedev/mame upstream found')
|
||||
print(" no profiles with mamedev/mame upstream found")
|
||||
return
|
||||
|
||||
for profile_path in profiles:
|
||||
is_main = profile_path.name == 'mame.yml'
|
||||
diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode='mame')
|
||||
is_main = profile_path.name == "mame.yml"
|
||||
diff = compute_diff(str(profile_path), str(_CACHE_PATH), mode="mame")
|
||||
lines = _format_diff(profile_path, diff, hashes, show_added=is_main)
|
||||
for line in lines:
|
||||
print(line)
|
||||
|
||||
if not args.dry_run:
|
||||
updated = diff.get('updated', [])
|
||||
added = diff.get('added', []) if is_main else []
|
||||
updated = diff.get("updated", [])
|
||||
added = diff.get("added", []) if is_main else []
|
||||
if added or updated:
|
||||
merge_mame_profile(
|
||||
str(profile_path),
|
||||
@@ -278,32 +288,32 @@ def _run(args: argparse.Namespace) -> None:
|
||||
write=True,
|
||||
add_new=is_main,
|
||||
)
|
||||
log.info('merged into %s', profile_path.name)
|
||||
log.info("merged into %s", profile_path.name)
|
||||
|
||||
print()
|
||||
if args.dry_run:
|
||||
print('(dry run, no files modified)')
|
||||
print("(dry run, no files modified)")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='mame_hash_scraper',
|
||||
description='Fetch MAME BIOS hashes from source and merge into profiles.',
|
||||
prog="mame_hash_scraper",
|
||||
description="Fetch MAME BIOS hashes from source and merge into profiles.",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='show diff only, do not modify profiles',
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="show diff only, do not modify profiles",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--json',
|
||||
action='store_true',
|
||||
help='output raw JSON to stdout',
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="output raw JSON to stdout",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--force',
|
||||
action='store_true',
|
||||
help='re-fetch even if cache is fresh',
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="re-fetch even if cache is fresh",
|
||||
)
|
||||
return parser
|
||||
|
||||
@@ -311,12 +321,12 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
def main() -> None:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(levelname)s: %(message)s',
|
||||
format="%(levelname)s: %(message)s",
|
||||
)
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
_run(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -14,27 +14,27 @@ from pathlib import Path
|
||||
|
||||
# Macros that declare a machine entry
|
||||
_MACHINE_MACROS = re.compile(
|
||||
r'\b(GAME|SYST|COMP|CONS)\s*\(',
|
||||
r"\b(GAME|SYST|COMP|CONS)\s*\(",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# ROM block boundaries
|
||||
_ROM_START = re.compile(r'ROM_START\s*\(\s*(\w+)\s*\)')
|
||||
_ROM_END = re.compile(r'ROM_END')
|
||||
_ROM_START = re.compile(r"ROM_START\s*\(\s*(\w+)\s*\)")
|
||||
_ROM_END = re.compile(r"ROM_END")
|
||||
|
||||
# ROM_REGION variants: ROM_REGION, ROM_REGION16_BE, ROM_REGION16_LE, ROM_REGION32_LE, etc.
|
||||
_ROM_REGION = re.compile(
|
||||
r'ROM_REGION\w*\s*\('
|
||||
r'\s*(0x[\da-fA-F]+|\d+)\s*,' # size
|
||||
r'\s*"([^"]+)"\s*,', # tag
|
||||
r"ROM_REGION\w*\s*\("
|
||||
r"\s*(0x[\da-fA-F]+|\d+)\s*," # size
|
||||
r'\s*"([^"]+)"\s*,', # tag
|
||||
)
|
||||
|
||||
# ROM_SYSTEM_BIOS( index, label, description )
|
||||
_ROM_SYSTEM_BIOS = re.compile(
|
||||
r'ROM_SYSTEM_BIOS\s*\('
|
||||
r'\s*(\d+)\s*,' # index
|
||||
r'\s*"([^"]+)"\s*,' # label
|
||||
r'\s*"([^"]+)"\s*\)', # description
|
||||
r"ROM_SYSTEM_BIOS\s*\("
|
||||
r"\s*(\d+)\s*," # index
|
||||
r'\s*"([^"]+)"\s*,' # label
|
||||
r'\s*"([^"]+)"\s*\)', # description
|
||||
)
|
||||
|
||||
# All ROM_LOAD variants including custom BIOS macros.
|
||||
@@ -44,23 +44,23 @@ _ROM_SYSTEM_BIOS = re.compile(
|
||||
# The key pattern: any macro containing "ROM_LOAD" or "ROMX_LOAD" in its name,
|
||||
# with the first quoted string being the ROM filename.
|
||||
_ROM_LOAD = re.compile(
|
||||
r'\b\w*ROMX?_LOAD\w*\s*\('
|
||||
r'[^"]*' # skip any args before the filename (e.g., bios index)
|
||||
r'"([^"]+)"\s*,' # name (first quoted string)
|
||||
r'\s*(0x[\da-fA-F]+|\d+)\s*,' # offset
|
||||
r'\s*(0x[\da-fA-F]+|\d+)\s*,', # size
|
||||
r"\b\w*ROMX?_LOAD\w*\s*\("
|
||||
r'[^"]*' # skip any args before the filename (e.g., bios index)
|
||||
r'"([^"]+)"\s*,' # name (first quoted string)
|
||||
r"\s*(0x[\da-fA-F]+|\d+)\s*," # offset
|
||||
r"\s*(0x[\da-fA-F]+|\d+)\s*,", # size
|
||||
)
|
||||
|
||||
# CRC32 and SHA1 within a ROM_LOAD line
|
||||
_CRC_SHA = re.compile(
|
||||
r'CRC\s*\(\s*([0-9a-fA-F]+)\s*\)'
|
||||
r'\s+'
|
||||
r'SHA1\s*\(\s*([0-9a-fA-F]+)\s*\)',
|
||||
r"CRC\s*\(\s*([0-9a-fA-F]+)\s*\)"
|
||||
r"\s+"
|
||||
r"SHA1\s*\(\s*([0-9a-fA-F]+)\s*\)",
|
||||
)
|
||||
|
||||
_NO_DUMP = re.compile(r'\bNO_DUMP\b')
|
||||
_BAD_DUMP = re.compile(r'\bBAD_DUMP\b')
|
||||
_ROM_BIOS = re.compile(r'ROM_BIOS\s*\(\s*(\d+)\s*\)')
|
||||
_NO_DUMP = re.compile(r"\bNO_DUMP\b")
|
||||
_BAD_DUMP = re.compile(r"\bBAD_DUMP\b")
|
||||
_ROM_BIOS = re.compile(r"ROM_BIOS\s*\(\s*(\d+)\s*\)")
|
||||
|
||||
|
||||
def find_bios_root_sets(source: str, filename: str) -> dict[str, dict]:
|
||||
@@ -77,8 +77,8 @@ def find_bios_root_sets(source: str, filename: str) -> dict[str, dict]:
|
||||
if block_end == -1:
|
||||
continue
|
||||
|
||||
block = source[start:block_end + 1]
|
||||
if 'MACHINE_IS_BIOS_ROOT' not in block:
|
||||
block = source[start : block_end + 1]
|
||||
if "MACHINE_IS_BIOS_ROOT" not in block:
|
||||
continue
|
||||
|
||||
# Extract set name: first arg after the opening paren
|
||||
@@ -97,11 +97,11 @@ def find_bios_root_sets(source: str, filename: str) -> dict[str, dict]:
|
||||
continue
|
||||
|
||||
set_name = args[1].strip()
|
||||
line_no = source[:match.start()].count('\n') + 1
|
||||
line_no = source[: match.start()].count("\n") + 1
|
||||
|
||||
results[set_name] = {
|
||||
'source_file': filename,
|
||||
'source_line': line_no,
|
||||
"source_file": filename,
|
||||
"source_line": line_no,
|
||||
}
|
||||
|
||||
return results
|
||||
@@ -115,7 +115,7 @@ def parse_rom_block(source: str, set_name: str) -> list[dict]:
|
||||
extracts all ROM entries. Skips NO_DUMP, flags BAD_DUMP.
|
||||
"""
|
||||
pattern = re.compile(
|
||||
r'ROM_START\s*\(\s*' + re.escape(set_name) + r'\s*\)',
|
||||
r"ROM_START\s*\(\s*" + re.escape(set_name) + r"\s*\)",
|
||||
)
|
||||
start_match = pattern.search(source)
|
||||
if not start_match:
|
||||
@@ -125,7 +125,7 @@ def parse_rom_block(source: str, set_name: str) -> list[dict]:
|
||||
if not end_match:
|
||||
return []
|
||||
|
||||
block = source[start_match.end():end_match.start()]
|
||||
block = source[start_match.end() : end_match.start()]
|
||||
|
||||
# Pre-expand macros: find #define macros in the file that contain
|
||||
# ROM_LOAD/ROM_REGION/ROM_SYSTEM_BIOS calls, then expand their
|
||||
@@ -144,26 +144,26 @@ def parse_mame_source_tree(base_path: str) -> dict[str, dict]:
|
||||
results: dict[str, dict] = {}
|
||||
root = Path(base_path)
|
||||
|
||||
search_dirs = [root / 'src' / 'mame', root / 'src' / 'devices']
|
||||
search_dirs = [root / "src" / "mame", root / "src" / "devices"]
|
||||
|
||||
for search_dir in search_dirs:
|
||||
if not search_dir.is_dir():
|
||||
continue
|
||||
for dirpath, _dirnames, filenames in os.walk(search_dir):
|
||||
for fname in filenames:
|
||||
if not fname.endswith(('.cpp', '.c', '.h', '.hxx')):
|
||||
if not fname.endswith((".cpp", ".c", ".h", ".hxx")):
|
||||
continue
|
||||
filepath = Path(dirpath) / fname
|
||||
rel_path = str(filepath.relative_to(root))
|
||||
content = filepath.read_text(encoding='utf-8', errors='replace')
|
||||
content = filepath.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
bios_sets = find_bios_root_sets(content, rel_path)
|
||||
for set_name, info in bios_sets.items():
|
||||
roms = parse_rom_block(content, set_name)
|
||||
results[set_name] = {
|
||||
'source_file': info['source_file'],
|
||||
'source_line': info['source_line'],
|
||||
'roms': roms,
|
||||
"source_file": info["source_file"],
|
||||
"source_line": info["source_line"],
|
||||
"roms": roms,
|
||||
}
|
||||
|
||||
return results
|
||||
@@ -171,13 +171,20 @@ def parse_mame_source_tree(base_path: str) -> dict[str, dict]:
|
||||
|
||||
# Regex for #define macros that span multiple lines (backslash continuation)
|
||||
_DEFINE_RE = re.compile(
|
||||
r'^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s*((?:.*\\\n)*.*)',
|
||||
r"^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s*((?:.*\\\n)*.*)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# ROM-related tokens that indicate a macro is relevant for expansion
|
||||
_ROM_TOKENS = {'ROM_LOAD', 'ROMX_LOAD', 'ROM_REGION', 'ROM_SYSTEM_BIOS',
|
||||
'ROM_FILL', 'ROM_COPY', 'ROM_RELOAD'}
|
||||
_ROM_TOKENS = {
|
||||
"ROM_LOAD",
|
||||
"ROMX_LOAD",
|
||||
"ROM_REGION",
|
||||
"ROM_SYSTEM_BIOS",
|
||||
"ROM_FILL",
|
||||
"ROM_COPY",
|
||||
"ROM_RELOAD",
|
||||
}
|
||||
|
||||
|
||||
def _collect_rom_macros(source: str) -> dict[str, str]:
|
||||
@@ -193,14 +200,14 @@ def _collect_rom_macros(source: str) -> dict[str, str]:
|
||||
name = m.group(1)
|
||||
body = m.group(2)
|
||||
# Join backslash-continued lines
|
||||
body = body.replace('\\\n', ' ')
|
||||
body = body.replace("\\\n", " ")
|
||||
# Only keep macros that contain ROM-related tokens
|
||||
if not any(tok in body for tok in _ROM_TOKENS):
|
||||
continue
|
||||
# Skip wrapper macros: if the body contains ROMX_LOAD/ROM_LOAD
|
||||
# with unquoted args (formal parameters), it's a wrapper.
|
||||
# These are already recognized by the _ROM_LOAD regex directly.
|
||||
if re.search(r'ROMX?_LOAD\s*\(\s*\w+\s*,\s*\w+\s*,', body):
|
||||
if re.search(r"ROMX?_LOAD\s*\(\s*\w+\s*,\s*\w+\s*,", body):
|
||||
continue
|
||||
macros[name] = body
|
||||
return macros
|
||||
@@ -223,7 +230,7 @@ def _expand_macros(block: str, macros: dict[str, str], depth: int = 5) -> str:
|
||||
iterations += 1
|
||||
for name, body in macros.items():
|
||||
# Match macro invocation: NAME or NAME(args)
|
||||
pattern = re.compile(r'\b' + re.escape(name) + r'(?:\s*\([^)]*\))?')
|
||||
pattern = re.compile(r"\b" + re.escape(name) + r"(?:\s*\([^)]*\))?")
|
||||
if pattern.search(block):
|
||||
block = pattern.sub(body, block)
|
||||
changed = True
|
||||
@@ -237,9 +244,9 @@ def _find_closing_paren(source: str, start: int) -> int:
|
||||
i = start
|
||||
while i < len(source):
|
||||
ch = source[i]
|
||||
if ch == '(':
|
||||
if ch == "(":
|
||||
depth += 1
|
||||
elif ch == ')':
|
||||
elif ch == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i
|
||||
@@ -268,24 +275,24 @@ def _split_macro_args(inner: str) -> list[str]:
|
||||
i += 1
|
||||
if i < len(inner):
|
||||
current.append(inner[i])
|
||||
elif ch == '(':
|
||||
elif ch == "(":
|
||||
depth += 1
|
||||
current.append(ch)
|
||||
elif ch == ')':
|
||||
elif ch == ")":
|
||||
if depth == 0:
|
||||
args.append(''.join(current))
|
||||
args.append("".join(current))
|
||||
break
|
||||
depth -= 1
|
||||
current.append(ch)
|
||||
elif ch == ',' and depth == 0:
|
||||
args.append(''.join(current))
|
||||
elif ch == "," and depth == 0:
|
||||
args.append("".join(current))
|
||||
current = []
|
||||
else:
|
||||
current.append(ch)
|
||||
i += 1
|
||||
|
||||
if current:
|
||||
remaining = ''.join(current).strip()
|
||||
remaining = "".join(current).strip()
|
||||
if remaining:
|
||||
args.append(remaining)
|
||||
|
||||
@@ -300,15 +307,15 @@ def _parse_rom_entries(block: str) -> list[dict]:
|
||||
Processes matches in order of appearance to track region and BIOS context.
|
||||
"""
|
||||
roms: list[dict] = []
|
||||
current_region = ''
|
||||
current_region = ""
|
||||
bios_labels: dict[int, tuple[str, str]] = {}
|
||||
|
||||
# Build a combined pattern that matches all interesting tokens
|
||||
# and process them in order of occurrence
|
||||
token_patterns = [
|
||||
('region', _ROM_REGION),
|
||||
('bios_label', _ROM_SYSTEM_BIOS),
|
||||
('rom_load', _ROM_LOAD),
|
||||
("region", _ROM_REGION),
|
||||
("bios_label", _ROM_SYSTEM_BIOS),
|
||||
("rom_load", _ROM_LOAD),
|
||||
]
|
||||
|
||||
# Collect all matches with their positions
|
||||
@@ -321,22 +328,22 @@ def _parse_rom_entries(block: str) -> list[dict]:
|
||||
events.sort(key=lambda e: e[0])
|
||||
|
||||
for _pos, tag, m in events:
|
||||
if tag == 'region':
|
||||
if tag == "region":
|
||||
current_region = m.group(2)
|
||||
elif tag == 'bios_label':
|
||||
elif tag == "bios_label":
|
||||
idx = int(m.group(1))
|
||||
bios_labels[idx] = (m.group(2), m.group(3))
|
||||
elif tag == 'rom_load':
|
||||
elif tag == "rom_load":
|
||||
# Get the full macro call as context (find closing paren)
|
||||
context_start = m.start()
|
||||
# Find the opening paren of the ROM_LOAD macro
|
||||
paren_pos = block.find('(', context_start)
|
||||
paren_pos = block.find("(", context_start)
|
||||
if paren_pos != -1:
|
||||
close_pos = _find_closing_paren(block, paren_pos)
|
||||
context_end = close_pos + 1 if close_pos != -1 else m.end() + 200
|
||||
else:
|
||||
context_end = m.end() + 200
|
||||
context = block[context_start:min(context_end, len(block))]
|
||||
context = block[context_start : min(context_end, len(block))]
|
||||
|
||||
if _NO_DUMP.search(context):
|
||||
continue
|
||||
@@ -345,8 +352,8 @@ def _parse_rom_entries(block: str) -> list[dict]:
|
||||
rom_size = _parse_int(m.group(3))
|
||||
|
||||
crc_sha_match = _CRC_SHA.search(context)
|
||||
crc32 = ''
|
||||
sha1 = ''
|
||||
crc32 = ""
|
||||
sha1 = ""
|
||||
if crc_sha_match:
|
||||
crc32 = crc_sha_match.group(1).lower()
|
||||
sha1 = crc_sha_match.group(2).lower()
|
||||
@@ -354,8 +361,8 @@ def _parse_rom_entries(block: str) -> list[dict]:
|
||||
bad_dump = bool(_BAD_DUMP.search(context))
|
||||
|
||||
bios_index = None
|
||||
bios_label = ''
|
||||
bios_description = ''
|
||||
bios_label = ""
|
||||
bios_description = ""
|
||||
bios_ref = _ROM_BIOS.search(context)
|
||||
if bios_ref:
|
||||
bios_index = int(bios_ref.group(1))
|
||||
@@ -363,18 +370,18 @@ def _parse_rom_entries(block: str) -> list[dict]:
|
||||
bios_label, bios_description = bios_labels[bios_index]
|
||||
|
||||
entry: dict = {
|
||||
'name': rom_name,
|
||||
'size': rom_size,
|
||||
'crc32': crc32,
|
||||
'sha1': sha1,
|
||||
'region': current_region,
|
||||
'bad_dump': bad_dump,
|
||||
"name": rom_name,
|
||||
"size": rom_size,
|
||||
"crc32": crc32,
|
||||
"sha1": sha1,
|
||||
"region": current_region,
|
||||
"bad_dump": bad_dump,
|
||||
}
|
||||
|
||||
if bios_index is not None:
|
||||
entry['bios_index'] = bios_index
|
||||
entry['bios_label'] = bios_label
|
||||
entry['bios_description'] = bios_description
|
||||
entry["bios_index"] = bios_index
|
||||
entry["bios_label"] = bios_label
|
||||
entry["bios_description"] = bios_description
|
||||
|
||||
roms.append(entry)
|
||||
|
||||
@@ -384,6 +391,6 @@ def _parse_rom_entries(block: str) -> list[dict]:
|
||||
def _parse_int(value: str) -> int:
|
||||
"""Parse an integer that may be hex (0x...) or decimal."""
|
||||
value = value.strip()
|
||||
if value.startswith('0x') or value.startswith('0X'):
|
||||
if value.startswith("0x") or value.startswith("0X"):
|
||||
return int(value, 16)
|
||||
return int(value)
|
||||
|
||||
@@ -16,8 +16,6 @@ Recalbox verification logic:
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag
|
||||
@@ -121,17 +119,19 @@ class Scraper(BaseScraper):
|
||||
for bios_elem in system_elem.findall("bios"):
|
||||
paths_str = bios_elem.get("path", "")
|
||||
md5_str = bios_elem.get("md5", "")
|
||||
core = bios_elem.get("core", "")
|
||||
bios_elem.get("core", "")
|
||||
mandatory = bios_elem.get("mandatory", "true") != "false"
|
||||
hash_match_mandatory = bios_elem.get("hashMatchMandatory", "true") != "false"
|
||||
note = bios_elem.get("note", "")
|
||||
bios_elem.get("hashMatchMandatory", "true") != "false"
|
||||
bios_elem.get("note", "")
|
||||
|
||||
paths = [p.strip() for p in paths_str.split("|") if p.strip()]
|
||||
if not paths:
|
||||
continue
|
||||
|
||||
primary_path = paths[0]
|
||||
name = primary_path.split("/")[-1] if "/" in primary_path else primary_path
|
||||
name = (
|
||||
primary_path.split("/")[-1] if "/" in primary_path else primary_path
|
||||
)
|
||||
|
||||
md5_list = [m.strip() for m in md5_str.split(",") if m.strip()]
|
||||
all_md5 = ",".join(md5_list) if md5_list else None
|
||||
@@ -141,14 +141,16 @@ class Scraper(BaseScraper):
|
||||
continue
|
||||
seen.add(dedup_key)
|
||||
|
||||
requirements.append(BiosRequirement(
|
||||
name=name,
|
||||
system=system_slug,
|
||||
md5=all_md5,
|
||||
destination=primary_path,
|
||||
required=mandatory,
|
||||
native_id=platform,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=name,
|
||||
system=system_slug,
|
||||
md5=all_md5,
|
||||
destination=primary_path,
|
||||
required=mandatory,
|
||||
native_id=platform,
|
||||
)
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
@@ -168,7 +170,9 @@ class Scraper(BaseScraper):
|
||||
md5_str = bios_elem.get("md5", "")
|
||||
core = bios_elem.get("core", "")
|
||||
mandatory = bios_elem.get("mandatory", "true") != "false"
|
||||
hash_match_mandatory = bios_elem.get("hashMatchMandatory", "true") != "false"
|
||||
hash_match_mandatory = (
|
||||
bios_elem.get("hashMatchMandatory", "true") != "false"
|
||||
)
|
||||
note = bios_elem.get("note", "")
|
||||
|
||||
paths = [p.strip() for p in paths_str.split("|") if p.strip()]
|
||||
@@ -179,17 +183,19 @@ class Scraper(BaseScraper):
|
||||
|
||||
name = paths[0].split("/")[-1] if "/" in paths[0] else paths[0]
|
||||
|
||||
requirements.append({
|
||||
"name": name,
|
||||
"system": system_slug,
|
||||
"system_name": system_name,
|
||||
"paths": paths,
|
||||
"md5_list": md5_list,
|
||||
"core": core,
|
||||
"mandatory": mandatory,
|
||||
"hash_match_mandatory": hash_match_mandatory,
|
||||
"note": note,
|
||||
})
|
||||
requirements.append(
|
||||
{
|
||||
"name": name,
|
||||
"system": system_slug,
|
||||
"system_name": system_name,
|
||||
"paths": paths,
|
||||
"md5_list": md5_list,
|
||||
"core": core,
|
||||
"mandatory": mandatory,
|
||||
"hash_match_mandatory": hash_match_mandatory,
|
||||
"note": note,
|
||||
}
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
@@ -245,7 +251,9 @@ def main():
|
||||
parser = argparse.ArgumentParser(description="Scrape Recalbox es_bios.xml")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--json", action="store_true")
|
||||
parser.add_argument("--full", action="store_true", help="Show full Recalbox-specific fields")
|
||||
parser.add_argument(
|
||||
"--full", action="store_true", help="Show full Recalbox-specific fields"
|
||||
)
|
||||
parser.add_argument("--output", "-o")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -264,6 +272,7 @@ def main():
|
||||
|
||||
if args.dry_run:
|
||||
from collections import defaultdict
|
||||
|
||||
by_system = defaultdict(list)
|
||||
for r in reqs:
|
||||
by_system[r.system].append(r)
|
||||
@@ -272,7 +281,7 @@ def main():
|
||||
for f in files[:5]:
|
||||
print(f" {f.name} (md5={f.md5[:12] if f.md5 else 'N/A'}...)")
|
||||
if len(files) > 5:
|
||||
print(f" ... +{len(files)-5} more")
|
||||
print(f" ... +{len(files) - 5} more")
|
||||
print(f"\nTotal: {len(reqs)} BIOS files across {len(by_system)} systems")
|
||||
return
|
||||
|
||||
|
||||
@@ -9,9 +9,6 @@ Hash: MD5 primary
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
try:
|
||||
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_version
|
||||
@@ -43,7 +40,6 @@ class Scraper(BaseScraper):
|
||||
super().__init__(url=url)
|
||||
self._parsed: dict | None = None
|
||||
|
||||
|
||||
def _parse_json(self) -> dict:
|
||||
if self._parsed is not None:
|
||||
return self._parsed
|
||||
@@ -89,13 +85,15 @@ class Scraper(BaseScraper):
|
||||
|
||||
name = file_path.split("/")[-1] if "/" in file_path else file_path
|
||||
|
||||
requirements.append(BiosRequirement(
|
||||
name=name,
|
||||
system=SYSTEM_SLUG_MAP.get(sys_key, sys_key),
|
||||
md5=md5 or None,
|
||||
destination=file_path,
|
||||
required=True,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=name,
|
||||
system=SYSTEM_SLUG_MAP.get(sys_key, sys_key),
|
||||
md5=md5 or None,
|
||||
destination=file_path,
|
||||
required=True,
|
||||
)
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
@@ -170,6 +168,7 @@ class Scraper(BaseScraper):
|
||||
|
||||
def main():
|
||||
from scripts.scraper.base_scraper import scraper_cli
|
||||
|
||||
scraper_cli(Scraper, "Scrape retrobat BIOS requirements")
|
||||
|
||||
|
||||
|
||||
@@ -29,8 +29,8 @@ import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
@@ -43,16 +43,16 @@ PLATFORM_NAME = "retrodeck"
|
||||
COMPONENTS_REPO = "RetroDECK/components"
|
||||
COMPONENTS_BRANCH = "main"
|
||||
COMPONENTS_API_URL = (
|
||||
f"https://api.github.com/repos/{COMPONENTS_REPO}"
|
||||
f"/git/trees/{COMPONENTS_BRANCH}"
|
||||
)
|
||||
RAW_BASE = (
|
||||
f"https://raw.githubusercontent.com/{COMPONENTS_REPO}"
|
||||
f"/{COMPONENTS_BRANCH}"
|
||||
f"https://api.github.com/repos/{COMPONENTS_REPO}/git/trees/{COMPONENTS_BRANCH}"
|
||||
)
|
||||
RAW_BASE = f"https://raw.githubusercontent.com/{COMPONENTS_REPO}/{COMPONENTS_BRANCH}"
|
||||
SKIP_DIRS = {"archive_later", "archive_old", "automation-tools", ".github"}
|
||||
NON_EMULATOR_COMPONENTS = {
|
||||
"framework", "es-de", "steam-rom-manager", "flips", "portmaster",
|
||||
"framework",
|
||||
"es-de",
|
||||
"steam-rom-manager",
|
||||
"flips",
|
||||
"portmaster",
|
||||
}
|
||||
|
||||
# RetroDECK system ID -> retrobios slug.
|
||||
@@ -358,13 +358,20 @@ class Scraper(BaseScraper):
|
||||
|
||||
required_raw = entry.get("required", "")
|
||||
required = bool(required_raw) and str(required_raw).lower() not in (
|
||||
"false", "no", "optional", "",
|
||||
"false",
|
||||
"no",
|
||||
"optional",
|
||||
"",
|
||||
)
|
||||
|
||||
key = (system, filename.lower())
|
||||
if key in seen:
|
||||
existing = next(
|
||||
(r for r in requirements if (r.system, r.name.lower()) == key),
|
||||
(
|
||||
r
|
||||
for r in requirements
|
||||
if (r.system, r.name.lower()) == key
|
||||
),
|
||||
None,
|
||||
)
|
||||
if existing and md5 and existing.md5 and md5 != existing.md5:
|
||||
@@ -376,13 +383,15 @@ class Scraper(BaseScraper):
|
||||
continue
|
||||
seen.add(key)
|
||||
|
||||
requirements.append(BiosRequirement(
|
||||
name=filename,
|
||||
system=system,
|
||||
destination=destination,
|
||||
md5=md5,
|
||||
required=required,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=filename,
|
||||
system=system,
|
||||
destination=destination,
|
||||
md5=md5,
|
||||
required=required,
|
||||
)
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
@@ -390,11 +399,14 @@ class Scraper(BaseScraper):
|
||||
reqs = self.fetch_requirements()
|
||||
manifests = self._get_manifests()
|
||||
|
||||
cores = sorted({
|
||||
comp_name for comp_name, _ in manifests
|
||||
if comp_name not in SKIP_DIRS
|
||||
and comp_name not in NON_EMULATOR_COMPONENTS
|
||||
})
|
||||
cores = sorted(
|
||||
{
|
||||
comp_name
|
||||
for comp_name, _ in manifests
|
||||
if comp_name not in SKIP_DIRS
|
||||
and comp_name not in NON_EMULATOR_COMPONENTS
|
||||
}
|
||||
)
|
||||
|
||||
systems: dict[str, dict] = {}
|
||||
for req in reqs:
|
||||
@@ -423,6 +435,7 @@ class Scraper(BaseScraper):
|
||||
|
||||
def main() -> None:
|
||||
from scraper.base_scraper import scraper_cli
|
||||
|
||||
scraper_cli(Scraper, "Scrape RetroDECK BIOS requirements")
|
||||
|
||||
|
||||
|
||||
@@ -138,16 +138,18 @@ class Scraper(BaseScraper):
|
||||
crc32 = (entry.get("crc") or "").strip() or None
|
||||
size = int(entry["size"]) if entry.get("size") else None
|
||||
|
||||
requirements.append(BiosRequirement(
|
||||
name=filename,
|
||||
system=system,
|
||||
sha1=sha1,
|
||||
md5=md5,
|
||||
crc32=crc32,
|
||||
size=size,
|
||||
destination=f"{igdb_slug}/{filename}",
|
||||
required=True,
|
||||
))
|
||||
requirements.append(
|
||||
BiosRequirement(
|
||||
name=filename,
|
||||
system=system,
|
||||
sha1=sha1,
|
||||
md5=md5,
|
||||
crc32=crc32,
|
||||
size=size,
|
||||
destination=f"{igdb_slug}/{filename}",
|
||||
required=True,
|
||||
)
|
||||
)
|
||||
|
||||
return requirements
|
||||
|
||||
@@ -164,7 +166,7 @@ class Scraper(BaseScraper):
|
||||
for key in list(data.keys())[:5]:
|
||||
if ":" not in key:
|
||||
return False
|
||||
_, entry = key.split(":", 1), data[key]
|
||||
_, _entry = key.split(":", 1), data[key]
|
||||
if not isinstance(data[key], dict):
|
||||
return False
|
||||
if "md5" not in data[key] and "sha1" not in data[key]:
|
||||
@@ -217,6 +219,7 @@ class Scraper(BaseScraper):
|
||||
|
||||
def main():
|
||||
from scripts.scraper.base_scraper import scraper_cli
|
||||
|
||||
scraper_cli(Scraper, "Scrape RomM BIOS requirements")
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
Auto-detects *_targets_scraper.py files and exposes their scrapers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
@@ -6,6 +6,7 @@ Sources (batocera-linux/batocera.linux):
|
||||
- package/batocera/emulationstation/batocera-es-system/es_systems.yml
|
||||
-- emulator requireAnyOf flag mapping
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
@@ -35,23 +36,23 @@ _HEADERS = {
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
|
||||
_TARGET_FLAG_RE = re.compile(r'^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y', re.MULTILINE)
|
||||
_TARGET_FLAG_RE = re.compile(r"^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y", re.MULTILINE)
|
||||
|
||||
# Matches: select BR2_PACKAGE_FOO (optional: if CONDITION)
|
||||
# Condition may span multiple lines (backslash continuation)
|
||||
_SELECT_RE = re.compile(
|
||||
r'^\s+select\s+(BR2_PACKAGE_\w+)' # package being selected
|
||||
r'(?:\s+if\s+((?:[^\n]|\\\n)+?))?' # optional "if CONDITION" (may continue with \)
|
||||
r'(?:\s*#[^\n]*)?$', # optional trailing comment
|
||||
r"^\s+select\s+(BR2_PACKAGE_\w+)" # package being selected
|
||||
r"(?:\s+if\s+((?:[^\n]|\\\n)+?))?" # optional "if CONDITION" (may continue with \)
|
||||
r"(?:\s*#[^\n]*)?$", # optional trailing comment
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# Meta-flag definition: "if COND\n\tconfig DERIVED_FLAG\n\t...\nendif"
|
||||
_META_BLOCK_RE = re.compile(
|
||||
r'^if\s+((?:[^\n]|\\\n)+?)\n' # condition (may span lines via \)
|
||||
r'(?:.*?\n)*?' # optional lines before the config
|
||||
r'\s+config\s+(BR2_PACKAGE_\w+)' # derived flag name
|
||||
r'.*?^endif', # end of block
|
||||
r"^if\s+((?:[^\n]|\\\n)+?)\n" # condition (may span lines via \)
|
||||
r"(?:.*?\n)*?" # optional lines before the config
|
||||
r"\s+config\s+(BR2_PACKAGE_\w+)" # derived flag name
|
||||
r".*?^endif", # end of block
|
||||
re.MULTILINE | re.DOTALL,
|
||||
)
|
||||
|
||||
@@ -80,7 +81,7 @@ def _fetch_json(url: str) -> list | dict | None:
|
||||
|
||||
def _normalise_condition(raw: str) -> str:
|
||||
"""Strip backslash-continuations and collapse whitespace."""
|
||||
return re.sub(r'\\\n\s*', ' ', raw).strip()
|
||||
return re.sub(r"\\\n\s*", " ", raw).strip()
|
||||
|
||||
|
||||
def _tokenise(condition: str) -> list[str]:
|
||||
@@ -89,14 +90,16 @@ def _tokenise(condition: str) -> list[str]:
|
||||
return token_re.findall(condition)
|
||||
|
||||
|
||||
def _check_condition(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
|
||||
def _check_condition(
|
||||
tokens: list[str], pos: int, active: frozenset[str]
|
||||
) -> tuple[bool, int]:
|
||||
"""Recursive descent check of a Kconfig boolean expression."""
|
||||
return _check_or(tokens, pos, active)
|
||||
|
||||
|
||||
def _check_or(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
|
||||
left, pos = _check_and(tokens, pos, active)
|
||||
while pos < len(tokens) and tokens[pos] == '||':
|
||||
while pos < len(tokens) and tokens[pos] == "||":
|
||||
pos += 1
|
||||
right, pos = _check_and(tokens, pos, active)
|
||||
left = left or right
|
||||
@@ -105,7 +108,7 @@ def _check_or(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool
|
||||
|
||||
def _check_and(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
|
||||
left, pos = _check_not(tokens, pos, active)
|
||||
while pos < len(tokens) and tokens[pos] == '&&':
|
||||
while pos < len(tokens) and tokens[pos] == "&&":
|
||||
pos += 1
|
||||
right, pos = _check_not(tokens, pos, active)
|
||||
left = left and right
|
||||
@@ -113,24 +116,26 @@ def _check_and(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[boo
|
||||
|
||||
|
||||
def _check_not(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
|
||||
if pos < len(tokens) and tokens[pos] == '!':
|
||||
if pos < len(tokens) and tokens[pos] == "!":
|
||||
pos += 1
|
||||
val, pos = _check_atom(tokens, pos, active)
|
||||
return not val, pos
|
||||
return _check_atom(tokens, pos, active)
|
||||
|
||||
|
||||
def _check_atom(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bool, int]:
|
||||
def _check_atom(
|
||||
tokens: list[str], pos: int, active: frozenset[str]
|
||||
) -> tuple[bool, int]:
|
||||
if pos >= len(tokens):
|
||||
return True, pos
|
||||
tok = tokens[pos]
|
||||
if tok == '(':
|
||||
if tok == "(":
|
||||
pos += 1
|
||||
val, pos = _check_or(tokens, pos, active)
|
||||
if pos < len(tokens) and tokens[pos] == ')':
|
||||
if pos < len(tokens) and tokens[pos] == ")":
|
||||
pos += 1
|
||||
return val, pos
|
||||
if tok.startswith('BR2_'):
|
||||
if tok.startswith("BR2_"):
|
||||
pos += 1
|
||||
return tok in active, pos
|
||||
if tok.startswith('"'):
|
||||
@@ -170,7 +175,9 @@ def _parse_meta_flags(text: str) -> list[tuple[str, str]]:
|
||||
return results
|
||||
|
||||
|
||||
def _expand_flags(primary_flag: str, meta_rules: list[tuple[str, str]]) -> frozenset[str]:
|
||||
def _expand_flags(
|
||||
primary_flag: str, meta_rules: list[tuple[str, str]]
|
||||
) -> frozenset[str]:
|
||||
"""Given a board's primary flag, expand to all active derived flags.
|
||||
|
||||
Iterates until stable (handles chained derivations like X86_64_ANY -> X86_ANY).
|
||||
@@ -194,7 +201,7 @@ def _parse_selects(text: str) -> list[tuple[str, str]]:
|
||||
results: list[tuple[str, str]] = []
|
||||
for m in _SELECT_RE.finditer(text):
|
||||
pkg = m.group(1)
|
||||
cond = _normalise_condition(m.group(2) or '')
|
||||
cond = _normalise_condition(m.group(2) or "")
|
||||
results.append((pkg, cond))
|
||||
return results
|
||||
|
||||
@@ -261,7 +268,8 @@ class Scraper(BaseTargetScraper):
|
||||
if not data or not isinstance(data, list):
|
||||
return []
|
||||
return [
|
||||
item["name"] for item in data
|
||||
item["name"]
|
||||
for item in data
|
||||
if isinstance(item, dict)
|
||||
and item.get("name", "").startswith("batocera-")
|
||||
and item.get("name", "").endswith(".board")
|
||||
|
||||
@@ -4,6 +4,7 @@ Sources:
|
||||
SteamOS: dragoonDorise/EmuDeck -functions/EmuScripts/*.sh
|
||||
Windows: EmuDeck/emudeck-we -functions/EmuScripts/*.ps1
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
@@ -20,8 +21,12 @@ from . import BaseTargetScraper
|
||||
|
||||
PLATFORM_NAME = "emudeck"
|
||||
|
||||
STEAMOS_API = "https://api.github.com/repos/dragoonDorise/EmuDeck/contents/functions/EmuScripts"
|
||||
WINDOWS_API = "https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts"
|
||||
STEAMOS_API = (
|
||||
"https://api.github.com/repos/dragoonDorise/EmuDeck/contents/functions/EmuScripts"
|
||||
)
|
||||
WINDOWS_API = (
|
||||
"https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts"
|
||||
)
|
||||
|
||||
# Map EmuDeck script names to emulator profile keys
|
||||
# Script naming: emuDeckDolphin.sh -> dolphin
|
||||
@@ -70,8 +75,8 @@ def _list_emuscripts(api_url: str) -> list[str]:
|
||||
def _script_to_core(filename: str) -> str | None:
|
||||
"""Convert EmuScripts filename to core profile key."""
|
||||
# Strip extension and emuDeck prefix
|
||||
name = re.sub(r'\.(sh|ps1)$', '', filename, flags=re.IGNORECASE)
|
||||
name = re.sub(r'^emuDeck', '', name, flags=re.IGNORECASE)
|
||||
name = re.sub(r"\.(sh|ps1)$", "", filename, flags=re.IGNORECASE)
|
||||
name = re.sub(r"^emuDeck", "", name, flags=re.IGNORECASE)
|
||||
if not name:
|
||||
return None
|
||||
key = name.lower()
|
||||
@@ -86,8 +91,9 @@ class Scraper(BaseTargetScraper):
|
||||
def __init__(self, url: str = "https://github.com/dragoonDorise/EmuDeck"):
|
||||
super().__init__(url=url)
|
||||
|
||||
def _fetch_cores_for_target(self, api_url: str, label: str,
|
||||
arch: str = "x86_64") -> list[str]:
|
||||
def _fetch_cores_for_target(
|
||||
self, api_url: str, label: str, arch: str = "x86_64"
|
||||
) -> list[str]:
|
||||
print(f" fetching {label} EmuScripts...", file=sys.stderr)
|
||||
scripts = _list_emuscripts(api_url)
|
||||
cores: list[str] = []
|
||||
@@ -99,7 +105,7 @@ class Scraper(BaseTargetScraper):
|
||||
seen.add(core)
|
||||
cores.append(core)
|
||||
# Detect RetroArch presence (provides all libretro cores)
|
||||
name = re.sub(r'\.(sh|ps1)$', '', script, flags=re.IGNORECASE)
|
||||
name = re.sub(r"\.(sh|ps1)$", "", script, flags=re.IGNORECASE)
|
||||
if name.lower() in ("emudeckretroarch", "retroarch_maincfg"):
|
||||
has_retroarch = True
|
||||
|
||||
@@ -112,15 +118,18 @@ class Scraper(BaseTargetScraper):
|
||||
seen.add(c)
|
||||
cores.append(c)
|
||||
|
||||
print(f" {label}: {standalone_count} standalone + "
|
||||
f"{len(cores) - standalone_count} via RetroArch = {len(cores)} total",
|
||||
file=sys.stderr)
|
||||
print(
|
||||
f" {label}: {standalone_count} standalone + "
|
||||
f"{len(cores) - standalone_count} via RetroArch = {len(cores)} total",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return sorted(cores)
|
||||
|
||||
@staticmethod
|
||||
def _load_retroarch_cores(arch: str) -> list[str]:
|
||||
"""Load RetroArch target cores for given architecture."""
|
||||
import os
|
||||
|
||||
target_path = os.path.join("platforms", "targets", "retroarch.yml")
|
||||
if not os.path.exists(target_path):
|
||||
return []
|
||||
@@ -157,9 +166,7 @@ class Scraper(BaseTargetScraper):
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Scrape EmuDeck emulator targets"
|
||||
)
|
||||
parser = argparse.ArgumentParser(description="Scrape EmuDeck emulator targets")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show target summary")
|
||||
parser.add_argument("--output", "-o", help="Output YAML file")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -16,6 +16,7 @@ Buildbot structure varies by platform:
|
||||
- ps2: playstation/ps2/latest/ -> *_libretro_ps2.elf.zip
|
||||
- vita: bundles only (VPK) - no individual cores
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
@@ -64,7 +65,9 @@ RECIPE_TARGETS: list[tuple[str, str, str]] = [
|
||||
("playstation/vita", "playstation-vita", "armv7"),
|
||||
]
|
||||
|
||||
RECIPE_BASE_URL = "https://raw.githubusercontent.com/libretro/libretro-super/master/recipes/"
|
||||
RECIPE_BASE_URL = (
|
||||
"https://raw.githubusercontent.com/libretro/libretro-super/master/recipes/"
|
||||
)
|
||||
|
||||
# Match any href containing _libretro followed by a platform-specific extension
|
||||
# Covers: .so.zip, .dll.zip, .dylib.zip, .nro.zip, .dol.zip, .rpx.zip,
|
||||
@@ -75,7 +78,7 @@ _HREF_RE = re.compile(
|
||||
)
|
||||
|
||||
# Extract core name: everything before _libretro
|
||||
_CORE_NAME_RE = re.compile(r'^(.+?)_libretro')
|
||||
_CORE_NAME_RE = re.compile(r"^(.+?)_libretro")
|
||||
|
||||
|
||||
class Scraper(BaseTargetScraper):
|
||||
@@ -180,12 +183,16 @@ def main() -> None:
|
||||
data = scraper.fetch_targets()
|
||||
|
||||
total_cores = sum(len(t["cores"]) for t in data["targets"].values())
|
||||
print(f"\n{len(data['targets'])} targets, {total_cores} total core entries",
|
||||
file=sys.stderr)
|
||||
print(
|
||||
f"\n{len(data['targets'])} targets, {total_cores} total core entries",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
for name, info in sorted(data["targets"].items()):
|
||||
print(f" {name:30s} {info['architecture']:10s} {len(info['cores']):>4d} cores")
|
||||
print(
|
||||
f" {name:30s} {info['architecture']:10s} {len(info['cores']):>4d} cores"
|
||||
)
|
||||
return
|
||||
|
||||
if args.output:
|
||||
|
||||
@@ -4,6 +4,7 @@ Source: https://github.com/RetroPie/RetroPie-Setup/tree/master/scriptmodules/lib
|
||||
Parses rp_module_id and rp_module_flags from each scriptmodule to determine
|
||||
which platforms each core supports.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
@@ -9,6 +9,7 @@ Curve: sect233r1 (NIST B-233, SEC 2 v2)
|
||||
Field: GF(2^233) with irreducible polynomial t^233 + t^74 + 1
|
||||
Equation: y^2 + xy = x^3 + x^2 + b
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
@@ -34,6 +35,7 @@ _H = 2
|
||||
|
||||
# GF(2^233) field arithmetic
|
||||
|
||||
|
||||
def _gf_reduce(a: int) -> int:
|
||||
"""Reduce polynomial a modulo t^233 + t^74 + 1."""
|
||||
while a.bit_length() > _M:
|
||||
@@ -171,6 +173,7 @@ def _ec_mul(k: int, p: tuple[int, int] | None) -> tuple[int, int] | None:
|
||||
|
||||
# ECDSA-SHA256 verification
|
||||
|
||||
|
||||
def _modinv(a: int, m: int) -> int:
|
||||
"""Modular inverse of a modulo m (integers, not GF(2^m))."""
|
||||
if a < 0:
|
||||
|
||||
108
scripts/truth.py
108
scripts/truth.py
@@ -13,7 +13,8 @@ from validation import filter_files_by_mode
|
||||
|
||||
|
||||
def _determine_core_mode(
|
||||
emu_name: str, profile: dict,
|
||||
emu_name: str,
|
||||
profile: dict,
|
||||
cores_config: str | list | None,
|
||||
standalone_set: set[str] | None,
|
||||
) -> str:
|
||||
@@ -62,7 +63,10 @@ def _enrich_hashes(entry: dict, db: dict) -> None:
|
||||
|
||||
|
||||
def _merge_file_into_system(
|
||||
system: dict, file_entry: dict, emu_name: str, db: dict | None,
|
||||
system: dict,
|
||||
file_entry: dict,
|
||||
emu_name: str,
|
||||
db: dict | None,
|
||||
) -> None:
|
||||
"""Merge a file entry into a system's file list, deduplicating by name."""
|
||||
files = system.setdefault("files", [])
|
||||
@@ -100,9 +104,22 @@ def _merge_file_into_system(
|
||||
entry: dict = {"name": file_entry["name"]}
|
||||
if file_entry.get("required") is not None:
|
||||
entry["required"] = file_entry["required"]
|
||||
for field in ("sha1", "md5", "sha256", "crc32", "size", "path",
|
||||
"description", "hle_fallback", "category", "note",
|
||||
"validation", "min_size", "max_size", "aliases"):
|
||||
for field in (
|
||||
"sha1",
|
||||
"md5",
|
||||
"sha256",
|
||||
"crc32",
|
||||
"size",
|
||||
"path",
|
||||
"description",
|
||||
"hle_fallback",
|
||||
"category",
|
||||
"note",
|
||||
"validation",
|
||||
"min_size",
|
||||
"max_size",
|
||||
"aliases",
|
||||
):
|
||||
val = file_entry.get(field)
|
||||
if val is not None:
|
||||
entry[field] = val
|
||||
@@ -206,7 +223,9 @@ def generate_platform_truth(
|
||||
if mode == "both":
|
||||
filtered = raw_files
|
||||
else:
|
||||
filtered = filter_files_by_mode(raw_files, standalone=(mode == "standalone"))
|
||||
filtered = filter_files_by_mode(
|
||||
raw_files, standalone=(mode == "standalone")
|
||||
)
|
||||
|
||||
for fe in filtered:
|
||||
profile_sid = fe.get("system", "")
|
||||
@@ -217,9 +236,13 @@ def generate_platform_truth(
|
||||
system = systems.setdefault(sys_id, {})
|
||||
_merge_file_into_system(system, fe, emu_name, db)
|
||||
# Track core contribution per system
|
||||
sys_cov = system_cores.setdefault(sys_id, {
|
||||
"profiled": set(), "unprofiled": set(),
|
||||
})
|
||||
sys_cov = system_cores.setdefault(
|
||||
sys_id,
|
||||
{
|
||||
"profiled": set(),
|
||||
"unprofiled": set(),
|
||||
},
|
||||
)
|
||||
sys_cov["profiled"].add(emu_name)
|
||||
|
||||
# Ensure all systems of resolved cores have entries (even with 0 files).
|
||||
@@ -230,17 +253,25 @@ def generate_platform_truth(
|
||||
for prof_sid in profile.get("systems", []):
|
||||
sys_id = _map_sys_id(prof_sid)
|
||||
systems.setdefault(sys_id, {})
|
||||
sys_cov = system_cores.setdefault(sys_id, {
|
||||
"profiled": set(), "unprofiled": set(),
|
||||
})
|
||||
sys_cov = system_cores.setdefault(
|
||||
sys_id,
|
||||
{
|
||||
"profiled": set(),
|
||||
"unprofiled": set(),
|
||||
},
|
||||
)
|
||||
sys_cov["profiled"].add(emu_name)
|
||||
|
||||
# Track unprofiled cores per system based on profile system lists
|
||||
for emu_name in cores_unprofiled:
|
||||
for sys_id in systems:
|
||||
sys_cov = system_cores.setdefault(sys_id, {
|
||||
"profiled": set(), "unprofiled": set(),
|
||||
})
|
||||
sys_cov = system_cores.setdefault(
|
||||
sys_id,
|
||||
{
|
||||
"profiled": set(),
|
||||
"unprofiled": set(),
|
||||
},
|
||||
)
|
||||
sys_cov["unprofiled"].add(emu_name)
|
||||
|
||||
# Convert sets to sorted lists for serialization
|
||||
@@ -269,6 +300,7 @@ def generate_platform_truth(
|
||||
|
||||
# Platform truth diffing
|
||||
|
||||
|
||||
def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
"""Compare files between truth and scraped for a single system."""
|
||||
# Build truth index: name.lower() -> entry, alias.lower() -> entry
|
||||
@@ -310,32 +342,38 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
t_set = {v.lower() for v in t_list}
|
||||
s_set = {v.lower() for v in s_list}
|
||||
if not t_set & s_set:
|
||||
hash_mismatch.append({
|
||||
"name": s_entry["name"],
|
||||
"hash_type": h,
|
||||
f"truth_{h}": t_hash,
|
||||
f"scraped_{h}": s_hash,
|
||||
"truth_cores": list(t_entry.get("_cores", [])),
|
||||
})
|
||||
hash_mismatch.append(
|
||||
{
|
||||
"name": s_entry["name"],
|
||||
"hash_type": h,
|
||||
f"truth_{h}": t_hash,
|
||||
f"scraped_{h}": s_hash,
|
||||
"truth_cores": list(t_entry.get("_cores", [])),
|
||||
}
|
||||
)
|
||||
break
|
||||
|
||||
# Required mismatch
|
||||
t_req = t_entry.get("required")
|
||||
s_req = s_entry.get("required")
|
||||
if t_req is not None and s_req is not None and t_req != s_req:
|
||||
required_mismatch.append({
|
||||
"name": s_entry["name"],
|
||||
"truth_required": t_req,
|
||||
"scraped_required": s_req,
|
||||
})
|
||||
required_mismatch.append(
|
||||
{
|
||||
"name": s_entry["name"],
|
||||
"truth_required": t_req,
|
||||
"scraped_required": s_req,
|
||||
}
|
||||
)
|
||||
|
||||
# Collect unmatched files from both sides
|
||||
unmatched_truth = [
|
||||
fe for fe in truth_sys.get("files", [])
|
||||
fe
|
||||
for fe in truth_sys.get("files", [])
|
||||
if fe["name"].lower() not in matched_truth_names
|
||||
]
|
||||
unmatched_scraped = {
|
||||
s_key: s_entry for s_key, s_entry in scraped_index.items()
|
||||
s_key: s_entry
|
||||
for s_key, s_entry in scraped_index.items()
|
||||
if s_key not in truth_index
|
||||
}
|
||||
|
||||
@@ -369,11 +407,13 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
# Truth files not matched (by name, alias, or hash) -> missing
|
||||
for fe in unmatched_truth:
|
||||
if fe["name"].lower() not in rename_matched_truth:
|
||||
missing.append({
|
||||
"name": fe["name"],
|
||||
"cores": list(fe.get("_cores", [])),
|
||||
"source_refs": list(fe.get("_source_refs", [])),
|
||||
})
|
||||
missing.append(
|
||||
{
|
||||
"name": fe["name"],
|
||||
"cores": list(fe.get("_cores", [])),
|
||||
"source_refs": list(fe.get("_source_refs", [])),
|
||||
}
|
||||
)
|
||||
|
||||
# Scraped files not in truth -> extra
|
||||
coverage = truth_sys.get("_coverage", {})
|
||||
|
||||
@@ -36,8 +36,20 @@ DEFAULT_DB = "database.json"
|
||||
DEFAULT_PLATFORMS_DIR = "platforms"
|
||||
|
||||
BLOCKED_EXTENSIONS = {
|
||||
".exe", ".bat", ".cmd", ".sh", ".ps1", ".vbs", ".js",
|
||||
".msi", ".dll", ".so", ".dylib", ".py", ".rb", ".pl",
|
||||
".exe",
|
||||
".bat",
|
||||
".cmd",
|
||||
".sh",
|
||||
".ps1",
|
||||
".vbs",
|
||||
".js",
|
||||
".msi",
|
||||
".dll",
|
||||
".so",
|
||||
".dylib",
|
||||
".py",
|
||||
".rb",
|
||||
".pl",
|
||||
}
|
||||
|
||||
MAX_FILE_SIZE = 100 * 1024 * 1024
|
||||
@@ -140,7 +152,10 @@ def validate_file(
|
||||
result.add_check(False, f"Blocked file extension: {ext}")
|
||||
|
||||
if result.size > MAX_FILE_SIZE:
|
||||
result.add_check(False, f"File too large for embedded storage ({result.size:,} > {MAX_FILE_SIZE:,} bytes). Use storage: external in platform config.")
|
||||
result.add_check(
|
||||
False,
|
||||
f"File too large for embedded storage ({result.size:,} > {MAX_FILE_SIZE:,} bytes). Use storage: external in platform config.",
|
||||
)
|
||||
elif result.size == 0:
|
||||
result.add_check(False, "File is empty (0 bytes)")
|
||||
else:
|
||||
@@ -149,7 +164,9 @@ def validate_file(
|
||||
if db:
|
||||
if result.sha1 in db.get("files", {}):
|
||||
existing = db["files"][result.sha1]
|
||||
result.add_warning(f"Duplicate: identical file already exists at `{existing['path']}`")
|
||||
result.add_warning(
|
||||
f"Duplicate: identical file already exists at `{existing['path']}`"
|
||||
)
|
||||
else:
|
||||
result.add_check(True, "Not a duplicate in database")
|
||||
|
||||
@@ -162,9 +179,13 @@ def validate_file(
|
||||
elif md5_known:
|
||||
result.add_check(True, "MD5 matches known platform requirement")
|
||||
elif name_known:
|
||||
result.add_warning("Filename matches a known requirement but hash differs - may be a variant")
|
||||
result.add_warning(
|
||||
"Filename matches a known requirement but hash differs - may be a variant"
|
||||
)
|
||||
else:
|
||||
result.add_warning("File not referenced in any platform config - needs manual review")
|
||||
result.add_warning(
|
||||
"File not referenced in any platform config - needs manual review"
|
||||
)
|
||||
|
||||
normalized = os.path.normpath(filepath)
|
||||
if os.path.islink(filepath):
|
||||
@@ -194,9 +215,15 @@ def get_changed_files() -> list[str]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", f"origin/{base}...HEAD"],
|
||||
capture_output=True, text=True, check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
files = [f for f in result.stdout.strip().split("\n") if f.startswith("bios/")]
|
||||
files = [
|
||||
f
|
||||
for f in result.stdout.strip().split("\n")
|
||||
if f.startswith("bios/")
|
||||
]
|
||||
if files:
|
||||
return files
|
||||
except subprocess.CalledProcessError:
|
||||
@@ -206,7 +233,8 @@ def get_changed_files() -> list[str]:
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--cached", "--name-only"],
|
||||
capture_output=True, text=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return [f for f in result.stdout.strip().split("\n") if f.startswith("bios/") and f]
|
||||
|
||||
@@ -214,10 +242,14 @@ def get_changed_files() -> list[str]:
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Validate BIOS file contributions")
|
||||
parser.add_argument("files", nargs="*", help="Files to validate")
|
||||
parser.add_argument("--changed", action="store_true", help="Auto-detect changed BIOS files")
|
||||
parser.add_argument(
|
||||
"--changed", action="store_true", help="Auto-detect changed BIOS files"
|
||||
)
|
||||
parser.add_argument("--db", default=DEFAULT_DB, help="Path to database.json")
|
||||
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
|
||||
parser.add_argument("--markdown", action="store_true", help="Output as markdown (for PR comments)")
|
||||
parser.add_argument(
|
||||
"--markdown", action="store_true", help="Output as markdown (for PR comments)"
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -250,14 +282,16 @@ def main():
|
||||
if args.json:
|
||||
output = []
|
||||
for r in results:
|
||||
output.append({
|
||||
"file": r.filepath,
|
||||
"passed": r.passed,
|
||||
"sha1": r.sha1,
|
||||
"md5": r.md5,
|
||||
"size": r.size,
|
||||
"checks": [{"status": s, "message": m} for s, m in r.checks],
|
||||
})
|
||||
output.append(
|
||||
{
|
||||
"file": r.filepath,
|
||||
"passed": r.passed,
|
||||
"sha1": r.sha1,
|
||||
"md5": r.md5,
|
||||
"size": r.size,
|
||||
"checks": [{"status": s, "message": m} for s, m in r.checks],
|
||||
}
|
||||
)
|
||||
print(json.dumps(output, indent=2))
|
||||
elif args.markdown:
|
||||
lines = ["## BIOS Validation Report", ""]
|
||||
@@ -278,7 +312,15 @@ def main():
|
||||
print(f" MD5: {r.md5}")
|
||||
print(f" Size: {r.size:,}")
|
||||
for s, m in r.checks:
|
||||
marker = "✓" if s == "PASS" else "✗" if s == "FAIL" else "!" if s == "WARN" else "i"
|
||||
marker = (
|
||||
"✓"
|
||||
if s == "PASS"
|
||||
else "✗"
|
||||
if s == "FAIL"
|
||||
else "!"
|
||||
if s == "WARN"
|
||||
else "i"
|
||||
)
|
||||
print(f" [{marker}] {m}")
|
||||
|
||||
if not all_passed:
|
||||
|
||||
@@ -63,28 +63,37 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
|
||||
continue
|
||||
if fname not in index:
|
||||
index[fname] = {
|
||||
"checks": set(), "sizes": set(),
|
||||
"min_size": None, "max_size": None,
|
||||
"crc32": set(), "md5": set(), "sha1": set(), "sha256": set(),
|
||||
"adler32": set(), "crypto_only": set(),
|
||||
"emulators": set(), "per_emulator": {},
|
||||
"checks": set(),
|
||||
"sizes": set(),
|
||||
"min_size": None,
|
||||
"max_size": None,
|
||||
"crc32": set(),
|
||||
"md5": set(),
|
||||
"sha1": set(),
|
||||
"sha256": set(),
|
||||
"adler32": set(),
|
||||
"crypto_only": set(),
|
||||
"emulators": set(),
|
||||
"per_emulator": {},
|
||||
}
|
||||
index[fname]["emulators"].add(emu_name)
|
||||
index[fname]["checks"].update(checks)
|
||||
# Track non-reproducible crypto checks
|
||||
index[fname]["crypto_only"].update(
|
||||
c for c in checks if c in _CRYPTO_CHECKS
|
||||
)
|
||||
index[fname]["crypto_only"].update(c for c in checks if c in _CRYPTO_CHECKS)
|
||||
# Size checks
|
||||
if "size" in checks:
|
||||
if f.get("size") is not None:
|
||||
index[fname]["sizes"].add(f["size"])
|
||||
if f.get("min_size") is not None:
|
||||
cur = index[fname]["min_size"]
|
||||
index[fname]["min_size"] = min(cur, f["min_size"]) if cur is not None else f["min_size"]
|
||||
index[fname]["min_size"] = (
|
||||
min(cur, f["min_size"]) if cur is not None else f["min_size"]
|
||||
)
|
||||
if f.get("max_size") is not None:
|
||||
cur = index[fname]["max_size"]
|
||||
index[fname]["max_size"] = max(cur, f["max_size"]) if cur is not None else f["max_size"]
|
||||
index[fname]["max_size"] = (
|
||||
max(cur, f["max_size"]) if cur is not None else f["max_size"]
|
||||
)
|
||||
# Hash checks -collect all accepted hashes as sets (multiple valid
|
||||
# versions of the same file, e.g. MT-32 ROM versions)
|
||||
if "crc32" in checks and f.get("crc32"):
|
||||
@@ -132,7 +141,9 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
|
||||
if emu_name in pe:
|
||||
# Merge checks from multiple file entries for same emulator
|
||||
existing = pe[emu_name]
|
||||
merged_checks = sorted(set(existing["checks"]) | set(pe_entry["checks"]))
|
||||
merged_checks = sorted(
|
||||
set(existing["checks"]) | set(pe_entry["checks"])
|
||||
)
|
||||
existing["checks"] = merged_checks
|
||||
existing["expected"].update(pe_entry["expected"])
|
||||
if pe_entry["source_ref"] and not existing["source_ref"]:
|
||||
@@ -160,17 +171,21 @@ def build_ground_truth(filename: str, validation_index: dict[str, dict]) -> list
|
||||
result = []
|
||||
for emu_name in sorted(entry["per_emulator"]):
|
||||
detail = entry["per_emulator"][emu_name]
|
||||
result.append({
|
||||
"emulator": emu_name,
|
||||
"checks": detail["checks"],
|
||||
"source_ref": detail.get("source_ref"),
|
||||
"expected": detail.get("expected", {}),
|
||||
})
|
||||
result.append(
|
||||
{
|
||||
"emulator": emu_name,
|
||||
"checks": detail["checks"],
|
||||
"source_ref": detail.get("source_ref"),
|
||||
"expected": detail.get("expected", {}),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def check_file_validation(
|
||||
local_path: str, filename: str, validation_index: dict[str, dict],
|
||||
local_path: str,
|
||||
filename: str,
|
||||
validation_index: dict[str, dict],
|
||||
bios_dir: str = "bios",
|
||||
) -> str | None:
|
||||
"""Check emulator-level validation on a resolved file.
|
||||
@@ -199,10 +214,9 @@ def check_file_validation(
|
||||
|
||||
# Hash checks -compute once, reuse for all hash types.
|
||||
# Each hash field is a set of accepted values (multiple valid ROM versions).
|
||||
need_hashes = (
|
||||
any(h in checks and entry.get(h) for h in ("crc32", "md5", "sha1", "sha256"))
|
||||
or entry.get("adler32")
|
||||
)
|
||||
need_hashes = any(
|
||||
h in checks and entry.get(h) for h in ("crc32", "md5", "sha1", "sha256")
|
||||
) or entry.get("adler32")
|
||||
if need_hashes:
|
||||
hashes = compute_hashes(local_path)
|
||||
for hash_type in ("crc32", "md5", "sha1", "sha256"):
|
||||
@@ -218,6 +232,7 @@ def check_file_validation(
|
||||
# Signature/crypto checks (3DS RSA, AES)
|
||||
if entry["crypto_only"]:
|
||||
from crypto_verify import check_crypto_validation
|
||||
|
||||
crypto_reason = check_crypto_validation(local_path, filename, bios_dir)
|
||||
if crypto_reason:
|
||||
return crypto_reason
|
||||
|
||||
@@ -21,28 +21,41 @@ Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import (
|
||||
build_target_cores_cache, build_zip_contents_index, check_inside_zip,
|
||||
compute_hashes, expand_platform_declared_names, filter_systems_by_target,
|
||||
group_identical_platforms, list_emulator_profiles, list_system_ids,
|
||||
load_data_dir_registry, load_emulator_profiles, load_platform_config,
|
||||
md5sum, md5_composite, require_yaml, resolve_local_file,
|
||||
build_target_cores_cache,
|
||||
build_zip_contents_index,
|
||||
check_inside_zip,
|
||||
compute_hashes,
|
||||
expand_platform_declared_names,
|
||||
filter_systems_by_target,
|
||||
group_identical_platforms,
|
||||
list_emulator_profiles,
|
||||
list_system_ids,
|
||||
load_data_dir_registry,
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
md5_composite,
|
||||
md5sum,
|
||||
require_yaml,
|
||||
resolve_local_file,
|
||||
resolve_platform_cores,
|
||||
)
|
||||
|
||||
yaml = require_yaml()
|
||||
from validation import (
|
||||
_build_validation_index, _parse_validation, build_ground_truth,
|
||||
check_file_validation, filter_files_by_mode,
|
||||
_build_validation_index,
|
||||
_parse_validation,
|
||||
build_ground_truth,
|
||||
check_file_validation,
|
||||
filter_files_by_mode,
|
||||
)
|
||||
|
||||
DEFAULT_DB = "database.json"
|
||||
DEFAULT_PLATFORMS_DIR = "platforms"
|
||||
DEFAULT_EMULATORS_DIR = "emulators"
|
||||
@@ -50,27 +63,36 @@ DEFAULT_EMULATORS_DIR = "emulators"
|
||||
|
||||
# Status model -aligned with Batocera BiosStatus (batocera-systems:967-969)
|
||||
|
||||
|
||||
class Status:
|
||||
OK = "ok"
|
||||
UNTESTED = "untested" # file present, hash not confirmed
|
||||
UNTESTED = "untested" # file present, hash not confirmed
|
||||
MISSING = "missing"
|
||||
|
||||
|
||||
# Severity for per-file required/optional distinction
|
||||
class Severity:
|
||||
CRITICAL = "critical" # required file missing or bad hash (Recalbox RED)
|
||||
WARNING = "warning" # optional missing or hash mismatch (Recalbox YELLOW)
|
||||
INFO = "info" # optional missing on existence-only platform
|
||||
OK = "ok" # file verified
|
||||
CRITICAL = "critical" # required file missing or bad hash (Recalbox RED)
|
||||
WARNING = "warning" # optional missing or hash mismatch (Recalbox YELLOW)
|
||||
INFO = "info" # optional missing on existence-only platform
|
||||
OK = "ok" # file verified
|
||||
|
||||
|
||||
_STATUS_ORDER = {Status.OK: 0, Status.UNTESTED: 1, Status.MISSING: 2}
|
||||
_SEVERITY_ORDER = {Severity.OK: 0, Severity.INFO: 1, Severity.WARNING: 2, Severity.CRITICAL: 3}
|
||||
_SEVERITY_ORDER = {
|
||||
Severity.OK: 0,
|
||||
Severity.INFO: 1,
|
||||
Severity.WARNING: 2,
|
||||
Severity.CRITICAL: 3,
|
||||
}
|
||||
|
||||
|
||||
# Verification functions
|
||||
|
||||
|
||||
def verify_entry_existence(
|
||||
file_entry: dict, local_path: str | None,
|
||||
file_entry: dict,
|
||||
local_path: str | None,
|
||||
validation_index: dict[str, dict] | None = None,
|
||||
) -> dict:
|
||||
"""RetroArch verification: path_is_valid() -file exists = OK."""
|
||||
@@ -120,13 +142,25 @@ def verify_entry_md5(
|
||||
elif result != "not_in_zip":
|
||||
found_in_zip = True
|
||||
if had_error and not found_in_zip:
|
||||
return {**base, "status": Status.UNTESTED, "path": local_path,
|
||||
"reason": f"{local_path} read error"}
|
||||
return {
|
||||
**base,
|
||||
"status": Status.UNTESTED,
|
||||
"path": local_path,
|
||||
"reason": f"{local_path} read error",
|
||||
}
|
||||
if not found_in_zip:
|
||||
return {**base, "status": Status.UNTESTED, "path": local_path,
|
||||
"reason": f"{zipped_file} not found inside ZIP"}
|
||||
return {**base, "status": Status.UNTESTED, "path": local_path,
|
||||
"reason": f"{zipped_file} MD5 mismatch inside ZIP"}
|
||||
return {
|
||||
**base,
|
||||
"status": Status.UNTESTED,
|
||||
"path": local_path,
|
||||
"reason": f"{zipped_file} not found inside ZIP",
|
||||
}
|
||||
return {
|
||||
**base,
|
||||
"status": Status.UNTESTED,
|
||||
"path": local_path,
|
||||
"reason": f"{zipped_file} MD5 mismatch inside ZIP",
|
||||
}
|
||||
|
||||
if not md5_list:
|
||||
return {**base, "status": Status.OK, "path": local_path}
|
||||
@@ -151,8 +185,12 @@ def verify_entry_md5(
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
pass
|
||||
|
||||
return {**base, "status": Status.UNTESTED, "path": local_path,
|
||||
"reason": f"expected {md5_list[0][:12]}… got {actual_md5[:12]}…"}
|
||||
return {
|
||||
**base,
|
||||
"status": Status.UNTESTED,
|
||||
"path": local_path,
|
||||
"reason": f"expected {md5_list[0][:12]}… got {actual_md5[:12]}…",
|
||||
}
|
||||
|
||||
|
||||
def verify_entry_sha1(
|
||||
@@ -176,14 +214,22 @@ def verify_entry_sha1(
|
||||
if actual_sha1 == expected_sha1.lower():
|
||||
return {**base, "status": Status.OK, "path": local_path}
|
||||
|
||||
return {**base, "status": Status.UNTESTED, "path": local_path,
|
||||
"reason": f"expected {expected_sha1[:12]}… got {actual_sha1[:12]}…"}
|
||||
return {
|
||||
**base,
|
||||
"status": Status.UNTESTED,
|
||||
"path": local_path,
|
||||
"reason": f"expected {expected_sha1[:12]}… got {actual_sha1[:12]}…",
|
||||
}
|
||||
|
||||
|
||||
# Severity mapping per platform
|
||||
|
||||
|
||||
def compute_severity(
|
||||
status: str, required: bool, mode: str, hle_fallback: bool = False,
|
||||
status: str,
|
||||
required: bool,
|
||||
mode: str,
|
||||
hle_fallback: bool = False,
|
||||
) -> str:
|
||||
"""Map (status, required, verification_mode, hle_fallback) -> severity.
|
||||
|
||||
@@ -235,8 +281,13 @@ def _build_expected(file_entry: dict, checks: list[str]) -> dict:
|
||||
expected["adler32"] = adler_val
|
||||
return expected
|
||||
|
||||
def _name_in_index(name: str, by_name: dict, by_path_suffix: dict | None = None,
|
||||
data_names: set[str] | None = None) -> bool:
|
||||
|
||||
def _name_in_index(
|
||||
name: str,
|
||||
by_name: dict,
|
||||
by_path_suffix: dict | None = None,
|
||||
data_names: set[str] | None = None,
|
||||
) -> bool:
|
||||
"""Check if a name is resolvable in the database indexes or data directories."""
|
||||
if name in by_name:
|
||||
return True
|
||||
@@ -248,7 +299,9 @@ def _name_in_index(name: str, by_name: dict, by_path_suffix: dict | None = None,
|
||||
if data_names:
|
||||
if name in data_names or name.lower() in data_names:
|
||||
return True
|
||||
if basename != name and (basename in data_names or basename.lower() in data_names):
|
||||
if basename != name and (
|
||||
basename in data_names or basename.lower() in data_names
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -260,11 +313,17 @@ def find_undeclared_files(
|
||||
emu_profiles: dict | None = None,
|
||||
target_cores: set[str] | None = None,
|
||||
data_names: set[str] | None = None,
|
||||
include_all: bool = False,
|
||||
declared_names: set[str] | None = None,
|
||||
) -> list[dict]:
|
||||
"""Find files needed by cores but not declared in platform config."""
|
||||
# Collect all filenames declared by this platform, enriched with
|
||||
# canonical names from DB via MD5 (handles platform renaming)
|
||||
declared_names = expand_platform_declared_names(config, db)
|
||||
"""Find files needed by cores but not declared in platform config.
|
||||
|
||||
declared_names overrides the default enriched set from
|
||||
expand_platform_declared_names. Pass a strict set (YAML names only)
|
||||
when building packs so alias-only names still get packed.
|
||||
"""
|
||||
if declared_names is None:
|
||||
declared_names = expand_platform_declared_names(config, db)
|
||||
|
||||
# Collect data_directory refs
|
||||
declared_dd: set[str] = set()
|
||||
@@ -276,7 +335,11 @@ def find_undeclared_files(
|
||||
|
||||
by_name = db.get("indexes", {}).get("by_name", {})
|
||||
by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
|
||||
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir)
|
||||
profiles = (
|
||||
emu_profiles
|
||||
if emu_profiles is not None
|
||||
else load_emulator_profiles(emulators_dir)
|
||||
)
|
||||
|
||||
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
|
||||
standalone_set = set(str(c) for c in config.get("standalone_cores", []))
|
||||
@@ -328,19 +391,22 @@ def find_undeclared_files(
|
||||
archive = f.get("archive")
|
||||
|
||||
# Skip files declared by the platform (by name or archive)
|
||||
if fname in declared_names:
|
||||
seen_files.add(fname)
|
||||
continue
|
||||
if archive and archive in declared_names:
|
||||
seen_files.add(fname)
|
||||
continue
|
||||
if not include_all:
|
||||
if fname in declared_names:
|
||||
seen_files.add(fname)
|
||||
continue
|
||||
if archive and archive in declared_names:
|
||||
seen_files.add(fname)
|
||||
continue
|
||||
|
||||
seen_files.add(fname)
|
||||
|
||||
# Archived files are grouped by archive
|
||||
if archive:
|
||||
if archive not in archive_entries:
|
||||
in_repo = _name_in_index(archive, by_name, by_path_suffix, data_names)
|
||||
in_repo = _name_in_index(
|
||||
archive, by_name, by_path_suffix, data_names
|
||||
)
|
||||
archive_entries[archive] = {
|
||||
"emulator": profile.get("emulator", emu_name),
|
||||
"name": archive,
|
||||
@@ -377,19 +443,21 @@ def find_undeclared_files(
|
||||
in_repo = _name_in_index(path_base, by_name, by_path_suffix, data_names)
|
||||
|
||||
checks = _parse_validation(f.get("validation"))
|
||||
undeclared.append({
|
||||
"emulator": profile.get("emulator", emu_name),
|
||||
"name": fname,
|
||||
"path": dest,
|
||||
"required": f.get("required", False),
|
||||
"hle_fallback": f.get("hle_fallback", False),
|
||||
"category": f.get("category", "bios"),
|
||||
"in_repo": in_repo,
|
||||
"note": f.get("note", ""),
|
||||
"checks": sorted(checks) if checks else [],
|
||||
"source_ref": f.get("source_ref"),
|
||||
"expected": _build_expected(f, checks),
|
||||
})
|
||||
undeclared.append(
|
||||
{
|
||||
"emulator": profile.get("emulator", emu_name),
|
||||
"name": fname,
|
||||
"path": dest,
|
||||
"required": f.get("required", False),
|
||||
"hle_fallback": f.get("hle_fallback", False),
|
||||
"category": f.get("category", "bios"),
|
||||
"in_repo": in_repo,
|
||||
"note": f.get("note", ""),
|
||||
"checks": sorted(checks) if checks else [],
|
||||
"source_ref": f.get("source_ref"),
|
||||
"expected": _build_expected(f, checks),
|
||||
}
|
||||
)
|
||||
|
||||
# Append grouped archive entries
|
||||
for entry in sorted(archive_entries.values(), key=lambda e: e["name"]):
|
||||
@@ -399,7 +467,9 @@ def find_undeclared_files(
|
||||
|
||||
|
||||
def find_exclusion_notes(
|
||||
config: dict, emulators_dir: str, emu_profiles: dict | None = None,
|
||||
config: dict,
|
||||
emulators_dir: str,
|
||||
emu_profiles: dict | None = None,
|
||||
target_cores: set[str] | None = None,
|
||||
) -> list[dict]:
|
||||
"""Document why certain emulator files are intentionally excluded.
|
||||
@@ -410,7 +480,11 @@ def find_exclusion_notes(
|
||||
- Frozen snapshots with files: [] (code doesn't load .info firmware)
|
||||
- Files covered by data_directories
|
||||
"""
|
||||
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir)
|
||||
profiles = (
|
||||
emu_profiles
|
||||
if emu_profiles is not None
|
||||
else load_emulator_profiles(emulators_dir)
|
||||
)
|
||||
platform_systems = set()
|
||||
for sys_id in config.get("systems", {}):
|
||||
platform_systems.add(sys_id)
|
||||
@@ -427,19 +501,27 @@ def find_exclusion_notes(
|
||||
|
||||
# Launcher excluded entirely
|
||||
if profile.get("type") == "launcher":
|
||||
notes.append({
|
||||
"emulator": emu_display, "reason": "launcher",
|
||||
"detail": profile.get("exclusion_note", "BIOS managed by standalone emulator"),
|
||||
})
|
||||
notes.append(
|
||||
{
|
||||
"emulator": emu_display,
|
||||
"reason": "launcher",
|
||||
"detail": profile.get(
|
||||
"exclusion_note", "BIOS managed by standalone emulator"
|
||||
),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Profile-level exclusion note (frozen snapshots, etc.)
|
||||
exclusion_note = profile.get("exclusion_note")
|
||||
if exclusion_note:
|
||||
notes.append({
|
||||
"emulator": emu_display, "reason": "exclusion_note",
|
||||
"detail": exclusion_note,
|
||||
})
|
||||
notes.append(
|
||||
{
|
||||
"emulator": emu_display,
|
||||
"reason": "exclusion_note",
|
||||
"detail": exclusion_note,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Count standalone-only files -but only report as excluded if the
|
||||
@@ -449,22 +531,34 @@ def find_exclusion_notes(
|
||||
standalone_set & {str(c) for c in profile.get("cores", [])}
|
||||
)
|
||||
if not is_standalone:
|
||||
standalone_files = [f for f in profile.get("files", []) if f.get("mode") == "standalone"]
|
||||
standalone_files = [
|
||||
f for f in profile.get("files", []) if f.get("mode") == "standalone"
|
||||
]
|
||||
if standalone_files:
|
||||
names = [f["name"] for f in standalone_files[:3]]
|
||||
more = f" +{len(standalone_files)-3}" if len(standalone_files) > 3 else ""
|
||||
notes.append({
|
||||
"emulator": emu_display, "reason": "standalone_only",
|
||||
"detail": f"{len(standalone_files)} files for standalone mode only ({', '.join(names)}{more})",
|
||||
})
|
||||
more = (
|
||||
f" +{len(standalone_files) - 3}"
|
||||
if len(standalone_files) > 3
|
||||
else ""
|
||||
)
|
||||
notes.append(
|
||||
{
|
||||
"emulator": emu_display,
|
||||
"reason": "standalone_only",
|
||||
"detail": f"{len(standalone_files)} files for standalone mode only ({', '.join(names)}{more})",
|
||||
}
|
||||
)
|
||||
|
||||
return notes
|
||||
|
||||
|
||||
# Platform verification
|
||||
|
||||
|
||||
def _find_best_variant(
|
||||
file_entry: dict, db: dict, current_path: str,
|
||||
file_entry: dict,
|
||||
db: dict,
|
||||
current_path: str,
|
||||
validation_index: dict,
|
||||
) -> str | None:
|
||||
"""Search for a repo file that passes both platform MD5 and emulator validation."""
|
||||
@@ -473,7 +567,11 @@ def _find_best_variant(
|
||||
return None
|
||||
|
||||
md5_expected = file_entry.get("md5", "")
|
||||
md5_set = {m.strip().lower() for m in md5_expected.split(",") if m.strip()} if md5_expected else set()
|
||||
md5_set = (
|
||||
{m.strip().lower() for m in md5_expected.split(",") if m.strip()}
|
||||
if md5_expected
|
||||
else set()
|
||||
)
|
||||
|
||||
by_name = db.get("indexes", {}).get("by_name", {})
|
||||
files_db = db.get("files", {})
|
||||
@@ -481,7 +579,11 @@ def _find_best_variant(
|
||||
for sha1 in by_name.get(fname, []):
|
||||
candidate = files_db.get(sha1, {})
|
||||
path = candidate.get("path", "")
|
||||
if not path or not os.path.exists(path) or os.path.realpath(path) == os.path.realpath(current_path):
|
||||
if (
|
||||
not path
|
||||
or not os.path.exists(path)
|
||||
or os.path.realpath(path) == os.path.realpath(current_path)
|
||||
):
|
||||
continue
|
||||
if md5_set and candidate.get("md5", "").lower() not in md5_set:
|
||||
continue
|
||||
@@ -492,7 +594,8 @@ def _find_best_variant(
|
||||
|
||||
|
||||
def verify_platform(
|
||||
config: dict, db: dict,
|
||||
config: dict,
|
||||
db: dict,
|
||||
emulators_dir: str = DEFAULT_EMULATORS_DIR,
|
||||
emu_profiles: dict | None = None,
|
||||
target_cores: set[str] | None = None,
|
||||
@@ -511,7 +614,11 @@ def verify_platform(
|
||||
zip_contents = build_zip_contents_index(db) if has_zipped else {}
|
||||
|
||||
# Build HLE + validation indexes from emulator profiles
|
||||
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir)
|
||||
profiles = (
|
||||
emu_profiles
|
||||
if emu_profiles is not None
|
||||
else load_emulator_profiles(emulators_dir)
|
||||
)
|
||||
hle_index: dict[str, bool] = {}
|
||||
for profile in profiles.values():
|
||||
for f in profile.get("files", []):
|
||||
@@ -522,7 +629,9 @@ def verify_platform(
|
||||
# Filter systems by target
|
||||
plat_cores = resolve_platform_cores(config, profiles) if target_cores else None
|
||||
verify_systems = filter_systems_by_target(
|
||||
config.get("systems", {}), profiles, target_cores,
|
||||
config.get("systems", {}),
|
||||
profiles,
|
||||
target_cores,
|
||||
platform_cores=plat_cores,
|
||||
)
|
||||
|
||||
@@ -536,12 +645,16 @@ def verify_platform(
|
||||
for sys_id, system in verify_systems.items():
|
||||
for file_entry in system.get("files", []):
|
||||
local_path, resolve_status = resolve_local_file(
|
||||
file_entry, db, zip_contents,
|
||||
file_entry,
|
||||
db,
|
||||
zip_contents,
|
||||
data_dir_registry=data_dir_registry,
|
||||
)
|
||||
if mode == "existence":
|
||||
result = verify_entry_existence(
|
||||
file_entry, local_path, validation_index,
|
||||
file_entry,
|
||||
local_path,
|
||||
validation_index,
|
||||
)
|
||||
elif mode == "sha1":
|
||||
result = verify_entry_sha1(file_entry, local_path)
|
||||
@@ -555,16 +668,22 @@ def verify_platform(
|
||||
reason = check_file_validation(local_path, fname, validation_index)
|
||||
if reason:
|
||||
better = _find_best_variant(
|
||||
file_entry, db, local_path, validation_index,
|
||||
file_entry,
|
||||
db,
|
||||
local_path,
|
||||
validation_index,
|
||||
)
|
||||
if not better:
|
||||
ventry = validation_index.get(fname, {})
|
||||
emus = ", ".join(ventry.get("emulators", []))
|
||||
result["discrepancy"] = f"{platform} says OK but {emus} says {reason}"
|
||||
result["discrepancy"] = (
|
||||
f"{platform} says OK but {emus} says {reason}"
|
||||
)
|
||||
result["system"] = sys_id
|
||||
result["hle_fallback"] = hle_index.get(file_entry.get("name", ""), False)
|
||||
result["ground_truth"] = build_ground_truth(
|
||||
file_entry.get("name", ""), validation_index,
|
||||
file_entry.get("name", ""),
|
||||
validation_index,
|
||||
)
|
||||
details.append(result)
|
||||
|
||||
@@ -581,11 +700,18 @@ def verify_platform(
|
||||
hle = hle_index.get(file_entry.get("name", ""), False)
|
||||
sev = compute_severity(cur, required, mode, hle)
|
||||
prev_sev = file_severity.get(dest)
|
||||
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(prev_sev, 0):
|
||||
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(
|
||||
prev_sev, 0
|
||||
):
|
||||
file_severity[dest] = sev
|
||||
|
||||
# Count by severity
|
||||
counts = {Severity.OK: 0, Severity.INFO: 0, Severity.WARNING: 0, Severity.CRITICAL: 0}
|
||||
counts = {
|
||||
Severity.OK: 0,
|
||||
Severity.INFO: 0,
|
||||
Severity.WARNING: 0,
|
||||
Severity.CRITICAL: 0,
|
||||
}
|
||||
for s in file_severity.values():
|
||||
counts[s] = counts.get(s, 0) + 1
|
||||
|
||||
@@ -597,10 +723,19 @@ def verify_platform(
|
||||
# Cross-reference undeclared files
|
||||
if supplemental_names is None:
|
||||
from cross_reference import _build_supplemental_index
|
||||
|
||||
supplemental_names = _build_supplemental_index()
|
||||
undeclared = find_undeclared_files(config, emulators_dir, db, emu_profiles,
|
||||
target_cores=target_cores, data_names=supplemental_names)
|
||||
exclusions = find_exclusion_notes(config, emulators_dir, emu_profiles, target_cores=target_cores)
|
||||
undeclared = find_undeclared_files(
|
||||
config,
|
||||
emulators_dir,
|
||||
db,
|
||||
emu_profiles,
|
||||
target_cores=target_cores,
|
||||
data_names=supplemental_names,
|
||||
)
|
||||
exclusions = find_exclusion_notes(
|
||||
config, emulators_dir, emu_profiles, target_cores=target_cores
|
||||
)
|
||||
|
||||
# Ground truth coverage
|
||||
gt_filenames = set(validation_index)
|
||||
@@ -635,6 +770,7 @@ def verify_platform(
|
||||
|
||||
# Output
|
||||
|
||||
|
||||
def _format_ground_truth_aggregate(ground_truth: list[dict]) -> str:
|
||||
"""Format ground truth as a single aggregated line.
|
||||
|
||||
@@ -759,8 +895,16 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
bios_files = [u for u in undeclared if u.get("category", "bios") == "bios"]
|
||||
game_data = [u for u in undeclared if u.get("category", "bios") == "game_data"]
|
||||
|
||||
req_not_in_repo = [u for u in bios_files if u["required"] and not u["in_repo"] and not u.get("hle_fallback")]
|
||||
req_hle_not_in_repo = [u for u in bios_files if u["required"] and not u["in_repo"] and u.get("hle_fallback")]
|
||||
req_not_in_repo = [
|
||||
u
|
||||
for u in bios_files
|
||||
if u["required"] and not u["in_repo"] and not u.get("hle_fallback")
|
||||
]
|
||||
req_hle_not_in_repo = [
|
||||
u
|
||||
for u in bios_files
|
||||
if u["required"] and not u["in_repo"] and u.get("hle_fallback")
|
||||
]
|
||||
req_in_repo = [u for u in bios_files if u["required"] and u["in_repo"]]
|
||||
opt_in_repo = [u for u in bios_files if not u["required"] and u["in_repo"]]
|
||||
opt_not_in_repo = [u for u in bios_files if not u["required"] and not u["in_repo"]]
|
||||
@@ -769,7 +913,9 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
core_missing_req = len(req_not_in_repo) + len(req_hle_not_in_repo)
|
||||
core_missing_opt = len(opt_not_in_repo)
|
||||
|
||||
print(f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, {core_missing_opt} optional missing")
|
||||
print(
|
||||
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, {core_missing_opt} optional missing"
|
||||
)
|
||||
|
||||
for u in req_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (required)", verbose)
|
||||
@@ -783,7 +929,9 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
print(f" Game data: {len(gd_present)} in pack, {len(gd_missing)} missing")
|
||||
|
||||
|
||||
def print_platform_result(result: dict, group: list[str], verbose: bool = False) -> None:
|
||||
def print_platform_result(
|
||||
result: dict, group: list[str], verbose: bool = False
|
||||
) -> None:
|
||||
mode = result["verification_mode"]
|
||||
total = result["total_files"]
|
||||
c = result["severity_counts"]
|
||||
@@ -827,13 +975,16 @@ def print_platform_result(result: dict, group: list[str], verbose: bool = False)
|
||||
gt_cov = result.get("ground_truth_coverage")
|
||||
if gt_cov and gt_cov["total"] > 0:
|
||||
pct = gt_cov["with_validation"] * 100 // gt_cov["total"]
|
||||
print(f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)")
|
||||
print(
|
||||
f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)"
|
||||
)
|
||||
if gt_cov["platform_only"]:
|
||||
print(f" {gt_cov['platform_only']} platform-only (no emulator profile)")
|
||||
|
||||
|
||||
# Emulator/system mode verification
|
||||
|
||||
|
||||
def _effective_validation_label(details: list[dict], validation_index: dict) -> str:
|
||||
"""Determine the bracket label for the report.
|
||||
|
||||
@@ -863,7 +1014,7 @@ def verify_emulator(
|
||||
standalone: bool = False,
|
||||
) -> dict:
|
||||
"""Verify files for specific emulator profiles."""
|
||||
profiles = load_emulator_profiles(emulators_dir)
|
||||
load_emulator_profiles(emulators_dir)
|
||||
zip_contents = build_zip_contents_index(db)
|
||||
|
||||
# Also load aliases for redirect messages
|
||||
@@ -873,26 +1024,35 @@ def verify_emulator(
|
||||
selected: list[tuple[str, dict]] = []
|
||||
for name in profile_names:
|
||||
if name not in all_profiles:
|
||||
available = sorted(k for k, v in all_profiles.items()
|
||||
if v.get("type") not in ("alias", "test"))
|
||||
available = sorted(
|
||||
k
|
||||
for k, v in all_profiles.items()
|
||||
if v.get("type") not in ("alias", "test")
|
||||
)
|
||||
print(f"Error: emulator '{name}' not found", file=sys.stderr)
|
||||
print(f"Available: {', '.join(available[:10])}...", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
p = all_profiles[name]
|
||||
if p.get("type") == "alias":
|
||||
alias_of = p.get("alias_of", "?")
|
||||
print(f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}",
|
||||
file=sys.stderr)
|
||||
print(
|
||||
f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
if p.get("type") == "launcher":
|
||||
print(f"Error: {name} is a launcher -use the emulator it launches",
|
||||
file=sys.stderr)
|
||||
print(
|
||||
f"Error: {name} is a launcher -use the emulator it launches",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
# Check standalone capability
|
||||
ptype = p.get("type", "libretro")
|
||||
if standalone and "standalone" not in ptype:
|
||||
print(f"Error: {name} ({ptype}) does not support --standalone",
|
||||
file=sys.stderr)
|
||||
print(
|
||||
f"Error: {name} ({ptype}) does not support --standalone",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
selected.append((name, p))
|
||||
|
||||
@@ -924,12 +1084,16 @@ def verify_emulator(
|
||||
data_dir_notices.append(ref)
|
||||
|
||||
if not files:
|
||||
details.append({
|
||||
"name": f"({emu_name})", "status": Status.OK,
|
||||
"required": False, "system": "",
|
||||
"note": f"No files needed for {profile.get('emulator', emu_name)}",
|
||||
"ground_truth": [],
|
||||
})
|
||||
details.append(
|
||||
{
|
||||
"name": f"({emu_name})",
|
||||
"status": Status.OK,
|
||||
"required": False,
|
||||
"system": "",
|
||||
"note": f"No files needed for {profile.get('emulator', emu_name)}",
|
||||
"ground_truth": [],
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Verify archives as units (e.g., neogeo.zip, aes.zip)
|
||||
@@ -940,7 +1104,9 @@ def verify_emulator(
|
||||
seen_archives.add(archive)
|
||||
archive_entry = {"name": archive}
|
||||
local_path, _ = resolve_local_file(
|
||||
archive_entry, db, zip_contents,
|
||||
archive_entry,
|
||||
db,
|
||||
zip_contents,
|
||||
data_dir_registry=data_registry,
|
||||
)
|
||||
required = any(
|
||||
@@ -948,11 +1114,18 @@ def verify_emulator(
|
||||
for f in files
|
||||
)
|
||||
if local_path:
|
||||
result = {"name": archive, "status": Status.OK,
|
||||
"required": required, "path": local_path}
|
||||
result = {
|
||||
"name": archive,
|
||||
"status": Status.OK,
|
||||
"required": required,
|
||||
"path": local_path,
|
||||
}
|
||||
else:
|
||||
result = {"name": archive, "status": Status.MISSING,
|
||||
"required": required}
|
||||
result = {
|
||||
"name": archive,
|
||||
"status": Status.MISSING,
|
||||
"required": required,
|
||||
}
|
||||
result["system"] = file_entry.get("system", "")
|
||||
result["hle_fallback"] = False
|
||||
result["ground_truth"] = build_ground_truth(archive, validation_index)
|
||||
@@ -961,11 +1134,15 @@ def verify_emulator(
|
||||
dest_to_name[dest] = archive
|
||||
cur = result["status"]
|
||||
prev = file_status.get(dest)
|
||||
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
|
||||
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(
|
||||
prev, 0
|
||||
):
|
||||
file_status[dest] = cur
|
||||
sev = compute_severity(cur, required, "existence", False)
|
||||
prev_sev = file_severity.get(dest)
|
||||
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(prev_sev, 0):
|
||||
if prev_sev is None or _SEVERITY_ORDER.get(
|
||||
sev, 0
|
||||
) > _SEVERITY_ORDER.get(prev_sev, 0):
|
||||
file_severity[dest] = sev
|
||||
|
||||
for file_entry in files:
|
||||
@@ -975,7 +1152,10 @@ def verify_emulator(
|
||||
|
||||
dest_hint = file_entry.get("path", "")
|
||||
local_path, resolve_status = resolve_local_file(
|
||||
file_entry, db, zip_contents, dest_hint=dest_hint,
|
||||
file_entry,
|
||||
db,
|
||||
zip_contents,
|
||||
dest_hint=dest_hint,
|
||||
data_dir_registry=data_registry,
|
||||
)
|
||||
name = file_entry.get("name", "")
|
||||
@@ -988,12 +1168,20 @@ def verify_emulator(
|
||||
# Apply emulator validation
|
||||
reason = check_file_validation(local_path, name, validation_index)
|
||||
if reason:
|
||||
result = {"name": name, "status": Status.UNTESTED,
|
||||
"required": required, "path": local_path,
|
||||
"reason": reason}
|
||||
result = {
|
||||
"name": name,
|
||||
"status": Status.UNTESTED,
|
||||
"required": required,
|
||||
"path": local_path,
|
||||
"reason": reason,
|
||||
}
|
||||
else:
|
||||
result = {"name": name, "status": Status.OK,
|
||||
"required": required, "path": local_path}
|
||||
result = {
|
||||
"name": name,
|
||||
"status": Status.OK,
|
||||
"required": required,
|
||||
"path": local_path,
|
||||
}
|
||||
|
||||
result["system"] = file_entry.get("system", "")
|
||||
result["hle_fallback"] = hle
|
||||
@@ -1009,10 +1197,17 @@ def verify_emulator(
|
||||
file_status[dest] = cur
|
||||
sev = compute_severity(cur, required, "existence", hle)
|
||||
prev_sev = file_severity.get(dest)
|
||||
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(prev_sev, 0):
|
||||
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(
|
||||
prev_sev, 0
|
||||
):
|
||||
file_severity[dest] = sev
|
||||
|
||||
counts = {Severity.OK: 0, Severity.INFO: 0, Severity.WARNING: 0, Severity.CRITICAL: 0}
|
||||
counts = {
|
||||
Severity.OK: 0,
|
||||
Severity.INFO: 0,
|
||||
Severity.WARNING: 0,
|
||||
Severity.CRITICAL: 0,
|
||||
}
|
||||
for s in file_severity.values():
|
||||
counts[s] = counts.get(s, 0) + 1
|
||||
status_counts: dict[str, int] = {}
|
||||
@@ -1067,13 +1262,19 @@ def verify_system(
|
||||
for p in profiles.values():
|
||||
all_systems.update(p.get("systems", []))
|
||||
if standalone:
|
||||
print(f"No standalone emulators found for system(s): {', '.join(system_ids)}",
|
||||
file=sys.stderr)
|
||||
print(
|
||||
f"No standalone emulators found for system(s): {', '.join(system_ids)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
print(f"No emulators found for system(s): {', '.join(system_ids)}",
|
||||
file=sys.stderr)
|
||||
print(f"Available systems: {', '.join(sorted(all_systems)[:20])}...",
|
||||
file=sys.stderr)
|
||||
print(
|
||||
f"No emulators found for system(s): {', '.join(system_ids)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f"Available systems: {', '.join(sorted(all_systems)[:20])}...",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
return verify_emulator(matching, emulators_dir, db, standalone)
|
||||
@@ -1147,13 +1348,17 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
|
||||
print(f" {line}")
|
||||
|
||||
for ref in result.get("data_dir_notices", []):
|
||||
print(f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)")
|
||||
print(
|
||||
f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)"
|
||||
)
|
||||
|
||||
# Ground truth coverage footer
|
||||
gt_cov = result.get("ground_truth_coverage")
|
||||
if gt_cov and gt_cov["total"] > 0:
|
||||
pct = gt_cov["with_validation"] * 100 // gt_cov["total"]
|
||||
print(f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)")
|
||||
print(
|
||||
f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)"
|
||||
)
|
||||
if gt_cov["platform_only"]:
|
||||
print(f" {gt_cov['platform_only']} platform-only (no emulator profile)")
|
||||
|
||||
@@ -1161,19 +1366,36 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Platform-native BIOS verification")
|
||||
parser.add_argument("--platform", "-p", help="Platform name")
|
||||
parser.add_argument("--all", action="store_true", help="Verify all active platforms")
|
||||
parser.add_argument("--emulator", "-e", help="Emulator profile name(s), comma-separated")
|
||||
parser.add_argument(
|
||||
"--all", action="store_true", help="Verify all active platforms"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--emulator", "-e", help="Emulator profile name(s), comma-separated"
|
||||
)
|
||||
parser.add_argument("--system", "-s", help="System ID(s), comma-separated")
|
||||
parser.add_argument("--standalone", action="store_true", help="Use standalone mode")
|
||||
parser.add_argument("--list-emulators", action="store_true", help="List available emulators")
|
||||
parser.add_argument("--list-systems", action="store_true", help="List available systems")
|
||||
parser.add_argument(
|
||||
"--list-emulators", action="store_true", help="List available emulators"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list-systems", action="store_true", help="List available systems"
|
||||
)
|
||||
parser.add_argument("--include-archived", action="store_true")
|
||||
parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)")
|
||||
parser.add_argument("--list-targets", action="store_true", help="List available targets for the platform")
|
||||
parser.add_argument(
|
||||
"--list-targets",
|
||||
action="store_true",
|
||||
help="List available targets for the platform",
|
||||
)
|
||||
parser.add_argument("--db", default=DEFAULT_DB)
|
||||
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
|
||||
parser.add_argument("--emulators-dir", default=DEFAULT_EMULATORS_DIR)
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Show emulator ground truth details")
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
"-v",
|
||||
action="store_true",
|
||||
help="Show emulator ground truth details",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="JSON output")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -1188,13 +1410,16 @@ def main():
|
||||
if not args.platform:
|
||||
parser.error("--list-targets requires --platform")
|
||||
from common import list_available_targets
|
||||
|
||||
targets = list_available_targets(args.platform, args.platforms_dir)
|
||||
if not targets:
|
||||
print(f"No targets configured for platform '{args.platform}'")
|
||||
return
|
||||
for t in targets:
|
||||
aliases = f" (aliases: {', '.join(t['aliases'])})" if t['aliases'] else ""
|
||||
print(f" {t['name']:30s} {t['architecture']:10s} {t['core_count']:>4d} cores{aliases}")
|
||||
aliases = f" (aliases: {', '.join(t['aliases'])})" if t["aliases"] else ""
|
||||
print(
|
||||
f" {t['name']:30s} {t['architecture']:10s} {t['core_count']:>4d} cores{aliases}"
|
||||
)
|
||||
return
|
||||
|
||||
# Mutual exclusion
|
||||
@@ -1202,7 +1427,9 @@ def main():
|
||||
if modes == 0:
|
||||
parser.error("Specify --platform, --all, --emulator, or --system")
|
||||
if modes > 1:
|
||||
parser.error("--platform, --all, --emulator, and --system are mutually exclusive")
|
||||
parser.error(
|
||||
"--platform, --all, --emulator, and --system are mutually exclusive"
|
||||
)
|
||||
if args.standalone and not (args.emulator or args.system):
|
||||
parser.error("--standalone requires --emulator or --system")
|
||||
if args.target and not (args.platform or args.all):
|
||||
@@ -1218,7 +1445,9 @@ def main():
|
||||
names = [n.strip() for n in args.emulator.split(",") if n.strip()]
|
||||
result = verify_emulator(names, args.emulators_dir, db, args.standalone)
|
||||
if args.json:
|
||||
result["details"] = [d for d in result["details"] if d["status"] != Status.OK]
|
||||
result["details"] = [
|
||||
d for d in result["details"] if d["status"] != Status.OK
|
||||
]
|
||||
print(json.dumps(result, indent=2))
|
||||
else:
|
||||
print_emulator_result(result, verbose=args.verbose)
|
||||
@@ -1229,7 +1458,9 @@ def main():
|
||||
system_ids = [s.strip() for s in args.system.split(",") if s.strip()]
|
||||
result = verify_system(system_ids, args.emulators_dir, db, args.standalone)
|
||||
if args.json:
|
||||
result["details"] = [d for d in result["details"] if d["status"] != Status.OK]
|
||||
result["details"] = [
|
||||
d for d in result["details"] if d["status"] != Status.OK
|
||||
]
|
||||
print(json.dumps(result, indent=2))
|
||||
else:
|
||||
print_emulator_result(result, verbose=args.verbose)
|
||||
@@ -1238,6 +1469,7 @@ def main():
|
||||
# Platform mode (existing)
|
||||
if args.all:
|
||||
from list_platforms import list_platforms as _list_platforms
|
||||
|
||||
platforms = _list_platforms(include_archived=args.include_archived)
|
||||
elif args.platform:
|
||||
platforms = [args.platform]
|
||||
@@ -1253,16 +1485,21 @@ def main():
|
||||
if args.target:
|
||||
try:
|
||||
target_cores_cache, platforms = build_target_cores_cache(
|
||||
platforms, args.target, args.platforms_dir, is_all=args.all,
|
||||
platforms,
|
||||
args.target,
|
||||
args.platforms_dir,
|
||||
is_all=args.all,
|
||||
)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Group identical platforms (same function as generate_pack)
|
||||
groups = group_identical_platforms(platforms, args.platforms_dir,
|
||||
target_cores_cache if args.target else None)
|
||||
groups = group_identical_platforms(
|
||||
platforms, args.platforms_dir, target_cores_cache if args.target else None
|
||||
)
|
||||
from cross_reference import _build_supplemental_index
|
||||
|
||||
suppl_names = _build_supplemental_index()
|
||||
|
||||
all_results = {}
|
||||
@@ -1271,11 +1508,18 @@ def main():
|
||||
config = load_platform_config(representative, args.platforms_dir)
|
||||
tc = target_cores_cache.get(representative) if args.target else None
|
||||
result = verify_platform(
|
||||
config, db, args.emulators_dir, emu_profiles,
|
||||
target_cores=tc, data_dir_registry=data_registry,
|
||||
config,
|
||||
db,
|
||||
args.emulators_dir,
|
||||
emu_profiles,
|
||||
target_cores=tc,
|
||||
data_dir_registry=data_registry,
|
||||
supplemental_names=suppl_names,
|
||||
)
|
||||
names = [load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms]
|
||||
names = [
|
||||
load_platform_config(p, args.platforms_dir).get("platform", p)
|
||||
for p in group_platforms
|
||||
]
|
||||
group_results.append((result, names))
|
||||
for p in group_platforms:
|
||||
all_results[p] = result
|
||||
|
||||
1831
tests/test_e2e.py
1831
tests/test_e2e.py
File diff suppressed because it is too large
Load Diff
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -84,87 +83,84 @@ struct BurnDriver BurnDrvmslug = {
|
||||
|
||||
|
||||
class TestFindBiosSets(unittest.TestCase):
|
||||
|
||||
def test_detects_neogeo(self) -> None:
|
||||
result = find_bios_sets(NEOGEO_FIXTURE, 'd_neogeo.cpp')
|
||||
self.assertIn('neogeo', result)
|
||||
self.assertEqual(result['neogeo']['source_file'], 'd_neogeo.cpp')
|
||||
result = find_bios_sets(NEOGEO_FIXTURE, "d_neogeo.cpp")
|
||||
self.assertIn("neogeo", result)
|
||||
self.assertEqual(result["neogeo"]["source_file"], "d_neogeo.cpp")
|
||||
|
||||
def test_detects_pgm(self) -> None:
|
||||
result = find_bios_sets(PGM_FIXTURE, 'd_pgm.cpp')
|
||||
self.assertIn('pgm', result)
|
||||
self.assertEqual(result['pgm']['source_file'], 'd_pgm.cpp')
|
||||
result = find_bios_sets(PGM_FIXTURE, "d_pgm.cpp")
|
||||
self.assertIn("pgm", result)
|
||||
self.assertEqual(result["pgm"]["source_file"], "d_pgm.cpp")
|
||||
|
||||
def test_ignores_non_bios(self) -> None:
|
||||
result = find_bios_sets(NON_BIOS_FIXTURE, 'd_neogeo.cpp')
|
||||
result = find_bios_sets(NON_BIOS_FIXTURE, "d_neogeo.cpp")
|
||||
self.assertEqual(result, {})
|
||||
|
||||
def test_source_line_positive(self) -> None:
|
||||
result = find_bios_sets(NEOGEO_FIXTURE, 'd_neogeo.cpp')
|
||||
self.assertGreater(result['neogeo']['source_line'], 0)
|
||||
result = find_bios_sets(NEOGEO_FIXTURE, "d_neogeo.cpp")
|
||||
self.assertGreater(result["neogeo"]["source_line"], 0)
|
||||
|
||||
|
||||
class TestParseRomInfo(unittest.TestCase):
|
||||
|
||||
def test_neogeo_rom_count(self) -> None:
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo')
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
|
||||
self.assertEqual(len(roms), 5)
|
||||
|
||||
def test_sentinel_skipped(self) -> None:
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo')
|
||||
names = [r['name'] for r in roms]
|
||||
self.assertNotIn('', names)
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
|
||||
names = [r["name"] for r in roms]
|
||||
self.assertNotIn("", names)
|
||||
|
||||
def test_crc32_lowercase_hex(self) -> None:
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo')
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
|
||||
first = roms[0]
|
||||
self.assertEqual(first['crc32'], '9036d879')
|
||||
self.assertRegex(first['crc32'], r'^[0-9a-f]{8}$')
|
||||
self.assertEqual(first["crc32"], "9036d879")
|
||||
self.assertRegex(first["crc32"], r"^[0-9a-f]{8}$")
|
||||
|
||||
def test_no_sha1(self) -> None:
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo')
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
|
||||
for rom in roms:
|
||||
self.assertNotIn('sha1', rom)
|
||||
self.assertNotIn("sha1", rom)
|
||||
|
||||
def test_neogeo_first_rom(self) -> None:
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, 'neogeo')
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, "neogeo")
|
||||
first = roms[0]
|
||||
self.assertEqual(first['name'], 'sp-s2.sp1')
|
||||
self.assertEqual(first['size'], 0x020000)
|
||||
self.assertEqual(first['crc32'], '9036d879')
|
||||
self.assertEqual(first["name"], "sp-s2.sp1")
|
||||
self.assertEqual(first["size"], 0x020000)
|
||||
self.assertEqual(first["crc32"], "9036d879")
|
||||
|
||||
def test_pgm_rom_count(self) -> None:
|
||||
roms = parse_rom_info(PGM_FIXTURE, 'pgm')
|
||||
roms = parse_rom_info(PGM_FIXTURE, "pgm")
|
||||
self.assertEqual(len(roms), 3)
|
||||
|
||||
def test_pgm_bios_entry(self) -> None:
|
||||
roms = parse_rom_info(PGM_FIXTURE, 'pgm')
|
||||
roms = parse_rom_info(PGM_FIXTURE, "pgm")
|
||||
bios = roms[2]
|
||||
self.assertEqual(bios['name'], 'pgm_p01s.rom')
|
||||
self.assertEqual(bios['crc32'], 'e42b166e')
|
||||
self.assertEqual(bios["name"], "pgm_p01s.rom")
|
||||
self.assertEqual(bios["crc32"], "e42b166e")
|
||||
|
||||
def test_unknown_set_returns_empty(self) -> None:
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, 'nonexistent')
|
||||
roms = parse_rom_info(NEOGEO_FIXTURE, "nonexistent")
|
||||
self.assertEqual(roms, [])
|
||||
|
||||
|
||||
class TestParseSourceTree(unittest.TestCase):
|
||||
|
||||
def test_walks_drv_directory(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
drv_dir = Path(tmpdir) / 'src' / 'burn' / 'drv' / 'neogeo'
|
||||
drv_dir = Path(tmpdir) / "src" / "burn" / "drv" / "neogeo"
|
||||
drv_dir.mkdir(parents=True)
|
||||
(drv_dir / 'd_neogeo.cpp').write_text(NEOGEO_FIXTURE)
|
||||
(drv_dir / "d_neogeo.cpp").write_text(NEOGEO_FIXTURE)
|
||||
|
||||
result = parse_fbneo_source_tree(tmpdir)
|
||||
self.assertIn('neogeo', result)
|
||||
self.assertEqual(len(result['neogeo']['roms']), 5)
|
||||
self.assertIn("neogeo", result)
|
||||
self.assertEqual(len(result["neogeo"]["roms"]), 5)
|
||||
|
||||
def test_skips_non_cpp(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
drv_dir = Path(tmpdir) / 'src' / 'burn' / 'drv'
|
||||
drv_dir = Path(tmpdir) / "src" / "burn" / "drv"
|
||||
drv_dir.mkdir(parents=True)
|
||||
(drv_dir / 'd_neogeo.h').write_text(NEOGEO_FIXTURE)
|
||||
(drv_dir / "d_neogeo.h").write_text(NEOGEO_FIXTURE)
|
||||
|
||||
result = parse_fbneo_source_tree(tmpdir)
|
||||
self.assertEqual(result, {})
|
||||
@@ -175,16 +171,16 @@ class TestParseSourceTree(unittest.TestCase):
|
||||
self.assertEqual(result, {})
|
||||
|
||||
def test_multiple_sets(self) -> None:
|
||||
combined = NEOGEO_FIXTURE + '\n' + PGM_FIXTURE
|
||||
combined = NEOGEO_FIXTURE + "\n" + PGM_FIXTURE
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
drv_dir = Path(tmpdir) / 'src' / 'burn' / 'drv'
|
||||
drv_dir = Path(tmpdir) / "src" / "burn" / "drv"
|
||||
drv_dir.mkdir(parents=True)
|
||||
(drv_dir / 'd_combined.cpp').write_text(combined)
|
||||
(drv_dir / "d_combined.cpp").write_text(combined)
|
||||
|
||||
result = parse_fbneo_source_tree(tmpdir)
|
||||
self.assertIn('neogeo', result)
|
||||
self.assertIn('pgm', result)
|
||||
self.assertIn("neogeo", result)
|
||||
self.assertIn("pgm", result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -18,35 +18,35 @@ from scripts.scraper._hash_merge import (
|
||||
|
||||
def _write_yaml(path: Path, data: dict) -> str:
|
||||
p = str(path)
|
||||
with open(p, 'w', encoding='utf-8') as f:
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
yaml.dump(data, f, default_flow_style=False, sort_keys=False)
|
||||
return p
|
||||
|
||||
|
||||
def _write_json(path: Path, data: dict) -> str:
|
||||
p = str(path)
|
||||
with open(p, 'w', encoding='utf-8') as f:
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f)
|
||||
return p
|
||||
|
||||
|
||||
def _make_mame_profile(**overrides: object) -> dict:
|
||||
base = {
|
||||
'emulator': 'MAME',
|
||||
'core_version': '0.285',
|
||||
'files': [
|
||||
"emulator": "MAME",
|
||||
"core_version": "0.285",
|
||||
"files": [
|
||||
{
|
||||
'name': 'neogeo.zip',
|
||||
'required': True,
|
||||
'category': 'bios_zip',
|
||||
'system': 'snk-neogeo-mvs',
|
||||
'source_ref': 'src/mame/neogeo/neogeo.cpp:2400',
|
||||
'contents': [
|
||||
"name": "neogeo.zip",
|
||||
"required": True,
|
||||
"category": "bios_zip",
|
||||
"system": "snk-neogeo-mvs",
|
||||
"source_ref": "src/mame/neogeo/neogeo.cpp:2400",
|
||||
"contents": [
|
||||
{
|
||||
'name': 'sp-s2.sp1',
|
||||
'size': 131072,
|
||||
'crc32': 'oldcrc32',
|
||||
'description': 'Europe MVS (Ver. 2)',
|
||||
"name": "sp-s2.sp1",
|
||||
"size": 131072,
|
||||
"crc32": "oldcrc32",
|
||||
"description": "Europe MVS (Ver. 2)",
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -58,23 +58,23 @@ def _make_mame_profile(**overrides: object) -> dict:
|
||||
|
||||
def _make_mame_hashes(**overrides: object) -> dict:
|
||||
base = {
|
||||
'source': 'mamedev/mame',
|
||||
'version': '0.286',
|
||||
'commit': 'abc123',
|
||||
'fetched_at': '2026-03-30T12:00:00Z',
|
||||
'bios_sets': {
|
||||
'neogeo': {
|
||||
'source_file': 'src/mame/neogeo/neogeo.cpp',
|
||||
'source_line': 2432,
|
||||
'roms': [
|
||||
"source": "mamedev/mame",
|
||||
"version": "0.286",
|
||||
"commit": "abc123",
|
||||
"fetched_at": "2026-03-30T12:00:00Z",
|
||||
"bios_sets": {
|
||||
"neogeo": {
|
||||
"source_file": "src/mame/neogeo/neogeo.cpp",
|
||||
"source_line": 2432,
|
||||
"roms": [
|
||||
{
|
||||
'name': 'sp-s2.sp1',
|
||||
'size': 131072,
|
||||
'crc32': '9036d879',
|
||||
'sha1': '4f834c55',
|
||||
'region': 'mainbios',
|
||||
'bios_label': 'euro',
|
||||
'bios_description': 'Europe MVS (Ver. 2)',
|
||||
"name": "sp-s2.sp1",
|
||||
"size": 131072,
|
||||
"crc32": "9036d879",
|
||||
"sha1": "4f834c55",
|
||||
"region": "mainbios",
|
||||
"bios_label": "euro",
|
||||
"bios_description": "Europe MVS (Ver. 2)",
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -86,21 +86,21 @@ def _make_mame_hashes(**overrides: object) -> dict:
|
||||
|
||||
def _make_fbneo_profile(**overrides: object) -> dict:
|
||||
base = {
|
||||
'emulator': 'FinalBurn Neo',
|
||||
'core_version': 'v1.0.0.02',
|
||||
'files': [
|
||||
"emulator": "FinalBurn Neo",
|
||||
"core_version": "v1.0.0.02",
|
||||
"files": [
|
||||
{
|
||||
'name': 'sp-s2.sp1',
|
||||
'archive': 'neogeo.zip',
|
||||
'system': 'snk-neogeo-mvs',
|
||||
'required': True,
|
||||
'size': 131072,
|
||||
'crc32': 'oldcrc32',
|
||||
'source_ref': 'src/burn/drv/neogeo/d_neogeo.cpp:1605',
|
||||
"name": "sp-s2.sp1",
|
||||
"archive": "neogeo.zip",
|
||||
"system": "snk-neogeo-mvs",
|
||||
"required": True,
|
||||
"size": 131072,
|
||||
"crc32": "oldcrc32",
|
||||
"source_ref": "src/burn/drv/neogeo/d_neogeo.cpp:1605",
|
||||
},
|
||||
{
|
||||
'name': 'hiscore.dat',
|
||||
'required': False,
|
||||
"name": "hiscore.dat",
|
||||
"required": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
@@ -110,20 +110,20 @@ def _make_fbneo_profile(**overrides: object) -> dict:
|
||||
|
||||
def _make_fbneo_hashes(**overrides: object) -> dict:
|
||||
base = {
|
||||
'source': 'finalburnneo/FBNeo',
|
||||
'version': 'v1.0.0.03',
|
||||
'commit': 'def456',
|
||||
'fetched_at': '2026-03-30T12:00:00Z',
|
||||
'bios_sets': {
|
||||
'neogeo': {
|
||||
'source_file': 'src/burn/drv/neogeo/d_neogeo.cpp',
|
||||
'source_line': 1604,
|
||||
'roms': [
|
||||
"source": "finalburnneo/FBNeo",
|
||||
"version": "v1.0.0.03",
|
||||
"commit": "def456",
|
||||
"fetched_at": "2026-03-30T12:00:00Z",
|
||||
"bios_sets": {
|
||||
"neogeo": {
|
||||
"source_file": "src/burn/drv/neogeo/d_neogeo.cpp",
|
||||
"source_line": 1604,
|
||||
"roms": [
|
||||
{
|
||||
'name': 'sp-s2.sp1',
|
||||
'size': 131072,
|
||||
'crc32': '9036d879',
|
||||
'sha1': 'aabbccdd',
|
||||
"name": "sp-s2.sp1",
|
||||
"size": 131072,
|
||||
"crc32": "9036d879",
|
||||
"sha1": "aabbccdd",
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -139,129 +139,129 @@ class TestMameMerge(unittest.TestCase):
|
||||
def test_merge_updates_contents(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes())
|
||||
profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
|
||||
|
||||
result = merge_mame_profile(profile_path, hashes_path)
|
||||
|
||||
bios_files = [f for f in result['files'] if f.get('category') == 'bios_zip']
|
||||
bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"]
|
||||
self.assertEqual(len(bios_files), 1)
|
||||
contents = bios_files[0]['contents']
|
||||
self.assertEqual(contents[0]['crc32'], '9036d879')
|
||||
self.assertEqual(contents[0]['sha1'], '4f834c55')
|
||||
self.assertEqual(contents[0]['description'], 'Europe MVS (Ver. 2)')
|
||||
contents = bios_files[0]["contents"]
|
||||
self.assertEqual(contents[0]["crc32"], "9036d879")
|
||||
self.assertEqual(contents[0]["sha1"], "4f834c55")
|
||||
self.assertEqual(contents[0]["description"], "Europe MVS (Ver. 2)")
|
||||
|
||||
def test_merge_preserves_manual_fields(self) -> None:
|
||||
profile = _make_mame_profile()
|
||||
profile['files'][0]['note'] = 'manually curated note'
|
||||
profile['files'][0]['system'] = 'snk-neogeo-mvs'
|
||||
profile['files'][0]['required'] = False
|
||||
profile["files"][0]["note"] = "manually curated note"
|
||||
profile["files"][0]["system"] = "snk-neogeo-mvs"
|
||||
profile["files"][0]["required"] = False
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', profile)
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes())
|
||||
profile_path = _write_yaml(p / "mame.yml", profile)
|
||||
hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
|
||||
|
||||
result = merge_mame_profile(profile_path, hashes_path)
|
||||
|
||||
entry = [f for f in result['files'] if f.get('category') == 'bios_zip'][0]
|
||||
self.assertEqual(entry['note'], 'manually curated note')
|
||||
self.assertEqual(entry['system'], 'snk-neogeo-mvs')
|
||||
self.assertFalse(entry['required'])
|
||||
entry = [f for f in result["files"] if f.get("category") == "bios_zip"][0]
|
||||
self.assertEqual(entry["note"], "manually curated note")
|
||||
self.assertEqual(entry["system"], "snk-neogeo-mvs")
|
||||
self.assertFalse(entry["required"])
|
||||
|
||||
def test_merge_adds_new_bios_set(self) -> None:
|
||||
hashes = _make_mame_hashes()
|
||||
hashes['bios_sets']['pgm'] = {
|
||||
'source_file': 'src/mame/igs/pgm.cpp',
|
||||
'source_line': 5515,
|
||||
'roms': [
|
||||
{'name': 'pgm_t01s.rom', 'size': 2097152, 'crc32': '1a7123a0'},
|
||||
hashes["bios_sets"]["pgm"] = {
|
||||
"source_file": "src/mame/igs/pgm.cpp",
|
||||
"source_line": 5515,
|
||||
"roms": [
|
||||
{"name": "pgm_t01s.rom", "size": 2097152, "crc32": "1a7123a0"},
|
||||
],
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', hashes)
|
||||
profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", hashes)
|
||||
|
||||
result = merge_mame_profile(profile_path, hashes_path)
|
||||
|
||||
bios_files = [f for f in result['files'] if f.get('category') == 'bios_zip']
|
||||
names = {f['name'] for f in bios_files}
|
||||
self.assertIn('pgm.zip', names)
|
||||
bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"]
|
||||
names = {f["name"] for f in bios_files}
|
||||
self.assertIn("pgm.zip", names)
|
||||
|
||||
pgm = next(f for f in bios_files if f['name'] == 'pgm.zip')
|
||||
self.assertIsNone(pgm['system'])
|
||||
self.assertTrue(pgm['required'])
|
||||
self.assertEqual(pgm['category'], 'bios_zip')
|
||||
pgm = next(f for f in bios_files if f["name"] == "pgm.zip")
|
||||
self.assertIsNone(pgm["system"])
|
||||
self.assertTrue(pgm["required"])
|
||||
self.assertEqual(pgm["category"], "bios_zip")
|
||||
|
||||
def test_merge_preserves_non_bios_files(self) -> None:
|
||||
profile = _make_mame_profile()
|
||||
profile['files'].append({'name': 'hiscore.dat', 'required': False})
|
||||
profile["files"].append({"name": "hiscore.dat", "required": False})
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', profile)
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes())
|
||||
profile_path = _write_yaml(p / "mame.yml", profile)
|
||||
hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
|
||||
|
||||
result = merge_mame_profile(profile_path, hashes_path)
|
||||
|
||||
non_bios = [f for f in result['files'] if f.get('category') != 'bios_zip']
|
||||
non_bios = [f for f in result["files"] if f.get("category") != "bios_zip"]
|
||||
self.assertEqual(len(non_bios), 1)
|
||||
self.assertEqual(non_bios[0]['name'], 'hiscore.dat')
|
||||
self.assertEqual(non_bios[0]["name"], "hiscore.dat")
|
||||
|
||||
def test_merge_keeps_unmatched_bios_set(self) -> None:
|
||||
"""Entries not in scraper scope stay untouched (no _upstream_removed)."""
|
||||
hashes = _make_mame_hashes()
|
||||
hashes['bios_sets'] = {} # nothing from scraper
|
||||
hashes["bios_sets"] = {} # nothing from scraper
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', hashes)
|
||||
profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", hashes)
|
||||
|
||||
result = merge_mame_profile(profile_path, hashes_path)
|
||||
|
||||
bios_files = [f for f in result['files'] if f.get('category') == 'bios_zip']
|
||||
bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"]
|
||||
self.assertEqual(len(bios_files), 1)
|
||||
self.assertNotIn('_upstream_removed', bios_files[0])
|
||||
self.assertEqual(bios_files[0]['name'], 'neogeo.zip')
|
||||
self.assertNotIn("_upstream_removed", bios_files[0])
|
||||
self.assertEqual(bios_files[0]["name"], "neogeo.zip")
|
||||
|
||||
def test_merge_updates_core_version(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes())
|
||||
profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
|
||||
|
||||
result = merge_mame_profile(profile_path, hashes_path)
|
||||
|
||||
self.assertEqual(result['core_version'], '0.286')
|
||||
self.assertEqual(result["core_version"], "0.286")
|
||||
|
||||
def test_merge_backup_created(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes())
|
||||
profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
|
||||
|
||||
merge_mame_profile(profile_path, hashes_path, write=True)
|
||||
|
||||
backup = p / 'mame.old.yml'
|
||||
backup = p / "mame.old.yml"
|
||||
self.assertTrue(backup.exists())
|
||||
|
||||
with open(backup, encoding='utf-8') as f:
|
||||
with open(backup, encoding="utf-8") as f:
|
||||
old = yaml.safe_load(f)
|
||||
self.assertEqual(old['core_version'], '0.285')
|
||||
self.assertEqual(old["core_version"], "0.285")
|
||||
|
||||
def test_merge_updates_source_ref(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_mame_hashes())
|
||||
profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", _make_mame_hashes())
|
||||
|
||||
result = merge_mame_profile(profile_path, hashes_path)
|
||||
|
||||
entry = [f for f in result['files'] if f.get('category') == 'bios_zip'][0]
|
||||
self.assertEqual(entry['source_ref'], 'src/mame/neogeo/neogeo.cpp:2432')
|
||||
entry = [f for f in result["files"] if f.get("category") == "bios_zip"][0]
|
||||
self.assertEqual(entry["source_ref"], "src/mame/neogeo/neogeo.cpp:2432")
|
||||
|
||||
|
||||
class TestFbneoMerge(unittest.TestCase):
|
||||
@@ -270,74 +270,76 @@ class TestFbneoMerge(unittest.TestCase):
|
||||
def test_merge_updates_rom_entries(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_fbneo_hashes())
|
||||
profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes())
|
||||
|
||||
result = merge_fbneo_profile(profile_path, hashes_path)
|
||||
|
||||
archive_files = [f for f in result['files'] if 'archive' in f]
|
||||
archive_files = [f for f in result["files"] if "archive" in f]
|
||||
self.assertEqual(len(archive_files), 1)
|
||||
self.assertEqual(archive_files[0]['crc32'], '9036d879')
|
||||
self.assertEqual(archive_files[0]['system'], 'snk-neogeo-mvs')
|
||||
self.assertEqual(archive_files[0]["crc32"], "9036d879")
|
||||
self.assertEqual(archive_files[0]["system"], "snk-neogeo-mvs")
|
||||
|
||||
def test_merge_adds_new_roms(self) -> None:
|
||||
hashes = _make_fbneo_hashes()
|
||||
hashes['bios_sets']['neogeo']['roms'].append({
|
||||
'name': 'sp-s3.sp1',
|
||||
'size': 131072,
|
||||
'crc32': '91b64be3',
|
||||
})
|
||||
hashes["bios_sets"]["neogeo"]["roms"].append(
|
||||
{
|
||||
"name": "sp-s3.sp1",
|
||||
"size": 131072,
|
||||
"crc32": "91b64be3",
|
||||
}
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', hashes)
|
||||
profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", hashes)
|
||||
|
||||
result = merge_fbneo_profile(profile_path, hashes_path)
|
||||
|
||||
archive_files = [f for f in result['files'] if 'archive' in f]
|
||||
archive_files = [f for f in result["files"] if "archive" in f]
|
||||
self.assertEqual(len(archive_files), 2)
|
||||
new_rom = next(f for f in archive_files if f['name'] == 'sp-s3.sp1')
|
||||
self.assertEqual(new_rom['archive'], 'neogeo.zip')
|
||||
self.assertTrue(new_rom['required'])
|
||||
new_rom = next(f for f in archive_files if f["name"] == "sp-s3.sp1")
|
||||
self.assertEqual(new_rom["archive"], "neogeo.zip")
|
||||
self.assertTrue(new_rom["required"])
|
||||
|
||||
def test_merge_preserves_non_archive_files(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_fbneo_hashes())
|
||||
profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes())
|
||||
|
||||
result = merge_fbneo_profile(profile_path, hashes_path)
|
||||
|
||||
non_archive = [f for f in result['files'] if 'archive' not in f]
|
||||
non_archive = [f for f in result["files"] if "archive" not in f]
|
||||
self.assertEqual(len(non_archive), 1)
|
||||
self.assertEqual(non_archive[0]['name'], 'hiscore.dat')
|
||||
self.assertEqual(non_archive[0]["name"], "hiscore.dat")
|
||||
|
||||
def test_merge_keeps_unmatched_roms(self) -> None:
|
||||
"""Entries not in scraper scope stay untouched (no _upstream_removed)."""
|
||||
hashes = _make_fbneo_hashes()
|
||||
hashes['bios_sets'] = {}
|
||||
hashes["bios_sets"] = {}
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', hashes)
|
||||
profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", hashes)
|
||||
|
||||
result = merge_fbneo_profile(profile_path, hashes_path)
|
||||
|
||||
archive_files = [f for f in result['files'] if 'archive' in f]
|
||||
archive_files = [f for f in result["files"] if "archive" in f]
|
||||
self.assertEqual(len(archive_files), 1)
|
||||
self.assertNotIn('_upstream_removed', archive_files[0])
|
||||
self.assertNotIn("_upstream_removed", archive_files[0])
|
||||
|
||||
def test_merge_updates_core_version(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', _make_fbneo_hashes())
|
||||
profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes())
|
||||
|
||||
result = merge_fbneo_profile(profile_path, hashes_path)
|
||||
|
||||
self.assertEqual(result['core_version'], 'v1.0.0.03')
|
||||
self.assertEqual(result["core_version"], "v1.0.0.03")
|
||||
|
||||
|
||||
class TestDiff(unittest.TestCase):
|
||||
@@ -345,79 +347,81 @@ class TestDiff(unittest.TestCase):
|
||||
|
||||
def test_diff_mame_detects_changes(self) -> None:
|
||||
hashes = _make_mame_hashes()
|
||||
hashes['bios_sets']['pgm'] = {
|
||||
'source_file': 'src/mame/igs/pgm.cpp',
|
||||
'source_line': 5515,
|
||||
'roms': [
|
||||
{'name': 'pgm_t01s.rom', 'size': 2097152, 'crc32': '1a7123a0'},
|
||||
hashes["bios_sets"]["pgm"] = {
|
||||
"source_file": "src/mame/igs/pgm.cpp",
|
||||
"source_line": 5515,
|
||||
"roms": [
|
||||
{"name": "pgm_t01s.rom", "size": 2097152, "crc32": "1a7123a0"},
|
||||
],
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', hashes)
|
||||
profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", hashes)
|
||||
|
||||
diff = compute_diff(profile_path, hashes_path, mode='mame')
|
||||
diff = compute_diff(profile_path, hashes_path, mode="mame")
|
||||
|
||||
self.assertIn('pgm', diff['added'])
|
||||
self.assertIn('neogeo', diff['updated'])
|
||||
self.assertEqual(len(diff['removed']), 0)
|
||||
self.assertEqual(diff['unchanged'], 0)
|
||||
self.assertIn("pgm", diff["added"])
|
||||
self.assertIn("neogeo", diff["updated"])
|
||||
self.assertEqual(len(diff["removed"]), 0)
|
||||
self.assertEqual(diff["unchanged"], 0)
|
||||
|
||||
def test_diff_mame_out_of_scope(self) -> None:
|
||||
"""Items in profile but not in scraper output = out of scope, not removed."""
|
||||
hashes = _make_mame_hashes()
|
||||
hashes['bios_sets'] = {}
|
||||
hashes["bios_sets"] = {}
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'mame.yml', _make_mame_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', hashes)
|
||||
profile_path = _write_yaml(p / "mame.yml", _make_mame_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", hashes)
|
||||
|
||||
diff = compute_diff(profile_path, hashes_path, mode='mame')
|
||||
diff = compute_diff(profile_path, hashes_path, mode="mame")
|
||||
|
||||
self.assertEqual(diff['removed'], [])
|
||||
self.assertEqual(diff['out_of_scope'], 1)
|
||||
self.assertEqual(len(diff['added']), 0)
|
||||
self.assertEqual(diff["removed"], [])
|
||||
self.assertEqual(diff["out_of_scope"], 1)
|
||||
self.assertEqual(len(diff["added"]), 0)
|
||||
|
||||
def test_diff_fbneo_detects_changes(self) -> None:
|
||||
hashes = _make_fbneo_hashes()
|
||||
hashes['bios_sets']['neogeo']['roms'].append({
|
||||
'name': 'sp-s3.sp1',
|
||||
'size': 131072,
|
||||
'crc32': '91b64be3',
|
||||
})
|
||||
hashes["bios_sets"]["neogeo"]["roms"].append(
|
||||
{
|
||||
"name": "sp-s3.sp1",
|
||||
"size": 131072,
|
||||
"crc32": "91b64be3",
|
||||
}
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'fbneo.yml', _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / 'hashes.json', hashes)
|
||||
profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile())
|
||||
hashes_path = _write_json(p / "hashes.json", hashes)
|
||||
|
||||
diff = compute_diff(profile_path, hashes_path, mode='fbneo')
|
||||
diff = compute_diff(profile_path, hashes_path, mode="fbneo")
|
||||
|
||||
self.assertIn('neogeo.zip:sp-s3.sp1', diff['added'])
|
||||
self.assertIn('neogeo.zip:sp-s2.sp1', diff['updated'])
|
||||
self.assertEqual(len(diff['removed']), 0)
|
||||
self.assertIn("neogeo.zip:sp-s3.sp1", diff["added"])
|
||||
self.assertIn("neogeo.zip:sp-s2.sp1", diff["updated"])
|
||||
self.assertEqual(len(diff["removed"]), 0)
|
||||
|
||||
def test_diff_fbneo_unchanged(self) -> None:
|
||||
profile = _make_fbneo_profile()
|
||||
profile['files'][0]['crc32'] = '9036d879'
|
||||
profile['files'][0]['size'] = 131072
|
||||
profile["files"][0]["crc32"] = "9036d879"
|
||||
profile["files"][0]["size"] = 131072
|
||||
|
||||
hashes = _make_fbneo_hashes()
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td)
|
||||
profile_path = _write_yaml(p / 'fbneo.yml', profile)
|
||||
hashes_path = _write_json(p / 'hashes.json', hashes)
|
||||
profile_path = _write_yaml(p / "fbneo.yml", profile)
|
||||
hashes_path = _write_json(p / "hashes.json", hashes)
|
||||
|
||||
diff = compute_diff(profile_path, hashes_path, mode='fbneo')
|
||||
diff = compute_diff(profile_path, hashes_path, mode="fbneo")
|
||||
|
||||
self.assertEqual(diff['unchanged'], 1)
|
||||
self.assertEqual(len(diff['added']), 0)
|
||||
self.assertEqual(len(diff['updated']), 0)
|
||||
self.assertEqual(diff["unchanged"], 1)
|
||||
self.assertEqual(len(diff["added"]), 0)
|
||||
self.assertEqual(len(diff["updated"]), 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -86,101 +86,101 @@ class TestFindBiosRootSets(unittest.TestCase):
|
||||
"""Tests for find_bios_root_sets."""
|
||||
|
||||
def test_detects_neogeo_from_game_macro(self) -> None:
|
||||
result = find_bios_root_sets(NEOGEO_FIXTURE, 'src/mame/snk/neogeo.cpp')
|
||||
self.assertIn('neogeo', result)
|
||||
self.assertEqual(result['neogeo']['source_file'], 'src/mame/snk/neogeo.cpp')
|
||||
self.assertIsInstance(result['neogeo']['source_line'], int)
|
||||
result = find_bios_root_sets(NEOGEO_FIXTURE, "src/mame/snk/neogeo.cpp")
|
||||
self.assertIn("neogeo", result)
|
||||
self.assertEqual(result["neogeo"]["source_file"], "src/mame/snk/neogeo.cpp")
|
||||
self.assertIsInstance(result["neogeo"]["source_line"], int)
|
||||
|
||||
def test_detects_from_comp_macro(self) -> None:
|
||||
result = find_bios_root_sets(DEVICE_FIXTURE, 'src/mame/acorn/bbc.cpp')
|
||||
self.assertIn('bbcb', result)
|
||||
result = find_bios_root_sets(DEVICE_FIXTURE, "src/mame/acorn/bbc.cpp")
|
||||
self.assertIn("bbcb", result)
|
||||
|
||||
def test_detects_from_cons_macro(self) -> None:
|
||||
result = find_bios_root_sets(CONS_FIXTURE, 'src/mame/sega/megadriv.cpp')
|
||||
self.assertIn('megadriv', result)
|
||||
result = find_bios_root_sets(CONS_FIXTURE, "src/mame/sega/megadriv.cpp")
|
||||
self.assertIn("megadriv", result)
|
||||
|
||||
def test_ignores_non_bios_games(self) -> None:
|
||||
result = find_bios_root_sets(NON_BIOS_FIXTURE, 'src/mame/pacman/pacman.cpp')
|
||||
result = find_bios_root_sets(NON_BIOS_FIXTURE, "src/mame/pacman/pacman.cpp")
|
||||
self.assertEqual(result, {})
|
||||
|
||||
def test_detects_from_nodump_fixture(self) -> None:
|
||||
result = find_bios_root_sets(NODUMP_FIXTURE, 'test.cpp')
|
||||
self.assertIn('testnd', result)
|
||||
result = find_bios_root_sets(NODUMP_FIXTURE, "test.cpp")
|
||||
self.assertIn("testnd", result)
|
||||
|
||||
def test_detects_from_baddump_fixture(self) -> None:
|
||||
result = find_bios_root_sets(BADDUMP_FIXTURE, 'test.cpp')
|
||||
self.assertIn('testbd', result)
|
||||
result = find_bios_root_sets(BADDUMP_FIXTURE, "test.cpp")
|
||||
self.assertIn("testbd", result)
|
||||
|
||||
|
||||
class TestParseRomBlock(unittest.TestCase):
|
||||
"""Tests for parse_rom_block."""
|
||||
|
||||
def test_extracts_rom_names(self) -> None:
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo')
|
||||
names = [r['name'] for r in roms]
|
||||
self.assertIn('sp-s2.sp1', names)
|
||||
self.assertIn('vs-bios.rom', names)
|
||||
self.assertIn('sm1.sm1', names)
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
|
||||
names = [r["name"] for r in roms]
|
||||
self.assertIn("sp-s2.sp1", names)
|
||||
self.assertIn("vs-bios.rom", names)
|
||||
self.assertIn("sm1.sm1", names)
|
||||
|
||||
def test_extracts_crc32_and_sha1(self) -> None:
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo')
|
||||
sp_s2 = next(r for r in roms if r['name'] == 'sp-s2.sp1')
|
||||
self.assertEqual(sp_s2['crc32'], '9036d879')
|
||||
self.assertEqual(sp_s2['sha1'], '4f5ed7105b7128794654ce82b51723e16e389543')
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
|
||||
sp_s2 = next(r for r in roms if r["name"] == "sp-s2.sp1")
|
||||
self.assertEqual(sp_s2["crc32"], "9036d879")
|
||||
self.assertEqual(sp_s2["sha1"], "4f5ed7105b7128794654ce82b51723e16e389543")
|
||||
|
||||
def test_extracts_size(self) -> None:
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo')
|
||||
sp_s2 = next(r for r in roms if r['name'] == 'sp-s2.sp1')
|
||||
self.assertEqual(sp_s2['size'], 0x020000)
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
|
||||
sp_s2 = next(r for r in roms if r["name"] == "sp-s2.sp1")
|
||||
self.assertEqual(sp_s2["size"], 0x020000)
|
||||
|
||||
def test_extracts_bios_metadata(self) -> None:
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo')
|
||||
sp_s2 = next(r for r in roms if r['name'] == 'sp-s2.sp1')
|
||||
self.assertEqual(sp_s2['bios_index'], 0)
|
||||
self.assertEqual(sp_s2['bios_label'], 'euro')
|
||||
self.assertEqual(sp_s2['bios_description'], 'Europe MVS (Ver. 2)')
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
|
||||
sp_s2 = next(r for r in roms if r["name"] == "sp-s2.sp1")
|
||||
self.assertEqual(sp_s2["bios_index"], 0)
|
||||
self.assertEqual(sp_s2["bios_label"], "euro")
|
||||
self.assertEqual(sp_s2["bios_description"], "Europe MVS (Ver. 2)")
|
||||
|
||||
def test_non_bios_rom_has_no_bios_fields(self) -> None:
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo')
|
||||
sm1 = next(r for r in roms if r['name'] == 'sm1.sm1')
|
||||
self.assertNotIn('bios_index', sm1)
|
||||
self.assertNotIn('bios_label', sm1)
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
|
||||
sm1 = next(r for r in roms if r["name"] == "sm1.sm1")
|
||||
self.assertNotIn("bios_index", sm1)
|
||||
self.assertNotIn("bios_label", sm1)
|
||||
|
||||
def test_skips_no_dump(self) -> None:
|
||||
roms = parse_rom_block(NODUMP_FIXTURE, 'testnd')
|
||||
names = [r['name'] for r in roms]
|
||||
self.assertIn('good.rom', names)
|
||||
self.assertNotIn('missing.rom', names)
|
||||
roms = parse_rom_block(NODUMP_FIXTURE, "testnd")
|
||||
names = [r["name"] for r in roms]
|
||||
self.assertIn("good.rom", names)
|
||||
self.assertNotIn("missing.rom", names)
|
||||
|
||||
def test_includes_bad_dump_with_flag(self) -> None:
|
||||
roms = parse_rom_block(BADDUMP_FIXTURE, 'testbd')
|
||||
roms = parse_rom_block(BADDUMP_FIXTURE, "testbd")
|
||||
self.assertEqual(len(roms), 1)
|
||||
self.assertEqual(roms[0]['name'], 'badrom.bin')
|
||||
self.assertTrue(roms[0]['bad_dump'])
|
||||
self.assertEqual(roms[0]['crc32'], 'deadbeef')
|
||||
self.assertEqual(roms[0]['sha1'], '0123456789abcdef0123456789abcdef01234567')
|
||||
self.assertEqual(roms[0]["name"], "badrom.bin")
|
||||
self.assertTrue(roms[0]["bad_dump"])
|
||||
self.assertEqual(roms[0]["crc32"], "deadbeef")
|
||||
self.assertEqual(roms[0]["sha1"], "0123456789abcdef0123456789abcdef01234567")
|
||||
|
||||
def test_handles_rom_load16_word(self) -> None:
|
||||
roms = parse_rom_block(CONS_FIXTURE, 'megadriv')
|
||||
roms = parse_rom_block(CONS_FIXTURE, "megadriv")
|
||||
self.assertEqual(len(roms), 1)
|
||||
self.assertEqual(roms[0]['name'], 'epr-6209.ic7')
|
||||
self.assertEqual(roms[0]['crc32'], 'cafebabe')
|
||||
self.assertEqual(roms[0]["name"], "epr-6209.ic7")
|
||||
self.assertEqual(roms[0]["crc32"], "cafebabe")
|
||||
|
||||
def test_tracks_rom_region(self) -> None:
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, 'neogeo')
|
||||
sp_s2 = next(r for r in roms if r['name'] == 'sp-s2.sp1')
|
||||
sm1 = next(r for r in roms if r['name'] == 'sm1.sm1')
|
||||
self.assertEqual(sp_s2['region'], 'mainbios')
|
||||
self.assertEqual(sm1['region'], 'audiocpu')
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, "neogeo")
|
||||
sp_s2 = next(r for r in roms if r["name"] == "sp-s2.sp1")
|
||||
sm1 = next(r for r in roms if r["name"] == "sm1.sm1")
|
||||
self.assertEqual(sp_s2["region"], "mainbios")
|
||||
self.assertEqual(sm1["region"], "audiocpu")
|
||||
|
||||
def test_returns_empty_for_unknown_set(self) -> None:
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, 'nonexistent')
|
||||
roms = parse_rom_block(NEOGEO_FIXTURE, "nonexistent")
|
||||
self.assertEqual(roms, [])
|
||||
|
||||
def test_good_rom_not_flagged_bad_dump(self) -> None:
|
||||
roms = parse_rom_block(NODUMP_FIXTURE, 'testnd')
|
||||
good = next(r for r in roms if r['name'] == 'good.rom')
|
||||
self.assertFalse(good['bad_dump'])
|
||||
roms = parse_rom_block(NODUMP_FIXTURE, "testnd")
|
||||
good = next(r for r in roms if r["name"] == "good.rom")
|
||||
self.assertFalse(good["bad_dump"])
|
||||
|
||||
def test_crc32_sha1_lowercase(self) -> None:
|
||||
fixture = """\
|
||||
@@ -189,9 +189,9 @@ ROM_START( upper )
|
||||
ROM_LOAD( "test.rom", 0x00000, 0x4000, CRC(AABBCCDD) SHA1(AABBCCDDEEFF00112233AABBCCDDEEFF00112233) )
|
||||
ROM_END
|
||||
"""
|
||||
roms = parse_rom_block(fixture, 'upper')
|
||||
self.assertEqual(roms[0]['crc32'], 'aabbccdd')
|
||||
self.assertEqual(roms[0]['sha1'], 'aabbccddeeff00112233aabbccddeeff00112233')
|
||||
roms = parse_rom_block(fixture, "upper")
|
||||
self.assertEqual(roms[0]["crc32"], "aabbccdd")
|
||||
self.assertEqual(roms[0]["sha1"], "aabbccddeeff00112233aabbccddeeff00112233")
|
||||
|
||||
|
||||
class TestParseMameSourceTree(unittest.TestCase):
|
||||
@@ -199,26 +199,26 @@ class TestParseMameSourceTree(unittest.TestCase):
|
||||
|
||||
def test_walks_source_tree(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
mame_dir = os.path.join(tmpdir, 'src', 'mame', 'snk')
|
||||
mame_dir = os.path.join(tmpdir, "src", "mame", "snk")
|
||||
os.makedirs(mame_dir)
|
||||
filepath = os.path.join(mame_dir, 'neogeo.cpp')
|
||||
with open(filepath, 'w') as f:
|
||||
filepath = os.path.join(mame_dir, "neogeo.cpp")
|
||||
with open(filepath, "w") as f:
|
||||
f.write(NEOGEO_FIXTURE)
|
||||
|
||||
results = parse_mame_source_tree(tmpdir)
|
||||
self.assertIn('neogeo', results)
|
||||
self.assertEqual(len(results['neogeo']['roms']), 3)
|
||||
self.assertIn("neogeo", results)
|
||||
self.assertEqual(len(results["neogeo"]["roms"]), 3)
|
||||
self.assertEqual(
|
||||
results['neogeo']['source_file'],
|
||||
'src/mame/snk/neogeo.cpp',
|
||||
results["neogeo"]["source_file"],
|
||||
"src/mame/snk/neogeo.cpp",
|
||||
)
|
||||
|
||||
def test_ignores_non_source_files(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
mame_dir = os.path.join(tmpdir, 'src', 'mame')
|
||||
mame_dir = os.path.join(tmpdir, "src", "mame")
|
||||
os.makedirs(mame_dir)
|
||||
# Write a .txt file that should be ignored
|
||||
with open(os.path.join(mame_dir, 'notes.txt'), 'w') as f:
|
||||
with open(os.path.join(mame_dir, "notes.txt"), "w") as f:
|
||||
f.write(NEOGEO_FIXTURE)
|
||||
|
||||
results = parse_mame_source_tree(tmpdir)
|
||||
@@ -226,13 +226,13 @@ class TestParseMameSourceTree(unittest.TestCase):
|
||||
|
||||
def test_scans_devices_dir(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
dev_dir = os.path.join(tmpdir, 'src', 'devices', 'bus')
|
||||
dev_dir = os.path.join(tmpdir, "src", "devices", "bus")
|
||||
os.makedirs(dev_dir)
|
||||
with open(os.path.join(dev_dir, 'test.cpp'), 'w') as f:
|
||||
with open(os.path.join(dev_dir, "test.cpp"), "w") as f:
|
||||
f.write(DEVICE_FIXTURE)
|
||||
|
||||
results = parse_mame_source_tree(tmpdir)
|
||||
self.assertIn('bbcb', results)
|
||||
self.assertIn("bbcb", results)
|
||||
|
||||
def test_empty_tree(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
@@ -240,5 +240,5 @@ class TestParseMameSourceTree(unittest.TestCase):
|
||||
self.assertEqual(results, {})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -25,11 +25,11 @@ def _platform_has_pack(platform_name: str) -> bool:
|
||||
return False
|
||||
sys.path.insert(0, os.path.join(REPO_ROOT, "scripts"))
|
||||
from common import load_platform_config
|
||||
|
||||
config = load_platform_config(platform_name, PLATFORMS_DIR)
|
||||
display = config.get("platform", platform_name).replace(" ", "_")
|
||||
return any(
|
||||
f.endswith("_BIOS_Pack.zip") and display in f
|
||||
for f in os.listdir(DIST_DIR)
|
||||
f.endswith("_BIOS_Pack.zip") and display in f for f in os.listdir(DIST_DIR)
|
||||
)
|
||||
|
||||
|
||||
@@ -40,10 +40,18 @@ class PackIntegrityTest(unittest.TestCase):
|
||||
if not _platform_has_pack(platform_name):
|
||||
self.skipTest(f"no pack found for {platform_name}")
|
||||
result = subprocess.run(
|
||||
[sys.executable, "scripts/generate_pack.py",
|
||||
"--platform", platform_name,
|
||||
"--verify-packs", "--output-dir", "dist/"],
|
||||
capture_output=True, text=True, cwd=REPO_ROOT,
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/generate_pack.py",
|
||||
"--platform",
|
||||
platform_name,
|
||||
"--verify-packs",
|
||||
"--output-dir",
|
||||
"dist/",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=REPO_ROOT,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
self.fail(
|
||||
|
||||
@@ -48,7 +48,7 @@ truth.py generates diff_truth.py export_native.py
|
||||
|
||||
Pipeline runs all steps in sequence: DB, data dirs, MAME/FBNeo hashes,
|
||||
verify, packs, install manifests, target manifests, consistency check,
|
||||
README, site. See [tools](tools.md) for the full pipeline reference.
|
||||
pack integrity, README, site. See [tools](tools.md) for the full pipeline reference.
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
@@ -59,12 +59,14 @@ graph LR
|
||||
E --> F[install manifests]
|
||||
F --> G[target manifests]
|
||||
G --> H[consistency check]
|
||||
H --> I[generate_readme]
|
||||
H --> H2[pack integrity]
|
||||
H2 --> I[generate_readme]
|
||||
I --> J[generate_site]
|
||||
|
||||
style A fill:#2d333b,stroke:#adbac7,color:#adbac7
|
||||
style D fill:#2d333b,stroke:#adbac7,color:#adbac7
|
||||
style E fill:#2d333b,stroke:#adbac7,color:#adbac7
|
||||
style H2 fill:#2d333b,stroke:#adbac7,color:#adbac7
|
||||
style J fill:#2d333b,stroke:#adbac7,color:#adbac7
|
||||
```
|
||||
|
||||
@@ -234,14 +236,15 @@ user's platform, filter files by hardware target, and download with SHA1 verific
|
||||
|
||||
## Tests
|
||||
|
||||
4 test files with synthetic fixtures:
|
||||
5 test files, 249 tests total:
|
||||
|
||||
| File | Coverage |
|
||||
|------|----------|
|
||||
| `test_e2e.py` | file resolution, verification, severity, cross-reference, aliases, inheritance, shared groups, data dirs, storage tiers, HLE, launchers, platform grouping, core resolution, target filtering, truth/diff, exporters |
|
||||
| `test_mame_parser.py` | BIOS root set detection, ROM block parsing, macro expansion |
|
||||
| `test_fbneo_parser.py` | BIOS set detection, ROM info parsing |
|
||||
| `test_hash_merge.py` | MAME/FBNeo YAML merge, diff detection |
|
||||
| File | Tests | Coverage |
|
||||
|------|-------|----------|
|
||||
| `test_e2e.py` | 186 | file resolution, verification, severity, cross-reference, aliases, inheritance, shared groups, data dirs, storage tiers, HLE, launchers, platform grouping, core resolution, target filtering, truth/diff, exporters |
|
||||
| `test_pack_integrity.py` | 8 | extract ZIP packs to disk, verify paths + hashes per platform's native mode |
|
||||
| `test_mame_parser.py` | 22 | BIOS root set detection, ROM block parsing, macro expansion |
|
||||
| `test_fbneo_parser.py` | 16 | BIOS set detection, ROM info parsing |
|
||||
| `test_hash_merge.py` | 17 | MAME/FBNeo YAML merge, diff detection |
|
||||
|
||||
```bash
|
||||
python -m unittest tests.test_e2e -v
|
||||
|
||||
@@ -11,6 +11,7 @@ Run a single test module:
|
||||
|
||||
```bash
|
||||
python -m unittest tests.test_e2e -v
|
||||
python -m unittest tests.test_pack_integrity -v
|
||||
python -m unittest tests.test_mame_parser -v
|
||||
python -m unittest tests.test_fbneo_parser -v
|
||||
python -m unittest tests.test_hash_merge -v
|
||||
@@ -119,6 +120,30 @@ def test_42_my_new_behavior(self):
|
||||
self.assertEqual(result[0]["status"], Status.OK)
|
||||
```
|
||||
|
||||
### test_pack_integrity.py
|
||||
|
||||
End-to-end pack verification. Extracts each platform ZIP to `tmp/` (in the
|
||||
repo, not `/tmp` which is tmpfs on WSL) and verifies that every file declared
|
||||
in the platform YAML:
|
||||
|
||||
1. Exists at the correct path on disk after extraction
|
||||
2. Has the correct hash per the platform's native verification mode
|
||||
|
||||
Handles inner ZIP verification for MAME/FBNeo ROM sets (checkInsideZip,
|
||||
md5_composite, inner ROM MD5) and path collision deduplication.
|
||||
|
||||
8 tests (one per active platform): RetroArch, Batocera, BizHawk, EmuDeck,
|
||||
Recalbox, RetroBat, RetroDECK, RomM.
|
||||
|
||||
```bash
|
||||
python -m unittest tests.test_pack_integrity -v
|
||||
# or via CLI:
|
||||
python scripts/generate_pack.py --all --verify-packs --output-dir dist/
|
||||
```
|
||||
|
||||
Integrated as pipeline step 6/8 (runs after consistency check, before
|
||||
README generation). Requires packs in `dist/` — skip with `--skip-packs`.
|
||||
|
||||
## Verification discipline
|
||||
|
||||
The test suite is one layer of verification. The full quality gate is:
|
||||
@@ -127,6 +152,7 @@ The test suite is one layer of verification. The full quality gate is:
|
||||
2. The full pipeline completes without error (`python scripts/pipeline.py --offline`)
|
||||
3. No unexpected CRITICAL entries in the verify output
|
||||
4. Pack file counts match verification file counts (consistency check)
|
||||
5. Pack integrity passes (every declared file extractable with correct hash)
|
||||
|
||||
If a change passes tests but breaks the pipeline, it's worth investigating before merging. Similarly, new CRITICAL entries in the verify output after a change usually indicate something to look into. The pipeline is designed so that all steps agree: if verify reports N files for a platform, the pack should contain exactly N files.
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ All tools are Python scripts in `scripts/`. Single dependency: `pyyaml`.
|
||||
Run everything in sequence:
|
||||
|
||||
```bash
|
||||
python scripts/pipeline.py --offline # DB + verify + packs + manifests + readme + site
|
||||
python scripts/pipeline.py --offline # DB + verify + packs + manifests + integrity + readme + site
|
||||
python scripts/pipeline.py --offline --skip-packs # DB + verify only
|
||||
python scripts/pipeline.py --offline --skip-docs # skip readme + site generation
|
||||
python scripts/pipeline.py --offline --target switch # filter by hardware target
|
||||
@@ -20,21 +20,22 @@ Pipeline steps:
|
||||
|
||||
| Step | Description | Skipped by |
|
||||
|------|-------------|------------|
|
||||
| 1/9 | Generate database | - |
|
||||
| 2/9 | Refresh data directories | `--offline` |
|
||||
| 1/8 | Generate database | - |
|
||||
| 2/8 | Refresh data directories | `--offline` |
|
||||
| 2a | Refresh MAME BIOS hashes | `--offline` |
|
||||
| 2a2 | Refresh FBNeo BIOS hashes | `--offline` |
|
||||
| 2b | Check buildbot staleness | only with `--check-buildbot` |
|
||||
| 2c | Generate truth YAMLs | only with `--with-truth` / `--with-export` |
|
||||
| 2d | Diff truth vs scraped | only with `--with-truth` / `--with-export` |
|
||||
| 2e | Export native formats | only with `--with-export` |
|
||||
| 3/9 | Verify all platforms | - |
|
||||
| 4/9 | Generate packs | `--skip-packs` |
|
||||
| 3/8 | Verify all platforms | - |
|
||||
| 4/8 | Generate packs | `--skip-packs` |
|
||||
| 4b | Generate install manifests | `--skip-packs` |
|
||||
| 4c | Generate target manifests | `--skip-packs` |
|
||||
| 5/9 | Consistency check | if verify or pack skipped |
|
||||
| 8/9 | Generate README | `--skip-docs` |
|
||||
| 9/9 | Generate site | `--skip-docs` |
|
||||
| 5/8 | Consistency check | if verify or pack skipped |
|
||||
| 6/8 | Pack integrity (extract + hash) | `--skip-packs` |
|
||||
| 7/8 | Generate README | `--skip-docs` |
|
||||
| 8/8 | Generate site | `--skip-docs` |
|
||||
|
||||
## Individual tools
|
||||
|
||||
@@ -103,6 +104,16 @@ python scripts/generate_pack.py --platform retroarch --list-systems
|
||||
python scripts/generate_pack.py --all --target x86_64
|
||||
python scripts/generate_pack.py --platform retroarch --target switch
|
||||
|
||||
# Source variants
|
||||
python scripts/generate_pack.py --platform retroarch --source platform # YAML baseline only
|
||||
python scripts/generate_pack.py --platform retroarch --source truth # emulator profiles only
|
||||
python scripts/generate_pack.py --platform retroarch --source full # both (default)
|
||||
python scripts/generate_pack.py --all --all-variants --output-dir dist/ # all 6 combinations
|
||||
python scripts/generate_pack.py --all --all-variants --verify-packs --output-dir dist/
|
||||
|
||||
# Data refresh
|
||||
python scripts/generate_pack.py --all --refresh-data # force re-download data dirs
|
||||
|
||||
# Install manifests (consumed by install.py)
|
||||
python scripts/generate_pack.py --all --manifest --output-dir install/
|
||||
python scripts/generate_pack.py --manifest-targets --output-dir install/targets/
|
||||
@@ -122,6 +133,9 @@ If none exists, the platform version is kept and the discrepancy is reported.
|
||||
- `--from-md5`: look up a hash in the database, or build a custom pack with `--platform`/`--emulator`
|
||||
- `--from-md5-file`: same, reading hashes from a file (one per line, comments with #)
|
||||
- `--target`: filter by hardware target (e.g. `switch`, `rpi4`, `x86_64`)
|
||||
- `--source {platform,truth,full}`: select file source (platform YAML only, emulator profiles only, or both)
|
||||
- `--all-variants`: generate all 6 combinations of source x required_only
|
||||
- `--refresh-data`: force re-download all data directories before packing
|
||||
|
||||
### cross_reference.py
|
||||
|
||||
@@ -132,6 +146,8 @@ Reports files that cores need beyond what platforms declare.
|
||||
python scripts/cross_reference.py # all
|
||||
python scripts/cross_reference.py --emulator dolphin # single
|
||||
python scripts/cross_reference.py --emulator dolphin --json # JSON output
|
||||
python scripts/cross_reference.py --platform batocera # single platform
|
||||
python scripts/cross_reference.py --platform retroarch --target switch
|
||||
```
|
||||
|
||||
### truth.py, generate_truth.py, diff_truth.py
|
||||
@@ -168,6 +184,9 @@ from upstream repositories into `data/`.
|
||||
```bash
|
||||
python scripts/refresh_data_dirs.py
|
||||
python scripts/refresh_data_dirs.py --key dolphin-sys --force
|
||||
python scripts/refresh_data_dirs.py --dry-run # preview without downloading
|
||||
python scripts/refresh_data_dirs.py --platform batocera # single platform only
|
||||
python scripts/refresh_data_dirs.py --registry path/to/_data_dirs.yml
|
||||
```
|
||||
|
||||
### Other tools
|
||||
@@ -175,10 +194,10 @@ python scripts/refresh_data_dirs.py --key dolphin-sys --force
|
||||
| Script | Purpose |
|
||||
|--------|---------|
|
||||
| `common.py` | Shared library: hash computation, file resolution, platform config loading, emulator profiles, target filtering |
|
||||
| `dedup.py` | Deduplicate `bios/`, move duplicates to `.variants/`. RPG Maker and ScummVM excluded (NODEDUP) |
|
||||
| `dedup.py` | Deduplicate `bios/` (`--dry-run`, `--bios-dir`), move duplicates to `.variants/`. RPG Maker and ScummVM excluded (NODEDUP) |
|
||||
| `validate_pr.py` | Validate BIOS files in pull requests, post markdown report |
|
||||
| `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline) |
|
||||
| `list_platforms.py` | List active platforms (used by CI) |
|
||||
| `list_platforms.py` | List active platforms (`--all` includes archived, used by CI) |
|
||||
| `download.py` | Download packs from GitHub releases (Python, multi-threaded) |
|
||||
| `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database |
|
||||
| `generate_site.py` | Generate all MkDocs site pages (this documentation) |
|
||||
|
||||
Reference in New Issue
Block a user