diff --git a/emulators/81.yml b/emulators/81.yml index ca923ee7..7860258d 100644 --- a/emulators/81.yml +++ b/emulators/81.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/81-libretro" upstream: "https://github.com/charlierobson/EightyOne" profiled_date: "2026-03-22" source_commit: "fa7094910d040baa5fd8b11dbf6a1a618330ecd9" +upstream_commit: "4de3cdb8cff0060ad03e43ae2681b0628a9c5f5d" core_version: "1.0a" display_name: "Sinclair - ZX 81 (EightyOne)" cores: ["81"] @@ -23,10 +24,10 @@ systems: notes: | Libretro port embeds zx81.rom and dkchr.rom as C arrays (xxd -i). Core never accesses filesystem for ROMs. Hardcodes MACHINEZX81. - ref: 81-libretro/81/zx81/snap.c:487-508, src/libretro.cpp:393 + ref: source 81/zx81/snap.c:487-508, source src/libretro.cpp:393 Original EightyOne supports 15+ machines, loads ROMs from ROM/ directory. - ref: EightyOne/Source/zx81/zx81.cpp:309, EightyOne/Source/HW_.cpp:2734 + ref: upstream Source/zx81/zx81.cpp:309, upstream Source/HW_.cpp:2734 .info has no firmware declarations (firmware_count absent). @@ -270,7 +271,7 @@ files: mode: standalone size: 8192 md5: 5c11c61a2dd2ca4bf39328d9ff42d289 - source_ref: "upstream EightyOne/Source/ROMs/FDCs/interface1.edition1.rom" + source_ref: "upstream Source/ROMs/FDCs/interface1.edition1.rom" - name: "+d.g+dos.v1-a.rom" system: sinclair-zx-spectrum required: false @@ -482,7 +483,7 @@ files: mode: standalone size: 16384 md5: 9f6f56003307b9d55be96f406fdbf2b1 - source_ref: "upstream EightyOne/Source/ROMs/spectrum48.arabic.rom" + source_ref: "upstream Source/ROMs/spectrum48.arabic.rom" - name: spectrum48.arabic.version1.rom system: sinclair-zx-spectrum required: false @@ -599,7 +600,7 @@ files: size: 16384 md5: b7893ad25728fbf6410f08eddd4a1cad note: "Microdigital TK90X (Spectrum clone)." - source_ref: "upstream EightyOne/Source/ROMs/tk90.rom" + source_ref: "upstream Source/ROMs/tk90.rom" - name: tk95.rom system: sinclair-zx-spectrum required: false @@ -608,4 +609,4 @@ files: size: 16384 md5: b7893ad25728fbf6410f08eddd4a1cad note: "Microdigital TK95 (Spectrum clone)." - source_ref: "upstream EightyOne/Source/ROMs/tk95.rom" + source_ref: "upstream Source/ROMs/tk95.rom" diff --git a/emulators/armsx2.yml b/emulators/armsx2.yml index da416bae..b9f3c099 100644 --- a/emulators/armsx2.yml +++ b/emulators/armsx2.yml @@ -6,6 +6,7 @@ source: "https://github.com/ARMSX2/ARMSX2" upstream: "https://github.com/PCSX2/pcsx2" profiled_date: "2026-10-04" source_commit: "d383b3636947565d9201f81e32178d0288623d8d" +upstream_commit: "36907f4b70d149b05c2700acae687bedb720a7af" core_version: "2.8" display_name: "Sony - PlayStation 2 (ARMSX2)" cores: diff --git a/emulators/azahar.yml b/emulators/azahar.yml index aed198ab..270b1eac 100644 --- a/emulators/azahar.yml +++ b/emulators/azahar.yml @@ -5,6 +5,7 @@ source: "https://github.com/johndoe6345789/azahar_libretro" upstream: "https://github.com/azahar-emu/azahar" profiled_date: "2026-03-23" source_commit: "9dd12da328c753806cbd4f2b0e048fced97ae97d" +upstream_commit: "9b045bf8370d035cacd99e5ad19a082df9e3b262" core_version: "Git" display_name: "Nintendo - 3DS (Azahar)" cores: [azahar] diff --git a/emulators/cannonball.yml b/emulators/cannonball.yml index 621b7496..9e0a1ebf 100644 --- a/emulators/cannonball.yml +++ b/emulators/cannonball.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/cannonball" upstream: "https://github.com/djyt/cannonball" profiled_date: "2026-03-23" source_commit: "0d83575d920ea4c0a280847a19d7802f673e6a68" +upstream_commit: "27493ebf62be3498dff93ed6a45e8e2db819bae1" core_version: "1.0" display_name: "Cannonball" cores: diff --git a/emulators/cdi2015.yml b/emulators/cdi2015.yml index 02fde949..e703ab99 100644 --- a/emulators/cdi2015.yml +++ b/emulators/cdi2015.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/mame2015-libretro" upstream: "https://github.com/mamedev/mame/tree/mame0160" profiled_date: "2026-03-23" source_commit: "7adbf440d5f554097f58f8c6ebb673adc4320a31" +upstream_commit: "0147bb4cc82e642cf75151f039f240eb20989fbc" core_version: "0.160" display_name: "Philips CDi (CDi 2015)" cores: [cdi2015] diff --git a/emulators/dolphin-mmjr.yml b/emulators/dolphin-mmjr.yml index 52189e58..ce9095ae 100644 --- a/emulators/dolphin-mmjr.yml +++ b/emulators/dolphin-mmjr.yml @@ -5,6 +5,7 @@ source: "https://github.com/acidtech/Dolphin-MMJR" upstream: "https://github.com/weihuoya/dolphin" profiled_date: "2026-08-11" source_commit: "1b095b64857c2eabb49a42cc56ffd9d7881bc263" +upstream_commit: "0f4c971326ae9389b3ad55b0fefacb708d148f4d" core_version: "11460 - bankaimaster999" display_name: "Nintendo - GameCube / Wii (Dolphin MMJR)" cores: diff --git a/emulators/dolphin-mmjr2.yml b/emulators/dolphin-mmjr2.yml index 82acc01a..bc7af2f7 100644 --- a/emulators/dolphin-mmjr2.yml +++ b/emulators/dolphin-mmjr2.yml @@ -5,6 +5,7 @@ source: "https://github.com/nachoverdon/Dolphin-MMJR2" upstream: "https://github.com/dolphin-emu/dolphin" profiled_date: "2026-08-11" source_commit: "4d0fdb8eea616320e967408cf5547c72a95196f3" +upstream_commit: "4ff214cdd26f135251d2266d676ed72dc1c2f8c9" core_version: "2.0-18025" display_name: "Nintendo - GameCube / Wii (Dolphin MMJR2)" cores: @@ -22,7 +23,8 @@ notes: | selective merge of Dolphin Official carrying that label rather than a checkout of it: at Dolphin 5.0-18025 the memory subsystem has already moved behind Core::System, and here it has not, so no single Dolphin revision - matches and no upstream pin is declared. + matches the tree. upstream_commit is the Dolphin revision the upstream + citations were read at, not a base of this tree. ref: Source/Android/app/build.gradle:32, Source/Android/app/build.gradle:135 Two roots. The user directory is mmjr-revamp at the root of external storage. diff --git a/emulators/dolphin.yml b/emulators/dolphin.yml index 5c80a76a..de95c334 100644 --- a/emulators/dolphin.yml +++ b/emulators/dolphin.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/dolphin-emu/dolphin" logo: "https://raw.githubusercontent.com/dolphin-emu/dolphin/master/Data/dolphin-emu.svg" profiled_date: "2026-03-23" source_commit: "4d23cf151640eb810cb1b8e9d9fc922cf59c0b87" +upstream_commit: "f5dfb7e3d36ce24551ed0bb886b79f3f192dac9d" core_version: "Git" display_name: "Nintendo - GameCube / Wii (Dolphin)" cores: diff --git a/emulators/dosbox_pure.yml b/emulators/dosbox_pure.yml index b6d84a53..a91810ec 100644 --- a/emulators/dosbox_pure.yml +++ b/emulators/dosbox_pure.yml @@ -6,6 +6,7 @@ upstream: "https://codeberg.org/schelling/dosbox-pure" logo: "https://raw.githubusercontent.com/schellingb/dosbox-pure/main/images/logo.png" profiled_date: "2026-03-23" source_commit: "cdaea10ce8cac231bc341ca05d4c5e4e5fe0fbd5" +upstream_commit: "cdaea10ce8cac231bc341ca05d4c5e4e5fe0fbd5" core_version: "1.0-preview7" display_name: "DOS (DOSBox-Pure)" cores: [dosbox_pure] diff --git a/emulators/easyrpg.yml b/emulators/easyrpg.yml index 4d5482fa..9a4231f8 100644 --- a/emulators/easyrpg.yml +++ b/emulators/easyrpg.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/easyrpg-libretro" upstream: "https://github.com/EasyRPG/Player" profiled_date: "2026-03-23" source_commit: "a3fbb7c24fd5a39fd95388d09135256a6b19f1e4" +upstream_commit: "8acd2e17c6a8a0a75cb7ce95625c6bdc89aebe1a" core_version: "0.8.1.0" display_name: "RPG Maker 2000/2003 (EasyRPG)" cores: diff --git a/emulators/fbalpha2019.yml b/emulators/fbalpha2019.yml index b385302e..2f64305e 100644 --- a/emulators/fbalpha2019.yml +++ b/emulators/fbalpha2019.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/fbalpha" upstream: "https://github.com/barry65536/FBAlpha" profiled_date: "2026-10-17" source_commit: "0581797db6fdffd826086b053ced4b6b29bb6678" +upstream_commit: "416948158d88ba17f87813419ddc3c1209ef68c0" pin_frozen: "ROCKNIX 20261001 builds this revision as fbalpha2019. Later commits on libretro/fbalpha are not part of that build." core_version: "v0.2.97.44" display_name: "Arcade (FB Alpha 2019)" diff --git a/emulators/fbneo.yml b/emulators/fbneo.yml index 77a8c3a0..9028f2c7 100644 --- a/emulators/fbneo.yml +++ b/emulators/fbneo.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/finalburnneo/FBNeo" logo: "https://raw.githubusercontent.com/finalburnneo/FBNeo/master/projectfiles/xcode/Emulator/Assets.xcassets/AppIcon.appiconset/icon_512.png" profiled_date: "2026-03-23" source_commit: "63c4190785cadd5ff84483399375871ed6e98754" +upstream_commit: "8d47450deff3802a61559108f12bb0c846c98e36" core_version: "v1.0.0.2" display_name: "Arcade (FinalBurn Neo)" cores: diff --git a/emulators/fsuae.yml b/emulators/fsuae.yml index 2e4ab2db..a5fd235f 100644 --- a/emulators/fsuae.yml +++ b/emulators/fsuae.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/libretro-fsuae" upstream: "https://github.com/FrodeSolheim/fs-uae" profiled_date: "2026-03-26" source_commit: "089bb69207fe8fa5eec0006d1d759fb6426f4027" +upstream_commit: "f362278ccd4c60991caac3b4d240d4a3f751bea2" core_version: "v5.0.7" display_name: "Commodore - Amiga (FS-UAE)" cores: diff --git a/emulators/groovymame.yml b/emulators/groovymame.yml index 187622a6..b650160c 100644 --- a/emulators/groovymame.yml +++ b/emulators/groovymame.yml @@ -4,9 +4,10 @@ core_classification: enhanced_fork source: "https://github.com/antonioginer/GroovyMAME" upstream: "https://github.com/mamedev/mame" source_commit: "953db38f4aaab1d75faa4ac79e1b45686214f987" +upstream_commit: "f34f02505e32c1993c6a782b6814232cbfc74e36" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-08-08" -core_version: "0.288" +core_version: "0.289" display_name: "Arcade (GroovyMAME)" cores: @@ -122,13 +123,14 @@ systems: notes: | MAME fork adding Switchres 2.2.2 for CRT modeline generation, frame delay - and low latency vsync. Branch groovymame0288, BARE_BUILD_VERSION 0.288 - (makefile:1508). TARGET := mame with no SUBTARGET override (makefile:274-282), + and low latency vsync. Branch groovymame0289, BARE_BUILD_VERSION 0.289 + (makefile:1513). TARGET := mame with no SUBTARGET override (makefile:274-282), so the build carries the full machine set. - src/mame and src/devices are byte identical to mamedev/mame tag mame0288, - so ROM requirements are MAME 0.288's. The fork's changes sit in src/osd and - 3rdparty/switchres and pull in no external data file: monitor presets are + src/mame and src/devices are byte identical to mamedev/mame tag mame0289, + so ROM requirements are MAME 0.289's. The fork's changes sit in src/osd, + src/emu, src/frontend, 3rdparty/switchres, 3rdparty/emusync and + 3rdparty/lz4, and pull in no external data file: monitor presets are compiled in (3rdparty/switchres/monitor.cpp:124-145), and switchres.ini plus display#.ini are optional user configuration read only when the switchres_ini option is set (src/osd/modules/switchres/switchres_module.cpp:166,173). diff --git a/emulators/holani.yml b/emulators/holani.yml index b85d17c1..ef05f37b 100644 --- a/emulators/holani.yml +++ b/emulators/holani.yml @@ -5,6 +5,7 @@ source: "https://github.com/LLeny/holani-retro" upstream: "https://github.com/LLeny/holani" profiled_date: "2026-03-24" source_commit: "19b974f77247a937369adf48280d2b34a5a66245" +upstream_commit: "4fa0ee920c63a9cf8d07b974ae77fb15161ff967" core_version: "0.9.9" display_name: "Atari - Lynx (Holani)" cores: [holani] diff --git a/emulators/hypseus-singe.yml b/emulators/hypseus-singe.yml index 792b2947..eb611232 100644 --- a/emulators/hypseus-singe.yml +++ b/emulators/hypseus-singe.yml @@ -4,6 +4,7 @@ core_classification: enhanced_fork source: "https://github.com/DirtBagXon/hypseus-singe" upstream: "https://github.com/mirror/daphne-emu" source_commit: "cda87965fe0a031d027b2253c6715d37cca6b56a" +upstream_commit: "cd214c6046365047704f8a0563928970b30444e6" profiled_date: "2026-08-08" core_version: "v3.0.0" display_name: "Arcade (Hypseus Singe)" diff --git a/emulators/ishiiruka.yml b/emulators/ishiiruka.yml index 225ccc05..f558a95b 100644 --- a/emulators/ishiiruka.yml +++ b/emulators/ishiiruka.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/ishiiruka" upstream: "https://github.com/Tinob/Ishiiruka" profiled_date: "2026-03-24" source_commit: "f228bde5fbf4861ba749a229803addcd6238663f" +upstream_commit: "cba6d7279b44894297d8c1dde1effdbba3605c32" core_version: "Git" display_name: "Nintendo - GameCube / Wii (Ishiiruka)" cores: diff --git a/emulators/jl-mod.yml b/emulators/jl-mod.yml index ee2c5f5f..c1775e76 100644 --- a/emulators/jl-mod.yml +++ b/emulators/jl-mod.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/nikita36078/J2ME-Loader" author: "Yury Kharchenko" profiled_date: "2026-08-11" source_commit: "f723a190c0bdb44b31c3bc0ead6f8665c7ea517d" +upstream_commit: "c2fdac77edba94709796b67ed5493915f31d620f" core_version: "0.87.1" display_name: "Mobile - J2ME (JL-Mod)" cores: diff --git a/emulators/kronos.yml b/emulators/kronos.yml index 9f0a90e9..4f55dc45 100644 --- a/emulators/kronos.yml +++ b/emulators/kronos.yml @@ -6,6 +6,7 @@ source_branch: kronos upstream: "https://github.com/FCare/Kronos" profiled_date: "2026-03-24" source_commit: "3791ffb2221fda61d70aec5cca376418caf05669" +upstream_commit: "d0384bc33aad06bfa9cf5cd308c4bd7ae6cfc814" core_version: "v2.6.0" display_name: "Sega - Saturn/ST-V (Kronos)" cores: diff --git a/emulators/lrps2.yml b/emulators/lrps2.yml index facc380a..8aad8b6b 100644 --- a/emulators/lrps2.yml +++ b/emulators/lrps2.yml @@ -57,7 +57,7 @@ files: required: false max_size: 4194304 description: "DVD player ROM" - source_ref: "upstream pcsx2/ps2/BiosTools.cpp:189-229, 339" + source_ref: "source pcsx2/ps2/BiosTools.cpp:189-229, 339" note: > DVD player ROM. Tries .rom1 (appended) then .rom1 (extension replaced). Silently skipped if not found. @@ -67,7 +67,7 @@ files: required: false max_size: 524288 description: "Chinese ROM extension" - source_ref: "upstream pcsx2/ps2/BiosTools.cpp:189-229, 340" + source_ref: "source pcsx2/ps2/BiosTools.cpp:189-229, 340" note: > Chinese region ROM extension. Same naming convention as rom1. Only present on Chinese region consoles. @@ -78,7 +78,7 @@ files: hle_fallback: true size: 1024 description: "NVRAM / EEPROM data" - source_ref: "upstream pcsx2/CDVD/CDVD.cpp:196-227" + source_ref: "source pcsx2/CDVD/CDVD.cpp:196-227" note: > Console EEPROM data (language, timezone, region, iLink ID, OSD settings). Path derived from BIOS path with extension replaced to .nvm. @@ -90,7 +90,7 @@ files: hle_fallback: true size: 4 description: "Mechacon version" - source_ref: "upstream pcsx2/CDVD/CDVD.cpp:229-245" + source_ref: "source pcsx2/CDVD/CDVD.cpp:229-245" note: > Mechacon (disc drive controller) version as u32. Auto-created with default 0x00020603 if missing. diff --git a/emulators/mame2009.yml b/emulators/mame2009.yml index 8d62395d..f78a0c27 100644 --- a/emulators/mame2009.yml +++ b/emulators/mame2009.yml @@ -5,6 +5,7 @@ source: "https://github.com/r-type/mame2009-libretro" upstream: "https://github.com/mamedev/mame/tree/mame0135u4" profiled_date: "2026-03-24" source_commit: "45048806fbc0fea82709a75a523f2eda945e0a47" +upstream_commit: "c285eb9bcd3505c14a338e29f251ba00776cc75c" core_version: "0.135u4" display_name: "Arcade (MAME 2009)" diff --git a/emulators/mame2010.yml b/emulators/mame2010.yml index 20edf39d..26b15b78 100644 --- a/emulators/mame2010.yml +++ b/emulators/mame2010.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0139" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-03-22" source_commit: "dff8aadd1c3f38215af3955746d6e19abe0ddcea" +upstream_commit: "f4f24ba7274010fb8ccb84548b5ca95c2be2f85b" core_version: "0.139" display_name: "Arcade (MAME 2010)" mame_version: "0.139" diff --git a/emulators/mame2014.yml b/emulators/mame2014.yml index 4fd821d0..fec8f2aa 100644 --- a/emulators/mame2014.yml +++ b/emulators/mame2014.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0160" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-10-20" source_commit: "c8fe6c7399f24466808b67933b2b96feea188e0b" +upstream_commit: "0147bb4cc82e642cf75151f039f240eb20989fbc" core_version: "0.160" display_name: "Arcade (MAME 2014)" mame_version: "0.160" diff --git a/emulators/mame2015.yml b/emulators/mame2015.yml index 2c45a9cf..6bc5ca7a 100644 --- a/emulators/mame2015.yml +++ b/emulators/mame2015.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0160" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-03-24" source_commit: "7adbf440d5f554097f58f8c6ebb673adc4320a31" +upstream_commit: "0147bb4cc82e642cf75151f039f240eb20989fbc" core_version: "0.160" display_name: "Arcade (MAME 2015)" mame_version: "0.160" diff --git a/emulators/mame2016.yml b/emulators/mame2016.yml index b6d30f1b..ae67052e 100644 --- a/emulators/mame2016.yml +++ b/emulators/mame2016.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0174" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-03-22" source_commit: "ae07c2f88ff2482ba9f50ffc8c9e7e6fbfe97d0a" +upstream_commit: "68785dccfee3208163c8169abc5c6d01becf36d4" core_version: "0.174" cores: diff --git a/emulators/melonds_ds.yml b/emulators/melonds_ds.yml index 64fcd777..066fd054 100644 --- a/emulators/melonds_ds.yml +++ b/emulators/melonds_ds.yml @@ -6,6 +6,7 @@ source: "https://github.com/JesseTG/melonds-ds" upstream: "https://github.com/melonDS-emu/melonDS" profiled_date: "2026-10-04" source_commit: "f394adbacb5722ee97c1b37c8064da9a25818310" +upstream_commit: "906e9ebb27da8c6a715cd7abab4abfe8a8d29427" core_version: "1.4.0" display_name: "Nintendo - DS (melonDS DS)" systems: diff --git a/emulators/mesen-s.yml b/emulators/mesen-s.yml index f23f5035..6bec6c52 100644 --- a/emulators/mesen-s.yml +++ b/emulators/mesen-s.yml @@ -9,6 +9,7 @@ source: "https://github.com/libretro/Mesen-S" upstream: "https://github.com/SourMesen/Mesen-S" profiled_date: "2026-03-24" source_commit: "9e4fdeb9b336470bc96beb8765b2e79c86a2da1e" +upstream_commit: "89199f298a95345a893d814525b6668567a6d490" core_version: "0.4.0" cores: [mesen-s, mesen_s] systems: [nintendo-snes, nintendo-gb, nintendo-gbc, nintendo-super-game-boy, nintendo-satellaview] diff --git a/emulators/mesence.yml b/emulators/mesence.yml index c31bbf94..0ea80bc6 100644 --- a/emulators/mesence.yml +++ b/emulators/mesence.yml @@ -9,6 +9,7 @@ profiled_date: "2026-09-17" source_commit: standalone: "20f497c9620a7f4e55b7752e1d22734f9d7492fa" libretro: "4c4e069f16a8ad6ed79e046046c56d51558c7daf" +upstream_commit: "fabc9a62174f8734a113df6d244f5539ef6b8fcf" core_version: standalone: "2.2.1" libretro: "2.0.0" @@ -620,7 +621,7 @@ files: validation: [size] sha256: "990bf1956f10207d8781b619eb74f89b00d921c8d45c95c334c16c8cceca09ad,fb4a898eb93b19b36773d87a6c70eb28f981b2686bebdd2d431b05dcdf9cffd4" note: "ColecoVision BIOS, NTSC or PAL console dump. The system boots through it, and games cannot start without it." - source_ref: "source Core/SMS/SmsConsole.cpp:71-74, source Core/Shared/FirmwareHelper.h:381-398, upstream UI/Interop/FirmwareTypeExtensions.cs:70" + source_ref: "source Core/SMS/SmsConsole.cpp:71-74, source Core/Shared/FirmwareHelper.h:381-398, source UI/Interop/FirmwareTypeExtensions.cs:70" # ------------------------------------------------------- # WonderSwan (standalone only: UseBootRom is never set by the libretro port) diff --git a/emulators/np2kai.yml b/emulators/np2kai.yml index e86f43d0..1f2214ab 100644 --- a/emulators/np2kai.yml +++ b/emulators/np2kai.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/NP2kai" upstream: "https://github.com/AZO234/NP2kai" profiled_date: "2026-03-24" source_commit: "1ea561cd7fd62b33f4e7b3f1c2e30fc3e6089f28" +upstream_commit: "5d47f3a04b0b8ea78f625eeec354ac132c80df06" core_version: "0.86" display_name: "NEC - PC-98 (Neko Project II Kai)" cores: [np2kai] diff --git a/emulators/primehack.yml b/emulators/primehack.yml index 71917d14..cc082c97 100644 --- a/emulators/primehack.yml +++ b/emulators/primehack.yml @@ -5,6 +5,7 @@ source: "https://github.com/shiiion/dolphin" upstream: "https://github.com/dolphin-emu/dolphin" profiled_date: "2026-03-26" source_commit: "53f53e0f5bad27ad62a807cb93b136d84f68777f" +upstream_commit: "d8558142ead21b3fa39809c4aedb982fbfc85feb" core_version: "1.0.8" display_name: "PrimeHack (Dolphin fork for Metroid Prime)" cores: @@ -41,7 +42,7 @@ files: required: false hle_fallback: true note: "GameCube NTSC-U boot ROM. HLE available but real IPL needed for GC menu boot and accurate fonts. Discs with the Brazilian country code B and SYSCONF countries of the Americas map to NTSC-U, so Latin America reads this directory too (Source/Core/DiscIO/Enums.cpp:198-199, 249-251)." - source_ref: "upstream Source/Core/Common/CommonPaths.h:138, source Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp:109-110" + source_ref: "upstream Source/Core/Common/CommonPaths.h:139, source Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp:109-110" - name: "IPL.bin" path: "GC/EUR/IPL.bin" @@ -78,7 +79,7 @@ files: adler32_pairs_with: "dsp_coef.bin" adler32_pairs: [["0x66f334fe", "0xf3b93527"], ["0xe789b5a5", "0xa4a575f5"]] note: "DSP instruction ROM for LLE audio. Free replacement (v0.4) included" - source_ref: "upstream Source/Core/Common/CommonPaths.h:135, source Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-117, source Source/Core/Core/DSP/DSPCore.cpp:28-102" + source_ref: "upstream Source/Core/Common/CommonPaths.h:136, source Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-117, source Source/Core/Core/DSP/DSPCore.cpp:28-102" - name: "dsp_coef.bin" path: "GC/dsp_coef.bin" @@ -106,7 +107,7 @@ files: sha1: "483a17ff25b68a8f18cae691da99f16beefbe370" md5: "9bef639b7426ef6948602a12b33522a8" note: "Windows-1252 font for GC/Wii text rendering. The free alternative ships in Sys/GC/ of the 1.0.8 Windows and macOS releases, and the real one from an IPL dump is preferred" - source_ref: "upstream Source/Core/Common/CommonPaths.h:132, source Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp:131, 203-218" + source_ref: "upstream Source/Core/Common/CommonPaths.h:133, source Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp:131, 203-218" - name: "font_japanese.bin" path: "GC/font_japanese.bin" @@ -127,7 +128,7 @@ files: required: false hle_fallback: true note: "GBA BIOS for GC-GBA link feature (uses integrated mGBA). Path configurable in settings" - source_ref: "upstream Source/Core/Common/CommonPaths.h:144, source Source/Core/Core/HW/GBACore.cpp:344-361" + source_ref: "upstream Source/Core/Common/CommonPaths.h:145, source Source/Core/Core/HW/GBACore.cpp:344-361" # -- Wii System Files -- - name: "SYSCONF" @@ -143,7 +144,7 @@ files: required: false hle_fallback: true note: "Wii region/language settings. Auto-generated during Wii boot emulation" - source_ref: "upstream Source/Core/Common/CommonPaths.h:152, source Source/Core/Core/Boot/Boot_BS2Emu.cpp:377-454" + source_ref: "upstream Source/Core/Common/CommonPaths.h:153, source Source/Core/Core/Boot/Boot_BS2Emu.cpp:377-454" # -- Wii NAND Backup (BootMii) -- - name: "nand.bin" @@ -167,7 +168,7 @@ files: required: false hle_fallback: true note: "Virtual SD card image for Wii homebrew. Auto-created, supports SD/SDHC up to 4GB" - source_ref: "upstream Source/Core/Common/CommonPaths.h:149" + source_ref: "upstream Source/Core/Common/CommonPaths.h:150" # -- Gecko Code Handler -- - name: "codehandler.bin" @@ -179,7 +180,7 @@ files: sha1: "e6c0050b3c67ee26198488fbbe4a5d019c15b628" md5: "1f118b8735fcf40c20b38c3184eebfd9" note: "Gecko/Ocarina cheat code handler. Shipped in Sys/ of the 1.0.8 Windows and macOS releases" - source_ref: "upstream Source/Core/Common/CommonPaths.h:154, source Source/Core/Core/GeckoCode.cpp:121" + source_ref: "upstream Source/Core/Common/CommonPaths.h:155, source Source/Core/Core/GeckoCode.cpp:121" # -- Wii System Menu (WAD) -- - name: "Wii System Menu" diff --git a/emulators/scummvm.yml b/emulators/scummvm.yml index b8ce7478..3d166653 100644 --- a/emulators/scummvm.yml +++ b/emulators/scummvm.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/scummvm/scummvm" logo: "https://raw.githubusercontent.com/scummvm/scummvm/master/icons/scummvm.svg" profiled_date: "2026-03-25" source_commit: "fcbce3ae815269dacdc309092bc92ccc6d3e13bb" +upstream_commit: "fcbce3ae815269dacdc309092bc92ccc6d3e13bb" core_version: "2.8.0git" display_name: "ScummVM" cores: [scummvm] diff --git a/emulators/snes9x.yml b/emulators/snes9x.yml index b488d503..d1dcc1b3 100644 --- a/emulators/snes9x.yml +++ b/emulators/snes9x.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/snes9xgit/snes9x" logo: "https://raw.githubusercontent.com/snes9xgit/snes9x/master/gtk/data/snes9x.svg" profiled_date: "2026-03-25" source_commit: "fae2fea08f74180759ef540ee94259213f503480" +upstream_commit: "85591435763dede7ef111aab8070cdf2d5a7da49" core_version: "1.61" display_name: "Nintendo - SNES / SFC (Snes9x)" systems: [nintendo-snes, nintendo-satellaview, nintendo-sufami-turbo] diff --git a/emulators/syobonaction.yml b/emulators/syobonaction.yml index 9200ffed..5b64195e 100644 --- a/emulators/syobonaction.yml +++ b/emulators/syobonaction.yml @@ -5,6 +5,7 @@ source: "https://github.com/iyzsong/syobonaction-libretro" upstream: "https://github.com/akemin-dayo/OpenSyobonAction" profiled_date: "2026-03-25" source_commit: "792c40cb590bcc7286cf720b5fb85807082b534e" +upstream_commit: "77ca2c128784512eca10e24a453ca19c270a8af4" core_version: "0.3" display_name: "Syobon Action" cores: [syobonaction] diff --git a/emulators/triforce.yml b/emulators/triforce.yml index a321a743..00f0f948 100644 --- a/emulators/triforce.yml +++ b/emulators/triforce.yml @@ -5,6 +5,7 @@ source: "https://github.com/REG-Linux/dolphin-triforce" upstream: "https://github.com/dolphin-emu/dolphin" profiled_date: "2026-08-08" source_commit: "5c456e8ff0da0235be2cf1d55ba7bb2115b0cae2" +upstream_commit: "b92e354389bb7c0bd114a8631b8af110d3cb3a14" core_version: "2407" display_name: "Namco / Sega / Nintendo - Triforce (Dolphin Triforce)" cores: diff --git a/emulators/uae4arm.yml b/emulators/uae4arm.yml index a2561b38..cbd2b97d 100644 --- a/emulators/uae4arm.yml +++ b/emulators/uae4arm.yml @@ -6,6 +6,7 @@ source: "https://github.com/libretro/uae4arm-libretro" upstream: "https://github.com/PandTomB/uae4arm" profiled_date: "2026-03-25" source_commit: "26f444f01771f0dfa2fa652906ff8222e92e476e" +upstream_commit: "161ddbaafe90a9747e9fb9c65ee4f38c163f335e" core_version: "v0.5" display_name: "Commodore - Amiga (UAE4ARM)" systems: diff --git a/emulators/ume2015.yml b/emulators/ume2015.yml index 71eea26e..8a94a11f 100644 --- a/emulators/ume2015.yml +++ b/emulators/ume2015.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0159" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-03-25" source_commit: "7adbf440d5f554097f58f8c6ebb673adc4320a31" +upstream_commit: "6ae44e46f91aa2bd74d9895e6bfd345baebd3d22" core_version: "0.159" display_name: "Multi (UME 2015)" mame_version: "0.159" diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index df64e518..18bf24ff 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -25,6 +25,7 @@ "catalog", "platforms", "emulators", + "systems", "verification-and-coverage-gaps", "file-locations" ] @@ -261,6 +262,76 @@ } } } + }, + { + "if": { + "properties": { + "kind": { + "const": "systems" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "consoles", + "id", + "name" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "consoles": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "path", + "sha1" + ], + "properties": { + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "sha1": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "md5": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "crc32": { + "type": "string" + } + } + } + } + } + } + } + } + } + } } ] } diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 840ca8b2..a235f8d7 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -367,6 +367,52 @@ def _content_matches(location: tuple[str, str], declared: dict[str, set[str]]) - return any(actual[field] in values for field, values in declared.items()) +_tree_cache: dict[tuple[str, str], list[str]] = {} + + +def _trees(data_root: str, bios_root: str) -> list[str]: + """Every directory under data/ and bios/, as a path with a trailing slash.""" + key = (data_root, bios_root) + if key not in _tree_cache: + _tree_cache[key] = sorted( + f"{path.as_posix()}/" + for root in (data_root, bios_root) if Path(root).is_dir() + for path in Path(root).rglob("*") + if path.is_dir() + and not any(p.startswith(".") for p in path.relative_to(root).parts) + ) + return _tree_cache[key] + + +def held_copy( + entry: dict, data_root: str = "data", bios_root: str = "bios" +) -> tuple[str, str] | None: + """The data/ file, directory or bios/ ZIP member holding an entry. + + The database indexes the files under bios/, and an entry held otherwise + (specmate.rom as a member of spectrum_opus.zip, Dolphin's fonts in its + data directory) read as held with no word of where. Returns (path, + member), the member empty for a file or a directory. A file entry that + declares a hash is held only by a copy carrying it, as _data_hit judges. + """ + names = [entry.get("name", ""), entry.get("path") or "", *(entry.get("aliases") or [])] + names = [str(n) for n in names if n] + if entry.get("type") == "directory" or any(n.endswith("/") for n in names[:1]): + for name in names: + tail = "/" + name.strip("/") + "/" + for tree in _trees(data_root, bios_root): + if tree.endswith(tail): + return tree, "" + return None + declared = _declared_content(entry) + locations = _supplemental_locations(data_root, bios_root) + for name in names: + for location in locations.get(name.rsplit("/", 1)[-1].lower(), ()): + if not declared or _content_matches(location, declared): + return location + return None + + class FileConsumers: """Who reads each collected file, joined through the resolver. @@ -469,6 +515,10 @@ class FileConsumers: self.installed.setdefault(sha1, {}).setdefault( platform, set()).add(str(entry.get("dest", ""))) + def sha1_at(self, path: str) -> str | None: + """The file the database records at a collection path.""" + return self._sha1_by_path.get(path) + def path_of(self, sha1: str) -> str: """The collection path the database records for a file.""" return str(self._db.get("files", {}).get(sha1, {}).get("path", "")) @@ -757,17 +807,18 @@ def _cross_reference_profile( continue # --- resolve source provenance --- - storage = f.get("storage", "") - if storage in ("release", "large_file"): + # A release asset is held when the database records it, as any other + # file: storage says where the bytes are served from, not that the + # collection has them (kenji-nx's user-picked firmware package). + source = next( + ( + found for version in versions[slot] + if (found := entry_source(version, index)) is not None + ), + "missing", + ) + if source != "missing" and f.get("storage", "") in ("release", "large_file"): source = "large_file" - else: - source = next( - ( - found for version in versions[slot] - if (found := entry_source(version, index)) is not None - ), - "missing", - ) in_repo = source != "missing" diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 7e938d3b..01b379ce 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -15,10 +15,12 @@ from siterender import WIKI_SRC_DIR import argparse import csv import hashlib +import html import io import json import os import re +import shlex import shutil import sqlite3 import sys @@ -48,7 +50,7 @@ from common import ( yaml_load, ) from artifacts import ArtifactLockBusy, artifact_lock -from cross_reference import FileConsumers +from cross_reference import FileConsumers, held_copy from nativemode import reads_file_contents @@ -125,19 +127,95 @@ CLS_LABELS = { } +def _copy_places(db: dict) -> dict[str, set[tuple[str, str, str]]]: + """sha1 -> every (manufacturer, console, file name) holding a copy. + + The database keeps one path per content, deduplication having left one + copy. by_path_suffix still records the other folders a copy is filed in, + under the name that folder gives it. + """ + files = db.get("files", {}) + bios_dir = Path(str(db.get("bios_dir") or "bios")) + places: dict[str, set[tuple[str, str, str]]] = {} + for sha1, entry in files.items(): + parts = str(entry.get("path", "")).split("/") + if len(parts) >= 3 and parts[0] == "bios": + places.setdefault(sha1, set()).add((parts[1], parts[2], parts[-1])) + for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items(): + parts = tail.split("/") + if len(parts) < 3 or parts[1].startswith(".") or len(sha1s) != 1: + continue + if sha1s[0] in files and (bios_dir / tail).is_file(): + places.setdefault(sha1s[0], set()).add((parts[0], parts[1], parts[-1])) + return places + + +# bios/Other/ and bios/Arcade/ file a copy by +# who loads it, not by machine. A copy also filed under a machine's maker +# places the system there. +_BUCKET_FOLDERS = frozenset({"Other", "Arcade"}) + + +def _slug(manufacturer: str) -> str: + return manufacturer.lower().replace(" ", "-") + + +def _names_manufacturer(sys_id: str, manufacturer: str) -> bool: + slug = _slug(manufacturer) + return sys_id == slug or sys_id.startswith(f"{slug}-") + + +def _copies_by_path(db: dict) -> dict[str, set[tuple[str, str, str]]]: + """The copy places of each content, keyed by the path the database keeps.""" + files = db.get("files", {}) + return { + str(files[sha1].get("path", "")): where + for sha1, where in _copy_places(db).items() + } + + def _system_folder( - files: list[dict], db: dict, name_consoles: dict[str, set[tuple[str, str]]] + files: list[dict], + db: dict, + name_consoles: dict[str, set[tuple[str, str]]], + sys_id: str = "", + copies: dict[str, set[tuple[str, str, str]]] | None = None, ) -> tuple[str, str] | None: - """The bios// folder a system's files live in.""" + """The bios// folder a system's files live in. + + A content votes for every folder holding a copy under the name the system + declares, not only the one deduplication kept: disksys.rom is kept as + Arcade/FBNeo/fdsbios.nes and filed again under Nintendo/Famicom Disk + System. A copy filed under another name, outside the manufacturer the id + names, is another machine's file and votes only when no copy carries the + declared name. An id that names its manufacturer (nintendo-fds) keeps the + votes cast inside that manufacturer when there are any, and a maker's + folder outranks the folders that file copies by emulator. + """ + if copies is None: + copies = _copies_by_path(db) votes: dict[tuple[str, str], int] = {} for fe in files: local, status = resolve_local_file(fe, db) - parts = (local or "").split("/") - if len(parts) >= 4 and parts[0] == "bios" and resolution_is_hash_exact(status): - place = (parts[1], parts[2]) + if not local or not resolution_is_hash_exact(status): + continue + held = copies.get(local, set()) + if not held: + parts = local.split("/") + if len(parts) >= 4 and parts[0] == "bios": + held = {(parts[1], parts[2], parts[-1])} + declared = str(fe.get("name", "")).casefold() + named = { + (m, c) for m, c, base in held + if base.casefold() == declared or _names_manufacturer(sys_id, m) + } + for place in named or {(m, c) for m, c, _ in held}: votes[place] = votes.get(place, 0) + 1 if votes: - return max(sorted(votes), key=lambda place: votes[place]) + own = [place for place in votes if _names_manufacturer(sys_id, place[0])] + makers = [place for place in votes if place[0] not in _BUCKET_FOLDERS] + candidates = own or makers or list(votes) + return max(sorted(candidates), key=lambda place: votes[place]) for fe in files: places = name_consoles.get(fe.get("name", ""), set()) if len(places) == 1: @@ -145,6 +223,32 @@ def _system_folder( return None +def _folders_by_name(manufacturers: dict) -> dict[str, set[tuple[str, str]]]: + """Normalized system name -> the console folders that spell it. + + nintendo-nes, nes and the NES folder normalize alike, as do + sega-mastersystem and Sega/Master System. + """ + from common import _norm_system_id + + index: dict[str, set[tuple[str, str]]] = {} + for mfr, consoles in manufacturers.items(): + for console in consoles: + spelled = console.lower().replace(" ", "-") + for key in { + _norm_system_id(f"{_slug(mfr)}-{spelled}"), _norm_system_id(spelled) + }: + index.setdefault(key, set()).add((mfr, console)) + return index + + +def _folder_named(sys_id: str, by_name: dict[str, set[tuple[str, str]]]) -> tuple[str, str] | None: + from common import _norm_system_id + + places = by_name.get(_norm_system_id(sys_id), set()) + return next(iter(places)) if len(places) == 1 else None + + def _system_pages_from_data( manufacturers: dict, coverages: dict, @@ -189,21 +293,28 @@ def _system_pages_from_data( console_norm[key] = entry # Map system_id -> (manufacturer, console) via platform file entries + copies = _copies_by_path(db) + folders_by_name = _folders_by_name(manufacturers) pages: dict[str, tuple[str, str]] = {} for cov in coverages.values(): config = cov["config"] for sys_id, system in config.get("systems", {}).items(): if sys_id in pages: continue - # Strategy 1: the folder holding the files the system declares, + # Strategy 1: the one console folder the id spells (nintendo-nes, + # NES). A copy of the disk system BIOS in the NES list does not + # move the NES to the folder that copy sits in. + # Strategy 2: the folder holding the files the system declares, # found by content, and most votes win. A bare name stands in only # where the collection files it in a single console. - place = _system_folder(system.get("files", []), db, name_consoles) + place = _folder_named(sys_id, folders_by_name) or _system_folder( + system.get("files", []), db, name_consoles, sys_id, copies + ) if place is not None: mfr, console = place pages[sys_id] = (mfr.lower().replace(" ", "-"), console) continue - # Strategy 2: fuzzy match system_id against console directory names + # Strategy 3: fuzzy match system_id against console directory names if sys_id in console_norm: pages[sys_id] = console_norm[sys_id] else: @@ -228,7 +339,9 @@ def _system_pages_from_data( if isinstance(entry, dict) and (entry.get("system") == sys_id or len(systems) == 1) ] - place = _system_folder(files, db, name_consoles) + place = _folder_named(sys_id, folders_by_name) or _system_folder( + files, db, name_consoles, sys_id, copies + ) if place is not None: mfr, console = place pages[sys_id] = (mfr.lower().replace(" ", "-"), console) @@ -948,6 +1061,35 @@ def _write_sqlite_export( os.replace(temp_path, destination) +def write_single_item(path: Path, kind: str, item: dict, generated_at: str) -> None: + """One item in the envelope of its collection, for a page to link.""" + path.parent.mkdir(parents=True, exist_ok=True) + write_if_changed( + str(path), + json.dumps( + {"schema_version": 1, "generated_at": generated_at, "kind": kind, + "count": 1, "items": [item]}, + indent=2, sort_keys=True, ensure_ascii=False, default=str, + ) + "\n", + ) + + +def system_export_item(slug: str, manufacturer: str, consoles: dict[str, list]) -> dict: + """A system page's files by console, with the hashes its cards show.""" + fields = ("name", "path", "size", "sha1", "md5", "sha256", "crc32") + return { + "id": slug, + "name": manufacturer, + "consoles": { + console: [ + {field: entry[field] for field in fields if entry.get(field) is not None} + for entry in sorted(entries, key=lambda e: (e.get("name", ""), e.get("sha1", ""))) + ] + for console, entries in sorted(consoles.items()) + }, + } + + def generate_data_exports( docs: Path, db: dict, @@ -988,6 +1130,11 @@ def generate_data_exports( str(api / filename), json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) + # One platform without the others, as api/v1/emulators/.json is one + # profile: platforms.json is 2.8 MB for twelve platforms. + for item in platform_items: + write_single_item(api / "platforms" / f"{item['id']}.json", + "platforms", item, generated_at) write_if_changed(str(api / "stats.json"), generate_stats(stats)) write_if_changed( str(api / "database.json"), @@ -1168,7 +1315,9 @@ def generate_data_page(exports: list[dict]) -> str: "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " "[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each " - "emulator page links its own.", + "emulator page links its own. `api/v1/platforms/.json` holds one " + "platform of `platforms.json`, and `api/v1/systems/.json` the files " + "of one system page by console. Each platform and system page links its own.", "", "## Bulk downloads", "", @@ -1261,10 +1410,26 @@ def _plain_markdown(text: str) -> str: text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"!\[([^]]*)\]\([^)]*\)", r"\1", text) text = re.sub(r"\[([^]]+)\]\([^)]*\)", r"\1", text) - text = re.sub(r"[`*_~]", "", text) + # Code spans keep their text whole: validate_schemas.py read + # validateschemas.py once every underscore was taken for emphasis. + pieces = re.split(r"(`[^`]*`)", text) + text = "".join( + piece.strip("`") if piece.startswith("`") + else re.sub(r"(? str: + """A description cut at a word, never inside one: six snippets ended + in "op..." or "Fo...".""" + if len(plain) <= limit: + return plain + cut = plain[: limit + 1].rsplit(" ", 1)[0] if " " in plain[:limit] else plain[:limit] + return cut.rstrip(" ,;:-(.") + "..." + + def _page_description(markdown: str, relative: Path, title: str) -> str: key = relative.as_posix() specific = { @@ -1286,6 +1451,12 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: about = PAGE_DATA.get(key, {}).get("about", {}) pinned = any(a.get("version") for a in (about if isinstance(about, list) else [about])) kind = "source-pinned emulator firmware profile" if pinned else "emulator firmware profile" + if not PAGE_DATA.get(key, {}).get("hasPart"): + # 152 cores that load no file announced paths and hashes. + kind = kind.replace(" firmware", "") + trees = (" The data directories it reads are listed." + if "\n## Data directories\n" in markdown else "") + return f"{title}: {kind}. No BIOS or firmware file is required.{trees}" return f"{title}: {kind} with paths, hashes, requirements and validation behavior." if key.startswith("systems/") and relative.stem != "index": return f"{title}: indexed firmware files, hashes, variants, provenance and platform or emulator usage." @@ -1300,7 +1471,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: continue plain = _plain_markdown(stripped) if len(plain) >= 35: - return plain[:157].rstrip(" ,;:-") + ("..." if len(plain) > 157 else "") + return _shorten(plain) return f"{title}: RetroBIOS source-traced retrogaming firmware reference." @@ -1481,11 +1652,14 @@ def dataset_structured_data(exports: list[dict], db: dict) -> dict: def emulator_file_anchors(files: list[dict]) -> list[str]: """The id of each file card on an emulator page, in profile order. - A path declared once keeps its plain anchor. One declared for several - versions (raze's nine DUKE3D.GRP) takes what tells the version apart: - its hash, else its size, else, when even the size is shared, its - description. The page and its JSON-LD both read this, so a structured - part links to its card. + An entry that declares a hash is addressed by it: its anchor depends on + its own content, never on the entries beside it, and a second version + added under the same path moves nothing. An entry without one keeps + its plain anchor when its path is declared once, and otherwise takes + its size, else, when even the size is shared, its description. The + page and its JSON-LD both read this, so a structured part links to its + card. The plain anchor of a hashed card stays reachable through + emulator_file_aliases. """ bases = [ "file-" + ( @@ -1504,12 +1678,15 @@ def emulator_file_anchors(files: list[dict]) -> list[str]: return digest[:8] return f"{f['size']}b" if f.get("size") else "" - keys = [(b, _version_key(f)) if b in repeated else (b, "") for f, b in zip(files, bases)] + keys = [ + (b, _version_key(f)) if b in repeated or _first_hash_value(f) else (b, "") + for f, b in zip(files, bases) + ] used: dict[str, int] = {} anchors = [] for f, (base, key) in zip(files, keys): anchor = f"{base}-{key}" if key else base - if base in repeated and keys.count((base, key)) > 1 and f.get("description"): + if keys.count((base, key)) > 1 and f.get("description"): described = re.sub(r"[^a-z0-9]+", "-", str(f["description"]).lower()).strip("-") anchor = f"{anchor}-{described[:40].strip('-')}" used[anchor] = used.get(anchor, 0) + 1 @@ -1519,6 +1696,30 @@ def emulator_file_anchors(files: list[dict]) -> list[str]: return anchors +def emulator_file_aliases(files: list[dict], anchors: list[str]) -> list[str]: + """The plain file- id each card also answers to, or "". + + A link cited as emulators/raze/#file-duke3d-grp must still land once the + card is addressed by its hash: the first card of that path keeps the + plain id, unless another card already is it. + """ + taken = set(anchors) + aliases = [] + for f, anchor in zip(files, anchors): + base = "file-" + ( + re.sub( + r"[^a-z0-9]+", "-", + str(f.get("path") or f.get("standalone_path") or f.get("name", "")).lower(), + ).strip("-") or "entry" + ) + if base in taken or anchor == base: + aliases.append("") + else: + taken.add(base) + aliases.append(base) + return aliases + + def emulator_structured_data(profile: dict) -> dict: """What an emulator page states, for a reader of its JSON-LD. @@ -1621,7 +1822,9 @@ def decorate_markdown_pages(docs: Path) -> None: page_path = "" if len(relative.parts) == 1 else "/".join(relative.parts[:-1]) + "/" else: page_path = relative.with_suffix("").as_posix() + "/" - canonical = urllib.parse.urljoin(SITE_URL, page_path) + # Encoded as mkdocs encodes the canonical link and the sitemap: + # idtech4a++ named .../idtech4a++/ here and .../idtech4a%2B%2B/ there. + canonical = urllib.parse.urljoin(SITE_URL, urllib.parse.quote(page_path)) schema_type = "WebSite" if relative.as_posix() == "index.md" else "TechArticle" if relative.as_posix() == "data.md": schema_type = "Dataset" @@ -1834,6 +2037,22 @@ def _render_platform_systems( _EXPORTED: set[str] = set() +def _platform_data_links(name: str) -> str: + """The platform's own data, machine-readable: the YAML transcribed from + its upstream list, this page's JSON and the manifest the installer reads. + + An emulator page linked its YAML and JSON, a platform page nothing: a + tool had to download all twelve platforms in platforms.json and filter. + """ + links = [ + f"[YAML]({REPO_URL}/blob/main/platforms/{name}.yml)", + f"[JSON](../api/v1/platforms/{name}.json)", + ] + if (_REPO_ROOT / "install" / f"{name}.json").is_file(): + links.append(f"[install manifest]({REPO_URL}/blob/main/install/{name}.json)") + return ", ".join(links) + + def generate_platform_page( name: str, cov: dict, @@ -1927,6 +2146,7 @@ def generate_platform_page( lines.append(f"| Transcribed from | <{transcribed}>{note} |") if name in _EXPORTED: lines.append(f"| Updated file | [Download it and see the changes](../exports.md#{name}) |") + lines.append(f"| Data | {_platform_data_links(name)} |") contrib_list = (registry or {}).get(name, {}).get("contributed_by", []) if contrib_list: for cb in contrib_list: @@ -1935,9 +2155,10 @@ def generate_platform_page( pr = cb.get("pr") pr_link = f" ([#{pr}]({REPO_URL}/pull/{pr}))" if pr else "" lines.append( - f"| Contributed by | [@{username}](https://github.com/{username})" - f" - {contribution}{pr_link} |" + f"| Contributed by | [@{username}](https://github.com/{username}):" + f" {contribution}{pr_link} |" ) + lines.extend(["", *_platform_install_block(name, display)]) lines.extend( [ "", @@ -2151,21 +2372,9 @@ def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]: read from the full paths by_path_suffix records. """ files = db.get("files", {}) - bios_dir = Path(str(db.get("bios_dir") or "bios")) - places: dict[str, set[tuple[str, str]]] = {} - for sha1, entry in files.items(): - parts = str(entry.get("path", "")).split("/") - if len(parts) >= 3 and parts[0] == "bios": - places.setdefault(sha1, set()).add((parts[1], parts[2])) - for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items(): - parts = tail.split("/") - if len(parts) < 3 or parts[1].startswith(".") or len(sha1s) != 1: - continue - if sha1s[0] in files and (bios_dir / tail).is_file(): - places.setdefault(sha1s[0], set()).add((parts[0], parts[1])) manufacturers: dict[str, dict[str, list]] = {} - for sha1, where in places.items(): - for manufacturer, console in sorted(where): + for sha1, where in _copy_places(db).items(): + for manufacturer, console in sorted({(m, c) for m, c, _ in where}): manufacturers.setdefault(manufacturer, {}).setdefault(console, []).append(files[sha1]) return manufacturers @@ -2635,15 +2844,53 @@ def _card_title(text: str, anchor: str) -> str: return f"#### {text} {{ {attributes} }}" +def system_install_choices(coverages: dict) -> dict[str, dict[str, list[tuple[str, str]]]]: + """slug -> console -> the (platform, system id) pairs linked to it. + + Read through the same table as every system link, so a command sits in + the section the platform's system links to. Only a system the + platform's install manifest records is offered: --system refuses any + other name. + """ + from siterender import _page_by_norm # noqa: PLC0415 + + choices: dict[str, dict[str, list[tuple[str, str]]]] = {} + for name, cov in sorted(coverages.items()): + manifest = _REPO_ROOT / "install" / f"{name}.json" + if not manifest.is_file(): + continue + document = json.loads(manifest.read_text(encoding="utf-8")) + recorded = { + str(system) + for entry in (document.get("files") or []) + (document.get("omitted_files") or []) + for system in entry.get("systems") or [] + } + for sys_id in sorted(set(cov["config"].get("systems") or {}) & recorded): + page = _system_page_map.get(sys_id) or _page_by_norm(sys_id) + if page: + slug, console = page + choices.setdefault(slug, {}).setdefault(console, []).append((name, sys_id)) + return choices + + def generate_system_page( manufacturer: str, consoles: dict[str, list], consumers: FileConsumers, + installs: dict[str, list[tuple[str, str]]] | None = None, ) -> str: - manufacturer.lower().replace(" ", "-") + """One manufacturer's page, a section per console. + + ``installs`` gives, per console, the (platform, system id) pairs whose + install command the section offers. + """ + slug = manufacturer.lower().replace(" ", "-") lines = [ f"# {manufacturer}", "", + f"The files below by console, with every hash, are also published as " + f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json).", + "", ] anchored: set[str] = set() @@ -2652,6 +2899,7 @@ def generate_system_page( icon_md = system_icon_markdown(manufacturer, console_name) lines.append(f"## {icon_md}{console_name}") lines.append("") + lines.extend(_console_install_block(console_name, (installs or {}).get(console_name, []))) main_files = [f for f in files if "/.variants/" not in f["path"]] variant_files = [f for f in files if "/.variants/" in f["path"]] @@ -2878,6 +3126,83 @@ def _first_hash_value(entry: dict) -> str: return "" +_LOAD_FROM = { + "system_dir": "the system (BIOS) directory", + "save_dir": "the save directory, not the BIOS directory", + "content_dir": "the folder of the loaded content, not the BIOS directory", + "working_directory": "the working directory, not the BIOS directory", + "data_dir": "the emulator's data directory, not the BIOS directory", + "hdd_image": "inside the emulated hard disk image, not the BIOS directory", + "internal_storage": "the app's internal storage, not the BIOS directory", +} + + +def _by_build(value: object, words: Callable[[str], str]) -> str: + """A field keyed by build, said per build, or a plain value said once.""" + if isinstance(value, dict): + return ", ".join(f"{words(str(v))} ({mode} build)" for mode, v in value.items() if v) + return words(str(value)) if value else "" + + +def _load_from_text(value: object) -> str: + """Where the code reads a file, in words. + + The card showed the Path alone: Dolphin's libretro core reads + gba_bios.bin from its save directory, and a reader put it under system/. + """ + if value in (None, "", "system_dir"): + return "" + return _by_build(value, lambda v: _LOAD_FROM.get(v, _code(v))) + + +def _found_by_text(value: object) -> str: + """How the code finds a file: by the name it opens, or by content.""" + words = { + "name": "file name (the code opens this name)", + "content": "content (the code scans the folder and identifies the file " + "by its hash, any file name works)", + } + return _by_build(value, lambda v: words.get(v, _code(v))) + + +def _on_mismatch_lines(value: object) -> list[str]: + """What the code does when a declared check fails, where it is not reject.""" + said = {"warn": "warns and runs with the file", "reject": "rejects the file"} + if value == "warn": + return [f"On a failed check: {said['warn']}"] + if isinstance(value, dict): + return [f"On a failed {check} check: {said.get(str(how), _code(str(how)))}" + for check, how in value.items()] + return [] + + +def _build_rows(profile: dict) -> list[str]: + """Meta card rows for who ships the build, how its files are found, why + its revision is held and where its releases are announced.""" + rows = [] + built_by = profile.get("built_by") or [] + if built_by: + names = ", ".join(f"[{p}](../platforms/{p}.md)" for p in map(str, built_by)) + rows.append(f"| Built by | {names}, under its core name. The libretro " + "buildbot does not publish this build. |") + found = _found_by_text(profile.get("found_by")) + if found: + rows.append(f"| Files found by | {found} |") + frozen = profile.get("pin_frozen") + if frozen: + held = _escape_tags(' '.join(str(frozen).split())).replace('|', r'\|') + rows.append(f"| Revision held | {held} |") + announced = profile.get("version_source") + if isinstance(announced, dict) and announced.get("url"): + url = str(announced["url"]) + row = f"| Releases announced at | [{url}]({url})" + if announced.get("no_genuine_copy"): + row += (f" (version {_escape_tags(str(announced['no_genuine_copy']))} " + "announced there, found only as rebuilt copies)") + rows.append(row + " |") + return rows + + def _render_emulator_file( f: dict, profile: dict, @@ -2886,10 +3211,12 @@ def _render_emulator_file( _file_available: Callable[[dict], bool], anchor: str = "", owner: str = "", + alias: str = "", ) -> list[str]: """Render one row of an emulator profile's file table. - `anchor` is the card's id on the page, so one file can be cited by URL. + `anchor` is the card's id on the page, so one file can be cited by URL, + and `alias` a second id it answers to. Split out of generate_emulator_page, where it was a 228-line loop body carrying most of that function's branching: thirty-odd optional fields, @@ -2927,7 +3254,8 @@ def _render_emulator_file( badge_str = " ".join(badges) lines.append('
') lines.append("") - lines.append(_card_title(f"{_code(fname)} {badge_str}", anchor)) + marker = f'' if alias else "" + lines.append(_card_title(f"{marker}{_code(fname)} {badge_str}", anchor)) lines.append("") if desc: lines.extend([_escape_tags(str(desc)), ""]) @@ -2942,6 +3270,12 @@ def _render_emulator_file( # under Firmware/, and a page naming only the core's path sent its # users to the wrong folder. details.append(f"Standalone path: `{standalone_path}`") + read_from = _load_from_text(f.get("load_from")) + if read_from: + details.append(f"Read from: {read_from}") + found = _found_by_text(f.get("found_by")) + if found: + details.append(f"Found by: {found}") if fsystem: details.append(f"System: {_system_link(fsystem, '../')}") if size: @@ -2999,6 +3333,7 @@ def _render_emulator_file( elif isinstance(validation, dict): for scope, checks in validation.items(): details.append(f"Validation ({scope}): {', '.join(checks)}") + details.extend(_on_mismatch_lines(f.get("on_mismatch"))) if f.get("installed_on"): details.append("Installed by the platform's own package on: " + ", ".join(sorted(f["installed_on"]))) @@ -3022,18 +3357,21 @@ def _render_emulator_file( details.extend(_directory_copies(f, consumers)) if consumers: sha1 = consumers.sha1_of(f, f.get("path") or "", owner=owner) - if sha1 and in_repo: + held = consumers.path_of(sha1) if sha1 and in_repo else "" + parts = held.split("/") + if len(parts) >= 3 and parts[0] == "bios": # Which copy answers this entry, linked to its collection card # (provenance, every hash, download): an entry declaring no hash # said "in repo" without naming the file. - held = consumers.path_of(sha1) - parts = held.split("/") - if len(parts) >= 3 and parts[0] == "bios": - page = parts[1].lower().replace(" ", "-") - details.append( - f"Collection copy: [{_code(held)}]" - f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" - ) + page = parts[1].lower().replace(" ", "-") + details.append( + f"Collection copy: [{_code(held)}]" + f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" + ) + elif in_repo and not any(d.startswith("Collection copies:") for d in details): + copy = held_copy(f) + if copy: + details.append(_held_copy_line(copy, consumers)) plat_links = _platform_links(sha1, consumers) if sha1 else [] if plat_links: details.append(f"Platforms: {', '.join(plat_links)}") @@ -3090,6 +3428,33 @@ def _render_emulator_file( return lines +def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str: + """Where a copy the database does not index sits: a ZIP member under + bios/, or a file or tree of a data directory the packs fetch.""" + path, member = copy + parts = path.rstrip("/").split("/") + if member: + archive = _code(path) + zsha1 = consumers.sha1_at(path) + if zsha1 and len(parts) >= 3 and parts[0] == "bios": + page = urllib.parse.quote(parts[1].lower().replace(" ", "-")) + archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})" + return f"Collection copy: member {_code(member)} of {archive}" + if parts[0] == "data" and len(parts) >= 2: + ref, inside = parts[1], "/".join(parts[2:]) + if path.endswith("/"): + where = _code(inside + "/") if inside else "the tree" + return f"Collection copy: {where} in the {_code(ref)} data directory" + info = (_DATA_MEMBERS.get(ref) or {}).get(inside) or {} + shown = _code(inside) + if info.get("sha1"): + shown = (f'' + f"{html.escape(inside)}") + return f"Collection copy: {shown} in the {_code(ref)} data directory" + tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(path.rstrip('/'))}" + return f"Collection copy: [{_code(path)}]({tree})" + + def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]: """What the collection holds under a directory entry, and where. @@ -3150,6 +3515,37 @@ def _render_metadata_rows(profile: dict) -> list[str]: return lines +INSTALL_BASE = "https://raw.githubusercontent.com/Abdess/retrobios/main" + + +def install_command(shell: str, args: str = "") -> str: + """The one-liner for a shell, with installer options when given. + + PowerShell passes options only through a scriptblock: `irm ... | iex` + takes none. + """ + if shell == "sh": + return f"curl -fsSL {INSTALL_BASE}/install.sh | sh" + (f" -s -- {args}" if args else "") + if args: + return f"&([scriptblock]::Create((irm {INSTALL_BASE}/install.ps1))) {args}" + return f"irm {INSTALL_BASE}/install.ps1 | iex" + + +def install_commands(args: str = "", indent: str = "") -> list[str]: + """The same install for each place a user runs it: one generator for the + download page and the platform, system and emulator pages, so none of + them loses the Android or Windows form.""" + lines = [] + for label, shell in ( + ("Linux, macOS, Steam Deck:", "sh"), + ("Windows (PowerShell):", "powershell"), + ("Android (Termux, after `pkg install python` and `termux-setup-storage`):", "sh"), + ): + lines += [f"{indent}{label}", "", f"{indent}```{shell}", + f"{indent}{install_command(shell, args)}", f"{indent}```", ""] + return lines + + def _install_block(name: str, profile: dict) -> list[str]: """One command that installs this emulator's files where it reads them. @@ -3159,26 +3555,49 @@ def _install_block(name: str, profile: dict) -> list[str]: """ if profile.get("type") in ("alias", "launcher", "test"): return [] - base = "https://raw.githubusercontent.com/Abdess/retrobios/main" return [ '??? tip "Install this emulator\'s files"', "", " The installer finds your platform and its BIOS folder, then downloads " "only the files this emulator reads there and checks each one.", "", - " Linux, macOS, Steam Deck:", + *install_commands(f"--emulator {name}", " "), + ] + + +def _platform_install_block(name: str, display: str) -> list[str]: + """The platform's pack, installed file by file: what the ZIP holds, with + resume, verification and only what is missing.""" + return [ + '???+ tip "Install with one command"', + "", + f" The installer downloads the files of the {display} pack that are " + "missing or differ from it, checks each one, and resumes where it " + "stopped. Add `--system` with the ids of the systems overview below " + "to install fewer systems.", + "", + *install_commands(f"--platform {name}", " "), + ] + + +def _console_install_block(console: str, declared: list[tuple[str, str]]) -> list[str]: + """Each platform's command for one console, by the system id it gives it.""" + if not declared: + return [] + lines = [ + f'??? tip "Install the {console} files"', + "", + " Each platform names the system its own way. On Windows, run the " + "same options through `&([scriptblock]::Create((irm " + f"{INSTALL_BASE}/install.ps1)))`.", "", " ```sh", - f" curl -fsSL {base}/install.sh | sh -s -- --emulator {name}", - " ```", - "", - " Windows (PowerShell):", - "", - " ```powershell", - f" &([scriptblock]::Create((irm {base}/install.ps1))) --emulator {name}", - " ```", - "", ] + lines += [ + f" {install_command('sh', f'--platform {platform} --system {shlex.quote(system)}')}" + for platform, system in declared + ] + return lines + [" ```", ""] def _render_platform_details(profile: dict) -> list[str]: @@ -3326,8 +3745,6 @@ def _availability_check( # under the name belongs to another game or release. if not f.get("name") or f.get("unsourceable"): return False - if f.get("storage", "") in ("release", "large_file"): - return True return entry_source(f, index) is not None return _file_available @@ -3494,6 +3911,7 @@ def generate_emulator_page( revision = revision_markdown(profile) if revision: lines.append(f"| Revision | {revision} |") + lines.extend(_build_rows(profile)) lines.append(f"| Profiled | {profiled} |") # The YAML is the reusable document this site renders, and the JSON is the # same profile for a program. @@ -3545,10 +3963,12 @@ def generate_emulator_page( lines.extend(_emulator_file_summary(files, _file_available)) - for f, anchor in zip(files, emulator_file_anchors(files)): + anchors = emulator_file_anchors(files) + for f, anchor, alias in zip(files, anchors, emulator_file_aliases(files, anchors)): lines.extend( _render_emulator_file( - f, profile, consumers, files, _file_available, anchor, name + f, profile, consumers, files, _file_available, anchor, name, + alias, ) ) @@ -3636,6 +4056,48 @@ def _held_accepted(discrepancy: str, db: dict | None) -> str: return "" +_PROBLEM_HASHES = ("sha1", "md5", "sha256", "crc32") + + +def _declared_hashes(config: dict, detail: dict) -> dict[str, list[str]]: + """The hashes the platform declares for a reported file, by algorithm. + + The untested table said "declared hash contradicted by the local file" + and named neither: the reader could not look for the dump RetroDECK asks + for, nor report the difference. + """ + from hashing import parse_hash_list # noqa: PLC0415 + + system = (config.get("systems") or {}).get(detail.get("system", "")) or {} + for entry in system.get("files") or []: + if isinstance(entry, dict) and entry.get("name") == detail.get("name"): + return { + algo: parse_hash_list(entry.get(algo)) + for algo in _PROBLEM_HASHES if parse_hash_list(entry.get(algo)) + } + return {} + + +def _hash_cell(hashes: dict[str, list[str]]) -> str: + return "
".join( + f"{algo} " + ", ".join(_code(v) for v in values) for algo, values in hashes.items() + ) + + +def _copy_cell(path: str, algos: list[str], db: dict | None) -> str: + """The file a path holds, with its hashes in the algorithms declared.""" + if not path: + return "-" + record = next( + (rec | {"sha1": sha1} for sha1, rec in (db or {}).get("files", {}).items() + if rec.get("path") == path), + {}, + ) + hashes = {algo: [str(record[algo]).lower()] for algo in algos if record.get(algo)} + shown = f"[{_code(path)}]({_fetch_url(path)})" + return f"{shown}
{_hash_cell(hashes)}" if hashes else shown + + def _render_problem_files( platform_problems: list, settled: list | None = None, db: dict | None = None ) -> list[str]: @@ -3664,9 +4126,14 @@ def _render_problem_files( if p["required"]: entry["required"] = True elif p["status"] == "untested": - entry = untested_files.setdefault(fname, { + # Keyed by what is declared and what is held: two platforms + # asking two dumps under one name are two rows. + declared = p.get("declared") or {} + key = f"{fname}\0{sorted(declared.items())}\0{p.get('path', '')}" + entry = untested_files.setdefault(key, { "name": fname, "required": p["required"], "platforms": [], "reason": p["reason"], + "declared": declared, "path": p.get("path", ""), }) entry["platforms"].append(p["platform"]) if p.get("discrepancy"): @@ -3707,15 +4174,21 @@ def _render_problem_files( f'### Untested ' f"{_count(len(untested_files), 'file')}", "", - "Present but hash not verified.", + "Present but hash not verified. Declared is what the platform " + "asks for, the collection copy is what the path holds.", "", - "| File | Platforms | Reason |", - "|------|----------|--------|", + "| File | Platforms | Reason | Declared | Collection copy |", + "|------|----------|--------|----------|-----------------|", ]) - for fname in sorted(untested_files): - f = untested_files[fname] + for key in sorted(untested_files): + f = untested_files[key] plats = ", ".join(sorted(set(f["platforms"]))) - lines.append(f"| `{fname}` | {plats} | {f['reason']} |") + declared = f["declared"] + lines.append( + f"| {_code(f['name'])} | {plats} | {f['reason']} " + f"| {_hash_cell(declared) or '-'} " + f"| {_copy_cell(f['path'], list(declared) or ['md5'], db)} |" + ) lines.append("") if mismatch_files: @@ -3757,6 +4230,30 @@ def _render_problem_files( return lines +def _platform_lists_sentence(total: int, verified: int, untested: int, missing: int) -> str: + """The platforms' own lists in one sentence, every count said. + + Four tiles opened the page: "Verified (100%)" for 5,437 of 5,438 and + "Missing 0", above a table of red cells, on the page about what is + missing. The gaps are in the files the emulators load, and the sentence + says so. + """ + def _is(count: int) -> str: + return "is" if count == 1 else "are" + + parts = [f"{verified:,} {_is(verified)} held and pass the platform's own check"] + if untested: + parts.append(f"{untested:,} {_is(untested)} held but not verified " + "(listed under Untested below)") + parts.append(f"{missing:,} {_is(missing)} missing") + listed = ", ".join(parts[:-1]) + f" and {parts[-1]}" + return ( + f"Of the {total:,} files on the platforms' own BIOS lists, {listed}. " + "The files the emulators load beyond those lists are counted per " + "platform in the table below." + ) + + def generate_gap_analysis( profiles: dict, coverages: dict, @@ -3813,46 +4310,23 @@ def generate_gap_analysis( "reason": d.get("reason", ""), "discrepancy": d.get("discrepancy", ""), "system": d.get("system", ""), + "declared": _declared_hashes(cov.get("config") or {}, d), + "path": d.get("path") or "", }) - pct_verified = ( - f"{total_verified / total_files_verify * 100:.0f}%" - if total_files_verify - else "0%" - ) - lines = [ "# Gap analysis", "", - "Unified view of BIOS verification, file provenance, and coverage gaps.", + "What each platform and its emulators still lack, and where the " + "collection's copies come from.", "", "[Download gaps CSV](downloads/gaps.csv){ .md-button } " "[Open gaps API](api/v1/gaps.json){ .md-button } " "[All data exports](data.md){ .md-button }", "", - '
', - "", - '
', - f'{total_files_verify:,}', - 'Total files (all platforms)', - "
", - "", - '
', - f'{total_verified:,}', - f'Verified ({pct_verified})', - "
", - "", - '
', - f'{total_untested:,}', - 'Untested', - "
", - "", - '
', - f'{total_missing_verify:,}', - 'Missing', - "
", - "", - "
", + _platform_lists_sentence( + total_files_verify, total_verified, total_untested, total_missing_verify + ), "", ] @@ -4663,6 +5137,10 @@ def _mkdocs_config(docs: Path, text: str) -> tuple[Path, str]: def generate_which_pack() -> str: """Generate the 'Which pack?' decision page.""" rel = RELEASE_URL + quick = "\n".join(install_commands()) + check_sh, check_ps = install_command("sh", "--check"), install_command("powershell", "--check") + sd_card = install_command( + "sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system") return _link_pack_cells(f"""\ # Download @@ -4681,18 +5159,7 @@ incorrect files, verifies their hashes, and installs them atomically. The small bootstrap checks the Python installer against an embedded SHA-256 before running it, and the installer reads its file list from that same revision. -**Linux / Mac / Steam Deck:** - -```sh -curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -``` - -**Windows (PowerShell):** - -```powershell -irm https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1 | iex -``` - +{quick} That is the complete default flow. Extra copies into detected standalone emulator directories are deliberately opt-in with `--standalone-copies`, so automatic setup never writes outside the selected platform tree unexpectedly. @@ -4700,7 +5167,19 @@ Adding `--check` reads the BIOS folder, hashes what is there and writes nothing: ```sh -curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --check +{check_sh} +``` + +```powershell +{check_ps} +``` + +An SD card mounted on a PC is installed with `--platform` and `--dest`, the +BIOS folder on the card (RetroArch on Switch: `retroarch/cores/system`, on +Android: `RetroArch/system`): + +```sh +{sd_card} ``` Options, environment overrides, how each platform is detected and what the @@ -4724,7 +5203,7 @@ ZIP: extract them all into the same folder. | Setup | What it is | Pack | Extract to | |-------|-----------|------|-----------| | [EmuDeck](https://www.emudeck.com/) | Installs and configures multiple emulators, adds each game to the Steam library | [EmuDeck]({rel}) | `~/Emulation/bios/` | -| [RetroDECK](https://retrodeck.net/) | Single Flatpak app, all emulators bundled, one-click install from Discover | [RetroDECK]({rel}) | `~/retrodeck/` (the pack carries its own `bios/`) | +| [RetroDECK](https://retrodeck.net/) | Single Flatpak app with its emulators bundled, installed from Flathub or Discover | [RetroDECK]({rel}) | `~/retrodeck/` (the pack carries its own `bios/`) | | RetroArch standalone | Installed from Discover, Steam, or Flatpak | [RetroArch]({rel}) | Open RetroArch > Settings > Directory > System, that is the folder | ### Windows @@ -4733,7 +5212,7 @@ ZIP: extract them all into the same folder. |-------|-----------|------|-----------| | [RetroArch](https://www.retroarch.com/) | Multi-system emulator, loads different cores for each console | [RetroArch]({rel}) | The `system` folder next to `retroarch.exe` | | [RetroBat](https://www.retrobat.org/) | Windows frontend with EmulationStation, includes RetroArch and standalone emulators | [RetroBat]({rel}) | The `bios` folder inside the RetroBat installation | -| [BizHawk](https://tasvideos.org/BizHawk) | Accuracy-focused multi-system emulator, popular for speedruns and TAS | [BizHawk]({rel}) | The `Firmware` folder inside the BizHawk installation | +| [BizHawk](https://tasvideos.org/BizHawk) | Multi-system emulator with rerecording and debugging tools, hosted by TASVideos | [BizHawk]({rel}) | The `Firmware` folder inside the BizHawk installation | | [LaunchBox](https://www.launchbox-app.com/) | Game library manager and launcher, uses RetroArch or standalone emulators behind the scenes | [RetroArch]({rel}) | Open RetroArch (via LaunchBox) > Settings > Directory > System | | [EmuDeck](https://www.emudeck.com/) | Installs and configures multiple emulators with ES-DE | [EmuDeck]({rel}) | `bios` in the Emulation folder chosen in EmuDeck, `C:\\Emulation\\bios\\` by default | @@ -4757,24 +5236,24 @@ ZIP: extract them all into the same folder. | Setup | What it is | Pack | Extract to | |-------|-----------|------|-----------| -| [RetroPie](https://retropie.org.uk/) | The classic Pi emulation setup, largest community, most online guides | [RetroPie]({rel}) | `~/RetroPie/BIOS/` | -| [Lakka](https://www.lakka.tv/) | Lightweight RetroArch OS, minimal config, boots straight into the UI | [Lakka]({rel}) | `/storage/system/` | -| [Batocera](https://batocera.org/) | Easy setup, works on Pi 3/4/5 and many other boards (Odroid, etc.) | [Batocera]({rel}) | `/userdata/bios/` | -| [Recalbox](https://www.recalbox.com/) | Plug-and-play experience, good for a first build | [Recalbox]({rel}) | `/recalbox/share/bios/` | +| [RetroPie](https://retropie.org.uk/) | Scripts that install RetroArch and standalone emulators on Raspberry Pi OS | [RetroPie]({rel}) | `~/RetroPie/BIOS/` | +| [Lakka](https://www.lakka.tv/) | Operating system built around RetroArch, boots into its menu | [Lakka]({rel}) | `/storage/system/` | +| [Batocera](https://batocera.org/) | Bootable OS, with images for Raspberry Pi 3, 4 and 5 and other boards (Odroid XU4 among them) | [Batocera]({rel}) | `/userdata/bios/` | +| [Recalbox](https://www.recalbox.com/) | Bootable OS for retro gaming that configures its emulators itself | [Recalbox]({rel}) | `/recalbox/share/bios/` | ### Handhelds | Setup | What it is | Pack | Extract to | |-------|-----------|------|-----------| -| Android (Retroid Pocket, Odin, etc.) | Most Android handhelds run RetroArch | [RetroArch]({rel}) | `RetroArch/system/` on internal storage or SD card | -| [ROCKNIX](https://rocknix.org/) | Linux OS for ARM and x86 handhelds (RG35XX, RG552, Deck) | [ROCKNIX]({rel}) | `/storage/roms/bios/`, over SSH or the network share | -| [Batocera](https://batocera.org/) | Also images many handhelds | [Batocera]({rel}) | `/userdata/bios/` | +| Android (Retroid Pocket, Odin, etc.) | RetroArch for Android | [RetroArch]({rel}) | `RetroArch/system/` on internal storage or SD card | +| [ROCKNIX](https://rocknix.org/) | Linux OS for ARM and x86 handhelds, among them the RG35XX (H700) and the RG552 (RK3399) | [ROCKNIX]({rel}) | `/storage/roms/bios/`, over SSH or the network share | +| [Batocera](https://batocera.org/) | Also has images for handheld chips, among them H700 and RK3326 | [Batocera]({rel}) | `/userdata/bios/` | ### FPGA | Setup | What it is | Pack | Extract to | |-------|-----------|------|-----------| -| [MiSTer FPGA](https://mister-devel.github.io/MkDocs_MiSTer/) | Hardware-level recreation of consoles on a DE10-Nano board | [MiSTer FPGA]({rel}) | `/media/fat/games/`, one subfolder per core | +| [MiSTer FPGA](https://mister-devel.github.io/MkDocs_MiSTer/) | FPGA cores that reimplement console hardware on a DE10-Nano board | [MiSTer FPGA]({rel}) | `/media/fat/games/`, one subfolder per core | ### Self-hosted ROM manager @@ -5108,12 +5587,18 @@ def _generate(args: argparse.Namespace) -> None: write_if_changed( str(docs / "systems" / "index.md"), generate_systems_index(manufacturers) ) + console_installs = system_install_choices(coverages) (docs / "files").mkdir(parents=True, exist_ok=True) write_if_changed(str(docs / "files" / "index.md"), generate_files_page()) for mfr, consoles in manufacturers.items(): slug = mfr.lower().replace(" ", "-") - page = generate_system_page(mfr, consoles, consumers) + page = generate_system_page(mfr, consoles, consumers, console_installs.get(slug)) PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles) + write_single_item( + docs / "api" / "v1" / "systems" / f"{slug}.json", "systems", + system_export_item(slug, mfr, consoles), + str(db.get("generated_at") or _timestamp()), + ) write_if_changed(str(docs / "systems" / f"{slug}.md"), page) print("Generating emulator pages...") diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index f00c0d90..8ffe065c 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -977,6 +977,9 @@ class ProfileReport: upstream_pin: str | None = None upstream_head: str | None = None upstream_origin: str | None = None + # The release tag the upstream view is pinned on, when it is one. That + # view is read at its tag, and a bump leaves upstream_commit there. + upstream_tag: str | None = None # Repositories whose pin is no ancestor of the branch the profile # tracks: history was rewritten, or the pin lies on another branch. orphaned: list[str] = field(default_factory=list) @@ -1345,7 +1348,23 @@ def detect_pinned_tag( The declared `core_version` names the tag, which is looked up by name so that a repository publishing nightly tags cannot hide an old release behind pagination. + + Only the repository of the documented build freezes the profile. An + `upstream` a port was cut from sits on a release tag as its base + (GroovyMAME on mame0289, triforce on Dolphin 2407) while the port itself + keeps moving: that view is judged against its tag by `tag_pinned_views`, + and the port is still compared with its HEAD and still bumped. """ + documented = [view for view in views if view.field != "upstream"] or views + tagged = tag_pinned_views(profile, documented, cache_dir, offline) + return next(iter(tagged.values()), None) + + +def tag_pinned_views( + profile: dict, views: list[RepoView], cache_dir: str, offline: bool +) -> dict[tuple[str, str], str]: + """Views whose pin is the tag `core_version` names, keyed by repository.""" + tagged: dict[tuple[str, str], str] = {} for view in views: if view.pin == view.head: continue @@ -1353,8 +1372,9 @@ def detect_pinned_tag( profile.get("core_version"), view.repo.name ): if upstream.tag_commit(view.repo, tag, cache_dir, offline) == view.pin: - return tag - return None + tagged[(view.repo.host, view.repo.slug)] = tag + break + return tagged def _resolve_project_words( @@ -1538,12 +1558,27 @@ def build_report( report.pinned_tag = frozen or detect_pinned_tag( profile, views, cache_dir, offline ) + # A repository pinned on its release tag is read at that tag, whichever + # role it plays: comparing mame0174 refs with mamedev HEAD says nothing. + tagged = tag_pinned_views(profile, views, cache_dir, offline) + report.upstream_tag = next( + ( + tagged[(view.repo.host, view.repo.slug)] + for view in views + if view.field == "upstream" and (view.repo.host, view.repo.slug) in tagged + ), + None, + ) + + def on_own_revision(view: RepoView) -> bool: + return view.pin == view.head or (view.repo.host, view.repo.slug) in tagged + if not report.pinned_tag: # A tag or a held pin is off the tracked branch by design. report.orphaned = [ f"{view.repo.slug}: pin {view.pin[:8]} is no ancestor of {view.head[:8]}" for view in views - if view.pin != view.head + if not on_own_revision(view) and upstream.compare( view.repo, view.pin, view.head, cache_dir, offline ).diverged @@ -1896,9 +1931,9 @@ def build_report( if report.pinned_tag: return True if not part.role and len(path_roles.get(part.path, ())) > 1: - return all(view.pin == view.head for view in views) + return all(on_own_revision(view) for view in views) view, _ = resolve_path(part.path, part.start, tuple(tokens), role_of(part)) - return view.pin == view.head + return on_own_revision(view) def judge( part: RefPart, flag: bool, tokens: Sequence[str], hashes: Sequence[str] @@ -1931,7 +1966,7 @@ def build_report( # port's 305-356 read past the end of the upstream's 345 # lines. As in a comparison, a ref fails only everywhere. for view in views: - if view is owner or view.pin != view.head: + if view is owner or not on_own_revision(view): continue lines = fetch_from(view.repo, view.pin, part.path) if lines is None: @@ -2048,6 +2083,11 @@ def format_report(report: ProfileReport, changed_only: bool = False) -> str: ) elif report.pin and report.pin == report.head: lines.append(" pin is HEAD: checked for self-consistency") + if report.upstream_tag and not report.pinned_tag: + lines.append( + f" upstream pinned to tag {report.upstream_tag}: checked against " + "its own revision, not against HEAD" + ) for url in report.unread or []: lines.append(f" not read (unsupported host): {url}") for line in report.orphaned: @@ -2891,8 +2931,12 @@ def bump_commit( file=sys.stderr, ) return False + # An upstream pinned on a release tag is the base the port was cut from: + # its HEAD is another program, so the bump leaves it on the tag. upstream_moves = bool( - report.upstream_head and report.upstream_pin != report.upstream_head + report.upstream_head + and report.upstream_pin != report.upstream_head + and not report.upstream_tag ) if document.get("source_commit") == report.head and not upstream_moves: # Rewriting the pin to the value it already holds is not a bump, and diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 2f9c3d52..9186078a 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -419,7 +419,9 @@ def main() -> int: api_dir / name for name in ("catalog.json", "platforms.json", "emulators.json", "gaps.json") if (api_dir / name).exists() - ] + ] + sorted((api_dir / "platforms").glob("*.json")) + sorted( + (api_dir / "systems").glob("*.json") + ) errors.extend( _validate_json_files(envelope_paths, "site-api-envelope.schema.json") ) diff --git a/scripts/verify.py b/scripts/verify.py index cf5ae83c..7b2f4111 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -644,11 +644,10 @@ def find_undeclared_files( entry["required"] = True continue - # Resolution: storage flag, then name, then path basename - storage = f.get("storage", "") - if storage in ("release", "large_file"): - in_repo = True - elif shipped_on_mismatch and (f.get("md5") or f.get("sha1")): + # Resolution: name, then path basename + # storage: release names where the bytes are served from, not + # that the collection holds them: resolved like any file. + if shipped_on_mismatch and (f.get("md5") or f.get("sha1")): # The entry states what its content should be and the builder # drops a copy that contradicts it, so content decides here # too. A name match would not do: generic names collide across diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index f6315ef8..44de3c9e 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -1700,6 +1700,49 @@ class TestBuildReport(unittest.TestCase): report = build_report("test", self._versioned("v0.78"), self.dir) self.assertEqual(report.pinned_tag, "0.78") + def test_an_upstream_on_its_release_tag_does_not_freeze_the_port(self): + """GroovyMAME declares 0.289 and its upstream_commit is the mame0289 + tag. Reading that as a frozen profile stopped every comparison and + bump of the port, which keeps moving. The upstream is read at its tag, + the port against its HEAD.""" + self.files[("portpin", "src/p.c")] = ["x", "load p.bin", "y"] + self.files[("porthead", "src/p.c")] = ["x", "load p.bin", "y"] + self.files[("uppin", "src/u.c")] = ["x", "ROM_LOAD(u.bin)", "y"] + self.files[("uphead", "src/u.c")] = ["x", "", "", "", "y"] + heads = {"o/port": "porthead", "o/up": "uphead"} + profile_sync.upstream.resolve_head = ( + lambda repo, cache, offline=False, branch=None: heads[repo.slug] + ) + # A tag off the tracked branch is not an orphaned pin. + profile_sync.upstream.compare = ( + lambda repo, base, head, cache_dir, offline=False: CompareResult( + [], False, repo.slug == "o/up" + ) + ) + self.tag_commits = {"0.289": "uppin"} + profile = { + "emulator": "Port", + "source": "https://github.com/o/port", + "upstream": "https://github.com/o/up", + "source_commit": "portpin", + "upstream_commit": "uppin", + "core_version": "0.289", + "files": [ + {"name": "p.bin", "source_ref": "source src/p.c:2"}, + {"name": "u.bin", "source_ref": "upstream src/u.c:2"}, + ], + } + report = build_report("port", profile, self.dir) + self.assertIsNone(report.pinned_tag) + self.assertEqual(report.upstream_tag, "0.289") + self.assertEqual(report.orphaned, []) + self.assertEqual( + [entry.status for entry in report.entries], ["ANCHORED", "ANCHORED"] + ) + self.assertIn( + "upstream pinned to tag 0.289", profile_sync.format_report(report) + ) + def test_a_tag_pointing_elsewhere_is_not_flagged(self): self.files[("pinsha", "a.c")] = ["x", "hit"] self.files[("headsha", "a.c")] = ["x", "hit"] @@ -2983,6 +3026,24 @@ class TestBumpCommit(unittest.TestCase): self.assertEqual(rebase_refs(self.path, report), ["a.c:10-12 -> a.c:30-32"]) self.assertTrue(bump_commit(self.path, report)) + def test_an_upstream_on_its_release_tag_stays_there(self): + """GroovyMAME's upstream_commit is the mame0289 tag it merged: a bump + of the port must not carry it to mamedev HEAD, another program.""" + part = PartResult( + RefPart("a.c", 10, 12, "a.c:10-12"), "ANCHORED", None, 10, 12, [] + ) + report = ProfileReport( + name="p", repo="o/n", pin="pin", head="newhead", + entries=[EntryReport("a.bin", "a.c:10-12", "ANCHORED", [part])], + counts={"ANCHORED": 1}, + upstream_pin="uptag", upstream_head="uphead", + upstream_origin="upstream_commit", upstream_tag="mame0289", + ) + self.assertTrue(bump_commit(self.path, report)) + written = yaml.safe_load(self.path.read_text()) + self.assertEqual(written["source_commit"], "newhead") + self.assertNotIn("upstream_commit", written) + def test_upstream_pin_moves_with_source_pin(self): """A recale onto upstream HEAD lines must advance upstream_commit too.""" part = PartResult( diff --git a/tests/test_site_availability.py b/tests/test_site_availability.py index a1e09fbe..12fa9052 100644 --- a/tests/test_site_availability.py +++ b/tests/test_site_availability.py @@ -33,6 +33,13 @@ class UnsourceableIsNotHeld(unittest.TestCase): self.assertTrue(available({"name": "pak0.pk3"})) self.assertFalse(available({"name": "pak0.pk3", "unsourceable": "retail data"})) + def test_a_release_asset_is_held_only_when_collected(self): + """storage: large_file said "in repo" for kenji-nx's user-picked + firmware package, which the collection does not hold.""" + available = _availability_check(DB, set()) + self.assertFalse(available({"name": ".zip", "storage": "large_file"})) + self.assertTrue(available({"name": "pak0.pk3", "storage": "release"})) + def test_the_badge_says_why(self): entry = {"name": "pak0.pk3", "unsourceable": "retail data"} badges = " ".join(_file_badges(entry, False)) @@ -57,5 +64,50 @@ class UnsourceableIsNotHeld(unittest.TestCase): self.assertEqual(shown_held, []) +class AHeldEntrySaysWhere(unittest.TestCase): + """164 entries wore the "in repo" badge with no copy named: they are held + as a ZIP member (zesarux's specmate.rom in spectrum_opus.zip) or in a + data directory (Dolphin's fonts), which the database does not index.""" + + def test_member_file_and_tree(self): + import hashlib + import tempfile + import zipfile + + import generate_site + from cross_reference import FileConsumers, held_copy + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + data, bios = Path(tmp) / "data", Path(tmp) / "bios" + (data / "dolphin-sys" / "GC").mkdir(parents=True) + (data / "dolphin-sys" / "GC" / "font_western.bin").write_bytes(b"font") + (bios / "Arcade" / "MAME").mkdir(parents=True) + archive = bios / "Arcade" / "MAME" / "spectrum_opus.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("specmate.rom", b"rom") + roots = {"data_root": str(data), "bios_root": str(bios)} + member = held_copy({"name": "specmate.rom", + "sha1": hashlib.sha1(b"rom").hexdigest()}, **roots) + self.assertEqual(member, (str(archive), "specmate.rom")) + # A declared hash the copy does not carry names no copy. + self.assertIsNone(held_copy({"name": "specmate.rom", "sha1": "0" * 40}, **roots)) + self.assertEqual(held_copy({"name": "font_western.bin"}, **roots), + (str(data / "dolphin-sys/GC/font_western.bin"), "")) + self.assertEqual(held_copy({"name": "GC/", "type": "directory"}, **roots), + (f"{(data / 'dolphin-sys/GC').as_posix()}/", "")) + + consumers = FileConsumers({"files": {"z" * 40: { + "path": "bios/Arcade/MAME/spectrum_opus.zip"}}}, data_dir_registry={}) + line = generate_site._held_copy_line( + ("bios/Arcade/MAME/spectrum_opus.zip", "specmate.rom"), consumers) + self.assertEqual(line, "Collection copy: member `specmate.rom` of " + "[`bios/Arcade/MAME/spectrum_opus.zip`]" + "(../systems/arcade.md#sha1-zzzzzzzzzzzz)") + line = generate_site._held_copy_line(("data/dolphin-sys/GC/font_western.bin", ""), + consumers) + self.assertEqual(line, "Collection copy: `GC/font_western.bin` in the " + "`dolphin-sys` data directory") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_site_emulator_page.py b/tests/test_site_emulator_page.py new file mode 100644 index 00000000..9c5df978 --- /dev/null +++ b/tests/test_site_emulator_page.py @@ -0,0 +1,142 @@ +"""What an emulator page says about a profile and its files. + +No network and no database: pages are rendered from small profiles. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site as gs # noqa: E402 + +EMPTY_DB = {"files": {}, "indexes": {"by_name": {}}} + + +def _page(name: str, profile: dict) -> str: + return gs.generate_emulator_page(name, profile, EMPTY_DB, None, set()) + + +class WhereAndHowAFileIsRead(unittest.TestCase): + """load_from, found_by and on_mismatch were in 99, 4 and 25 entries and + on no page: Dolphin's gba_bios.bin showed a Path a RetroArch user put + under system/, while the libretro core reads it from its save directory.""" + + def test_load_from(self): + page = _page("dolphin", {"type": "standalone + libretro", "files": [ + {"name": "gba_bios.bin", "path": "GBA/gba_bios.bin", + "load_from": {"libretro": "save_dir"}}, + {"name": "hdd.rom", "load_from": "hdd_image"}, + {"name": "bios.bin", "load_from": "system_dir"}, + ]}) + self.assertIn("- Read from: the save directory, not the BIOS directory " + "(libretro build)", page) + self.assertIn("- Read from: inside the emulated hard disk image", page) + self.assertEqual(page.count("Read from:"), 2) + + def test_found_by_and_on_mismatch(self): + page = _page("x", {"type": "standalone", "found_by": "content", "files": [ + {"name": "scph5501.bin", "found_by": "content", "on_mismatch": "warn"}, + {"name": "boot.bin", "on_mismatch": {"signature": "warn"}}, + ]}) + self.assertIn("| Files found by | content (the code scans the folder", page) + self.assertIn("- Found by: content (", page) + self.assertIn("- On a failed check: warns and runs with the file", page) + self.assertIn("- On a failed signature check: warns and runs with the file", page) + + +class WhoShipsTheBuildAndWhy(unittest.TestCase): + """built_by, pin_frozen and version_source were in 9, 13 and 40 profiles + and on no page: mame0278 did not say only Recalbox ships it.""" + + def test_meta_rows(self): + page = _page("mame0278", { + "type": "libretro", "built_by": ["recalbox"], + "pin_frozen": "ROCKNIX 20261001 builds this revision.", + "version_source": {"url": "https://example.org/news", "pattern": "v(\\S+)", + "no_genuine_copy": "13.0"}, + "files": [], + }) + self.assertIn("| Built by | [recalbox](../platforms/recalbox.md), under its " + "core name.", page) + self.assertIn("| Revision held | ROCKNIX 20261001 builds this revision. |", page) + self.assertIn("| Releases announced at | [https://example.org/news]" + "(https://example.org/news) (version 13.0 announced there, " + "found only as rebuilt copies) |", page) + + +class ACardKeepsItsAddress(unittest.TestCase): + """emulators/raze/#file-duke3d-grp stopped pointing at its card once a + second version of the path was declared: the anchor of the first one + became file-duke3d-grp-.""" + + def test_a_second_version_moves_nothing(self): + one = {"name": "DUKE3D.GRP", "md5": "aaaa1111bbbb"} + two = {"name": "DUKE3D.GRP", "md5": "cccc2222dddd"} + before = gs.emulator_file_anchors([one]) + after = gs.emulator_file_anchors([two, one]) + self.assertEqual(before, ["file-duke3d-grp-aaaa1111"]) + self.assertEqual(after[1], before[0]) + + def test_the_plain_address_still_lands(self): + files = [{"name": "DUKE3D.GRP", "md5": "aaaa1111bbbb"}] + page = _page("raze", {"type": "standalone", "files": files}) + self.assertIn('', page) + self.assertIn("{ #file-duke3d-grp-aaaa1111 .rb-file-title }", page) + # A card already holding the plain id gives it to no one else. + files = [{"name": "x.bin"}, {"name": "x.bin", "sha1": "ab" * 20}] + anchors = gs.emulator_file_anchors(files) + self.assertEqual(anchors, ["file-x-bin", "file-x-bin-abababab"]) + self.assertEqual(gs.emulator_file_aliases(files, anchors), ["", ""]) + + +class WhatASearchSnippetSays(unittest.TestCase): + def test_a_core_loading_no_file_promises_no_hash(self): + """152 pages of cores that load no file announced paths and hashes.""" + rel = Path("emulators/lutro.md") + saved = dict(gs.PAGE_DATA) + self.addCleanup(lambda: (gs.PAGE_DATA.clear(), gs.PAGE_DATA.update(saved))) + gs.PAGE_DATA[rel.as_posix()] = {} + text = gs._page_description("# Lutro\n", rel, "Lutro") + self.assertNotIn("hashes", text) + self.assertIn("No BIOS or firmware file is required.", text) + gs.PAGE_DATA[rel.as_posix()] = {"hasPart": [{"name": "a.bin"}]} + self.assertIn("paths, hashes", gs._page_description("# Lutro\n", rel, "Lutro")) + + def test_a_long_paragraph_is_cut_at_a_word(self): + plain = ("files/?sha1= is the address that does not move: it opens the card " + "wherever the file is now, " * 4) + text = gs._shorten(plain) + self.assertTrue(text.endswith("...")) + self.assertLessEqual(len(text), 160) + self.assertTrue(plain.startswith(text[:-3])) + self.assertIn(plain[len(text) - 3], " ,:") + + def test_code_keeps_its_underscores(self): + self.assertEqual(gs._plain_markdown("only `validate_schemas.py` and _this_"), + "only validate_schemas.py and this") + + def test_structured_url_is_encoded_like_the_canonical(self): + """mkdocs encodes idtech4a++ as idtech4a%2B%2B in the canonical link + and the sitemap, and the JSON-LD named the unencoded URL.""" + import json + import tempfile + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + page = Path(tmp) / "emulators" / "idtech4a++.md" + page.parent.mkdir() + page.write_text("# idTech4A++\n\nSome text long enough to be a description.\n") + gs.decorate_markdown_pages(Path(tmp)) + text = page.read_text() + data = json.loads(text.split('")[0].replace("<\\/", " FBNeo, where + disksys.rom is kept as fdsbios.nes, sega-mastersystem and + sinclair-zx-spectrum to Other > zesarux.""" + + def _db(self, tmp: str) -> dict: + for rel in ("Arcade/FBNeo/fdsbios.nes", "Nintendo/Famicom Disk System/disksys.rom", + "Nintendo/NES/other.rom"): + (Path(tmp) / rel).parent.mkdir(parents=True, exist_ok=True) + (Path(tmp) / rel).write_bytes(b"x") + return { + "bios_dir": tmp, + "files": { + "d" * 40: {"path": "bios/Arcade/FBNeo/fdsbios.nes", "sha1": "d" * 40, + "name": "fdsbios.nes"}, + "e" * 40: {"path": "bios/Nintendo/NES/other.rom", "sha1": "e" * 40, + "name": "other.rom"}, + }, + "indexes": { + "by_sha1": {}, "by_md5": {}, "by_name": {}, + "by_path_suffix": { + "Arcade/FBNeo/fdsbios.nes": ["d" * 40], + "Nintendo/Famicom Disk System/disksys.rom": ["d" * 40], + "Nintendo/NES/other.rom": ["e" * 40], + }, + }, + } + + def test_the_copy_under_the_declared_name_places_the_system(self): + import tempfile + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + db = self._db(tmp) + files = [{"name": "disksys.rom", "sha1": "d" * 40}] + self.assertEqual( + gs._system_folder(files, db, {}, "nintendo-fds"), + ("Nintendo", "Famicom Disk System"), + ) + # A file named otherwise still finds the maker's copy before the + # emulator's folder. + files = [{"name": "FDS_disksys-nintendo.rom", "sha1": "d" * 40}] + self.assertEqual( + gs._system_folder(files, db, {}, "fds"), + ("Nintendo", "Famicom Disk System"), + ) + + def test_an_id_that_spells_a_folder_links_to_it(self): + import tempfile + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + db = self._db(tmp) + manufacturers = gs._group_by_manufacturer(db) + config = {"systems": {"nintendo-nes": {"files": [ + {"name": "FDS_disksys-nintendo.rom", "sha1": "d" * 40}]}}} + pages = gs._system_pages_from_data( + manufacturers, {"p": {"config": config}}, db) + self.assertEqual(pages["nintendo-nes"], ("nintendo", "NES")) + + def test_repository_systems(self): + if not (REPO_ROOT / "database.json").exists(): + self.skipTest("database.json is not built") + previous = os.getcwd() + os.chdir(REPO_ROOT) + self.addCleanup(os.chdir, previous) + from common import load_database, load_emulator_profiles + + db = load_database("database.json") + manufacturers = gs._group_by_manufacturer(db) + pages = gs._system_pages_from_data( + manufacturers, {}, db, load_emulator_profiles("emulators", skip_aliases=False)) + expected = { + "nintendo-nes": ("nintendo", "NES"), + "nintendo-fds": ("nintendo", "Famicom Disk System"), + "sega-mastersystem": ("sega", "Master System"), + "sinclair-zx-spectrum": ("sinclair", "ZX Spectrum"), + "commodore-vic20": ("commodore", "VIC-20"), + } + for sys_id, page in expected.items(): + with self.subTest(system=sys_id): + self.assertEqual(pages.get(sys_id), page) + + class SystemEmulatorsWhenNoIdMatches(unittest.TestCase): """RetroArch files NAOMI and Atomiswave as sega-dreamcast-arcade and the Enterprise as enterprise-64-128, ids no profile uses: the emulator cell diff --git a/wiki/profiling.md b/wiki/profiling.md index 6684eb45..809d79f3 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -301,7 +301,10 @@ only hold at one revision, so a few rules keep them checkable. it. - **A frozen line is pinned to its tag.** A profile documenting a release that lives as a tag in a still active repository (`pcsx2-legacy` at PCSX2 v1.6.0) - is checked against its own revision and never rebased onto the head. + is checked against its own revision and never rebased onto the head. An + `upstream` whose pin is the tag `core_version` names (GroovyMAME's base, + `mame0289`) is read at that tag the same way, while the port itself is still + compared with its head, and a bump leaves `upstream_commit` on the tag. - **A fork's base comes from the forge.** `upstream_commit` of a fork is the merge base the compare API returns. A vendored `version.h` can name the project the fork once came from. When a package ships prebuilt binaries diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 30abcf46..08e226c7 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -46,7 +46,7 @@ Every module in `tests/`, by the area it guards. | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture