diff --git a/database.json b/database.json index 5d19d83d..c95716ad 100644 --- a/database.json +++ b/database.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-03-18T16:46:00Z", + "generated_at": "2026-03-18T19:57:34Z", "total_files": 5593, "total_size": 4909044289, "files": { diff --git a/emulators/amiarcadia.yml b/emulators/amiarcadia.yml index 517bcee2..c09585c5 100644 --- a/emulators/amiarcadia.yml +++ b/emulators/amiarcadia.yml @@ -35,4 +35,8 @@ notes: | No BIOS files required. The emulator has no external firmware dependencies. Game ROMs are identified by CRC32 for automatic machine configuration. -bios_files: [] +files: [] + +# closed-source core — cannot verify via source code scan. +# firmware_count=0 in .info and docs confirm no BIOS needed. +# signetics 2636/2637 systems have no separate system ROM. diff --git a/emulators/amiberry.yml b/emulators/amiberry.yml new file mode 100644 index 00000000..335aef9d --- /dev/null +++ b/emulators/amiberry.yml @@ -0,0 +1,122 @@ +emulator: Amiberry +type: standalone + libretro +source: "https://github.com/BlitterStudio/amiberry" +upstream: "https://github.com/tonioni/WinUAE" +profiled_date: "2026-03-18" +core_version: "v6.3.4" +display_name: "Commodore - Amiga (Amiberry)" +cores: [amiberry] +systems: + - commodore-amiga + - commodore-cd32 + - commodore-cdtv + +notes: | + Amiberry is an Amiga emulator based on WinUAE, available as both + standalone and libretro core. No built-in kickstart fallback — a real + Kickstart ROM is required. + + Core option "amiberry_kickstart" selects the ROM file: + auto, kick.rom, kick13.rom, kick20.rom, kick31.rom, kick205.rom, + kick40068.A1200, kick40068.A4000, cd32.rom, cdtv.rom. + "auto" selects based on the model option. + ref: BlitterStudio/amiberry/libretro/libretro.cpp:558,621-634 + + Standalone mode ships with data/ directory (UI icons, virtual keyboards, + floppy sounds, AmigaTopaz.ttf font, gamecontrollerdb.txt). These are + compiled into the standalone binary but the libretro core may look for + them in the system directory. + +files: + # --- Required Kickstart ROMs (from .info) --- + # ref: BlitterStudio/amiberry/libretro/libretro.cpp:558 + # ref: WinUAE rommgr.cpp for ROM identification + + - name: kick34005.A500 + system: commodore-amiga + required: true + size: 262144 + crc32: c4f0f55f + note: "A500 Kickstart v1.3 rev 34.5" + source_ref: "libretro-super/dist/info/amiberry_libretro.info firmware0" + + - name: kick37350.A600 + system: commodore-amiga + required: true + size: 524288 + crc32: 43b0df7b + note: "A600 Kickstart v2.05 rev 37.350" + source_ref: "amiberry_libretro.info firmware1" + + - name: kick40068.A1200 + system: commodore-amiga + required: true + size: 524288 + crc32: 1483a091 + note: "A1200 Kickstart v3.1 rev 40.68" + source_ref: "amiberry_libretro.info firmware2" + + # --- Optional Kickstart ROMs --- + + - name: kick33180.A500 + system: commodore-amiga + required: false + size: 262144 + crc32: a6ce1636 + note: "A500 Kickstart v1.2 rev 33.180" + source_ref: "amiberry_libretro.info firmware3" + + - name: kick40068.A4000 + system: commodore-amiga + required: false + size: 524288 + crc32: d6bae334 + note: "A4000 Kickstart v3.1 rev 40.68" + source_ref: "amiberry_libretro.info firmware4" + + - name: kick40060.CD32 + system: commodore-cd32 + required: false + size: 524288 + crc32: 1e62d4a5 + note: "CD32 Kickstart v3.1 rev 40.60" + source_ref: "amiberry_libretro.info firmware5" + + - name: kick40060.CD32.ext + system: commodore-cd32 + required: false + size: 524288 + crc32: 87746be2 + note: "CD32 Extended ROM rev 40.60" + source_ref: "amiberry_libretro.info firmware6" + + # --- Alternative names accepted by core option --- + # these are common user-facing names, same ROMs as above + + - name: kick13.rom + system: commodore-amiga + required: false + note: "alias for kick34005.A500 (v1.3)" + source_ref: "libretro.cpp:630" + + - name: kick34005.CDTV + system: commodore-cdtv + required: false + note: "CDTV extended ROM" + source_ref: "libretro.cpp:558 cdtv.rom option" + + # --- Standalone data files --- + + - name: AmigaTopaz.ttf + system: commodore-amiga + required: false + mode: standalone + note: "Amiga Topaz font for UI rendering" + source_ref: "BlitterStudio/amiberry/data/AmigaTopaz.ttf" + + - name: gamecontrollerdb.txt + system: commodore-amiga + required: false + mode: standalone + note: "SDL gamepad mapping database" + source_ref: "BlitterStudio/amiberry/controllers/gamecontrollerdb.txt" diff --git a/emulators/atari800.yml b/emulators/atari800.yml index cfd759fa..a899f3df 100644 --- a/emulators/atari800.yml +++ b/emulators/atari800.yml @@ -1,6 +1,7 @@ emulator: Atari800 -type: libretro +type: standalone + libretro source: "https://github.com/libretro/libretro-atari800" +upstream: "https://github.com/atari800/atari800" profiled_date: "2026-03-18" core_version: "3.1.0" display_name: "Atari - 400/800/600XL/800XL/130XE/5200 (Atari800)" @@ -80,8 +81,13 @@ files: required: false size: 10240 md5: 4177f386a3bac989a981d3fe3388cb6c + crc32: 0e86d61d note: > - Atari 400/800 OS Rev B (NTSC). Preferred over Rev A for 400/800 mode. + Atari 400/800 OS Rev B (NTSC). CRC32 0x0e86d61d matches sysrom.c + SYSROM_B_NTSC. Preferred over Rev A for 400/800 mode. + Note: .info lists md5 a3e8d617c95d08031fe1b20d541434b2 which is a + different dump (crc32 3e28a1fe) — both in repo, the CRC32-matched + version is the one sysrom.c auto-detects. source_ref: "atari800/src/sysrom.c:89, libretro info firmware3" # -- Atari XL/XE OS -- diff --git a/emulators/azahar.yml b/emulators/azahar.yml index 80880a60..f0561173 100644 --- a/emulators/azahar.yml +++ b/emulators/azahar.yml @@ -1,8 +1,28 @@ -emulator: "azahar" +emulator: Azahar type: alias -alias_of: "citra" +alias_of: citra +source: "https://github.com/azahar-emu/azahar" +upstream: "https://github.com/azahar-emu/azahar" profiled_date: "2026-03-18" core_version: "Git" display_name: "Nintendo - 3DS (Azahar)" -note: "This core uses the same BIOS/firmware as citra. See emulators/citra.yml for details." + +notes: | + Azahar is the successor to Citra (via Lime3DS). Same codebase, same + BIOS/firmware requirements. See emulators/citra.yml for the full file list. + + Key difference: data directory changed from citra-emu/lime3ds-emu to + azahar-emu. Legacy dirs are still scanned as fallback. + ref: azahar/src/common/common_paths.h:41-47 + + Files needed (all optional, HLE fallback for most): + sysdata/keys.txt, sysdata/boot9.bin, sysdata/sector0x96.bin, + sysdata/shared_font.bin, sysdata/seeddb.bin, sysdata/otp.bin, + nand/private/movable.sed, nand/rw/sys/SecureInfo_A, + nand/rw/sys/LocalFriendCodeSeed_B + + .info has firmware_count=0 but core uses RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY + (src/citra_libretro/environment.cpp:194) to locate system files. + files: [] +# all files documented in emulators/citra.yml — same codebase diff --git a/emulators/b2.yml b/emulators/b2.yml index e51cda0f..1dc929b0 100644 --- a/emulators/b2.yml +++ b/emulators/b2.yml @@ -1,106 +1,45 @@ emulator: b2 -type: standalone -source: "https://github.com/tom-seddon/b2" +type: standalone + libretro +source: "https://github.com/zoltanvb/b2-libretro" +upstream: "https://github.com/tom-seddon/b2" profiled_date: "2026-03-18" core_version: "v0.1" display_name: "Acorn - BBC Micro (b2)" -cores: [] +cores: [b2] systems: - bbc-micro-b - bbc-micro-bplus - bbc-master-128 - bbc-master-compact -# b2 is a standalone BBC Micro emulator by Tom Seddon. It is NOT a libretro -# core -- there is no libretro port. The emulator uses SDL2 and Dear ImGui. -# -# Emulated models (src/b2/BeebConfig.cpp:211-433): -# - BBC Model B (with various disc interfaces: Acorn 1770, Watford, Opus) -# - BBC Model B+ / B+128 -# - BBC Master 128 (MOS 3.20 and 3.50) -# - BBC Master Turbo (MOS 3.20 and 3.50, with 65C102 parasite) -# - BBC Model B + 6502 second processor -# - BBC Master Compact (MOS 5.00, 5.10, I5.10C) -# - Olivetti PC 128 S (Master Compact variant) -# -# ROM handling: -# All ROMs ship in the repo under etc/roms/. The emulator loads them via -# GetAssetPath("roms", path) at runtime. No external BIOS files needed. -# ROM paths are defined in src/b2/roms.cpp as BeebROM structs with -# {filename, display_name, StandardROM_enum}. -# -# ROM inventory (etc/roms/): -# -# BBC Model B: -# OS12.ROM 16384 md5:0a59a5ba15fe8557b5f7fee32bbd393a OS 1.20 -# BASIC2.ROM 16384 md5:2cc67be4624df4dc66617742571a8e3d BASIC II -# -# BBC Model B+: -# B+MOS.rom 16384 md5:47a1eff6c7fbb47be780c87dfd50ece1 B+ MOS -# BASIC2.ROM 16384 (shared with Model B) -# -# Disc interfaces (sideways ROMs, slot 14): -# acorn/DFS-2.26.rom 16384 md5:f083f49d6fe66344c650d7e74249cb96 Acorn 1770 DFS -# watford/DDFS-1.53.rom 16384 md5:d2b71fa664f6cdf8c24b9b304777998c Watford DDFS (DDB2) -# watford/DDFS-1.54T.rom 16384 md5:e73a0417278504a6dfac2ccd9a73e9a9 Watford DDFS (DDB3) -# opus/OPUS-DDOS-3.45.rom 16384 md5:d98316eaff56ceab4fe3bfe54d2f92b2 Opus DDOS -# opus/challenger-1.01.rom 16384 md5:67e086a11834f8cc69175b601c4e5a5f Opus Challenger -# -# BBC Master 128 MOS 3.20 (M128/3.20/): -# mos.rom 16384 md5:9087f772447038a00178488b3591347b MOS 3.20 OS -# terminal.rom 16384 md5:7e053353b78b4631c274280931d96f02 Terminal (slot f) -# view.rom 16384 md5:178d09285727d596f8863f94a2fc8dad View (slot e) -# adfs.rom 16384 md5:10f08b255d6a6e3e888631946b628f1a ADFS (slot d) -# basic4.rom 16384 md5:63a6f43cbf491533cb03ceca2d2728d4 BASIC IV (slot c) -# edit.rom 16384 md5:cb3d762eaab6130d1360e1ac575589de Edit (slot b) -# viewsht.rom 16384 md5:7d5a80a8ce929d0a0f6ca8626cb9f436 Viewsheet (slot a) -# dfs.rom 16384 md5:865a0f7a5d6f10f48798801bf514d761 DFS (slot 9) -# -# BBC Master 128 MOS 3.50 (M128/3.50/): -# mos.rom 16384 md5:d7a046b1e6a9991748cee05332f1e258 MOS 3.50 OS -# terminal.rom 16384 md5:b1fa51fd49d31224ecb7e10e6908ec01 Terminal (slot f) -# view.rom 16384 md5:1566fab2f1827fb7ba61b283b828731d View (slot e) -# adfs.rom 16384 md5:fb8ceb72c5fbd9dd80eedd10d076b1a2 ADFS (slot d) -# basic4.rom 16384 md5:e11eed95d1caba8aa9772e9001590585 BASIC IV (slot c) -# edit.rom 16384 md5:8c72c7f24567cb4a2fb15eb0b0aeb55b Edit (slot b) -# viewsht.rom 16384 md5:1bc68b7c5eba6a2de2a43c9d1d9d983b Viewsheet (slot a) -# dfs.rom 16384 md5:22c2b7f14f244da21d630a7828541c79 DFS (slot 9) -# -# Parasite processors: -# MasterTurboParasite.rom 2048 md5:83d73e0e78693bb4b43e7cb18e58d556 65C102 TUBE 1.20 -# TUBE110.rom 2048 md5:a77c7946128ca0b2c5e5a66b17c99dae 6502 TUBE 1.10 -# -# BBC Master Compact MOS 5.00 (MCompact/5.00/): -# mos.rom 16384 md5:5d1b635e39aa461f17db2344870dda36 MOS 5.00 OS -# utils.rom 16384 md5:4f9ded16f44278a4ca0ef4e66822e316 UTILS (slot f) -# basic4.rom 16384 md5:64b3cd52e36b5262ac6ca3a9823c6ee2 BASIC IV (slot e) -# adfs.rom 16384 md5:bfbd0a7dbbfb1f1f4ba5ac0343c1f8ce ADFS (slot d) -# -# BBC Master Compact MOS 5.10 (MCompact/5.10/): -# mos.rom 16384 md5:f65eb89c7044e328b79f2ef8e51b7fd8 MOS 5.10 OS -# utils.rom 16384 md5:48a197850113b00a1eb141fb6d3ea135 UTILS (slot f) -# basic4.rom 16384 md5:64b3cd52e36b5262ac6ca3a9823c6ee2 BASIC IV (slot e, same as 5.00) -# adfs.rom 16384 md5:c2afc1cabb135e6f2254bcf9300a9b83 ADFS (slot d) -# -# Olivetti PC 128 S / MOS I5.10C (MCompact/I5.10C/): -# mos.rom 16384 md5:5e16a38bd7fd6f07f26b5a64826e4ec6 MOS I5.10C OS -# utils.rom 16384 md5:75364aa6fd5e134a40051d9ee772df37 UTILS (slot f) -# basic4.rom 16384 md5:717df34edae5b6bf5bbd8879c0d13022 BASIC IV (slot e) -# adfs.rom 16384 md5:10f23a239dde39b7fe09d72083709e93 ADFS (slot d) -# -# ROM type system (src/beeb/include/beeb/roms.inl): -# Sideways ROMs support banked mappers: 16KB (standard), CCIWORD (32KB), -# CCIBASE (64KB), CCISPELL (128KB), PALQST/PALWAP/PALTED, ABEP, ABE, -# Trilogy (64KB), MO2 (128KB). OS ROM types: 16KB, Compact (64KB), -# MegaROM (128KB), Multi-OS (512KB with 4 banks). -# -# Memory map (src/beeb/include/beeb/type.h): -# OS ROM mapped at $C000-$FFFF (MOS big pages). Sideways ROMs in 16 banks -# at $8000-$BFFF, selected via ROMSEL register at $FE30. - notes: | - b2 is a standalone BBC Micro emulator (SDL2 + Dear ImGui). Not a libretro core. - All required ROMs are bundled in the repository under etc/roms/. - No external BIOS files need to be provided by the user. + b2 is a BBC Micro emulator by Tom Seddon. Libretro port by Zoltan Balogh. + ref: zoltanvb/b2-libretro, tom-seddon/b2 -bios_files: [] + Standalone: loads ROMs from etc/roms/ via GetAssetPath(). + ref: tom-seddon/b2/src/b2/roms.cpp:9-10 + + Libretro port (experimental): GetAssetPath stubbed in + src/libretro/adapters.cpp to return relative paths. LoadFile also + stubbed (returns false). ROM loading mechanism unclear — the 256 ROM + files are in the repo but the stub suggests they may not load at runtime. + is_experimental=true in .info. + ref: zoltanvb/b2-libretro/src/libretro/adapters.cpp + + 256 ROM files in etc/roms/ cover: BBC Model B (OS 1.20, BASIC II), + Model B+ (B+MOS), Master 128 (MOS 3.20, 3.50), Master Compact + (MOS 5.00, 5.10, I5.10C/Olivetti), parasite processors (65C102 TUBE, + 6502 TUBE), disc interfaces (Acorn 1770, Watford, Opus). + + All ROMs ship with both standalone and libretro repos. + No external BIOS files needed by the user for either mode. + +files: [] +# 256 ROMs bundled in repo etc/roms/ — not loaded from system dir +# standalone loads via GetAssetPath("roms", path) -> etc/roms/ +# libretro port has LoadFile stubbed — ROM loading mechanism TBD +# is_experimental=true, core may not fully function + +# closed-source analysis note: core binary on buildbot contains +# "Retro ROM DIRECTORY" and "system_dir" strings but LoadFile stub +# returns false. needs further investigation when port matures. diff --git a/emulators/bk.yml b/emulators/bk.yml index 9d514a4d..c6a827e8 100644 --- a/emulators/bk.yml +++ b/emulators/bk.yml @@ -19,16 +19,16 @@ systems: # Slow BK-0011M (model 4) - same ROMs as model 3 # Terak 8510/a (model 9) # -# ROM loading (boot.c:83-116): +# ROM loading (boot.c:84-117): # BK-0010 models use load_rom() which maps ROM into emulated memory at # specific addresses: monitor at 0100000 (8 KB), BASIC/FOCAL at 0120000 # (24448-24576 bytes), disk controller at 0160000 (4 KB). # BK-0011M uses load_rom11() which loads into separate ROM buffer arrays. # -# ROM file resolution (libretro.c:1051-1108): -# Files are loaded from {system_dir}/bk/{filename} (libretro.c:724-726). +# ROM file resolution (libretro.c:1051-1094): +# Files are loaded from {system_dir}/bk/{filename} (libretro.c:720-728). # If not found with original casing, the loader retries with lowercase -# filename (libretro.c:1074-1079). +# filename (libretro.c:1071-1081). # # All ROM files are required for their respective model. The core calls # environ_cb(RETRO_ENVIRONMENT_SHUTDOWN) if any ROM file is missing. diff --git a/emulators/blastem.yml b/emulators/blastem.yml index b6709870..7f6e2333 100644 --- a/emulators/blastem.yml +++ b/emulators/blastem.yml @@ -1,6 +1,7 @@ emulator: BlastEm -type: libretro +type: standalone + libretro source: "https://github.com/libretro/blastem" +upstream: "https://www.retrodev.com/blastem/" profiled_date: "2026-03-18" core_version: "v0.6.3-pre" display_name: "Sega - Mega Drive - Genesis (BlastEm)" @@ -28,7 +29,28 @@ notes: | Sega CD / Mega CD is not supported (only a TODO comment in system.c:16). No BIOS files are required or loaded by the libretro core. -files: [] +files: + # rom.db is embedded in the libretro binary (libblastem.c:532-540, + # extern const char rom_db_data[]). The external file is an optional + # override — .info lists it as firmware0_opt=true. + # .info sha1 E5414FB1C4CC7D7F5101C07E4547316779BA3D97 does not match + # the version in our repo (sha1 02c287d1...) — rom.db changes between + # BlastEm releases. + - name: rom.db + system: sega-megadrive + required: false + note: "ROM feature database (game-specific mapper/save config). Embedded in core, external is optional override." + source_ref: "libblastem.c:532-540 read_bundled_file()" + + # TMSS ROM — only used in standalone mode with tmss-enabled model + # libretro build returns NULL for tmss.md (libblastem.c:541) + - name: tmss.md + system: sega-megadrive + required: false + mode: standalone + size: 2048 + note: "TMSS startup ROM. Standalone only (models md1va6, md2va1+). Libretro core uses md1va3 (tmss off), no way to change." + source_ref: "genesis.c:1910-1923" analysis: tmss: diff --git a/emulators/bluemsx.yml b/emulators/bluemsx.yml index f52bed4a..1a369102 100644 --- a/emulators/bluemsx.yml +++ b/emulators/bluemsx.yml @@ -1,6 +1,7 @@ emulator: blueMSX -type: libretro +type: standalone + libretro source: "https://github.com/libretro/blueMSX-libretro" +upstream: "http://bluemsx.msxblue.com/" profiled_date: "2026-03-18" core_version: "SVN" display_name: "MSX/SVI/ColecoVision/SG-1000 (blueMSX)" @@ -52,6 +53,13 @@ notes: | C-BIOS machines work for cartridge-based games without any copyrighted BIOS. For disk/tape support, real BIOS ROMs are required. +data_directories: + # 302 files: 296 in Machines/ (200 machine configs + ROMs) + 6 in Databases/ + # buildbot source: https://buildbot.libretro.com/assets/system/blueMSX.zip + # ref: libretro.c:1118-1119 — system_dir/Machines/ + system_dir/Databases/ + - ref: bluemsx + source_ref: "libretro.c:1118-1119" + files: # ============================================================ # Shared ROMs (Machines/Shared Roms/) diff --git a/emulators/freeintv_ts_overlay.yml b/emulators/freeintv_ts_overlay.yml index 12d45cfe..d2be9b1c 100644 --- a/emulators/freeintv_ts_overlay.yml +++ b/emulators/freeintv_ts_overlay.yml @@ -26,7 +26,10 @@ notes: | ROM-specific overlays are optional 370x600 PNG files placed by the user in system/freeintv_overlays/.png. These are cosmetic per-game keypad - images and are not BIOS files. + images loaded via stbi_load() (src/libretro.c:273). Core falls back to + embedded default_keypad_image.h if no per-game overlay found. + 90 overlays available in Assets/Overlays.zip in the core repo. + Not on RetroArch buildbot — sourced from core repo via _data_dirs.yml. BIOS loading is unchanged from the base FreeIntv core. retro_load_game() (src/libretro.c:1169-1174) loads exec.bin and grom.bin from the system @@ -34,6 +37,10 @@ notes: | bios_identical_to: freeintv +data_directories: + - ref: freeintv-overlays + destination: freeintv_overlays + files: - name: "exec.bin" system: intellivision diff --git a/platforms/_data_dirs.yml b/platforms/_data_dirs.yml index 29e86748..9044697f 100644 --- a/platforms/_data_dirs.yml +++ b/platforms/_data_dirs.yml @@ -1,38 +1,166 @@ # Data directory sources for libretro cores. # Platforms reference entries by key via data_directories: [{ref: key, destination: path}]. # Pack generator auto-refreshes from source. Use --offline to skip. +# +# Source rules: +# - RetroArch/Lakka/RetroPie: use buildbot.libretro.com (official RetroArch channel) +# ref: RetroArch/config.def.h DEFAULT_BUILDBOT_ASSETS_SERVER_URL +# - Other platforms (Batocera, Recalbox, RetroBat): use their own upstream sources +# - for_platforms: restricts which platforms use this entry +# if absent, entry is available to all platforms +# # Each entry cites the original emulator source code that requires the directory. data_directories: + + # ========================================================================= + # RetroArch buildbot sources (retroarch, lakka, retropie only) + # https://buildbot.libretro.com/assets/system/ + # ========================================================================= + # ref: DolphinLibretro/Boot.cpp:72-73 — system/dolphin-emu/Sys/ dolphin-sys: - source_url: "https://github.com/libretro/dolphin/archive/{version}.tar.gz" - source_path: "dolphin-{version}/Data/Sys" - version: master + source_url: "https://buildbot.libretro.com/assets/system/Dolphin.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] local_cache: data/dolphin-sys exclude: [Themes] description: "Dolphin system data (GameSettings, DSP, fonts, shaders)" # ref: ppsspp/ext/native/ui/ui_screen.cpp — system/PPSSPP/ ppsspp-assets: - source_url: "https://github.com/hrydgard/ppsspp/archive/{version}.tar.gz" - source_path: "ppsspp-{version}/assets" - version: master + source_url: "https://buildbot.libretro.com/assets/system/PPSSPP.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] local_cache: data/ppsspp-assets description: "PPSSPP fonts, backgrounds, shaders, lang files" - # ref: bluemsx-libretro/system/ — system/Databases/ - bluemsx-databases: - source_url: "https://github.com/libretro/blueMSX-libretro/archive/{version}.tar.gz" - source_path: "blueMSX-libretro-{version}/system/Databases" - version: master - local_cache: data/bluemsx-databases - description: "blueMSX machine database" + # ref: bluemsx-libretro/system/ — system/Databases/ + system/Machines/ + bluemsx: + source_url: "https://buildbot.libretro.com/assets/system/blueMSX.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/bluemsx + description: "blueMSX Databases + Machines configs" - # ref: bluemsx-libretro/system/ — system/Machines/ - bluemsx-machines: - source_url: "https://github.com/libretro/blueMSX-libretro/archive/{version}.tar.gz" - source_path: "blueMSX-libretro-{version}/system/Machines" + # ref: pcsx2/libretro/main.cpp — system/pcsx2/ + lrps2: + source_url: "https://buildbot.libretro.com/assets/system/LRPS2.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/lrps2 + description: "LRPS2 GameIndex.yaml + resources" + + # ref: scummvm/backends/platform/libretro — system/scummvm/ + scummvm: + source_url: "https://buildbot.libretro.com/assets/system/ScummVM.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/scummvm + description: "ScummVM extra data files (themes, translations, shaders)" + + # ref: nxengine-libretro — system/ + nxengine: + source_url: "https://buildbot.libretro.com/assets/system/NXEngine%20%28Cave%20Story%29.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/nxengine + description: "NXEngine Cave Story game data" + + # ref: ecwolf/src/wl_main.cpp — system/ecwolf.pk3 + ecwolf: + source_url: "https://buildbot.libretro.com/assets/system/ECWolf.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/ecwolf + description: "ECWolf game engine data" + + # ref: prboom/src/d_main.c — system/prboom.wad + prboom: + source_url: "https://buildbot.libretro.com/assets/system/PrBoom.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/prboom + description: "PrBoom Doom engine data" + + # ref: xrick/src/data.c — system/xrick/data.zip + xrick: + source_url: "https://buildbot.libretro.com/assets/system/XRick%20%28Rick%20Dangerous%29.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/xrick + description: "XRick Rick Dangerous game data" + + # ref: dinothawr/libretro.cpp — system/dinothawr/ + dinothawr: + source_url: "https://buildbot.libretro.com/assets/system/Dinothawr.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/dinothawr + description: "Dinothawr game data" + + # ref: qemu — system firmware + qemu: + source_url: "https://buildbot.libretro.com/assets/system/QEMU.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/qemu + description: "QEMU firmware (SeaBIOS, VGA BIOS, OpenSBI, etc.)" + + # ref: cannonball/src/main.cpp — system/ + cannonball: + source_url: "https://buildbot.libretro.com/assets/system/Cannonball%20%28ROMs%20Required%29.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/cannonball + description: "Cannonball OutRun engine config" + + # ref: fbneo hiscore.dat — system/fbneo/ + fbneo-hiscore: + source_url: "https://buildbot.libretro.com/assets/system/FinalBurn%20Neo%20%28hiscore%29.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/fbneo-hiscore + description: "FinalBurn Neo hiscore database" + + # ref: mame2003/src — system/mame2003/ + mame2003: + source_url: "https://buildbot.libretro.com/assets/system/MAME%202003.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/mame2003 + description: "MAME 2003 samples + cheat" + + # ref: mame2003-plus — system/mame2003-plus/ + mame2003-plus: + source_url: "https://buildbot.libretro.com/assets/system/MAME%202003-Plus.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/mame2003-plus + description: "MAME 2003-Plus samples + cheat + history" + + # ========================================================================= + # Upstream repo sources (not on buildbot, available to all platforms) + # ========================================================================= + + # ref: FreeIntv/src/libretro.c:273 — system/freeintv_overlays/.png + # 90 per-game controller overlay PNGs, loaded via stbi_load() + # optional: core falls back to embedded default keypad image + # freeintv overlays: Assets/Overlays.zip in repo (not a directory) + # Already archived in bios/Mattel/Intellivision/freeintv_overlays/ + # No buildbot source — sourced manually from core repo + freeintv-overlays: + source_url: "https://github.com/libretro/FreeIntv/raw/{version}/Assets/Overlays.zip" + source_type: zip version: master - local_cache: data/bluemsx-machines - description: "blueMSX machine ROM configs" + local_cache: data/freeintv-overlays + strip_components: 1 + description: "FreeIntv per-game controller overlay PNGs" + + # ref: dirksimple — system/ + dirksimple: + source_url: "https://buildbot.libretro.com/assets/system/DirkSimple.zip" + source_type: zip + for_platforms: [retroarch, lakka, retropie] + local_cache: data/dirksimple + description: "DirkSimple Dragon's Lair engine data" diff --git a/platforms/retroarch.yml b/platforms/retroarch.yml index 9320fcf4..c3ef0f0d 100644 --- a/platforms/retroarch.yml +++ b/platforms/retroarch.yml @@ -814,6 +814,9 @@ systems: md5: 0cd5946c6473e42e8e4c2137785e427f crc32: 683a4158 size: 2048 + data_directories: + - ref: freeintv-overlays + destination: freeintv_overlays microsoft-msx: files: - name: CARTS.SHA @@ -926,10 +929,8 @@ systems: Labs docs: https://docs.libretro.com/library/bluemsx/ data_directories: - - ref: bluemsx-databases - destination: Databases - - ref: bluemsx-machines - destination: Machines + - ref: bluemsx + destination: '' nec-pc-engine: files: - name: gecard.pce diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 659d31aa..a10b24da 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -46,9 +46,10 @@ def load_emulator_profiles(emulators_dir: str) -> dict[str, dict]: return profiles -def load_platform_files(platforms_dir: str) -> dict[str, set[str]]: - """Load all platform configs and collect declared filenames per system.""" +def load_platform_files(platforms_dir: str) -> tuple[dict[str, set[str]], dict[str, set[str]]]: + """Load all platform configs and collect declared filenames + data_directories per system.""" declared = {} + platform_data_dirs = {} for f in sorted(Path(platforms_dir).glob("*.yml")): if f.name.startswith("_"): continue @@ -58,7 +59,11 @@ def load_platform_files(platforms_dir: str) -> dict[str, set[str]]: name = fe.get("name", "") if name: declared.setdefault(sys_id, set()).add(name) - return declared + for dd in system.get("data_directories", []): + ref = dd.get("ref", "") + if ref: + platform_data_dirs.setdefault(sys_id, set()).add(ref) + return declared, platform_data_dirs def _find_in_repo(fname: str, by_name: dict[str, list], by_name_lower: dict[str, str]) -> bool: @@ -81,12 +86,15 @@ def cross_reference( profiles: dict[str, dict], declared: dict[str, set[str]], db: dict, + platform_data_dirs: dict[str, set[str]] | None = None, ) -> dict: """Compare emulator profiles against platform declarations. Returns a report with gaps (files emulators need but platforms don't list) - and coverage stats. + and coverage stats. Files covered by matching data_directories between + emulator profile and platform config are not reported as gaps. """ + platform_data_dirs = platform_data_dirs or {} by_name = db.get("indexes", {}).get("by_name", {}) by_name_lower = {k.lower(): k for k in by_name} report = {} @@ -99,6 +107,15 @@ def cross_reference( for sys_id in systems: platform_names.update(declared.get(sys_id, set())) + # data_directories: check if the emulator's data_dir refs are provided + # by ANY platform for ANY system (not limited to matching system IDs, + # since emulator profiles and platforms use different ID conventions) + all_plat_dd_refs = set() + for dd_set in platform_data_dirs.values(): + all_plat_dd_refs.update(dd_set) + emu_dd_refs = {dd.get("ref", "") for dd in profile.get("data_directories", [])} + covered_dd = emu_dd_refs & all_plat_dd_refs + gaps = [] covered = [] for f in emu_files: @@ -114,6 +131,9 @@ def cross_reference( continue in_platform = fname in platform_names + # files covered by shared data_directories are effectively in the platform pack + if not in_platform and covered_dd: + in_platform = True in_repo = _find_in_repo(fname, by_name, by_name_lower) entry = { @@ -200,9 +220,9 @@ def main(): print("No emulator profiles found.", file=sys.stderr) return - declared = load_platform_files(args.platforms_dir) + declared, plat_data_dirs = load_platform_files(args.platforms_dir) db = load_database(args.db) - report = cross_reference(profiles, declared, db) + report = cross_reference(profiles, declared, db, plat_data_dirs) if args.json: print(json.dumps(report, indent=2)) diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 55bbd883..0f3d421d 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -388,8 +388,12 @@ def generate_pack( if not ref_key or not data_registry or ref_key not in data_registry: continue entry = data_registry[ref_key] + allowed = entry.get("for_platforms") + if allowed and platform_name not in allowed: + continue local_path = entry.get("local_cache", "") if not local_path or not os.path.isdir(local_path): + print(f" WARNING: data directory '{ref_key}' not cached at {local_path} — run refresh_data_dirs.py") continue dd_dest = dd.get("destination", "") dd_prefix = f"{base_dest}/{dd_dest}" if base_dest else dd_dest diff --git a/scripts/refresh_data_dirs.py b/scripts/refresh_data_dirs.py index 7674236c..08d33d56 100644 --- a/scripts/refresh_data_dirs.py +++ b/scripts/refresh_data_dirs.py @@ -21,6 +21,7 @@ import tarfile import tempfile import urllib.error import urllib.request +import zipfile from pathlib import Path try: @@ -203,6 +204,81 @@ def _download_and_extract( return file_count +def _download_and_extract_zip( + source_url: str, + local_cache: str, + exclude: list[str] | None = None, + strip_components: int = 0, +) -> int: + """Download ZIP, extract to local_cache. Returns file count. + + strip_components removes N leading path components from each entry + (like tar --strip-components). Useful when a ZIP has a single root + directory that should be flattened. + """ + exclude = exclude or [] + cache_dir = Path(local_cache) + + with tempfile.TemporaryDirectory() as tmpdir: + zip_path = Path(tmpdir) / "archive.zip" + log.info("downloading %s", source_url) + + req = urllib.request.Request(source_url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp: + with open(zip_path, "wb") as f: + while True: + chunk = resp.read(65536) + if not chunk: + break + f.write(chunk) + + extract_dir = Path(tmpdir) / "extract" + extract_dir.mkdir() + file_count = 0 + + with zipfile.ZipFile(zip_path) as zf: + for info in zf.infolist(): + if info.is_dir(): + continue + name = info.filename + if ".." in name or name.startswith("/"): + continue + # strip leading path components + parts = name.split("/") + if strip_components > 0: + if len(parts) <= strip_components: + continue + parts = parts[strip_components:] + name = "/".join(parts) + # skip excludes (check against stripped path) + top = parts[0] if parts else "" + if top in exclude: + continue + dest = extract_dir / name + dest.parent.mkdir(parents=True, exist_ok=True) + with zf.open(info) as src, open(dest, "wb") as dst: + shutil.copyfileobj(src, dst) + file_count += 1 + + if cache_dir.exists(): + shutil.rmtree(cache_dir) + cache_dir.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(extract_dir), str(cache_dir)) + + return file_count + + +def _get_remote_etag(source_url: str) -> str | None: + """HEAD request to get ETag or Last-Modified for freshness check.""" + try: + req = urllib.request.Request(source_url, method="HEAD", + headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp: + return resp.headers.get("ETag") or resp.headers.get("Last-Modified") or "" + except (urllib.error.URLError, OSError): + return None + + def refresh_entry( key: str, entry: dict, @@ -215,46 +291,55 @@ def refresh_entry( Returns True if the entry was refreshed (or would be in dry-run mode). """ + source_type = entry.get("source_type", "tarball") version = entry.get("version", "master") source_url = entry["source_url"].format(version=version) - source_path = entry["source_path"].format(version=version) local_cache = entry["local_cache"] exclude = entry.get("exclude", []) versions = _load_versions(versions_path) cached = versions.get(key, {}) - cached_sha = cached.get("sha") + cached_tag = cached.get("sha") or cached.get("etag") needs_refresh = force or not Path(local_cache).exists() + remote_tag: str | None = None if not needs_refresh: - remote_sha = get_remote_sha(entry["source_url"], version) - if remote_sha is None: + if source_type == "zip": + remote_tag = _get_remote_etag(source_url) + else: + remote_tag = get_remote_sha(entry["source_url"], version) + if remote_tag is None: log.warning("[%s] could not check remote, skipping", key) return False - needs_refresh = remote_sha != cached_sha - else: - remote_sha = get_remote_sha(entry["source_url"], version) if not force else None + needs_refresh = remote_tag != cached_tag if not needs_refresh: - log.info("[%s] up to date (sha: %s)", key, cached_sha[:12] if cached_sha else "?") + log.info("[%s] up to date (tag: %s)", key, (cached_tag or "?")[:12]) return False if dry_run: - log.info("[%s] would refresh (version: %s, cached sha: %s)", key, version, cached_sha or "none") + log.info("[%s] would refresh (type: %s, cached: %s)", key, source_type, cached_tag or "none") return True try: - file_count = _download_and_extract(source_url, source_path, local_cache, exclude) - except (urllib.error.URLError, OSError, tarfile.TarError) as exc: + if source_type == "zip": + strip = entry.get("strip_components", 0) + file_count = _download_and_extract_zip(source_url, local_cache, exclude, strip) + else: + source_path = entry["source_path"].format(version=version) + file_count = _download_and_extract(source_url, source_path, local_cache, exclude) + except (urllib.error.URLError, OSError, tarfile.TarError, zipfile.BadZipFile) as exc: log.warning("[%s] download failed: %s", key, exc) return False - # update version tracking - if remote_sha is None: - remote_sha = get_remote_sha(entry["source_url"], version) + if remote_tag is None: + if source_type == "zip": + remote_tag = _get_remote_etag(source_url) + else: + remote_tag = get_remote_sha(entry["source_url"], version) versions = _load_versions(versions_path) - versions[key] = {"sha": remote_sha or "", "version": version} + versions[key] = {"sha": remote_tag or "", "version": version} _save_versions(versions, versions_path) log.info("[%s] refreshed: %d files extracted to %s", key, file_count, local_cache) @@ -267,13 +352,19 @@ def refresh_all( force: bool = False, dry_run: bool = False, versions_path: str = VERSIONS_FILE, + platform: str | None = None, ) -> dict[str, bool]: """Refresh all entries in the registry. + If platform is set, only refresh entries whose for_platforms + includes that platform (or entries with no for_platforms restriction). Returns a dict mapping key -> whether it was refreshed. """ results = {} for key, entry in registry.items(): + allowed = entry.get("for_platforms") + if platform and allowed and platform not in allowed: + continue results[key] = refresh_entry( key, entry, force=force, dry_run=dry_run, versions_path=versions_path, ) @@ -285,6 +376,7 @@ def main() -> None: parser.add_argument("--key", help="Refresh only this entry") parser.add_argument("--force", action="store_true", help="Re-download even if up to date") parser.add_argument("--dry-run", action="store_true", help="Preview without downloading") + parser.add_argument("--platform", help="Only refresh entries for this platform") parser.add_argument("--registry", default=DEFAULT_REGISTRY, help="Path to _data_dirs.yml") args = parser.parse_args() @@ -301,7 +393,7 @@ def main() -> None: raise SystemExit(1) refresh_entry(args.key, registry[args.key], force=args.force, dry_run=args.dry_run) else: - refresh_all(registry, force=args.force, dry_run=args.dry_run) + refresh_all(registry, force=args.force, dry_run=args.dry_run, platform=args.platform) if __name__ == "__main__": diff --git a/scripts/scraper/libretro_scraper.py b/scripts/scraper/libretro_scraper.py index 628fd64f..89f2ed41 100644 --- a/scripts/scraper/libretro_scraper.py +++ b/scripts/scraper/libretro_scraper.py @@ -416,9 +416,20 @@ class Scraper(BaseScraper): "sony-psp": [ {"ref": "ppsspp-assets", "destination": "PPSSPP"}, ], + # single buildbot ZIP contains both Databases/ and Machines/ + # ref: libretro.c:1118-1119 — system_dir/Machines + system_dir/Databases "microsoft-msx": [ - {"ref": "bluemsx-databases", "destination": "Databases"}, - {"ref": "bluemsx-machines", "destination": "Machines"}, + {"ref": "bluemsx", "destination": ""}, + ], + # FreeIntv overlays — system/freeintv_overlays/.png + # ref: FreeIntv/src/libretro.c:273 — stbi_load from system dir + # ZIP contains FreeIntvTS_Overlays/ subfolder, cache preserves it + # pack destination maps cache root to system/freeintv_overlays + # so final path is system/freeintv_overlays/FreeIntvTS_Overlays/.png + # but core expects system/freeintv_overlays/.png + # fix: point destination into the subfolder + "mattel-intellivision": [ + {"ref": "freeintv-overlays", "destination": "freeintv_overlays"}, ], } for sys_id, data_dirs in SYSTEM_DATA_DIRS.items():