feat: core profiles, data_dirs buildbot, cross_ref fix

profiles: amiberry (new), amiarcadia, atari800, azahar, b2,
bk, blastem, bluemsx, freeintv updated with source refs,
upstream field, mode field, data_directories.

_data_dirs.yml: buildbot source for retroarch platforms,
strip_components for nested ZIPs, freeintv-overlays fixed.

cross_reference.py: data_directories-aware gap analysis,
suppresses false gaps when emulator+platform share refs.

refresh_data_dirs.py: ZIP strip_components support,
for_platforms filter, ETag freshness for buildbot.

scraper: bluemsx single ref, freeintv overlays injection.
generate_pack.py: warning on missing data directory cache.

database.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-03-18T16:46:00Z",
+  "generated_at": "2026-03-18T19:57:34Z",
   "total_files": 5593,
   "total_size": 4909044289,
   "files": {

emulators/amiarcadia.yml

@@ -35,4 +35,8 @@ notes: |
   No BIOS files required. The emulator has no external firmware dependencies.
   Game ROMs are identified by CRC32 for automatic machine configuration.
 
-bios_files: []
+files: []
+
+# closed-source core — cannot verify via source code scan.
+# firmware_count=0 in .info and docs confirm no BIOS needed.
+# signetics 2636/2637 systems have no separate system ROM.

emulators/amiberry.yml

@@ -0,0 +1,122 @@
+emulator: Amiberry
+type: standalone + libretro
+source: "https://github.com/BlitterStudio/amiberry"
+upstream: "https://github.com/tonioni/WinUAE"
+profiled_date: "2026-03-18"
+core_version: "v6.3.4"
+display_name: "Commodore - Amiga (Amiberry)"
+cores: [amiberry]
+systems:
+  - commodore-amiga
+  - commodore-cd32
+  - commodore-cdtv
+
+notes: |
+  Amiberry is an Amiga emulator based on WinUAE, available as both
+  standalone and libretro core. No built-in kickstart fallback — a real
+  Kickstart ROM is required.
+
+  Core option "amiberry_kickstart" selects the ROM file:
+  auto, kick.rom, kick13.rom, kick20.rom, kick31.rom, kick205.rom,
+  kick40068.A1200, kick40068.A4000, cd32.rom, cdtv.rom.
+  "auto" selects based on the model option.
+  ref: BlitterStudio/amiberry/libretro/libretro.cpp:558,621-634
+
+  Standalone mode ships with data/ directory (UI icons, virtual keyboards,
+  floppy sounds, AmigaTopaz.ttf font, gamecontrollerdb.txt). These are
+  compiled into the standalone binary but the libretro core may look for
+  them in the system directory.
+
+files:
+  # --- Required Kickstart ROMs (from .info) ---
+  # ref: BlitterStudio/amiberry/libretro/libretro.cpp:558
+  # ref: WinUAE rommgr.cpp for ROM identification
+
+  - name: kick34005.A500
+    system: commodore-amiga
+    required: true
+    size: 262144
+    crc32: c4f0f55f
+    note: "A500 Kickstart v1.3 rev 34.5"
+    source_ref: "libretro-super/dist/info/amiberry_libretro.info firmware0"
+
+  - name: kick37350.A600
+    system: commodore-amiga
+    required: true
+    size: 524288
+    crc32: 43b0df7b
+    note: "A600 Kickstart v2.05 rev 37.350"
+    source_ref: "amiberry_libretro.info firmware1"
+
+  - name: kick40068.A1200
+    system: commodore-amiga
+    required: true
+    size: 524288
+    crc32: 1483a091
+    note: "A1200 Kickstart v3.1 rev 40.68"
+    source_ref: "amiberry_libretro.info firmware2"
+
+  # --- Optional Kickstart ROMs ---
+
+  - name: kick33180.A500
+    system: commodore-amiga
+    required: false
+    size: 262144
+    crc32: a6ce1636
+    note: "A500 Kickstart v1.2 rev 33.180"
+    source_ref: "amiberry_libretro.info firmware3"
+
+  - name: kick40068.A4000
+    system: commodore-amiga
+    required: false
+    size: 524288
+    crc32: d6bae334
+    note: "A4000 Kickstart v3.1 rev 40.68"
+    source_ref: "amiberry_libretro.info firmware4"
+
+  - name: kick40060.CD32
+    system: commodore-cd32
+    required: false
+    size: 524288
+    crc32: 1e62d4a5
+    note: "CD32 Kickstart v3.1 rev 40.60"
+    source_ref: "amiberry_libretro.info firmware5"
+
+  - name: kick40060.CD32.ext
+    system: commodore-cd32
+    required: false
+    size: 524288
+    crc32: 87746be2
+    note: "CD32 Extended ROM rev 40.60"
+    source_ref: "amiberry_libretro.info firmware6"
+
+  # --- Alternative names accepted by core option ---
+  # these are common user-facing names, same ROMs as above
+
+  - name: kick13.rom
+    system: commodore-amiga
+    required: false
+    note: "alias for kick34005.A500 (v1.3)"
+    source_ref: "libretro.cpp:630"
+
+  - name: kick34005.CDTV
+    system: commodore-cdtv
+    required: false
+    note: "CDTV extended ROM"
+    source_ref: "libretro.cpp:558 cdtv.rom option"
+
+  # --- Standalone data files ---
+
+  - name: AmigaTopaz.ttf
+    system: commodore-amiga
+    required: false
+    mode: standalone
+    note: "Amiga Topaz font for UI rendering"
+    source_ref: "BlitterStudio/amiberry/data/AmigaTopaz.ttf"
+
+  - name: gamecontrollerdb.txt
+    system: commodore-amiga
+    required: false
+    mode: standalone
+    note: "SDL gamepad mapping database"
+    source_ref: "BlitterStudio/amiberry/controllers/gamecontrollerdb.txt"

emulators/atari800.yml

@@ -1,6 +1,7 @@
 emulator: Atari800
-type: libretro
+type: standalone + libretro
 source: "https://github.com/libretro/libretro-atari800"
+upstream: "https://github.com/atari800/atari800"
 profiled_date: "2026-03-18"
 core_version: "3.1.0"
 display_name: "Atari - 400/800/600XL/800XL/130XE/5200 (Atari800)"
@@ -80,8 +81,13 @@ files:
     required: false
     size: 10240
     md5: 4177f386a3bac989a981d3fe3388cb6c
+    crc32: 0e86d61d
     note: >
-      Atari 400/800 OS Rev B (NTSC). Preferred over Rev A for 400/800 mode.
+      Atari 400/800 OS Rev B (NTSC). CRC32 0x0e86d61d matches sysrom.c
+      SYSROM_B_NTSC. Preferred over Rev A for 400/800 mode.
+      Note: .info lists md5 a3e8d617c95d08031fe1b20d541434b2 which is a
+      different dump (crc32 3e28a1fe) — both in repo, the CRC32-matched
+      version is the one sysrom.c auto-detects.
     source_ref: "atari800/src/sysrom.c:89, libretro info firmware3"
 
   # -- Atari XL/XE OS --

emulators/azahar.yml

@@ -1,8 +1,28 @@
-emulator: "azahar"
+emulator: Azahar
 type: alias
-alias_of: "citra"
+alias_of: citra
+source: "https://github.com/azahar-emu/azahar"
+upstream: "https://github.com/azahar-emu/azahar"
 profiled_date: "2026-03-18"
 core_version: "Git"
 display_name: "Nintendo - 3DS (Azahar)"
-note: "This core uses the same BIOS/firmware as citra. See emulators/citra.yml for details."
+
+notes: |
+  Azahar is the successor to Citra (via Lime3DS). Same codebase, same
+  BIOS/firmware requirements. See emulators/citra.yml for the full file list.
+
+  Key difference: data directory changed from citra-emu/lime3ds-emu to
+  azahar-emu. Legacy dirs are still scanned as fallback.
+  ref: azahar/src/common/common_paths.h:41-47
+
+  Files needed (all optional, HLE fallback for most):
+    sysdata/keys.txt, sysdata/boot9.bin, sysdata/sector0x96.bin,
+    sysdata/shared_font.bin, sysdata/seeddb.bin, sysdata/otp.bin,
+    nand/private/movable.sed, nand/rw/sys/SecureInfo_A,
+    nand/rw/sys/LocalFriendCodeSeed_B
+
+  .info has firmware_count=0 but core uses RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY
+  (src/citra_libretro/environment.cpp:194) to locate system files.
+
 files: []
+# all files documented in emulators/citra.yml — same codebase

emulators/b2.yml

@@ -1,106 +1,45 @@
 emulator: b2
-type: standalone
+type: standalone + libretro
-source: "https://github.com/tom-seddon/b2"
+source: "https://github.com/zoltanvb/b2-libretro"
+upstream: "https://github.com/tom-seddon/b2"
 profiled_date: "2026-03-18"
 core_version: "v0.1"
 display_name: "Acorn - BBC Micro (b2)"
-cores: []
+cores: [b2]
 systems:
   - bbc-micro-b
   - bbc-micro-bplus
   - bbc-master-128
   - bbc-master-compact
 
-# b2 is a standalone BBC Micro emulator by Tom Seddon. It is NOT a libretro
-# core -- there is no libretro port. The emulator uses SDL2 and Dear ImGui.
-#
-# Emulated models (src/b2/BeebConfig.cpp:211-433):
-#   - BBC Model B (with various disc interfaces: Acorn 1770, Watford, Opus)
-#   - BBC Model B+ / B+128
-#   - BBC Master 128 (MOS 3.20 and 3.50)
-#   - BBC Master Turbo (MOS 3.20 and 3.50, with 65C102 parasite)
-#   - BBC Model B + 6502 second processor
-#   - BBC Master Compact (MOS 5.00, 5.10, I5.10C)
-#   - Olivetti PC 128 S (Master Compact variant)
-#
-# ROM handling:
-#   All ROMs ship in the repo under etc/roms/. The emulator loads them via
-#   GetAssetPath("roms", path) at runtime. No external BIOS files needed.
-#   ROM paths are defined in src/b2/roms.cpp as BeebROM structs with
-#   {filename, display_name, StandardROM_enum}.
-#
-# ROM inventory (etc/roms/):
-#
-#   BBC Model B:
-#     OS12.ROM          16384  md5:0a59a5ba15fe8557b5f7fee32bbd393a  OS 1.20
-#     BASIC2.ROM        16384  md5:2cc67be4624df4dc66617742571a8e3d  BASIC II
-#
-#   BBC Model B+:
-#     B+MOS.rom         16384  md5:47a1eff6c7fbb47be780c87dfd50ece1  B+ MOS
-#     BASIC2.ROM        16384  (shared with Model B)
-#
-#   Disc interfaces (sideways ROMs, slot 14):
-#     acorn/DFS-2.26.rom          16384  md5:f083f49d6fe66344c650d7e74249cb96  Acorn 1770 DFS
-#     watford/DDFS-1.53.rom       16384  md5:d2b71fa664f6cdf8c24b9b304777998c  Watford DDFS (DDB2)
-#     watford/DDFS-1.54T.rom      16384  md5:e73a0417278504a6dfac2ccd9a73e9a9  Watford DDFS (DDB3)
-#     opus/OPUS-DDOS-3.45.rom     16384  md5:d98316eaff56ceab4fe3bfe54d2f92b2  Opus DDOS
-#     opus/challenger-1.01.rom    16384  md5:67e086a11834f8cc69175b601c4e5a5f  Opus Challenger
-#
-#   BBC Master 128 MOS 3.20 (M128/3.20/):
-#     mos.rom           16384  md5:9087f772447038a00178488b3591347b  MOS 3.20 OS
-#     terminal.rom      16384  md5:7e053353b78b4631c274280931d96f02  Terminal (slot f)
-#     view.rom          16384  md5:178d09285727d596f8863f94a2fc8dad  View (slot e)
-#     adfs.rom          16384  md5:10f08b255d6a6e3e888631946b628f1a  ADFS (slot d)
-#     basic4.rom        16384  md5:63a6f43cbf491533cb03ceca2d2728d4  BASIC IV (slot c)
-#     edit.rom          16384  md5:cb3d762eaab6130d1360e1ac575589de  Edit (slot b)
-#     viewsht.rom       16384  md5:7d5a80a8ce929d0a0f6ca8626cb9f436  Viewsheet (slot a)
-#     dfs.rom           16384  md5:865a0f7a5d6f10f48798801bf514d761  DFS (slot 9)
-#
-#   BBC Master 128 MOS 3.50 (M128/3.50/):
-#     mos.rom           16384  md5:d7a046b1e6a9991748cee05332f1e258  MOS 3.50 OS
-#     terminal.rom      16384  md5:b1fa51fd49d31224ecb7e10e6908ec01  Terminal (slot f)
-#     view.rom          16384  md5:1566fab2f1827fb7ba61b283b828731d  View (slot e)
-#     adfs.rom          16384  md5:fb8ceb72c5fbd9dd80eedd10d076b1a2  ADFS (slot d)
-#     basic4.rom        16384  md5:e11eed95d1caba8aa9772e9001590585  BASIC IV (slot c)
-#     edit.rom          16384  md5:8c72c7f24567cb4a2fb15eb0b0aeb55b  Edit (slot b)
-#     viewsht.rom       16384  md5:1bc68b7c5eba6a2de2a43c9d1d9d983b  Viewsheet (slot a)
-#     dfs.rom           16384  md5:22c2b7f14f244da21d630a7828541c79  DFS (slot 9)
-#
-#   Parasite processors:
-#     MasterTurboParasite.rom  2048  md5:83d73e0e78693bb4b43e7cb18e58d556  65C102 TUBE 1.20
-#     TUBE110.rom              2048  md5:a77c7946128ca0b2c5e5a66b17c99dae  6502 TUBE 1.10
-#
-#   BBC Master Compact MOS 5.00 (MCompact/5.00/):
-#     mos.rom           16384  md5:5d1b635e39aa461f17db2344870dda36  MOS 5.00 OS
-#     utils.rom         16384  md5:4f9ded16f44278a4ca0ef4e66822e316  UTILS (slot f)
-#     basic4.rom        16384  md5:64b3cd52e36b5262ac6ca3a9823c6ee2  BASIC IV (slot e)
-#     adfs.rom          16384  md5:bfbd0a7dbbfb1f1f4ba5ac0343c1f8ce  ADFS (slot d)
-#
-#   BBC Master Compact MOS 5.10 (MCompact/5.10/):
-#     mos.rom           16384  md5:f65eb89c7044e328b79f2ef8e51b7fd8  MOS 5.10 OS
-#     utils.rom         16384  md5:48a197850113b00a1eb141fb6d3ea135  UTILS (slot f)
-#     basic4.rom        16384  md5:64b3cd52e36b5262ac6ca3a9823c6ee2  BASIC IV (slot e, same as 5.00)
-#     adfs.rom          16384  md5:c2afc1cabb135e6f2254bcf9300a9b83  ADFS (slot d)
-#
-#   Olivetti PC 128 S / MOS I5.10C (MCompact/I5.10C/):
-#     mos.rom           16384  md5:5e16a38bd7fd6f07f26b5a64826e4ec6  MOS I5.10C OS
-#     utils.rom         16384  md5:75364aa6fd5e134a40051d9ee772df37  UTILS (slot f)
-#     basic4.rom        16384  md5:717df34edae5b6bf5bbd8879c0d13022  BASIC IV (slot e)
-#     adfs.rom          16384  md5:10f23a239dde39b7fe09d72083709e93  ADFS (slot d)
-#
-# ROM type system (src/beeb/include/beeb/roms.inl):
-#   Sideways ROMs support banked mappers: 16KB (standard), CCIWORD (32KB),
-#   CCIBASE (64KB), CCISPELL (128KB), PALQST/PALWAP/PALTED, ABEP, ABE,
-#   Trilogy (64KB), MO2 (128KB). OS ROM types: 16KB, Compact (64KB),
-#   MegaROM (128KB), Multi-OS (512KB with 4 banks).
-#
-# Memory map (src/beeb/include/beeb/type.h):
-#   OS ROM mapped at $C000-$FFFF (MOS big pages). Sideways ROMs in 16 banks
-#   at $8000-$BFFF, selected via ROMSEL register at $FE30.
-
 notes: |
-  b2 is a standalone BBC Micro emulator (SDL2 + Dear ImGui). Not a libretro core.
+  b2 is a BBC Micro emulator by Tom Seddon. Libretro port by Zoltan Balogh.
-  All required ROMs are bundled in the repository under etc/roms/.
+  ref: zoltanvb/b2-libretro, tom-seddon/b2
-  No external BIOS files need to be provided by the user.
 
-bios_files: []
+  Standalone: loads ROMs from etc/roms/ via GetAssetPath().
+  ref: tom-seddon/b2/src/b2/roms.cpp:9-10
+
+  Libretro port (experimental): GetAssetPath stubbed in
+  src/libretro/adapters.cpp to return relative paths. LoadFile also
+  stubbed (returns false). ROM loading mechanism unclear — the 256 ROM
+  files are in the repo but the stub suggests they may not load at runtime.
+  is_experimental=true in .info.
+  ref: zoltanvb/b2-libretro/src/libretro/adapters.cpp
+
+  256 ROM files in etc/roms/ cover: BBC Model B (OS 1.20, BASIC II),
+  Model B+ (B+MOS), Master 128 (MOS 3.20, 3.50), Master Compact
+  (MOS 5.00, 5.10, I5.10C/Olivetti), parasite processors (65C102 TUBE,
+  6502 TUBE), disc interfaces (Acorn 1770, Watford, Opus).
+
+  All ROMs ship with both standalone and libretro repos.
+  No external BIOS files needed by the user for either mode.
+
+files: []
+# 256 ROMs bundled in repo etc/roms/ — not loaded from system dir
+# standalone loads via GetAssetPath("roms", path) -> etc/roms/
+# libretro port has LoadFile stubbed — ROM loading mechanism TBD
+# is_experimental=true, core may not fully function
+
+# closed-source analysis note: core binary on buildbot contains
+# "Retro ROM DIRECTORY" and "system_dir" strings but LoadFile stub
+# returns false. needs further investigation when port matures.

emulators/bk.yml

@@ -19,16 +19,16 @@ systems:
 #   Slow BK-0011M     (model 4) - same ROMs as model 3
 #   Terak 8510/a      (model 9)
 #
-# ROM loading (boot.c:83-116):
+# ROM loading (boot.c:84-117):
 #   BK-0010 models use load_rom() which maps ROM into emulated memory at
 #   specific addresses: monitor at 0100000 (8 KB), BASIC/FOCAL at 0120000
 #   (24448-24576 bytes), disk controller at 0160000 (4 KB).
 #   BK-0011M uses load_rom11() which loads into separate ROM buffer arrays.
 #
-# ROM file resolution (libretro.c:1051-1108):
+# ROM file resolution (libretro.c:1051-1094):
-#   Files are loaded from {system_dir}/bk/{filename} (libretro.c:724-726).
+#   Files are loaded from {system_dir}/bk/{filename} (libretro.c:720-728).
 #   If not found with original casing, the loader retries with lowercase
-#   filename (libretro.c:1074-1079).
+#   filename (libretro.c:1071-1081).
 #
 # All ROM files are required for their respective model. The core calls
 # environ_cb(RETRO_ENVIRONMENT_SHUTDOWN) if any ROM file is missing.

emulators/blastem.yml

@@ -1,6 +1,7 @@
 emulator: BlastEm
-type: libretro
+type: standalone + libretro
 source: "https://github.com/libretro/blastem"
+upstream: "https://www.retrodev.com/blastem/"
 profiled_date: "2026-03-18"
 core_version: "v0.6.3-pre"
 display_name: "Sega - Mega Drive - Genesis (BlastEm)"
@@ -28,7 +29,28 @@ notes: |
   Sega CD / Mega CD is not supported (only a TODO comment in system.c:16).
   No BIOS files are required or loaded by the libretro core.
 
-files: []
+files:
+  # rom.db is embedded in the libretro binary (libblastem.c:532-540,
+  # extern const char rom_db_data[]). The external file is an optional
+  # override — .info lists it as firmware0_opt=true.
+  # .info sha1 E5414FB1C4CC7D7F5101C07E4547316779BA3D97 does not match
+  # the version in our repo (sha1 02c287d1...) — rom.db changes between
+  # BlastEm releases.
+  - name: rom.db
+    system: sega-megadrive
+    required: false
+    note: "ROM feature database (game-specific mapper/save config). Embedded in core, external is optional override."
+    source_ref: "libblastem.c:532-540 read_bundled_file()"
+
+  # TMSS ROM — only used in standalone mode with tmss-enabled model
+  # libretro build returns NULL for tmss.md (libblastem.c:541)
+  - name: tmss.md
+    system: sega-megadrive
+    required: false
+    mode: standalone
+    size: 2048
+    note: "TMSS startup ROM. Standalone only (models md1va6, md2va1+). Libretro core uses md1va3 (tmss off), no way to change."
+    source_ref: "genesis.c:1910-1923"
 
 analysis:
   tmss:

emulators/bluemsx.yml

@@ -1,6 +1,7 @@
 emulator: blueMSX
-type: libretro
+type: standalone + libretro
 source: "https://github.com/libretro/blueMSX-libretro"
+upstream: "http://bluemsx.msxblue.com/"
 profiled_date: "2026-03-18"
 core_version: "SVN"
 display_name: "MSX/SVI/ColecoVision/SG-1000 (blueMSX)"
@@ -52,6 +53,13 @@ notes: |
   C-BIOS machines work for cartridge-based games without any copyrighted BIOS.
   For disk/tape support, real BIOS ROMs are required.
 
+data_directories:
+  # 302 files: 296 in Machines/ (200 machine configs + ROMs) + 6 in Databases/
+  # buildbot source: https://buildbot.libretro.com/assets/system/blueMSX.zip
+  # ref: libretro.c:1118-1119 — system_dir/Machines/ + system_dir/Databases/
+  - ref: bluemsx
+    source_ref: "libretro.c:1118-1119"
+
 files:
   # ============================================================
   # Shared ROMs (Machines/Shared Roms/)

emulators/freeintv_ts_overlay.yml

@@ -26,7 +26,10 @@ notes: |
 
   ROM-specific overlays are optional 370x600 PNG files placed by the user in
   system/freeintv_overlays/<rom_name>.png. These are cosmetic per-game keypad
-  images and are not BIOS files.
+  images loaded via stbi_load() (src/libretro.c:273). Core falls back to
+  embedded default_keypad_image.h if no per-game overlay found.
+  90 overlays available in Assets/Overlays.zip in the core repo.
+  Not on RetroArch buildbot — sourced from core repo via _data_dirs.yml.
 
   BIOS loading is unchanged from the base FreeIntv core. retro_load_game()
   (src/libretro.c:1169-1174) loads exec.bin and grom.bin from the system
@@ -34,6 +37,10 @@ notes: |
 
 bios_identical_to: freeintv
 
+data_directories:
+  - ref: freeintv-overlays
+    destination: freeintv_overlays
+
 files:
   - name: "exec.bin"
     system: intellivision

platforms/_data_dirs.yml

@@ -1,38 +1,166 @@
 # Data directory sources for libretro cores.
 # Platforms reference entries by key via data_directories: [{ref: key, destination: path}].
 # Pack generator auto-refreshes from source. Use --offline to skip.
+#
+# Source rules:
+#   - RetroArch/Lakka/RetroPie: use buildbot.libretro.com (official RetroArch channel)
+#     ref: RetroArch/config.def.h DEFAULT_BUILDBOT_ASSETS_SERVER_URL
+#   - Other platforms (Batocera, Recalbox, RetroBat): use their own upstream sources
+#   - for_platforms: restricts which platforms use this entry
+#     if absent, entry is available to all platforms
+#
 # Each entry cites the original emulator source code that requires the directory.
 
 data_directories:
+
+  # =========================================================================
+  # RetroArch buildbot sources (retroarch, lakka, retropie only)
+  # https://buildbot.libretro.com/assets/system/
+  # =========================================================================
+
   # ref: DolphinLibretro/Boot.cpp:72-73 — system/dolphin-emu/Sys/
   dolphin-sys:
-    source_url: "https://github.com/libretro/dolphin/archive/{version}.tar.gz"
+    source_url: "https://buildbot.libretro.com/assets/system/Dolphin.zip"
-    source_path: "dolphin-{version}/Data/Sys"
+    source_type: zip
-    version: master
+    for_platforms: [retroarch, lakka, retropie]
     local_cache: data/dolphin-sys
     exclude: [Themes]
     description: "Dolphin system data (GameSettings, DSP, fonts, shaders)"
 
   # ref: ppsspp/ext/native/ui/ui_screen.cpp — system/PPSSPP/
   ppsspp-assets:
-    source_url: "https://github.com/hrydgard/ppsspp/archive/{version}.tar.gz"
+    source_url: "https://buildbot.libretro.com/assets/system/PPSSPP.zip"
-    source_path: "ppsspp-{version}/assets"
+    source_type: zip
-    version: master
+    for_platforms: [retroarch, lakka, retropie]
     local_cache: data/ppsspp-assets
     description: "PPSSPP fonts, backgrounds, shaders, lang files"
 
-  # ref: bluemsx-libretro/system/ — system/Databases/
+  # ref: bluemsx-libretro/system/ — system/Databases/ + system/Machines/
-  bluemsx-databases:
+  bluemsx:
-    source_url: "https://github.com/libretro/blueMSX-libretro/archive/{version}.tar.gz"
+    source_url: "https://buildbot.libretro.com/assets/system/blueMSX.zip"
-    source_path: "blueMSX-libretro-{version}/system/Databases"
+    source_type: zip
-    version: master
+    for_platforms: [retroarch, lakka, retropie]
-    local_cache: data/bluemsx-databases
+    local_cache: data/bluemsx
-    description: "blueMSX machine database"
+    description: "blueMSX Databases + Machines configs"
 
-  # ref: bluemsx-libretro/system/ — system/Machines/
+  # ref: pcsx2/libretro/main.cpp — system/pcsx2/
-  bluemsx-machines:
+  lrps2:
-    source_url: "https://github.com/libretro/blueMSX-libretro/archive/{version}.tar.gz"
+    source_url: "https://buildbot.libretro.com/assets/system/LRPS2.zip"
-    source_path: "blueMSX-libretro-{version}/system/Machines"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/lrps2
+    description: "LRPS2 GameIndex.yaml + resources"
+
+  # ref: scummvm/backends/platform/libretro — system/scummvm/
+  scummvm:
+    source_url: "https://buildbot.libretro.com/assets/system/ScummVM.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/scummvm
+    description: "ScummVM extra data files (themes, translations, shaders)"
+
+  # ref: nxengine-libretro — system/
+  nxengine:
+    source_url: "https://buildbot.libretro.com/assets/system/NXEngine%20%28Cave%20Story%29.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/nxengine
+    description: "NXEngine Cave Story game data"
+
+  # ref: ecwolf/src/wl_main.cpp — system/ecwolf.pk3
+  ecwolf:
+    source_url: "https://buildbot.libretro.com/assets/system/ECWolf.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/ecwolf
+    description: "ECWolf game engine data"
+
+  # ref: prboom/src/d_main.c — system/prboom.wad
+  prboom:
+    source_url: "https://buildbot.libretro.com/assets/system/PrBoom.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/prboom
+    description: "PrBoom Doom engine data"
+
+  # ref: xrick/src/data.c — system/xrick/data.zip
+  xrick:
+    source_url: "https://buildbot.libretro.com/assets/system/XRick%20%28Rick%20Dangerous%29.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/xrick
+    description: "XRick Rick Dangerous game data"
+
+  # ref: dinothawr/libretro.cpp — system/dinothawr/
+  dinothawr:
+    source_url: "https://buildbot.libretro.com/assets/system/Dinothawr.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/dinothawr
+    description: "Dinothawr game data"
+
+  # ref: qemu — system firmware
+  qemu:
+    source_url: "https://buildbot.libretro.com/assets/system/QEMU.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/qemu
+    description: "QEMU firmware (SeaBIOS, VGA BIOS, OpenSBI, etc.)"
+
+  # ref: cannonball/src/main.cpp — system/
+  cannonball:
+    source_url: "https://buildbot.libretro.com/assets/system/Cannonball%20%28ROMs%20Required%29.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/cannonball
+    description: "Cannonball OutRun engine config"
+
+  # ref: fbneo hiscore.dat — system/fbneo/
+  fbneo-hiscore:
+    source_url: "https://buildbot.libretro.com/assets/system/FinalBurn%20Neo%20%28hiscore%29.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/fbneo-hiscore
+    description: "FinalBurn Neo hiscore database"
+
+  # ref: mame2003/src — system/mame2003/
+  mame2003:
+    source_url: "https://buildbot.libretro.com/assets/system/MAME%202003.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/mame2003
+    description: "MAME 2003 samples + cheat"
+
+  # ref: mame2003-plus — system/mame2003-plus/
+  mame2003-plus:
+    source_url: "https://buildbot.libretro.com/assets/system/MAME%202003-Plus.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/mame2003-plus
+    description: "MAME 2003-Plus samples + cheat + history"
+
+  # =========================================================================
+  # Upstream repo sources (not on buildbot, available to all platforms)
+  # =========================================================================
+
+  # ref: FreeIntv/src/libretro.c:273 — system/freeintv_overlays/<rom>.png
+  # 90 per-game controller overlay PNGs, loaded via stbi_load()
+  # optional: core falls back to embedded default keypad image
+  # freeintv overlays: Assets/Overlays.zip in repo (not a directory)
+  # Already archived in bios/Mattel/Intellivision/freeintv_overlays/
+  # No buildbot source — sourced manually from core repo
+  freeintv-overlays:
+    source_url: "https://github.com/libretro/FreeIntv/raw/{version}/Assets/Overlays.zip"
+    source_type: zip
     version: master
-    local_cache: data/bluemsx-machines
+    local_cache: data/freeintv-overlays
-    description: "blueMSX machine ROM configs"
+    strip_components: 1
+    description: "FreeIntv per-game controller overlay PNGs"
+
+  # ref: dirksimple — system/
+  dirksimple:
+    source_url: "https://buildbot.libretro.com/assets/system/DirkSimple.zip"
+    source_type: zip
+    for_platforms: [retroarch, lakka, retropie]
+    local_cache: data/dirksimple
+    description: "DirkSimple Dragon's Lair engine data"

platforms/retroarch.yml

@@ -814,6 +814,9 @@ systems:
       md5: 0cd5946c6473e42e8e4c2137785e427f
       crc32: 683a4158
       size: 2048
+    data_directories:
+    - ref: freeintv-overlays
+      destination: freeintv_overlays
   microsoft-msx:
     files:
     - name: CARTS.SHA
@@ -926,10 +929,8 @@ systems:
       Labs
     docs: https://docs.libretro.com/library/bluemsx/
     data_directories:
-    - ref: bluemsx-databases
+    - ref: bluemsx
-      destination: Databases
+      destination: ''
-    - ref: bluemsx-machines
-      destination: Machines
   nec-pc-engine:
     files:
     - name: gecard.pce

scripts/cross_reference.py

@@ -46,9 +46,10 @@ def load_emulator_profiles(emulators_dir: str) -> dict[str, dict]:
     return profiles
 
 
-def load_platform_files(platforms_dir: str) -> dict[str, set[str]]:
+def load_platform_files(platforms_dir: str) -> tuple[dict[str, set[str]], dict[str, set[str]]]:
-    """Load all platform configs and collect declared filenames per system."""
+    """Load all platform configs and collect declared filenames + data_directories per system."""
     declared = {}
+    platform_data_dirs = {}
     for f in sorted(Path(platforms_dir).glob("*.yml")):
         if f.name.startswith("_"):
             continue
@@ -58,7 +59,11 @@ def load_platform_files(platforms_dir: str) -> dict[str, set[str]]:
                 name = fe.get("name", "")
                 if name:
                     declared.setdefault(sys_id, set()).add(name)
-    return declared
+            for dd in system.get("data_directories", []):
+                ref = dd.get("ref", "")
+                if ref:
+                    platform_data_dirs.setdefault(sys_id, set()).add(ref)
+    return declared, platform_data_dirs
 
 
 def _find_in_repo(fname: str, by_name: dict[str, list], by_name_lower: dict[str, str]) -> bool:
@@ -81,12 +86,15 @@ def cross_reference(
     profiles: dict[str, dict],
     declared: dict[str, set[str]],
     db: dict,
+    platform_data_dirs: dict[str, set[str]] | None = None,
 ) -> dict:
     """Compare emulator profiles against platform declarations.
 
     Returns a report with gaps (files emulators need but platforms don't list)
-    and coverage stats.
+    and coverage stats. Files covered by matching data_directories between
+    emulator profile and platform config are not reported as gaps.
     """
+    platform_data_dirs = platform_data_dirs or {}
     by_name = db.get("indexes", {}).get("by_name", {})
     by_name_lower = {k.lower(): k for k in by_name}
     report = {}
@@ -99,6 +107,15 @@ def cross_reference(
         for sys_id in systems:
             platform_names.update(declared.get(sys_id, set()))
 
+        # data_directories: check if the emulator's data_dir refs are provided
+        # by ANY platform for ANY system (not limited to matching system IDs,
+        # since emulator profiles and platforms use different ID conventions)
+        all_plat_dd_refs = set()
+        for dd_set in platform_data_dirs.values():
+            all_plat_dd_refs.update(dd_set)
+        emu_dd_refs = {dd.get("ref", "") for dd in profile.get("data_directories", [])}
+        covered_dd = emu_dd_refs & all_plat_dd_refs
+
         gaps = []
         covered = []
         for f in emu_files:
@@ -114,6 +131,9 @@ def cross_reference(
                 continue
 
             in_platform = fname in platform_names
+            # files covered by shared data_directories are effectively in the platform pack
+            if not in_platform and covered_dd:
+                in_platform = True
             in_repo = _find_in_repo(fname, by_name, by_name_lower)
 
             entry = {
@@ -200,9 +220,9 @@ def main():
         print("No emulator profiles found.", file=sys.stderr)
         return
 
-    declared = load_platform_files(args.platforms_dir)
+    declared, plat_data_dirs = load_platform_files(args.platforms_dir)
     db = load_database(args.db)
-    report = cross_reference(profiles, declared, db)
+    report = cross_reference(profiles, declared, db, plat_data_dirs)
 
     if args.json:
         print(json.dumps(report, indent=2))

scripts/generate_pack.py

@@ -388,8 +388,12 @@ def generate_pack(
                 if not ref_key or not data_registry or ref_key not in data_registry:
                     continue
                 entry = data_registry[ref_key]
+                allowed = entry.get("for_platforms")
+                if allowed and platform_name not in allowed:
+                    continue
                 local_path = entry.get("local_cache", "")
                 if not local_path or not os.path.isdir(local_path):
+                    print(f"  WARNING: data directory '{ref_key}' not cached at {local_path} — run refresh_data_dirs.py")
                     continue
                 dd_dest = dd.get("destination", "")
                 dd_prefix = f"{base_dest}/{dd_dest}" if base_dest else dd_dest

scripts/refresh_data_dirs.py

@@ -21,6 +21,7 @@ import tarfile
 import tempfile
 import urllib.error
 import urllib.request
+import zipfile
 from pathlib import Path
 
 try:
@@ -203,6 +204,81 @@ def _download_and_extract(
     return file_count
 
 
+def _download_and_extract_zip(
+    source_url: str,
+    local_cache: str,
+    exclude: list[str] | None = None,
+    strip_components: int = 0,
+) -> int:
+    """Download ZIP, extract to local_cache. Returns file count.
+
+    strip_components removes N leading path components from each entry
+    (like tar --strip-components). Useful when a ZIP has a single root
+    directory that should be flattened.
+    """
+    exclude = exclude or []
+    cache_dir = Path(local_cache)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        zip_path = Path(tmpdir) / "archive.zip"
+        log.info("downloading %s", source_url)
+
+        req = urllib.request.Request(source_url, headers={"User-Agent": USER_AGENT})
+        with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp:
+            with open(zip_path, "wb") as f:
+                while True:
+                    chunk = resp.read(65536)
+                    if not chunk:
+                        break
+                    f.write(chunk)
+
+        extract_dir = Path(tmpdir) / "extract"
+        extract_dir.mkdir()
+        file_count = 0
+
+        with zipfile.ZipFile(zip_path) as zf:
+            for info in zf.infolist():
+                if info.is_dir():
+                    continue
+                name = info.filename
+                if ".." in name or name.startswith("/"):
+                    continue
+                # strip leading path components
+                parts = name.split("/")
+                if strip_components > 0:
+                    if len(parts) <= strip_components:
+                        continue
+                    parts = parts[strip_components:]
+                    name = "/".join(parts)
+                # skip excludes (check against stripped path)
+                top = parts[0] if parts else ""
+                if top in exclude:
+                    continue
+                dest = extract_dir / name
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                with zf.open(info) as src, open(dest, "wb") as dst:
+                    shutil.copyfileobj(src, dst)
+                file_count += 1
+
+        if cache_dir.exists():
+            shutil.rmtree(cache_dir)
+        cache_dir.parent.mkdir(parents=True, exist_ok=True)
+        shutil.move(str(extract_dir), str(cache_dir))
+
+    return file_count
+
+
+def _get_remote_etag(source_url: str) -> str | None:
+    """HEAD request to get ETag or Last-Modified for freshness check."""
+    try:
+        req = urllib.request.Request(source_url, method="HEAD",
+                                     headers={"User-Agent": USER_AGENT})
+        with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
+            return resp.headers.get("ETag") or resp.headers.get("Last-Modified") or ""
+    except (urllib.error.URLError, OSError):
+        return None
+
+
 def refresh_entry(
     key: str,
     entry: dict,
@@ -215,46 +291,55 @@ def refresh_entry(
 
     Returns True if the entry was refreshed (or would be in dry-run mode).
     """
+    source_type = entry.get("source_type", "tarball")
     version = entry.get("version", "master")
     source_url = entry["source_url"].format(version=version)
-    source_path = entry["source_path"].format(version=version)
     local_cache = entry["local_cache"]
     exclude = entry.get("exclude", [])
 
     versions = _load_versions(versions_path)
     cached = versions.get(key, {})
-    cached_sha = cached.get("sha")
+    cached_tag = cached.get("sha") or cached.get("etag")
 
     needs_refresh = force or not Path(local_cache).exists()
 
+    remote_tag: str | None = None
     if not needs_refresh:
-        remote_sha = get_remote_sha(entry["source_url"], version)
+        if source_type == "zip":
-        if remote_sha is None:
+            remote_tag = _get_remote_etag(source_url)
+        else:
+            remote_tag = get_remote_sha(entry["source_url"], version)
+        if remote_tag is None:
             log.warning("[%s] could not check remote, skipping", key)
             return False
-        needs_refresh = remote_sha != cached_sha
+        needs_refresh = remote_tag != cached_tag
-    else:
-        remote_sha = get_remote_sha(entry["source_url"], version) if not force else None
 
     if not needs_refresh:
-        log.info("[%s] up to date (sha: %s)", key, cached_sha[:12] if cached_sha else "?")
+        log.info("[%s] up to date (tag: %s)", key, (cached_tag or "?")[:12])
         return False
 
     if dry_run:
-        log.info("[%s] would refresh (version: %s, cached sha: %s)", key, version, cached_sha or "none")
+        log.info("[%s] would refresh (type: %s, cached: %s)", key, source_type, cached_tag or "none")
         return True
 
     try:
-        file_count = _download_and_extract(source_url, source_path, local_cache, exclude)
+        if source_type == "zip":
-    except (urllib.error.URLError, OSError, tarfile.TarError) as exc:
+            strip = entry.get("strip_components", 0)
+            file_count = _download_and_extract_zip(source_url, local_cache, exclude, strip)
+        else:
+            source_path = entry["source_path"].format(version=version)
+            file_count = _download_and_extract(source_url, source_path, local_cache, exclude)
+    except (urllib.error.URLError, OSError, tarfile.TarError, zipfile.BadZipFile) as exc:
         log.warning("[%s] download failed: %s", key, exc)
         return False
 
-    # update version tracking
+    if remote_tag is None:
-    if remote_sha is None:
+        if source_type == "zip":
-        remote_sha = get_remote_sha(entry["source_url"], version)
+            remote_tag = _get_remote_etag(source_url)
+        else:
+            remote_tag = get_remote_sha(entry["source_url"], version)
     versions = _load_versions(versions_path)
-    versions[key] = {"sha": remote_sha or "", "version": version}
+    versions[key] = {"sha": remote_tag or "", "version": version}
     _save_versions(versions, versions_path)
 
     log.info("[%s] refreshed: %d files extracted to %s", key, file_count, local_cache)
@@ -267,13 +352,19 @@ def refresh_all(
     force: bool = False,
     dry_run: bool = False,
     versions_path: str = VERSIONS_FILE,
+    platform: str | None = None,
 ) -> dict[str, bool]:
     """Refresh all entries in the registry.
 
+    If platform is set, only refresh entries whose for_platforms
+    includes that platform (or entries with no for_platforms restriction).
     Returns a dict mapping key -> whether it was refreshed.
     """
     results = {}
     for key, entry in registry.items():
+        allowed = entry.get("for_platforms")
+        if platform and allowed and platform not in allowed:
+            continue
         results[key] = refresh_entry(
             key, entry, force=force, dry_run=dry_run, versions_path=versions_path,
         )
@@ -285,6 +376,7 @@ def main() -> None:
     parser.add_argument("--key", help="Refresh only this entry")
     parser.add_argument("--force", action="store_true", help="Re-download even if up to date")
     parser.add_argument("--dry-run", action="store_true", help="Preview without downloading")
+    parser.add_argument("--platform", help="Only refresh entries for this platform")
     parser.add_argument("--registry", default=DEFAULT_REGISTRY, help="Path to _data_dirs.yml")
     args = parser.parse_args()
 
@@ -301,7 +393,7 @@ def main() -> None:
             raise SystemExit(1)
         refresh_entry(args.key, registry[args.key], force=args.force, dry_run=args.dry_run)
     else:
-        refresh_all(registry, force=args.force, dry_run=args.dry_run)
+        refresh_all(registry, force=args.force, dry_run=args.dry_run, platform=args.platform)
 
 
 if __name__ == "__main__":

scripts/scraper/libretro_scraper.py

@@ -416,9 +416,20 @@ class Scraper(BaseScraper):
             "sony-psp": [
                 {"ref": "ppsspp-assets", "destination": "PPSSPP"},
             ],
+            # single buildbot ZIP contains both Databases/ and Machines/
+            # ref: libretro.c:1118-1119 — system_dir/Machines + system_dir/Databases
             "microsoft-msx": [
-                {"ref": "bluemsx-databases", "destination": "Databases"},
+                {"ref": "bluemsx", "destination": ""},
-                {"ref": "bluemsx-machines", "destination": "Machines"},
+            ],
+            # FreeIntv overlays — system/freeintv_overlays/<rom>.png
+            # ref: FreeIntv/src/libretro.c:273 — stbi_load from system dir
+            # ZIP contains FreeIntvTS_Overlays/ subfolder, cache preserves it
+            # pack destination maps cache root to system/freeintv_overlays
+            # so final path is system/freeintv_overlays/FreeIntvTS_Overlays/<rom>.png
+            # but core expects system/freeintv_overlays/<rom>.png
+            # fix: point destination into the subfolder
+            "mattel-intellivision": [
+                {"ref": "freeintv-overlays", "destination": "freeintv_overlays"},
             ],
         }
         for sys_id, data_dirs in SYSTEM_DATA_DIRS.items():