feat: platform-core registry for exact pack generation

resolve_platform_cores() links platforms to their cores via
three strategies: all_libretro, explicit list, system ID
fallback. Pack generation always includes core requirements
beyond platform baseline. Case-insensitive dedup prevents
conflicts on Windows/macOS. Data dir strip_components fixes
doubled paths for Dolphin and PPSSPP caches.
This commit is contained in:
Abdessamad Derraz
2026-03-19 16:10:43 +01:00
parent 257ec1a527
commit 6a21a99c22
15 changed files with 758 additions and 57 deletions

View File

@@ -1,5 +1,5 @@
{
"generated_at": "2026-03-19T13:30:04Z",
"generated_at": "2026-03-19T15:04:46Z",
"total_files": 5593,
"total_size": 4909044289,
"files": {

View File

@@ -24,6 +24,7 @@ data_directories:
source_type: zip
for_platforms: [retroarch, lakka, retropie]
local_cache: data/dolphin-sys
strip_components: 2
exclude: [Themes]
description: "Dolphin system data (GameSettings, DSP, fonts, shaders)"
@@ -33,6 +34,7 @@ data_directories:
source_type: zip
for_platforms: [retroarch, lakka, retropie]
local_cache: data/ppsspp-assets
strip_components: 1
description: "PPSSPP fonts, backgrounds, shaders, lang files"
# ref: bluemsx-libretro/system/ — system/Databases/ + system/Machines/

View File

@@ -16,7 +16,7 @@ platforms:
source_format: clrmamepro_dat
hash_type: sha1
schedule: weekly
emulators: [pcsx_rearmed, beetle_psx, genesis_plus_gx, flycast, melonds, mgba, snes9x, mupen64plus, beetle_saturn, dolphin]
cores: all_libretro
batocera:
config: batocera.yml
@@ -27,7 +27,7 @@ platforms:
source_format: python_dict
hash_type: md5
schedule: weekly
emulators: [flycast, dolphin, pcsx2, duckstation, rpcs3, ppsspp, beetle_psx, beetle_saturn, genesis_plus_gx, picodrive, fbneo, puae, hatari, fuse, opera, bluemsx, fmsx, np2kai, quasi88]
cores: [81, a5200, abuse, arduous, atari800, azahar, bennugd, bk, bluemsx, bsnes, bstone, cannonball, cap32, catacombgl, cdogs, cemu, cgenius, citron, clk, corsixth, demul, devilutionx, dhewm3, dice, dolphin, dosbox_pure, dxx-rebirth, easyrpg, ecwolf, eduke32, eka2l1, emuscv, etlegacy, fake08, fallout1-ce, fallout2-ce, fbneo, fceumm, flatpak, flycast, freechaf, freeintv, fury, fuse, gambatte, gearsystem, genesisplusgx, glide64mk2, gong, gsplus, gw, gzdoom, hatari, hcl, hurrican, hypseus-singe, ikemen, ioquake3, iortcw, jazz2-native, lindbergh-loader, lowresnx, lutro, mame, mame078plus, mednafen_lynx, mednafen_ngp, mednafen_supergrafx, mednafen_wswan, melonds, mgba, minivmac, model2emu, moonlight, mrboom, neocd, np2kai, nxengine, o2em, odcommander, openbor6412, openjazz, openjk, openjkdf2, openmohaa, opera, pce_fast, pcfx, pcsx2, pcsx_rearmed, pd777, picodrive, play, pokemini, potator, ppsspp, prboom, prosystem, puae, px68k, pygame, pyxel, quasi88, raze, reminiscence, rpcs3, ruffle, samcoupe, sameduck, scummvm, sdlpop, sh, shadps4, snes9x, solarus, sonic2013, sonic3-air, sonic-mania, steam, stella, superbroswar, supermodel, taradino, tgbdual, theforceengine, theodore, thextech, tic80, tr1x, tr2x, tsugaru, tyrian, tyrquake, uqm, uzem, vb, vecx, vice_x64, vircon32, virtualjaguar, vita3k, vox_official, vpinball, wasm4, wine-tkg, x1, x128, x16emu, xash3d_fwgs, xemu, xenia-canary, xpet, xplus4, xrick, xvic, yabasanshiro, yquake2, zc210]
recalbox:
config: recalbox.yml
@@ -38,7 +38,7 @@ platforms:
source_format: xml
hash_type: md5
schedule: monthly
emulators: [flycast, dolphin, pcsx2, beetle_psx, beetle_saturn, genesis_plus_gx, picodrive, fbneo, puae, hatari, opera, bluemsx, fmsx]
cores: ["2048", 81, a5200, advancemame, amiberry, applewin, arduous, atari800, b2, beebem, bk, bluemsx, boom3, bsnes, bsneshd, cannonball, cap32, cdi2015, corsixth, craft, crocods, daphne, desmume, dice, dinothawr, dirksimple, dolphin, dolphin-gui, dosbox, dosbox_pure, duckstation, easyrpg, ecwolf, emuscv, fake08, fba2x, fbneo, fceumm, flycast, flycast-next, fmsx, freechaf, freeintv, frotz, fuse, gambatte, gearcoleco, geargrafx, gearsystem, genesisplusgx, genesisplusgx_ex, genesisplusgxwide, geolith, glide64mk2, gliden64, gliden64_20, gong, gpsp, gsplus, gw, handy, hatari, hatarib, holani, imageviewer, julius, kronos, lowresnx, lutro, mame0258, mame0278, mame2000, mame2003, mame2003_plus, mame2010, mame2015, mame2016, mednafen_lynx, mednafen_ngp, mednafen_pce_fast, mednafen_pcfx, mednafen_psx, mednafen_psx_hw, mednafen_saturn, mednafen_supafaust, mednafen_supergrafx, mednafen_vb, mednafen_wswan, melonds, mesen, mesen_s, meteor, mgba, minivmac, mojozork, moonlight, mrboom, mu, mupen64plus, mupen64plus_next, n64_gles2, neocd, nestopia, np2kai, nxengine, o2em, openbor, openlara, opera, oricutron, parallel_n64, pcsx2, pcsx_rearmed, pico8, picodrive, pisnes, pokemini, potator, ppsspp, prboom, prosystem, ps2, puae, px68k, quasi88, quicknes, race, rb5000, reicast, reminiscence, retro8, retrodream, rice, rice_gles2, sameboy, same_cdi, sameduck, scummvm, sdlpop, simcoupe, snes9x, snes9x2002, snes9x2005, snes9x2010, solarus, stella, stella2014, stonesoup, supermodel, swanstation, tamalibretro, tgbdual, theodore, thepowdertoy, ti99sim, tic80, tyrquake, uae4all, uae4arm, uzem, vecx, vice_x128, vice_x64, vice_x64sc, vice_xcbm2, vice_xcbm5x0, vice_xpet, vice_xplus4, vice_xscpu64, vice_xvic, virtualjaguar, vitaquake2, vitaquake3, vitavoyager, vpinball, vvvvvv, wasm4, x1, x128, x64, x64sx, xcbm2, xcbm5x0, xemu, xpet, xplus4, xrick, xroar, xscpu64, xvic, yabasanshiro, yabause]
retrobat:
config: retrobat.yml
@@ -49,7 +49,7 @@ platforms:
source_format: json
hash_type: md5
schedule: weekly
emulators: [duckstation, pcsx2, dolphin, rpcs3, ppsspp, cemu, xemu, flycast, beetle_psx, beetle_saturn, genesis_plus_gx, puae, opera]
cores: [81, a5200, abuse, arduous, atari800, azahar, bennugd, bk, bluemsx, bsnes, bstone, cannonball, cap32, catacombgl, cdogs, cemu, cgenius, citron, clk, corsixth, demul, devilutionx, dhewm3, dice, dolphin, dosbox_pure, dxx-rebirth, easyrpg, ecwolf, eduke32, eka2l1, emuscv, etlegacy, fake08, fallout1-ce, fallout2-ce, fbneo, fceumm, flatpak, flycast, freechaf, freeintv, fury, fuse, gambatte, gearsystem, genesisplusgx, glide64mk2, gong, gsplus, gw, gzdoom, hatari, hcl, hurrican, hypseus-singe, ikemen, ioquake3, iortcw, jazz2-native, lindbergh-loader, lowresnx, lutro, mame, mame078plus, mednafen_lynx, mednafen_ngp, mednafen_supergrafx, mednafen_wswan, melonds, mgba, minivmac, model2emu, moonlight, mrboom, neocd, np2kai, nxengine, o2em, odcommander, openbor6412, openjazz, openjk, openjkdf2, openmohaa, opera, pce_fast, pcfx, pcsx2, pcsx_rearmed, pd777, picodrive, play, pokemini, potator, ppsspp, prboom, prosystem, puae, px68k, pygame, pyxel, quasi88, raze, reminiscence, rpcs3, ruffle, samcoupe, sameduck, scummvm, sdlpop, sh, shadps4, snes9x, solarus, sonic2013, sonic3-air, sonic-mania, steam, stella, superbroswar, supermodel, taradino, tgbdual, theforceengine, theodore, thextech, tic80, tr1x, tr2x, tsugaru, tyrian, tyrquake, uqm, uzem, vb, vecx, vice_x64, vircon32, virtualjaguar, vita3k, vox_official, vpinball, wasm4, wine-tkg, x1, x128, x16emu, xash3d_fwgs, xemu, xenia-canary, xpet, xplus4, xrick, xvic, yabasanshiro, yquake2, zc210]
emudeck:
config: emudeck.yml
@@ -61,7 +61,6 @@ platforms:
source_format: bash_script+csv
hash_type: md5
schedule: weekly
emulators: [duckstation, pcsx2, dolphin, rpcs3, ppsspp, cemu, xemu, vita3k, citra, melonds]
# dragoonDorise/EmuDeck = official repo (creator's account, 3.4k stars)
# EmuDeck/emudeck.github.io = official wiki (org account)
@@ -71,6 +70,7 @@ platforms:
logo: "https://raw.githubusercontent.com/libretro/retroarch-assets/master/src/xmb/flatui/lakka.svg"
scraper: libretro
inherits_from: retroarch
cores: all_libretro
schedule: weekly
retropie:
@@ -78,4 +78,5 @@ platforms:
status: archived # Last release: v4.8 (March 2022) - no update in 4 years
logo: "https://avatars.githubusercontent.com/u/11378204"
scraper: null
cores: all_libretro
schedule: null

View File

@@ -76,11 +76,6 @@ shared_groups:
md5: "7da1e5b7c482d4108d22a5b09631d967"
crc32: "d271798b"
size: 524350
# NP2kai also accepts FONT.ROM (uppercase) — ref: libretro.c:1813
- name: FONT.ROM
destination: np2kai/FONT.ROM
required: true
md5: "2af6179d7de4893ea0b705c00e9a98d6"
- name: 2608_bd.wav
destination: np2kai/2608_bd.wav
required: true

View File

@@ -5,6 +5,165 @@ source: "https://raw.githubusercontent.com/batocera-linux/batocera.linux/master/
base_destination: bios
hash_type: md5
verification_mode: md5
cores:
- 81
- a5200
- abuse
- arduous
- atari800
- azahar
- bennugd
- bk
- bluemsx
- bsnes
- bstone
- cannonball
- cap32
- catacombgl
- cdogs
- cemu
- cgenius
- citron
- clk
- corsixth
- demul
- devilutionx
- dhewm3
- dice
- dolphin
- dosbox_pure
- dxx-rebirth
- easyrpg
- ecwolf
- eduke32
- eka2l1
- emuscv
- etlegacy
- fake08
- fallout1-ce
- fallout2-ce
- fbneo
- fceumm
- flatpak
- flycast
- freechaf
- freeintv
- fury
- fuse
- gambatte
- gearsystem
- genesisplusgx
- glide64mk2
- gong
- gsplus
- gw
- gzdoom
- hatari
- hcl
- hurrican
- hypseus-singe
- ikemen
- ioquake3
- iortcw
- jazz2-native
- lindbergh-loader
- lowresnx
- lutro
- mame
- mame078plus
- mednafen_lynx
- mednafen_ngp
- mednafen_supergrafx
- mednafen_wswan
- melonds
- mgba
- minivmac
- model2emu
- moonlight
- mrboom
- neocd
- np2kai
- nxengine
- o2em
- odcommander
- openbor6412
- openjazz
- openjk
- openjkdf2
- openmohaa
- opera
- pce_fast
- pcfx
- pcsx2
- pcsx_rearmed
- pd777
- picodrive
- play
- pokemini
- potator
- ppsspp
- prboom
- prosystem
- puae
- px68k
- pygame
- pyxel
- quasi88
- raze
- reminiscence
- rpcs3
- ruffle
- samcoupe
- sameduck
- scummvm
- sdlpop
- sh
- shadps4
- snes9x
- solarus
- sonic2013
- sonic3-air
- sonic-mania
- steam
- stella
- superbroswar
- supermodel
- taradino
- tgbdual
- theforceengine
- theodore
- thextech
- tic80
- tr1x
- tr2x
- tsugaru
- tyrian
- tyrquake
- uqm
- uzem
- vb
- vecx
- vice_x64
- vircon32
- virtualjaguar
- vita3k
- vox_official
- vpinball
- wasm4
- wine-tkg
- x1
- x128
- x16emu
- xash3d_fwgs
- xemu
- xenia-canary
- xpet
- xplus4
- xrick
- xvic
- yabasanshiro
- yquake2
- zc210
systems:
atari-400-800:
files:

View File

@@ -5,6 +5,199 @@ source: "https://gitlab.com/recalbox/recalbox/-/raw/master/board/recalbox/fsover
base_destination: bios
hash_type: md5
verification_mode: md5
cores:
- "2048"
- 81
- a5200
- advancemame
- amiberry
- applewin
- arduous
- atari800
- b2
- beebem
- bk
- bluemsx
- boom3
- bsnes
- bsneshd
- cannonball
- cap32
- cdi2015
- corsixth
- craft
- crocods
- daphne
- desmume
- dice
- dinothawr
- dirksimple
- dolphin
- dolphin-gui
- dosbox
- dosbox_pure
- duckstation
- easyrpg
- ecwolf
- emuscv
- fake08
- fba2x
- fbneo
- fceumm
- flycast
- flycast-next
- fmsx
- freechaf
- freeintv
- frotz
- fuse
- gambatte
- gearcoleco
- geargrafx
- gearsystem
- genesisplusgx
- genesisplusgx_ex
- genesisplusgxwide
- geolith
- glide64mk2
- gliden64
- gliden64_20
- gong
- gpsp
- gsplus
- gw
- handy
- hatari
- hatarib
- holani
- imageviewer
- julius
- kronos
- lowresnx
- lutro
- mame0258
- mame0278
- mame2000
- mame2003
- mame2003_plus
- mame2010
- mame2015
- mame2016
- mednafen_lynx
- mednafen_ngp
- mednafen_pce_fast
- mednafen_pcfx
- mednafen_psx
- mednafen_psx_hw
- mednafen_saturn
- mednafen_supafaust
- mednafen_supergrafx
- mednafen_vb
- mednafen_wswan
- melonds
- mesen
- mesen_s
- meteor
- mgba
- minivmac
- mojozork
- moonlight
- mrboom
- mu
- mupen64plus
- mupen64plus_next
- n64_gles2
- neocd
- nestopia
- np2kai
- nxengine
- o2em
- openbor
- openlara
- opera
- oricutron
- parallel_n64
- pcsx2
- pcsx_rearmed
- pico8
- picodrive
- pisnes
- pokemini
- potator
- ppsspp
- prboom
- prosystem
- ps2
- puae
- px68k
- quasi88
- quicknes
- race
- rb5000
- reicast
- reminiscence
- retro8
- retrodream
- rice
- rice_gles2
- sameboy
- same_cdi
- sameduck
- scummvm
- sdlpop
- simcoupe
- snes9x
- snes9x2002
- snes9x2005
- snes9x2010
- solarus
- stella
- stella2014
- stonesoup
- supermodel
- swanstation
- tamalibretro
- tgbdual
- theodore
- thepowdertoy
- ti99sim
- tic80
- tyrquake
- uae4all
- uae4arm
- uzem
- vecx
- vice_x128
- vice_x64
- vice_x64sc
- vice_xcbm2
- vice_xcbm5x0
- vice_xpet
- vice_xplus4
- vice_xscpu64
- vice_xvic
- virtualjaguar
- vitaquake2
- vitaquake3
- vitavoyager
- vpinball
- vvvvvv
- wasm4
- x1
- x128
- x64
- x64sx
- xcbm2
- xcbm5x0
- xemu
- xpet
- xplus4
- xrick
- xroar
- xscpu64
- xvic
- yabasanshiro
- yabause
systems:
commodore-amiga:
files:

View File

@@ -4,6 +4,7 @@ dat_version: v1.19.0
homepage: https://www.retroarch.com
source: https://github.com/libretro/libretro-database/blob/master/dat/System.dat
base_destination: system
cores: all_libretro
hash_type: sha1
verification_mode: existence
systems:

View File

@@ -5,6 +5,165 @@ source: "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/ma
base_destination: bios
hash_type: md5
verification_mode: md5
cores:
- 81
- a5200
- abuse
- arduous
- atari800
- azahar
- bennugd
- bk
- bluemsx
- bsnes
- bstone
- cannonball
- cap32
- catacombgl
- cdogs
- cemu
- cgenius
- citron
- clk
- corsixth
- demul
- devilutionx
- dhewm3
- dice
- dolphin
- dosbox_pure
- dxx-rebirth
- easyrpg
- ecwolf
- eduke32
- eka2l1
- emuscv
- etlegacy
- fake08
- fallout1-ce
- fallout2-ce
- fbneo
- fceumm
- flatpak
- flycast
- freechaf
- freeintv
- fury
- fuse
- gambatte
- gearsystem
- genesisplusgx
- glide64mk2
- gong
- gsplus
- gw
- gzdoom
- hatari
- hcl
- hurrican
- hypseus-singe
- ikemen
- ioquake3
- iortcw
- jazz2-native
- lindbergh-loader
- lowresnx
- lutro
- mame
- mame078plus
- mednafen_lynx
- mednafen_ngp
- mednafen_supergrafx
- mednafen_wswan
- melonds
- mgba
- minivmac
- model2emu
- moonlight
- mrboom
- neocd
- np2kai
- nxengine
- o2em
- odcommander
- openbor6412
- openjazz
- openjk
- openjkdf2
- openmohaa
- opera
- pce_fast
- pcfx
- pcsx2
- pcsx_rearmed
- pd777
- picodrive
- play
- pokemini
- potator
- ppsspp
- prboom
- prosystem
- puae
- px68k
- pygame
- pyxel
- quasi88
- raze
- reminiscence
- rpcs3
- ruffle
- samcoupe
- sameduck
- scummvm
- sdlpop
- sh
- shadps4
- snes9x
- solarus
- sonic2013
- sonic3-air
- sonic-mania
- steam
- stella
- superbroswar
- supermodel
- taradino
- tgbdual
- theforceengine
- theodore
- thextech
- tic80
- tr1x
- tr2x
- tsugaru
- tyrian
- tyrquake
- uqm
- uzem
- vb
- vecx
- vice_x64
- vircon32
- virtualjaguar
- vita3k
- vox_official
- vpinball
- wasm4
- wine-tkg
- x1
- x128
- x16emu
- xash3d_fwgs
- xemu
- xenia-canary
- xpet
- xplus4
- xrick
- xvic
- yabasanshiro
- yquake2
- zc210
systems:
3do:
files:

View File

@@ -125,9 +125,14 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
(f.get("name"), f.get("destination", f.get("name")))
for f in system.get("files", [])
}
existing_lower = {
f.get("destination", f.get("name", "")).lower()
for f in system.get("files", [])
}
for gf in shared_groups[group_name]:
key = (gf.get("name"), gf.get("destination", gf.get("name")))
if key not in existing:
dest_lower = gf.get("destination", gf.get("name", "")).lower()
if key not in existing and dest_lower not in existing_lower:
system.setdefault("files", []).append(gf)
existing.add(key)
@@ -348,6 +353,44 @@ def group_identical_platforms(
return [(group, representatives[fp]) for fp, group in fingerprints.items()]
def resolve_platform_cores(
config: dict, profiles: dict[str, dict],
) -> set[str]:
"""Resolve which emulator profiles are relevant for a platform.
Resolution strategies (by priority):
1. cores: "all_libretro" — all profiles with libretro in type
2. cores: [list] — profiles whose dict key matches a core name
3. cores: absent — fallback to systems intersection
Alias profiles are always excluded (they point to another profile).
"""
cores_config = config.get("cores")
if cores_config == "all_libretro":
return {
name for name, p in profiles.items()
if "libretro" in p.get("type", "")
and p.get("type") != "alias"
}
if isinstance(cores_config, list):
core_set = set(cores_config)
return {
name for name in profiles
if name in core_set
and profiles[name].get("type") != "alias"
}
# Fallback: system ID intersection
platform_systems = set(config.get("systems", {}).keys())
return {
name for name, p in profiles.items()
if set(p.get("systems", [])) & platform_systems
and p.get("type") != "alias"
}
def safe_extract_zip(zip_path: str, dest_dir: str) -> None:
"""Extract a ZIP file safely, preventing zip-slip path traversal."""
dest = os.path.realpath(dest_dir)

View File

@@ -182,7 +182,7 @@ def _collect_emulator_extras(
base_dest: str,
emu_profiles: dict | None = None,
) -> list[dict]:
"""Collect extra files from emulator profiles not in the platform pack.
"""Collect core requirement files from emulator profiles not in the platform pack.
Uses the same system-overlap matching as verify.py cross-reference:
- Matches emulators by shared system IDs with the platform
@@ -237,15 +237,15 @@ def generate_pack(
platform_display = config.get("platform", platform_name)
base_dest = config.get("base_destination", "")
suffix = "Complete_Pack" if include_extras else "BIOS_Pack"
zip_name = f"{platform_display.replace(' ', '_')}_{suffix}.zip"
zip_name = f"{platform_display.replace(' ', '_')}_BIOS_Pack.zip"
zip_path = os.path.join(output_dir, zip_name)
os.makedirs(output_dir, exist_ok=True)
total_files = 0
missing_files = []
user_provided = []
seen_destinations = set()
seen_destinations: set[str] = set()
seen_lower: set[str] = set() # case-insensitive dedup for Windows/macOS
# Per-file status: worst status wins (missing > untested > ok)
file_status: dict[str, str] = {}
file_reasons: dict[str, str] = {}
@@ -277,6 +277,7 @@ def generate_pack(
if already_packed:
continue
seen_destinations.add(dedup_key)
seen_lower.add(dedup_key.lower())
file_status.setdefault(dedup_key, "ok")
instructions = file_entry.get("instructions", "Please provide this file manually.")
instr_name = f"INSTRUCTIONS_{file_entry['name']}.txt"
@@ -301,6 +302,7 @@ def generate_pack(
else:
zf.write(tmp_path, full_dest)
seen_destinations.add(dedup_key)
seen_lower.add(dedup_key.lower())
file_status.setdefault(dedup_key, "ok")
total_files += 1
else:
@@ -352,6 +354,7 @@ def generate_pack(
if already_packed:
continue
seen_destinations.add(dedup_key)
seen_lower.add(dedup_key.lower())
extract = file_entry.get("extract", False)
if extract and local_path.endswith(".zip"):
@@ -360,21 +363,23 @@ def generate_pack(
zf.write(local_path, full_dest)
total_files += 1
# Tier 2: emulator extras (files cores need but platform doesn't declare)
extra_count = 0
if include_extras:
# Core requirements: files platform's cores need but YAML doesn't declare
emu_profiles = load_emulator_profiles(emulators_dir)
extras = _collect_emulator_extras(
core_files = _collect_emulator_extras(
config, emulators_dir, db,
seen_destinations, base_dest, emu_profiles,
)
for fe in extras:
core_count = 0
for fe in core_files:
dest = _sanitize_path(fe.get("destination", fe["name"]))
if not dest:
continue
full_dest = f"{base_dest}/{dest}" if base_dest else dest
if full_dest in seen_destinations:
continue
# Skip case-insensitive duplicates (Windows/macOS FS safety)
if full_dest.lower() in seen_lower:
continue
local_path, status = resolve_file(fe, db, bios_dir, zip_contents)
if status in ("not_found", "external", "user_provided"):
@@ -382,7 +387,8 @@ def generate_pack(
zf.write(local_path, full_dest)
seen_destinations.add(full_dest)
extra_count += 1
seen_lower.add(full_dest.lower())
core_count += 1
total_files += 1
# Data directories from _data_dirs.yml
@@ -406,9 +412,10 @@ def generate_pack(
src = os.path.join(root, fname)
rel = os.path.relpath(src, local_path)
full = f"{dd_prefix}/{rel}"
if full in seen_destinations:
if full in seen_destinations or full.lower() in seen_lower:
continue
seen_destinations.add(full)
seen_lower.add(full.lower())
zf.write(src, full)
total_files += 1
@@ -422,8 +429,8 @@ def generate_pack(
parts.append(f"{files_untested} untested")
if files_miss:
parts.append(f"{files_miss} missing")
extras_msg = f", {extra_count} extras" if extra_count else ""
print(f" {zip_path}: {total_files} files packed{extras_msg}, {', '.join(parts)} [{verification_mode}]")
baseline = total_files - core_count
print(f" {zip_path}: {total_files} files packed ({baseline} baseline + {core_count} from cores), {', '.join(parts)} [{verification_mode}]")
for key, reason in sorted(file_reasons.items()):
status = file_status.get(key, "")
@@ -467,8 +474,9 @@ def main():
parser.add_argument("--db", default=DEFAULT_DB_FILE, help="Path to database.json")
parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR)
parser.add_argument("--output-dir", "-o", default=DEFAULT_OUTPUT_DIR)
# --include-extras is now a no-op: core requirements are always included
parser.add_argument("--include-extras", action="store_true",
help="Include emulator-recommended files not declared by platform")
help="(no-op) Core requirements are always included")
parser.add_argument("--emulators-dir", default="emulators")
parser.add_argument("--offline", action="store_true",
help="Skip data directory freshness check, use cache only")

View File

@@ -73,9 +73,18 @@ def parse_pack_counts(output: str) -> dict[str, tuple[int, int]]:
if m:
current_label = m.group(1)
continue
frac_m = re.search(r"(\d+)/(\d+) files OK", line)
if frac_m and "files packed" in line:
ok, total = int(frac_m.group(1)), int(frac_m.group(2))
if "files packed" not in line:
continue
# New format: "622 files packed (359 baseline + 263 from cores), 358/359 files OK"
base_m = re.search(r"\((\d+) baseline", line)
ok_m = re.search(r"(\d+)/(\d+) files OK", line)
if base_m and ok_m:
baseline = int(base_m.group(1))
ok, total = int(ok_m.group(1)), int(ok_m.group(2))
counts[current_label] = (ok, total)
elif ok_m:
# Fallback: old format without baseline
ok, total = int(ok_m.group(1)), int(ok_m.group(2))
counts[current_label] = (ok, total)
return counts
@@ -123,8 +132,9 @@ def main():
help="Skip data directory refresh")
parser.add_argument("--output-dir", default="dist",
help="Pack output directory (default: dist/)")
# --include-extras is now a no-op: core requirements are always included
parser.add_argument("--include-extras", action="store_true",
help="Include Tier 2 emulator extras in packs")
help="(no-op) Core requirements are always included")
args = parser.parse_args()
results = {}

View File

@@ -14,6 +14,8 @@ import sys
import urllib.request
import urllib.error
import yaml
from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag
PLATFORM_NAME = "batocera"
@@ -23,6 +25,12 @@ SOURCE_URL = (
"master/package/batocera/core/batocera-scripts/scripts/batocera-systems"
)
CONFIGGEN_DEFAULTS_URL = (
"https://raw.githubusercontent.com/batocera-linux/batocera.linux/"
"master/package/batocera/core/batocera-configgen/configs/"
"configgen-defaults.yml"
)
SYSTEM_SLUG_MAP = {
"atari800": "atari-400-800",
"atari5200": "atari-5200",
@@ -91,6 +99,28 @@ class Scraper(BaseScraper):
def __init__(self, url: str = SOURCE_URL):
super().__init__(url=url)
def _fetch_cores(self) -> list[str]:
"""Extract core names from Batocera configgen-defaults.yml."""
try:
req = urllib.request.Request(
CONFIGGEN_DEFAULTS_URL,
headers={"User-Agent": "retrobios-scraper/1.0"},
)
with urllib.request.urlopen(req, timeout=30) as resp:
raw = resp.read().decode("utf-8")
except urllib.error.URLError as e:
raise ConnectionError(
f"Failed to fetch {CONFIGGEN_DEFAULTS_URL}: {e}"
) from e
data = yaml.safe_load(raw)
cores: set[str] = set()
for system, cfg in data.items():
if system == "default" or not isinstance(cfg, dict):
continue
core = cfg.get("core")
if core:
cores.add(core)
return sorted(cores)
def _extract_systems_dict(self, raw: str) -> dict:
"""Extract and parse the 'systems' dict from the Python source via ast.literal_eval."""
@@ -244,6 +274,7 @@ class Scraper(BaseScraper):
"base_destination": "bios",
"hash_type": "md5",
"verification_mode": "md5",
"cores": self._fetch_cores(),
"systems": systems,
}

View File

@@ -88,6 +88,20 @@ class Scraper(BaseScraper):
def __init__(self, url: str = SOURCE_URL):
super().__init__(url=url)
def _fetch_cores(self) -> list[str]:
"""Extract unique core names from es_bios.xml bios elements."""
raw = self._fetch_raw()
root = ET.fromstring(raw)
cores: set[str] = set()
for bios_elem in root.findall(".//system/bios"):
raw_core = bios_elem.get("core", "").strip()
if not raw_core:
continue
for part in raw_core.split(","):
name = part.strip()
if name:
cores.add(name)
return sorted(cores)
def fetch_requirements(self) -> list[BiosRequirement]:
"""Parse es_bios.xml and return BIOS requirements."""
@@ -214,6 +228,7 @@ class Scraper(BaseScraper):
"base_destination": "bios",
"hash_type": "md5",
"verification_mode": "md5",
"cores": self._fetch_cores(),
"systems": systems,
}

View File

@@ -37,7 +37,7 @@ sys.path.insert(0, os.path.dirname(__file__))
from common import (
build_zip_contents_index, check_inside_zip, group_identical_platforms,
load_emulator_profiles, load_platform_config, md5sum, md5_composite,
resolve_local_file,
resolve_local_file, resolve_platform_cores,
)
DEFAULT_DB = "database.json"
@@ -198,9 +198,7 @@ def find_undeclared_files(
"""Find files needed by cores but not declared in platform config."""
# Collect all filenames declared by this platform
declared_names: set[str] = set()
platform_systems: set[str] = set()
for sys_id, system in config.get("systems", {}).items():
platform_systems.add(sys_id)
for fe in system.get("files", []):
name = fe.get("name", "")
if name:
@@ -217,15 +215,13 @@ def find_undeclared_files(
by_name = db.get("indexes", {}).get("by_name", {})
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir)
relevant = resolve_platform_cores(config, profiles)
undeclared = []
seen = set()
for emu_name, profile in sorted(profiles.items()):
# Skip launchers — they don't use system_dir for BIOS
if profile.get("type") == "launcher":
if profile.get("type") in ("launcher", "alias"):
continue
emu_systems = set(profile.get("systems", []))
# Only check emulators whose systems overlap with this platform
if not emu_systems & platform_systems:
if emu_name not in relevant:
continue
for f in profile.get("files", []):
@@ -268,10 +264,12 @@ def find_exclusion_notes(
for sys_id in config.get("systems", {}):
platform_systems.add(sys_id)
relevant = resolve_platform_cores(config, profiles)
notes = []
for emu_name, profile in sorted(profiles.items()):
emu_systems = set(profile.get("systems", []))
if not emu_systems & platform_systems:
# Match by core resolution OR system intersection (documents all potential emulators)
if emu_name not in relevant and not (emu_systems & platform_systems):
continue
emu_display = profile.get("emulator", emu_name)

View File

@@ -31,9 +31,9 @@ import yaml
from common import (
build_zip_contents_index, check_inside_zip, group_identical_platforms,
load_emulator_profiles, load_platform_config, md5_composite, md5sum,
resolve_local_file,
resolve_local_file, resolve_platform_cores,
)
from verify import Severity, Status, verify_platform, find_undeclared_files
from verify import Severity, Status, verify_platform, find_undeclared_files, find_exclusion_notes
def _h(data: bytes) -> dict:
@@ -555,5 +555,91 @@ class TestE2E(unittest.TestCase):
self.assertEqual(status, "user_provided")
def test_resolve_cores_all_libretro(self):
"""all_libretro resolves to all libretro-type profiles, excludes alias/standalone."""
config = {"cores": "all_libretro", "systems": {"nes": {"files": []}}}
profiles = {
"fceumm": {"type": "libretro", "systems": ["nes"], "files": []},
"dolphin_standalone": {"type": "standalone", "systems": ["gc"], "files": []},
"gambatte": {"type": "pure_libretro", "systems": ["gb"], "files": []},
"mednafen_psx_hw": {"type": "alias", "alias_of": "beetle_psx", "files": []},
}
result = resolve_platform_cores(config, profiles)
self.assertEqual(result, {"fceumm", "gambatte"})
def test_resolve_cores_explicit_list(self):
"""Explicit cores list matches against profile dict keys."""
config = {"cores": ["fbneo", "opera"], "systems": {"arcade": {"files": []}}}
profiles = {
"fbneo": {"type": "pure_libretro", "systems": ["arcade"], "files": []},
"opera": {"type": "libretro", "systems": ["3do"], "files": []},
"mame": {"type": "libretro", "systems": ["arcade"], "files": []},
}
result = resolve_platform_cores(config, profiles)
self.assertEqual(result, {"fbneo", "opera"})
def test_resolve_cores_fallback_systems(self):
"""Missing cores: field falls back to system ID intersection."""
config = {"systems": {"nes": {"files": []}}}
profiles = {
"fceumm": {"type": "libretro", "systems": ["nes"], "files": []},
"dolphin": {"type": "libretro", "systems": ["gc"], "files": []},
}
result = resolve_platform_cores(config, profiles)
self.assertEqual(result, {"fceumm"})
def test_resolve_cores_excludes_alias(self):
"""Alias profiles never included even if name matches cores list."""
config = {"cores": ["mednafen_psx_hw"], "systems": {}}
profiles = {
"mednafen_psx_hw": {"type": "alias", "alias_of": "beetle_psx", "files": []},
}
result = resolve_platform_cores(config, profiles)
self.assertEqual(result, set())
def test_cross_reference_uses_core_resolution(self):
"""Cross-reference matches by cores: field, not system intersection."""
config = {
"cores": ["fbneo"],
"systems": {
"arcade": {"files": [{"name": "neogeo.zip", "md5": "abc"}]}
}
}
profiles = {
"fbneo": {
"emulator": "FBNeo", "systems": ["snk-neogeo-mvs"],
"type": "pure_libretro",
"files": [
{"name": "neogeo.zip", "required": True},
{"name": "neocdz.zip", "required": True},
],
},
}
db = {"indexes": {"by_name": {"neocdz.zip": {"sha1": "x"}}}}
undeclared = find_undeclared_files(config, self.emulators_dir, db, profiles)
names = [u["name"] for u in undeclared]
self.assertIn("neocdz.zip", names)
self.assertNotIn("neogeo.zip", names)
def test_exclusion_notes_uses_core_resolution(self):
"""Exclusion notes match by cores: field, not system intersection."""
config = {
"cores": ["desmume2015"],
"systems": {"nds": {"files": []}}
}
profiles = {
"desmume2015": {
"emulator": "DeSmuME 2015", "type": "frozen_snapshot",
"systems": ["nintendo-ds"],
"files": [],
"exclusion_note": "Frozen snapshot, code never loads BIOS",
},
}
notes = find_exclusion_notes(config, self.emulators_dir, profiles)
emu_names = [n["emulator"] for n in notes]
self.assertIn("DeSmuME 2015", emu_names)
if __name__ == "__main__":
unittest.main()