diff --git a/database.json b/database.json index 043aa029..b94b1ded 100644 --- a/database.json +++ b/database.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-03-19T13:30:04Z", + "generated_at": "2026-03-19T15:04:46Z", "total_files": 5593, "total_size": 4909044289, "files": { diff --git a/platforms/_data_dirs.yml b/platforms/_data_dirs.yml index 9044697f..55407b7a 100644 --- a/platforms/_data_dirs.yml +++ b/platforms/_data_dirs.yml @@ -24,6 +24,7 @@ data_directories: source_type: zip for_platforms: [retroarch, lakka, retropie] local_cache: data/dolphin-sys + strip_components: 2 exclude: [Themes] description: "Dolphin system data (GameSettings, DSP, fonts, shaders)" @@ -33,6 +34,7 @@ data_directories: source_type: zip for_platforms: [retroarch, lakka, retropie] local_cache: data/ppsspp-assets + strip_components: 1 description: "PPSSPP fonts, backgrounds, shaders, lang files" # ref: bluemsx-libretro/system/ — system/Databases/ + system/Machines/ diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 0043e254..e46f5260 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -16,7 +16,7 @@ platforms: source_format: clrmamepro_dat hash_type: sha1 schedule: weekly - emulators: [pcsx_rearmed, beetle_psx, genesis_plus_gx, flycast, melonds, mgba, snes9x, mupen64plus, beetle_saturn, dolphin] + cores: all_libretro batocera: config: batocera.yml @@ -27,7 +27,7 @@ platforms: source_format: python_dict hash_type: md5 schedule: weekly - emulators: [flycast, dolphin, pcsx2, duckstation, rpcs3, ppsspp, beetle_psx, beetle_saturn, genesis_plus_gx, picodrive, fbneo, puae, hatari, fuse, opera, bluemsx, fmsx, np2kai, quasi88] + cores: [81, a5200, abuse, arduous, atari800, azahar, bennugd, bk, bluemsx, bsnes, bstone, cannonball, cap32, catacombgl, cdogs, cemu, cgenius, citron, clk, corsixth, demul, devilutionx, dhewm3, dice, dolphin, dosbox_pure, dxx-rebirth, easyrpg, ecwolf, eduke32, eka2l1, emuscv, etlegacy, fake08, fallout1-ce, fallout2-ce, fbneo, fceumm, flatpak, flycast, freechaf, freeintv, fury, fuse, gambatte, gearsystem, genesisplusgx, glide64mk2, gong, gsplus, gw, gzdoom, hatari, hcl, hurrican, hypseus-singe, ikemen, ioquake3, iortcw, jazz2-native, lindbergh-loader, lowresnx, lutro, mame, mame078plus, mednafen_lynx, mednafen_ngp, mednafen_supergrafx, mednafen_wswan, melonds, mgba, minivmac, model2emu, moonlight, mrboom, neocd, np2kai, nxengine, o2em, odcommander, openbor6412, openjazz, openjk, openjkdf2, openmohaa, opera, pce_fast, pcfx, pcsx2, pcsx_rearmed, pd777, picodrive, play, pokemini, potator, ppsspp, prboom, prosystem, puae, px68k, pygame, pyxel, quasi88, raze, reminiscence, rpcs3, ruffle, samcoupe, sameduck, scummvm, sdlpop, sh, shadps4, snes9x, solarus, sonic2013, sonic3-air, sonic-mania, steam, stella, superbroswar, supermodel, taradino, tgbdual, theforceengine, theodore, thextech, tic80, tr1x, tr2x, tsugaru, tyrian, tyrquake, uqm, uzem, vb, vecx, vice_x64, vircon32, virtualjaguar, vita3k, vox_official, vpinball, wasm4, wine-tkg, x1, x128, x16emu, xash3d_fwgs, xemu, xenia-canary, xpet, xplus4, xrick, xvic, yabasanshiro, yquake2, zc210] recalbox: config: recalbox.yml @@ -38,7 +38,7 @@ platforms: source_format: xml hash_type: md5 schedule: monthly - emulators: [flycast, dolphin, pcsx2, beetle_psx, beetle_saturn, genesis_plus_gx, picodrive, fbneo, puae, hatari, opera, bluemsx, fmsx] + cores: ["2048", 81, a5200, advancemame, amiberry, applewin, arduous, atari800, b2, beebem, bk, bluemsx, boom3, bsnes, bsneshd, cannonball, cap32, cdi2015, corsixth, craft, crocods, daphne, desmume, dice, dinothawr, dirksimple, dolphin, dolphin-gui, dosbox, dosbox_pure, duckstation, easyrpg, ecwolf, emuscv, fake08, fba2x, fbneo, fceumm, flycast, flycast-next, fmsx, freechaf, freeintv, frotz, fuse, gambatte, gearcoleco, geargrafx, gearsystem, genesisplusgx, genesisplusgx_ex, genesisplusgxwide, geolith, glide64mk2, gliden64, gliden64_20, gong, gpsp, gsplus, gw, handy, hatari, hatarib, holani, imageviewer, julius, kronos, lowresnx, lutro, mame0258, mame0278, mame2000, mame2003, mame2003_plus, mame2010, mame2015, mame2016, mednafen_lynx, mednafen_ngp, mednafen_pce_fast, mednafen_pcfx, mednafen_psx, mednafen_psx_hw, mednafen_saturn, mednafen_supafaust, mednafen_supergrafx, mednafen_vb, mednafen_wswan, melonds, mesen, mesen_s, meteor, mgba, minivmac, mojozork, moonlight, mrboom, mu, mupen64plus, mupen64plus_next, n64_gles2, neocd, nestopia, np2kai, nxengine, o2em, openbor, openlara, opera, oricutron, parallel_n64, pcsx2, pcsx_rearmed, pico8, picodrive, pisnes, pokemini, potator, ppsspp, prboom, prosystem, ps2, puae, px68k, quasi88, quicknes, race, rb5000, reicast, reminiscence, retro8, retrodream, rice, rice_gles2, sameboy, same_cdi, sameduck, scummvm, sdlpop, simcoupe, snes9x, snes9x2002, snes9x2005, snes9x2010, solarus, stella, stella2014, stonesoup, supermodel, swanstation, tamalibretro, tgbdual, theodore, thepowdertoy, ti99sim, tic80, tyrquake, uae4all, uae4arm, uzem, vecx, vice_x128, vice_x64, vice_x64sc, vice_xcbm2, vice_xcbm5x0, vice_xpet, vice_xplus4, vice_xscpu64, vice_xvic, virtualjaguar, vitaquake2, vitaquake3, vitavoyager, vpinball, vvvvvv, wasm4, x1, x128, x64, x64sx, xcbm2, xcbm5x0, xemu, xpet, xplus4, xrick, xroar, xscpu64, xvic, yabasanshiro, yabause] retrobat: config: retrobat.yml @@ -49,7 +49,7 @@ platforms: source_format: json hash_type: md5 schedule: weekly - emulators: [duckstation, pcsx2, dolphin, rpcs3, ppsspp, cemu, xemu, flycast, beetle_psx, beetle_saturn, genesis_plus_gx, puae, opera] + cores: [81, a5200, abuse, arduous, atari800, azahar, bennugd, bk, bluemsx, bsnes, bstone, cannonball, cap32, catacombgl, cdogs, cemu, cgenius, citron, clk, corsixth, demul, devilutionx, dhewm3, dice, dolphin, dosbox_pure, dxx-rebirth, easyrpg, ecwolf, eduke32, eka2l1, emuscv, etlegacy, fake08, fallout1-ce, fallout2-ce, fbneo, fceumm, flatpak, flycast, freechaf, freeintv, fury, fuse, gambatte, gearsystem, genesisplusgx, glide64mk2, gong, gsplus, gw, gzdoom, hatari, hcl, hurrican, hypseus-singe, ikemen, ioquake3, iortcw, jazz2-native, lindbergh-loader, lowresnx, lutro, mame, mame078plus, mednafen_lynx, mednafen_ngp, mednafen_supergrafx, mednafen_wswan, melonds, mgba, minivmac, model2emu, moonlight, mrboom, neocd, np2kai, nxengine, o2em, odcommander, openbor6412, openjazz, openjk, openjkdf2, openmohaa, opera, pce_fast, pcfx, pcsx2, pcsx_rearmed, pd777, picodrive, play, pokemini, potator, ppsspp, prboom, prosystem, puae, px68k, pygame, pyxel, quasi88, raze, reminiscence, rpcs3, ruffle, samcoupe, sameduck, scummvm, sdlpop, sh, shadps4, snes9x, solarus, sonic2013, sonic3-air, sonic-mania, steam, stella, superbroswar, supermodel, taradino, tgbdual, theforceengine, theodore, thextech, tic80, tr1x, tr2x, tsugaru, tyrian, tyrquake, uqm, uzem, vb, vecx, vice_x64, vircon32, virtualjaguar, vita3k, vox_official, vpinball, wasm4, wine-tkg, x1, x128, x16emu, xash3d_fwgs, xemu, xenia-canary, xpet, xplus4, xrick, xvic, yabasanshiro, yquake2, zc210] emudeck: config: emudeck.yml @@ -61,7 +61,6 @@ platforms: source_format: bash_script+csv hash_type: md5 schedule: weekly - emulators: [duckstation, pcsx2, dolphin, rpcs3, ppsspp, cemu, xemu, vita3k, citra, melonds] # dragoonDorise/EmuDeck = official repo (creator's account, 3.4k stars) # EmuDeck/emudeck.github.io = official wiki (org account) @@ -71,6 +70,7 @@ platforms: logo: "https://raw.githubusercontent.com/libretro/retroarch-assets/master/src/xmb/flatui/lakka.svg" scraper: libretro inherits_from: retroarch + cores: all_libretro schedule: weekly retropie: @@ -78,4 +78,5 @@ platforms: status: archived # Last release: v4.8 (March 2022) - no update in 4 years logo: "https://avatars.githubusercontent.com/u/11378204" scraper: null + cores: all_libretro schedule: null diff --git a/platforms/_shared.yml b/platforms/_shared.yml index 1cc4777f..96043049 100644 --- a/platforms/_shared.yml +++ b/platforms/_shared.yml @@ -76,11 +76,6 @@ shared_groups: md5: "7da1e5b7c482d4108d22a5b09631d967" crc32: "d271798b" size: 524350 - # NP2kai also accepts FONT.ROM (uppercase) — ref: libretro.c:1813 - - name: FONT.ROM - destination: np2kai/FONT.ROM - required: true - md5: "2af6179d7de4893ea0b705c00e9a98d6" - name: 2608_bd.wav destination: np2kai/2608_bd.wav required: true diff --git a/platforms/batocera.yml b/platforms/batocera.yml index 108018a0..3d7cea7a 100644 --- a/platforms/batocera.yml +++ b/platforms/batocera.yml @@ -5,6 +5,165 @@ source: "https://raw.githubusercontent.com/batocera-linux/batocera.linux/master/ base_destination: bios hash_type: md5 verification_mode: md5 +cores: + - 81 + - a5200 + - abuse + - arduous + - atari800 + - azahar + - bennugd + - bk + - bluemsx + - bsnes + - bstone + - cannonball + - cap32 + - catacombgl + - cdogs + - cemu + - cgenius + - citron + - clk + - corsixth + - demul + - devilutionx + - dhewm3 + - dice + - dolphin + - dosbox_pure + - dxx-rebirth + - easyrpg + - ecwolf + - eduke32 + - eka2l1 + - emuscv + - etlegacy + - fake08 + - fallout1-ce + - fallout2-ce + - fbneo + - fceumm + - flatpak + - flycast + - freechaf + - freeintv + - fury + - fuse + - gambatte + - gearsystem + - genesisplusgx + - glide64mk2 + - gong + - gsplus + - gw + - gzdoom + - hatari + - hcl + - hurrican + - hypseus-singe + - ikemen + - ioquake3 + - iortcw + - jazz2-native + - lindbergh-loader + - lowresnx + - lutro + - mame + - mame078plus + - mednafen_lynx + - mednafen_ngp + - mednafen_supergrafx + - mednafen_wswan + - melonds + - mgba + - minivmac + - model2emu + - moonlight + - mrboom + - neocd + - np2kai + - nxengine + - o2em + - odcommander + - openbor6412 + - openjazz + - openjk + - openjkdf2 + - openmohaa + - opera + - pce_fast + - pcfx + - pcsx2 + - pcsx_rearmed + - pd777 + - picodrive + - play + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - puae + - px68k + - pygame + - pyxel + - quasi88 + - raze + - reminiscence + - rpcs3 + - ruffle + - samcoupe + - sameduck + - scummvm + - sdlpop + - sh + - shadps4 + - snes9x + - solarus + - sonic2013 + - sonic3-air + - sonic-mania + - steam + - stella + - superbroswar + - supermodel + - taradino + - tgbdual + - theforceengine + - theodore + - thextech + - tic80 + - tr1x + - tr2x + - tsugaru + - tyrian + - tyrquake + - uqm + - uzem + - vb + - vecx + - vice_x64 + - vircon32 + - virtualjaguar + - vita3k + - vox_official + - vpinball + - wasm4 + - wine-tkg + - x1 + - x128 + - x16emu + - xash3d_fwgs + - xemu + - xenia-canary + - xpet + - xplus4 + - xrick + - xvic + - yabasanshiro + - yquake2 + - zc210 systems: atari-400-800: files: diff --git a/platforms/recalbox.yml b/platforms/recalbox.yml index aae926a3..b9e96e72 100644 --- a/platforms/recalbox.yml +++ b/platforms/recalbox.yml @@ -5,6 +5,199 @@ source: "https://gitlab.com/recalbox/recalbox/-/raw/master/board/recalbox/fsover base_destination: bios hash_type: md5 verification_mode: md5 +cores: + - "2048" + - 81 + - a5200 + - advancemame + - amiberry + - applewin + - arduous + - atari800 + - b2 + - beebem + - bk + - bluemsx + - boom3 + - bsnes + - bsneshd + - cannonball + - cap32 + - cdi2015 + - corsixth + - craft + - crocods + - daphne + - desmume + - dice + - dinothawr + - dirksimple + - dolphin + - dolphin-gui + - dosbox + - dosbox_pure + - duckstation + - easyrpg + - ecwolf + - emuscv + - fake08 + - fba2x + - fbneo + - fceumm + - flycast + - flycast-next + - fmsx + - freechaf + - freeintv + - frotz + - fuse + - gambatte + - gearcoleco + - geargrafx + - gearsystem + - genesisplusgx + - genesisplusgx_ex + - genesisplusgxwide + - geolith + - glide64mk2 + - gliden64 + - gliden64_20 + - gong + - gpsp + - gsplus + - gw + - handy + - hatari + - hatarib + - holani + - imageviewer + - julius + - kronos + - lowresnx + - lutro + - mame0258 + - mame0278 + - mame2000 + - mame2003 + - mame2003_plus + - mame2010 + - mame2015 + - mame2016 + - mednafen_lynx + - mednafen_ngp + - mednafen_pce_fast + - mednafen_pcfx + - mednafen_psx + - mednafen_psx_hw + - mednafen_saturn + - mednafen_supafaust + - mednafen_supergrafx + - mednafen_vb + - mednafen_wswan + - melonds + - mesen + - mesen_s + - meteor + - mgba + - minivmac + - mojozork + - moonlight + - mrboom + - mu + - mupen64plus + - mupen64plus_next + - n64_gles2 + - neocd + - nestopia + - np2kai + - nxengine + - o2em + - openbor + - openlara + - opera + - oricutron + - parallel_n64 + - pcsx2 + - pcsx_rearmed + - pico8 + - picodrive + - pisnes + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - ps2 + - puae + - px68k + - quasi88 + - quicknes + - race + - rb5000 + - reicast + - reminiscence + - retro8 + - retrodream + - rice + - rice_gles2 + - sameboy + - same_cdi + - sameduck + - scummvm + - sdlpop + - simcoupe + - snes9x + - snes9x2002 + - snes9x2005 + - snes9x2010 + - solarus + - stella + - stella2014 + - stonesoup + - supermodel + - swanstation + - tamalibretro + - tgbdual + - theodore + - thepowdertoy + - ti99sim + - tic80 + - tyrquake + - uae4all + - uae4arm + - uzem + - vecx + - vice_x128 + - vice_x64 + - vice_x64sc + - vice_xcbm2 + - vice_xcbm5x0 + - vice_xpet + - vice_xplus4 + - vice_xscpu64 + - vice_xvic + - virtualjaguar + - vitaquake2 + - vitaquake3 + - vitavoyager + - vpinball + - vvvvvv + - wasm4 + - x1 + - x128 + - x64 + - x64sx + - xcbm2 + - xcbm5x0 + - xemu + - xpet + - xplus4 + - xrick + - xroar + - xscpu64 + - xvic + - yabasanshiro + - yabause systems: commodore-amiga: files: diff --git a/platforms/retroarch.yml b/platforms/retroarch.yml index c3ef0f0d..528ceac5 100644 --- a/platforms/retroarch.yml +++ b/platforms/retroarch.yml @@ -4,6 +4,7 @@ dat_version: v1.19.0 homepage: https://www.retroarch.com source: https://github.com/libretro/libretro-database/blob/master/dat/System.dat base_destination: system +cores: all_libretro hash_type: sha1 verification_mode: existence systems: diff --git a/platforms/retrobat.yml b/platforms/retrobat.yml index 5a13f136..cb96c4dd 100644 --- a/platforms/retrobat.yml +++ b/platforms/retrobat.yml @@ -5,6 +5,165 @@ source: "https://raw.githubusercontent.com/RetroBat-Official/emulatorlauncher/ma base_destination: bios hash_type: md5 verification_mode: md5 +cores: + - 81 + - a5200 + - abuse + - arduous + - atari800 + - azahar + - bennugd + - bk + - bluemsx + - bsnes + - bstone + - cannonball + - cap32 + - catacombgl + - cdogs + - cemu + - cgenius + - citron + - clk + - corsixth + - demul + - devilutionx + - dhewm3 + - dice + - dolphin + - dosbox_pure + - dxx-rebirth + - easyrpg + - ecwolf + - eduke32 + - eka2l1 + - emuscv + - etlegacy + - fake08 + - fallout1-ce + - fallout2-ce + - fbneo + - fceumm + - flatpak + - flycast + - freechaf + - freeintv + - fury + - fuse + - gambatte + - gearsystem + - genesisplusgx + - glide64mk2 + - gong + - gsplus + - gw + - gzdoom + - hatari + - hcl + - hurrican + - hypseus-singe + - ikemen + - ioquake3 + - iortcw + - jazz2-native + - lindbergh-loader + - lowresnx + - lutro + - mame + - mame078plus + - mednafen_lynx + - mednafen_ngp + - mednafen_supergrafx + - mednafen_wswan + - melonds + - mgba + - minivmac + - model2emu + - moonlight + - mrboom + - neocd + - np2kai + - nxengine + - o2em + - odcommander + - openbor6412 + - openjazz + - openjk + - openjkdf2 + - openmohaa + - opera + - pce_fast + - pcfx + - pcsx2 + - pcsx_rearmed + - pd777 + - picodrive + - play + - pokemini + - potator + - ppsspp + - prboom + - prosystem + - puae + - px68k + - pygame + - pyxel + - quasi88 + - raze + - reminiscence + - rpcs3 + - ruffle + - samcoupe + - sameduck + - scummvm + - sdlpop + - sh + - shadps4 + - snes9x + - solarus + - sonic2013 + - sonic3-air + - sonic-mania + - steam + - stella + - superbroswar + - supermodel + - taradino + - tgbdual + - theforceengine + - theodore + - thextech + - tic80 + - tr1x + - tr2x + - tsugaru + - tyrian + - tyrquake + - uqm + - uzem + - vb + - vecx + - vice_x64 + - vircon32 + - virtualjaguar + - vita3k + - vox_official + - vpinball + - wasm4 + - wine-tkg + - x1 + - x128 + - x16emu + - xash3d_fwgs + - xemu + - xenia-canary + - xpet + - xplus4 + - xrick + - xvic + - yabasanshiro + - yquake2 + - zc210 systems: 3do: files: diff --git a/scripts/common.py b/scripts/common.py index af05fcce..52c0ad6e 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -125,9 +125,14 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") - (f.get("name"), f.get("destination", f.get("name"))) for f in system.get("files", []) } + existing_lower = { + f.get("destination", f.get("name", "")).lower() + for f in system.get("files", []) + } for gf in shared_groups[group_name]: key = (gf.get("name"), gf.get("destination", gf.get("name"))) - if key not in existing: + dest_lower = gf.get("destination", gf.get("name", "")).lower() + if key not in existing and dest_lower not in existing_lower: system.setdefault("files", []).append(gf) existing.add(key) @@ -348,6 +353,44 @@ def group_identical_platforms( return [(group, representatives[fp]) for fp, group in fingerprints.items()] +def resolve_platform_cores( + config: dict, profiles: dict[str, dict], +) -> set[str]: + """Resolve which emulator profiles are relevant for a platform. + + Resolution strategies (by priority): + 1. cores: "all_libretro" — all profiles with libretro in type + 2. cores: [list] — profiles whose dict key matches a core name + 3. cores: absent — fallback to systems intersection + + Alias profiles are always excluded (they point to another profile). + """ + cores_config = config.get("cores") + + if cores_config == "all_libretro": + return { + name for name, p in profiles.items() + if "libretro" in p.get("type", "") + and p.get("type") != "alias" + } + + if isinstance(cores_config, list): + core_set = set(cores_config) + return { + name for name in profiles + if name in core_set + and profiles[name].get("type") != "alias" + } + + # Fallback: system ID intersection + platform_systems = set(config.get("systems", {}).keys()) + return { + name for name, p in profiles.items() + if set(p.get("systems", [])) & platform_systems + and p.get("type") != "alias" + } + + def safe_extract_zip(zip_path: str, dest_dir: str) -> None: """Extract a ZIP file safely, preventing zip-slip path traversal.""" dest = os.path.realpath(dest_dir) diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index b57081c5..1c873c40 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -182,7 +182,7 @@ def _collect_emulator_extras( base_dest: str, emu_profiles: dict | None = None, ) -> list[dict]: - """Collect extra files from emulator profiles not in the platform pack. + """Collect core requirement files from emulator profiles not in the platform pack. Uses the same system-overlap matching as verify.py cross-reference: - Matches emulators by shared system IDs with the platform @@ -237,15 +237,15 @@ def generate_pack( platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") - suffix = "Complete_Pack" if include_extras else "BIOS_Pack" - zip_name = f"{platform_display.replace(' ', '_')}_{suffix}.zip" + zip_name = f"{platform_display.replace(' ', '_')}_BIOS_Pack.zip" zip_path = os.path.join(output_dir, zip_name) os.makedirs(output_dir, exist_ok=True) total_files = 0 missing_files = [] user_provided = [] - seen_destinations = set() + seen_destinations: set[str] = set() + seen_lower: set[str] = set() # case-insensitive dedup for Windows/macOS # Per-file status: worst status wins (missing > untested > ok) file_status: dict[str, str] = {} file_reasons: dict[str, str] = {} @@ -277,6 +277,7 @@ def generate_pack( if already_packed: continue seen_destinations.add(dedup_key) + seen_lower.add(dedup_key.lower()) file_status.setdefault(dedup_key, "ok") instructions = file_entry.get("instructions", "Please provide this file manually.") instr_name = f"INSTRUCTIONS_{file_entry['name']}.txt" @@ -301,6 +302,7 @@ def generate_pack( else: zf.write(tmp_path, full_dest) seen_destinations.add(dedup_key) + seen_lower.add(dedup_key.lower()) file_status.setdefault(dedup_key, "ok") total_files += 1 else: @@ -352,6 +354,7 @@ def generate_pack( if already_packed: continue seen_destinations.add(dedup_key) + seen_lower.add(dedup_key.lower()) extract = file_entry.get("extract", False) if extract and local_path.endswith(".zip"): @@ -360,30 +363,33 @@ def generate_pack( zf.write(local_path, full_dest) total_files += 1 - # Tier 2: emulator extras (files cores need but platform doesn't declare) - extra_count = 0 - if include_extras: - emu_profiles = load_emulator_profiles(emulators_dir) - extras = _collect_emulator_extras( - config, emulators_dir, db, - seen_destinations, base_dest, emu_profiles, - ) - for fe in extras: - dest = _sanitize_path(fe.get("destination", fe["name"])) - if not dest: - continue - full_dest = f"{base_dest}/{dest}" if base_dest else dest - if full_dest in seen_destinations: - continue + # Core requirements: files platform's cores need but YAML doesn't declare + emu_profiles = load_emulator_profiles(emulators_dir) + core_files = _collect_emulator_extras( + config, emulators_dir, db, + seen_destinations, base_dest, emu_profiles, + ) + core_count = 0 + for fe in core_files: + dest = _sanitize_path(fe.get("destination", fe["name"])) + if not dest: + continue + full_dest = f"{base_dest}/{dest}" if base_dest else dest + if full_dest in seen_destinations: + continue + # Skip case-insensitive duplicates (Windows/macOS FS safety) + if full_dest.lower() in seen_lower: + continue - local_path, status = resolve_file(fe, db, bios_dir, zip_contents) - if status in ("not_found", "external", "user_provided"): - continue + local_path, status = resolve_file(fe, db, bios_dir, zip_contents) + if status in ("not_found", "external", "user_provided"): + continue - zf.write(local_path, full_dest) - seen_destinations.add(full_dest) - extra_count += 1 - total_files += 1 + zf.write(local_path, full_dest) + seen_destinations.add(full_dest) + seen_lower.add(full_dest.lower()) + core_count += 1 + total_files += 1 # Data directories from _data_dirs.yml for sys_id, system in sorted(config.get("systems", {}).items()): @@ -406,9 +412,10 @@ def generate_pack( src = os.path.join(root, fname) rel = os.path.relpath(src, local_path) full = f"{dd_prefix}/{rel}" - if full in seen_destinations: + if full in seen_destinations or full.lower() in seen_lower: continue seen_destinations.add(full) + seen_lower.add(full.lower()) zf.write(src, full) total_files += 1 @@ -422,8 +429,8 @@ def generate_pack( parts.append(f"{files_untested} untested") if files_miss: parts.append(f"{files_miss} missing") - extras_msg = f", {extra_count} extras" if extra_count else "" - print(f" {zip_path}: {total_files} files packed{extras_msg}, {', '.join(parts)} [{verification_mode}]") + baseline = total_files - core_count + print(f" {zip_path}: {total_files} files packed ({baseline} baseline + {core_count} from cores), {', '.join(parts)} [{verification_mode}]") for key, reason in sorted(file_reasons.items()): status = file_status.get(key, "") @@ -467,8 +474,9 @@ def main(): parser.add_argument("--db", default=DEFAULT_DB_FILE, help="Path to database.json") parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR) parser.add_argument("--output-dir", "-o", default=DEFAULT_OUTPUT_DIR) + # --include-extras is now a no-op: core requirements are always included parser.add_argument("--include-extras", action="store_true", - help="Include emulator-recommended files not declared by platform") + help="(no-op) Core requirements are always included") parser.add_argument("--emulators-dir", default="emulators") parser.add_argument("--offline", action="store_true", help="Skip data directory freshness check, use cache only") diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 9c7ba44d..1d428ece 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -73,9 +73,18 @@ def parse_pack_counts(output: str) -> dict[str, tuple[int, int]]: if m: current_label = m.group(1) continue - frac_m = re.search(r"(\d+)/(\d+) files OK", line) - if frac_m and "files packed" in line: - ok, total = int(frac_m.group(1)), int(frac_m.group(2)) + if "files packed" not in line: + continue + # New format: "622 files packed (359 baseline + 263 from cores), 358/359 files OK" + base_m = re.search(r"\((\d+) baseline", line) + ok_m = re.search(r"(\d+)/(\d+) files OK", line) + if base_m and ok_m: + baseline = int(base_m.group(1)) + ok, total = int(ok_m.group(1)), int(ok_m.group(2)) + counts[current_label] = (ok, total) + elif ok_m: + # Fallback: old format without baseline + ok, total = int(ok_m.group(1)), int(ok_m.group(2)) counts[current_label] = (ok, total) return counts @@ -123,8 +132,9 @@ def main(): help="Skip data directory refresh") parser.add_argument("--output-dir", default="dist", help="Pack output directory (default: dist/)") + # --include-extras is now a no-op: core requirements are always included parser.add_argument("--include-extras", action="store_true", - help="Include Tier 2 emulator extras in packs") + help="(no-op) Core requirements are always included") args = parser.parse_args() results = {} diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index 42373d35..283e8aa5 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -14,6 +14,8 @@ import sys import urllib.request import urllib.error +import yaml + from .base_scraper import BaseScraper, BiosRequirement, fetch_github_latest_tag PLATFORM_NAME = "batocera" @@ -23,6 +25,12 @@ SOURCE_URL = ( "master/package/batocera/core/batocera-scripts/scripts/batocera-systems" ) +CONFIGGEN_DEFAULTS_URL = ( + "https://raw.githubusercontent.com/batocera-linux/batocera.linux/" + "master/package/batocera/core/batocera-configgen/configs/" + "configgen-defaults.yml" +) + SYSTEM_SLUG_MAP = { "atari800": "atari-400-800", "atari5200": "atari-5200", @@ -91,6 +99,28 @@ class Scraper(BaseScraper): def __init__(self, url: str = SOURCE_URL): super().__init__(url=url) + def _fetch_cores(self) -> list[str]: + """Extract core names from Batocera configgen-defaults.yml.""" + try: + req = urllib.request.Request( + CONFIGGEN_DEFAULTS_URL, + headers={"User-Agent": "retrobios-scraper/1.0"}, + ) + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read().decode("utf-8") + except urllib.error.URLError as e: + raise ConnectionError( + f"Failed to fetch {CONFIGGEN_DEFAULTS_URL}: {e}" + ) from e + data = yaml.safe_load(raw) + cores: set[str] = set() + for system, cfg in data.items(): + if system == "default" or not isinstance(cfg, dict): + continue + core = cfg.get("core") + if core: + cores.add(core) + return sorted(cores) def _extract_systems_dict(self, raw: str) -> dict: """Extract and parse the 'systems' dict from the Python source via ast.literal_eval.""" @@ -244,6 +274,7 @@ class Scraper(BaseScraper): "base_destination": "bios", "hash_type": "md5", "verification_mode": "md5", + "cores": self._fetch_cores(), "systems": systems, } diff --git a/scripts/scraper/recalbox_scraper.py b/scripts/scraper/recalbox_scraper.py index 3dc379cd..e551ac0a 100644 --- a/scripts/scraper/recalbox_scraper.py +++ b/scripts/scraper/recalbox_scraper.py @@ -88,6 +88,20 @@ class Scraper(BaseScraper): def __init__(self, url: str = SOURCE_URL): super().__init__(url=url) + def _fetch_cores(self) -> list[str]: + """Extract unique core names from es_bios.xml bios elements.""" + raw = self._fetch_raw() + root = ET.fromstring(raw) + cores: set[str] = set() + for bios_elem in root.findall(".//system/bios"): + raw_core = bios_elem.get("core", "").strip() + if not raw_core: + continue + for part in raw_core.split(","): + name = part.strip() + if name: + cores.add(name) + return sorted(cores) def fetch_requirements(self) -> list[BiosRequirement]: """Parse es_bios.xml and return BIOS requirements.""" @@ -214,6 +228,7 @@ class Scraper(BaseScraper): "base_destination": "bios", "hash_type": "md5", "verification_mode": "md5", + "cores": self._fetch_cores(), "systems": systems, } diff --git a/scripts/verify.py b/scripts/verify.py index 49792321..2bbee3ff 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -37,7 +37,7 @@ sys.path.insert(0, os.path.dirname(__file__)) from common import ( build_zip_contents_index, check_inside_zip, group_identical_platforms, load_emulator_profiles, load_platform_config, md5sum, md5_composite, - resolve_local_file, + resolve_local_file, resolve_platform_cores, ) DEFAULT_DB = "database.json" @@ -198,9 +198,7 @@ def find_undeclared_files( """Find files needed by cores but not declared in platform config.""" # Collect all filenames declared by this platform declared_names: set[str] = set() - platform_systems: set[str] = set() for sys_id, system in config.get("systems", {}).items(): - platform_systems.add(sys_id) for fe in system.get("files", []): name = fe.get("name", "") if name: @@ -217,15 +215,13 @@ def find_undeclared_files( by_name = db.get("indexes", {}).get("by_name", {}) profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir) + relevant = resolve_platform_cores(config, profiles) undeclared = [] seen = set() for emu_name, profile in sorted(profiles.items()): - # Skip launchers — they don't use system_dir for BIOS - if profile.get("type") == "launcher": + if profile.get("type") in ("launcher", "alias"): continue - emu_systems = set(profile.get("systems", [])) - # Only check emulators whose systems overlap with this platform - if not emu_systems & platform_systems: + if emu_name not in relevant: continue for f in profile.get("files", []): @@ -268,10 +264,12 @@ def find_exclusion_notes( for sys_id in config.get("systems", {}): platform_systems.add(sys_id) + relevant = resolve_platform_cores(config, profiles) notes = [] for emu_name, profile in sorted(profiles.items()): emu_systems = set(profile.get("systems", [])) - if not emu_systems & platform_systems: + # Match by core resolution OR system intersection (documents all potential emulators) + if emu_name not in relevant and not (emu_systems & platform_systems): continue emu_display = profile.get("emulator", emu_name) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ac60a967..e638282c 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -31,9 +31,9 @@ import yaml from common import ( build_zip_contents_index, check_inside_zip, group_identical_platforms, load_emulator_profiles, load_platform_config, md5_composite, md5sum, - resolve_local_file, + resolve_local_file, resolve_platform_cores, ) -from verify import Severity, Status, verify_platform, find_undeclared_files +from verify import Severity, Status, verify_platform, find_undeclared_files, find_exclusion_notes def _h(data: bytes) -> dict: @@ -555,5 +555,91 @@ class TestE2E(unittest.TestCase): self.assertEqual(status, "user_provided") + def test_resolve_cores_all_libretro(self): + """all_libretro resolves to all libretro-type profiles, excludes alias/standalone.""" + config = {"cores": "all_libretro", "systems": {"nes": {"files": []}}} + profiles = { + "fceumm": {"type": "libretro", "systems": ["nes"], "files": []}, + "dolphin_standalone": {"type": "standalone", "systems": ["gc"], "files": []}, + "gambatte": {"type": "pure_libretro", "systems": ["gb"], "files": []}, + "mednafen_psx_hw": {"type": "alias", "alias_of": "beetle_psx", "files": []}, + } + result = resolve_platform_cores(config, profiles) + self.assertEqual(result, {"fceumm", "gambatte"}) + + def test_resolve_cores_explicit_list(self): + """Explicit cores list matches against profile dict keys.""" + config = {"cores": ["fbneo", "opera"], "systems": {"arcade": {"files": []}}} + profiles = { + "fbneo": {"type": "pure_libretro", "systems": ["arcade"], "files": []}, + "opera": {"type": "libretro", "systems": ["3do"], "files": []}, + "mame": {"type": "libretro", "systems": ["arcade"], "files": []}, + } + result = resolve_platform_cores(config, profiles) + self.assertEqual(result, {"fbneo", "opera"}) + + def test_resolve_cores_fallback_systems(self): + """Missing cores: field falls back to system ID intersection.""" + config = {"systems": {"nes": {"files": []}}} + profiles = { + "fceumm": {"type": "libretro", "systems": ["nes"], "files": []}, + "dolphin": {"type": "libretro", "systems": ["gc"], "files": []}, + } + result = resolve_platform_cores(config, profiles) + self.assertEqual(result, {"fceumm"}) + + def test_resolve_cores_excludes_alias(self): + """Alias profiles never included even if name matches cores list.""" + config = {"cores": ["mednafen_psx_hw"], "systems": {}} + profiles = { + "mednafen_psx_hw": {"type": "alias", "alias_of": "beetle_psx", "files": []}, + } + result = resolve_platform_cores(config, profiles) + self.assertEqual(result, set()) + + + def test_cross_reference_uses_core_resolution(self): + """Cross-reference matches by cores: field, not system intersection.""" + config = { + "cores": ["fbneo"], + "systems": { + "arcade": {"files": [{"name": "neogeo.zip", "md5": "abc"}]} + } + } + profiles = { + "fbneo": { + "emulator": "FBNeo", "systems": ["snk-neogeo-mvs"], + "type": "pure_libretro", + "files": [ + {"name": "neogeo.zip", "required": True}, + {"name": "neocdz.zip", "required": True}, + ], + }, + } + db = {"indexes": {"by_name": {"neocdz.zip": {"sha1": "x"}}}} + undeclared = find_undeclared_files(config, self.emulators_dir, db, profiles) + names = [u["name"] for u in undeclared] + self.assertIn("neocdz.zip", names) + self.assertNotIn("neogeo.zip", names) + + def test_exclusion_notes_uses_core_resolution(self): + """Exclusion notes match by cores: field, not system intersection.""" + config = { + "cores": ["desmume2015"], + "systems": {"nds": {"files": []}} + } + profiles = { + "desmume2015": { + "emulator": "DeSmuME 2015", "type": "frozen_snapshot", + "systems": ["nintendo-ds"], + "files": [], + "exclusion_note": "Frozen snapshot, code never loads BIOS", + }, + } + notes = find_exclusion_notes(config, self.emulators_dir, profiles) + emu_names = [n["emulator"] for n in notes] + self.assertIn("DeSmuME 2015", emu_names) + + if __name__ == "__main__": unittest.main()