fix: sha1-based large file restore, fix broken data dir urls

Replace grep-based restore with SHA1 matching via database.json.
The old grep heuristic failed for assets with renamed basenames
(dsi_nand_batocera42.bin) or special characters (MAME dots vs
spaces), and only restored to the first .gitignore match when
multiple paths shared a basename.

Fix 3 broken data directory sources:
- opentyrian: buildbot URL 404, use release asset
- syobonaction: invalid git_subtree URL, use GitHub archive
- stonesoup: same fix, adds 532 game data files

.github/workflows/build.yml

@@ -58,16 +58,32 @@ jobs:
         run: |
           mkdir -p .cache/large
           gh release download large-files -D .cache/large/ 2>/dev/null || true
-          for f in .cache/large/*; do
-            [ -f "$f" ] || continue
-            name=$(basename "$f")
-            target=$(grep "$name" .gitignore | head -1)
-            if [ -n "$target" ] && [ ! -f "$target" ]; then
-              mkdir -p "$(dirname "$target")"
-              cp "$f" "$target"
-              echo "Restored: $target"
-            fi
-          done
+          python3 -c "
+          import hashlib, json, os, shutil
+          db = json.load(open('database.json'))
+          with open('.gitignore') as f:
+              ignored = {l.strip() for l in f if l.strip().startswith('bios/')}
+          cache = '.cache/large'
+          if not os.path.isdir(cache):
+              exit(0)
+          idx = {}
+          for fn in os.listdir(cache):
+              fp = os.path.join(cache, fn)
+              if os.path.isfile(fp):
+                  h = hashlib.sha1(open(fp, 'rb').read()).hexdigest()
+                  idx[h] = fp
+          restored = 0
+          for sha1, entry in db['files'].items():
+              path = entry['path']
+              if path in ignored and not os.path.exists(path):
+                  src = idx.get(sha1)
+                  if src:
+                      os.makedirs(os.path.dirname(path), exist_ok=True)
+                      shutil.copy2(src, path)
+                      print(f'Restored: {path}')
+                      restored += 1
+          print(f'Total: {restored} files restored')
+          "
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/deploy-site.yml

@@ -36,6 +36,43 @@ jobs:
 
       - run: pip install pyyaml mkdocs-material pymdown-extensions
 
+      - name: Restore large files from release
+        run: |
+          mkdir -p .cache/large
+          gh release download large-files -D .cache/large/ 2>/dev/null || true
+          python3 -c "
+          import hashlib, json, os, shutil
+          db = json.load(open('database.json'))
+          with open('.gitignore') as f:
+              ignored = {l.strip() for l in f if l.strip().startswith('bios/')}
+          cache = '.cache/large'
+          if not os.path.isdir(cache):
+              exit(0)
+          idx = {}
+          for fn in os.listdir(cache):
+              fp = os.path.join(cache, fn)
+              if os.path.isfile(fp):
+                  h = hashlib.sha1(open(fp, 'rb').read()).hexdigest()
+                  idx[h] = fp
+          restored = 0
+          for sha1, entry in db['files'].items():
+              path = entry['path']
+              if path in ignored and not os.path.exists(path):
+                  src = idx.get(sha1)
+                  if src:
+                      os.makedirs(os.path.dirname(path), exist_ok=True)
+                      shutil.copy2(src, path)
+                      print(f'Restored: {path}')
+                      restored += 1
+          print(f'Total: {restored} files restored')
+          "
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Refresh data directories
+        run: python scripts/refresh_data_dirs.py
+        continue-on-error: true
+
       - name: Generate site
         run: |
           python scripts/generate_site.py

.gitignore

@@ -29,13 +29,14 @@ data/
 # Large files stored as GitHub Release assets (additional)
 bios/Arcade/MAME/artwork/snspell.zip
 bios/Arcade/MAME/MAME 0.174 Arcade XML.dat
-bios/Sony/PlayStation Vita/.variants/PSP2UPDAT.PUP
+bios/Sony/PlayStation Vita/.variants/PSP2UPDAT.PUP.3ae832c9
 bios/Nintendo/DS/DSi_Nand_JPN.bin
 bios/Nintendo/DS/DSi_Nand_EUR.bin
 bios/Nintendo/DS/DSi_Nand_USA.bin
 bios/Nintendo/DS/DSi_Nand_AUS.bin
 bios/Nintendo/DS/DSi_Nand_CHN.bin
 bios/Nintendo/DS/DSi_Nand_KOR.bin
+bios/Nintendo/DS/dsi_nand.bin
 
 # QEMU EDK2 firmware (64MB each)
 bios/QEMU/edk2-aarch64-code.fd

CONTRIBUTING.md

@@ -7,6 +7,16 @@
 3. Variants (alternate hashes): `bios/Manufacturer/Console/.variants/`
 4. Create a Pull Request - checksums are verified automatically
 
+## Add a new platform
+
+1. Write a scraper in `scripts/scraper/`
+2. Create the platform YAML in `platforms/`
+3. Register in `platforms/_registry.yml`
+4. Submit a Pull Request
+
+Contributors who add platform support are credited in the README,
+on the documentation site, and in the BIOS packs.
+
 ## File conventions
 
 - Files >50 MB go in GitHub release assets (`large-files` release)

README.md

@@ -2,7 +2,7 @@
 
 Complete BIOS and firmware packs for Batocera, BizHawk, EmuDeck, Lakka, Recalbox, RetroArch, RetroBat, RetroDECK, RetroPie, and RomM.
 
-**7,631** verified files across **352** systems, ready to extract into your emulator's BIOS directory.
+**7,302** verified files across **396** systems, ready to extract into your emulator's BIOS directory.
 
 ## Quick Install
 
@@ -27,14 +27,14 @@ Pick your platform, download the ZIP, extract to the BIOS path.
 
 | Platform | BIOS files | Extract to | Download |
 |----------|-----------|-----------|----------|
-| Batocera | 359 | `/userdata/bios/` | [Download](../../releases/latest) |
+| Batocera | 361 | `/userdata/bios/` | [Download](../../releases/latest) |
 | BizHawk | 118 | `Firmware/` | [Download](../../releases/latest) |
 | EmuDeck | 161 | `Emulation/bios/` | [Download](../../releases/latest) |
 | Lakka | 448 | `system/` | [Download](../../releases/latest) |
 | Recalbox | 346 | `/recalbox/share/bios/` | [Download](../../releases/latest) |
 | RetroArch | 448 | `system/` | [Download](../../releases/latest) |
-| RetroBat | 331 | `bios/` | [Download](../../releases/latest) |
-| RetroDECK | 2007 | `~/retrodeck/bios/` | [Download](../../releases/latest) |
+| RetroBat | 339 | `bios/` | [Download](../../releases/latest) |
+| RetroDECK | 2006 | `~/retrodeck/bios/` | [Download](../../releases/latest) |
 | RetroPie | 448 | `BIOS/` | [Download](../../releases/latest) |
 | RomM | 374 | `bios/{platform_slug}/` | [Download](../../releases/latest) |
 
@@ -44,14 +44,14 @@ BIOS, firmware, and system files for consoles from Atari to PlayStation 3.
 Each file is checked against the emulator's source code to match what the code actually loads at runtime.
 
 - **10 platforms** supported with platform-specific verification
-- **328 emulators** profiled from source (RetroArch cores + standalone)
-- **352 systems** covered (NES, SNES, PlayStation, Saturn, Dreamcast, ...)
-- **7,631 files** verified with MD5, SHA1, CRC32 checksums
-- **9295 MB** total collection size
+- **329 emulators** profiled from source (RetroArch cores + standalone)
+- **396 systems** covered (NES, SNES, PlayStation, Saturn, Dreamcast, ...)
+- **7,302 files** verified with MD5, SHA1, CRC32 checksums
+- **8765 MB** total collection size
 
 ## Supported systems
 
-NES, SNES, Nintendo 64, GameCube, Wii, Game Boy, Game Boy Advance, Nintendo DS, Nintendo 3DS, Switch, PlayStation, PlayStation 2, PlayStation 3, PSP, PS Vita, Mega Drive, Saturn, Dreamcast, Game Gear, Master System, Neo Geo, Atari 2600, Atari 7800, Atari Lynx, Atari ST, MSX, PC Engine, TurboGrafx-16, ColecoVision, Intellivision, Commodore 64, Amiga, ZX Spectrum, Arcade (MAME), and 318+ more.
+NES, SNES, Nintendo 64, GameCube, Wii, Game Boy, Game Boy Advance, Nintendo DS, Nintendo 3DS, Switch, PlayStation, PlayStation 2, PlayStation 3, PSP, PS Vita, Mega Drive, Saturn, Dreamcast, Game Gear, Master System, Neo Geo, Atari 2600, Atari 7800, Atari Lynx, Atari ST, MSX, PC Engine, TurboGrafx-16, ColecoVision, Intellivision, Commodore 64, Amiga, ZX Spectrum, Arcade (MAME), and 362+ more.
 
 Full list with per-file details: **[https://abdess.github.io/retrobios/](https://abdess.github.io/retrobios/)**
 
@@ -59,14 +59,14 @@ Full list with per-file details: **[https://abdess.github.io/retrobios/](https:/
 
 | Platform | Coverage | Verified | Untested | Missing |
 |----------|----------|----------|----------|---------|
-| Batocera | 359/359 (100.0%) | 359 | 0 | 0 |
+| Batocera | 361/361 (100.0%) | 361 | 0 | 0 |
 | BizHawk | 118/118 (100.0%) | 118 | 0 | 0 |
 | EmuDeck | 161/161 (100.0%) | 161 | 0 | 0 |
 | Lakka | 448/448 (100.0%) | 448 | 0 | 0 |
 | Recalbox | 346/346 (100.0%) | 346 | 0 | 0 |
 | RetroArch | 448/448 (100.0%) | 448 | 0 | 0 |
-| RetroBat | 331/331 (100.0%) | 331 | 0 | 0 |
-| RetroDECK | 2007/2007 (100.0%) | 2007 | 0 | 0 |
+| RetroBat | 339/339 (100.0%) | 339 | 0 | 0 |
+| RetroDECK | 2006/2006 (100.0%) | 2006 | 0 | 0 |
 | RetroPie | 448/448 (100.0%) | 448 | 0 | 0 |
 | RomM | 374/374 (100.0%) | 374 | 0 | 0 |
 
@@ -104,7 +104,7 @@ The [documentation site](https://abdess.github.io/retrobios/) provides:
 - **Per-emulator profiles** with source code references for every file
 - **Per-system pages** showing which emulators and platforms cover each console
 - **Gap analysis** identifying missing files and undeclared core requirements
-- **Cross-reference** mapping files across 10 platforms and 328 emulators
+- **Cross-reference** mapping files across 10 platforms and 329 emulators
 
 ## How it works
 
@@ -130,4 +130,4 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
 
 This repository provides BIOS files for personal backup and archival purposes.
 
-*Auto-generated on 2026-03-29T07:51:39Z*
+*Auto-generated on 2026-04-02T13:38:26Z*

bios/Apple/Apple II/DiskII/state-machine-13.rom

@@ -0,0 +1,65 @@
+ИИ<EFBFBD>Ш
+
+
+
+<EFBFBD>Щ<EFBFBD>Щ<EFBFBD>Ы<EFBFBD>Ы<EFBFBD>ШШ
+
+
+
+<EFBFBD>Щ<EFBFBD>Щ<EFBFBD>Ы<EFBFBD>ЫИ=ИИ
+
+
+
+<EFBFBD>й<EFBFBD>й<EFBFBD>л<EFBFBD>л<EFBFBD>н<EFBFBD>и
+
+
+
+<EFBFBD>й<EFBFBD>й<EFBFBD>л<EFBFBD>лИИИИ
+
+
+
+ЈшЈшЈшЈшЈшЈш
+
+
+
+ЈшЈшЈшЈшЙ§Иј
+
+
+
+ИјИјИјИјЙ§Pј
+
+
+
+ИјИјИјИјИИHИ
+
+
+
+H(H(H(H(H(H(
+
+
+
+H(H(H(H(ИЙИИ
+
+
+
+X8X8X8X8	ЩX8
+
+
+
+X8X8X8X8ИИИИ
+
+
+
+hhhhhh
+
+
+
+hhhh<18>Нxp
+
+
+
+xxxx
+-xp
+
+
+

bios/Apple/Apple II/DiskII/state-machine-16.rom

@@ -0,0 +1,65 @@
+Ш
+
+
+
+99;;8(
+
+
+
+99;;-Ш8H
+
+
+
+(H(H(H(H-H8H
+
+
+
+(H(H(H(HШШШШ
+
+
+
+XxXxXxXxXxXx
+
+
+
+XxXxXxXxШШШШ
+
+
+
+hh€hh€h€h€
+
+
+
+hh€hh€ШНШШ
+
+
+
+<EFBFBD>№<EFBFBD>№<EFBFBD>»<EFBFBD>»<EFBFBD>Ѕ<EFBFBD>ё
+
+
+
+<EFBFBD>№<EFBFBD>№<EFBFBD>»<EFBFBD>»ШЩШШ
+
+
+
+ЁИЁИЁИЁИ)YЁИ
+
+
+
+ЁИЁИЁИЁИЩэШш
+
+
+
+ШшШшШшШшЩэ ш
+
+
+
+ШшШшШшШшШЭиа
+
+
+
+и€ии€иMиа
+
+
+
+и€ии€и

bios/Microsoft/MSX/Machines/COL - Bit Corporation Dina/config.ini

@@ -1,25 +0,0 @@
-
-[CMOS]
-Enable CMOS=0
-Battery Backed=0
-[Video]
-version=TMS99x8A
-vram size=16kB
-[Subslotted Slots]
-slot 0=0
-slot 1=0
-slot 2=0
-slot 3=0
-[External Slots]
-slot A=1 0
-slot B=1 0
-[FDC]
-Count=0
-[CPU]
-Z80 Frequency=3579545Hz
-[Board]
-type=ColecoVision
-[Slots]
-0 0 0 1 20 "Machines/COL - Bit Corporation Dina/czz50-1.rom" ""
-0 0 3 1 93 "" ""
-0 0 4 1 20 "Machines/COL - Bit Corporation Dina/czz50-2.rom" ""

bios/Microsoft/MSX/Machines/COL - ColecoVision with Opcode Memory Extension/config.ini

@@ -1,25 +0,0 @@
-
-[CMOS]
-Enable CMOS=0
-Battery Backed=0
-[Video]
-version=TMS99x8A
-vram size=16kB
-[Subslotted Slots]
-slot 0=0
-slot 1=0
-slot 2=0
-slot 3=0
-[External Slots]
-slot A=1 0
-slot B=1 0
-[FDC]
-Count=2
-[CPU]
-Z80 Frequency=3579545Hz
-[Board]
-type=ColecoVision
-[Slots]
-0 0 0 1 20 "Machines/COL - ColecoVision with Opcode Memory Extension/coleco.rom" ""
-0 0 1 2 23 "" ""
-0 0 3 1 93 "" ""

bios/Microsoft/MSX/Machines/COL - ColecoVision/config.ini

@@ -1,24 +0,0 @@
-
-[CMOS]
-Enable CMOS=0
-Battery Backed=0
-[Video]
-version=TMS99x8A
-vram size=16kB
-[Subslotted Slots]
-slot 0=0
-slot 1=0
-slot 2=0
-slot 3=0
-[External Slots]
-slot A=1 0
-slot B=1 0
-[FDC]
-Count=2
-[CPU]
-Z80 Frequency=3579545Hz
-[Board]
-type=ColecoVision
-[Slots]
-0 0 0 1 20 "Machines/COL - ColecoVision/coleco.rom" ""
-0 0 3 1 93 "" ""

bios/Microsoft/MSX/Machines/COL - Spectravideo SVI-603 Coleco/config.ini

@@ -1,24 +0,0 @@
-
-[CMOS]
-Enable CMOS=0
-Battery Backed=0
-[Video]
-version=TMS99x8A
-vram size=16kB
-[Subslotted Slots]
-slot 0=0
-slot 1=0
-slot 2=0
-slot 3=0
-[External Slots]
-slot A=1 0
-slot B=1 0
-[FDC]
-Count=2
-[CPU]
-Z80 Frequency=3579545Hz
-[Board]
-type=ColecoVision
-[Slots]
-0 0 0 1 20 "Machines/COL - Spectravideo SVI-603 Coleco/SVI603.ROM" ""
-0 0 3 1 93 "" ""