From eb2c06451851929e3ae2fe28bc9685ab8b0aee35 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:05:23 +0200 Subject: [PATCH] fix: create scratch dir and scope catalog ratio --- README.md | 18 +++++++++--------- scripts/generate_pack.py | 3 +++ scripts/generate_readme.py | 23 +++++++++++++++++++---- scripts/generate_site.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2c4390b4..3c93e588 100644 --- a/README.md +++ b/README.md @@ -56,17 +56,17 @@ The RetroDECK pack already contains its own `bios/` folder, so it extracts into BIOS, firmware, and system files for consoles from Atari to PlayStation 3. Every file passes its platform's own verification; where an emulator profile exists, the expected hashes and sizes are read from the emulator's source code (the Source-backed column below). -- **77 files** the platforms' emulators load are not in the collection yet, named in the [gap analysis](https://abdess.github.io/retrobios/gaps/) +- **91 files** the platforms' emulators load are not in the collection yet, named in the [gap analysis](https://abdess.github.io/retrobios/gaps/) - **12 platforms** supported with platform-specific verification -- **328 emulators** profiled from source (RetroArch cores + standalone) -- **409 systems** covered (NES, SNES, PlayStation, Saturn, Dreamcast, ...) +- **336 emulators** profiled from source (RetroArch cores + standalone) +- **410 systems** covered (NES, SNES, PlayStation, Saturn, Dreamcast, ...) - **7,763 files** indexed with SHA1, MD5, SHA256 and CRC32 checksums: 2,478 system files, 2,746 arcade ROM sets, 2,539 game and engine data files -- **542 files** matched to dump-preservation catalogs (No-Intro, Redump, TOSEC) +- **531 of 2,478 system files** matched to dump-preservation catalogs (No-Intro, Redump, TOSEC); arcade sets and engine data fall outside what those catalogs index - **9854 MB** total collection size ## Supported systems -NES, SNES, Nintendo 64, GameCube, Wii, Game Boy, Game Boy Advance, Nintendo DS, Nintendo 3DS, Switch, PlayStation, PlayStation 2, PlayStation 3, PSP, PS Vita, Mega Drive, Saturn, Dreamcast, Game Gear, Master System, Neo Geo, Atari 2600, Atari 7800, Atari Lynx, Atari ST, MSX, PC Engine, TurboGrafx-16, ColecoVision, Intellivision, Commodore 64, Amiga, ZX Spectrum, Arcade (MAME), and 375+ more. +NES, SNES, Nintendo 64, GameCube, Wii, Game Boy, Game Boy Advance, Nintendo DS, Nintendo 3DS, Switch, PlayStation, PlayStation 2, PlayStation 3, PSP, PS Vita, Mega Drive, Saturn, Dreamcast, Game Gear, Master System, Neo Geo, Atari 2600, Atari 7800, Atari Lynx, Atari ST, MSX, PC Engine, TurboGrafx-16, ColecoVision, Intellivision, Commodore 64, Amiga, ZX Spectrum, Arcade (MAME), and 376+ more. Full list with per-file details: **[https://abdess.github.io/retrobios/](https://abdess.github.io/retrobios/)** @@ -74,7 +74,7 @@ Full list with per-file details: **[https://abdess.github.io/retrobios/](https:/ | Platform | Platform list | Read from emulator code | Verified by | |----------|--------------:|------------------------:|-------------| -| Batocera | 353/353 | 1,145/1,222 | MD5 hash | +| Batocera | 353/353 | 1,145/1,236 | MD5 hash | | BizHawk | 118/118 | 366/367 | SHA1 hash | | EmuDeck | 161/161 | 404/404 | MD5 hash | | Lakka | 530/530 | 1,148/1,153 | file presence | @@ -82,7 +82,7 @@ Full list with per-file details: **[https://abdess.github.io/retrobios/](https:/ | ROCKNIX | 38/38 | 1,483/1,488 | MD5 hash | | Recalbox | 346/346 | 714/719 | MD5 hash | | RetroArch | 530/530 | 1,148/1,153 | file presence | -| RetroBat | 341/341 | 808/884 | MD5 hash | +| RetroBat | 341/341 | 808/898 | MD5 hash | | RetroDECK | 2,008/2,008 | 1,182/1,188 | MD5 hash | | RetroPie * | 530/530 | 1,148/1,153 | file presence | | RomM | 374/374 | 266/269 | MD5 hash | @@ -125,7 +125,7 @@ The [documentation site](https://abdess.github.io/retrobios/) provides: - **Per-emulator profiles** with source code references for every file - **Per-system pages** showing which emulators and platforms cover each console - **Gap analysis** identifying missing files and undeclared core requirements -- **Cross-reference** mapping files across 12 platforms and 328 emulators +- **Cross-reference** mapping files across 12 platforms and 336 emulators ## How it works @@ -162,4 +162,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. The legal reasoning is laid out in the [FAQ](https://abdess.github.io/retrobios/wiki/faq/#is-this-legal). -*Auto-generated on 2026-08-08T07:00:03Z* +*Auto-generated on 2026-08-08T09:02:08Z* diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 87cdf6fa..2943d564 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -1672,6 +1672,9 @@ def _normalize_zip_for_pack( """ import tempfile as _tmp + # tmp/ is gitignored, so a fresh clone does not carry it. WSL keeps /tmp + # on a 4 GB tmpfs that MAME sets overflow, hence the repo-local scratch. + os.makedirs("tmp", exist_ok=True) tmp_fd, tmp_path = _tmp.mkstemp(suffix=".zip", dir="tmp") os.close(tmp_fd) try: diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 2caf0e70..aa2b3c56 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -18,6 +18,7 @@ from datetime import datetime, timezone sys.path.insert(0, os.path.dirname(__file__)) from common import ( + GAME_DATA_TOPS, compute_composition, list_registered_platforms, load_database, @@ -154,18 +155,32 @@ _CATALOG_LABELS = {"redump": "Redump", "no-intro": "No-Intro", "tosec": "TOSEC"} def _catalog_matched_line(db: dict) -> list[str]: - """Bullet line for files matched to dump-preservation catalogs.""" + """Bullet line for files matched to dump-preservation catalogs. + + Counted against the system files alone: those catalogs index console and + computer dumps, so arcade ROM sets and engine data sit outside their + scope and would only dilute the ratio. + """ sources: set[str] = set() matched = 0 for entry in db.get("files", {}).values(): provenance = entry.get("provenance") - if provenance: + if not provenance: + continue + sources.update(provenance) + parts = entry.get("path", "").split("/") + top = parts[1] if len(parts) > 1 else "" + if top != "Arcade" and top not in GAME_DATA_TOPS: matched += 1 - sources.update(provenance) if not matched: return [] + system_files = compute_composition(db)["systems"]["files"] labels = ", ".join(_CATALOG_LABELS.get(s, s) for s in sorted(sources)) - return [f"- **{matched:,} files** matched to dump-preservation catalogs ({labels})"] + return [ + f"- **{matched:,} of {system_files:,} system files** matched to" + f" dump-preservation catalogs ({labels}); arcade sets and engine data" + f" fall outside what those catalogs index" + ] def generate_readme(db: dict, platforms_dir: str) -> str: diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 22ecf663..8710b431 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -54,6 +54,35 @@ ICON_CACHE_PATH = Path(".cache") / "system_icons.json" _icon_available: dict[str, bool] = {} +def _content_check_ceiling(profiles: dict) -> str: + """How far content checking can reach across every profiled entry.""" + hashed = sized = neither = 0 + for profile in unique_emulator_profiles(profiles).values(): + for f in profile.get("files", []) or []: + if any( + f.get(k) + for k in ("md5", "sha1", "crc32", "sha256", "known_hash_adler32") + ): + hashed += 1 + elif any(f.get(k) for k in ("size", "min_size", "max_size")): + sized += 1 + else: + neither += 1 + total = hashed + sized + neither + if not total: + return "" + return ( + f"Across every profiled entry, {hashed:,} of {total:,} " + f"({hashed / total * 100:.0f}%) carry a hash the code checks, " + f"{sized:,} ({sized / total * 100:.0f}%) only a size, and " + f"{neither:,} ({neither / total * 100:.0f}%) neither. That last share " + "is the ceiling of the method: where an emulator validates nothing, " + "reading its source establishes which file it loads, never whether " + "the content is the right dump. The provenance field answers that " + "other question, for the files a dump catalog indexes." + ) + + def _icon_name(manufacturer: str, console_name: str) -> str: return f"{manufacturer} - {console_name}".replace("/", " ") @@ -1884,6 +1913,8 @@ def generate_gap_analysis( "profiled emulator applies to the platform, whose own source is then " "the only authority.", "", + _content_check_ceiling(profiles), + "", ]) # ---- Section 1b: platform lists vs emulator source ----