From 02a7c58fca609f8387ec3ef056dc9e19d29caea0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 25 Mar 2026 15:02:23 +0100 Subject: [PATCH] docs: complete wiki coverage, document all scripts and edge cases --- scripts/generate_site.py | 85 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index b629cb24..7b1a71a0 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1284,6 +1284,59 @@ def generate_wiki_architecture() -> str: "The pack combines platform baseline (layer 1) with core requirements (layer 3).", "Neither too much (no files from unused cores) nor too few (no missing files for active cores).", "", + "## Pack grouping", + "", + "Platforms that produce identical packs are grouped automatically.", + "RetroArch and Lakka share the same files and `base_destination` (`system/`),", + "so they produce one combined pack (`RetroArch_Lakka_BIOS_Pack.zip`).", + "RetroPie uses `BIOS/` as base path, so it gets a separate pack.", + "", + "## Storage tiers", + "", + "| Tier | Meaning |", + "|------|---------|", + "| `embedded` (default) | file is in the `bios/` directory, included in packs |", + "| `external` | file has a `source_url`, downloaded at pack build time |", + "| `user_provided` | user must provide the file (instructions included in pack) |", + "", + "## Verification severity", + "", + "How missing or mismatched files are reported:", + "", + "| Mode | required + missing | optional + missing | hash mismatch |", + "|------|-------------------|-------------------|--------------|", + "| existence | WARNING | INFO | N/A |", + "| md5 | CRITICAL | WARNING | UNTESTED |", + "", + "Files with `hle_fallback: true` are downgraded to INFO when missing", + "(the emulator has a software fallback).", + "", + "## Discrepancy detection", + "", + "When a file passes platform verification (MD5 match) but fails", + "emulator-level validation (wrong CRC32, wrong size), a DISCREPANCY is reported.", + "The pack generator searches the repo for a variant that satisfies both.", + "If none exists, the platform version is kept.", + "", + "## Security", + "", + "- `safe_extract_zip()` prevents zip-slip path traversal attacks", + "- `deterministic_zip` rebuilds MAME ZIPs so same ROMs always produce the same hash", + "- `crypto_verify.py` and `sect233r1.py` verify 3DS RSA-2048 signatures and AES-128-CBC integrity", + "- ZIP inner ROM verification via `checkInsideZip()` replicates Batocera's behavior", + "- `md5_composite()` replicates Recalbox's composite ZIP hash", + "", + "## Edge cases", + "", + "| Case | Handling |", + "|------|---------|", + "| Batocera truncated MD5 (29 chars) | prefix match in resolution |", + "| `zippedFile` entries | MD5 is of the ROM inside the ZIP, not the ZIP itself |", + "| Regional variants (same filename) | `by_path_suffix` index disambiguates |", + "| MAME BIOS ZIPs | `contents` field documents inner structure |", + "| RPG Maker/ScummVM | excluded from dedup (NODEDUP) to preserve directory structure |", + "| `strip_components` in data dirs | flattens cache prefix to match expected path |", + "", ] return "\n".join(lines) + "\n" @@ -1377,11 +1430,41 @@ def generate_wiki_tools() -> str: "", "| Script | Purpose |", "|--------|---------|", - "| `dedup.py` | Deduplicate `bios/`, move duplicates to `.variants/` |", + "| `dedup.py` | Deduplicate `bios/`, move duplicates to `.variants/`. RPG Maker and ScummVM excluded (NODEDUP) |", "| `validate_pr.py` | Validate BIOS files in pull requests |", "| `auto_fetch.py` | Fetch missing BIOS files from known sources |", "| `list_platforms.py` | List active platforms (used by CI) |", "| `download.py` | Download packs from GitHub releases |", + "| `common.py` | Shared library: hash computation, file resolution, platform config loading, emulator profiles |", + "| `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database |", + "| `generate_site.py` | Generate all MkDocs site pages (this documentation) |", + "| `deterministic_zip.py` | Rebuild MAME BIOS ZIPs deterministically (same ROMs = same hash) |", + "| `crypto_verify.py` | 3DS RSA signature and AES crypto verification |", + "| `sect233r1.py` | Pure Python ECDSA verification on sect233r1 curve (3DS OTP cert) |", + "| `batch_profile.py` | Batch profiling automation for libretro cores |", + "| `migrate.py` | Migrate flat bios structure to Manufacturer/Console/ hierarchy |", + "", + "## Large files", + "", + "Files over 50 MB are stored as assets on the `large-files` GitHub release.", + "They are listed in `.gitignore` so they don't bloat the git repository.", + "`generate_db.py` downloads them from the release when rebuilding the database,", + "using `fetch_large_file()` from `common.py`. The same function is used by", + "`generate_pack.py` when a file has a hash mismatch with the local variant.", + "", + "## Scrapers", + "", + "Located in `scripts/scraper/`. Each inherits `BaseScraper` and implements `fetch_requirements()`.", + "", + "| Scraper | Source | Format |", + "|---------|--------|--------|", + "| `libretro_scraper` | System.dat + core-info .info files | clrmamepro DAT |", + "| `batocera_scraper` | batocera-systems script | Python dict |", + "| `recalbox_scraper` | es_bios.xml | XML |", + "| `retrobat_scraper` | batocera-systems.json | JSON |", + "| `emudeck_scraper` | checkBIOS.sh | Bash + CSV |", + "| `retrodeck_scraper` | component manifests | JSON per component |", + "| `coreinfo_scraper` | .info files from libretro-core-info | INI-like |", "", ] return "\n".join(lines) + "\n"