docs: complete wiki coverage, document all scripts and edge cases

scripts/generate_site.py

@@ -1284,6 +1284,59 @@ def generate_wiki_architecture() -> str:
         "The pack combines platform baseline (layer 1) with core requirements (layer 3).",
         "Neither too much (no files from unused cores) nor too few (no missing files for active cores).",
         "",
+        "## Pack grouping",
+        "",
+        "Platforms that produce identical packs are grouped automatically.",
+        "RetroArch and Lakka share the same files and `base_destination` (`system/`),",
+        "so they produce one combined pack (`RetroArch_Lakka_BIOS_Pack.zip`).",
+        "RetroPie uses `BIOS/` as base path, so it gets a separate pack.",
+        "",
+        "## Storage tiers",
+        "",
+        "| Tier | Meaning |",
+        "|------|---------|",
+        "| `embedded` (default) | file is in the `bios/` directory, included in packs |",
+        "| `external` | file has a `source_url`, downloaded at pack build time |",
+        "| `user_provided` | user must provide the file (instructions included in pack) |",
+        "",
+        "## Verification severity",
+        "",
+        "How missing or mismatched files are reported:",
+        "",
+        "| Mode | required + missing | optional + missing | hash mismatch |",
+        "|------|-------------------|-------------------|--------------|",
+        "| existence | WARNING | INFO | N/A |",
+        "| md5 | CRITICAL | WARNING | UNTESTED |",
+        "",
+        "Files with `hle_fallback: true` are downgraded to INFO when missing",
+        "(the emulator has a software fallback).",
+        "",
+        "## Discrepancy detection",
+        "",
+        "When a file passes platform verification (MD5 match) but fails",
+        "emulator-level validation (wrong CRC32, wrong size), a DISCREPANCY is reported.",
+        "The pack generator searches the repo for a variant that satisfies both.",
+        "If none exists, the platform version is kept.",
+        "",
+        "## Security",
+        "",
+        "- `safe_extract_zip()` prevents zip-slip path traversal attacks",
+        "- `deterministic_zip` rebuilds MAME ZIPs so same ROMs always produce the same hash",
+        "- `crypto_verify.py` and `sect233r1.py` verify 3DS RSA-2048 signatures and AES-128-CBC integrity",
+        "- ZIP inner ROM verification via `checkInsideZip()` replicates Batocera's behavior",
+        "- `md5_composite()` replicates Recalbox's composite ZIP hash",
+        "",
+        "## Edge cases",
+        "",
+        "| Case | Handling |",
+        "|------|---------|",
+        "| Batocera truncated MD5 (29 chars) | prefix match in resolution |",
+        "| `zippedFile` entries | MD5 is of the ROM inside the ZIP, not the ZIP itself |",
+        "| Regional variants (same filename) | `by_path_suffix` index disambiguates |",
+        "| MAME BIOS ZIPs | `contents` field documents inner structure |",
+        "| RPG Maker/ScummVM | excluded from dedup (NODEDUP) to preserve directory structure |",
+        "| `strip_components` in data dirs | flattens cache prefix to match expected path |",
+        "",
     ]
     return "\n".join(lines) + "\n"
 
@@ -1377,11 +1430,41 @@ def generate_wiki_tools() -> str:
         "",
         "| Script | Purpose |",
         "|--------|---------|",
-        "| `dedup.py` | Deduplicate `bios/`, move duplicates to `.variants/` |",
+        "| `dedup.py` | Deduplicate `bios/`, move duplicates to `.variants/`. RPG Maker and ScummVM excluded (NODEDUP) |",
         "| `validate_pr.py` | Validate BIOS files in pull requests |",
         "| `auto_fetch.py` | Fetch missing BIOS files from known sources |",
         "| `list_platforms.py` | List active platforms (used by CI) |",
         "| `download.py` | Download packs from GitHub releases |",
+        "| `common.py` | Shared library: hash computation, file resolution, platform config loading, emulator profiles |",
+        "| `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database |",
+        "| `generate_site.py` | Generate all MkDocs site pages (this documentation) |",
+        "| `deterministic_zip.py` | Rebuild MAME BIOS ZIPs deterministically (same ROMs = same hash) |",
+        "| `crypto_verify.py` | 3DS RSA signature and AES crypto verification |",
+        "| `sect233r1.py` | Pure Python ECDSA verification on sect233r1 curve (3DS OTP cert) |",
+        "| `batch_profile.py` | Batch profiling automation for libretro cores |",
+        "| `migrate.py` | Migrate flat bios structure to Manufacturer/Console/ hierarchy |",
+        "",
+        "## Large files",
+        "",
+        "Files over 50 MB are stored as assets on the `large-files` GitHub release.",
+        "They are listed in `.gitignore` so they don't bloat the git repository.",
+        "`generate_db.py` downloads them from the release when rebuilding the database,",
+        "using `fetch_large_file()` from `common.py`. The same function is used by",
+        "`generate_pack.py` when a file has a hash mismatch with the local variant.",
+        "",
+        "## Scrapers",
+        "",
+        "Located in `scripts/scraper/`. Each inherits `BaseScraper` and implements `fetch_requirements()`.",
+        "",
+        "| Scraper | Source | Format |",
+        "|---------|--------|--------|",
+        "| `libretro_scraper` | System.dat + core-info .info files | clrmamepro DAT |",
+        "| `batocera_scraper` | batocera-systems script | Python dict |",
+        "| `recalbox_scraper` | es_bios.xml | XML |",
+        "| `retrobat_scraper` | batocera-systems.json | JSON |",
+        "| `emudeck_scraper` | checkBIOS.sh | Bash + CSV |",
+        "| `retrodeck_scraper` | component manifests | JSON per component |",
+        "| `coreinfo_scraper` | .info files from libretro-core-info | INI-like |",
         "",
     ]
     return "\n".join(lines) + "\n"