From 18e093182d66bc38fe1de2dcccfe80206582b8fc Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:35:02 +0200 Subject: [PATCH] docs: name the registries behind assets and dedup The guides said files over 50 MB become release assets and that RPG Maker and ScummVM were the trees kept out of dedup. .gitignore decides the first, four committed files are over 50 MB, and NODEDUP_DIRS holds eleven trees. --- CONTRIBUTING.md | 6 ++++-- scripts/generate_readme.py | 6 ++++-- tests/test_audit_regressions.py | 27 +++++++++++++++++++++++++++ wiki/architecture.md | 2 +- wiki/profiling.md | 2 +- wiki/release-process.md | 6 ++++-- wiki/tools.md | 6 ++++-- wiki/troubleshooting.md | 2 +- 8 files changed, 46 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a066f461..f92f3942 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,8 +34,10 @@ Full walkthrough: [profiling guide](https://abdess.github.io/retrobios/wiki/prof - `bios/Manufacturer/Console/filename` for canonical files - `bios/Manufacturer/Console/.variants/filename.sha1prefix` for alternate versions -- Files >50 MB go in GitHub release assets (`large-files` release) -- RPG Maker and ScummVM directories are excluded from deduplication +- A file too large for git goes to the `large-files` GitHub release and its + path into `.gitignore`, the list that decides what is served from the release +- Trees addressed by path are kept out of deduplication (`NODEDUP_DIRS` in + `scripts/dedup.py`) - Two paths differing only by case break clones on Windows and macOS, and `tests/test_no_case_collisions.py` enforces this diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index b090d9f5..f665ab3a 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -719,8 +719,10 @@ Full walkthrough: [profiling guide]({link("wiki/profiling")}). - `bios/Manufacturer/Console/filename` for canonical files - `bios/Manufacturer/Console/.variants/filename.sha1prefix` for alternate versions -- Files >50 MB go in GitHub release assets (`large-files` release) -- RPG Maker and ScummVM directories are excluded from deduplication +- A file too large for git goes to the `large-files` GitHub release and its + path into `.gitignore`, the list that decides what is served from the release +- Trees addressed by path are kept out of deduplication (`NODEDUP_DIRS` in + `scripts/dedup.py`) - Two paths differing only by case break clones on Windows and macOS, and `tests/test_no_case_collisions.py` enforces this diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index f85d1eb5..b979f3af 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5456,5 +5456,32 @@ class AnUnknownTargetIsRefusedUnderAll(unittest.TestCase): self.assertIn("retroarch", kept) +class DocsDescribeTheRegistriesTheyName(unittest.TestCase): + """The contributing guide said files over 50 MB are release assets and that + RPG Maker and ScummVM were the trees kept out of dedup. `.gitignore` + decides the first, four committed files exceed 50 MB, and NODEDUP_DIRS + holds eleven trees.""" + + SIZE_RULE = re.compile(r"(over|larger than|>)\s*50\s*MB[^\n]*(release|asset)", re.I) + + def _texts(self): + paths = sorted((ROOT / "wiki").glob("*.md")) + paths += [ROOT / "scripts" / "generate_readme.py", ROOT / "CONTRIBUTING.md"] + return [(p, p.read_text(encoding="utf-8")) for p in paths] + + def test_no_page_says_size_decides_a_release_asset(self): + offenders = [str(p.relative_to(ROOT)) for p, text in self._texts() + if self.SIZE_RULE.search(text)] + self.assertEqual(offenders, []) + + def test_the_dedup_list_is_named_whole_or_by_its_constant(self): + from dedup import NODEDUP_DIRS + tools = (ROOT / "wiki" / "tools.md").read_text(encoding="utf-8") + line = next(l for l in tools.splitlines() if "NODEDUP_DIRS" in l) + self.assertEqual(sorted(d for d in NODEDUP_DIRS if d not in line), []) + readme_gen = (ROOT / "scripts" / "generate_readme.py").read_text(encoding="utf-8") + self.assertIn("NODEDUP_DIRS", readme_gen) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/architecture.md b/wiki/architecture.md index ff62e513..c9efa1d4 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -241,7 +241,7 @@ platform's own verification mode. | `zippedFile` entries | MD5 is of the ROM inside the ZIP, not the ZIP itself | | Regional variants (same filename) | `by_path_suffix` index disambiguates | | MAME BIOS ZIPs | `contents` field documents inner structure | -| RPG Maker/ScummVM | excluded from dedup (NODEDUP) to preserve directory structure | +| Trees addressed by path (RPG Maker, ScummVM, tzdata, c64-emu and the rest of `NODEDUP_DIRS`) | excluded from dedup to keep files that are equal by design | | `strip_components` in data dirs | flattens cache prefix to match expected path | | case-insensitive dedup | prevents `font.rom` + `FONT.ROM` conflicts on Windows/macOS | | frozen snapshot cores | `.info` may reflect current version while code is pinned to an old one. Only the frozen source at the pinned tag is reliable (e.g. desmume2015, mame2003) | diff --git a/wiki/profiling.md b/wiki/profiling.md index a1c91de5..10106dbf 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -411,7 +411,7 @@ partitions), and `game_count`. | `description` | what this file is | | `note` | additional context | | `contents` | structure of files inside a BIOS ZIP (`name`, `description`, `size`, `crc32`) | -| `storage` | `embedded` (default), `external`, `user_provided`, or `large_file`/`release` for files > 50 MB stored as release assets | +| `storage` | `embedded` (default), `external`, `user_provided`, or `large_file`/`release` for files stored as release assets (listed in `.gitignore`) | | `agnostic` | true if any file under the system path within size constraints satisfies the requirement | | `unsourceable` | reason why the file cannot be sourced (acknowledged gap) | | `destination` | target path within the BIOS directory | diff --git a/wiki/release-process.md b/wiki/release-process.md index 9be05d9e..865b7d0c 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -124,8 +124,10 @@ merge, and again on the commit a direct push puts at the head of main. ## Large files management -Files larger than 50 MB are stored as assets on a permanent GitHub release -named `large-files` (to keep the git repository lightweight). +Files too large for git are stored as assets on a permanent GitHub release +named `large-files`. Size does not decide it: four committed files are over +50 MB (`history.xml`, `default-GM.sf3`, `5.avi`, `tbblue.mmc`) and are served +from the repository like any other. Examples: PS3UPDAT.PUP, PSVUPDAT.PUP, PSP2UPDAT.PUP, the DSi NAND images, maclc3.zip, Firmware.19.0.0.zip (Switch), the QEMU EDK2 firmware, the ScummVM diff --git a/wiki/tools.md b/wiki/tools.md index 79e923f6..6adb1ffd 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -709,8 +709,10 @@ at. Writing is explicit: the pipeline only ever reports. ## Large files -Files over 50 MB are stored as assets on the `large-files` GitHub release. -They are listed in `.gitignore` to keep the git repository lightweight. +Files too large for git are stored as assets on the `large-files` GitHub +release. `.gitignore` is the list of them: a `bios/` path listed there is a +release asset, whatever its size, and a committed file is served from the +repository even past 50 MB. `generate_db.py` downloads them from the release when rebuilding the database, using `fetch_large_file()` from `common.py`. The same function is used by `generate_pack.py` when a file has a hash mismatch with the local variant. diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md index ed7fcb0a..493b3191 100644 --- a/wiki/troubleshooting.md +++ b/wiki/troubleshooting.md @@ -154,7 +154,7 @@ in [verification modes](verification-modes.md#severity-matrix). Focus on CRITICAL issues first. WARNING files improve the experience but aren't strictly necessary. INFO files are nice to have. -**Large files (over 50 MB):** +**Large files (release assets):** Files like PS3UPDAT.PUP, PSVUPDAT.PUP, and Switch firmware are too large for the git repository. They are stored as GitHub release assets under the `large-files`