fix: scan hidden engine dirs, anchor data ignore

.gitignore

@@ -8,6 +8,8 @@ __pycache__/
 dist/
 site/
 *.tmp
+# px68k reads its fallback font under this name: a collection file, not scratch.
+!bios/**/*.tmp
 *.log
 node_modules/
 
@@ -20,10 +22,13 @@ bios/Sony/PlayStation 3/.variants/PS3UPDAT.PUP.ed8ab192
 bios/Sony/PlayStation Vita/PSVUPDAT.PUP
 bios/Sony/PlayStation Vita/PSP2UPDAT.PUP
 
-# Data directories (fetched at build time)
-data/
+# Data directories (fetched at build time). Anchored: an engine tree under
+# bios/ has a data/ directory of its own, and that one is committed.
+/data/
 
 # Large files stored as GitHub Release assets (additional)
+bios/Other/mame4droid-2024/data/history.db
+bios/Other/mame4droid-2024/data/.variants/history.db.615619e8
 bios/Arcade/MAME/artwork/snspell.zip
 bios/Arcade/MAME/MAME 0.174 Arcade XML.dat
 bios/Sony/PlayStation Vita/.variants/PSP2UPDAT.PUP.3ae832c9

scripts/dedup.py

@@ -50,6 +50,13 @@ NODEDUP_DIRS = {
     # BBK dictionary ROMs are addressed by model directory, and the OS ROM
     # the A4988 folder needs is the one the gam4980 core is given.
     "BBK",
+    # Engine data trees read by path: one sound or one sprite sits under
+    # several characters, campaigns or episodes, and each copy is opened
+    # where it lies.
+    "C-Dogs SDL",
+    "Ikemen GO",
+    "Theme Hospital",
+    "TheXTech",
 }
 
 

scripts/generate_db.py

@@ -43,13 +43,15 @@ CACHED_HASHES = ("sha1", "md5", "sha256", "crc32", "adler32")
 
 
 def should_skip(path: Path) -> bool:
-    """Check if a path should be skipped. Allows .variants/ directories."""
-    for part in path.parts:
-        if part in SKIP_PATTERNS:
-            return True
-        if part.startswith(".") and part != ".variants":
-            return True
-    return False
+    """Whether a path stays out of the collection.
+
+    Tooling directories and hidden files do. A hidden directory inside the
+    collection does not: `.variants/` holds alternate dumps, and an engine
+    tree can carry one of its own (C-Dogs SDL reads `data/.wolf3d/`).
+    """
+    if any(part in SKIP_PATTERNS for part in path.parts):
+        return True
+    return path.name.startswith(".")
 
 
 def _canonical_name(filepath: Path) -> str:

tests/test_directory_entries.py

@@ -292,5 +292,50 @@ class ProfileContract(unittest.TestCase):
         return name.endswith("/") or "type" in entry
 
 
+class CollectionScanAndIgnoreRules(unittest.TestCase):
+    """Two rules that kept collected engine trees out of the repository."""
+
+    def test_a_hidden_directory_of_an_engine_tree_is_scanned(self):
+        """C-Dogs SDL reads its Wolfenstein campaigns from `data/.wolf3d/`.
+        The scan dropped every dot-directory but `.variants`, so 695 files
+        of a collected tree could not enter the database."""
+        self.assertFalse(
+            generate_db.should_skip(Path("Game Engines/cdogs/data/.wolf3d/N3D.json"))
+        )
+        self.assertFalse(generate_db.should_skip(Path("Sony/.variants/scph.bin.1234")))
+
+    def test_hidden_files_and_tooling_directories_stay_out(self):
+        for skipped in (
+            "RPG Maker/mkxp-z/Fonts/.gitkeep",
+            "Atari/.DS_Store",
+            ".cache/large/file.bin",
+            ".git/config",
+        ):
+            self.assertTrue(generate_db.should_skip(Path(skipped)), skipped)
+
+    def test_an_ignored_file_under_bios_is_a_registered_release_asset(self):
+        """`.gitignore` is the register of release assets, path by path. A
+        bare `data/` rule also matched every engine's own `data/` directory:
+        1 525 collected files would have stayed uncommitted while the
+        manifests pointed installers at them."""
+        import subprocess
+
+        listed = subprocess.run(
+            ["git", "ls-files", "--others", "--ignored", "--exclude-standard", "bios"],
+            cwd=REPO_ROOT, capture_output=True, text=True,
+        )
+        if listed.returncode != 0:
+            self.skipTest("not a git checkout")
+        registered = {
+            line.strip()
+            for line in (REPO_ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()
+            if line.strip() and not line.startswith("#")
+        }
+        strays = [
+            path for path in listed.stdout.splitlines() if path not in registered
+        ]
+        self.assertEqual(strays[:10], [], f"{len(strays)} ignored files not registered")
+
+
 if __name__ == "__main__":
     unittest.main()