refactor: consolidate CI pipeline, remove third-party deps

scripts/auto_fetch.py

@@ -26,7 +26,7 @@ import urllib.error
 from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from common import load_platform_config
+from common import load_database, load_platform_config
 
 try:
     import yaml
@@ -62,11 +62,6 @@ ARCHIVE_ORG_COLLECTIONS = [
 ]
 
 
-def load_database(db_path: str) -> dict:
-    with open(db_path) as f:
-        return json.load(f)
-
-
 def find_missing(config: dict, db: dict) -> list[dict]:
     """Find BIOS files required by platform but not in database."""
     missing = []

scripts/common.py

@@ -7,6 +7,7 @@ and file resolution - eliminates DRY violations across scripts.
 from __future__ import annotations
 
 import hashlib
+import json
 import os
 import zipfile
 import zlib
@@ -38,6 +39,12 @@ def compute_hashes(filepath: str | Path) -> dict[str, str]:
     }
 
 
+def load_database(db_path: str) -> dict:
+    """Load database.json and return parsed dict."""
+    with open(db_path) as f:
+        return json.load(f)
+
+
 def md5sum(filepath: str | Path) -> str:
     """Compute MD5 of a file - matches Batocera's md5sum()."""
     h = hashlib.md5()

scripts/dedup.py

@@ -87,7 +87,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
         if len(paths) <= 1:
             continue
 
-            paths.sort(key=path_priority)
+        paths.sort(key=path_priority)
         canonical = paths[0]
         duplicates = paths[1:]
 

scripts/generate_db.py

@@ -11,14 +11,15 @@ Use --force to rehash all files.
 from __future__ import annotations
 
 import argparse
-import hashlib
 import json
 import os
 import sys
-import zlib
 from datetime import datetime, timezone
 from pathlib import Path
 
+sys.path.insert(0, os.path.dirname(__file__))
+from common import compute_hashes
+
 CACHE_DIR = ".cache"
 CACHE_FILE = os.path.join(CACHE_DIR, "db_cache.json")
 DEFAULT_BIOS_DIR = "bios"
@@ -27,31 +28,6 @@ DEFAULT_OUTPUT = "database.json"
 SKIP_PATTERNS = {".git", ".github", "__pycache__", ".cache", ".DS_Store", "desktop.ini"}
 
 
-def compute_hashes(filepath: Path) -> dict:
-    """Compute SHA1, MD5, SHA256, CRC32 for a file."""
-    sha1 = hashlib.sha1()
-    md5 = hashlib.md5()
-    sha256 = hashlib.sha256()
-    crc = 0
-
-    with open(filepath, "rb") as f:
-        while True:
-            chunk = f.read(65536)
-            if not chunk:
-                break
-            sha1.update(chunk)
-            md5.update(chunk)
-            sha256.update(chunk)
-            crc = zlib.crc32(chunk, crc)
-
-    return {
-        "sha1": sha1.hexdigest(),
-        "md5": md5.hexdigest(),
-        "sha256": sha256.hexdigest(),
-        "crc32": format(crc & 0xFFFFFFFF, "08x"),
-    }
-
-
 def should_skip(path: Path) -> bool:
     """Check if a path should be skipped. Allows .variants/ directories."""
     for part in path.parts:
@@ -276,8 +252,7 @@ def _collect_all_aliases(files: dict) -> dict:
             pass
 
     try:
-        import sys as _sys
-        _sys.path.insert(0, "scripts")
+        sys.path.insert(0, "scripts")
         from scraper.coreinfo_scraper import Scraper as CoreInfoScraper
         ci_reqs = CoreInfoScraper().fetch_requirements()
         for r in ci_reqs:

scripts/generate_pack.py

@@ -24,7 +24,7 @@ import zipfile
 from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
-from common import load_platform_config
+from common import load_database, load_platform_config
 
 try:
     import yaml
@@ -40,11 +40,6 @@ LARGE_FILES_RELEASE = "large-files"
 LARGE_FILES_REPO = "Abdess/retrobios"
 
 
-def load_database(db_path: str) -> dict:
-    with open(db_path) as f:
-        return json.load(f)
-
-
 def fetch_large_file(name: str, dest_dir: str = ".cache/large") -> str | None:
     """Download a large file from the 'large-files' GitHub release if not cached."""
     cached = os.path.join(dest_dir, name)
@@ -380,9 +375,7 @@ def main():
                 if new_path != zip_path:
                     os.rename(zip_path, new_path)
                     print(f"  Renamed -> {os.path.basename(new_path)}")
-        except FileNotFoundError as e:
-            print(f"  ERROR: {e}")
-        except Exception as e:
+        except (FileNotFoundError, OSError, yaml.YAMLError) as e:
             print(f"  ERROR: {e}")
 
 
@@ -392,8 +385,6 @@ def _group_identical_platforms(platforms: list[str], platforms_dir: str) -> list
     Returns [(group_of_platform_names, representative_platform), ...].
     Platforms with the same resolved systems+files+base_destination are grouped.
     """
-    import hashlib as _hl
-
     fingerprints = {}
     representatives = {}
 
@@ -413,7 +404,7 @@ def _group_identical_platforms(platforms: list[str], platforms_dir: str) -> list
                 full_dest = f"{base_dest}/{dest}" if base_dest else dest
                 entries.append(full_dest)
 
-        fingerprint = _hl.sha1("|".join(sorted(entries)).encode()).hexdigest()
+        fingerprint = hashlib.sha1("|".join(sorted(entries)).encode()).hexdigest()
         fingerprints.setdefault(fingerprint, []).append(platform)
         representatives.setdefault(fingerprint, platform)
 

scripts/generate_readme.py

@@ -15,7 +15,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
-from common import load_platform_config
+from common import load_database, load_platform_config
 
 try:
     import yaml
@@ -24,11 +24,6 @@ except ImportError:
     sys.exit(1)
 
 
-def load_database(db_path: str) -> dict:
-    with open(db_path) as f:
-        return json.load(f)
-
-
 def load_platform_configs(platforms_dir: str) -> dict:
     """Load all platform configs with inheritance resolved."""
     configs = {}

scripts/scraper/base_scraper.py

@@ -111,7 +111,7 @@ class BaseScraper(ABC):
         try:
             self.fetch_requirements()
             return True
-        except Exception:
+        except (ConnectionError, ValueError, OSError):
             return False
 
     @abstractmethod

scripts/scraper/batocera_scraper.py

@@ -239,7 +239,7 @@ class Scraper(BaseScraper):
                         _versions.append(int(_num))
             if _versions:
                 batocera_version = str(max(_versions))
-        except Exception:
+        except (ConnectionError, ValueError, OSError):
             pass
 
         return {

scripts/scraper/libretro_scraper.py

@@ -208,7 +208,7 @@ class Scraper(BaseScraper):
                         }
                 except (urllib.error.URLError, urllib.error.HTTPError):
                     continue
-        except Exception:
+        except (ConnectionError, ValueError, OSError):
             pass
 
         return metadata

scripts/validate_pr.py

@@ -18,16 +18,14 @@ Outputs a structured report suitable for PR comments.
 from __future__ import annotations
 
 import argparse
-import hashlib
 import json
 import os
 import subprocess
 import sys
-import zlib
 from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
-from common import compute_hashes
+from common import compute_hashes, load_database as _load_database
 
 try:
     import yaml
@@ -93,11 +91,10 @@ class ValidationResult:
 
 
 def load_database(db_path: str) -> dict | None:
-    """Load database.json if available."""
-    if os.path.exists(db_path):
-        with open(db_path) as f:
-            return json.load(f)
-    return None
+    try:
+        return _load_database(db_path)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return None
 
 
 def load_platform_hashes(platforms_dir: str) -> dict:

scripts/verify.py

@@ -67,7 +67,7 @@ def check_inside_zip(container: str, file_name: str, expected_md5: str) -> str:
                         return Status.UNTESTED
 
             return "not_in_zip"
-    except Exception:
+    except (zipfile.BadZipFile, OSError, KeyError):
         return "error"