refactor: centralize hash logic, fix circular imports and perf bottlenecks

scripts/scraper/base_scraper.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import sys
 import urllib.request
 import urllib.error
 from abc import ABC, abstractmethod
@@ -168,7 +169,6 @@ def scraper_cli(scraper_class: type, description: str = "Scrape BIOS requirement
         reqs = scraper.fetch_requirements()
     except (ConnectionError, ValueError) as e:
         print(f"Error: {e}", file=sys.stderr)
-        import sys
         sys.exit(1)
 
     if args.dry_run:

scripts/scraper/batocera_scraper.py

@@ -211,28 +211,12 @@ class Scraper(BaseScraper):
 
             systems[req.system]["files"].append(entry)
 
-        # Sort numerically since API returns by commit date, not version
-        import json as _json
+        tag = fetch_github_latest_tag("batocera-linux/batocera.linux", prefix="batocera-")
         batocera_version = ""
-        try:
-            _url = "https://api.github.com/repos/batocera-linux/batocera.linux/tags?per_page=50"
-            _req = urllib.request.Request(_url, headers={
-                "User-Agent": "retrobios-scraper/1.0",
-                "Accept": "application/vnd.github.v3+json",
-            })
-            with urllib.request.urlopen(_req, timeout=15) as _resp:
-                _tags = _json.loads(_resp.read())
-            _versions = []
-            for _t in _tags:
-                _name = _t["name"]
-                if _name.startswith("batocera-"):
-                    _num = _name.replace("batocera-", "")
-                    if _num.isdigit():
-                        _versions.append(int(_num))
-            if _versions:
-                batocera_version = str(max(_versions))
-        except (ConnectionError, ValueError, OSError):
-            pass
+        if tag:
+            num = tag.removeprefix("batocera-")
+            if num.isdigit():
+                batocera_version = num
 
         return {
             "platform": "Batocera",

scripts/scraper/recalbox_scraper.py

@@ -262,7 +262,6 @@ def main():
         print(json.dumps(config, indent=2))
         return
 
-    reqs = scraper.fetch_requirements()
     by_system = {}
     for r in reqs:
         by_system.setdefault(r.system, []).append(r)