chore: merge audit fixes (scrapers)

scripts/scraper/targets/__init__.py

@@ -43,6 +43,7 @@ class BaseTargetScraper(ABC):
             r"^https://raw\.githubusercontent\.com/[^/]+/[^/]+/([^/]+)/",
             r"^https://github\.com/[^/]+/[^/]+/(?:blob|tree)/([^/]+)/",
             r"^https://gitlab\.com/.+?/-/(?:raw|blob|tree)/([^/]+)/",
+            r"^https://codeload\.github\.com/[^/]+/[^/]+/(?:tar\.gz|zip)/([^/]+)$",
         ):
             match = re.match(pattern, source)
             if match:

scripts/scraper/targets/emudeck_targets_scraper.py

@@ -1,8 +1,11 @@
 """Scraper for EmuDeck emulator targets.
 
 Sources:
-  SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh
-  Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1
+  SteamOS: dragoonDorise/EmuDeck, functions/EmuScripts/*.sh, read at the
+           revision platforms/emudeck.yml was scraped at
+  Windows: EmuDeck/emudeck-we, functions/EmuScripts/*.ps1, default branch:
+           the platform file is read from dragoonDorise/EmuDeck alone and
+           pins no emudeck-we revision
 """
 
 from __future__ import annotations
@@ -89,8 +92,18 @@ def _script_to_core(filename: str) -> str | None:
 class Scraper(BaseTargetScraper):
     """Fetches emulator lists for EmuDeck SteamOS and Windows targets."""
 
-    def __init__(self, url: str = "https://github.com/dragoonDorise/EmuDeck") -> None:
+    def __init__(
+        self, url: str = "https://github.com/dragoonDorise/EmuDeck", revision: str | None = None
+    ) -> None:
         super().__init__(url=url)
+        self._revision = revision
+
+    @property
+    def revision(self) -> str:
+        """The checkBIOS.sh commit platforms/emudeck.yml was read at."""
+        if self._revision is None:
+            self._revision = self.pinned_revision(PLATFORM_NAME)
+        return self._revision
 
     def _fetch_cores_for_target(
         self, api_url: str, label: str, arch: str = "x86_64"
@@ -141,7 +154,9 @@ class Scraper(BaseTargetScraper):
         raise RuntimeError(f"no {arch} core list in {target_path}")
 
     def fetch_targets(self) -> dict:
-        steamos_cores = self._fetch_cores_for_target(STEAMOS_API, "SteamOS")
+        steamos_cores = self._fetch_cores_for_target(
+            f"{STEAMOS_API}?ref={self.revision}", "SteamOS"
+        )
         windows_cores = self._fetch_cores_for_target(WINDOWS_API, "Windows")
 
         targets: dict[str, dict] = {}
@@ -158,7 +173,7 @@ class Scraper(BaseTargetScraper):
 
         return {
             "platform": "emudeck",
-            "source": self.url,
+            "source": f"{self.url}/tree/{self.revision}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
             "targets": targets,
         }

scripts/scraper/targets/retropie_targets_scraper.py

@@ -1,6 +1,7 @@
 """Scraper for RetroPie package availability per platform.
 
-Source: https://github.com/RetroPie/RetroPie-Setup/tree/master/scriptmodules
+Source: https://github.com/RetroPie/RetroPie-Setup/tree/<rev>/scriptmodules,
+read at the revision platforms/retropie.yml was scraped at.
 Parses rp_module_id and rp_module_flags from each scriptmodule of the
 package sections the platform list reads (emulators, libretrocores, ports)
 to determine which platforms each package supports.
@@ -26,7 +27,7 @@ GITHUB_API_URL = (
     "https://api.github.com/repos/RetroPie/RetroPie-Setup/contents/scriptmodules"
 )
 RAW_BASE_URL = (
-    "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/master/scriptmodules/"
+    "https://raw.githubusercontent.com/RetroPie/RetroPie-Setup/{rev}/scriptmodules/"
 )
 # The sections the platform scraper reads: a standalone package missing here
 # is filtered out of every targeted pack.
@@ -114,14 +115,23 @@ def _parse_module(content: str) -> tuple[str | None, str]:
 class Scraper(BaseTargetScraper):
     """Fetches RetroPie package availability by parsing scriptmodules."""
 
-    def __init__(self, url: str = GITHUB_API_URL) -> None:
+    def __init__(self, url: str = GITHUB_API_URL, revision: str | None = None) -> None:
         super().__init__(url=url)
+        self._revision = revision
+
+    @property
+    def revision(self) -> str:
+        """The commit the package list was read at: the flags of another
+        revision filter packages the platform file does not describe."""
+        if self._revision is None:
+            self._revision = self.pinned_revision(PLATFORM_NAME)
+        return self._revision
 
     def _list_scriptmodules(self) -> list[str]:
         """Return section/filename for every .sh of the package sections."""
         names: list[str] = []
         for section in PACKAGE_DIRS:
-            url = f"{self.url}/{section}"
+            url = f"{self.url}/{section}?ref={self.revision}"
             entries = json.loads(_fetch(url, accept="application/vnd.github+json"))
             listed = [
                 f"{section}/{e['name']}" for e in entries if e.get("name", "").endswith(".sh")
@@ -132,7 +142,7 @@ class Scraper(BaseTargetScraper):
         return names
 
     def _fetch_module(self, filename: str) -> str:
-        return _fetch(f"{RAW_BASE_URL}{filename}")
+        return _fetch(f"{RAW_BASE_URL.format(rev=self.revision)}{filename}")
 
     def fetch_targets(self) -> dict:
         print("  listing RetroPie scriptmodules...", file=sys.stderr)
@@ -170,7 +180,7 @@ class Scraper(BaseTargetScraper):
 
         return {
             "platform": "retropie",
-            "source": self.url,
+            "source": f"{self.url}?ref={self.revision}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
             "targets": targets,
         }