mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
fix: keep external downloads under the run policy
download_external stayed in generate_pack, outside the offline switch: it fetched under --offline, wrote to the system /tmp and held the body in memory. It now lives in packresolve, honours the switch, streams into the repository tmp/ and hashes on the way.
This commit is contained in:
1 parent
4fec09a014
commit
8ed1a737a8
3 files changed
+126
-68
No files matched your search
+61
-2
@@ -6,9 +6,14 @@ and the offline switch that forbids reaching for the network."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from largefiles import fetch_large_file
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from largefiles import fetch_large_file
|
||||
from common import (
|
||||
declared_hash_verdict,
|
||||
resolution_is_hash_exact,
|
||||
@@ -26,6 +31,60 @@ def set_offline(value: bool) -> None:
|
||||
global _OFFLINE
|
||||
_OFFLINE = value
|
||||
|
||||
|
||||
def scratch_dir() -> str:
|
||||
"""The repository's tmp/, never the system /tmp (a 4 GB tmpfs on WSL)."""
|
||||
path = Path(__file__).resolve().parent.parent / "tmp"
|
||||
path.mkdir(exist_ok=True)
|
||||
return str(path)
|
||||
|
||||
|
||||
def download_external(
|
||||
file_entry: dict, dest_path: str, offline: bool | None = None
|
||||
) -> bool:
|
||||
"""Download a `storage: external` file, verify its hash, write dest_path.
|
||||
|
||||
The run's network policy applies: under --offline nothing is fetched and
|
||||
the entry stays missing, as verify reports it. The body is streamed to
|
||||
dest_path and hashed on the way, and removed when the hash disagrees.
|
||||
"""
|
||||
url = file_entry.get("source_url")
|
||||
if not url:
|
||||
return False
|
||||
name = file_entry.get("name", url)
|
||||
declared = next(
|
||||
(
|
||||
(algo, str(file_entry[algo]).lower())
|
||||
for algo in ("sha256", "sha1", "md5")
|
||||
if file_entry.get(algo)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if declared is None:
|
||||
print(f" WARNING: no hash for {name}, skipping unverifiable download")
|
||||
return False
|
||||
if _OFFLINE if offline is None else offline:
|
||||
print(f" WARNING: {name} is external and the run is offline, not fetched")
|
||||
return False
|
||||
algo, expected = declared
|
||||
digest = hashlib.new(algo)
|
||||
Path(dest_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "retrobios-pack-gen/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=120) as resp, open(dest_path, "wb") as out:
|
||||
for chunk in iter(lambda: resp.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
out.write(chunk)
|
||||
except (urllib.error.URLError, OSError) as exc:
|
||||
print(f" WARNING: Failed to download {url}: {exc}")
|
||||
Path(dest_path).unlink(missing_ok=True)
|
||||
return False
|
||||
if digest.hexdigest() != expected:
|
||||
print(f" WARNING: {algo.upper()} mismatch for {name}")
|
||||
Path(dest_path).unlink(missing_ok=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _values(raw: object) -> list[str]:
|
||||
"""A hash field's values: one, a list, or a comma-separated string."""
|
||||
items = raw if isinstance(raw, list) else [raw] if raw else []
|
||||
|
||||
Reference in new issue
Block a user